@syncular/server 0.96.0 → 0.98.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -245,7 +245,7 @@ class D1Transaction extends SqliteFamilyTransaction {
245
245
  while (values.length > 0 && persisted.length < persistedLimit) {
246
246
  const { results: records } = await this.#db
247
247
  .prepare(sql)
248
- .bind(this.partition, query.table, variable, JSON.stringify(values), afterRowId, batchSize, this.partition)
248
+ .bind(this.partition, query.table, variable, JSON.stringify(values), afterRowId, batchSize)
249
249
  .all();
250
250
  if (records.length === 0)
251
251
  break;
package/dist/errors.d.ts CHANGED
@@ -34,6 +34,17 @@ export declare function errorBody(error: SyncError): {
34
34
  message: string;
35
35
  details?: unknown;
36
36
  };
37
+ /**
38
+ * HTTP headers of a §10.1 error answer: `sync.segment_store_unavailable`
39
+ * carries a fixed `Retry-After: 5` (§10.2).
40
+ */
41
+ export declare function errorHeaders(error: SyncError): Record<string, string>;
42
+ /**
43
+ * The rejection handler of a `SegmentStore` call (§5.4, §5.5): the store's
44
+ * error goes to `onError`, and the request fails with the retryable
45
+ * `sync.segment_store_unavailable`, its `details.operation` naming the call.
46
+ */
47
+ export declare function segmentStoreFailure(onError: SyncularErrorHandler | undefined, route: 'sync' | 'segments', operation: 'put' | 'find' | 'get' | 'open'): (error: unknown) => never;
37
48
  /** The adapter surface that caught an exception reported through `onError`. */
38
49
  export type SyncularErrorRoute = 'sync' | 'operations' | 'segments' | 'blobs' | 'admin' | 'realtime';
39
50
  /**
package/dist/errors.js CHANGED
@@ -234,6 +234,14 @@ export const ERROR_CATALOG = {
234
234
  recommendedAction: 'retryLater',
235
235
  httpStatus: 503,
236
236
  },
237
+ // §5.4/§5.5: a SegmentStore call rejected (an object store outage). The
238
+ // HTTP answer carries `Retry-After: 5` (`errorHeaders`).
239
+ 'sync.segment_store_unavailable': {
240
+ category: 'internal',
241
+ retryable: true,
242
+ recommendedAction: 'retryLater',
243
+ httpStatus: 503,
244
+ },
237
245
  'sync.segment_expired': {
238
246
  category: 'not-found',
239
247
  retryable: true,
@@ -341,6 +349,26 @@ export function errorBody(error) {
341
349
  : {}),
342
350
  };
343
351
  }
352
+ /**
353
+ * HTTP headers of a §10.1 error answer: `sync.segment_store_unavailable`
354
+ * carries a fixed `Retry-After: 5` (§10.2).
355
+ */
356
+ export function errorHeaders(error) {
357
+ return error.code === 'sync.segment_store_unavailable'
358
+ ? { 'retry-after': '5' }
359
+ : {};
360
+ }
361
+ /**
362
+ * The rejection handler of a `SegmentStore` call (§5.4, §5.5): the store's
363
+ * error goes to `onError`, and the request fails with the retryable
364
+ * `sync.segment_store_unavailable`, its `details.operation` naming the call.
365
+ */
366
+ export function segmentStoreFailure(onError, route, operation) {
367
+ return (error) => {
368
+ reportError(error, onError, route);
369
+ throw new SyncError('sync.segment_store_unavailable', 'the segment store failed', JSON.stringify({ operation }));
370
+ };
371
+ }
344
372
  /**
345
373
  * The protocol error for an exception an adapter caught. A `SyncError`
346
374
  * passes through unchanged. Anything else goes to `onError` and answers
package/dist/events.d.ts CHANGED
@@ -290,8 +290,9 @@ export interface PruneCompletedEvent {
290
290
  readonly advanced: boolean;
291
291
  readonly removedCommits: number;
292
292
  }
293
- /** The host `resolveScopes` callback threw — the §3.2/§3.4 fail-loud
294
- * path (units revoke, writes reject, downloads deny). */
293
+ /** The host `resolveScopes` callback threw (§3.2 rule 3): a sync request
294
+ * fails with `sync.internal_error`, a download or realtime registration is
295
+ * denied. */
295
296
  export interface ScopesResolveFailedEvent {
296
297
  readonly type: 'scopes.resolve_failed';
297
298
  readonly atMs: number;
package/dist/handler.js CHANGED
@@ -76,9 +76,8 @@ async function resolveOnce(ctx, clientId, schema) {
76
76
  catch (error) {
77
77
  if (error instanceof SyncError)
78
78
  throw error;
79
- // §3.2 rule 5 / §3.4 step 4: fail loud, never leak — units revoke and
80
- // writes reject with sync.forbidden. A throw is NOT an
81
- // outage (§7.3.3): it never reaches the lease.
79
+ // §3.2 rule 3: a throw fails the whole request; nothing is revoked,
80
+ // rejected or leased. A throw is NOT an outage (§7.3.3).
82
81
  const events = ctx.events;
83
82
  if (events !== undefined) {
84
83
  emitEvent(events, {
@@ -90,7 +89,7 @@ async function resolveOnce(ctx, clientId, schema) {
90
89
  message: error instanceof Error ? error.message : String(error),
91
90
  });
92
91
  }
93
- return { resolved: { ok: false }, leaseToEmit: undefined };
92
+ throw error;
94
93
  }
95
94
  if (allowed === RESOLVER_OUTAGE) {
96
95
  // §7.3.3: the host opted this request into lease authorization. No
@@ -964,7 +963,10 @@ async function* streamResponse(plan, ctx, schema, report) {
964
963
  }
965
964
  }
966
965
  catch (error) {
967
- if (error instanceof SyncError) {
966
+ // §5.4: a segment store failure leaves the build, so `createStreamCore`
967
+ // answers it as HTTP 503 before the header frames and in-band after them.
968
+ if (error instanceof SyncError &&
969
+ error.code !== 'sync.segment_store_unavailable') {
968
970
  report.outcome = 'error';
969
971
  report.errorCode = error.code;
970
972
  await recordRoundFailure(ctx, clientId, error);
package/dist/pull.js CHANGED
@@ -3,7 +3,7 @@
3
3
  */
4
4
  import { encodeRowsSegment, unitId, } from '@syncular/core';
5
5
  import { clockOf, limitsOf } from './context.js';
6
- import { syncError } from './errors.js';
6
+ import { segmentStoreFailure, syncError } from './errors.js';
7
7
  import { compileSchema, } from './schema.js';
8
8
  import { computeEffective, scopeDigest } from './scopes.js';
9
9
  import { issueSegmentUrl } from './signed-url.js';
@@ -237,7 +237,8 @@ function sqliteLaneEligible(limits, table, startRowCursor, imageLane) {
237
237
  async function findImage(ctx, schema, limits, pair, scopePin, rowCursor, probe, logEpoch) {
238
238
  if (probe.length <= limits.limitSnapshotRows)
239
239
  return undefined;
240
- const stored = await ctx.segments.find({
240
+ const stored = await ctx.segments
241
+ .find({
241
242
  partition: ctx.partition,
242
243
  logEpoch,
243
244
  table: pair.table.name,
@@ -246,7 +247,8 @@ async function findImage(ctx, schema, limits, pair, scopePin, rowCursor, probe,
246
247
  scopeDigest: await scopeDigest(pair.effective),
247
248
  asOfCommitSeq: scopePin,
248
249
  rowCursor,
249
- }, clockOf(ctx)());
250
+ }, clockOf(ctx)())
251
+ .catch(segmentStoreFailure(ctx.onError, 'sync', 'find'));
250
252
  return stored !== undefined &&
251
253
  (stored.byteLength <= limitsOf(ctx).maxImageSegmentBytes ||
252
254
  stored.rowCount === 1)
@@ -377,11 +379,13 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, rowCurso
377
379
  throw new Error('sqliteImageBuilder consumed rows out of order');
378
380
  }
379
381
  const last = index < 0 ? beforeBatch : batch[index].rowId;
380
- return segments.put({
382
+ return segments
383
+ .put({
381
384
  ...identity,
382
385
  rowCount: image.rowCount,
383
386
  nextRowCursor: image.rowCount < yielded || !exhausted ? last : null,
384
- }, image.bytes, clockOf(ctx)());
387
+ }, image.bytes, clockOf(ctx)())
388
+ .catch(segmentStoreFailure(ctx.onError, 'sync', 'put'));
385
389
  })();
386
390
  builds.set(key, building);
387
391
  }
@@ -495,6 +499,7 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
495
499
  rowCursor,
496
500
  nextRowCursor,
497
501
  }, bytes, now)
502
+ .catch(segmentStoreFailure(ctx.onError, 'sync', 'put'))
498
503
  .then(async (record) => segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now)));
499
504
  yield { type: 'PENDING_REF', ref };
500
505
  }
@@ -210,12 +210,22 @@ export interface IndexRowPageStatement {
210
210
  export declare function indexRowPageStatement(table: CompiledTable, index: IndexSchema, values: readonly RowValue[], partition: string, afterRowId: string | null | undefined, limit: number, dialect: RelationalDialect): IndexRowPageStatement;
211
211
  /**
212
212
  * One-round-trip page scan for `scanRows`: candidates from the inverted
213
- * scope index (ordered + LIMITed at the covering `sync_row_scopes` PK —
214
- * exactly the old candidate query, so the index-first posture is unchanged)
215
- * LEFT JOINed to the row table, so a whole page arrives in ONE statement
216
- * instead of one lookup per candidate. That per-candidate lookup was the
217
- * cold-bootstrap hot path: a 100k-row snapshot on Postgres paid ~100k
218
- * network round-trips (~10 s at 0.1 ms each) before this join.
213
+ * scope index LEFT JOINed to the row table, so a whole page arrives in ONE
214
+ * statement instead of one lookup per candidate. That per-candidate lookup
215
+ * was the cold-bootstrap hot path: a 100k-row snapshot on Postgres paid
216
+ * ~100k network round-trips (~10 s at 0.1 ms each) before this join.
217
+ *
218
+ * The candidates read at most `limit` entries of the covering
219
+ * `sync_row_scopes` PK per scope value. A bare `value IN (...)` (or
220
+ * `= ANY`) predicate gives up the PK's `row_id` order whenever the planner
221
+ * cannot prove a single value, and then every entry after the cursor is read
222
+ * and sorted before the LIMIT: a cost that grows with the entries left, so a
223
+ * bootstrap is quadratic. PostgreSQL reads one ordered, LIMITed range per
224
+ * value through a LATERAL join and merges them. SQLite has no LATERAL; it
225
+ * bounds the ranges from above instead: the smallest of the values' `limit`th
226
+ * entries after the cursor. That value alone has `limit` entries up to its
227
+ * bound, so the page ends there; a value with fewer entries has no bound, and
228
+ * when no value has one the empty blob `x''` sorts after every TEXT row id.
219
229
  *
220
230
  * LEFT JOIN (not INNER): a candidate whose row vanished (index entry
221
231
  * without a row) must still reach the caller — its `row_id` advances the
@@ -223,11 +233,9 @@ export declare function indexRowPageStatement(table: CompiledTable, index: Index
223
233
  * disappearing from the page.
224
234
  *
225
235
  * Bind order:
226
- * - postgres: [partition, tbl, var, ...values, afterRowId, limit]
227
- * (the join reuses `$1` for the partition);
236
+ * - postgres: [partition, tbl, var, ...values, afterRowId, limit];
228
237
  * - sqlite: [partition, tbl, var, values as JSON array text, afterRowId,
229
- * limit, partition] (positional `?`; the partition binds again for the
230
- * join). The values expand through `json_each`, so the statement text and
238
+ * limit]. The values expand through `json_each`, so the statement text and
231
239
  * parameter count do not depend on `valueCount` and D1's 100-parameter
232
240
  * cap is unreachable through scope values.
233
241
  */
@@ -264,7 +272,7 @@ export declare function scanProbeSql(dialect: RelationalDialect): string;
264
272
  * PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
265
273
  * page of every subscription scanning any table in a pull. `$2` is a JSON
266
274
  * array of `{i, tbl, var, vals, after, lim}` records; each record runs the
267
- * same index-first candidate subquery through a LATERAL join, so every
275
+ * same per-value candidate ranges through a LATERAL join, so every
268
276
  * subscription keeps its own keyset cursor and LIMIT. The candidate side is
269
277
  * table-agnostic (a spec names its table), so the statement joins the row
270
278
  * table once per table present in the batch through a UNION ALL branch that
@@ -538,12 +538,22 @@ export function indexRowPageStatement(table, index, values, partition, afterRowI
538
538
  }
539
539
  /**
540
540
  * One-round-trip page scan for `scanRows`: candidates from the inverted
541
- * scope index (ordered + LIMITed at the covering `sync_row_scopes` PK —
542
- * exactly the old candidate query, so the index-first posture is unchanged)
543
- * LEFT JOINed to the row table, so a whole page arrives in ONE statement
544
- * instead of one lookup per candidate. That per-candidate lookup was the
545
- * cold-bootstrap hot path: a 100k-row snapshot on Postgres paid ~100k
546
- * network round-trips (~10 s at 0.1 ms each) before this join.
541
+ * scope index LEFT JOINed to the row table, so a whole page arrives in ONE
542
+ * statement instead of one lookup per candidate. That per-candidate lookup
543
+ * was the cold-bootstrap hot path: a 100k-row snapshot on Postgres paid
544
+ * ~100k network round-trips (~10 s at 0.1 ms each) before this join.
545
+ *
546
+ * The candidates read at most `limit` entries of the covering
547
+ * `sync_row_scopes` PK per scope value. A bare `value IN (...)` (or
548
+ * `= ANY`) predicate gives up the PK's `row_id` order whenever the planner
549
+ * cannot prove a single value, and then every entry after the cursor is read
550
+ * and sorted before the LIMIT: a cost that grows with the entries left, so a
551
+ * bootstrap is quadratic. PostgreSQL reads one ordered, LIMITed range per
552
+ * value through a LATERAL join and merges them. SQLite has no LATERAL; it
553
+ * bounds the ranges from above instead: the smallest of the values' `limit`th
554
+ * entries after the cursor. That value alone has `limit` entries up to its
555
+ * bound, so the page ends there; a value with fewer entries has no bound, and
556
+ * when no value has one the empty blob `x''` sorts after every TEXT row id.
547
557
  *
548
558
  * LEFT JOIN (not INNER): a candidate whose row vanished (index entry
549
559
  * without a row) must still reach the caller — its `row_id` advances the
@@ -551,29 +561,32 @@ export function indexRowPageStatement(table, index, values, partition, afterRowI
551
561
  * disappearing from the page.
552
562
  *
553
563
  * Bind order:
554
- * - postgres: [partition, tbl, var, ...values, afterRowId, limit]
555
- * (the join reuses `$1` for the partition);
564
+ * - postgres: [partition, tbl, var, ...values, afterRowId, limit];
556
565
  * - sqlite: [partition, tbl, var, values as JSON array text, afterRowId,
557
- * limit, partition] (positional `?`; the partition binds again for the
558
- * join). The values expand through `json_each`, so the statement text and
566
+ * limit]. The values expand through `json_each`, so the statement text and
559
567
  * parameter count do not depend on `valueCount` and D1's 100-parameter
560
568
  * cap is unreachable through scope values.
561
569
  */
562
570
  export function scanRowPageSql(table, valueCount, dialect) {
563
- const p = (n) => (dialect === 'sqlite' ? '?' : `$${n}`);
564
- const values = dialect === 'sqlite'
565
- ? 'SELECT value FROM json_each(?)'
566
- : Array.from({ length: valueCount }, (_, i) => p(4 + i)).join(',');
567
- const after = p(4 + valueCount);
568
- const limit = p(5 + valueCount);
569
- const joinPartition = dialect === 'sqlite' ? '?' : '$1';
571
+ const after = dialect === 'sqlite' ? '?5' : `$${4 + valueCount}`;
572
+ const limit = dialect === 'sqlite' ? '?6' : `$${5 + valueCount}`;
573
+ const candidates = dialect === 'sqlite'
574
+ ? `SELECT DISTINCT row_id FROM sync_row_scopes
575
+ WHERE partition=?1 AND tbl=?2 AND var=?3 AND value IN (SELECT value FROM json_each(?4))
576
+ AND row_id>?5
577
+ AND row_id<=coalesce((SELECT min((SELECT row_id FROM sync_row_scopes
578
+ WHERE partition=?1 AND tbl=?2 AND var=?3 AND value=v.value AND row_id>?5
579
+ ORDER BY row_id LIMIT 1 OFFSET ?6-1)) FROM json_each(?4) v), x'')
580
+ ORDER BY row_id LIMIT ?6`
581
+ : `SELECT DISTINCT x.row_id FROM (VALUES ${Array.from({ length: valueCount }, (_, i) => `($${4 + i})`).join(',')}) v(value)
582
+ CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
583
+ WHERE partition=$1 AND tbl=$2 AND var=$3 AND value=v.value AND row_id>${after}
584
+ ORDER BY row_id LIMIT ${limit}) x
585
+ ORDER BY x.row_id LIMIT ${limit}`;
570
586
  return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
571
- FROM (SELECT DISTINCT row_id FROM sync_row_scopes
572
- WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
573
- AND row_id>${after}
574
- ORDER BY row_id LIMIT ${limit}) c
587
+ FROM (${candidates}) c
575
588
  LEFT JOIN ${quoteIdent(table.name)} r
576
- ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${joinPartition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
589
+ ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${dialect === 'sqlite' ? '?1' : '$1'} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
577
590
  ORDER BY c.row_id`;
578
591
  }
579
592
  /**
@@ -629,7 +642,7 @@ export function scanProbeSql(dialect) {
629
642
  * PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
630
643
  * page of every subscription scanning any table in a pull. `$2` is a JSON
631
644
  * array of `{i, tbl, var, vals, after, lim}` records; each record runs the
632
- * same index-first candidate subquery through a LATERAL join, so every
645
+ * same per-value candidate ranges through a LATERAL join, so every
633
646
  * subscription keeps its own keyset cursor and LIMIT. The candidate side is
634
647
  * table-agnostic (a spec names its table), so the statement joins the row
635
648
  * table once per table present in the batch through a UNION ALL branch that
@@ -655,10 +668,11 @@ export function scanRowBatchSql(tables) {
655
668
  return `WITH cand AS (
656
669
  SELECT s.i AS sub, c.row_id AS row_id, s.tbl AS tbl
657
670
  FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after text, lim int)
658
- CROSS JOIN LATERAL (SELECT DISTINCT row_id FROM sync_row_scopes
659
- WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
660
- AND row_id>s.after
661
- ORDER BY row_id LIMIT s.lim) c)
671
+ CROSS JOIN LATERAL (SELECT DISTINCT x.row_id FROM unnest(s.vals) v(value)
672
+ CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
673
+ WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value AND row_id>s.after
674
+ ORDER BY row_id LIMIT s.lim) x
675
+ ORDER BY x.row_id LIMIT s.lim) c)
662
676
  ${projection}
663
677
  ORDER BY sub, row_id`;
664
678
  }
@@ -1,5 +1,5 @@
1
1
  import { clockOf, RESOLVER_OUTAGE, touchAuthenticatedPartition, } from './context.js';
2
- import { SyncError, syncError } from './errors.js';
2
+ import { SyncError, segmentStoreFailure, syncError } from './errors.js';
3
3
  import { emitEvent } from './events.js';
4
4
  import { compileSchema } from './schema.js';
5
5
  import { schemaWindowOf } from './schema-window.js';
@@ -79,7 +79,9 @@ async function withDownloadEvent(ctx, request, run, bytesOf) {
79
79
  }
80
80
  async function downloadSegment(ctx, request) {
81
81
  const registry = await touchAuthenticatedPartition(ctx);
82
- const entry = await ctx.segments.get(request.segmentId);
82
+ const entry = await ctx.segments
83
+ .get(request.segmentId)
84
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'get'));
83
85
  if (entry === undefined) {
84
86
  throw syncError('sync.not_found', 'unknown segment (§5.5)');
85
87
  }
@@ -89,8 +91,13 @@ async function downloadSegment(ctx, request) {
89
91
  async function openSegment(ctx, request) {
90
92
  const registry = await touchAuthenticatedPartition(ctx);
91
93
  const opened = ctx.segments.open !== undefined
92
- ? await ctx.segments.open(request.segmentId)
93
- : await ctx.segments.get(request.segmentId).then((entry) => entry === undefined
94
+ ? await ctx.segments
95
+ .open(request.segmentId)
96
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'open'))
97
+ : await ctx.segments
98
+ .get(request.segmentId)
99
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'get'))
100
+ .then((entry) => entry === undefined
94
101
  ? undefined
95
102
  : {
96
103
  record: entry.record,
@@ -738,7 +738,6 @@ export class SqliteFamilyStorage {
738
738
  JSON.stringify(values),
739
739
  afterRowId,
740
740
  batchSize,
741
- partition,
742
741
  ]);
743
742
  if (records.length === 0)
744
743
  break;
@@ -20,8 +20,11 @@ export interface ScanProbe {
20
20
  *
21
21
  * A single-variable filter needs no probe. Otherwise `count` counts each
22
22
  * variable's entries in one statement, up to `cap`. The cap starts at
23
- * max(64, limit) and grows fourfold, to at most 65,536, while the fewest
24
- * count is shared by several variables at the cap.
23
+ * max(64, limit) and grows fourfold while the fewest count is shared by
24
+ * several variables at the cap. A round runs only while every tied variable
25
+ * has at least `cap` entries, so the probe ends at the first cap above the
26
+ * smallest count `n`: at most 4n entries per variable in the last round, and
27
+ * the pick is the variable with the fewest entries whatever the starting cap.
25
28
  *
26
29
  * `chosen` memoizes the pick per (partition, table, filter), at most 1,024
27
30
  * entries, so the page loop of a bootstrap probes once. A stale pick only
@@ -10,8 +10,11 @@ import { StorageQueryError } from './storage-errors.js';
10
10
  *
11
11
  * A single-variable filter needs no probe. Otherwise `count` counts each
12
12
  * variable's entries in one statement, up to `cap`. The cap starts at
13
- * max(64, limit) and grows fourfold, to at most 65,536, while the fewest
14
- * count is shared by several variables at the cap.
13
+ * max(64, limit) and grows fourfold while the fewest count is shared by
14
+ * several variables at the cap. A round runs only while every tied variable
15
+ * has at least `cap` entries, so the probe ends at the first cap above the
16
+ * smallest count `n`: at most 4n entries per variable in the last round, and
17
+ * the pick is the variable with the fewest entries whatever the starting cap.
15
18
  *
16
19
  * `chosen` memoizes the pick per (partition, table, filter), at most 1,024
17
20
  * entries, so the page loop of a bootstrap probes once. A stale pick only
@@ -42,7 +45,7 @@ export async function chooseScanVariable(partition, query, chosen, count) {
42
45
  if (cached !== undefined)
43
46
  return cached;
44
47
  let tied = variables;
45
- for (let cap = Math.max(64, query.limit);; cap = Math.min(cap * 4, 65_536)) {
48
+ for (let cap = Math.max(64, query.limit);; cap *= 4) {
46
49
  const counts = tied.map(() => 0);
47
50
  const probes = tied.map((variable, i) => ({
48
51
  i,
@@ -57,7 +60,7 @@ export async function chooseScanVariable(partition, query, chosen, count) {
57
60
  if (fewest === 0)
58
61
  return undefined;
59
62
  tied = tied.filter((_, i) => counts[i] === fewest);
60
- if (fewest < cap || tied.length === 1 || cap >= 65_536)
63
+ if (fewest < cap || tied.length === 1)
61
64
  break;
62
65
  }
63
66
  if (chosen.size >= 1_024)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.96.0",
3
+ "version": "0.98.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.96.0"
71
+ "@syncular/core": "0.98.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
package/src/d1-storage.ts CHANGED
@@ -443,7 +443,6 @@ class D1Transaction
443
443
  JSON.stringify(values),
444
444
  afterRowId,
445
445
  batchSize,
446
- this.partition,
447
446
  )
448
447
  .all<SqliteRowRecord & { payload: Uint8Array | null }>();
449
448
  if (records.length === 0) break;
package/src/errors.ts CHANGED
@@ -242,6 +242,14 @@ export const ERROR_CATALOG: Readonly<Record<string, ErrorCatalogEntry>> = {
242
242
  recommendedAction: 'retryLater',
243
243
  httpStatus: 503,
244
244
  },
245
+ // §5.4/§5.5: a SegmentStore call rejected (an object store outage). The
246
+ // HTTP answer carries `Retry-After: 5` (`errorHeaders`).
247
+ 'sync.segment_store_unavailable': {
248
+ category: 'internal',
249
+ retryable: true,
250
+ recommendedAction: 'retryLater',
251
+ httpStatus: 503,
252
+ },
245
253
  'sync.segment_expired': {
246
254
  category: 'not-found',
247
255
  retryable: true,
@@ -364,6 +372,36 @@ export function errorBody(error: SyncError): {
364
372
  };
365
373
  }
366
374
 
375
+ /**
376
+ * HTTP headers of a §10.1 error answer: `sync.segment_store_unavailable`
377
+ * carries a fixed `Retry-After: 5` (§10.2).
378
+ */
379
+ export function errorHeaders(error: SyncError): Record<string, string> {
380
+ return error.code === 'sync.segment_store_unavailable'
381
+ ? { 'retry-after': '5' }
382
+ : {};
383
+ }
384
+
385
+ /**
386
+ * The rejection handler of a `SegmentStore` call (§5.4, §5.5): the store's
387
+ * error goes to `onError`, and the request fails with the retryable
388
+ * `sync.segment_store_unavailable`, its `details.operation` naming the call.
389
+ */
390
+ export function segmentStoreFailure(
391
+ onError: SyncularErrorHandler | undefined,
392
+ route: 'sync' | 'segments',
393
+ operation: 'put' | 'find' | 'get' | 'open',
394
+ ): (error: unknown) => never {
395
+ return (error) => {
396
+ reportError(error, onError, route);
397
+ throw new SyncError(
398
+ 'sync.segment_store_unavailable',
399
+ 'the segment store failed',
400
+ JSON.stringify({ operation }),
401
+ );
402
+ };
403
+ }
404
+
367
405
  /** The adapter surface that caught an exception reported through `onError`. */
368
406
  export type SyncularErrorRoute =
369
407
  | 'sync'
package/src/events.ts CHANGED
@@ -324,8 +324,9 @@ export interface PruneCompletedEvent {
324
324
  readonly removedCommits: number;
325
325
  }
326
326
 
327
- /** The host `resolveScopes` callback threw — the §3.2/§3.4 fail-loud
328
- * path (units revoke, writes reject, downloads deny). */
327
+ /** The host `resolveScopes` callback threw (§3.2 rule 3): a sync request
328
+ * fails with `sync.internal_error`, a download or realtime registration is
329
+ * denied. */
329
330
  export interface ScopesResolveFailedEvent {
330
331
  readonly type: 'scopes.resolve_failed';
331
332
  readonly atMs: number;
package/src/handler.ts CHANGED
@@ -205,9 +205,8 @@ async function resolveOnce(
205
205
  });
206
206
  } catch (error) {
207
207
  if (error instanceof SyncError) throw error;
208
- // §3.2 rule 5 / §3.4 step 4: fail loud, never leak — units revoke and
209
- // writes reject with sync.forbidden. A throw is NOT an
210
- // outage (§7.3.3): it never reaches the lease.
208
+ // §3.2 rule 3: a throw fails the whole request; nothing is revoked,
209
+ // rejected or leased. A throw is NOT an outage (§7.3.3).
211
210
  const events = ctx.events;
212
211
  if (events !== undefined) {
213
212
  emitEvent(events, {
@@ -219,7 +218,7 @@ async function resolveOnce(
219
218
  message: error instanceof Error ? error.message : String(error),
220
219
  });
221
220
  }
222
- return { resolved: { ok: false }, leaseToEmit: undefined };
221
+ throw error;
223
222
  }
224
223
 
225
224
  if (allowed === RESOLVER_OUTAGE) {
@@ -1357,7 +1356,12 @@ async function* streamResponse(
1357
1356
  }
1358
1357
  }
1359
1358
  } catch (error) {
1360
- if (error instanceof SyncError) {
1359
+ // §5.4: a segment store failure leaves the build, so `createStreamCore`
1360
+ // answers it as HTTP 503 before the header frames and in-band after them.
1361
+ if (
1362
+ error instanceof SyncError &&
1363
+ error.code !== 'sync.segment_store_unavailable'
1364
+ ) {
1361
1365
  report.outcome = 'error';
1362
1366
  report.errorCode = error.code;
1363
1367
  await recordRoundFailure(ctx, clientId, error);
package/src/pull.ts CHANGED
@@ -16,7 +16,7 @@ import {
16
16
  } from '@syncular/core';
17
17
  import type { SyncRequestContext, SyncServerConfig } from './context';
18
18
  import { clockOf, limitsOf } from './context';
19
- import { syncError } from './errors';
19
+ import { segmentStoreFailure, syncError } from './errors';
20
20
  import type { PullSegmentSummary } from './events';
21
21
  import {
22
22
  type CompiledSchema,
@@ -382,19 +382,21 @@ async function findImage(
382
382
  logEpoch: string,
383
383
  ): Promise<SegmentRecord | undefined> {
384
384
  if (probe.length <= limits.limitSnapshotRows) return undefined;
385
- const stored = await ctx.segments.find(
386
- {
387
- partition: ctx.partition,
388
- logEpoch,
389
- table: pair.table.name,
390
- schemaVersion: schema.version,
391
- mediaType: 'sqlite',
392
- scopeDigest: await scopeDigest(pair.effective),
393
- asOfCommitSeq: scopePin,
394
- rowCursor,
395
- },
396
- clockOf(ctx)(),
397
- );
385
+ const stored = await ctx.segments
386
+ .find(
387
+ {
388
+ partition: ctx.partition,
389
+ logEpoch,
390
+ table: pair.table.name,
391
+ schemaVersion: schema.version,
392
+ mediaType: 'sqlite',
393
+ scopeDigest: await scopeDigest(pair.effective),
394
+ asOfCommitSeq: scopePin,
395
+ rowCursor,
396
+ },
397
+ clockOf(ctx)(),
398
+ )
399
+ .catch(segmentStoreFailure(ctx.onError, 'sync', 'find'));
398
400
  return stored !== undefined &&
399
401
  (stored.byteLength <= limitsOf(ctx).maxImageSegmentBytes ||
400
402
  stored.rowCount === 1)
@@ -583,15 +585,17 @@ async function* sqliteImageSegment(
583
585
  throw new Error('sqliteImageBuilder consumed rows out of order');
584
586
  }
585
587
  const last = index < 0 ? beforeBatch : batch[index]!.rowId;
586
- return segments.put(
587
- {
588
- ...identity,
589
- rowCount: image.rowCount,
590
- nextRowCursor: image.rowCount < yielded || !exhausted ? last : null,
591
- },
592
- image.bytes,
593
- clockOf(ctx)(),
594
- );
588
+ return segments
589
+ .put(
590
+ {
591
+ ...identity,
592
+ rowCount: image.rowCount,
593
+ nextRowCursor: image.rowCount < yielded || !exhausted ? last : null,
594
+ },
595
+ image.bytes,
596
+ clockOf(ctx)(),
597
+ )
598
+ .catch(segmentStoreFailure(ctx.onError, 'sync', 'put'));
595
599
  })();
596
600
  builds.set(key, building);
597
601
  }
@@ -764,6 +768,7 @@ async function* tableSegments(
764
768
  bytes,
765
769
  now,
766
770
  )
771
+ .catch(segmentStoreFailure(ctx.onError, 'sync', 'put'))
767
772
  .then(async (record) =>
768
773
  segmentRefFrame(
769
774
  record,
@@ -675,12 +675,22 @@ export function indexRowPageStatement(
675
675
 
676
676
  /**
677
677
  * One-round-trip page scan for `scanRows`: candidates from the inverted
678
- * scope index (ordered + LIMITed at the covering `sync_row_scopes` PK —
679
- * exactly the old candidate query, so the index-first posture is unchanged)
680
- * LEFT JOINed to the row table, so a whole page arrives in ONE statement
681
- * instead of one lookup per candidate. That per-candidate lookup was the
682
- * cold-bootstrap hot path: a 100k-row snapshot on Postgres paid ~100k
683
- * network round-trips (~10 s at 0.1 ms each) before this join.
678
+ * scope index LEFT JOINed to the row table, so a whole page arrives in ONE
679
+ * statement instead of one lookup per candidate. That per-candidate lookup
680
+ * was the cold-bootstrap hot path: a 100k-row snapshot on Postgres paid
681
+ * ~100k network round-trips (~10 s at 0.1 ms each) before this join.
682
+ *
683
+ * The candidates read at most `limit` entries of the covering
684
+ * `sync_row_scopes` PK per scope value. A bare `value IN (...)` (or
685
+ * `= ANY`) predicate gives up the PK's `row_id` order whenever the planner
686
+ * cannot prove a single value, and then every entry after the cursor is read
687
+ * and sorted before the LIMIT: a cost that grows with the entries left, so a
688
+ * bootstrap is quadratic. PostgreSQL reads one ordered, LIMITed range per
689
+ * value through a LATERAL join and merges them. SQLite has no LATERAL; it
690
+ * bounds the ranges from above instead: the smallest of the values' `limit`th
691
+ * entries after the cursor. That value alone has `limit` entries up to its
692
+ * bound, so the page ends there; a value with fewer entries has no bound, and
693
+ * when no value has one the empty blob `x''` sorts after every TEXT row id.
684
694
  *
685
695
  * LEFT JOIN (not INNER): a candidate whose row vanished (index entry
686
696
  * without a row) must still reach the caller — its `row_id` advances the
@@ -688,11 +698,9 @@ export function indexRowPageStatement(
688
698
  * disappearing from the page.
689
699
  *
690
700
  * Bind order:
691
- * - postgres: [partition, tbl, var, ...values, afterRowId, limit]
692
- * (the join reuses `$1` for the partition);
701
+ * - postgres: [partition, tbl, var, ...values, afterRowId, limit];
693
702
  * - sqlite: [partition, tbl, var, values as JSON array text, afterRowId,
694
- * limit, partition] (positional `?`; the partition binds again for the
695
- * join). The values expand through `json_each`, so the statement text and
703
+ * limit]. The values expand through `json_each`, so the statement text and
696
704
  * parameter count do not depend on `valueCount` and D1's 100-parameter
697
705
  * cap is unreachable through scope values.
698
706
  */
@@ -701,21 +709,26 @@ export function scanRowPageSql(
701
709
  valueCount: number,
702
710
  dialect: RelationalDialect,
703
711
  ): string {
704
- const p = (n: number) => (dialect === 'sqlite' ? '?' : `$${n}`);
705
- const values =
712
+ const after = dialect === 'sqlite' ? '?5' : `$${4 + valueCount}`;
713
+ const limit = dialect === 'sqlite' ? '?6' : `$${5 + valueCount}`;
714
+ const candidates =
706
715
  dialect === 'sqlite'
707
- ? 'SELECT value FROM json_each(?)'
708
- : Array.from({ length: valueCount }, (_, i) => p(4 + i)).join(',');
709
- const after = p(4 + valueCount);
710
- const limit = p(5 + valueCount);
711
- const joinPartition = dialect === 'sqlite' ? '?' : '$1';
716
+ ? `SELECT DISTINCT row_id FROM sync_row_scopes
717
+ WHERE partition=?1 AND tbl=?2 AND var=?3 AND value IN (SELECT value FROM json_each(?4))
718
+ AND row_id>?5
719
+ AND row_id<=coalesce((SELECT min((SELECT row_id FROM sync_row_scopes
720
+ WHERE partition=?1 AND tbl=?2 AND var=?3 AND value=v.value AND row_id>?5
721
+ ORDER BY row_id LIMIT 1 OFFSET ?6-1)) FROM json_each(?4) v), x'')
722
+ ORDER BY row_id LIMIT ?6`
723
+ : `SELECT DISTINCT x.row_id FROM (VALUES ${Array.from({ length: valueCount }, (_, i) => `($${4 + i})`).join(',')}) v(value)
724
+ CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
725
+ WHERE partition=$1 AND tbl=$2 AND var=$3 AND value=v.value AND row_id>${after}
726
+ ORDER BY row_id LIMIT ${limit}) x
727
+ ORDER BY x.row_id LIMIT ${limit}`;
712
728
  return `SELECT c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
713
- FROM (SELECT DISTINCT row_id FROM sync_row_scopes
714
- WHERE partition=${p(1)} AND tbl=${p(2)} AND var=${p(3)} AND value IN (${values})
715
- AND row_id>${after}
716
- ORDER BY row_id LIMIT ${limit}) c
729
+ FROM (${candidates}) c
717
730
  LEFT JOIN ${quoteIdent(table.name)} r
718
- ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${joinPartition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
731
+ ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${dialect === 'sqlite' ? '?1' : '$1'} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
719
732
  ORDER BY c.row_id`;
720
733
  }
721
734
 
@@ -774,7 +787,7 @@ export function scanProbeSql(dialect: RelationalDialect): string {
774
787
  * PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
775
788
  * page of every subscription scanning any table in a pull. `$2` is a JSON
776
789
  * array of `{i, tbl, var, vals, after, lim}` records; each record runs the
777
- * same index-first candidate subquery through a LATERAL join, so every
790
+ * same per-value candidate ranges through a LATERAL join, so every
778
791
  * subscription keeps its own keyset cursor and LIMIT. The candidate side is
779
792
  * table-agnostic (a spec names its table), so the statement joins the row
780
793
  * table once per table present in the batch through a UNION ALL branch that
@@ -800,10 +813,11 @@ export function scanRowBatchSql(tables: readonly CompiledTable[]): string {
800
813
  return `WITH cand AS (
801
814
  SELECT s.i AS sub, c.row_id AS row_id, s.tbl AS tbl
802
815
  FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after text, lim int)
803
- CROSS JOIN LATERAL (SELECT DISTINCT row_id FROM sync_row_scopes
804
- WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
805
- AND row_id>s.after
806
- ORDER BY row_id LIMIT s.lim) c)
816
+ CROSS JOIN LATERAL (SELECT DISTINCT x.row_id FROM unnest(s.vals) v(value)
817
+ CROSS JOIN LATERAL (SELECT row_id FROM sync_row_scopes
818
+ WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value AND row_id>s.after
819
+ ORDER BY row_id LIMIT s.lim) x
820
+ ORDER BY x.row_id LIMIT s.lim) c)
807
821
  ${projection}
808
822
  ORDER BY sub, row_id`;
809
823
  }
@@ -15,7 +15,7 @@ import {
15
15
  RESOLVER_OUTAGE,
16
16
  touchAuthenticatedPartition,
17
17
  } from './context';
18
- import { SyncError, syncError } from './errors';
18
+ import { SyncError, segmentStoreFailure, syncError } from './errors';
19
19
  import { emitEvent } from './events';
20
20
  import { compileSchema } from './schema';
21
21
  import { schemaWindowOf } from './schema-window';
@@ -158,7 +158,9 @@ async function downloadSegment(
158
158
  request: SegmentDownloadRequest,
159
159
  ): Promise<SegmentDownloadResult> {
160
160
  const registry = await touchAuthenticatedPartition(ctx);
161
- const entry = await ctx.segments.get(request.segmentId);
161
+ const entry = await ctx.segments
162
+ .get(request.segmentId)
163
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'get'));
162
164
  if (entry === undefined) {
163
165
  throw syncError('sync.not_found', 'unknown segment (§5.5)');
164
166
  }
@@ -178,16 +180,21 @@ async function openSegment(
178
180
  const registry = await touchAuthenticatedPartition(ctx);
179
181
  const opened =
180
182
  ctx.segments.open !== undefined
181
- ? await ctx.segments.open(request.segmentId)
182
- : await ctx.segments.get(request.segmentId).then((entry) =>
183
- entry === undefined
184
- ? undefined
185
- : {
186
- record: entry.record,
187
- body: bytesStream(entry.bytes),
188
- byteLength: entry.bytes.length,
189
- },
190
- );
183
+ ? await ctx.segments
184
+ .open(request.segmentId)
185
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'open'))
186
+ : await ctx.segments
187
+ .get(request.segmentId)
188
+ .catch(segmentStoreFailure(ctx.onError, 'segments', 'get'))
189
+ .then((entry) =>
190
+ entry === undefined
191
+ ? undefined
192
+ : {
193
+ record: entry.record,
194
+ body: bytesStream(entry.bytes),
195
+ byteLength: entry.bytes.length,
196
+ },
197
+ );
191
198
  if (opened === undefined) {
192
199
  throw syncError('sync.not_found', 'unknown segment (§5.5)');
193
200
  }
@@ -1161,7 +1161,6 @@ export abstract class SqliteFamilyStorage {
1161
1161
  JSON.stringify(values),
1162
1162
  afterRowId,
1163
1163
  batchSize,
1164
- partition,
1165
1164
  ]);
1166
1165
  if (records.length === 0) break;
1167
1166
  for (const record of records) {
@@ -35,8 +35,11 @@ export interface ScanProbe {
35
35
  *
36
36
  * A single-variable filter needs no probe. Otherwise `count` counts each
37
37
  * variable's entries in one statement, up to `cap`. The cap starts at
38
- * max(64, limit) and grows fourfold, to at most 65,536, while the fewest
39
- * count is shared by several variables at the cap.
38
+ * max(64, limit) and grows fourfold while the fewest count is shared by
39
+ * several variables at the cap. A round runs only while every tied variable
40
+ * has at least `cap` entries, so the probe ends at the first cap above the
41
+ * smallest count `n`: at most 4n entries per variable in the last round, and
42
+ * the pick is the variable with the fewest entries whatever the starting cap.
40
43
  *
41
44
  * `chosen` memoizes the pick per (partition, table, filter), at most 1,024
42
45
  * entries, so the page loop of a bootstrap probes once. A stale pick only
@@ -78,7 +81,7 @@ export async function chooseScanVariable(
78
81
  const cached = chosen.get(key);
79
82
  if (cached !== undefined) return cached;
80
83
  let tied = variables;
81
- for (let cap = Math.max(64, query.limit); ; cap = Math.min(cap * 4, 65_536)) {
84
+ for (let cap = Math.max(64, query.limit); ; cap *= 4) {
82
85
  const counts = tied.map(() => 0);
83
86
  const probes = tied.map((variable, i) => ({
84
87
  i,
@@ -92,7 +95,7 @@ export async function chooseScanVariable(
92
95
  const fewest = Math.min(...counts);
93
96
  if (fewest === 0) return undefined;
94
97
  tied = tied.filter((_, i) => counts[i] === fewest);
95
- if (fewest < cap || tied.length === 1 || cap >= 65_536) break;
98
+ if (fewest < cap || tied.length === 1) break;
96
99
  }
97
100
  if (chosen.size >= 1_024) chosen.delete(chosen.keys().next().value!);
98
101
  chosen.set(key, tied[0]!);