@syncular/server 0.85.0 → 0.90.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -94,6 +94,16 @@ function wrapperFor(frame) {
94
94
  frames: [stubHeader(), STUB_LIVE_START, frame, STUB_LIVE_END],
95
95
  index: 2,
96
96
  };
97
+ case 'UNIT_EXTENT':
98
+ return {
99
+ frames: [
100
+ stubHeader(),
101
+ { ...STUB_SUB_START, bootstrap: true },
102
+ frame,
103
+ STUB_SUB_END,
104
+ ],
105
+ index: 2,
106
+ };
97
107
  case 'SEGMENT_REF':
98
108
  case 'SEGMENT_INLINE':
99
109
  return {
package/dist/handler.js CHANGED
@@ -117,7 +117,11 @@ async function planRequest(request, ctx, schema, registry) {
117
117
  let pullHeader;
118
118
  const unitFrames = [];
119
119
  const progressFrames = [];
120
+ // §4.3: a RESTRICT belongs to the PROGRESS it immediately follows.
121
+ const restricts = new Map();
120
122
  for (const frame of request.frames) {
123
+ if (frame.type === 'RESTRICT')
124
+ restricts.set(progressFrames.at(-1).id, frame);
121
125
  if (frame.type === 'ROUND_FAILURE')
122
126
  roundFailure = frame;
123
127
  else if (frame.type === 'PUSH_COMMIT')
@@ -234,6 +238,14 @@ async function planRequest(request, ctx, schema, registry) {
234
238
  }
235
239
  progressed.add(frame.id);
236
240
  }
241
+ for (const { id, tables } of list) {
242
+ const restrict = restricts.get(id);
243
+ if (restrict !== undefined &&
244
+ (restrict.tables.length >= tables.length ||
245
+ restrict.tables.some((name) => !tables.some((table) => table.name === name)))) {
246
+ throw syncError('sync.invalid_subscription', 'a RESTRICT frame names a table the unit does not cover, or every covered table (§4.3)', JSON.stringify({ reason: 'restrict_tables', id }));
247
+ }
248
+ }
237
249
  }
238
250
  }
239
251
  const { resolved, leaseToEmit } = await resolveOnce(ctx, header.clientId, compileSchema(ctx.schema));
@@ -253,7 +265,11 @@ async function planRequest(request, ctx, schema, registry) {
253
265
  registration,
254
266
  units: list?.map(({ scopes, tables, id }) => {
255
267
  const plan = planUnit(scopes, tables, resolved, id);
256
- return { plan, progress: progress.get(plan.id) };
268
+ return {
269
+ plan,
270
+ progress: progress.get(plan.id),
271
+ restrict: restricts.get(plan.id),
272
+ };
257
273
  }),
258
274
  },
259
275
  record,
@@ -551,11 +567,22 @@ async function* streamResponse(plan, ctx, schema, report) {
551
567
  const liveUnits = units.some((unit) => unit.progress === undefined && unit.plan.status === 'active');
552
568
  // §4.6 replica reset: a live unit with R outside [horizon, max].
553
569
  let liveReset = liveUnits && (R < horizonSeq || R > maxSeq);
554
- const states = units.map(({ plan: unit, progress }) => {
570
+ const states = units.map(({ plan: unit, progress, restrict }) => {
555
571
  if (unit.status === 'revoked')
556
572
  return { kind: 'revoked' };
557
573
  if (progress === undefined)
558
574
  return { kind: 'live' };
575
+ // §4.6: a kept cursor outside [horizon, max] cannot join.
576
+ if (restrict !== undefined)
577
+ return restrict.cursor < horizonSeq || restrict.cursor > maxSeq
578
+ ? { kind: 'reset' }
579
+ : {
580
+ kind: 'bootstrap',
581
+ pin: -1,
582
+ restricted: true,
583
+ frames: [],
584
+ token: '',
585
+ };
559
586
  // §4.7: a token, cursor -1 or a cursor from the future bootstraps.
560
587
  if (progress.bootstrapState === undefined &&
561
588
  progress.cursor >= 0 &&
@@ -565,7 +592,13 @@ async function* streamResponse(plan, ctx, schema, report) {
565
592
  : { kind: 'post', cursor: progress.cursor };
566
593
  }
567
594
  // Replaced by the section the bootstrap below produces.
568
- return { kind: 'bootstrap', pin: -1, frames: [], token: '' };
595
+ return {
596
+ kind: 'bootstrap',
597
+ pin: -1,
598
+ restricted: false,
599
+ frames: [],
600
+ token: '',
601
+ };
569
602
  });
570
603
  const traces = units.map(() => events !== undefined ? { segments: [] } : undefined);
571
604
  // Bootstrap sections run before the live read: a bootstrap that
@@ -589,10 +622,21 @@ async function* streamResponse(plan, ctx, schema, report) {
589
622
  pages += reads;
590
623
  next += 1;
591
624
  }
592
- const batch = window.map((index) => ({
593
- index,
594
- plan: planBootstrap(ctx, schema, limits, units[index].plan, units[index].progress?.bootstrapState, maxSeq, horizonSeq, plan.logEpoch),
595
- }));
625
+ const batch = window.map((index) => {
626
+ const { plan: unit, progress, restrict } = units[index];
627
+ // §4.7: a restricted bootstrap serves only the RESTRICT tables.
628
+ const only = restrict === undefined ? undefined : new Set(restrict.tables);
629
+ return {
630
+ index,
631
+ plan: planBootstrap(ctx, schema, limits, only === undefined
632
+ ? unit
633
+ : {
634
+ ...unit,
635
+ tables: unit.tables.filter((table) => only.has(table.name)),
636
+ pairs: new Map([...unit.pairs].filter(([name]) => only.has(name))),
637
+ }, progress?.bootstrapState, maxSeq, horizonSeq, plan.logEpoch, restrict?.cursor),
638
+ };
639
+ });
596
640
  for (const { index, plan: bootstrap } of batch) {
597
641
  const frames = [];
598
642
  const segments = bootstrapSegments(ctx, schema, limits, bootstrap, traces[index], plan.logEpoch);
@@ -601,9 +645,11 @@ async function* streamResponse(plan, ctx, schema, report) {
601
645
  frames.push(step.value);
602
646
  step = await segments.next();
603
647
  }
648
+ const restrict = units[index].restrict;
604
649
  states[index] = {
605
650
  kind: 'bootstrap',
606
- pin: bootstrap.pin,
651
+ pin: restrict?.cursor ?? bootstrap.pin,
652
+ restricted: restrict !== undefined,
607
653
  frames,
608
654
  token: step.value,
609
655
  };
@@ -651,7 +697,11 @@ async function* streamResponse(plan, ctx, schema, report) {
651
697
  const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
652
698
  let expired = false;
653
699
  for (const [index, state] of states.entries()) {
654
- const afterSeq = bound(state);
700
+ // §4.6: a restricted bootstrap's kept cursor is rechecked too.
701
+ const afterSeq = bound(state) ??
702
+ (state.kind === 'bootstrap' && state.restricted
703
+ ? state.pin
704
+ : undefined);
655
705
  if (afterSeq === undefined || afterSeq >= recheck)
656
706
  continue;
657
707
  expired = true;
@@ -131,6 +131,7 @@ export declare class PostgresServerStorage implements ServerStorage {
131
131
  readonly scopes: UnitScopes;
132
132
  }): Promise<number>;
133
133
  readLiveWindow(partition: string, query: LiveWindowQuery): Promise<LiveWindow>;
134
+ countScopedRows(partition: string, query: Pick<RowScanQuery, 'table' | 'scopeFilter'>): Promise<number>;
134
135
  scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
135
136
  scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
136
137
  getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
@@ -43,7 +43,7 @@ import { decodeRow } from '@syncular/core';
43
43
  import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
44
44
  import { syncError } from './errors.js';
45
45
  import { asBytes, asNumber, } from './pg-executor.js';
46
- import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanProbeSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
46
+ import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, countRowsSql, scanRowBatchSql, scanProbeSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
47
47
  import { matchesEffective, sha256Hex } from './scopes.js';
48
48
  import { ClosedCommitConstraintError, isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
49
49
  import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
@@ -2242,10 +2242,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2242
2242
  ORDER BY commit_seq LIMIT $4) s
2243
2243
  ORDER BY s.commit_seq LIMIT $4)
2244
2244
  SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
2245
- z.key AS key, z.bytes AS bytes
2245
+ z.key AS key, z.bytes AS bytes, z.changes AS changes
2246
2246
  FROM c
2247
2247
  LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
2248
- LEFT JOIN LATERAL (SELECT k.i AS key,
2248
+ LEFT JOIN LATERAL (SELECT k.i AS key, count(*) AS changes,
2249
2249
  sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
2250
2250
  FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value
2251
2251
  WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
@@ -2263,7 +2263,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2263
2263
  }),
2264
2264
  ...(record.key === null
2265
2265
  ? {}
2266
- : { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
2266
+ : {
2267
+ key: asNumber(record.key),
2268
+ bytes: asNumber(record.bytes),
2269
+ changes: asNumber(record.changes),
2270
+ }),
2267
2271
  }));
2268
2272
  }, async (keys, commitSeqs) => {
2269
2273
  const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
@@ -2279,12 +2283,32 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2279
2283
  }));
2280
2284
  });
2281
2285
  }
2282
- async scanRows(partition, query) {
2286
+ #scanVariable(partition, query) {
2283
2287
  // The probes of concurrent scans leave as one statement, like their pages.
2284
- const variable = await chooseScanVariable(partition, query, this.#scanVariables, (probes, cap) => Promise.all(probes.map(async (probe) => {
2288
+ return chooseScanVariable(partition, query, this.#scanVariables, (probes, cap) => Promise.all(probes.map(async (probe) => {
2285
2289
  const [row] = await this.#batchedPage('scanProbe', scanProbeSql('postgres'), partition, { tbl: probe.tbl, var: probe.var, vals: probe.vals, lim: cap });
2286
2290
  return { sub: probe.i, n: row?.n };
2287
2291
  })));
2292
+ }
2293
+ async countScopedRows(partition, query) {
2294
+ const variable = await this.#scanVariable(partition, {
2295
+ ...query,
2296
+ afterRowId: null,
2297
+ limit: 0,
2298
+ });
2299
+ if (variable === undefined)
2300
+ return 0;
2301
+ // The counts of concurrent bootstraps leave as one statement.
2302
+ const [row] = await this.#batchedPage('countRows', countRowsSql('postgres'), partition, {
2303
+ tbl: query.table,
2304
+ var: variable,
2305
+ vals: query.scopeFilter[variable],
2306
+ rest: Object.entries(query.scopeFilter).flatMap(([name, vals]) => name === variable ? [] : [{ var: name, vals }]),
2307
+ });
2308
+ return row === undefined ? 0 : asNumber(row.n);
2309
+ }
2310
+ async scanRows(partition, query) {
2311
+ const variable = await this.#scanVariable(partition, query);
2288
2312
  if (variable === undefined)
2289
2313
  return [];
2290
2314
  // Candidates via the inverted index (ordered + LIMITed at the covering
package/dist/pull.d.ts CHANGED
@@ -78,6 +78,11 @@ export interface BootstrapPlan {
78
78
  readonly image: boolean;
79
79
  /** Table index → its start reads. */
80
80
  readonly starts: ReadonlyMap<number, Promise<TableStart>>;
81
+ /** The §4.7 extent the section states, absent for an uncounted bootstrap. */
82
+ readonly extent: Promise<{
83
+ readonly rowsTotal: number;
84
+ readonly rowsDone: number;
85
+ }> | undefined;
81
86
  }
82
87
  /**
83
88
  * Plan one unit's bootstrap and start the reads of every table it can reach
@@ -86,7 +91,7 @@ export interface BootstrapPlan {
86
91
  * is awaited, so a storage that batches concurrent page reads (PostgreSQL,
87
92
  * D1) answers them in one statement per table.
88
93
  */
89
- export declare function planBootstrap(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, unit: UnitPlan, bootstrapState: string | undefined, maxSeq: number, horizonSeq: number, logEpoch: string): BootstrapPlan;
94
+ export declare function planBootstrap(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, unit: UnitPlan, bootstrapState: string | undefined, maxSeq: number, horizonSeq: number, logEpoch: string, keptCursor?: number): BootstrapPlan;
90
95
  /**
91
96
  * The segments of one bootstrap section (§4.7): tables in bootstrap order,
92
97
  * each active pair's rows or image, up to `maxSnapshotPages` pages. Returns
package/dist/pull.js CHANGED
@@ -146,7 +146,9 @@ function parseBootstrapToken(raw, tables, maxSeq) {
146
146
  parsed.tableIndex >= tables.length ||
147
147
  (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
148
148
  (parsed.image !== undefined &&
149
- (parsed.image !== true || parsed.rowCursor === null))) {
149
+ (parsed.image !== true || parsed.rowCursor === null)) ||
150
+ (parsed.rowsTotal === undefined) !== (parsed.rowsDone === undefined) ||
151
+ [parsed.rowsTotal, parsed.rowsDone].some((count) => count !== undefined && (!Number.isSafeInteger(count) || count < 0))) {
150
152
  return undefined;
151
153
  }
152
154
  return {
@@ -155,6 +157,9 @@ function parseBootstrapToken(raw, tables, maxSeq) {
155
157
  tableIndex: parsed.tableIndex,
156
158
  rowCursor: parsed.rowCursor ?? null,
157
159
  ...(parsed.image === true ? { image: true } : {}),
160
+ ...(parsed.rowsTotal !== undefined && parsed.rowsDone !== undefined
161
+ ? { rowsTotal: parsed.rowsTotal, rowsDone: parsed.rowsDone }
162
+ : {}),
158
163
  };
159
164
  }
160
165
  catch {
@@ -412,6 +417,7 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
412
417
  const current = compileSchema(ctx.schema).tables.get(pair.table.name);
413
418
  let rowCursor = startRowCursor;
414
419
  let image = start.image;
420
+ let rows = 0;
415
421
  for (let page = 0; page < pageBudget; page++) {
416
422
  const scanned = page === 0
417
423
  ? start.firstPage
@@ -428,8 +434,9 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
428
434
  ? image.stored
429
435
  : await findImage(ctx, schema, limits, pair, image.scopePin, rowCursor, scanned, logEpoch), scanned);
430
436
  if (record !== undefined) {
437
+ rows += record.rowCount;
431
438
  if (record.nextRowCursor === null)
432
- return { pages: page + 1, rowCursor: null, image: false };
439
+ return { pages: page + 1, rows, rowCursor: null, image: false };
433
440
  rowCursor = record.nextRowCursor;
434
441
  continue;
435
442
  }
@@ -437,6 +444,7 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
437
444
  }
438
445
  const pageRows = scanned.slice(0, limits.limitSnapshotRows);
439
446
  const hasMore = scanned.length > limits.limitSnapshotRows;
447
+ rows += pageRows.length;
440
448
  // §5.2: every row record carries the row's current server_version.
441
449
  const decoded = pageRows.map((row) => ({
442
450
  serverVersion: row.serverVersion,
@@ -487,10 +495,10 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
487
495
  yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
488
496
  }
489
497
  if (!hasMore)
490
- return { pages: page + 1, rowCursor: null, image: false };
498
+ return { pages: page + 1, rows, rowCursor: null, image: false };
491
499
  rowCursor = nextRowCursor;
492
500
  }
493
- return { pages: pageBudget, rowCursor, image: image !== undefined };
501
+ return { pages: pageBudget, rows, rowCursor, image: image !== undefined };
494
502
  }
495
503
  /**
496
504
  * Plan one unit's bootstrap and start the reads of every table it can reach
@@ -499,11 +507,13 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
499
507
  * is awaited, so a storage that batches concurrent page reads (PostgreSQL,
500
508
  * D1) answers them in one statement per table.
501
509
  */
502
- export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq, horizonSeq, logEpoch) {
503
- // §4.7: resume at the token's pin unless the token is unusable or the pin
504
- // fell behind the horizon; then restart from a fresh pin.
510
+ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq, horizonSeq, logEpoch, keptCursor = 0) {
511
+ // §4.7: resume at the token's pin unless the token is unusable, or the pin
512
+ // fell behind the horizon or a restricted bootstrap's kept cursor; then
513
+ // restart from a fresh pin.
505
514
  const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
506
- const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
515
+ const resume = token !== undefined &&
516
+ token.asOfCommitSeq >= Math.max(horizonSeq, keptCursor)
507
517
  ? token
508
518
  : undefined;
509
519
  const tableIndex = resume?.tableIndex ?? 0;
@@ -520,6 +530,20 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
520
530
  start.catch(() => undefined);
521
531
  starts.set(index, start);
522
532
  }
533
+ // §4.7: a fresh pin counts every active pair once; a resume carries the
534
+ // token's counts, and a token minted without them stays without.
535
+ const extent = resume === undefined
536
+ ? Promise.all([...unit.pairs].map(([table, scopeFilter]) => ctx.storage.countScopedRows(ctx.partition, { table, scopeFilter }))).then((counts) => ({
537
+ rowsTotal: counts.reduce((sum, count) => sum + count, 0),
538
+ rowsDone: 0,
539
+ }))
540
+ : resume.rowsTotal !== undefined && resume.rowsDone !== undefined
541
+ ? Promise.resolve({
542
+ rowsTotal: resume.rowsTotal,
543
+ rowsDone: resume.rowsDone,
544
+ })
545
+ : undefined;
546
+ extent?.catch(() => undefined);
523
547
  return {
524
548
  unit,
525
549
  pin: resume?.asOfCommitSeq ?? maxSeq,
@@ -527,6 +551,7 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
527
551
  rowCursor,
528
552
  image,
529
553
  starts,
554
+ extent,
530
555
  };
531
556
  }
532
557
  /**
@@ -539,6 +564,10 @@ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEp
539
564
  const { unit, pin } = plan;
540
565
  let { tableIndex, rowCursor, image } = plan;
541
566
  let pages = 0;
567
+ const extent = await plan.extent;
568
+ if (extent !== undefined)
569
+ yield { type: 'UNIT_EXTENT', ...extent };
570
+ let rowsDone = extent?.rowsDone ?? 0;
542
571
  for (;;) {
543
572
  while (tableIndex < unit.tables.length &&
544
573
  !unit.pairs.has(unit.tables[tableIndex].name)) {
@@ -555,12 +584,16 @@ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEp
555
584
  tableIndex,
556
585
  rowCursor,
557
586
  ...(image ? { image: true } : {}),
587
+ ...(extent !== undefined
588
+ ? { rowsTotal: extent.rowsTotal, rowsDone }
589
+ : {}),
558
590
  };
559
591
  return JSON.stringify(token);
560
592
  }
561
593
  const table = unit.tables[tableIndex];
562
594
  const outcome = yield* tableSegments(ctx, schema, limits, { table, effective: unit.pairs.get(table.name) }, pin, rowCursor, await plan.starts.get(tableIndex), limits.maxSnapshotPages - pages, trace, logEpoch);
563
595
  pages += outcome.pages;
596
+ rowsDone += outcome.rows;
564
597
  rowCursor = outcome.rowCursor;
565
598
  image = outcome.image;
566
599
  if (rowCursor === null)
@@ -232,6 +232,21 @@ export declare function indexRowPageStatement(table: CompiledTable, index: Index
232
232
  * cap is unreachable through scope values.
233
233
  */
234
234
  export declare function scanRowPageSql(table: CompiledTable, valueCount: number, dialect: RelationalDialect): string;
235
+ /**
236
+ * The §4.7 bootstrap extent of one (table, filter): the `sync_row_scopes`
237
+ * entries of the seek variable whose row also holds an entry of every other
238
+ * variable of the filter. A row holds one entry per variable, so the count is
239
+ * the matching rows; the index changes in the transaction of each row write.
240
+ *
241
+ * Bind order:
242
+ * - postgres: [partition, records as JSON text]; one `(sub, n)` row per
243
+ * `{i, tbl, var, vals, rest}` record, so the counts of concurrent
244
+ * bootstraps share one statement;
245
+ * - sqlite: [partition, tbl, var, values as JSON array text, rest as
246
+ * JSON text]; one `n` row.
247
+ * `rest` is the JSON array of `{var, vals}` for the other variables.
248
+ */
249
+ export declare function countRowsSql(dialect: RelationalDialect): string;
235
250
  /**
236
251
  * Selectivity probe for `chooseScanVariable`: per `{i, tbl, var, vals}`
237
252
  * record of the JSON array, the count of its `sync_row_scopes` entries up
@@ -576,6 +576,33 @@ export function scanRowPageSql(table, valueCount, dialect) {
576
576
  ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${joinPartition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
577
577
  ORDER BY c.row_id`;
578
578
  }
579
+ /**
580
+ * The §4.7 bootstrap extent of one (table, filter): the `sync_row_scopes`
581
+ * entries of the seek variable whose row also holds an entry of every other
582
+ * variable of the filter. A row holds one entry per variable, so the count is
583
+ * the matching rows; the index changes in the transaction of each row write.
584
+ *
585
+ * Bind order:
586
+ * - postgres: [partition, records as JSON text]; one `(sub, n)` row per
587
+ * `{i, tbl, var, vals, rest}` record, so the counts of concurrent
588
+ * bootstraps share one statement;
589
+ * - sqlite: [partition, tbl, var, values as JSON array text, rest as
590
+ * JSON text]; one `n` row.
591
+ * `rest` is the JSON array of `{var, vals}` for the other variables.
592
+ */
593
+ export function countRowsSql(dialect) {
594
+ return dialect === 'sqlite'
595
+ ? `SELECT count(*) AS n FROM sync_row_scopes a
596
+ WHERE a.partition=?1 AND a.tbl=?2 AND a.var=?3 AND a.value IN (SELECT value FROM json_each(?4))
597
+ AND NOT EXISTS (SELECT 1 FROM json_each(?5) f WHERE NOT EXISTS (SELECT 1 FROM sync_row_scopes b
598
+ WHERE b.partition=?1 AND b.tbl=?2 AND b.var=json_extract(f.value, '$.var')
599
+ AND b.value IN (SELECT value FROM json_each(f.value, '$.vals')) AND b.row_id=a.row_id))`
600
+ : `SELECT s.i AS sub, (SELECT count(*) FROM sync_row_scopes a
601
+ WHERE a.partition=$1 AND a.tbl=s.tbl AND a.var=s.var AND a.value=ANY(s.vals)
602
+ AND NOT EXISTS (SELECT 1 FROM jsonb_to_recordset(s.rest) AS f(var text, vals text[]) WHERE NOT EXISTS (SELECT 1 FROM sync_row_scopes b
603
+ WHERE b.partition=$1 AND b.tbl=s.tbl AND b.var=f.var AND b.value=ANY(f.vals) AND b.row_id=a.row_id))) AS n
604
+ FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], rest jsonb)`;
605
+ }
579
606
  /**
580
607
  * Selectivity probe for `chooseScanVariable`: per `{i, tbl, var, vals}`
581
608
  * record of the JSON array, the count of its `sync_row_scopes` entries up
@@ -139,6 +139,8 @@ export declare abstract class SqliteFamilyStorage {
139
139
  readonly scopes: UnitScopes;
140
140
  }): Promise<number>;
141
141
  readLiveWindow(partition: string, query: LiveWindowQuery): Promise<LiveWindow>;
142
+ private scanVariable;
143
+ countScopedRows(partition: string, query: Pick<RowScanQuery, 'table' | 'scopeFilter'>): Promise<number>;
142
144
  scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
143
145
  scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
144
146
  getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
@@ -2,7 +2,7 @@ import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './author
2
2
  import { syncError } from './errors.js';
3
3
  import { validateCommitPruneQuery } from './prune.js';
4
4
  import { StorageQueryError } from './storage-errors.js';
5
- import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, scanProbeSql, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
5
+ import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, countRowsSql, scanProbeSql, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
6
6
  import { matchesEffective } from './scopes.js';
7
7
  import { CLIENT_RECORD_COLUMNS, PARTITION_TOUCH_INTERVAL_MS, pushResultFromJson, pushResultToJson, rowsLiveAfterDelete, toClientRecord, toStoredChange, toStoredReaction, toStoredRow, toTombstone, } from './storage.js';
8
8
  import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
@@ -623,10 +623,10 @@ export class SqliteFamilyStorage {
623
623
  AND s.commit_seq>k.after AND s.commit_seq<=?3
624
624
  ORDER BY s.commit_seq LIMIT ?4)
625
625
  SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
626
- z.i AS key, z.bytes AS bytes
626
+ z.i AS key, z.bytes AS bytes, z.changes AS changes
627
627
  FROM c
628
628
  LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
629
- LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
629
+ LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i, count(*) AS changes,
630
630
  sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
631
631
  FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
632
632
  CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
@@ -642,7 +642,11 @@ export class SqliteFamilyStorage {
642
642
  }
643
643
  : {}),
644
644
  ...(record.key !== null
645
- ? { key: record.key, bytes: record.bytes ?? 0 }
645
+ ? {
646
+ key: record.key,
647
+ bytes: record.bytes ?? 0,
648
+ changes: record.changes ?? 0,
649
+ }
646
650
  : {}),
647
651
  })), async (keys, commitSeqs) => (await this.guardedAll(`WITH k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
648
652
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
@@ -654,12 +658,32 @@ export class SqliteFamilyStorage {
654
658
  change: toStoredChange(record),
655
659
  })));
656
660
  }
657
- async scanRows(partition, query) {
658
- const variable = await chooseScanVariable(partition, query, this.scanVariables, (probes, cap) => this.guardedAll(scanProbeSql('sqlite'), [
661
+ scanVariable(partition, query) {
662
+ return chooseScanVariable(partition, query, this.scanVariables, (probes, cap) => this.guardedAll(scanProbeSql('sqlite'), [
659
663
  partition,
660
664
  JSON.stringify(probes),
661
665
  cap,
662
666
  ]));
667
+ }
668
+ async countScopedRows(partition, query) {
669
+ const variable = await this.scanVariable(partition, {
670
+ ...query,
671
+ afterRowId: null,
672
+ limit: 0,
673
+ });
674
+ if (variable === undefined)
675
+ return 0;
676
+ const [row] = await this.guardedAll(countRowsSql('sqlite'), [
677
+ partition,
678
+ query.table,
679
+ variable,
680
+ JSON.stringify(query.scopeFilter[variable]),
681
+ JSON.stringify(Object.entries(query.scopeFilter).flatMap(([name, vals]) => name === variable ? [] : [{ var: name, vals }])),
682
+ ]);
683
+ return Number(row?.n ?? 0);
684
+ }
685
+ async scanRows(partition, query) {
686
+ const variable = await this.scanVariable(partition, query);
663
687
  if (variable === undefined)
664
688
  return [];
665
689
  const values = query.scopeFilter[variable];
@@ -54,10 +54,11 @@ export interface LiveKey {
54
54
  }
55
55
  /**
56
56
  * One row of a live-window candidate page: a candidate commit joined to its
57
- * metadata (absent when the commit vanished) and, per key, the stored bytes
58
- * (payload, row id, scopes) of its changes that key reads: a covered table
59
- * and the key's value in the stored scopes. A commit with no such change
60
- * comes back once without `key`. Rows are in (commitSeq, key) order.
57
+ * metadata (absent when the commit vanished) and, per key, the count and
58
+ * stored bytes (payload, row id, scopes) of its changes that key reads: a
59
+ * covered table and the key's value in the stored scopes. A commit with no
60
+ * such change comes back once without `key`. Rows are in (commitSeq, key)
61
+ * order.
61
62
  */
62
63
  export interface LiveCandidateRow {
63
64
  readonly commitSeq: number;
@@ -67,6 +68,7 @@ export interface LiveCandidateRow {
67
68
  };
68
69
  readonly key?: number;
69
70
  readonly bytes?: number;
71
+ readonly changes?: number;
70
72
  }
71
73
  /**
72
74
  * The §4.5 live scan over a backend's two reads. `readCandidates` returns
@@ -81,6 +83,11 @@ export interface LiveCandidateRow {
81
83
  * bound. Every candidate advances the scan, so a page of non-matching
82
84
  * candidates never repeats.
83
85
  *
86
+ * The counts bound the delivered changes from above, because every change
87
+ * a unit keeps is one its key reads: the scan loads pending candidates once
88
+ * their counts could reach the rest of `limitChanges`, so a page loads
89
+ * about the changes it delivers and never a candidate past the cut.
90
+ *
84
91
  * The sizes bound the loaded bytes by `limitBytes` before any payload is
85
92
  * read: the section stops before the commit that would pass it, and a
86
93
  * commit that passes it alone ends the read as `oversized`, naming the
@@ -110,6 +110,11 @@ export async function transactionQuery(tx, query) {
110
110
  * bound. Every candidate advances the scan, so a page of non-matching
111
111
  * candidates never repeats.
112
112
  *
113
+ * The counts bound the delivered changes from above, because every change
114
+ * a unit keeps is one its key reads: the scan loads pending candidates once
115
+ * their counts could reach the rest of `limitChanges`, so a page loads
116
+ * about the changes it delivers and never a candidate past the cut.
117
+ *
113
118
  * The sizes bound the loaded bytes by `limitBytes` before any payload is
114
119
  * read: the section stops before the commit that would pass it, and a
115
120
  * commit that passes it alone ends the read as `oversized`, naming the
@@ -150,6 +155,8 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
150
155
  const limit = Math.max(64, query.limitChanges);
151
156
  let delivered = 0;
152
157
  let loaded = 0;
158
+ // Upper bound of the changes the pending candidates deliver.
159
+ let bound = 0;
153
160
  let scanned = -1;
154
161
  // Candidates sized but not loaded yet: (scanned, pending.at(-1)].
155
162
  let pending = [];
@@ -162,6 +169,7 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
162
169
  const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
163
170
  const rows = await readChanges(scanKeys(), pending.map((c) => c.commitSeq));
164
171
  pending = [];
172
+ bound = 0;
165
173
  scanned = last.commitSeq;
166
174
  for (let i = 0; i < rows.length;) {
167
175
  const seq = rows[i].commitSeq;
@@ -194,12 +202,14 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
194
202
  const head = rows[i];
195
203
  candidates += 1;
196
204
  let bytes = 0;
205
+ let changes = 0;
197
206
  const sizes = [];
198
207
  for (; rows[i]?.commitSeq === head.commitSeq; i++) {
199
208
  const row = rows[i];
200
209
  if (row.key === undefined)
201
210
  continue;
202
211
  bytes += row.bytes ?? 0;
212
+ changes += row.changes ?? 0;
203
213
  sizes.push([row.key, row.bytes ?? 0]);
204
214
  }
205
215
  if (bytes > query.limitBytes) {
@@ -233,7 +243,13 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
233
243
  return { commits, throughSeq: cut ?? scanned };
234
244
  }
235
245
  loaded += bytes;
246
+ bound += changes;
236
247
  pending.push({ commitSeq: head.commitSeq, meta: head.meta });
248
+ if (delivered + bound >= query.limitChanges) {
249
+ const cut = await load();
250
+ if (cut !== undefined)
251
+ return { commits, throughSeq: cut };
252
+ }
237
253
  }
238
254
  const cut = await load();
239
255
  if (cut !== undefined)
package/dist/storage.d.ts CHANGED
@@ -960,6 +960,13 @@ export interface ServerStorage {
960
960
  }): Promise<number>;
961
961
  /** Scope-filtered snapshot scan, ordered by rowId (bootstrap paging). */
962
962
  scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
963
+ /**
964
+ * The rows `scanRows` returns for the filter from the start of the table,
965
+ * counted (§4.7 bootstrap extent). Reads only the row-scope index: one
966
+ * index range of the variable `scanRows` seeks plus one index seek per
967
+ * other variable and candidate, never the row table.
968
+ */
969
+ countScopedRows(partition: string, query: Pick<RowScanQuery, 'table' | 'scopeFilter'>): Promise<number>;
963
970
  /**
964
971
  * Optional registered-query capability. The implementation MUST replace
965
972
  * every generated app-table relation with a partition-filtered relation and
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.85.0",
3
+ "version": "0.90.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.85.0"
71
+ "@syncular/core": "0.90.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"