@syncular/server 0.85.0 → 0.86.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -94,6 +94,16 @@ function wrapperFor(frame) {
94
94
  frames: [stubHeader(), STUB_LIVE_START, frame, STUB_LIVE_END],
95
95
  index: 2,
96
96
  };
97
+ case 'UNIT_EXTENT':
98
+ return {
99
+ frames: [
100
+ stubHeader(),
101
+ { ...STUB_SUB_START, bootstrap: true },
102
+ frame,
103
+ STUB_SUB_END,
104
+ ],
105
+ index: 2,
106
+ };
97
107
  case 'SEGMENT_REF':
98
108
  case 'SEGMENT_INLINE':
99
109
  return {
@@ -131,6 +131,7 @@ export declare class PostgresServerStorage implements ServerStorage {
131
131
  readonly scopes: UnitScopes;
132
132
  }): Promise<number>;
133
133
  readLiveWindow(partition: string, query: LiveWindowQuery): Promise<LiveWindow>;
134
+ countScopedRows(partition: string, query: Pick<RowScanQuery, 'table' | 'scopeFilter'>): Promise<number>;
134
135
  scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
135
136
  scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
136
137
  getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
@@ -43,7 +43,7 @@ import { decodeRow } from '@syncular/core';
43
43
  import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
44
44
  import { syncError } from './errors.js';
45
45
  import { asBytes, asNumber, } from './pg-executor.js';
46
- import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanProbeSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
46
+ import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, countRowsSql, scanRowBatchSql, scanProbeSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
47
47
  import { matchesEffective, sha256Hex } from './scopes.js';
48
48
  import { ClosedCommitConstraintError, isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
49
49
  import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
@@ -2279,12 +2279,32 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2279
2279
  }));
2280
2280
  });
2281
2281
  }
2282
- async scanRows(partition, query) {
2282
+ #scanVariable(partition, query) {
2283
2283
  // The probes of concurrent scans leave as one statement, like their pages.
2284
- const variable = await chooseScanVariable(partition, query, this.#scanVariables, (probes, cap) => Promise.all(probes.map(async (probe) => {
2284
+ return chooseScanVariable(partition, query, this.#scanVariables, (probes, cap) => Promise.all(probes.map(async (probe) => {
2285
2285
  const [row] = await this.#batchedPage('scanProbe', scanProbeSql('postgres'), partition, { tbl: probe.tbl, var: probe.var, vals: probe.vals, lim: cap });
2286
2286
  return { sub: probe.i, n: row?.n };
2287
2287
  })));
2288
+ }
2289
+ async countScopedRows(partition, query) {
2290
+ const variable = await this.#scanVariable(partition, {
2291
+ ...query,
2292
+ afterRowId: null,
2293
+ limit: 0,
2294
+ });
2295
+ if (variable === undefined)
2296
+ return 0;
2297
+ // The counts of concurrent bootstraps leave as one statement.
2298
+ const [row] = await this.#batchedPage('countRows', countRowsSql('postgres'), partition, {
2299
+ tbl: query.table,
2300
+ var: variable,
2301
+ vals: query.scopeFilter[variable],
2302
+ rest: Object.entries(query.scopeFilter).flatMap(([name, vals]) => name === variable ? [] : [{ var: name, vals }]),
2303
+ });
2304
+ return row === undefined ? 0 : asNumber(row.n);
2305
+ }
2306
+ async scanRows(partition, query) {
2307
+ const variable = await this.#scanVariable(partition, query);
2288
2308
  if (variable === undefined)
2289
2309
  return [];
2290
2310
  // Candidates via the inverted index (ordered + LIMITed at the covering
package/dist/pull.d.ts CHANGED
@@ -78,6 +78,11 @@ export interface BootstrapPlan {
78
78
  readonly image: boolean;
79
79
  /** Table index → its start reads. */
80
80
  readonly starts: ReadonlyMap<number, Promise<TableStart>>;
81
+ /** The §4.7 extent the section states, absent for an uncounted bootstrap. */
82
+ readonly extent: Promise<{
83
+ readonly rowsTotal: number;
84
+ readonly rowsDone: number;
85
+ }> | undefined;
81
86
  }
82
87
  /**
83
88
  * Plan one unit's bootstrap and start the reads of every table it can reach
package/dist/pull.js CHANGED
@@ -146,7 +146,9 @@ function parseBootstrapToken(raw, tables, maxSeq) {
146
146
  parsed.tableIndex >= tables.length ||
147
147
  (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
148
148
  (parsed.image !== undefined &&
149
- (parsed.image !== true || parsed.rowCursor === null))) {
149
+ (parsed.image !== true || parsed.rowCursor === null)) ||
150
+ (parsed.rowsTotal === undefined) !== (parsed.rowsDone === undefined) ||
151
+ [parsed.rowsTotal, parsed.rowsDone].some((count) => count !== undefined && (!Number.isSafeInteger(count) || count < 0))) {
150
152
  return undefined;
151
153
  }
152
154
  return {
@@ -155,6 +157,9 @@ function parseBootstrapToken(raw, tables, maxSeq) {
155
157
  tableIndex: parsed.tableIndex,
156
158
  rowCursor: parsed.rowCursor ?? null,
157
159
  ...(parsed.image === true ? { image: true } : {}),
160
+ ...(parsed.rowsTotal !== undefined && parsed.rowsDone !== undefined
161
+ ? { rowsTotal: parsed.rowsTotal, rowsDone: parsed.rowsDone }
162
+ : {}),
158
163
  };
159
164
  }
160
165
  catch {
@@ -412,6 +417,7 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
412
417
  const current = compileSchema(ctx.schema).tables.get(pair.table.name);
413
418
  let rowCursor = startRowCursor;
414
419
  let image = start.image;
420
+ let rows = 0;
415
421
  for (let page = 0; page < pageBudget; page++) {
416
422
  const scanned = page === 0
417
423
  ? start.firstPage
@@ -428,8 +434,9 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
428
434
  ? image.stored
429
435
  : await findImage(ctx, schema, limits, pair, image.scopePin, rowCursor, scanned, logEpoch), scanned);
430
436
  if (record !== undefined) {
437
+ rows += record.rowCount;
431
438
  if (record.nextRowCursor === null)
432
- return { pages: page + 1, rowCursor: null, image: false };
439
+ return { pages: page + 1, rows, rowCursor: null, image: false };
433
440
  rowCursor = record.nextRowCursor;
434
441
  continue;
435
442
  }
@@ -437,6 +444,7 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
437
444
  }
438
445
  const pageRows = scanned.slice(0, limits.limitSnapshotRows);
439
446
  const hasMore = scanned.length > limits.limitSnapshotRows;
447
+ rows += pageRows.length;
440
448
  // §5.2: every row record carries the row's current server_version.
441
449
  const decoded = pageRows.map((row) => ({
442
450
  serverVersion: row.serverVersion,
@@ -487,10 +495,10 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
487
495
  yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
488
496
  }
489
497
  if (!hasMore)
490
- return { pages: page + 1, rowCursor: null, image: false };
498
+ return { pages: page + 1, rows, rowCursor: null, image: false };
491
499
  rowCursor = nextRowCursor;
492
500
  }
493
- return { pages: pageBudget, rowCursor, image: image !== undefined };
501
+ return { pages: pageBudget, rows, rowCursor, image: image !== undefined };
494
502
  }
495
503
  /**
496
504
  * Plan one unit's bootstrap and start the reads of every table it can reach
@@ -520,6 +528,20 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
520
528
  start.catch(() => undefined);
521
529
  starts.set(index, start);
522
530
  }
531
+ // §4.7: a fresh pin counts every active pair once; a resume carries the
532
+ // token's counts, and a token minted without them stays without.
533
+ const extent = resume === undefined
534
+ ? Promise.all([...unit.pairs].map(([table, scopeFilter]) => ctx.storage.countScopedRows(ctx.partition, { table, scopeFilter }))).then((counts) => ({
535
+ rowsTotal: counts.reduce((sum, count) => sum + count, 0),
536
+ rowsDone: 0,
537
+ }))
538
+ : resume.rowsTotal !== undefined && resume.rowsDone !== undefined
539
+ ? Promise.resolve({
540
+ rowsTotal: resume.rowsTotal,
541
+ rowsDone: resume.rowsDone,
542
+ })
543
+ : undefined;
544
+ extent?.catch(() => undefined);
523
545
  return {
524
546
  unit,
525
547
  pin: resume?.asOfCommitSeq ?? maxSeq,
@@ -527,6 +549,7 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
527
549
  rowCursor,
528
550
  image,
529
551
  starts,
552
+ extent,
530
553
  };
531
554
  }
532
555
  /**
@@ -539,6 +562,10 @@ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEp
539
562
  const { unit, pin } = plan;
540
563
  let { tableIndex, rowCursor, image } = plan;
541
564
  let pages = 0;
565
+ const extent = await plan.extent;
566
+ if (extent !== undefined)
567
+ yield { type: 'UNIT_EXTENT', ...extent };
568
+ let rowsDone = extent?.rowsDone ?? 0;
542
569
  for (;;) {
543
570
  while (tableIndex < unit.tables.length &&
544
571
  !unit.pairs.has(unit.tables[tableIndex].name)) {
@@ -555,12 +582,16 @@ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEp
555
582
  tableIndex,
556
583
  rowCursor,
557
584
  ...(image ? { image: true } : {}),
585
+ ...(extent !== undefined
586
+ ? { rowsTotal: extent.rowsTotal, rowsDone }
587
+ : {}),
558
588
  };
559
589
  return JSON.stringify(token);
560
590
  }
561
591
  const table = unit.tables[tableIndex];
562
592
  const outcome = yield* tableSegments(ctx, schema, limits, { table, effective: unit.pairs.get(table.name) }, pin, rowCursor, await plan.starts.get(tableIndex), limits.maxSnapshotPages - pages, trace, logEpoch);
563
593
  pages += outcome.pages;
594
+ rowsDone += outcome.rows;
564
595
  rowCursor = outcome.rowCursor;
565
596
  image = outcome.image;
566
597
  if (rowCursor === null)
@@ -232,6 +232,21 @@ export declare function indexRowPageStatement(table: CompiledTable, index: Index
232
232
  * cap is unreachable through scope values.
233
233
  */
234
234
  export declare function scanRowPageSql(table: CompiledTable, valueCount: number, dialect: RelationalDialect): string;
235
+ /**
236
+ * The §4.7 bootstrap extent of one (table, filter): the `sync_row_scopes`
237
+ * entries of the seek variable whose row also holds an entry of every other
238
+ * variable of the filter. A row holds one entry per variable, so the count is
239
+ * the matching rows; the index changes in the transaction of each row write.
240
+ *
241
+ * Bind order:
242
+ * - postgres: [partition, records as JSON text]; one `(sub, n)` row per
243
+ * `{i, tbl, var, vals, rest}` record, so the counts of concurrent
244
+ * bootstraps share one statement;
245
+ * - sqlite: [partition, tbl, var, values as JSON array text, rest as
246
+ * JSON text]; one `n` row.
247
+ * `rest` is the JSON array of `{var, vals}` for the other variables.
248
+ */
249
+ export declare function countRowsSql(dialect: RelationalDialect): string;
235
250
  /**
236
251
  * Selectivity probe for `chooseScanVariable`: per `{i, tbl, var, vals}`
237
252
  * record of the JSON array, the count of its `sync_row_scopes` entries up
@@ -576,6 +576,33 @@ export function scanRowPageSql(table, valueCount, dialect) {
576
576
  ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${joinPartition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
577
577
  ORDER BY c.row_id`;
578
578
  }
579
+ /**
580
+ * The §4.7 bootstrap extent of one (table, filter): the `sync_row_scopes`
581
+ * entries of the seek variable whose row also holds an entry of every other
582
+ * variable of the filter. A row holds one entry per variable, so the count is
583
+ * the matching rows; the index changes in the transaction of each row write.
584
+ *
585
+ * Bind order:
586
+ * - postgres: [partition, records as JSON text]; one `(sub, n)` row per
587
+ * `{i, tbl, var, vals, rest}` record, so the counts of concurrent
588
+ * bootstraps share one statement;
589
+ * - sqlite: [partition, tbl, var, values as JSON array text, rest as
590
+ * JSON text]; one `n` row.
591
+ * `rest` is the JSON array of `{var, vals}` for the other variables.
592
+ */
593
+ export function countRowsSql(dialect) {
594
+ return dialect === 'sqlite'
595
+ ? `SELECT count(*) AS n FROM sync_row_scopes a
596
+ WHERE a.partition=?1 AND a.tbl=?2 AND a.var=?3 AND a.value IN (SELECT value FROM json_each(?4))
597
+ AND NOT EXISTS (SELECT 1 FROM json_each(?5) f WHERE NOT EXISTS (SELECT 1 FROM sync_row_scopes b
598
+ WHERE b.partition=?1 AND b.tbl=?2 AND b.var=json_extract(f.value, '$.var')
599
+ AND b.value IN (SELECT value FROM json_each(f.value, '$.vals')) AND b.row_id=a.row_id))`
600
+ : `SELECT s.i AS sub, (SELECT count(*) FROM sync_row_scopes a
601
+ WHERE a.partition=$1 AND a.tbl=s.tbl AND a.var=s.var AND a.value=ANY(s.vals)
602
+ AND NOT EXISTS (SELECT 1 FROM jsonb_to_recordset(s.rest) AS f(var text, vals text[]) WHERE NOT EXISTS (SELECT 1 FROM sync_row_scopes b
603
+ WHERE b.partition=$1 AND b.tbl=s.tbl AND b.var=f.var AND b.value=ANY(f.vals) AND b.row_id=a.row_id))) AS n
604
+ FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], rest jsonb)`;
605
+ }
579
606
  /**
580
607
  * Selectivity probe for `chooseScanVariable`: per `{i, tbl, var, vals}`
581
608
  * record of the JSON array, the count of its `sync_row_scopes` entries up
@@ -139,6 +139,8 @@ export declare abstract class SqliteFamilyStorage {
139
139
  readonly scopes: UnitScopes;
140
140
  }): Promise<number>;
141
141
  readLiveWindow(partition: string, query: LiveWindowQuery): Promise<LiveWindow>;
142
+ private scanVariable;
143
+ countScopedRows(partition: string, query: Pick<RowScanQuery, 'table' | 'scopeFilter'>): Promise<number>;
142
144
  scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
143
145
  scanRowsByIndex(partition: string, query: IndexRowScanQuery): Promise<StoredRow[]>;
144
146
  getClientRecord(partition: string, clientId: string): Promise<ClientRecord | undefined>;
@@ -2,7 +2,7 @@ import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './author
2
2
  import { syncError } from './errors.js';
3
3
  import { validateCommitPruneQuery } from './prune.js';
4
4
  import { StorageQueryError } from './storage-errors.js';
5
- import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, scanProbeSql, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
5
+ import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, countRowsSql, scanProbeSql, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
6
6
  import { matchesEffective } from './scopes.js';
7
7
  import { CLIENT_RECORD_COLUMNS, PARTITION_TOUCH_INTERVAL_MS, pushResultFromJson, pushResultToJson, rowsLiveAfterDelete, toClientRecord, toStoredChange, toStoredReaction, toStoredRow, toTombstone, } from './storage.js';
8
8
  import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
@@ -654,12 +654,32 @@ export class SqliteFamilyStorage {
654
654
  change: toStoredChange(record),
655
655
  })));
656
656
  }
657
- async scanRows(partition, query) {
658
- const variable = await chooseScanVariable(partition, query, this.scanVariables, (probes, cap) => this.guardedAll(scanProbeSql('sqlite'), [
657
+ scanVariable(partition, query) {
658
+ return chooseScanVariable(partition, query, this.scanVariables, (probes, cap) => this.guardedAll(scanProbeSql('sqlite'), [
659
659
  partition,
660
660
  JSON.stringify(probes),
661
661
  cap,
662
662
  ]));
663
+ }
664
+ async countScopedRows(partition, query) {
665
+ const variable = await this.scanVariable(partition, {
666
+ ...query,
667
+ afterRowId: null,
668
+ limit: 0,
669
+ });
670
+ if (variable === undefined)
671
+ return 0;
672
+ const [row] = await this.guardedAll(countRowsSql('sqlite'), [
673
+ partition,
674
+ query.table,
675
+ variable,
676
+ JSON.stringify(query.scopeFilter[variable]),
677
+ JSON.stringify(Object.entries(query.scopeFilter).flatMap(([name, vals]) => name === variable ? [] : [{ var: name, vals }])),
678
+ ]);
679
+ return Number(row?.n ?? 0);
680
+ }
681
+ async scanRows(partition, query) {
682
+ const variable = await this.scanVariable(partition, query);
663
683
  if (variable === undefined)
664
684
  return [];
665
685
  const values = query.scopeFilter[variable];
package/dist/storage.d.ts CHANGED
@@ -960,6 +960,13 @@ export interface ServerStorage {
960
960
  }): Promise<number>;
961
961
  /** Scope-filtered snapshot scan, ordered by rowId (bootstrap paging). */
962
962
  scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
963
+ /**
964
+ * The rows `scanRows` returns for the filter from the start of the table,
965
+ * counted (§4.7 bootstrap extent). Reads only the row-scope index: one
966
+ * index range of the variable `scanRows` seeks plus one index seek per
967
+ * other variable and candidate, never the row table.
968
+ */
969
+ countScopedRows(partition: string, query: Pick<RowScanQuery, 'table' | 'scopeFilter'>): Promise<number>;
963
970
  /**
964
971
  * Optional registered-query capability. The implementation MUST replace
965
972
  * every generated app-table relation with a partition-filtered relation and
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.85.0",
3
+ "version": "0.86.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.85.0"
71
+ "@syncular/core": "0.86.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
@@ -112,6 +112,16 @@ function wrapperFor(frame: ResponseFrame): {
112
112
  frames: [stubHeader(), STUB_LIVE_START, frame, STUB_LIVE_END],
113
113
  index: 2,
114
114
  };
115
+ case 'UNIT_EXTENT':
116
+ return {
117
+ frames: [
118
+ stubHeader(),
119
+ { ...STUB_SUB_START, bootstrap: true },
120
+ frame,
121
+ STUB_SUB_END,
122
+ ],
123
+ index: 2,
124
+ };
115
125
  case 'SEGMENT_REF':
116
126
  case 'SEGMENT_INLINE':
117
127
  return {
@@ -100,6 +100,7 @@ import {
100
100
  SYNC_SCOPES_COLUMN,
101
101
  SYNC_VERSION_COLUMN,
102
102
  type StoredColumnLayout,
103
+ countRowsSql,
103
104
  scanRowBatchSql,
104
105
  scanProbeSql,
105
106
  scanRowPageSql,
@@ -3357,9 +3358,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3357
3358
  );
3358
3359
  }
3359
3360
 
3360
- async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
3361
+ #scanVariable(
3362
+ partition: string,
3363
+ query: RowScanQuery,
3364
+ ): Promise<string | undefined> {
3361
3365
  // The probes of concurrent scans leave as one statement, like their pages.
3362
- const variable = await chooseScanVariable(
3366
+ return chooseScanVariable(
3363
3367
  partition,
3364
3368
  query,
3365
3369
  this.#scanVariables,
@@ -3376,6 +3380,37 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3376
3380
  }),
3377
3381
  ),
3378
3382
  );
3383
+ }
3384
+
3385
+ async countScopedRows(
3386
+ partition: string,
3387
+ query: Pick<RowScanQuery, 'table' | 'scopeFilter'>,
3388
+ ): Promise<number> {
3389
+ const variable = await this.#scanVariable(partition, {
3390
+ ...query,
3391
+ afterRowId: null,
3392
+ limit: 0,
3393
+ });
3394
+ if (variable === undefined) return 0;
3395
+ // The counts of concurrent bootstraps leave as one statement.
3396
+ const [row] = await this.#batchedPage<{ n: unknown }>(
3397
+ 'countRows',
3398
+ countRowsSql('postgres'),
3399
+ partition,
3400
+ {
3401
+ tbl: query.table,
3402
+ var: variable,
3403
+ vals: query.scopeFilter[variable],
3404
+ rest: Object.entries(query.scopeFilter).flatMap(([name, vals]) =>
3405
+ name === variable ? [] : [{ var: name, vals }],
3406
+ ),
3407
+ },
3408
+ );
3409
+ return row === undefined ? 0 : asNumber(row.n);
3410
+ }
3411
+
3412
+ async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
3413
+ const variable = await this.#scanVariable(partition, query);
3379
3414
  if (variable === undefined) return [];
3380
3415
  // Candidates via the inverted index (ordered + LIMITed at the covering
3381
3416
  // PK) LEFT JOINed to the row table — one round trip per page, never one
package/src/pull.ts CHANGED
@@ -216,6 +216,9 @@ interface BootstrapToken {
216
216
  rowCursor: string | null;
217
217
  /** Present when the table at `tableIndex` continues on the image lane. */
218
218
  image?: true;
219
+ /** The §4.7 extent: present together, or absent for the bootstrap. */
220
+ rowsTotal?: number;
221
+ rowsDone?: number;
219
222
  }
220
223
 
221
224
  /**
@@ -246,7 +249,12 @@ function parseBootstrapToken(
246
249
  parsed.tableIndex >= tables.length ||
247
250
  (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
248
251
  (parsed.image !== undefined &&
249
- (parsed.image !== true || parsed.rowCursor === null))
252
+ (parsed.image !== true || parsed.rowCursor === null)) ||
253
+ (parsed.rowsTotal === undefined) !== (parsed.rowsDone === undefined) ||
254
+ [parsed.rowsTotal, parsed.rowsDone].some(
255
+ (count) =>
256
+ count !== undefined && (!Number.isSafeInteger(count) || count < 0),
257
+ )
250
258
  ) {
251
259
  return undefined;
252
260
  }
@@ -256,6 +264,9 @@ function parseBootstrapToken(
256
264
  tableIndex: parsed.tableIndex,
257
265
  rowCursor: parsed.rowCursor ?? null,
258
266
  ...(parsed.image === true ? { image: true as const } : {}),
267
+ ...(parsed.rowsTotal !== undefined && parsed.rowsDone !== undefined
268
+ ? { rowsTotal: parsed.rowsTotal, rowsDone: parsed.rowsDone }
269
+ : {}),
259
270
  };
260
271
  } catch {
261
272
  return undefined;
@@ -625,7 +636,7 @@ async function* tableSegments(
625
636
  logEpoch: string,
626
637
  ): AsyncGenerator<
627
638
  ResponseFrame,
628
- { pages: number; rowCursor: string | null; image: boolean }
639
+ { pages: number; rows: number; rowCursor: string | null; image: boolean }
629
640
  > {
630
641
  const { storage, segments, partition } = ctx;
631
642
  const serverLimits = limitsOf(ctx);
@@ -634,6 +645,7 @@ async function* tableSegments(
634
645
  const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
635
646
  let rowCursor = startRowCursor;
636
647
  let image = start.image;
648
+ let rows = 0;
637
649
  for (let page = 0; page < pageBudget; page++) {
638
650
  const scanned =
639
651
  page === 0
@@ -672,8 +684,9 @@ async function* tableSegments(
672
684
  scanned,
673
685
  );
674
686
  if (record !== undefined) {
687
+ rows += record.rowCount;
675
688
  if (record.nextRowCursor === null)
676
- return { pages: page + 1, rowCursor: null, image: false };
689
+ return { pages: page + 1, rows, rowCursor: null, image: false };
677
690
  rowCursor = record.nextRowCursor;
678
691
  continue;
679
692
  }
@@ -681,6 +694,7 @@ async function* tableSegments(
681
694
  }
682
695
  const pageRows = scanned.slice(0, limits.limitSnapshotRows);
683
696
  const hasMore = scanned.length > limits.limitSnapshotRows;
697
+ rows += pageRows.length;
684
698
  // §5.2: every row record carries the row's current server_version.
685
699
  const decoded = pageRows.map((row: StoredRow) => ({
686
700
  serverVersion: row.serverVersion,
@@ -740,10 +754,11 @@ async function* tableSegments(
740
754
  await signedUrlFields(ctx, limits, record.segmentId, digest, now),
741
755
  );
742
756
  }
743
- if (!hasMore) return { pages: page + 1, rowCursor: null, image: false };
757
+ if (!hasMore)
758
+ return { pages: page + 1, rows, rowCursor: null, image: false };
744
759
  rowCursor = nextRowCursor;
745
760
  }
746
- return { pages: pageBudget, rowCursor, image: image !== undefined };
761
+ return { pages: pageBudget, rows, rowCursor, image: image !== undefined };
747
762
  }
748
763
 
749
764
  /** A bootstrapping unit's position and its table starts, read ahead (§4.7). */
@@ -757,6 +772,10 @@ export interface BootstrapPlan {
757
772
  readonly image: boolean;
758
773
  /** Table index → its start reads. */
759
774
  readonly starts: ReadonlyMap<number, Promise<TableStart>>;
775
+ /** The §4.7 extent the section states, absent for an uncounted bootstrap. */
776
+ readonly extent:
777
+ | Promise<{ readonly rowsTotal: number; readonly rowsDone: number }>
778
+ | undefined;
760
779
  }
761
780
 
762
781
  /**
@@ -811,6 +830,25 @@ export function planBootstrap(
811
830
  start.catch(() => undefined);
812
831
  starts.set(index, start);
813
832
  }
833
+ // §4.7: a fresh pin counts every active pair once; a resume carries the
834
+ // token's counts, and a token minted without them stays without.
835
+ const extent =
836
+ resume === undefined
837
+ ? Promise.all(
838
+ [...unit.pairs].map(([table, scopeFilter]) =>
839
+ ctx.storage.countScopedRows(ctx.partition, { table, scopeFilter }),
840
+ ),
841
+ ).then((counts) => ({
842
+ rowsTotal: counts.reduce((sum, count) => sum + count, 0),
843
+ rowsDone: 0,
844
+ }))
845
+ : resume.rowsTotal !== undefined && resume.rowsDone !== undefined
846
+ ? Promise.resolve({
847
+ rowsTotal: resume.rowsTotal,
848
+ rowsDone: resume.rowsDone,
849
+ })
850
+ : undefined;
851
+ extent?.catch(() => undefined);
814
852
  return {
815
853
  unit,
816
854
  pin: resume?.asOfCommitSeq ?? maxSeq,
@@ -818,6 +856,7 @@ export function planBootstrap(
818
856
  rowCursor,
819
857
  image,
820
858
  starts,
859
+ extent,
821
860
  };
822
861
  }
823
862
 
@@ -838,6 +877,9 @@ export async function* bootstrapSegments(
838
877
  const { unit, pin } = plan;
839
878
  let { tableIndex, rowCursor, image } = plan;
840
879
  let pages = 0;
880
+ const extent = await plan.extent;
881
+ if (extent !== undefined) yield { type: 'UNIT_EXTENT', ...extent };
882
+ let rowsDone = extent?.rowsDone ?? 0;
841
883
  for (;;) {
842
884
  while (
843
885
  tableIndex < unit.tables.length &&
@@ -855,6 +897,9 @@ export async function* bootstrapSegments(
855
897
  tableIndex,
856
898
  rowCursor,
857
899
  ...(image ? { image: true as const } : {}),
900
+ ...(extent !== undefined
901
+ ? { rowsTotal: extent.rowsTotal, rowsDone }
902
+ : {}),
858
903
  };
859
904
  return JSON.stringify(token);
860
905
  }
@@ -872,6 +917,7 @@ export async function* bootstrapSegments(
872
917
  logEpoch,
873
918
  );
874
919
  pages += outcome.pages;
920
+ rowsDone += outcome.rows;
875
921
  rowCursor = outcome.rowCursor;
876
922
  image = outcome.image;
877
923
  if (rowCursor === null) tableIndex += 1;
@@ -719,6 +719,34 @@ export function scanRowPageSql(
719
719
  ORDER BY c.row_id`;
720
720
  }
721
721
 
722
+ /**
723
+ * The §4.7 bootstrap extent of one (table, filter): the `sync_row_scopes`
724
+ * entries of the seek variable whose row also holds an entry of every other
725
+ * variable of the filter. A row holds one entry per variable, so the count is
726
+ * the matching rows; the index changes in the transaction of each row write.
727
+ *
728
+ * Bind order:
729
+ * - postgres: [partition, records as JSON text]; one `(sub, n)` row per
730
+ * `{i, tbl, var, vals, rest}` record, so the counts of concurrent
731
+ * bootstraps share one statement;
732
+ * - sqlite: [partition, tbl, var, values as JSON array text, rest as
733
+ * JSON text]; one `n` row.
734
+ * `rest` is the JSON array of `{var, vals}` for the other variables.
735
+ */
736
+ export function countRowsSql(dialect: RelationalDialect): string {
737
+ return dialect === 'sqlite'
738
+ ? `SELECT count(*) AS n FROM sync_row_scopes a
739
+ WHERE a.partition=?1 AND a.tbl=?2 AND a.var=?3 AND a.value IN (SELECT value FROM json_each(?4))
740
+ AND NOT EXISTS (SELECT 1 FROM json_each(?5) f WHERE NOT EXISTS (SELECT 1 FROM sync_row_scopes b
741
+ WHERE b.partition=?1 AND b.tbl=?2 AND b.var=json_extract(f.value, '$.var')
742
+ AND b.value IN (SELECT value FROM json_each(f.value, '$.vals')) AND b.row_id=a.row_id))`
743
+ : `SELECT s.i AS sub, (SELECT count(*) FROM sync_row_scopes a
744
+ WHERE a.partition=$1 AND a.tbl=s.tbl AND a.var=s.var AND a.value=ANY(s.vals)
745
+ AND NOT EXISTS (SELECT 1 FROM jsonb_to_recordset(s.rest) AS f(var text, vals text[]) WHERE NOT EXISTS (SELECT 1 FROM sync_row_scopes b
746
+ WHERE b.partition=$1 AND b.tbl=s.tbl AND b.var=f.var AND b.value=ANY(f.vals) AND b.row_id=a.row_id))) AS n
747
+ FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], rest jsonb)`;
748
+ }
749
+
722
750
  /**
723
751
  * Selectivity probe for `chooseScanVariable`: per `{i, tbl, var, vals}`
724
752
  * record of the JSON array, the count of its `sync_row_scopes` entries up
@@ -28,6 +28,7 @@ import {
28
28
  deleteSqliteRowScopesSql,
29
29
  indexRowPageStatement,
30
30
  quoteIdent,
31
+ countRowsSql,
31
32
  scanProbeSql,
32
33
  scanRowPageSql,
33
34
  rowsReferencingBlobSql,
@@ -1033,8 +1034,11 @@ export abstract class SqliteFamilyStorage {
1033
1034
  );
1034
1035
  }
1035
1036
 
1036
- async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
1037
- const variable = await chooseScanVariable(
1037
+ private scanVariable(
1038
+ partition: string,
1039
+ query: RowScanQuery,
1040
+ ): Promise<string | undefined> {
1041
+ return chooseScanVariable(
1038
1042
  partition,
1039
1043
  query,
1040
1044
  this.scanVariables,
@@ -1045,6 +1049,34 @@ export abstract class SqliteFamilyStorage {
1045
1049
  cap,
1046
1050
  ]),
1047
1051
  );
1052
+ }
1053
+
1054
+ async countScopedRows(
1055
+ partition: string,
1056
+ query: Pick<RowScanQuery, 'table' | 'scopeFilter'>,
1057
+ ): Promise<number> {
1058
+ const variable = await this.scanVariable(partition, {
1059
+ ...query,
1060
+ afterRowId: null,
1061
+ limit: 0,
1062
+ });
1063
+ if (variable === undefined) return 0;
1064
+ const [row] = await this.guardedAll<{ n: number }>(countRowsSql('sqlite'), [
1065
+ partition,
1066
+ query.table,
1067
+ variable,
1068
+ JSON.stringify(query.scopeFilter[variable]),
1069
+ JSON.stringify(
1070
+ Object.entries(query.scopeFilter).flatMap(([name, vals]) =>
1071
+ name === variable ? [] : [{ var: name, vals }],
1072
+ ),
1073
+ ),
1074
+ ]);
1075
+ return Number(row?.n ?? 0);
1076
+ }
1077
+
1078
+ async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
1079
+ const variable = await this.scanVariable(partition, query);
1048
1080
  if (variable === undefined) return [];
1049
1081
  const values = query.scopeFilter[variable]!;
1050
1082
  // Candidates via the inverted index LEFT JOINed to the row table — one
package/src/storage.ts CHANGED
@@ -1494,6 +1494,17 @@ export interface ServerStorage {
1494
1494
  /** Scope-filtered snapshot scan, ordered by rowId (bootstrap paging). */
1495
1495
  scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]>;
1496
1496
 
1497
+ /**
1498
+ * The rows `scanRows` returns for the filter from the start of the table,
1499
+ * counted (§4.7 bootstrap extent). Reads only the row-scope index: one
1500
+ * index range of the variable `scanRows` seeks plus one index seek per
1501
+ * other variable and candidate, never the row table.
1502
+ */
1503
+ countScopedRows(
1504
+ partition: string,
1505
+ query: Pick<RowScanQuery, 'table' | 'scopeFilter'>,
1506
+ ): Promise<number>;
1507
+
1497
1508
  /**
1498
1509
  * Optional registered-query capability. The implementation MUST replace
1499
1510
  * every generated app-table relation with a partition-filtered relation and