@syncular/server 0.85.0 → 0.90.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -112,6 +112,16 @@ function wrapperFor(frame: ResponseFrame): {
112
112
  frames: [stubHeader(), STUB_LIVE_START, frame, STUB_LIVE_END],
113
113
  index: 2,
114
114
  };
115
+ case 'UNIT_EXTENT':
116
+ return {
117
+ frames: [
118
+ stubHeader(),
119
+ { ...STUB_SUB_START, bootstrap: true },
120
+ frame,
121
+ STUB_SUB_END,
122
+ ],
123
+ index: 2,
124
+ };
115
125
  case 'SEGMENT_REF':
116
126
  case 'SEGMENT_INLINE':
117
127
  return {
package/src/handler.ts CHANGED
@@ -14,6 +14,7 @@ import {
14
14
  decodeRequest,
15
15
  echoDigest,
16
16
  type ProgressFrame,
17
+ type RestrictFrame,
17
18
  type PullHeaderFrame,
18
19
  type PushCommitFrame,
19
20
  type PushResultDetailsFrame,
@@ -82,6 +83,8 @@ interface PlannedUnit {
82
83
  readonly plan: UnitPlan;
83
84
  /** Absent for a live unit. */
84
85
  readonly progress: ProgressFrame | undefined;
86
+ /** §4.3: the restriction that follows the unit's `PROGRESS`. */
87
+ readonly restrict: RestrictFrame | undefined;
85
88
  }
86
89
 
87
90
  interface PullPlan {
@@ -259,7 +262,11 @@ async function planRequest(
259
262
  let pullHeader: PullHeaderFrame | undefined;
260
263
  const unitFrames: UnitFrame[] = [];
261
264
  const progressFrames: ProgressFrame[] = [];
265
+ // §4.3: a RESTRICT belongs to the PROGRESS it immediately follows.
266
+ const restricts = new Map<string, RestrictFrame>();
262
267
  for (const frame of request.frames) {
268
+ if (frame.type === 'RESTRICT')
269
+ restricts.set(progressFrames.at(-1)!.id, frame);
263
270
  if (frame.type === 'ROUND_FAILURE') roundFailure = frame;
264
271
  else if (frame.type === 'PUSH_COMMIT') pushes.push(frame);
265
272
  else if (frame.type === 'PULL_HEADER') pullHeader = frame;
@@ -425,6 +432,22 @@ async function planRequest(
425
432
  }
426
433
  progressed.add(frame.id);
427
434
  }
435
+ for (const { id, tables } of list) {
436
+ const restrict = restricts.get(id);
437
+ if (
438
+ restrict !== undefined &&
439
+ (restrict.tables.length >= tables.length ||
440
+ restrict.tables.some(
441
+ (name) => !tables.some((table) => table.name === name),
442
+ ))
443
+ ) {
444
+ throw syncError(
445
+ 'sync.invalid_subscription',
446
+ 'a RESTRICT frame names a table the unit does not cover, or every covered table (§4.3)',
447
+ JSON.stringify({ reason: 'restrict_tables', id }),
448
+ );
449
+ }
450
+ }
428
451
  }
429
452
  }
430
453
 
@@ -450,7 +473,11 @@ async function planRequest(
450
473
  registration,
451
474
  units: list?.map(({ scopes, tables, id }) => {
452
475
  const plan = planUnit(scopes, tables, resolved, id);
453
- return { plan, progress: progress.get(plan.id) };
476
+ return {
477
+ plan,
478
+ progress: progress.get(plan.id),
479
+ restrict: restricts.get(plan.id),
480
+ };
454
481
  }),
455
482
  },
456
483
  record,
@@ -848,26 +875,47 @@ async function* streamResponse(
848
875
  | { kind: 'post'; cursor: number }
849
876
  | {
850
877
  kind: 'bootstrap';
878
+ /** The join cursor (§4.7): the pin, or the kept cursor. */
851
879
  pin: number;
880
+ restricted: boolean;
852
881
  frames: ResponseFrame[];
853
882
  token: string | undefined;
854
883
  };
855
- const states: UnitState[] = units.map(({ plan: unit, progress }) => {
856
- if (unit.status === 'revoked') return { kind: 'revoked' };
857
- if (progress === undefined) return { kind: 'live' };
858
- // §4.7: a token, cursor -1 or a cursor from the future bootstraps.
859
- if (
860
- progress.bootstrapState === undefined &&
861
- progress.cursor >= 0 &&
862
- progress.cursor <= maxSeq
863
- ) {
864
- return progress.cursor < horizonSeq
865
- ? { kind: 'reset' }
866
- : { kind: 'post', cursor: progress.cursor };
867
- }
868
- // Replaced by the section the bootstrap below produces.
869
- return { kind: 'bootstrap', pin: -1, frames: [], token: '' };
870
- });
884
+ const states: UnitState[] = units.map(
885
+ ({ plan: unit, progress, restrict }): UnitState => {
886
+ if (unit.status === 'revoked') return { kind: 'revoked' };
887
+ if (progress === undefined) return { kind: 'live' };
888
+ // §4.6: a kept cursor outside [horizon, max] cannot join.
889
+ if (restrict !== undefined)
890
+ return restrict.cursor < horizonSeq || restrict.cursor > maxSeq
891
+ ? { kind: 'reset' }
892
+ : {
893
+ kind: 'bootstrap',
894
+ pin: -1,
895
+ restricted: true,
896
+ frames: [],
897
+ token: '',
898
+ };
899
+ // §4.7: a token, cursor -1 or a cursor from the future bootstraps.
900
+ if (
901
+ progress.bootstrapState === undefined &&
902
+ progress.cursor >= 0 &&
903
+ progress.cursor <= maxSeq
904
+ ) {
905
+ return progress.cursor < horizonSeq
906
+ ? { kind: 'reset' }
907
+ : { kind: 'post', cursor: progress.cursor };
908
+ }
909
+ // Replaced by the section the bootstrap below produces.
910
+ return {
911
+ kind: 'bootstrap',
912
+ pin: -1,
913
+ restricted: false,
914
+ frames: [],
915
+ token: '',
916
+ };
917
+ },
918
+ );
871
919
  const traces = units.map((): PullSectionTrace | undefined =>
872
920
  events !== undefined ? { segments: [] } : undefined,
873
921
  );
@@ -899,19 +947,36 @@ async function* streamResponse(
899
947
  pages += reads;
900
948
  next += 1;
901
949
  }
902
- const batch = window.map((index) => ({
903
- index,
904
- plan: planBootstrap(
905
- ctx,
906
- schema,
907
- limits,
908
- units[index]!.plan,
909
- units[index]!.progress?.bootstrapState,
910
- maxSeq,
911
- horizonSeq,
912
- plan.logEpoch,
913
- ),
914
- }));
950
+ const batch = window.map((index) => {
951
+ const { plan: unit, progress, restrict } = units[index]!;
952
+ // §4.7: a restricted bootstrap serves only the RESTRICT tables.
953
+ const only =
954
+ restrict === undefined ? undefined : new Set(restrict.tables);
955
+ return {
956
+ index,
957
+ plan: planBootstrap(
958
+ ctx,
959
+ schema,
960
+ limits,
961
+ only === undefined
962
+ ? unit
963
+ : {
964
+ ...unit,
965
+ tables: unit.tables.filter((table) =>
966
+ only.has(table.name),
967
+ ),
968
+ pairs: new Map(
969
+ [...unit.pairs].filter(([name]) => only.has(name)),
970
+ ),
971
+ },
972
+ progress?.bootstrapState,
973
+ maxSeq,
974
+ horizonSeq,
975
+ plan.logEpoch,
976
+ restrict?.cursor,
977
+ ),
978
+ };
979
+ });
915
980
  for (const { index, plan: bootstrap } of batch) {
916
981
  const frames: ResponseFrame[] = [];
917
982
  const segments = bootstrapSegments(
@@ -927,9 +992,11 @@ async function* streamResponse(
927
992
  frames.push(step.value);
928
993
  step = await segments.next();
929
994
  }
995
+ const restrict = units[index]!.restrict;
930
996
  states[index] = {
931
997
  kind: 'bootstrap',
932
- pin: bootstrap.pin,
998
+ pin: restrict?.cursor ?? bootstrap.pin,
999
+ restricted: restrict !== undefined,
933
1000
  frames,
934
1001
  token: step.value,
935
1002
  };
@@ -975,7 +1042,12 @@ async function* streamResponse(
975
1042
  const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
976
1043
  let expired = false;
977
1044
  for (const [index, state] of states.entries()) {
978
- const afterSeq = bound(state);
1045
+ // §4.6: a restricted bootstrap's kept cursor is rechecked too.
1046
+ const afterSeq =
1047
+ bound(state) ??
1048
+ (state.kind === 'bootstrap' && state.restricted
1049
+ ? state.pin
1050
+ : undefined);
979
1051
  if (afterSeq === undefined || afterSeq >= recheck) continue;
980
1052
  expired = true;
981
1053
  if (state.kind === 'live') liveReset = true;
@@ -100,6 +100,7 @@ import {
100
100
  SYNC_SCOPES_COLUMN,
101
101
  SYNC_VERSION_COLUMN,
102
102
  type StoredColumnLayout,
103
+ countRowsSql,
103
104
  scanRowBatchSql,
104
105
  scanProbeSql,
105
106
  scanRowPageSql,
@@ -3301,6 +3302,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3301
3302
  created_at_ms: unknown;
3302
3303
  key: unknown;
3303
3304
  bytes: unknown;
3305
+ changes: unknown;
3304
3306
  }>(
3305
3307
  `WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
3306
3308
  c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
@@ -3310,10 +3312,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3310
3312
  ORDER BY commit_seq LIMIT $4) s
3311
3313
  ORDER BY s.commit_seq LIMIT $4)
3312
3314
  SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
3313
- z.key AS key, z.bytes AS bytes
3315
+ z.key AS key, z.bytes AS bytes, z.changes AS changes
3314
3316
  FROM c
3315
3317
  LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
3316
- LEFT JOIN LATERAL (SELECT k.i AS key,
3318
+ LEFT JOIN LATERAL (SELECT k.i AS key, count(*) AS changes,
3317
3319
  sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
3318
3320
  FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value
3319
3321
  WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
@@ -3333,7 +3335,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3333
3335
  }),
3334
3336
  ...(record.key === null
3335
3337
  ? {}
3336
- : { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
3338
+ : {
3339
+ key: asNumber(record.key),
3340
+ bytes: asNumber(record.bytes),
3341
+ changes: asNumber(record.changes),
3342
+ }),
3337
3343
  }));
3338
3344
  },
3339
3345
  async (keys, commitSeqs) => {
@@ -3357,9 +3363,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3357
3363
  );
3358
3364
  }
3359
3365
 
3360
- async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
3366
+ #scanVariable(
3367
+ partition: string,
3368
+ query: RowScanQuery,
3369
+ ): Promise<string | undefined> {
3361
3370
  // The probes of concurrent scans leave as one statement, like their pages.
3362
- const variable = await chooseScanVariable(
3371
+ return chooseScanVariable(
3363
3372
  partition,
3364
3373
  query,
3365
3374
  this.#scanVariables,
@@ -3376,6 +3385,37 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3376
3385
  }),
3377
3386
  ),
3378
3387
  );
3388
+ }
3389
+
3390
+ async countScopedRows(
3391
+ partition: string,
3392
+ query: Pick<RowScanQuery, 'table' | 'scopeFilter'>,
3393
+ ): Promise<number> {
3394
+ const variable = await this.#scanVariable(partition, {
3395
+ ...query,
3396
+ afterRowId: null,
3397
+ limit: 0,
3398
+ });
3399
+ if (variable === undefined) return 0;
3400
+ // The counts of concurrent bootstraps leave as one statement.
3401
+ const [row] = await this.#batchedPage<{ n: unknown }>(
3402
+ 'countRows',
3403
+ countRowsSql('postgres'),
3404
+ partition,
3405
+ {
3406
+ tbl: query.table,
3407
+ var: variable,
3408
+ vals: query.scopeFilter[variable],
3409
+ rest: Object.entries(query.scopeFilter).flatMap(([name, vals]) =>
3410
+ name === variable ? [] : [{ var: name, vals }],
3411
+ ),
3412
+ },
3413
+ );
3414
+ return row === undefined ? 0 : asNumber(row.n);
3415
+ }
3416
+
3417
+ async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
3418
+ const variable = await this.#scanVariable(partition, query);
3379
3419
  if (variable === undefined) return [];
3380
3420
  // Candidates via the inverted index (ordered + LIMITed at the covering
3381
3421
  // PK) LEFT JOINed to the row table — one round trip per page, never one
package/src/pull.ts CHANGED
@@ -216,6 +216,9 @@ interface BootstrapToken {
216
216
  rowCursor: string | null;
217
217
  /** Present when the table at `tableIndex` continues on the image lane. */
218
218
  image?: true;
219
+ /** The §4.7 extent: present together, or absent for the bootstrap. */
220
+ rowsTotal?: number;
221
+ rowsDone?: number;
219
222
  }
220
223
 
221
224
  /**
@@ -246,7 +249,12 @@ function parseBootstrapToken(
246
249
  parsed.tableIndex >= tables.length ||
247
250
  (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
248
251
  (parsed.image !== undefined &&
249
- (parsed.image !== true || parsed.rowCursor === null))
252
+ (parsed.image !== true || parsed.rowCursor === null)) ||
253
+ (parsed.rowsTotal === undefined) !== (parsed.rowsDone === undefined) ||
254
+ [parsed.rowsTotal, parsed.rowsDone].some(
255
+ (count) =>
256
+ count !== undefined && (!Number.isSafeInteger(count) || count < 0),
257
+ )
250
258
  ) {
251
259
  return undefined;
252
260
  }
@@ -256,6 +264,9 @@ function parseBootstrapToken(
256
264
  tableIndex: parsed.tableIndex,
257
265
  rowCursor: parsed.rowCursor ?? null,
258
266
  ...(parsed.image === true ? { image: true as const } : {}),
267
+ ...(parsed.rowsTotal !== undefined && parsed.rowsDone !== undefined
268
+ ? { rowsTotal: parsed.rowsTotal, rowsDone: parsed.rowsDone }
269
+ : {}),
259
270
  };
260
271
  } catch {
261
272
  return undefined;
@@ -625,7 +636,7 @@ async function* tableSegments(
625
636
  logEpoch: string,
626
637
  ): AsyncGenerator<
627
638
  ResponseFrame,
628
- { pages: number; rowCursor: string | null; image: boolean }
639
+ { pages: number; rows: number; rowCursor: string | null; image: boolean }
629
640
  > {
630
641
  const { storage, segments, partition } = ctx;
631
642
  const serverLimits = limitsOf(ctx);
@@ -634,6 +645,7 @@ async function* tableSegments(
634
645
  const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
635
646
  let rowCursor = startRowCursor;
636
647
  let image = start.image;
648
+ let rows = 0;
637
649
  for (let page = 0; page < pageBudget; page++) {
638
650
  const scanned =
639
651
  page === 0
@@ -672,8 +684,9 @@ async function* tableSegments(
672
684
  scanned,
673
685
  );
674
686
  if (record !== undefined) {
687
+ rows += record.rowCount;
675
688
  if (record.nextRowCursor === null)
676
- return { pages: page + 1, rowCursor: null, image: false };
689
+ return { pages: page + 1, rows, rowCursor: null, image: false };
677
690
  rowCursor = record.nextRowCursor;
678
691
  continue;
679
692
  }
@@ -681,6 +694,7 @@ async function* tableSegments(
681
694
  }
682
695
  const pageRows = scanned.slice(0, limits.limitSnapshotRows);
683
696
  const hasMore = scanned.length > limits.limitSnapshotRows;
697
+ rows += pageRows.length;
684
698
  // §5.2: every row record carries the row's current server_version.
685
699
  const decoded = pageRows.map((row: StoredRow) => ({
686
700
  serverVersion: row.serverVersion,
@@ -740,10 +754,11 @@ async function* tableSegments(
740
754
  await signedUrlFields(ctx, limits, record.segmentId, digest, now),
741
755
  );
742
756
  }
743
- if (!hasMore) return { pages: page + 1, rowCursor: null, image: false };
757
+ if (!hasMore)
758
+ return { pages: page + 1, rows, rowCursor: null, image: false };
744
759
  rowCursor = nextRowCursor;
745
760
  }
746
- return { pages: pageBudget, rowCursor, image: image !== undefined };
761
+ return { pages: pageBudget, rows, rowCursor, image: image !== undefined };
747
762
  }
748
763
 
749
764
  /** A bootstrapping unit's position and its table starts, read ahead (§4.7). */
@@ -757,6 +772,10 @@ export interface BootstrapPlan {
757
772
  readonly image: boolean;
758
773
  /** Table index → its start reads. */
759
774
  readonly starts: ReadonlyMap<number, Promise<TableStart>>;
775
+ /** The §4.7 extent the section states, absent for an uncounted bootstrap. */
776
+ readonly extent:
777
+ | Promise<{ readonly rowsTotal: number; readonly rowsDone: number }>
778
+ | undefined;
760
779
  }
761
780
 
762
781
  /**
@@ -775,12 +794,15 @@ export function planBootstrap(
775
794
  maxSeq: number,
776
795
  horizonSeq: number,
777
796
  logEpoch: string,
797
+ keptCursor = 0,
778
798
  ): BootstrapPlan {
779
- // §4.7: resume at the token's pin unless the token is unusable or the pin
780
- // fell behind the horizon; then restart from a fresh pin.
799
+ // §4.7: resume at the token's pin unless the token is unusable, or the pin
800
+ // fell behind the horizon or a restricted bootstrap's kept cursor; then
801
+ // restart from a fresh pin.
781
802
  const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
782
803
  const resume =
783
- token !== undefined && token.asOfCommitSeq >= horizonSeq
804
+ token !== undefined &&
805
+ token.asOfCommitSeq >= Math.max(horizonSeq, keptCursor)
784
806
  ? token
785
807
  : undefined;
786
808
  const tableIndex = resume?.tableIndex ?? 0;
@@ -811,6 +833,25 @@ export function planBootstrap(
811
833
  start.catch(() => undefined);
812
834
  starts.set(index, start);
813
835
  }
836
+ // §4.7: a fresh pin counts every active pair once; a resume carries the
837
+ // token's counts, and a token minted without them stays without.
838
+ const extent =
839
+ resume === undefined
840
+ ? Promise.all(
841
+ [...unit.pairs].map(([table, scopeFilter]) =>
842
+ ctx.storage.countScopedRows(ctx.partition, { table, scopeFilter }),
843
+ ),
844
+ ).then((counts) => ({
845
+ rowsTotal: counts.reduce((sum, count) => sum + count, 0),
846
+ rowsDone: 0,
847
+ }))
848
+ : resume.rowsTotal !== undefined && resume.rowsDone !== undefined
849
+ ? Promise.resolve({
850
+ rowsTotal: resume.rowsTotal,
851
+ rowsDone: resume.rowsDone,
852
+ })
853
+ : undefined;
854
+ extent?.catch(() => undefined);
814
855
  return {
815
856
  unit,
816
857
  pin: resume?.asOfCommitSeq ?? maxSeq,
@@ -818,6 +859,7 @@ export function planBootstrap(
818
859
  rowCursor,
819
860
  image,
820
861
  starts,
862
+ extent,
821
863
  };
822
864
  }
823
865
 
@@ -838,6 +880,9 @@ export async function* bootstrapSegments(
838
880
  const { unit, pin } = plan;
839
881
  let { tableIndex, rowCursor, image } = plan;
840
882
  let pages = 0;
883
+ const extent = await plan.extent;
884
+ if (extent !== undefined) yield { type: 'UNIT_EXTENT', ...extent };
885
+ let rowsDone = extent?.rowsDone ?? 0;
841
886
  for (;;) {
842
887
  while (
843
888
  tableIndex < unit.tables.length &&
@@ -855,6 +900,9 @@ export async function* bootstrapSegments(
855
900
  tableIndex,
856
901
  rowCursor,
857
902
  ...(image ? { image: true as const } : {}),
903
+ ...(extent !== undefined
904
+ ? { rowsTotal: extent.rowsTotal, rowsDone }
905
+ : {}),
858
906
  };
859
907
  return JSON.stringify(token);
860
908
  }
@@ -872,6 +920,7 @@ export async function* bootstrapSegments(
872
920
  logEpoch,
873
921
  );
874
922
  pages += outcome.pages;
923
+ rowsDone += outcome.rows;
875
924
  rowCursor = outcome.rowCursor;
876
925
  image = outcome.image;
877
926
  if (rowCursor === null) tableIndex += 1;
@@ -719,6 +719,34 @@ export function scanRowPageSql(
719
719
  ORDER BY c.row_id`;
720
720
  }
721
721
 
722
+ /**
723
+ * The §4.7 bootstrap extent of one (table, filter): the `sync_row_scopes`
724
+ * entries of the seek variable whose row also holds an entry of every other
725
+ * variable of the filter. A row holds one entry per variable, so the count is
726
+ * the matching rows; the index changes in the transaction of each row write.
727
+ *
728
+ * Bind order:
729
+ * - postgres: [partition, records as JSON text]; one `(sub, n)` row per
730
+ * `{i, tbl, var, vals, rest}` record, so the counts of concurrent
731
+ * bootstraps share one statement;
732
+ * - sqlite: [partition, tbl, var, values as JSON array text, rest as
733
+ * JSON text]; one `n` row.
734
+ * `rest` is the JSON array of `{var, vals}` for the other variables.
735
+ */
736
+ export function countRowsSql(dialect: RelationalDialect): string {
737
+ return dialect === 'sqlite'
738
+ ? `SELECT count(*) AS n FROM sync_row_scopes a
739
+ WHERE a.partition=?1 AND a.tbl=?2 AND a.var=?3 AND a.value IN (SELECT value FROM json_each(?4))
740
+ AND NOT EXISTS (SELECT 1 FROM json_each(?5) f WHERE NOT EXISTS (SELECT 1 FROM sync_row_scopes b
741
+ WHERE b.partition=?1 AND b.tbl=?2 AND b.var=json_extract(f.value, '$.var')
742
+ AND b.value IN (SELECT value FROM json_each(f.value, '$.vals')) AND b.row_id=a.row_id))`
743
+ : `SELECT s.i AS sub, (SELECT count(*) FROM sync_row_scopes a
744
+ WHERE a.partition=$1 AND a.tbl=s.tbl AND a.var=s.var AND a.value=ANY(s.vals)
745
+ AND NOT EXISTS (SELECT 1 FROM jsonb_to_recordset(s.rest) AS f(var text, vals text[]) WHERE NOT EXISTS (SELECT 1 FROM sync_row_scopes b
746
+ WHERE b.partition=$1 AND b.tbl=s.tbl AND b.var=f.var AND b.value=ANY(f.vals) AND b.row_id=a.row_id))) AS n
747
+ FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], rest jsonb)`;
748
+ }
749
+
722
750
  /**
723
751
  * Selectivity probe for `chooseScanVariable`: per `{i, tbl, var, vals}`
724
752
  * record of the JSON array, the count of its `sync_row_scopes` entries up
@@ -28,6 +28,7 @@ import {
28
28
  deleteSqliteRowScopesSql,
29
29
  indexRowPageStatement,
30
30
  quoteIdent,
31
+ countRowsSql,
31
32
  scanProbeSql,
32
33
  scanRowPageSql,
33
34
  rowsReferencingBlobSql,
@@ -982,6 +983,7 @@ export abstract class SqliteFamilyStorage {
982
983
  created_at_ms: number | null;
983
984
  key: number | null;
984
985
  bytes: number | null;
986
+ changes: number | null;
985
987
  }>(
986
988
  `WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
987
989
  c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
@@ -990,10 +992,10 @@ export abstract class SqliteFamilyStorage {
990
992
  AND s.commit_seq>k.after AND s.commit_seq<=?3
991
993
  ORDER BY s.commit_seq LIMIT ?4)
992
994
  SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
993
- z.i AS key, z.bytes AS bytes
995
+ z.i AS key, z.bytes AS bytes, z.changes AS changes
994
996
  FROM c
995
997
  LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
996
- LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
998
+ LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i, count(*) AS changes,
997
999
  sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
998
1000
  FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
999
1001
  CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
@@ -1012,7 +1014,11 @@ export abstract class SqliteFamilyStorage {
1012
1014
  }
1013
1015
  : {}),
1014
1016
  ...(record.key !== null
1015
- ? { key: record.key, bytes: record.bytes ?? 0 }
1017
+ ? {
1018
+ key: record.key,
1019
+ bytes: record.bytes ?? 0,
1020
+ changes: record.changes ?? 0,
1021
+ }
1016
1022
  : {}),
1017
1023
  })),
1018
1024
  async (keys, commitSeqs) =>
@@ -1033,8 +1039,11 @@ export abstract class SqliteFamilyStorage {
1033
1039
  );
1034
1040
  }
1035
1041
 
1036
- async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
1037
- const variable = await chooseScanVariable(
1042
+ private scanVariable(
1043
+ partition: string,
1044
+ query: RowScanQuery,
1045
+ ): Promise<string | undefined> {
1046
+ return chooseScanVariable(
1038
1047
  partition,
1039
1048
  query,
1040
1049
  this.scanVariables,
@@ -1045,6 +1054,34 @@ export abstract class SqliteFamilyStorage {
1045
1054
  cap,
1046
1055
  ]),
1047
1056
  );
1057
+ }
1058
+
1059
+ async countScopedRows(
1060
+ partition: string,
1061
+ query: Pick<RowScanQuery, 'table' | 'scopeFilter'>,
1062
+ ): Promise<number> {
1063
+ const variable = await this.scanVariable(partition, {
1064
+ ...query,
1065
+ afterRowId: null,
1066
+ limit: 0,
1067
+ });
1068
+ if (variable === undefined) return 0;
1069
+ const [row] = await this.guardedAll<{ n: number }>(countRowsSql('sqlite'), [
1070
+ partition,
1071
+ query.table,
1072
+ variable,
1073
+ JSON.stringify(query.scopeFilter[variable]),
1074
+ JSON.stringify(
1075
+ Object.entries(query.scopeFilter).flatMap(([name, vals]) =>
1076
+ name === variable ? [] : [{ var: name, vals }],
1077
+ ),
1078
+ ),
1079
+ ]);
1080
+ return Number(row?.n ?? 0);
1081
+ }
1082
+
1083
+ async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
1084
+ const variable = await this.scanVariable(partition, query);
1048
1085
  if (variable === undefined) return [];
1049
1086
  const values = query.scopeFilter[variable]!;
1050
1087
  // Candidates via the inverted index LEFT JOINed to the row table — one