@syncular/server 0.86.0 → 0.90.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/handler.js CHANGED
@@ -117,7 +117,11 @@ async function planRequest(request, ctx, schema, registry) {
117
117
  let pullHeader;
118
118
  const unitFrames = [];
119
119
  const progressFrames = [];
120
+ // §4.3: a RESTRICT belongs to the PROGRESS it immediately follows.
121
+ const restricts = new Map();
120
122
  for (const frame of request.frames) {
123
+ if (frame.type === 'RESTRICT')
124
+ restricts.set(progressFrames.at(-1).id, frame);
121
125
  if (frame.type === 'ROUND_FAILURE')
122
126
  roundFailure = frame;
123
127
  else if (frame.type === 'PUSH_COMMIT')
@@ -234,6 +238,14 @@ async function planRequest(request, ctx, schema, registry) {
234
238
  }
235
239
  progressed.add(frame.id);
236
240
  }
241
+ for (const { id, tables } of list) {
242
+ const restrict = restricts.get(id);
243
+ if (restrict !== undefined &&
244
+ (restrict.tables.length >= tables.length ||
245
+ restrict.tables.some((name) => !tables.some((table) => table.name === name)))) {
246
+ throw syncError('sync.invalid_subscription', 'a RESTRICT frame names a table the unit does not cover, or every covered table (§4.3)', JSON.stringify({ reason: 'restrict_tables', id }));
247
+ }
248
+ }
237
249
  }
238
250
  }
239
251
  const { resolved, leaseToEmit } = await resolveOnce(ctx, header.clientId, compileSchema(ctx.schema));
@@ -253,7 +265,11 @@ async function planRequest(request, ctx, schema, registry) {
253
265
  registration,
254
266
  units: list?.map(({ scopes, tables, id }) => {
255
267
  const plan = planUnit(scopes, tables, resolved, id);
256
- return { plan, progress: progress.get(plan.id) };
268
+ return {
269
+ plan,
270
+ progress: progress.get(plan.id),
271
+ restrict: restricts.get(plan.id),
272
+ };
257
273
  }),
258
274
  },
259
275
  record,
@@ -551,11 +567,22 @@ async function* streamResponse(plan, ctx, schema, report) {
551
567
  const liveUnits = units.some((unit) => unit.progress === undefined && unit.plan.status === 'active');
552
568
  // §4.6 replica reset: a live unit with R outside [horizon, max].
553
569
  let liveReset = liveUnits && (R < horizonSeq || R > maxSeq);
554
- const states = units.map(({ plan: unit, progress }) => {
570
+ const states = units.map(({ plan: unit, progress, restrict }) => {
555
571
  if (unit.status === 'revoked')
556
572
  return { kind: 'revoked' };
557
573
  if (progress === undefined)
558
574
  return { kind: 'live' };
575
+ // §4.6: a kept cursor outside [horizon, max] cannot join.
576
+ if (restrict !== undefined)
577
+ return restrict.cursor < horizonSeq || restrict.cursor > maxSeq
578
+ ? { kind: 'reset' }
579
+ : {
580
+ kind: 'bootstrap',
581
+ pin: -1,
582
+ restricted: true,
583
+ frames: [],
584
+ token: '',
585
+ };
559
586
  // §4.7: a token, cursor -1 or a cursor from the future bootstraps.
560
587
  if (progress.bootstrapState === undefined &&
561
588
  progress.cursor >= 0 &&
@@ -565,7 +592,13 @@ async function* streamResponse(plan, ctx, schema, report) {
565
592
  : { kind: 'post', cursor: progress.cursor };
566
593
  }
567
594
  // Replaced by the section the bootstrap below produces.
568
- return { kind: 'bootstrap', pin: -1, frames: [], token: '' };
595
+ return {
596
+ kind: 'bootstrap',
597
+ pin: -1,
598
+ restricted: false,
599
+ frames: [],
600
+ token: '',
601
+ };
569
602
  });
570
603
  const traces = units.map(() => events !== undefined ? { segments: [] } : undefined);
571
604
  // Bootstrap sections run before the live read: a bootstrap that
@@ -589,10 +622,21 @@ async function* streamResponse(plan, ctx, schema, report) {
589
622
  pages += reads;
590
623
  next += 1;
591
624
  }
592
- const batch = window.map((index) => ({
593
- index,
594
- plan: planBootstrap(ctx, schema, limits, units[index].plan, units[index].progress?.bootstrapState, maxSeq, horizonSeq, plan.logEpoch),
595
- }));
625
+ const batch = window.map((index) => {
626
+ const { plan: unit, progress, restrict } = units[index];
627
+ // §4.7: a restricted bootstrap serves only the RESTRICT tables.
628
+ const only = restrict === undefined ? undefined : new Set(restrict.tables);
629
+ return {
630
+ index,
631
+ plan: planBootstrap(ctx, schema, limits, only === undefined
632
+ ? unit
633
+ : {
634
+ ...unit,
635
+ tables: unit.tables.filter((table) => only.has(table.name)),
636
+ pairs: new Map([...unit.pairs].filter(([name]) => only.has(name))),
637
+ }, progress?.bootstrapState, maxSeq, horizonSeq, plan.logEpoch, restrict?.cursor),
638
+ };
639
+ });
596
640
  for (const { index, plan: bootstrap } of batch) {
597
641
  const frames = [];
598
642
  const segments = bootstrapSegments(ctx, schema, limits, bootstrap, traces[index], plan.logEpoch);
@@ -601,9 +645,11 @@ async function* streamResponse(plan, ctx, schema, report) {
601
645
  frames.push(step.value);
602
646
  step = await segments.next();
603
647
  }
648
+ const restrict = units[index].restrict;
604
649
  states[index] = {
605
650
  kind: 'bootstrap',
606
- pin: bootstrap.pin,
651
+ pin: restrict?.cursor ?? bootstrap.pin,
652
+ restricted: restrict !== undefined,
607
653
  frames,
608
654
  token: step.value,
609
655
  };
@@ -651,7 +697,11 @@ async function* streamResponse(plan, ctx, schema, report) {
651
697
  const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
652
698
  let expired = false;
653
699
  for (const [index, state] of states.entries()) {
654
- const afterSeq = bound(state);
700
+ // §4.6: a restricted bootstrap's kept cursor is rechecked too.
701
+ const afterSeq = bound(state) ??
702
+ (state.kind === 'bootstrap' && state.restricted
703
+ ? state.pin
704
+ : undefined);
655
705
  if (afterSeq === undefined || afterSeq >= recheck)
656
706
  continue;
657
707
  expired = true;
@@ -2242,10 +2242,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2242
2242
  ORDER BY commit_seq LIMIT $4) s
2243
2243
  ORDER BY s.commit_seq LIMIT $4)
2244
2244
  SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
2245
- z.key AS key, z.bytes AS bytes
2245
+ z.key AS key, z.bytes AS bytes, z.changes AS changes
2246
2246
  FROM c
2247
2247
  LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
2248
- LEFT JOIN LATERAL (SELECT k.i AS key,
2248
+ LEFT JOIN LATERAL (SELECT k.i AS key, count(*) AS changes,
2249
2249
  sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
2250
2250
  FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value
2251
2251
  WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
@@ -2263,7 +2263,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2263
2263
  }),
2264
2264
  ...(record.key === null
2265
2265
  ? {}
2266
- : { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
2266
+ : {
2267
+ key: asNumber(record.key),
2268
+ bytes: asNumber(record.bytes),
2269
+ changes: asNumber(record.changes),
2270
+ }),
2267
2271
  }));
2268
2272
  }, async (keys, commitSeqs) => {
2269
2273
  const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
package/dist/pull.d.ts CHANGED
@@ -91,7 +91,7 @@ export interface BootstrapPlan {
91
91
  * is awaited, so a storage that batches concurrent page reads (PostgreSQL,
92
92
  * D1) answers them in one statement per table.
93
93
  */
94
- export declare function planBootstrap(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, unit: UnitPlan, bootstrapState: string | undefined, maxSeq: number, horizonSeq: number, logEpoch: string): BootstrapPlan;
94
+ export declare function planBootstrap(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, unit: UnitPlan, bootstrapState: string | undefined, maxSeq: number, horizonSeq: number, logEpoch: string, keptCursor?: number): BootstrapPlan;
95
95
  /**
96
96
  * The segments of one bootstrap section (§4.7): tables in bootstrap order,
97
97
  * each active pair's rows or image, up to `maxSnapshotPages` pages. Returns
package/dist/pull.js CHANGED
@@ -507,11 +507,13 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
507
507
  * is awaited, so a storage that batches concurrent page reads (PostgreSQL,
508
508
  * D1) answers them in one statement per table.
509
509
  */
510
- export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq, horizonSeq, logEpoch) {
511
- // §4.7: resume at the token's pin unless the token is unusable or the pin
512
- // fell behind the horizon; then restart from a fresh pin.
510
+ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq, horizonSeq, logEpoch, keptCursor = 0) {
511
+ // §4.7: resume at the token's pin unless the token is unusable, or the pin
512
+ // fell behind the horizon or a restricted bootstrap's kept cursor; then
513
+ // restart from a fresh pin.
513
514
  const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
514
- const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
515
+ const resume = token !== undefined &&
516
+ token.asOfCommitSeq >= Math.max(horizonSeq, keptCursor)
515
517
  ? token
516
518
  : undefined;
517
519
  const tableIndex = resume?.tableIndex ?? 0;
@@ -623,10 +623,10 @@ export class SqliteFamilyStorage {
623
623
  AND s.commit_seq>k.after AND s.commit_seq<=?3
624
624
  ORDER BY s.commit_seq LIMIT ?4)
625
625
  SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
626
- z.i AS key, z.bytes AS bytes
626
+ z.i AS key, z.bytes AS bytes, z.changes AS changes
627
627
  FROM c
628
628
  LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
629
- LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
629
+ LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i, count(*) AS changes,
630
630
  sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
631
631
  FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
632
632
  CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
@@ -642,7 +642,11 @@ export class SqliteFamilyStorage {
642
642
  }
643
643
  : {}),
644
644
  ...(record.key !== null
645
- ? { key: record.key, bytes: record.bytes ?? 0 }
645
+ ? {
646
+ key: record.key,
647
+ bytes: record.bytes ?? 0,
648
+ changes: record.changes ?? 0,
649
+ }
646
650
  : {}),
647
651
  })), async (keys, commitSeqs) => (await this.guardedAll(`WITH k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
648
652
  SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
@@ -54,10 +54,11 @@ export interface LiveKey {
54
54
  }
55
55
  /**
56
56
  * One row of a live-window candidate page: a candidate commit joined to its
57
- * metadata (absent when the commit vanished) and, per key, the stored bytes
58
- * (payload, row id, scopes) of its changes that key reads: a covered table
59
- * and the key's value in the stored scopes. A commit with no such change
60
- * comes back once without `key`. Rows are in (commitSeq, key) order.
57
+ * metadata (absent when the commit vanished) and, per key, the count and
58
+ * stored bytes (payload, row id, scopes) of its changes that key reads: a
59
+ * covered table and the key's value in the stored scopes. A commit with no
60
+ * such change comes back once without `key`. Rows are in (commitSeq, key)
61
+ * order.
61
62
  */
62
63
  export interface LiveCandidateRow {
63
64
  readonly commitSeq: number;
@@ -67,6 +68,7 @@ export interface LiveCandidateRow {
67
68
  };
68
69
  readonly key?: number;
69
70
  readonly bytes?: number;
71
+ readonly changes?: number;
70
72
  }
71
73
  /**
72
74
  * The §4.5 live scan over a backend's two reads. `readCandidates` returns
@@ -81,6 +83,11 @@ export interface LiveCandidateRow {
81
83
  * bound. Every candidate advances the scan, so a page of non-matching
82
84
  * candidates never repeats.
83
85
  *
86
+ * The counts bound the delivered changes from above, because every change
87
+ * a unit keeps is one its key reads: the scan loads pending candidates once
88
+ * their counts could reach the rest of `limitChanges`, so a page loads
89
+ * about the changes it delivers and never a candidate past the cut.
90
+ *
84
91
  * The sizes bound the loaded bytes by `limitBytes` before any payload is
85
92
  * read: the section stops before the commit that would pass it, and a
86
93
  * commit that passes it alone ends the read as `oversized`, naming the
@@ -110,6 +110,11 @@ export async function transactionQuery(tx, query) {
110
110
  * bound. Every candidate advances the scan, so a page of non-matching
111
111
  * candidates never repeats.
112
112
  *
113
+ * The counts bound the delivered changes from above, because every change
114
+ * a unit keeps is one its key reads: the scan loads pending candidates once
115
+ * their counts could reach the rest of `limitChanges`, so a page loads
116
+ * about the changes it delivers and never a candidate past the cut.
117
+ *
113
118
  * The sizes bound the loaded bytes by `limitBytes` before any payload is
114
119
  * read: the section stops before the commit that would pass it, and a
115
120
  * commit that passes it alone ends the read as `oversized`, naming the
@@ -150,6 +155,8 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
150
155
  const limit = Math.max(64, query.limitChanges);
151
156
  let delivered = 0;
152
157
  let loaded = 0;
158
+ // Upper bound of the changes the pending candidates deliver.
159
+ let bound = 0;
153
160
  let scanned = -1;
154
161
  // Candidates sized but not loaded yet: (scanned, pending.at(-1)].
155
162
  let pending = [];
@@ -162,6 +169,7 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
162
169
  const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
163
170
  const rows = await readChanges(scanKeys(), pending.map((c) => c.commitSeq));
164
171
  pending = [];
172
+ bound = 0;
165
173
  scanned = last.commitSeq;
166
174
  for (let i = 0; i < rows.length;) {
167
175
  const seq = rows[i].commitSeq;
@@ -194,12 +202,14 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
194
202
  const head = rows[i];
195
203
  candidates += 1;
196
204
  let bytes = 0;
205
+ let changes = 0;
197
206
  const sizes = [];
198
207
  for (; rows[i]?.commitSeq === head.commitSeq; i++) {
199
208
  const row = rows[i];
200
209
  if (row.key === undefined)
201
210
  continue;
202
211
  bytes += row.bytes ?? 0;
212
+ changes += row.changes ?? 0;
203
213
  sizes.push([row.key, row.bytes ?? 0]);
204
214
  }
205
215
  if (bytes > query.limitBytes) {
@@ -233,7 +243,13 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
233
243
  return { commits, throughSeq: cut ?? scanned };
234
244
  }
235
245
  loaded += bytes;
246
+ bound += changes;
236
247
  pending.push({ commitSeq: head.commitSeq, meta: head.meta });
248
+ if (delivered + bound >= query.limitChanges) {
249
+ const cut = await load();
250
+ if (cut !== undefined)
251
+ return { commits, throughSeq: cut };
252
+ }
237
253
  }
238
254
  const cut = await load();
239
255
  if (cut !== undefined)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.86.0",
3
+ "version": "0.90.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.86.0"
71
+ "@syncular/core": "0.90.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
package/src/handler.ts CHANGED
@@ -14,6 +14,7 @@ import {
14
14
  decodeRequest,
15
15
  echoDigest,
16
16
  type ProgressFrame,
17
+ type RestrictFrame,
17
18
  type PullHeaderFrame,
18
19
  type PushCommitFrame,
19
20
  type PushResultDetailsFrame,
@@ -82,6 +83,8 @@ interface PlannedUnit {
82
83
  readonly plan: UnitPlan;
83
84
  /** Absent for a live unit. */
84
85
  readonly progress: ProgressFrame | undefined;
86
+ /** §4.3: the restriction that follows the unit's `PROGRESS`. */
87
+ readonly restrict: RestrictFrame | undefined;
85
88
  }
86
89
 
87
90
  interface PullPlan {
@@ -259,7 +262,11 @@ async function planRequest(
259
262
  let pullHeader: PullHeaderFrame | undefined;
260
263
  const unitFrames: UnitFrame[] = [];
261
264
  const progressFrames: ProgressFrame[] = [];
265
+ // §4.3: a RESTRICT belongs to the PROGRESS it immediately follows.
266
+ const restricts = new Map<string, RestrictFrame>();
262
267
  for (const frame of request.frames) {
268
+ if (frame.type === 'RESTRICT')
269
+ restricts.set(progressFrames.at(-1)!.id, frame);
263
270
  if (frame.type === 'ROUND_FAILURE') roundFailure = frame;
264
271
  else if (frame.type === 'PUSH_COMMIT') pushes.push(frame);
265
272
  else if (frame.type === 'PULL_HEADER') pullHeader = frame;
@@ -425,6 +432,22 @@ async function planRequest(
425
432
  }
426
433
  progressed.add(frame.id);
427
434
  }
435
+ for (const { id, tables } of list) {
436
+ const restrict = restricts.get(id);
437
+ if (
438
+ restrict !== undefined &&
439
+ (restrict.tables.length >= tables.length ||
440
+ restrict.tables.some(
441
+ (name) => !tables.some((table) => table.name === name),
442
+ ))
443
+ ) {
444
+ throw syncError(
445
+ 'sync.invalid_subscription',
446
+ 'a RESTRICT frame names a table the unit does not cover, or every covered table (§4.3)',
447
+ JSON.stringify({ reason: 'restrict_tables', id }),
448
+ );
449
+ }
450
+ }
428
451
  }
429
452
  }
430
453
 
@@ -450,7 +473,11 @@ async function planRequest(
450
473
  registration,
451
474
  units: list?.map(({ scopes, tables, id }) => {
452
475
  const plan = planUnit(scopes, tables, resolved, id);
453
- return { plan, progress: progress.get(plan.id) };
476
+ return {
477
+ plan,
478
+ progress: progress.get(plan.id),
479
+ restrict: restricts.get(plan.id),
480
+ };
454
481
  }),
455
482
  },
456
483
  record,
@@ -848,26 +875,47 @@ async function* streamResponse(
848
875
  | { kind: 'post'; cursor: number }
849
876
  | {
850
877
  kind: 'bootstrap';
878
+ /** The join cursor (§4.7): the pin, or the kept cursor. */
851
879
  pin: number;
880
+ restricted: boolean;
852
881
  frames: ResponseFrame[];
853
882
  token: string | undefined;
854
883
  };
855
- const states: UnitState[] = units.map(({ plan: unit, progress }) => {
856
- if (unit.status === 'revoked') return { kind: 'revoked' };
857
- if (progress === undefined) return { kind: 'live' };
858
- // §4.7: a token, cursor -1 or a cursor from the future bootstraps.
859
- if (
860
- progress.bootstrapState === undefined &&
861
- progress.cursor >= 0 &&
862
- progress.cursor <= maxSeq
863
- ) {
864
- return progress.cursor < horizonSeq
865
- ? { kind: 'reset' }
866
- : { kind: 'post', cursor: progress.cursor };
867
- }
868
- // Replaced by the section the bootstrap below produces.
869
- return { kind: 'bootstrap', pin: -1, frames: [], token: '' };
870
- });
884
+ const states: UnitState[] = units.map(
885
+ ({ plan: unit, progress, restrict }): UnitState => {
886
+ if (unit.status === 'revoked') return { kind: 'revoked' };
887
+ if (progress === undefined) return { kind: 'live' };
888
+ // §4.6: a kept cursor outside [horizon, max] cannot join.
889
+ if (restrict !== undefined)
890
+ return restrict.cursor < horizonSeq || restrict.cursor > maxSeq
891
+ ? { kind: 'reset' }
892
+ : {
893
+ kind: 'bootstrap',
894
+ pin: -1,
895
+ restricted: true,
896
+ frames: [],
897
+ token: '',
898
+ };
899
+ // §4.7: a token, cursor -1 or a cursor from the future bootstraps.
900
+ if (
901
+ progress.bootstrapState === undefined &&
902
+ progress.cursor >= 0 &&
903
+ progress.cursor <= maxSeq
904
+ ) {
905
+ return progress.cursor < horizonSeq
906
+ ? { kind: 'reset' }
907
+ : { kind: 'post', cursor: progress.cursor };
908
+ }
909
+ // Replaced by the section the bootstrap below produces.
910
+ return {
911
+ kind: 'bootstrap',
912
+ pin: -1,
913
+ restricted: false,
914
+ frames: [],
915
+ token: '',
916
+ };
917
+ },
918
+ );
871
919
  const traces = units.map((): PullSectionTrace | undefined =>
872
920
  events !== undefined ? { segments: [] } : undefined,
873
921
  );
@@ -899,19 +947,36 @@ async function* streamResponse(
899
947
  pages += reads;
900
948
  next += 1;
901
949
  }
902
- const batch = window.map((index) => ({
903
- index,
904
- plan: planBootstrap(
905
- ctx,
906
- schema,
907
- limits,
908
- units[index]!.plan,
909
- units[index]!.progress?.bootstrapState,
910
- maxSeq,
911
- horizonSeq,
912
- plan.logEpoch,
913
- ),
914
- }));
950
+ const batch = window.map((index) => {
951
+ const { plan: unit, progress, restrict } = units[index]!;
952
+ // §4.7: a restricted bootstrap serves only the RESTRICT tables.
953
+ const only =
954
+ restrict === undefined ? undefined : new Set(restrict.tables);
955
+ return {
956
+ index,
957
+ plan: planBootstrap(
958
+ ctx,
959
+ schema,
960
+ limits,
961
+ only === undefined
962
+ ? unit
963
+ : {
964
+ ...unit,
965
+ tables: unit.tables.filter((table) =>
966
+ only.has(table.name),
967
+ ),
968
+ pairs: new Map(
969
+ [...unit.pairs].filter(([name]) => only.has(name)),
970
+ ),
971
+ },
972
+ progress?.bootstrapState,
973
+ maxSeq,
974
+ horizonSeq,
975
+ plan.logEpoch,
976
+ restrict?.cursor,
977
+ ),
978
+ };
979
+ });
915
980
  for (const { index, plan: bootstrap } of batch) {
916
981
  const frames: ResponseFrame[] = [];
917
982
  const segments = bootstrapSegments(
@@ -927,9 +992,11 @@ async function* streamResponse(
927
992
  frames.push(step.value);
928
993
  step = await segments.next();
929
994
  }
995
+ const restrict = units[index]!.restrict;
930
996
  states[index] = {
931
997
  kind: 'bootstrap',
932
- pin: bootstrap.pin,
998
+ pin: restrict?.cursor ?? bootstrap.pin,
999
+ restricted: restrict !== undefined,
933
1000
  frames,
934
1001
  token: step.value,
935
1002
  };
@@ -975,7 +1042,12 @@ async function* streamResponse(
975
1042
  const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
976
1043
  let expired = false;
977
1044
  for (const [index, state] of states.entries()) {
978
- const afterSeq = bound(state);
1045
+ // §4.6: a restricted bootstrap's kept cursor is rechecked too.
1046
+ const afterSeq =
1047
+ bound(state) ??
1048
+ (state.kind === 'bootstrap' && state.restricted
1049
+ ? state.pin
1050
+ : undefined);
979
1051
  if (afterSeq === undefined || afterSeq >= recheck) continue;
980
1052
  expired = true;
981
1053
  if (state.kind === 'live') liveReset = true;
@@ -3302,6 +3302,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3302
3302
  created_at_ms: unknown;
3303
3303
  key: unknown;
3304
3304
  bytes: unknown;
3305
+ changes: unknown;
3305
3306
  }>(
3306
3307
  `WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
3307
3308
  c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
@@ -3311,10 +3312,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3311
3312
  ORDER BY commit_seq LIMIT $4) s
3312
3313
  ORDER BY s.commit_seq LIMIT $4)
3313
3314
  SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
3314
- z.key AS key, z.bytes AS bytes
3315
+ z.key AS key, z.bytes AS bytes, z.changes AS changes
3315
3316
  FROM c
3316
3317
  LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
3317
- LEFT JOIN LATERAL (SELECT k.i AS key,
3318
+ LEFT JOIN LATERAL (SELECT k.i AS key, count(*) AS changes,
3318
3319
  sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
3319
3320
  FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value
3320
3321
  WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
@@ -3334,7 +3335,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
3334
3335
  }),
3335
3336
  ...(record.key === null
3336
3337
  ? {}
3337
- : { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
3338
+ : {
3339
+ key: asNumber(record.key),
3340
+ bytes: asNumber(record.bytes),
3341
+ changes: asNumber(record.changes),
3342
+ }),
3338
3343
  }));
3339
3344
  },
3340
3345
  async (keys, commitSeqs) => {
package/src/pull.ts CHANGED
@@ -794,12 +794,15 @@ export function planBootstrap(
794
794
  maxSeq: number,
795
795
  horizonSeq: number,
796
796
  logEpoch: string,
797
+ keptCursor = 0,
797
798
  ): BootstrapPlan {
798
- // §4.7: resume at the token's pin unless the token is unusable or the pin
799
- // fell behind the horizon; then restart from a fresh pin.
799
+ // §4.7: resume at the token's pin unless the token is unusable, or the pin
800
+ // fell behind the horizon or a restricted bootstrap's kept cursor; then
801
+ // restart from a fresh pin.
800
802
  const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
801
803
  const resume =
802
- token !== undefined && token.asOfCommitSeq >= horizonSeq
804
+ token !== undefined &&
805
+ token.asOfCommitSeq >= Math.max(horizonSeq, keptCursor)
803
806
  ? token
804
807
  : undefined;
805
808
  const tableIndex = resume?.tableIndex ?? 0;
@@ -983,6 +983,7 @@ export abstract class SqliteFamilyStorage {
983
983
  created_at_ms: number | null;
984
984
  key: number | null;
985
985
  bytes: number | null;
986
+ changes: number | null;
986
987
  }>(
987
988
  `WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
988
989
  c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
@@ -991,10 +992,10 @@ export abstract class SqliteFamilyStorage {
991
992
  AND s.commit_seq>k.after AND s.commit_seq<=?3
992
993
  ORDER BY s.commit_seq LIMIT ?4)
993
994
  SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
994
- z.i AS key, z.bytes AS bytes
995
+ z.i AS key, z.bytes AS bytes, z.changes AS changes
995
996
  FROM c
996
997
  LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
997
- LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
998
+ LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i, count(*) AS changes,
998
999
  sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
999
1000
  FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
1000
1001
  CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
@@ -1013,7 +1014,11 @@ export abstract class SqliteFamilyStorage {
1013
1014
  }
1014
1015
  : {}),
1015
1016
  ...(record.key !== null
1016
- ? { key: record.key, bytes: record.bytes ?? 0 }
1017
+ ? {
1018
+ key: record.key,
1019
+ bytes: record.bytes ?? 0,
1020
+ changes: record.changes ?? 0,
1021
+ }
1017
1022
  : {}),
1018
1023
  })),
1019
1024
  async (keys, commitSeqs) =>
@@ -159,16 +159,18 @@ export interface LiveKey {
159
159
 
160
160
  /**
161
161
  * One row of a live-window candidate page: a candidate commit joined to its
162
- * metadata (absent when the commit vanished) and, per key, the stored bytes
163
- * (payload, row id, scopes) of its changes that key reads: a covered table
164
- * and the key's value in the stored scopes. A commit with no such change
165
- * comes back once without `key`. Rows are in (commitSeq, key) order.
162
+ * metadata (absent when the commit vanished) and, per key, the count and
163
+ * stored bytes (payload, row id, scopes) of its changes that key reads: a
164
+ * covered table and the key's value in the stored scopes. A commit with no
165
+ * such change comes back once without `key`. Rows are in (commitSeq, key)
166
+ * order.
166
167
  */
167
168
  export interface LiveCandidateRow {
168
169
  readonly commitSeq: number;
169
170
  readonly meta?: { readonly actorId: string; readonly createdAtMs: number };
170
171
  readonly key?: number;
171
172
  readonly bytes?: number;
173
+ readonly changes?: number;
172
174
  }
173
175
 
174
176
  /**
@@ -184,6 +186,11 @@ export interface LiveCandidateRow {
184
186
  * bound. Every candidate advances the scan, so a page of non-matching
185
187
  * candidates never repeats.
186
188
  *
189
+ * The counts bound the delivered changes from above, because every change
190
+ * a unit keeps is one its key reads: the scan loads pending candidates once
191
+ * their counts could reach the rest of `limitChanges`, so a page loads
192
+ * about the changes it delivers and never a candidate past the cut.
193
+ *
187
194
  * The sizes bound the loaded bytes by `limitBytes` before any payload is
188
195
  * read: the section stops before the commit that would pass it, and a
189
196
  * commit that passes it alone ends the read as `oversized`, naming the
@@ -236,6 +243,8 @@ export async function readLiveWindowPages(
236
243
  const limit = Math.max(64, query.limitChanges);
237
244
  let delivered = 0;
238
245
  let loaded = 0;
246
+ // Upper bound of the changes the pending candidates deliver.
247
+ let bound = 0;
239
248
  let scanned = -1;
240
249
  // Candidates sized but not loaded yet: (scanned, pending.at(-1)].
241
250
  let pending: { commitSeq: number; meta?: LiveCandidateRow['meta'] }[] = [];
@@ -251,6 +260,7 @@ export async function readLiveWindowPages(
251
260
  pending.map((c) => c.commitSeq),
252
261
  );
253
262
  pending = [];
263
+ bound = 0;
254
264
  scanned = last.commitSeq;
255
265
  for (let i = 0; i < rows.length;) {
256
266
  const seq = rows[i]!.commitSeq;
@@ -287,11 +297,13 @@ export async function readLiveWindowPages(
287
297
  const head = rows[i]!;
288
298
  candidates += 1;
289
299
  let bytes = 0;
300
+ let changes = 0;
290
301
  const sizes: [key: number, bytes: number][] = [];
291
302
  for (; rows[i]?.commitSeq === head.commitSeq; i++) {
292
303
  const row = rows[i]!;
293
304
  if (row.key === undefined) continue;
294
305
  bytes += row.bytes ?? 0;
306
+ changes += row.changes ?? 0;
295
307
  sizes.push([row.key, row.bytes ?? 0]);
296
308
  }
297
309
  if (bytes > query.limitBytes) {
@@ -324,7 +336,12 @@ export async function readLiveWindowPages(
324
336
  return { commits, throughSeq: cut ?? scanned };
325
337
  }
326
338
  loaded += bytes;
339
+ bound += changes;
327
340
  pending.push({ commitSeq: head.commitSeq, meta: head.meta });
341
+ if (delivered + bound >= query.limitChanges) {
342
+ const cut = await load();
343
+ if (cut !== undefined) return { commits, throughSeq: cut };
344
+ }
328
345
  }
329
346
  const cut = await load();
330
347
  if (cut !== undefined) return { commits, throughSeq: cut };