@syncular/server 0.59.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +20 -16
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +42 -40
  4. package/dist/context.d.ts +6 -8
  5. package/dist/context.js +1 -3
  6. package/dist/d1-storage.js +42 -24
  7. package/dist/errors.js +8 -0
  8. package/dist/events.d.ts +35 -17
  9. package/dist/frame-bytes.js +24 -6
  10. package/dist/handler.js +486 -223
  11. package/dist/local-authority.js +7 -9
  12. package/dist/operations.d.ts +5 -2
  13. package/dist/operations.js +20 -29
  14. package/dist/postgres-storage.d.ts +24 -19
  15. package/dist/postgres-storage.js +249 -168
  16. package/dist/pull.d.ts +68 -53
  17. package/dist/pull.js +258 -287
  18. package/dist/push.d.ts +12 -3
  19. package/dist/push.js +95 -32
  20. package/dist/realtime.d.ts +17 -11
  21. package/dist/realtime.js +65 -90
  22. package/dist/relational-rows.d.ts +0 -57
  23. package/dist/relational-rows.js +0 -97
  24. package/dist/seed.js +10 -15
  25. package/dist/segment-download.d.ts +4 -1
  26. package/dist/segment-download.js +4 -2
  27. package/dist/sqlite-dialect.d.ts +36 -59
  28. package/dist/sqlite-dialect.js +255 -159
  29. package/dist/sqlite-storage.js +24 -20
  30. package/dist/storage-query.d.ts +30 -1
  31. package/dist/storage-query.js +68 -0
  32. package/dist/storage.d.ts +156 -54
  33. package/dist/storage.js +15 -0
  34. package/package.json +2 -2
  35. package/src/admin.ts +87 -59
  36. package/src/context.ts +7 -11
  37. package/src/d1-storage.ts +49 -33
  38. package/src/errors.ts +8 -0
  39. package/src/events.ts +36 -17
  40. package/src/frame-bytes.ts +26 -6
  41. package/src/handler.ts +628 -286
  42. package/src/local-authority.ts +10 -11
  43. package/src/operations.ts +44 -42
  44. package/src/postgres-storage.ts +336 -226
  45. package/src/pull.ts +394 -414
  46. package/src/push.ts +116 -21
  47. package/src/realtime.ts +85 -129
  48. package/src/relational-rows.ts +0 -105
  49. package/src/seed.ts +13 -19
  50. package/src/segment-download.ts +8 -3
  51. package/src/sqlite-dialect.ts +363 -204
  52. package/src/sqlite-storage.ts +45 -25
  53. package/src/storage-query.ts +102 -0
  54. package/src/storage.ts +185 -58
package/src/pull.ts CHANGED
@@ -1,26 +1,29 @@
1
1
  /**
2
- * Pull: incremental commit delivery, cursors, bootstrap segments
3
- * (SPEC.md §4, §5).
2
+ * Pull: units, the live section, bootstrap segments (SPEC.md §4, §5).
4
3
  */
5
4
  import {
6
5
  type CommitChange,
6
+ type CommitFrame,
7
7
  encodeRowsSegment,
8
+ type FencedRecord,
8
9
  type PullHeaderFrame,
9
10
  type ResponseFrame,
10
11
  type ScopeMap,
11
12
  type SegmentRefFrame,
12
13
  type SegmentRow,
13
- type SubscriptionFrame,
14
+ type UnitScopes,
15
+ unitId,
14
16
  } from '@syncular/core';
15
17
  import type { SyncRequestContext, SyncServerConfig } from './context';
16
18
  import { clockOf, limitsOf } from './context';
19
+ import { syncError } from './errors';
17
20
  import type { PullSegmentSummary } from './events';
18
21
  import {
19
22
  type CompiledSchema,
20
23
  type CompiledTable,
21
24
  compileSchema,
22
25
  } from './schema';
23
- import { scopeDigest } from './scopes';
26
+ import { computeEffective, type ResolvedScopes, scopeDigest } from './scopes';
24
27
  import type { SegmentRecord, SegmentStore } from './segment-store';
25
28
  import { issueSegmentUrl } from './signed-url';
26
29
  import type { ServerStorage, StoredCommit, StoredRow } from './storage';
@@ -50,7 +53,7 @@ function clamp(value: number, min: number, max: number, dflt: number): number {
50
53
  return Math.min(max, Math.max(min, value));
51
54
  }
52
55
 
53
- /** §4.2 defaults and silent clamps (the v1 values). */
56
+ /** §4.2 defaults and silent clamps. */
54
57
  export function clampPullLimits(header: PullHeaderFrame): PullLimits {
55
58
  return {
56
59
  limitCommits: clamp(header.limitCommits, 1, 1000, 1000),
@@ -60,16 +63,140 @@ export function clampPullLimits(header: PullHeaderFrame): PullLimits {
60
63
  };
61
64
  }
62
65
 
63
- export interface SubscriptionPlan {
64
- readonly frame: SubscriptionFrame;
65
- readonly table: CompiledTable;
66
+ /** One unit of the effective list, resolved against the request (§3.2, §4.1). */
67
+ export interface UnitPlan {
68
+ readonly id: string;
69
+ readonly scopes: UnitScopes;
70
+ /** Covered tables in bootstrap order (§4.1, §4.7). */
71
+ readonly tables: readonly CompiledTable[];
72
+ /** Active (unit, table) pairs: table → effective scopes (§3.2). */
73
+ readonly pairs: ReadonlyMap<string, ScopeMap>;
66
74
  readonly status: 'active' | 'revoked';
67
- readonly effective: ScopeMap;
75
+ /** The echo (§4.4): ascending table order, empty when revoked. */
76
+ readonly fenced: readonly FencedRecord[];
77
+ }
78
+
79
+ /**
80
+ * §4.1 unit validation against one schema version: the covered tables, or
81
+ * `sync.invalid_subscription` for an empty scope map, a requested `'*'`, a
82
+ * variable no table declares, or no covered table.
83
+ */
84
+ export function coveredTables(
85
+ schema: CompiledSchema,
86
+ scopes: UnitScopes,
87
+ ): CompiledTable[] {
88
+ const keys = Object.keys(scopes);
89
+ const reason =
90
+ keys.length === 0
91
+ ? 'empty_scope_map'
92
+ : keys.some((key) => scopes[key] === '*')
93
+ ? 'wildcard_value'
94
+ : keys.some((key) => !schema.declaredVariables.has(key))
95
+ ? 'undeclared_variable'
96
+ : undefined;
97
+ const tables =
98
+ reason === undefined
99
+ ? [...schema.tables.values()].filter((table) =>
100
+ keys.every((key) => table.declaredVariables.has(key)),
101
+ )
102
+ : [];
103
+ if (tables.length === 0) {
104
+ throw syncError(
105
+ 'sync.invalid_subscription',
106
+ 'the server cannot serve this unit (§4.1)',
107
+ JSON.stringify({ reason: reason ?? 'no_covered_table', scopes }),
108
+ );
109
+ }
110
+ return tables;
111
+ }
112
+
113
+ /** §3.2 per covered table: the unit's active pairs, status and echo. */
114
+ export function planUnit(
115
+ scopes: UnitScopes,
116
+ tables: readonly CompiledTable[],
117
+ resolved: ResolvedScopes,
118
+ ): UnitPlan {
119
+ const requested: ScopeMap = {};
120
+ for (const [key, value] of Object.entries(scopes)) requested[key] = [value];
121
+ const pairs = new Map<string, ScopeMap>();
122
+ const fenced: FencedRecord[] = [];
123
+ for (const table of tables) {
124
+ const outcome = computeEffective(
125
+ table.declaredVariables,
126
+ requested,
127
+ resolved,
128
+ );
129
+ if (outcome.status === 'revoked') {
130
+ fenced.push({ table: table.name, status: 'revoked', scopes: {} });
131
+ continue;
132
+ }
133
+ pairs.set(table.name, outcome.effective);
134
+ const added = Object.entries(outcome.effective).filter(
135
+ ([key]) => !Object.hasOwn(scopes, key),
136
+ );
137
+ if (added.length > 0) {
138
+ fenced.push({
139
+ table: table.name,
140
+ status: 'active',
141
+ scopes: Object.fromEntries(added),
142
+ });
143
+ }
144
+ }
145
+ const active = pairs.size > 0;
146
+ return {
147
+ id: unitId(scopes),
148
+ scopes,
149
+ tables,
150
+ pairs,
151
+ status: active ? 'active' : 'revoked',
152
+ fenced: active ? fenced.sort((a, b) => (a.table < b.table ? -1 : 1)) : [],
153
+ };
68
154
  }
69
155
 
70
- export interface SubscriptionResult {
71
- readonly nextCursor: number;
72
- readonly active: boolean;
156
+ /**
157
+ * The `COMMIT` frame of one live-section commit (§4.5): its frame-local
158
+ * table dictionary and every change, projected onto the client's codec.
159
+ */
160
+ export function commitFrame(
161
+ commit: StoredCommit,
162
+ current: CompiledSchema,
163
+ target: CompiledSchema,
164
+ ): CommitFrame {
165
+ const tables: string[] = [];
166
+ const changes: CommitChange[] = commit.changes.map((change) => {
167
+ // §4.5: apply is version-guarded, so every change carries its version:
168
+ // an upsert the row's version after the write, a delete the version its
169
+ // tombstone records. A change without one cannot be served.
170
+ if (change.rowVersion === undefined) {
171
+ throw new Error('a change record carries no row version (§4.5)');
172
+ }
173
+ let tableIndex = tables.indexOf(change.table);
174
+ if (tableIndex === -1) tableIndex = tables.push(change.table) - 1;
175
+ return {
176
+ tableIndex,
177
+ rowId: change.rowId,
178
+ op: change.op,
179
+ rowVersion: change.rowVersion,
180
+ scopes: change.scopes,
181
+ ...(change.payload !== undefined
182
+ ? {
183
+ row: projectRowPayload(
184
+ current.tables.get(change.table)!,
185
+ target.tables.get(change.table)!,
186
+ change.payload,
187
+ ),
188
+ }
189
+ : {}),
190
+ };
191
+ });
192
+ return {
193
+ type: 'COMMIT',
194
+ commitSeq: commit.commitSeq,
195
+ createdAtMs: commit.createdAtMs,
196
+ actorId: commit.actorId,
197
+ tables,
198
+ changes,
199
+ };
73
200
  }
74
201
 
75
202
  /**
@@ -89,13 +216,14 @@ interface BootstrapToken {
89
216
  }
90
217
 
91
218
  /**
92
- * §4.7: a usable resume token for `table`, or `undefined`. A token whose pin
93
- * is not a commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`) or
94
- * whose position does not name one of its tables is unusable.
219
+ * §4.7: a usable resume token, or `undefined`. A token whose pin is not a
220
+ * commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`), whose tables
221
+ * differ from the unit's covered tables in bootstrap order, or whose position
222
+ * names none of them is unusable.
95
223
  */
96
224
  function parseBootstrapToken(
97
225
  raw: string | undefined,
98
- table: string,
226
+ tables: readonly CompiledTable[],
99
227
  maxSeq: number,
100
228
  ): BootstrapToken | undefined {
101
229
  if (raw === undefined) return undefined;
@@ -107,11 +235,12 @@ function parseBootstrapToken(
107
235
  parsed.asOfCommitSeq < 0 ||
108
236
  parsed.asOfCommitSeq > maxSeq ||
109
237
  !Array.isArray(parsed.tables) ||
110
- parsed.tables[0] !== table ||
238
+ parsed.tables.length !== tables.length ||
239
+ parsed.tables.some((name, i) => name !== tables[i]?.name) ||
111
240
  !Number.isSafeInteger(parsed.tableIndex) ||
112
241
  parsed.tableIndex === undefined ||
113
242
  parsed.tableIndex < 0 ||
114
- parsed.tableIndex >= parsed.tables.length ||
243
+ parsed.tableIndex >= tables.length ||
115
244
  (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')
116
245
  ) {
117
246
  return undefined;
@@ -127,34 +256,6 @@ function parseBootstrapToken(
127
256
  }
128
257
  }
129
258
 
130
- function commitFrame(
131
- table: string,
132
- commit: StoredCommit,
133
- current: CompiledTable,
134
- target: CompiledTable,
135
- ): ResponseFrame {
136
- const changes: CommitChange[] = commit.changes.map((change) => ({
137
- tableIndex: 0,
138
- rowId: change.rowId,
139
- op: change.op,
140
- ...(change.rowVersion !== undefined
141
- ? { rowVersion: change.rowVersion }
142
- : {}),
143
- scopes: change.scopes,
144
- ...(change.payload !== undefined
145
- ? { row: projectRowPayload(current, target, change.payload) }
146
- : {}),
147
- }));
148
- return {
149
- type: 'COMMIT',
150
- commitSeq: commit.commitSeq,
151
- createdAtMs: commit.createdAtMs,
152
- actorId: commit.actorId,
153
- tables: [table],
154
- changes,
155
- };
156
- }
157
-
158
259
  function chunkRows(rows: SegmentRow[], size: number): SegmentRow[][] {
159
260
  if (rows.length === 0) return [];
160
261
  const blocks: SegmentRow[][] = [];
@@ -194,10 +295,10 @@ function segmentRefFrame(
194
295
  byteLength: record.byteLength,
195
296
  rowCount: record.rowCount,
196
297
  asOfCommitSeq: record.asOfCommitSeq,
197
- // The frame reports the digest THIS subscription is authorized under. A
198
- // stored entry can carry several digests when identical content was
199
- // published under different scopes (§5.1), and the record's primary
200
- // digest is not necessarily the caller's (§5.5).
298
+ // The frame reports the digest THIS pair is authorized under. A stored
299
+ // entry can carry several digests when identical content was published
300
+ // under different scopes (§5.1), and the record's primary digest is not
301
+ // necessarily the caller's (§5.5).
201
302
  scopeDigest,
202
303
  ...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
203
304
  ...(record.nextRowCursor !== null
@@ -210,6 +311,12 @@ function segmentRefFrame(
210
311
  };
211
312
  }
212
313
 
314
+ /** One active (unit, table) pair of a bootstrap (§3.2). */
315
+ interface BootstrapPair {
316
+ readonly table: CompiledTable;
317
+ readonly effective: ScopeMap;
318
+ }
319
+
213
320
  /**
214
321
  * §5.3: the sqlite-image lane — whole-table, chosen only at the start of a
215
322
  * table (a mid-table resume stays on the rows lane, never switching lanes),
@@ -220,30 +327,96 @@ function segmentRefFrame(
220
327
  */
221
328
  function sqliteLaneEligible(
222
329
  limits: PullLimits,
223
- plan: SubscriptionPlan,
330
+ table: CompiledTable,
224
331
  startRowCursor: string | null,
225
332
  ): boolean {
226
333
  return (
227
334
  (limits.accept & ACCEPT_SQLITE) !== 0 &&
228
335
  startRowCursor === null &&
229
- plan.table.encryptedColumnIndices.length === 0
336
+ table.encryptedColumnIndices.length === 0
230
337
  );
231
338
  }
232
339
 
340
+ /** The reads a table's bootstrap needs before its first frame (§4.7, §5.3). */
341
+ interface TableStart {
342
+ /**
343
+ * `limitSnapshotRows + 1` rows from the start cursor: the first page and
344
+ * the §5.3 image eligibility probe.
345
+ */
346
+ readonly firstPage: StoredRow[];
347
+ /** On the sqlite lane: the §4.7 scope pin and a stored image under it. */
348
+ readonly image?: {
349
+ readonly scopePin: number;
350
+ readonly stored: SegmentRecord | undefined;
351
+ };
352
+ }
353
+
354
+ async function readTableStart(
355
+ ctx: SyncRequestContext,
356
+ schema: CompiledSchema,
357
+ limits: PullLimits,
358
+ unit: UnitPlan,
359
+ pair: BootstrapPair,
360
+ startRowCursor: string | null,
361
+ maxSeq: number,
362
+ horizonSeq: number,
363
+ logEpoch: string,
364
+ ): Promise<TableStart> {
365
+ const scan = ctx.storage.scanRows(ctx.partition, {
366
+ table: pair.table.name,
367
+ scopeFilter: pair.effective,
368
+ afterRowId: startRowCursor,
369
+ limit: limits.limitSnapshotRows + 1,
370
+ });
371
+ if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
372
+ return { firstPage: await scan };
373
+ }
374
+ // §4.7: a sqlite-lane table pins at the newest change carrying the unit's
375
+ // index key (never below the horizon, never above maxCommitSeq), so a
376
+ // stored image stays the reuse key's match until this scope changes
377
+ // (§5.3). The server computes the pin; a client token never supplies it.
378
+ const [firstPage, latest] = await Promise.all([
379
+ scan,
380
+ ctx.storage.latestScopedChangeSeq(ctx.partition, {
381
+ table: pair.table.name,
382
+ scopes: unit.scopes,
383
+ }),
384
+ ]);
385
+ const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
386
+ // §5.3 eligibility: a stored image only matches a snapshot larger than one
387
+ // rows page, so the probe decides before the reuse lookup; a remote `find`
388
+ // on a small table would be a wasted round trip.
389
+ const stored =
390
+ firstPage.length <= limits.limitSnapshotRows
391
+ ? undefined
392
+ : await ctx.segments.find(
393
+ {
394
+ partition: ctx.partition,
395
+ logEpoch,
396
+ table: pair.table.name,
397
+ schemaVersion: schema.version,
398
+ mediaType: 'sqlite',
399
+ scopeDigest: await scopeDigest(pair.effective),
400
+ asOfCommitSeq: scopePin,
401
+ },
402
+ clockOf(ctx)(),
403
+ );
404
+ return { firstPage, image: { scopePin, stored } };
405
+ }
406
+
233
407
  /**
234
- * §5.3 sqlite-image lane: only at the start of a table, only when the
235
- * client advertised bit 2, and only when the snapshot exceeds one rows
236
- * page. Reuses an unexpired stored image for the same (partition, table,
237
- * schemaVersion, scope digest, pin) instead of rebuilding — the
238
- * bootstrap-storm rule. Returns false when the table is not eligible
239
- * (the rows lane takes over).
408
+ * §5.3 sqlite-image lane: only when the snapshot exceeds one rows page.
409
+ * Reuses an unexpired stored image for the same (partition, table,
410
+ * schemaVersion, scope digest, scope pin) instead of rebuilding — the
411
+ * bootstrap-storm rule. Returns false when no image is served (the rows lane
412
+ * takes over).
240
413
  */
241
414
  async function* sqliteImageSegment(
242
415
  ctx: SyncRequestContext,
243
416
  schema: CompiledSchema,
244
417
  limits: PullLimits,
245
- plan: SubscriptionPlan,
246
- asOf: number,
418
+ pair: BootstrapPair,
419
+ scopePin: number,
247
420
  digest: string,
248
421
  trace: PullSectionTrace | undefined,
249
422
  logEpoch: string,
@@ -267,9 +440,6 @@ async function* sqliteImageSegment(
267
440
  );
268
441
  return true;
269
442
  }
270
- // Eligibility probe (§5.3): image only when the snapshot exceeds one
271
- // rows page — smaller tables stay on the (typically inline) rows lane.
272
- // The probe is the rows lane's first page, read by `prepareSections`.
273
443
  if (probe.length <= limits.limitSnapshotRows) return false;
274
444
  // §5.3: hosts opt into image construction with a runtime-specific builder.
275
445
  // Without one, matching stored images remain usable and cold pulls use rows.
@@ -288,12 +458,13 @@ async function* sqliteImageSegment(
288
458
  const identity = {
289
459
  partition,
290
460
  logEpoch,
291
- table: plan.table.name,
461
+ table: pair.table.name,
292
462
  schemaVersion: schema.version,
293
463
  mediaType: 'sqlite' as const,
294
464
  scopeDigest: digest,
295
- asOfCommitSeq: asOf,
465
+ asOfCommitSeq: scopePin,
296
466
  };
467
+ const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
297
468
  const key = JSON.stringify(identity);
298
469
  let building = builds.get(key);
299
470
  let builtHere = false;
@@ -305,36 +476,28 @@ async function* sqliteImageSegment(
305
476
  builtHere = true;
306
477
  let rowCount = 0;
307
478
  const bytes = await buildImage({
308
- table: plan.table,
479
+ table: pair.table,
309
480
  schemaVersion: schema.version,
310
- asOfCommitSeq: asOf,
481
+ asOfCommitSeq: scopePin,
311
482
  scopeDigest: digest,
312
483
  rowBatches: (async function* () {
313
484
  rowCount += probe.length;
314
485
  yield probe.map((row) => ({
315
486
  ...row,
316
- payload: projectRowPayload(
317
- compileSchema(ctx.schema).tables.get(plan.table.name)!,
318
- plan.table,
319
- row.payload,
320
- ),
487
+ payload: projectRowPayload(current, pair.table, row.payload),
321
488
  }));
322
489
  let afterRowId = probe[probe.length - 1]!.rowId;
323
490
  for (;;) {
324
491
  const rows = await storage.scanRows(partition, {
325
- table: plan.table.name,
326
- scopeFilter: plan.effective,
492
+ table: pair.table.name,
493
+ scopeFilter: pair.effective,
327
494
  afterRowId,
328
495
  limit: 5_000,
329
496
  });
330
497
  rowCount += rows.length;
331
498
  yield rows.map((row) => ({
332
499
  ...row,
333
- payload: projectRowPayload(
334
- compileSchema(ctx.schema).tables.get(plan.table.name)!,
335
- plan.table,
336
- row.payload,
337
- ),
500
+ payload: projectRowPayload(current, pair.table, row.payload),
338
501
  }));
339
502
  const last = rows[rows.length - 1];
340
503
  if (rows.length < 5_000 || last === undefined) break;
@@ -377,48 +540,50 @@ async function* sqliteImageSegment(
377
540
  return true;
378
541
  }
379
542
 
380
- async function* bootstrapSegments(
543
+ /**
544
+ * One table's segments within the page budget. Returns the pages it used
545
+ * and the row cursor to resume at, `null` once the table is exhausted.
546
+ */
547
+ async function* tableSegments(
381
548
  ctx: SyncRequestContext,
382
549
  schema: CompiledSchema,
383
550
  limits: PullLimits,
384
- plan: SubscriptionPlan,
385
- asOf: number,
551
+ pair: BootstrapPair,
552
+ pin: number,
386
553
  startRowCursor: string | null,
554
+ start: TableStart,
555
+ pageBudget: number,
387
556
  trace: PullSectionTrace | undefined,
388
557
  logEpoch: string,
389
- firstPage: StoredRow[],
390
- image: SegmentRecord | undefined,
391
- ): AsyncGenerator<
392
- ResponseFrame,
393
- { complete: boolean; rowCursor: string | null }
394
- > {
558
+ ): AsyncGenerator<ResponseFrame, { pages: number; rowCursor: string | null }> {
395
559
  const { storage, segments, partition } = ctx;
396
560
  const serverLimits = limitsOf(ctx);
397
- const digest = await scopeDigest(plan.effective);
561
+ const digest = await scopeDigest(pair.effective);
398
562
  const now = clockOf(ctx)();
399
- if (sqliteLaneEligible(limits, plan, startRowCursor)) {
563
+ if (start.image !== undefined) {
400
564
  const imaged = yield* sqliteImageSegment(
401
565
  ctx,
402
566
  schema,
403
567
  limits,
404
- plan,
405
- asOf,
568
+ pair,
569
+ start.image.scopePin,
406
570
  digest,
407
571
  trace,
408
572
  logEpoch,
409
- image,
410
- firstPage,
573
+ start.image.stored,
574
+ start.firstPage,
411
575
  );
412
- if (imaged) return { complete: true, rowCursor: null };
576
+ if (imaged) return { pages: 1, rowCursor: null };
413
577
  }
578
+ const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
414
579
  let rowCursor = startRowCursor;
415
- for (let page = 0; page < limits.maxSnapshotPages; page++) {
580
+ for (let page = 0; page < pageBudget; page++) {
416
581
  const scanned =
417
582
  page === 0
418
- ? firstPage
583
+ ? start.firstPage
419
584
  : await storage.scanRows(partition, {
420
- table: plan.table.name,
421
- scopeFilter: plan.effective,
585
+ table: pair.table.name,
586
+ scopeFilter: pair.effective,
422
587
  afterRowId: rowCursor,
423
588
  limit: limits.limitSnapshotRows + 1,
424
589
  });
@@ -427,16 +592,12 @@ async function* bootstrapSegments(
427
592
  // §5.2: every row record carries the row's current server_version.
428
593
  const decoded = pageRows.map((row: StoredRow) => ({
429
594
  serverVersion: row.serverVersion,
430
- values: projectRowValues(
431
- compileSchema(ctx.schema).tables.get(plan.table.name)!,
432
- plan.table,
433
- row.payload,
434
- ),
595
+ values: projectRowValues(current, pair.table, row.payload),
435
596
  }));
436
597
  const bytes = encodeRowsSegment({
437
- table: plan.table.name,
598
+ table: pair.table.name,
438
599
  schemaVersion: schema.version,
439
- columns: plan.table.columns,
600
+ columns: pair.table.columns,
440
601
  blocks: chunkRows(decoded, 1000),
441
602
  });
442
603
  const lastRow = pageRows[pageRows.length - 1];
@@ -462,11 +623,11 @@ async function* bootstrapSegments(
462
623
  {
463
624
  partition,
464
625
  logEpoch,
465
- table: plan.table.name,
626
+ table: pair.table.name,
466
627
  schemaVersion: schema.version,
467
628
  mediaType: 'rows',
468
629
  scopeDigest: digest,
469
- asOfCommitSeq: asOf,
630
+ asOfCommitSeq: pin,
470
631
  rowCount: pageRows.length,
471
632
  rowCursor,
472
633
  nextRowCursor,
@@ -487,314 +648,134 @@ async function* bootstrapSegments(
487
648
  await signedUrlFields(ctx, limits, record.segmentId, digest, now),
488
649
  );
489
650
  }
490
- if (!hasMore) return { complete: true, rowCursor: null };
651
+ if (!hasMore) return { pages: page + 1, rowCursor: null };
491
652
  rowCursor = nextRowCursor;
492
653
  }
493
- return { complete: false, rowCursor };
654
+ return { pages: pageBudget, rowCursor };
494
655
  }
495
656
 
496
- /**
497
- * One subscription's section decision (§4.5–§4.7) with the storage reads it
498
- * needs before its first frame.
499
- */
500
- export type PreparedSection =
501
- | { readonly kind: 'revoked' }
502
- /** §4.6: the cursor is behind the horizon. */
503
- | { readonly kind: 'reset' }
504
- /** §4.5 window read for an in-range cursor. */
505
- | { readonly kind: 'incremental'; readonly commits: StoredCommit[] }
506
- | {
507
- readonly kind: 'bootstrap';
508
- /** §4.7 pin: a usable resume token's, the sqlite lane's scope pin, or `maxCommitSeq`. */
509
- readonly asOf: number;
510
- readonly startCursor: string | null;
511
- /**
512
- * The first snapshot page (`limitSnapshotRows + 1` rows from
513
- * `startCursor`), which is also the §5.3 image eligibility probe.
514
- */
515
- readonly firstPage: StoredRow[];
516
- /** §5.3 stored image under the reuse key, read only for an eligible probe. */
517
- readonly image: SegmentRecord | undefined;
518
- };
657
+ /** A bootstrapping unit's position and its table starts, read ahead (§4.7). */
658
+ export interface BootstrapPlan {
659
+ readonly unit: UnitPlan;
660
+ /** The unit's pin: a usable resume token's, or `maxCommitSeq`. */
661
+ readonly pin: number;
662
+ readonly tableIndex: number;
663
+ readonly rowCursor: string | null;
664
+ /** Table index → its start reads. */
665
+ readonly starts: ReadonlyMap<number, Promise<TableStart>>;
666
+ }
519
667
 
520
668
  /**
521
- * Decide the section of every plan and run its pre-frame storage reads. The
522
- * reads of all plans start before any is awaited, so a storage that batches
523
- * concurrent page reads (PostgreSQL, D1) answers the first pages of every
524
- * plan in one statement per table, and the horizon re-check runs once.
669
+ * Plan one unit's bootstrap and start the reads of every table it can reach
670
+ * in this section: each visited table costs at least one page, so at most
671
+ * `maxSnapshotPages` tables. The reads of every planned unit start before any
672
+ * is awaited, so a storage that batches concurrent page reads (PostgreSQL,
673
+ * D1) answers them in one statement per table.
525
674
  */
526
- export async function prepareSections(
675
+ export function planBootstrap(
527
676
  ctx: SyncRequestContext,
528
677
  schema: CompiledSchema,
529
678
  limits: PullLimits,
530
- plans: readonly SubscriptionPlan[],
679
+ unit: UnitPlan,
680
+ bootstrapState: string | undefined,
531
681
  maxSeq: number,
532
682
  horizonSeq: number,
533
683
  logEpoch: string,
534
- ): Promise<PreparedSection[]> {
535
- const prepared = await Promise.all(
536
- plans.map(async (plan): Promise<PreparedSection> => {
537
- const sub = plan.frame;
538
- if (plan.status === 'revoked') return { kind: 'revoked' };
539
- // §4.7: a present resume token keeps the subscription bootstrapping,
540
- // whether or not it is usable.
541
- if (
542
- sub.bootstrapState === undefined &&
543
- sub.cursor >= 0 &&
544
- sub.cursor <= maxSeq
545
- ) {
546
- if (sub.cursor < horizonSeq) return { kind: 'reset' };
547
- return {
548
- kind: 'incremental',
549
- commits: await ctx.storage.readCommitWindow(ctx.partition, {
550
- table: sub.table,
551
- scopeFilter: plan.effective,
552
- afterSeq: sub.cursor,
553
- throughSeq: maxSeq,
554
- limitChanges: limits.limitCommits + 1,
555
- }),
556
- };
557
- }
558
- // §4.7: resume at the token's pin unless the token is unusable or the
559
- // pin fell behind the horizon; then restart from a fresh pin.
560
- const token = parseBootstrapToken(sub.bootstrapState, sub.table, maxSeq);
561
- const resume =
562
- token !== undefined && token.asOfCommitSeq >= horizonSeq
563
- ? token
564
- : undefined;
565
- const startCursor = resume?.rowCursor ?? null;
566
- const scan = ctx.storage.scanRows(ctx.partition, {
567
- table: plan.table.name,
568
- scopeFilter: plan.effective,
569
- afterRowId: startCursor,
570
- limit: limits.limitSnapshotRows + 1,
571
- });
572
- if (!sqliteLaneEligible(limits, plan, startCursor)) {
573
- return {
574
- kind: 'bootstrap',
575
- asOf: resume?.asOfCommitSeq ?? maxSeq,
576
- startCursor,
577
- firstPage: await scan,
578
- image: undefined,
579
- };
580
- }
581
- // §4.7: a sqlite-lane bootstrap starts at the table start and pins at
582
- // the newest change in its scope (never below the horizon, never above
583
- // maxCommitSeq), so a stored image stays the reuse key's match until
584
- // this scope changes (§5.3). The server computes the pin; a client
585
- // token never supplies the reuse key.
586
- const [firstPage, latest] = await Promise.all([
587
- scan,
588
- ctx.storage.latestScopedChangeSeq(ctx.partition, {
589
- table: plan.table.name,
590
- scopeFilter: plan.effective,
591
- }),
592
- ]);
593
- const asOf = Math.min(maxSeq, Math.max(latest, horizonSeq));
594
- // §5.3 eligibility: a stored image only matches a snapshot larger than
595
- // one rows page, so the probe decides before the reuse lookup; a remote
596
- // `find` on a small table would be a wasted round trip.
597
- const image =
598
- firstPage.length <= limits.limitSnapshotRows
599
- ? undefined
600
- : await ctx.segments.find(
601
- {
602
- partition: ctx.partition,
603
- logEpoch,
604
- table: plan.table.name,
605
- schemaVersion: schema.version,
606
- mediaType: 'sqlite',
607
- scopeDigest: await scopeDigest(plan.effective),
608
- asOfCommitSeq: asOf,
609
- },
610
- clockOf(ctx)(),
611
- );
612
- return { kind: 'bootstrap', asOf, startCursor, firstPage, image };
613
- }),
614
- );
615
- // Validate continuity before committing to an active section. A prune
616
- // during a paged read can otherwise make an incomplete window look empty.
617
- if (!prepared.some((section) => section.kind === 'incremental')) {
618
- return prepared;
684
+ ): BootstrapPlan {
685
+ // §4.7: resume at the token's pin unless the token is unusable or the pin
686
+ // fell behind the horizon; then restart from a fresh pin.
687
+ const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
688
+ const resume =
689
+ token !== undefined && token.asOfCommitSeq >= horizonSeq
690
+ ? token
691
+ : undefined;
692
+ const tableIndex = resume?.tableIndex ?? 0;
693
+ const rowCursor = resume?.rowCursor ?? null;
694
+ const starts = new Map<number, Promise<TableStart>>();
695
+ for (
696
+ let index = tableIndex;
697
+ index < unit.tables.length && starts.size < limits.maxSnapshotPages;
698
+ index++
699
+ ) {
700
+ const table = unit.tables[index]!;
701
+ const effective = unit.pairs.get(table.name);
702
+ if (effective === undefined) continue;
703
+ const start = readTableStart(
704
+ ctx,
705
+ schema,
706
+ limits,
707
+ unit,
708
+ { table, effective },
709
+ index === tableIndex ? rowCursor : null,
710
+ maxSeq,
711
+ horizonSeq,
712
+ logEpoch,
713
+ );
714
+ // A section that stops early never awaits the later starts.
715
+ start.catch(() => undefined);
716
+ starts.set(index, start);
619
717
  }
620
- const windowHorizon = await ctx.storage.getHorizonSeq(ctx.partition);
621
- return prepared.map((section, index) =>
622
- section.kind === 'incremental' &&
623
- (plans[index]?.frame.cursor ?? -1) < windowHorizon
624
- ? { kind: 'reset' }
625
- : section,
626
- );
718
+ return {
719
+ unit,
720
+ pin: resume?.asOfCommitSeq ?? maxSeq,
721
+ tableIndex,
722
+ rowCursor,
723
+ starts,
724
+ };
627
725
  }
628
726
 
629
727
  /**
630
- * Produce the `SUB_START … SUB_END` section for one subscription (§1.6),
631
- * returning the cursor recorded for the retention watermark (§4.5).
728
+ * The segments of one bootstrap section (§4.7): tables in bootstrap order,
729
+ * each active pair's rows or image, up to `maxSnapshotPages` pages. Returns
730
+ * the resume token, or `undefined` once every table is exhausted. A pair the
731
+ * wildcard fence revoked receives no segment (§3.3).
632
732
  */
633
- export async function* subscriptionSection(
733
+ export async function* bootstrapSegments(
634
734
  ctx: SyncRequestContext,
635
735
  schema: CompiledSchema,
636
736
  limits: PullLimits,
637
- plan: SubscriptionPlan,
638
- maxSeq: number,
639
- horizonSeq: number,
640
- trace?: PullSectionTrace,
641
- logEpoch?: string,
642
- ): AsyncGenerator<ResponseFrame, SubscriptionResult> {
643
- if (logEpoch === undefined || logEpoch.length === 0) {
644
- throw new Error('subscriptionSection requires a non-empty log epoch');
645
- }
646
- const [prepared] = await prepareSections(
647
- ctx,
648
- schema,
649
- limits,
650
- [plan],
651
- maxSeq,
652
- horizonSeq,
653
- logEpoch,
654
- );
655
- if (prepared === undefined) throw new Error('prepareSections lost a plan');
656
- return yield* preparedSection(
657
- ctx,
658
- schema,
659
- limits,
660
- plan,
661
- maxSeq,
662
- prepared,
663
- trace,
664
- logEpoch,
665
- );
666
- }
667
-
668
- /** The section of one subscription whose reads `prepareSections` ran. */
669
- export async function* preparedSection(
670
- ctx: SyncRequestContext,
671
- schema: CompiledSchema,
672
- limits: PullLimits,
673
- plan: SubscriptionPlan,
674
- maxSeq: number,
675
- prepared: PreparedSection,
676
- trace?: PullSectionTrace,
677
- logEpoch?: string,
678
- ): AsyncGenerator<ResponseFrame, SubscriptionResult> {
679
- const sub = plan.frame;
680
- if (logEpoch === undefined || logEpoch.length === 0) {
681
- throw new Error('subscriptionSection requires a non-empty log epoch');
682
- }
683
-
684
- if (prepared.kind === 'revoked') {
685
- yield {
686
- type: 'SUB_START',
687
- id: sub.id,
688
- status: 'revoked',
689
- reasonCode: 'sync.scope_revoked',
690
- effectiveScopes: {},
691
- bootstrap: false,
692
- };
693
- yield { type: 'SUB_END', nextCursor: sub.cursor };
694
- return { nextCursor: sub.cursor, active: false };
695
- }
696
-
697
- // §4.6: a cursor behind the horizon (and not resuming a bootstrap)
698
- // cannot compute deltas — answer `reset` and echo the cursor.
699
- if (prepared.kind === 'reset') {
700
- yield {
701
- type: 'SUB_START',
702
- id: sub.id,
703
- status: 'reset',
704
- reasonCode: 'sync.cursor_expired',
705
- effectiveScopes: {},
706
- bootstrap: false,
707
- };
708
- yield { type: 'SUB_END', nextCursor: sub.cursor };
709
- return { nextCursor: sub.cursor, active: false };
710
- }
711
-
712
- if (prepared.kind === 'bootstrap') {
713
- const { asOf } = prepared;
714
- yield {
715
- type: 'SUB_START',
716
- id: sub.id,
717
- status: 'active',
718
- reasonCode: '',
719
- effectiveScopes: plan.effective,
720
- bootstrap: true,
721
- };
722
- const outcome = yield* bootstrapSegments(
737
+ plan: BootstrapPlan,
738
+ trace: PullSectionTrace | undefined,
739
+ logEpoch: string,
740
+ ): AsyncGenerator<ResponseFrame, string | undefined> {
741
+ const { unit, pin } = plan;
742
+ let { tableIndex, rowCursor } = plan;
743
+ let pages = 0;
744
+ for (;;) {
745
+ while (
746
+ tableIndex < unit.tables.length &&
747
+ !unit.pairs.has(unit.tables[tableIndex]!.name)
748
+ ) {
749
+ tableIndex += 1;
750
+ rowCursor = null;
751
+ }
752
+ if (tableIndex === unit.tables.length) return undefined;
753
+ if (pages === limits.maxSnapshotPages) {
754
+ const token: BootstrapToken = {
755
+ asOfCommitSeq: pin,
756
+ tables: unit.tables.map((table) => table.name),
757
+ tableIndex,
758
+ rowCursor,
759
+ };
760
+ return JSON.stringify(token);
761
+ }
762
+ const table = unit.tables[tableIndex]!;
763
+ const outcome = yield* tableSegments(
723
764
  ctx,
724
765
  schema,
725
766
  limits,
726
- plan,
727
- asOf,
728
- prepared.startCursor,
767
+ { table, effective: unit.pairs.get(table.name)! },
768
+ pin,
769
+ rowCursor,
770
+ await plan.starts.get(tableIndex)!,
771
+ limits.maxSnapshotPages - pages,
729
772
  trace,
730
773
  logEpoch,
731
- prepared.firstPage,
732
- prepared.image,
733
774
  );
734
- if (outcome.complete) {
735
- yield { type: 'SUB_END', nextCursor: asOf };
736
- } else {
737
- const nextToken: BootstrapToken = {
738
- asOfCommitSeq: asOf,
739
- tables: [sub.table],
740
- tableIndex: 0,
741
- rowCursor: outcome.rowCursor,
742
- };
743
- yield {
744
- type: 'SUB_END',
745
- nextCursor: asOf,
746
- bootstrapState: JSON.stringify(nextToken),
747
- };
748
- }
749
- return { nextCursor: asOf, active: true };
775
+ pages += outcome.pages;
776
+ rowCursor = outcome.rowCursor;
777
+ if (rowCursor === null) tableIndex += 1;
750
778
  }
751
-
752
- // Incremental (§4.5): window cursor < commitSeq <= maxSeq, oldest first,
753
- // cut off at limitCommits total changes, never splitting a commit.
754
- yield {
755
- type: 'SUB_START',
756
- id: sub.id,
757
- status: 'active',
758
- reasonCode: '',
759
- effectiveScopes: plan.effective,
760
- bootstrap: false,
761
- };
762
- let delivered = 0;
763
- let deliveredCommits = 0;
764
- let lastDeliveredSeq = sub.cursor;
765
- const { commits } = prepared;
766
- for (const commit of commits) {
767
- if (
768
- delivered > 0 &&
769
- delivered + commit.changes.length > limits.limitCommits
770
- ) {
771
- break;
772
- }
773
- yield commitFrame(
774
- sub.table,
775
- commit,
776
- compileSchema(ctx.schema).tables.get(sub.table)!,
777
- plan.table,
778
- );
779
- delivered += commit.changes.length;
780
- deliveredCommits += 1;
781
- lastDeliveredSeq = commit.commitSeq;
782
- if (delivered >= limits.limitCommits) break;
783
- }
784
- const totalReturned = commits.reduce((n, c) => n + c.changes.length, 0);
785
- // The window is proven exhausted only when the storage lookahead came
786
- // back under budget (it scanned to `throughSeq`) and every returned
787
- // commit was delivered.
788
- const exhausted =
789
- totalReturned <= limits.limitCommits && deliveredCommits === commits.length;
790
- // §4.5: the cursor advances even when no matching changes exist; when
791
- // the change limit truncated the window it stops at the last fully
792
- // delivered commit.
793
- const nextCursor = exhausted
794
- ? Math.max(sub.cursor, maxSeq)
795
- : Math.max(sub.cursor, lastDeliveredSeq);
796
- yield { type: 'SUB_END', nextCursor };
797
- return { nextCursor, active: true };
798
779
  }
799
780
 
800
781
  export interface PublishSqliteImageInput {
@@ -802,12 +783,15 @@ export interface PublishSqliteImageInput {
802
783
  readonly config: SyncServerConfig;
803
784
  readonly partition: string;
804
785
  readonly table: string;
786
+ /** The unit whose clients bootstrap the table (§4.1). */
787
+ readonly scopes: UnitScopes;
805
788
  /**
806
- * The effective scopes the subscribing clients hold for `table` (§3.2):
807
- * the reuse key carries their digest, so publish once per distinct scope
808
- * set that clients subscribe with.
789
+ * The allowed scopes of the actors that register the unit (§3.2). The
790
+ * reuse key carries the digest of the pair's effective scopes, so a
791
+ * wildcard fence on the table changes the image. Default: `'*'` for every
792
+ * declared variable, which leaves the unit's own scope map.
809
793
  */
810
- readonly scopes: ScopeMap;
794
+ readonly allowed?: ScopeMap;
811
795
  }
812
796
 
813
797
  export interface PublishedSqliteImage {
@@ -822,8 +806,8 @@ export interface PublishedSqliteImage {
822
806
  }
823
807
 
824
808
  /**
825
- * Publish the §5.3 sqlite image that a pull for (`table`, `scopes`) looks
826
- * up, from a host with a SQLite engine (a Bun process with the production
809
+ * Publish the §5.3 sqlite image that a pull of (`scopes`, `table`) looks up,
810
+ * from a host with a SQLite engine (a Bun process with the production
827
811
  * storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
828
812
  * reuse key, the build), so the stored identity is exactly the one a
829
813
  * serving host without a builder finds. The image stays current until a
@@ -838,9 +822,19 @@ export async function publishSqliteImage(
838
822
  throw new Error('publishSqliteImage requires config.sqliteImageBuilder');
839
823
  }
840
824
  const schema = compileSchema(config.schema);
841
- const table = schema.tables.get(input.table);
825
+ const table = coveredTables(schema, input.scopes).find(
826
+ (candidate) => candidate.name === input.table,
827
+ );
842
828
  if (table === undefined) {
843
- throw new Error('publishSqliteImage: the schema has no such table');
829
+ throw new Error('publishSqliteImage: the unit does not cover the table');
830
+ }
831
+ const allowed =
832
+ input.allowed ??
833
+ Object.fromEntries([...schema.declaredVariables].map((v) => [v, ['*']]));
834
+ const unit = planUnit(input.scopes, [table], { ok: true, allowed });
835
+ const effective = unit.pairs.get(table.name);
836
+ if (effective === undefined) {
837
+ throw new Error('publishSqliteImage: the allowed scopes revoke the pair');
844
838
  }
845
839
  await config.storage.ensureSchema(schema, config.checkpoints);
846
840
  const logEpoch = await config.storage.getPartitionLogEpoch(partition);
@@ -852,18 +846,6 @@ export async function publishSqliteImage(
852
846
  partition,
853
847
  actorId: 'syncular.image-publisher',
854
848
  };
855
- const plan: SubscriptionPlan = {
856
- frame: {
857
- type: 'SUBSCRIPTION',
858
- id: 'image',
859
- table: input.table,
860
- scopes: input.scopes,
861
- cursor: -1,
862
- },
863
- table,
864
- status: 'active',
865
- effective: input.scopes,
866
- };
867
849
  // One-row pages make every table above one row image-eligible (§5.3).
868
850
  const limits: PullLimits = {
869
851
  limitCommits: 1,
@@ -874,24 +856,22 @@ export async function publishSqliteImage(
874
856
  const maxSeq = await config.storage.getMaxCommitSeq(partition);
875
857
  const horizonSeq = await config.storage.getHorizonSeq(partition);
876
858
  const trace: PullSectionTrace = { segments: [] };
877
- const [prepared] = await prepareSections(
859
+ const plan = planBootstrap(
878
860
  ctx,
879
861
  schema,
880
862
  limits,
881
- [plan],
863
+ unit,
864
+ undefined,
882
865
  maxSeq,
883
866
  horizonSeq,
884
867
  logEpoch,
885
868
  );
886
- if (prepared === undefined) throw new Error('prepareSections lost a plan');
887
869
  let ref: SegmentRefFrame | undefined;
888
- for await (const frame of preparedSection(
870
+ for await (const frame of bootstrapSegments(
889
871
  ctx,
890
872
  schema,
891
873
  limits,
892
874
  plan,
893
- maxSeq,
894
- prepared,
895
875
  trace,
896
876
  logEpoch,
897
877
  )) {