@syncular/server 0.59.0 → 0.60.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +20 -16
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +42 -40
  4. package/dist/context.d.ts +6 -8
  5. package/dist/context.js +1 -3
  6. package/dist/d1-storage.js +42 -24
  7. package/dist/errors.js +8 -0
  8. package/dist/events.d.ts +35 -17
  9. package/dist/frame-bytes.js +24 -6
  10. package/dist/handler.js +486 -223
  11. package/dist/local-authority.js +7 -9
  12. package/dist/operations.d.ts +5 -2
  13. package/dist/operations.js +20 -29
  14. package/dist/postgres-storage.d.ts +24 -19
  15. package/dist/postgres-storage.js +249 -168
  16. package/dist/pull.d.ts +68 -53
  17. package/dist/pull.js +258 -287
  18. package/dist/push.d.ts +12 -3
  19. package/dist/push.js +95 -32
  20. package/dist/realtime.d.ts +17 -11
  21. package/dist/realtime.js +65 -90
  22. package/dist/relational-rows.d.ts +0 -57
  23. package/dist/relational-rows.js +4 -100
  24. package/dist/seed.js +10 -15
  25. package/dist/segment-download.d.ts +4 -1
  26. package/dist/segment-download.js +4 -2
  27. package/dist/sqlite-dialect.d.ts +36 -59
  28. package/dist/sqlite-dialect.js +255 -159
  29. package/dist/sqlite-storage.js +24 -20
  30. package/dist/storage-query.d.ts +30 -1
  31. package/dist/storage-query.js +68 -0
  32. package/dist/storage.d.ts +156 -54
  33. package/dist/storage.js +15 -0
  34. package/package.json +2 -2
  35. package/src/admin.ts +87 -59
  36. package/src/context.ts +7 -11
  37. package/src/d1-storage.ts +49 -33
  38. package/src/errors.ts +8 -0
  39. package/src/events.ts +36 -17
  40. package/src/frame-bytes.ts +26 -6
  41. package/src/handler.ts +628 -286
  42. package/src/local-authority.ts +10 -11
  43. package/src/operations.ts +44 -42
  44. package/src/postgres-storage.ts +336 -226
  45. package/src/pull.ts +394 -414
  46. package/src/push.ts +116 -21
  47. package/src/realtime.ts +85 -129
  48. package/src/relational-rows.ts +4 -107
  49. package/src/seed.ts +13 -19
  50. package/src/segment-download.ts +8 -3
  51. package/src/sqlite-dialect.ts +363 -204
  52. package/src/sqlite-storage.ts +45 -25
  53. package/src/storage-query.ts +102 -0
  54. package/src/storage.ts +185 -58
package/dist/pull.js CHANGED
@@ -1,11 +1,11 @@
1
1
  /**
2
- * Pull: incremental commit delivery, cursors, bootstrap segments
3
- * (SPEC.md §4, §5).
2
+ * Pull: units, the live section, bootstrap segments (SPEC.md §4, §5).
4
3
  */
5
- import { encodeRowsSegment, } from '@syncular/core';
4
+ import { encodeRowsSegment, unitId, } from '@syncular/core';
6
5
  import { clockOf, limitsOf } from './context.js';
6
+ import { syncError } from './errors.js';
7
7
  import { compileSchema, } from './schema.js';
8
- import { scopeDigest } from './scopes.js';
8
+ import { computeEffective, scopeDigest } from './scopes.js';
9
9
  import { issueSegmentUrl } from './signed-url.js';
10
10
  import { projectRowPayload, projectRowValues } from './schema-window.js';
11
11
  // One artifact build per owning storage pair and complete immutable identity.
@@ -20,7 +20,7 @@ function clamp(value, min, max, dflt) {
20
20
  return dflt;
21
21
  return Math.min(max, Math.max(min, value));
22
22
  }
23
- /** §4.2 defaults and silent clamps (the v1 values). */
23
+ /** §4.2 defaults and silent clamps. */
24
24
  export function clampPullLimits(header) {
25
25
  return {
26
26
  limitCommits: clamp(header.limitCommits, 1, 1000, 1000),
@@ -30,11 +30,105 @@ export function clampPullLimits(header) {
30
30
  };
31
31
  }
32
32
  /**
33
- * §4.7: a usable resume token for `table`, or `undefined`. A token whose pin
34
- * is not a commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`) or
35
- * whose position does not name one of its tables is unusable.
33
+ * §4.1 unit validation against one schema version: the covered tables, or
34
+ * `sync.invalid_subscription` for an empty scope map, a requested `'*'`, a
35
+ * variable no table declares, or no covered table.
36
36
  */
37
- function parseBootstrapToken(raw, table, maxSeq) {
37
+ export function coveredTables(schema, scopes) {
38
+ const keys = Object.keys(scopes);
39
+ const reason = keys.length === 0
40
+ ? 'empty_scope_map'
41
+ : keys.some((key) => scopes[key] === '*')
42
+ ? 'wildcard_value'
43
+ : keys.some((key) => !schema.declaredVariables.has(key))
44
+ ? 'undeclared_variable'
45
+ : undefined;
46
+ const tables = reason === undefined
47
+ ? [...schema.tables.values()].filter((table) => keys.every((key) => table.declaredVariables.has(key)))
48
+ : [];
49
+ if (tables.length === 0) {
50
+ throw syncError('sync.invalid_subscription', 'the server cannot serve this unit (§4.1)', JSON.stringify({ reason: reason ?? 'no_covered_table', scopes }));
51
+ }
52
+ return tables;
53
+ }
54
+ /** §3.2 per covered table: the unit's active pairs, status and echo. */
55
+ export function planUnit(scopes, tables, resolved) {
56
+ const requested = {};
57
+ for (const [key, value] of Object.entries(scopes))
58
+ requested[key] = [value];
59
+ const pairs = new Map();
60
+ const fenced = [];
61
+ for (const table of tables) {
62
+ const outcome = computeEffective(table.declaredVariables, requested, resolved);
63
+ if (outcome.status === 'revoked') {
64
+ fenced.push({ table: table.name, status: 'revoked', scopes: {} });
65
+ continue;
66
+ }
67
+ pairs.set(table.name, outcome.effective);
68
+ const added = Object.entries(outcome.effective).filter(([key]) => !Object.hasOwn(scopes, key));
69
+ if (added.length > 0) {
70
+ fenced.push({
71
+ table: table.name,
72
+ status: 'active',
73
+ scopes: Object.fromEntries(added),
74
+ });
75
+ }
76
+ }
77
+ const active = pairs.size > 0;
78
+ return {
79
+ id: unitId(scopes),
80
+ scopes,
81
+ tables,
82
+ pairs,
83
+ status: active ? 'active' : 'revoked',
84
+ fenced: active ? fenced.sort((a, b) => (a.table < b.table ? -1 : 1)) : [],
85
+ };
86
+ }
87
+ /**
88
+ * The `COMMIT` frame of one live-section commit (§4.5): its frame-local
89
+ * table dictionary and every change, projected onto the client's codec.
90
+ */
91
+ export function commitFrame(commit, current, target) {
92
+ const tables = [];
93
+ const changes = commit.changes.map((change) => {
94
+ // §4.5: apply is version-guarded, so every change carries its version:
95
+ // an upsert the row's version after the write, a delete the version its
96
+ // tombstone records. A change without one cannot be served.
97
+ if (change.rowVersion === undefined) {
98
+ throw new Error('a change record carries no row version (§4.5)');
99
+ }
100
+ let tableIndex = tables.indexOf(change.table);
101
+ if (tableIndex === -1)
102
+ tableIndex = tables.push(change.table) - 1;
103
+ return {
104
+ tableIndex,
105
+ rowId: change.rowId,
106
+ op: change.op,
107
+ rowVersion: change.rowVersion,
108
+ scopes: change.scopes,
109
+ ...(change.payload !== undefined
110
+ ? {
111
+ row: projectRowPayload(current.tables.get(change.table), target.tables.get(change.table), change.payload),
112
+ }
113
+ : {}),
114
+ };
115
+ });
116
+ return {
117
+ type: 'COMMIT',
118
+ commitSeq: commit.commitSeq,
119
+ createdAtMs: commit.createdAtMs,
120
+ actorId: commit.actorId,
121
+ tables,
122
+ changes,
123
+ };
124
+ }
125
+ /**
126
+ * §4.7: a usable resume token, or `undefined`. A token whose pin is not a
127
+ * commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`), whose tables
128
+ * differ from the unit's covered tables in bootstrap order, or whose position
129
+ * names none of them is unusable.
130
+ */
131
+ function parseBootstrapToken(raw, tables, maxSeq) {
38
132
  if (raw === undefined)
39
133
  return undefined;
40
134
  try {
@@ -44,11 +138,12 @@ function parseBootstrapToken(raw, table, maxSeq) {
44
138
  parsed.asOfCommitSeq < 0 ||
45
139
  parsed.asOfCommitSeq > maxSeq ||
46
140
  !Array.isArray(parsed.tables) ||
47
- parsed.tables[0] !== table ||
141
+ parsed.tables.length !== tables.length ||
142
+ parsed.tables.some((name, i) => name !== tables[i]?.name) ||
48
143
  !Number.isSafeInteger(parsed.tableIndex) ||
49
144
  parsed.tableIndex === undefined ||
50
145
  parsed.tableIndex < 0 ||
51
- parsed.tableIndex >= parsed.tables.length ||
146
+ parsed.tableIndex >= tables.length ||
52
147
  (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')) {
53
148
  return undefined;
54
149
  }
@@ -63,28 +158,6 @@ function parseBootstrapToken(raw, table, maxSeq) {
63
158
  return undefined;
64
159
  }
65
160
  }
66
- function commitFrame(table, commit, current, target) {
67
- const changes = commit.changes.map((change) => ({
68
- tableIndex: 0,
69
- rowId: change.rowId,
70
- op: change.op,
71
- ...(change.rowVersion !== undefined
72
- ? { rowVersion: change.rowVersion }
73
- : {}),
74
- scopes: change.scopes,
75
- ...(change.payload !== undefined
76
- ? { row: projectRowPayload(current, target, change.payload) }
77
- : {}),
78
- }));
79
- return {
80
- type: 'COMMIT',
81
- commitSeq: commit.commitSeq,
82
- createdAtMs: commit.createdAtMs,
83
- actorId: commit.actorId,
84
- tables: [table],
85
- changes,
86
- };
87
- }
88
161
  function chunkRows(rows, size) {
89
162
  if (rows.length === 0)
90
163
  return [];
@@ -114,10 +187,10 @@ function segmentRefFrame(record, scopeDigest, extra) {
114
187
  byteLength: record.byteLength,
115
188
  rowCount: record.rowCount,
116
189
  asOfCommitSeq: record.asOfCommitSeq,
117
- // The frame reports the digest THIS subscription is authorized under. A
118
- // stored entry can carry several digests when identical content was
119
- // published under different scopes (§5.1), and the record's primary
120
- // digest is not necessarily the caller's (§5.5).
190
+ // The frame reports the digest THIS pair is authorized under. A stored
191
+ // entry can carry several digests when identical content was published
192
+ // under different scopes (§5.1), and the record's primary digest is not
193
+ // necessarily the caller's (§5.5).
121
194
  scopeDigest,
122
195
  ...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
123
196
  ...(record.nextRowCursor !== null
@@ -137,20 +210,57 @@ function segmentRefFrame(record, scopeDigest, extra) {
137
210
  * wholesale with no per-row decrypt pass, so it MUST be served on the rows
138
211
  * lane (which decrypts per row on the client).
139
212
  */
140
- function sqliteLaneEligible(limits, plan, startRowCursor) {
213
+ function sqliteLaneEligible(limits, table, startRowCursor) {
141
214
  return ((limits.accept & ACCEPT_SQLITE) !== 0 &&
142
215
  startRowCursor === null &&
143
- plan.table.encryptedColumnIndices.length === 0);
216
+ table.encryptedColumnIndices.length === 0);
217
+ }
218
+ async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, maxSeq, horizonSeq, logEpoch) {
219
+ const scan = ctx.storage.scanRows(ctx.partition, {
220
+ table: pair.table.name,
221
+ scopeFilter: pair.effective,
222
+ afterRowId: startRowCursor,
223
+ limit: limits.limitSnapshotRows + 1,
224
+ });
225
+ if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
226
+ return { firstPage: await scan };
227
+ }
228
+ // §4.7: a sqlite-lane table pins at the newest change carrying the unit's
229
+ // index key (never below the horizon, never above maxCommitSeq), so a
230
+ // stored image stays the reuse key's match until this scope changes
231
+ // (§5.3). The server computes the pin; a client token never supplies it.
232
+ const [firstPage, latest] = await Promise.all([
233
+ scan,
234
+ ctx.storage.latestScopedChangeSeq(ctx.partition, {
235
+ table: pair.table.name,
236
+ scopes: unit.scopes,
237
+ }),
238
+ ]);
239
+ const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
240
+ // §5.3 eligibility: a stored image only matches a snapshot larger than one
241
+ // rows page, so the probe decides before the reuse lookup; a remote `find`
242
+ // on a small table would be a wasted round trip.
243
+ const stored = firstPage.length <= limits.limitSnapshotRows
244
+ ? undefined
245
+ : await ctx.segments.find({
246
+ partition: ctx.partition,
247
+ logEpoch,
248
+ table: pair.table.name,
249
+ schemaVersion: schema.version,
250
+ mediaType: 'sqlite',
251
+ scopeDigest: await scopeDigest(pair.effective),
252
+ asOfCommitSeq: scopePin,
253
+ }, clockOf(ctx)());
254
+ return { firstPage, image: { scopePin, stored } };
144
255
  }
145
256
  /**
146
- * §5.3 sqlite-image lane: only at the start of a table, only when the
147
- * client advertised bit 2, and only when the snapshot exceeds one rows
148
- * page. Reuses an unexpired stored image for the same (partition, table,
149
- * schemaVersion, scope digest, pin) instead of rebuilding — the
150
- * bootstrap-storm rule. Returns false when the table is not eligible
151
- * (the rows lane takes over).
257
+ * §5.3 sqlite-image lane: only when the snapshot exceeds one rows page.
258
+ * Reuses an unexpired stored image for the same (partition, table,
259
+ * schemaVersion, scope digest, scope pin) instead of rebuilding — the
260
+ * bootstrap-storm rule. Returns false when no image is served (the rows lane
261
+ * takes over).
152
262
  */
153
- async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trace, logEpoch, existing, probe) {
263
+ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest, trace, logEpoch, existing, probe) {
154
264
  const { storage, segments, partition } = ctx;
155
265
  const now = clockOf(ctx)();
156
266
  if (existing !== undefined) {
@@ -164,9 +274,6 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
164
274
  yield segmentRefFrame(existing, digest, await signedUrlFields(ctx, limits, existing.segmentId, digest, now));
165
275
  return true;
166
276
  }
167
- // Eligibility probe (§5.3): image only when the snapshot exceeds one
168
- // rows page — smaller tables stay on the (typically inline) rows lane.
169
- // The probe is the rows lane's first page, read by `prepareSections`.
170
277
  if (probe.length <= limits.limitSnapshotRows)
171
278
  return false;
172
279
  // §5.3: hosts opt into image construction with a runtime-specific builder.
@@ -187,12 +294,13 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
187
294
  const identity = {
188
295
  partition,
189
296
  logEpoch,
190
- table: plan.table.name,
297
+ table: pair.table.name,
191
298
  schemaVersion: schema.version,
192
299
  mediaType: 'sqlite',
193
300
  scopeDigest: digest,
194
- asOfCommitSeq: asOf,
301
+ asOfCommitSeq: scopePin,
195
302
  };
303
+ const current = compileSchema(ctx.schema).tables.get(pair.table.name);
196
304
  const key = JSON.stringify(identity);
197
305
  let building = builds.get(key);
198
306
  let builtHere = false;
@@ -205,28 +313,28 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
205
313
  builtHere = true;
206
314
  let rowCount = 0;
207
315
  const bytes = await buildImage({
208
- table: plan.table,
316
+ table: pair.table,
209
317
  schemaVersion: schema.version,
210
- asOfCommitSeq: asOf,
318
+ asOfCommitSeq: scopePin,
211
319
  scopeDigest: digest,
212
320
  rowBatches: (async function* () {
213
321
  rowCount += probe.length;
214
322
  yield probe.map((row) => ({
215
323
  ...row,
216
- payload: projectRowPayload(compileSchema(ctx.schema).tables.get(plan.table.name), plan.table, row.payload),
324
+ payload: projectRowPayload(current, pair.table, row.payload),
217
325
  }));
218
326
  let afterRowId = probe[probe.length - 1].rowId;
219
327
  for (;;) {
220
328
  const rows = await storage.scanRows(partition, {
221
- table: plan.table.name,
222
- scopeFilter: plan.effective,
329
+ table: pair.table.name,
330
+ scopeFilter: pair.effective,
223
331
  afterRowId,
224
332
  limit: 5_000,
225
333
  });
226
334
  rowCount += rows.length;
227
335
  yield rows.map((row) => ({
228
336
  ...row,
229
- payload: projectRowPayload(compileSchema(ctx.schema).tables.get(plan.table.name), plan.table, row.payload),
337
+ payload: projectRowPayload(current, pair.table, row.payload),
230
338
  }));
231
339
  const last = rows[rows.length - 1];
232
340
  if (rows.length < 5_000 || last === undefined)
@@ -257,23 +365,28 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
257
365
  yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, clockOf(ctx)()));
258
366
  return true;
259
367
  }
260
- async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCursor, trace, logEpoch, firstPage, image) {
368
+ /**
369
+ * One table's segments within the page budget. Returns the pages it used
370
+ * and the row cursor to resume at, `null` once the table is exhausted.
371
+ */
372
+ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, start, pageBudget, trace, logEpoch) {
261
373
  const { storage, segments, partition } = ctx;
262
374
  const serverLimits = limitsOf(ctx);
263
- const digest = await scopeDigest(plan.effective);
375
+ const digest = await scopeDigest(pair.effective);
264
376
  const now = clockOf(ctx)();
265
- if (sqliteLaneEligible(limits, plan, startRowCursor)) {
266
- const imaged = yield* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trace, logEpoch, image, firstPage);
377
+ if (start.image !== undefined) {
378
+ const imaged = yield* sqliteImageSegment(ctx, schema, limits, pair, start.image.scopePin, digest, trace, logEpoch, start.image.stored, start.firstPage);
267
379
  if (imaged)
268
- return { complete: true, rowCursor: null };
380
+ return { pages: 1, rowCursor: null };
269
381
  }
382
+ const current = compileSchema(ctx.schema).tables.get(pair.table.name);
270
383
  let rowCursor = startRowCursor;
271
- for (let page = 0; page < limits.maxSnapshotPages; page++) {
384
+ for (let page = 0; page < pageBudget; page++) {
272
385
  const scanned = page === 0
273
- ? firstPage
386
+ ? start.firstPage
274
387
  : await storage.scanRows(partition, {
275
- table: plan.table.name,
276
- scopeFilter: plan.effective,
388
+ table: pair.table.name,
389
+ scopeFilter: pair.effective,
277
390
  afterRowId: rowCursor,
278
391
  limit: limits.limitSnapshotRows + 1,
279
392
  });
@@ -282,12 +395,12 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
282
395
  // §5.2: every row record carries the row's current server_version.
283
396
  const decoded = pageRows.map((row) => ({
284
397
  serverVersion: row.serverVersion,
285
- values: projectRowValues(compileSchema(ctx.schema).tables.get(plan.table.name), plan.table, row.payload),
398
+ values: projectRowValues(current, pair.table, row.payload),
286
399
  }));
287
400
  const bytes = encodeRowsSegment({
288
- table: plan.table.name,
401
+ table: pair.table.name,
289
402
  schemaVersion: schema.version,
290
- columns: plan.table.columns,
403
+ columns: pair.table.columns,
291
404
  blocks: chunkRows(decoded, 1000),
292
405
  });
293
406
  const lastRow = pageRows[pageRows.length - 1];
@@ -310,11 +423,11 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
310
423
  const record = await segments.put({
311
424
  partition,
312
425
  logEpoch,
313
- table: plan.table.name,
426
+ table: pair.table.name,
314
427
  schemaVersion: schema.version,
315
428
  mediaType: 'rows',
316
429
  scopeDigest: digest,
317
- asOfCommitSeq: asOf,
430
+ asOfCommitSeq: pin,
318
431
  rowCount: pageRows.length,
319
432
  rowCursor,
320
433
  nextRowCursor,
@@ -329,219 +442,84 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
329
442
  yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
330
443
  }
331
444
  if (!hasMore)
332
- return { complete: true, rowCursor: null };
445
+ return { pages: page + 1, rowCursor: null };
333
446
  rowCursor = nextRowCursor;
334
447
  }
335
- return { complete: false, rowCursor };
448
+ return { pages: pageBudget, rowCursor };
336
449
  }
337
450
  /**
338
- * Decide the section of every plan and run its pre-frame storage reads. The
339
- * reads of all plans start before any is awaited, so a storage that batches
340
- * concurrent page reads (PostgreSQL, D1) answers the first pages of every
341
- * plan in one statement per table, and the horizon re-check runs once.
451
+ * Plan one unit's bootstrap and start the reads of every table it can reach
452
+ * in this section: each visited table costs at least one page, so at most
453
+ * `maxSnapshotPages` tables. The reads of every planned unit start before any
454
+ * is awaited, so a storage that batches concurrent page reads (PostgreSQL,
455
+ * D1) answers them in one statement per table.
342
456
  */
343
- export async function prepareSections(ctx, schema, limits, plans, maxSeq, horizonSeq, logEpoch) {
344
- const prepared = await Promise.all(plans.map(async (plan) => {
345
- const sub = plan.frame;
346
- if (plan.status === 'revoked')
347
- return { kind: 'revoked' };
348
- // §4.7: a present resume token keeps the subscription bootstrapping,
349
- // whether or not it is usable.
350
- if (sub.bootstrapState === undefined &&
351
- sub.cursor >= 0 &&
352
- sub.cursor <= maxSeq) {
353
- if (sub.cursor < horizonSeq)
354
- return { kind: 'reset' };
355
- return {
356
- kind: 'incremental',
357
- commits: await ctx.storage.readCommitWindow(ctx.partition, {
358
- table: sub.table,
359
- scopeFilter: plan.effective,
360
- afterSeq: sub.cursor,
361
- throughSeq: maxSeq,
362
- limitChanges: limits.limitCommits + 1,
363
- }),
364
- };
365
- }
366
- // §4.7: resume at the token's pin unless the token is unusable or the
367
- // pin fell behind the horizon; then restart from a fresh pin.
368
- const token = parseBootstrapToken(sub.bootstrapState, sub.table, maxSeq);
369
- const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
370
- ? token
371
- : undefined;
372
- const startCursor = resume?.rowCursor ?? null;
373
- const scan = ctx.storage.scanRows(ctx.partition, {
374
- table: plan.table.name,
375
- scopeFilter: plan.effective,
376
- afterRowId: startCursor,
377
- limit: limits.limitSnapshotRows + 1,
378
- });
379
- if (!sqliteLaneEligible(limits, plan, startCursor)) {
380
- return {
381
- kind: 'bootstrap',
382
- asOf: resume?.asOfCommitSeq ?? maxSeq,
383
- startCursor,
384
- firstPage: await scan,
385
- image: undefined,
386
- };
387
- }
388
- // §4.7: a sqlite-lane bootstrap starts at the table start and pins at
389
- // the newest change in its scope (never below the horizon, never above
390
- // maxCommitSeq), so a stored image stays the reuse key's match until
391
- // this scope changes (§5.3). The server computes the pin; a client
392
- // token never supplies the reuse key.
393
- const [firstPage, latest] = await Promise.all([
394
- scan,
395
- ctx.storage.latestScopedChangeSeq(ctx.partition, {
396
- table: plan.table.name,
397
- scopeFilter: plan.effective,
398
- }),
399
- ]);
400
- const asOf = Math.min(maxSeq, Math.max(latest, horizonSeq));
401
- // §5.3 eligibility: a stored image only matches a snapshot larger than
402
- // one rows page, so the probe decides before the reuse lookup; a remote
403
- // `find` on a small table would be a wasted round trip.
404
- const image = firstPage.length <= limits.limitSnapshotRows
405
- ? undefined
406
- : await ctx.segments.find({
407
- partition: ctx.partition,
408
- logEpoch,
409
- table: plan.table.name,
410
- schemaVersion: schema.version,
411
- mediaType: 'sqlite',
412
- scopeDigest: await scopeDigest(plan.effective),
413
- asOfCommitSeq: asOf,
414
- }, clockOf(ctx)());
415
- return { kind: 'bootstrap', asOf, startCursor, firstPage, image };
416
- }));
417
- // Validate continuity before committing to an active section. A prune
418
- // during a paged read can otherwise make an incomplete window look empty.
419
- if (!prepared.some((section) => section.kind === 'incremental')) {
420
- return prepared;
457
+ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq, horizonSeq, logEpoch) {
458
+ // §4.7: resume at the token's pin unless the token is unusable or the pin
459
+ // fell behind the horizon; then restart from a fresh pin.
460
+ const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
461
+ const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
462
+ ? token
463
+ : undefined;
464
+ const tableIndex = resume?.tableIndex ?? 0;
465
+ const rowCursor = resume?.rowCursor ?? null;
466
+ const starts = new Map();
467
+ for (let index = tableIndex; index < unit.tables.length && starts.size < limits.maxSnapshotPages; index++) {
468
+ const table = unit.tables[index];
469
+ const effective = unit.pairs.get(table.name);
470
+ if (effective === undefined)
471
+ continue;
472
+ const start = readTableStart(ctx, schema, limits, unit, { table, effective }, index === tableIndex ? rowCursor : null, maxSeq, horizonSeq, logEpoch);
473
+ // A section that stops early never awaits the later starts.
474
+ start.catch(() => undefined);
475
+ starts.set(index, start);
421
476
  }
422
- const windowHorizon = await ctx.storage.getHorizonSeq(ctx.partition);
423
- return prepared.map((section, index) => section.kind === 'incremental' &&
424
- (plans[index]?.frame.cursor ?? -1) < windowHorizon
425
- ? { kind: 'reset' }
426
- : section);
477
+ return {
478
+ unit,
479
+ pin: resume?.asOfCommitSeq ?? maxSeq,
480
+ tableIndex,
481
+ rowCursor,
482
+ starts,
483
+ };
427
484
  }
428
485
  /**
429
- * Produce the `SUB_START … SUB_END` section for one subscription (§1.6),
430
- * returning the cursor recorded for the retention watermark (§4.5).
486
+ * The segments of one bootstrap section (§4.7): tables in bootstrap order,
487
+ * each active pair's rows or image, up to `maxSnapshotPages` pages. Returns
488
+ * the resume token, or `undefined` once every table is exhausted. A pair the
489
+ * wildcard fence revoked receives no segment (§3.3).
431
490
  */
432
- export async function* subscriptionSection(ctx, schema, limits, plan, maxSeq, horizonSeq, trace, logEpoch) {
433
- if (logEpoch === undefined || logEpoch.length === 0) {
434
- throw new Error('subscriptionSection requires a non-empty log epoch');
435
- }
436
- const [prepared] = await prepareSections(ctx, schema, limits, [plan], maxSeq, horizonSeq, logEpoch);
437
- if (prepared === undefined)
438
- throw new Error('prepareSections lost a plan');
439
- return yield* preparedSection(ctx, schema, limits, plan, maxSeq, prepared, trace, logEpoch);
440
- }
441
- /** The section of one subscription whose reads `prepareSections` ran. */
442
- export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepared, trace, logEpoch) {
443
- const sub = plan.frame;
444
- if (logEpoch === undefined || logEpoch.length === 0) {
445
- throw new Error('subscriptionSection requires a non-empty log epoch');
446
- }
447
- if (prepared.kind === 'revoked') {
448
- yield {
449
- type: 'SUB_START',
450
- id: sub.id,
451
- status: 'revoked',
452
- reasonCode: 'sync.scope_revoked',
453
- effectiveScopes: {},
454
- bootstrap: false,
455
- };
456
- yield { type: 'SUB_END', nextCursor: sub.cursor };
457
- return { nextCursor: sub.cursor, active: false };
458
- }
459
- // §4.6: a cursor behind the horizon (and not resuming a bootstrap)
460
- // cannot compute deltas — answer `reset` and echo the cursor.
461
- if (prepared.kind === 'reset') {
462
- yield {
463
- type: 'SUB_START',
464
- id: sub.id,
465
- status: 'reset',
466
- reasonCode: 'sync.cursor_expired',
467
- effectiveScopes: {},
468
- bootstrap: false,
469
- };
470
- yield { type: 'SUB_END', nextCursor: sub.cursor };
471
- return { nextCursor: sub.cursor, active: false };
472
- }
473
- if (prepared.kind === 'bootstrap') {
474
- const { asOf } = prepared;
475
- yield {
476
- type: 'SUB_START',
477
- id: sub.id,
478
- status: 'active',
479
- reasonCode: '',
480
- effectiveScopes: plan.effective,
481
- bootstrap: true,
482
- };
483
- const outcome = yield* bootstrapSegments(ctx, schema, limits, plan, asOf, prepared.startCursor, trace, logEpoch, prepared.firstPage, prepared.image);
484
- if (outcome.complete) {
485
- yield { type: 'SUB_END', nextCursor: asOf };
491
+ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch) {
492
+ const { unit, pin } = plan;
493
+ let { tableIndex, rowCursor } = plan;
494
+ let pages = 0;
495
+ for (;;) {
496
+ while (tableIndex < unit.tables.length &&
497
+ !unit.pairs.has(unit.tables[tableIndex].name)) {
498
+ tableIndex += 1;
499
+ rowCursor = null;
486
500
  }
487
- else {
488
- const nextToken = {
489
- asOfCommitSeq: asOf,
490
- tables: [sub.table],
491
- tableIndex: 0,
492
- rowCursor: outcome.rowCursor,
493
- };
494
- yield {
495
- type: 'SUB_END',
496
- nextCursor: asOf,
497
- bootstrapState: JSON.stringify(nextToken),
501
+ if (tableIndex === unit.tables.length)
502
+ return undefined;
503
+ if (pages === limits.maxSnapshotPages) {
504
+ const token = {
505
+ asOfCommitSeq: pin,
506
+ tables: unit.tables.map((table) => table.name),
507
+ tableIndex,
508
+ rowCursor,
498
509
  };
510
+ return JSON.stringify(token);
499
511
  }
500
- return { nextCursor: asOf, active: true };
501
- }
502
- // Incremental (§4.5): window cursor < commitSeq <= maxSeq, oldest first,
503
- // cut off at limitCommits total changes, never splitting a commit.
504
- yield {
505
- type: 'SUB_START',
506
- id: sub.id,
507
- status: 'active',
508
- reasonCode: '',
509
- effectiveScopes: plan.effective,
510
- bootstrap: false,
511
- };
512
- let delivered = 0;
513
- let deliveredCommits = 0;
514
- let lastDeliveredSeq = sub.cursor;
515
- const { commits } = prepared;
516
- for (const commit of commits) {
517
- if (delivered > 0 &&
518
- delivered + commit.changes.length > limits.limitCommits) {
519
- break;
520
- }
521
- yield commitFrame(sub.table, commit, compileSchema(ctx.schema).tables.get(sub.table), plan.table);
522
- delivered += commit.changes.length;
523
- deliveredCommits += 1;
524
- lastDeliveredSeq = commit.commitSeq;
525
- if (delivered >= limits.limitCommits)
526
- break;
512
+ const table = unit.tables[tableIndex];
513
+ const outcome = yield* tableSegments(ctx, schema, limits, { table, effective: unit.pairs.get(table.name) }, pin, rowCursor, await plan.starts.get(tableIndex), limits.maxSnapshotPages - pages, trace, logEpoch);
514
+ pages += outcome.pages;
515
+ rowCursor = outcome.rowCursor;
516
+ if (rowCursor === null)
517
+ tableIndex += 1;
527
518
  }
528
- const totalReturned = commits.reduce((n, c) => n + c.changes.length, 0);
529
- // The window is proven exhausted only when the storage lookahead came
530
- // back under budget (it scanned to `throughSeq`) and every returned
531
- // commit was delivered.
532
- const exhausted = totalReturned <= limits.limitCommits && deliveredCommits === commits.length;
533
- // §4.5: the cursor advances even when no matching changes exist; when
534
- // the change limit truncated the window it stops at the last fully
535
- // delivered commit.
536
- const nextCursor = exhausted
537
- ? Math.max(sub.cursor, maxSeq)
538
- : Math.max(sub.cursor, lastDeliveredSeq);
539
- yield { type: 'SUB_END', nextCursor };
540
- return { nextCursor, active: true };
541
519
  }
542
520
  /**
543
- * Publish the §5.3 sqlite image that a pull for (`table`, `scopes`) looks
544
- * up, from a host with a SQLite engine (a Bun process with the production
521
+ * Publish the §5.3 sqlite image that a pull of (`scopes`, `table`) looks up,
522
+ * from a host with a SQLite engine (a Bun process with the production
545
523
  * storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
546
524
  * reuse key, the build), so the stored identity is exactly the one a
547
525
  * serving host without a builder finds. The image stays current until a
@@ -554,9 +532,16 @@ export async function publishSqliteImage(input) {
554
532
  throw new Error('publishSqliteImage requires config.sqliteImageBuilder');
555
533
  }
556
534
  const schema = compileSchema(config.schema);
557
- const table = schema.tables.get(input.table);
535
+ const table = coveredTables(schema, input.scopes).find((candidate) => candidate.name === input.table);
558
536
  if (table === undefined) {
559
- throw new Error('publishSqliteImage: the schema has no such table');
537
+ throw new Error('publishSqliteImage: the unit does not cover the table');
538
+ }
539
+ const allowed = input.allowed ??
540
+ Object.fromEntries([...schema.declaredVariables].map((v) => [v, ['*']]));
541
+ const unit = planUnit(input.scopes, [table], { ok: true, allowed });
542
+ const effective = unit.pairs.get(table.name);
543
+ if (effective === undefined) {
544
+ throw new Error('publishSqliteImage: the allowed scopes revoke the pair');
560
545
  }
561
546
  await config.storage.ensureSchema(schema, config.checkpoints);
562
547
  const logEpoch = await config.storage.getPartitionLogEpoch(partition);
@@ -568,18 +553,6 @@ export async function publishSqliteImage(input) {
568
553
  partition,
569
554
  actorId: 'syncular.image-publisher',
570
555
  };
571
- const plan = {
572
- frame: {
573
- type: 'SUBSCRIPTION',
574
- id: 'image',
575
- table: input.table,
576
- scopes: input.scopes,
577
- cursor: -1,
578
- },
579
- table,
580
- status: 'active',
581
- effective: input.scopes,
582
- };
583
556
  // One-row pages make every table above one row image-eligible (§5.3).
584
557
  const limits = {
585
558
  limitCommits: 1,
@@ -590,11 +563,9 @@ export async function publishSqliteImage(input) {
590
563
  const maxSeq = await config.storage.getMaxCommitSeq(partition);
591
564
  const horizonSeq = await config.storage.getHorizonSeq(partition);
592
565
  const trace = { segments: [] };
593
- const [prepared] = await prepareSections(ctx, schema, limits, [plan], maxSeq, horizonSeq, logEpoch);
594
- if (prepared === undefined)
595
- throw new Error('prepareSections lost a plan');
566
+ const plan = planBootstrap(ctx, schema, limits, unit, undefined, maxSeq, horizonSeq, logEpoch);
596
567
  let ref;
597
- for await (const frame of preparedSection(ctx, schema, limits, plan, maxSeq, prepared, trace, logEpoch)) {
568
+ for await (const frame of bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch)) {
598
569
  if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
599
570
  ref = frame;
600
571
  }