@syncular/server 0.58.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/README.md +49 -36
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +46 -40
  4. package/dist/blob-handlers.d.ts +25 -31
  5. package/dist/blob-handlers.js +63 -41
  6. package/dist/blob-store.d.ts +42 -24
  7. package/dist/blob-store.js +0 -0
  8. package/dist/bucket-blob-store.d.ts +12 -8
  9. package/dist/bucket-blob-store.js +54 -45
  10. package/dist/bucket-segment-store.js +18 -16
  11. package/dist/content-encoding.d.ts +5 -2
  12. package/dist/content-encoding.js +42 -13
  13. package/dist/context.d.ts +19 -5
  14. package/dist/context.js +19 -1
  15. package/dist/d1-storage.d.ts +14 -98
  16. package/dist/d1-storage.js +249 -1119
  17. package/dist/errors.js +16 -0
  18. package/dist/events.d.ts +36 -18
  19. package/dist/frame-bytes.js +24 -6
  20. package/dist/handler.js +503 -208
  21. package/dist/local-authority.d.ts +2 -0
  22. package/dist/local-authority.js +24 -12
  23. package/dist/object-bucket.d.ts +2 -0
  24. package/dist/operations-realtime.js +69 -50
  25. package/dist/operations.d.ts +5 -2
  26. package/dist/operations.js +20 -29
  27. package/dist/pg-executor-pglite.d.ts +5 -6
  28. package/dist/pg-executor-pglite.js +22 -14
  29. package/dist/pg-executor.d.ts +4 -1
  30. package/dist/postgres-storage.d.ts +24 -19
  31. package/dist/postgres-storage.js +420 -262
  32. package/dist/pull.d.ts +70 -47
  33. package/dist/pull.js +270 -301
  34. package/dist/push.d.ts +12 -3
  35. package/dist/push.js +361 -173
  36. package/dist/realtime.d.ts +85 -39
  37. package/dist/realtime.js +355 -219
  38. package/dist/relational-rows.d.ts +21 -66
  39. package/dist/relational-rows.js +28 -109
  40. package/dist/s3-blob-store.d.ts +11 -12
  41. package/dist/s3-blob-store.js +20 -13
  42. package/dist/s3-object-bucket.d.ts +8 -1
  43. package/dist/s3-object-bucket.js +16 -3
  44. package/dist/schema.d.ts +14 -0
  45. package/dist/schema.js +23 -0
  46. package/dist/scopes.d.ts +6 -2
  47. package/dist/scopes.js +22 -4
  48. package/dist/seed.js +10 -16
  49. package/dist/segment-download.d.ts +4 -1
  50. package/dist/segment-download.js +22 -6
  51. package/dist/signed-url.d.ts +2 -2
  52. package/dist/signed-url.js +9 -2
  53. package/dist/sigv4.d.ts +8 -2
  54. package/dist/sigv4.js +8 -4
  55. package/dist/sqlite-blob-store.d.ts +1 -1
  56. package/dist/sqlite-blob-store.js +23 -22
  57. package/dist/sqlite-dialect.d.ts +164 -84
  58. package/dist/sqlite-dialect.js +1145 -150
  59. package/dist/sqlite-lease-store.js +13 -18
  60. package/dist/sqlite-storage.d.ts +9 -62
  61. package/dist/sqlite-storage.js +303 -1123
  62. package/dist/storage-query.d.ts +30 -1
  63. package/dist/storage-query.js +68 -0
  64. package/dist/storage.d.ts +185 -63
  65. package/dist/storage.js +15 -0
  66. package/package.json +2 -2
  67. package/src/admin.ts +99 -57
  68. package/src/blob-handlers.ts +91 -72
  69. package/src/blob-store.ts +0 -0
  70. package/src/bucket-blob-store.ts +75 -48
  71. package/src/bucket-segment-store.ts +20 -15
  72. package/src/content-encoding.ts +48 -16
  73. package/src/context.ts +33 -6
  74. package/src/d1-storage.ts +350 -1700
  75. package/src/errors.ts +16 -0
  76. package/src/events.ts +37 -18
  77. package/src/frame-bytes.ts +26 -6
  78. package/src/handler.ts +654 -261
  79. package/src/local-authority.ts +27 -14
  80. package/src/object-bucket.ts +2 -0
  81. package/src/operations-realtime.ts +85 -61
  82. package/src/operations.ts +44 -42
  83. package/src/pg-executor-pglite.ts +27 -23
  84. package/src/pg-executor.ts +5 -1
  85. package/src/postgres-storage.ts +557 -354
  86. package/src/pull.ts +406 -440
  87. package/src/push.ts +466 -220
  88. package/src/realtime.ts +447 -290
  89. package/src/relational-rows.ts +43 -121
  90. package/src/s3-blob-store.ts +23 -13
  91. package/src/s3-object-bucket.ts +20 -2
  92. package/src/schema.ts +31 -0
  93. package/src/scopes.ts +23 -3
  94. package/src/seed.ts +13 -20
  95. package/src/segment-download.ts +33 -7
  96. package/src/signed-url.ts +10 -4
  97. package/src/sigv4.ts +17 -4
  98. package/src/sqlite-blob-store.ts +27 -26
  99. package/src/sqlite-dialect.ts +1851 -199
  100. package/src/sqlite-lease-store.ts +12 -23
  101. package/src/sqlite-storage.ts +390 -1821
  102. package/src/storage-query.ts +102 -0
  103. package/src/storage.ts +215 -67
package/dist/pull.js CHANGED
@@ -1,11 +1,11 @@
1
1
  /**
2
- * Pull: incremental commit delivery, cursors, bootstrap segments
3
- * (SPEC.md §4, §5).
2
+ * Pull: units, the live section, bootstrap segments (SPEC.md §4, §5).
4
3
  */
5
- import { encodeRowsSegment, } from '@syncular/core';
4
+ import { encodeRowsSegment, unitId, } from '@syncular/core';
6
5
  import { clockOf, limitsOf } from './context.js';
6
+ import { syncError } from './errors.js';
7
7
  import { compileSchema, } from './schema.js';
8
- import { scopeDigest } from './scopes.js';
8
+ import { computeEffective, scopeDigest } from './scopes.js';
9
9
  import { issueSegmentUrl } from './signed-url.js';
10
10
  import { projectRowPayload, projectRowValues } from './schema-window.js';
11
11
  // One artifact build per owning storage pair and complete immutable identity.
@@ -20,7 +20,7 @@ function clamp(value, min, max, dflt) {
20
20
  return dflt;
21
21
  return Math.min(max, Math.max(min, value));
22
22
  }
23
- /** §4.2 defaults and silent clamps (the v1 values). */
23
+ /** §4.2 defaults and silent clamps. */
24
24
  export function clampPullLimits(header) {
25
25
  return {
26
26
  limitCommits: clamp(header.limitCommits, 1, 1000, 1000),
@@ -29,16 +29,122 @@ export function clampPullLimits(header) {
29
29
  accept: header.accept,
30
30
  };
31
31
  }
32
- function parseBootstrapToken(raw, table) {
32
+ /**
33
+ * §4.1 unit validation against one schema version: the covered tables, or
34
+ * `sync.invalid_subscription` for an empty scope map, a requested `'*'`, a
35
+ * variable no table declares, or no covered table.
36
+ */
37
+ export function coveredTables(schema, scopes) {
38
+ const keys = Object.keys(scopes);
39
+ const reason = keys.length === 0
40
+ ? 'empty_scope_map'
41
+ : keys.some((key) => scopes[key] === '*')
42
+ ? 'wildcard_value'
43
+ : keys.some((key) => !schema.declaredVariables.has(key))
44
+ ? 'undeclared_variable'
45
+ : undefined;
46
+ const tables = reason === undefined
47
+ ? [...schema.tables.values()].filter((table) => keys.every((key) => table.declaredVariables.has(key)))
48
+ : [];
49
+ if (tables.length === 0) {
50
+ throw syncError('sync.invalid_subscription', 'the server cannot serve this unit (§4.1)', JSON.stringify({ reason: reason ?? 'no_covered_table', scopes }));
51
+ }
52
+ return tables;
53
+ }
54
+ /** §3.2 per covered table: the unit's active pairs, status and echo. */
55
+ export function planUnit(scopes, tables, resolved) {
56
+ const requested = {};
57
+ for (const [key, value] of Object.entries(scopes))
58
+ requested[key] = [value];
59
+ const pairs = new Map();
60
+ const fenced = [];
61
+ for (const table of tables) {
62
+ const outcome = computeEffective(table.declaredVariables, requested, resolved);
63
+ if (outcome.status === 'revoked') {
64
+ fenced.push({ table: table.name, status: 'revoked', scopes: {} });
65
+ continue;
66
+ }
67
+ pairs.set(table.name, outcome.effective);
68
+ const added = Object.entries(outcome.effective).filter(([key]) => !Object.hasOwn(scopes, key));
69
+ if (added.length > 0) {
70
+ fenced.push({
71
+ table: table.name,
72
+ status: 'active',
73
+ scopes: Object.fromEntries(added),
74
+ });
75
+ }
76
+ }
77
+ const active = pairs.size > 0;
78
+ return {
79
+ id: unitId(scopes),
80
+ scopes,
81
+ tables,
82
+ pairs,
83
+ status: active ? 'active' : 'revoked',
84
+ fenced: active ? fenced.sort((a, b) => (a.table < b.table ? -1 : 1)) : [],
85
+ };
86
+ }
87
+ /**
88
+ * The `COMMIT` frame of one live-section commit (§4.5): its frame-local
89
+ * table dictionary and every change, projected onto the client's codec.
90
+ */
91
+ export function commitFrame(commit, current, target) {
92
+ const tables = [];
93
+ const changes = commit.changes.map((change) => {
94
+ // §4.5: apply is version-guarded, so every change carries its version:
95
+ // an upsert the row's version after the write, a delete the version its
96
+ // tombstone records. A change without one cannot be served.
97
+ if (change.rowVersion === undefined) {
98
+ throw new Error('a change record carries no row version (§4.5)');
99
+ }
100
+ let tableIndex = tables.indexOf(change.table);
101
+ if (tableIndex === -1)
102
+ tableIndex = tables.push(change.table) - 1;
103
+ return {
104
+ tableIndex,
105
+ rowId: change.rowId,
106
+ op: change.op,
107
+ rowVersion: change.rowVersion,
108
+ scopes: change.scopes,
109
+ ...(change.payload !== undefined
110
+ ? {
111
+ row: projectRowPayload(current.tables.get(change.table), target.tables.get(change.table), change.payload),
112
+ }
113
+ : {}),
114
+ };
115
+ });
116
+ return {
117
+ type: 'COMMIT',
118
+ commitSeq: commit.commitSeq,
119
+ createdAtMs: commit.createdAtMs,
120
+ actorId: commit.actorId,
121
+ tables,
122
+ changes,
123
+ };
124
+ }
125
+ /**
126
+ * §4.7: a usable resume token, or `undefined`. A token whose pin is not a
127
+ * commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`), whose tables
128
+ * differ from the unit's covered tables in bootstrap order, or whose position
129
+ * names none of them is unusable.
130
+ */
131
+ function parseBootstrapToken(raw, tables, maxSeq) {
33
132
  if (raw === undefined)
34
133
  return undefined;
35
134
  try {
36
135
  const parsed = JSON.parse(raw);
37
- if (typeof parsed.asOfCommitSeq !== 'number' ||
136
+ if (!Number.isSafeInteger(parsed.asOfCommitSeq) ||
137
+ parsed.asOfCommitSeq === undefined ||
138
+ parsed.asOfCommitSeq < 0 ||
139
+ parsed.asOfCommitSeq > maxSeq ||
38
140
  !Array.isArray(parsed.tables) ||
39
- typeof parsed.tableIndex !== 'number' ||
40
- (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
41
- parsed.tables[0] !== table) {
141
+ parsed.tables.length !== tables.length ||
142
+ parsed.tables.some((name, i) => name !== tables[i]?.name) ||
143
+ !Number.isSafeInteger(parsed.tableIndex) ||
144
+ parsed.tableIndex === undefined ||
145
+ parsed.tableIndex < 0 ||
146
+ parsed.tableIndex >= tables.length ||
147
+ (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')) {
42
148
  return undefined;
43
149
  }
44
150
  return {
@@ -52,28 +158,6 @@ function parseBootstrapToken(raw, table) {
52
158
  return undefined;
53
159
  }
54
160
  }
55
- function commitFrame(table, commit, current, target) {
56
- const changes = commit.changes.map((change) => ({
57
- tableIndex: 0,
58
- rowId: change.rowId,
59
- op: change.op,
60
- ...(change.rowVersion !== undefined
61
- ? { rowVersion: change.rowVersion }
62
- : {}),
63
- scopes: change.scopes,
64
- ...(change.payload !== undefined
65
- ? { row: projectRowPayload(current, target, change.payload) }
66
- : {}),
67
- }));
68
- return {
69
- type: 'COMMIT',
70
- commitSeq: commit.commitSeq,
71
- createdAtMs: commit.createdAtMs,
72
- actorId: commit.actorId,
73
- tables: [table],
74
- changes,
75
- };
76
- }
77
161
  function chunkRows(rows, size) {
78
162
  if (rows.length === 0)
79
163
  return [];
@@ -103,10 +187,10 @@ function segmentRefFrame(record, scopeDigest, extra) {
103
187
  byteLength: record.byteLength,
104
188
  rowCount: record.rowCount,
105
189
  asOfCommitSeq: record.asOfCommitSeq,
106
- // The frame reports the digest THIS subscription is authorized under. A
107
- // stored entry can carry several digests when identical content was
108
- // published under different scopes (§5.1), and the record's primary
109
- // digest is not necessarily the caller's (§5.5).
190
+ // The frame reports the digest THIS pair is authorized under. A stored
191
+ // entry can carry several digests when identical content was published
192
+ // under different scopes (§5.1), and the record's primary digest is not
193
+ // necessarily the caller's (§5.5).
110
194
  scopeDigest,
111
195
  ...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
112
196
  ...(record.nextRowCursor !== null
@@ -126,20 +210,57 @@ function segmentRefFrame(record, scopeDigest, extra) {
126
210
  * wholesale with no per-row decrypt pass, so it MUST be served on the rows
127
211
  * lane (which decrypts per row on the client).
128
212
  */
129
- function sqliteLaneEligible(limits, plan, startRowCursor) {
213
+ function sqliteLaneEligible(limits, table, startRowCursor) {
130
214
  return ((limits.accept & ACCEPT_SQLITE) !== 0 &&
131
215
  startRowCursor === null &&
132
- plan.table.encryptedColumnIndices.length === 0);
216
+ table.encryptedColumnIndices.length === 0);
217
+ }
218
+ async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, maxSeq, horizonSeq, logEpoch) {
219
+ const scan = ctx.storage.scanRows(ctx.partition, {
220
+ table: pair.table.name,
221
+ scopeFilter: pair.effective,
222
+ afterRowId: startRowCursor,
223
+ limit: limits.limitSnapshotRows + 1,
224
+ });
225
+ if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
226
+ return { firstPage: await scan };
227
+ }
228
+ // §4.7: a sqlite-lane table pins at the newest change carrying the unit's
229
+ // index key (never below the horizon, never above maxCommitSeq), so a
230
+ // stored image stays the reuse key's match until this scope changes
231
+ // (§5.3). The server computes the pin; a client token never supplies it.
232
+ const [firstPage, latest] = await Promise.all([
233
+ scan,
234
+ ctx.storage.latestScopedChangeSeq(ctx.partition, {
235
+ table: pair.table.name,
236
+ scopes: unit.scopes,
237
+ }),
238
+ ]);
239
+ const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
240
+ // §5.3 eligibility: a stored image only matches a snapshot larger than one
241
+ // rows page, so the probe decides before the reuse lookup; a remote `find`
242
+ // on a small table would be a wasted round trip.
243
+ const stored = firstPage.length <= limits.limitSnapshotRows
244
+ ? undefined
245
+ : await ctx.segments.find({
246
+ partition: ctx.partition,
247
+ logEpoch,
248
+ table: pair.table.name,
249
+ schemaVersion: schema.version,
250
+ mediaType: 'sqlite',
251
+ scopeDigest: await scopeDigest(pair.effective),
252
+ asOfCommitSeq: scopePin,
253
+ }, clockOf(ctx)());
254
+ return { firstPage, image: { scopePin, stored } };
133
255
  }
134
256
  /**
135
- * §5.3 sqlite-image lane: only at the start of a table, only when the
136
- * client advertised bit 2, and only when the snapshot exceeds one rows
137
- * page. Reuses an unexpired stored image for the same (partition, table,
138
- * schemaVersion, scope digest, pin) instead of rebuilding — the
139
- * bootstrap-storm rule. Returns false when the table is not eligible
140
- * (the rows lane takes over).
257
+ * §5.3 sqlite-image lane: only when the snapshot exceeds one rows page.
258
+ * Reuses an unexpired stored image for the same (partition, table,
259
+ * schemaVersion, scope digest, scope pin) instead of rebuilding — the
260
+ * bootstrap-storm rule. Returns false when no image is served (the rows lane
261
+ * takes over).
141
262
  */
142
- async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trace, logEpoch, existing, probe) {
263
+ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest, trace, logEpoch, existing, probe) {
143
264
  const { storage, segments, partition } = ctx;
144
265
  const now = clockOf(ctx)();
145
266
  if (existing !== undefined) {
@@ -153,9 +274,6 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
153
274
  yield segmentRefFrame(existing, digest, await signedUrlFields(ctx, limits, existing.segmentId, digest, now));
154
275
  return true;
155
276
  }
156
- // Eligibility probe (§5.3): image only when the snapshot exceeds one
157
- // rows page — smaller tables stay on the (typically inline) rows lane.
158
- // The probe is the rows lane's first page, read by `prepareSections`.
159
277
  if (probe.length <= limits.limitSnapshotRows)
160
278
  return false;
161
279
  // §5.3: hosts opt into image construction with a runtime-specific builder.
@@ -176,12 +294,13 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
176
294
  const identity = {
177
295
  partition,
178
296
  logEpoch,
179
- table: plan.table.name,
297
+ table: pair.table.name,
180
298
  schemaVersion: schema.version,
181
299
  mediaType: 'sqlite',
182
300
  scopeDigest: digest,
183
- asOfCommitSeq: asOf,
301
+ asOfCommitSeq: scopePin,
184
302
  };
303
+ const current = compileSchema(ctx.schema).tables.get(pair.table.name);
185
304
  const key = JSON.stringify(identity);
186
305
  let building = builds.get(key);
187
306
  let builtHere = false;
@@ -194,28 +313,28 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
194
313
  builtHere = true;
195
314
  let rowCount = 0;
196
315
  const bytes = await buildImage({
197
- table: plan.table,
316
+ table: pair.table,
198
317
  schemaVersion: schema.version,
199
- asOfCommitSeq: asOf,
318
+ asOfCommitSeq: scopePin,
200
319
  scopeDigest: digest,
201
320
  rowBatches: (async function* () {
202
321
  rowCount += probe.length;
203
322
  yield probe.map((row) => ({
204
323
  ...row,
205
- payload: projectRowPayload(compileSchema(ctx.schema).tables.get(plan.table.name), plan.table, row.payload),
324
+ payload: projectRowPayload(current, pair.table, row.payload),
206
325
  }));
207
326
  let afterRowId = probe[probe.length - 1].rowId;
208
327
  for (;;) {
209
328
  const rows = await storage.scanRows(partition, {
210
- table: plan.table.name,
211
- scopeFilter: plan.effective,
329
+ table: pair.table.name,
330
+ scopeFilter: pair.effective,
212
331
  afterRowId,
213
332
  limit: 5_000,
214
333
  });
215
334
  rowCount += rows.length;
216
335
  yield rows.map((row) => ({
217
336
  ...row,
218
- payload: projectRowPayload(compileSchema(ctx.schema).tables.get(plan.table.name), plan.table, row.payload),
337
+ payload: projectRowPayload(current, pair.table, row.payload),
219
338
  }));
220
339
  const last = rows[rows.length - 1];
221
340
  if (rows.length < 5_000 || last === undefined)
@@ -246,23 +365,28 @@ async function* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trac
246
365
  yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, clockOf(ctx)()));
247
366
  return true;
248
367
  }
249
- async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCursor, trace, logEpoch, firstPage, image) {
368
+ /**
369
+ * One table's segments within the page budget. Returns the pages it used
370
+ * and the row cursor to resume at, `null` once the table is exhausted.
371
+ */
372
+ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, start, pageBudget, trace, logEpoch) {
250
373
  const { storage, segments, partition } = ctx;
251
374
  const serverLimits = limitsOf(ctx);
252
- const digest = await scopeDigest(plan.effective);
375
+ const digest = await scopeDigest(pair.effective);
253
376
  const now = clockOf(ctx)();
254
- if (sqliteLaneEligible(limits, plan, startRowCursor)) {
255
- const imaged = yield* sqliteImageSegment(ctx, schema, limits, plan, asOf, digest, trace, logEpoch, image, firstPage);
377
+ if (start.image !== undefined) {
378
+ const imaged = yield* sqliteImageSegment(ctx, schema, limits, pair, start.image.scopePin, digest, trace, logEpoch, start.image.stored, start.firstPage);
256
379
  if (imaged)
257
- return { complete: true, rowCursor: null };
380
+ return { pages: 1, rowCursor: null };
258
381
  }
382
+ const current = compileSchema(ctx.schema).tables.get(pair.table.name);
259
383
  let rowCursor = startRowCursor;
260
- for (let page = 0; page < limits.maxSnapshotPages; page++) {
384
+ for (let page = 0; page < pageBudget; page++) {
261
385
  const scanned = page === 0
262
- ? firstPage
386
+ ? start.firstPage
263
387
  : await storage.scanRows(partition, {
264
- table: plan.table.name,
265
- scopeFilter: plan.effective,
388
+ table: pair.table.name,
389
+ scopeFilter: pair.effective,
266
390
  afterRowId: rowCursor,
267
391
  limit: limits.limitSnapshotRows + 1,
268
392
  });
@@ -271,12 +395,12 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
271
395
  // §5.2: every row record carries the row's current server_version.
272
396
  const decoded = pageRows.map((row) => ({
273
397
  serverVersion: row.serverVersion,
274
- values: projectRowValues(compileSchema(ctx.schema).tables.get(plan.table.name), plan.table, row.payload),
398
+ values: projectRowValues(current, pair.table, row.payload),
275
399
  }));
276
400
  const bytes = encodeRowsSegment({
277
- table: plan.table.name,
401
+ table: pair.table.name,
278
402
  schemaVersion: schema.version,
279
- columns: plan.table.columns,
403
+ columns: pair.table.columns,
280
404
  blocks: chunkRows(decoded, 1000),
281
405
  });
282
406
  const lastRow = pageRows[pageRows.length - 1];
@@ -299,11 +423,11 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
299
423
  const record = await segments.put({
300
424
  partition,
301
425
  logEpoch,
302
- table: plan.table.name,
426
+ table: pair.table.name,
303
427
  schemaVersion: schema.version,
304
428
  mediaType: 'rows',
305
429
  scopeDigest: digest,
306
- asOfCommitSeq: asOf,
430
+ asOfCommitSeq: pin,
307
431
  rowCount: pageRows.length,
308
432
  rowCursor,
309
433
  nextRowCursor,
@@ -318,232 +442,84 @@ async function* bootstrapSegments(ctx, schema, limits, plan, asOf, startRowCurso
318
442
  yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
319
443
  }
320
444
  if (!hasMore)
321
- return { complete: true, rowCursor: null };
445
+ return { pages: page + 1, rowCursor: null };
322
446
  rowCursor = nextRowCursor;
323
447
  }
324
- return { complete: false, rowCursor };
325
- }
326
- /** Run the pre-frame storage reads of every subscription concurrently. */
327
- export async function prepareSections(ctx, schema, limits, plans, maxSeq, horizonSeq, logEpoch) {
328
- const reads = plans.map((plan) => {
329
- const sub = plan.frame;
330
- if (plan.status === 'revoked')
331
- return {};
332
- const token = parseBootstrapToken(sub.bootstrapState, sub.table);
333
- if (token === undefined &&
334
- sub.cursor >= 0 &&
335
- sub.cursor >= horizonSeq &&
336
- sub.cursor <= maxSeq) {
337
- return {
338
- commits: ctx.storage.readCommitWindow(ctx.partition, {
339
- table: sub.table,
340
- scopeFilter: plan.effective,
341
- afterSeq: sub.cursor,
342
- throughSeq: maxSeq,
343
- limitChanges: limits.limitCommits + 1,
344
- }),
345
- };
346
- }
347
- // §4.6 reset: no reads.
348
- if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
349
- return {};
350
- }
351
- // Bootstrapping: the same resume decision `preparedSection` makes (§4.7).
352
- const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
353
- ? token
354
- : undefined;
355
- const startCursor = resume?.rowCursor ?? null;
356
- const firstPage = ctx.storage.scanRows(ctx.partition, {
357
- table: plan.table.name,
358
- scopeFilter: plan.effective,
359
- afterRowId: startCursor,
360
- limit: limits.limitSnapshotRows + 1,
361
- });
362
- if (!sqliteLaneEligible(limits, plan, startCursor))
363
- return { firstPage };
364
- // §4.7: a fresh sqlite-lane bootstrap pins at the newest change in its
365
- // scope (never below the horizon, never above maxCommitSeq), so a stored
366
- // image stays the reuse key's match until this scope changes (§5.3).
367
- const pinned = resume !== undefined
368
- ? Promise.resolve({ asOf: resume.asOfCommitSeq, pin: undefined })
369
- : ctx.storage
370
- .latestScopedChangeSeq(ctx.partition, {
371
- table: plan.table.name,
372
- scopeFilter: plan.effective,
373
- })
374
- .then((latest) => {
375
- const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
376
- return { asOf: pin, pin };
377
- });
378
- const lane = pinned.then(async ({ asOf, pin }) => {
379
- // §5.3 eligibility: a stored image only matches a snapshot larger than
380
- // one rows page, so the probe decides before the reuse lookup; a remote
381
- // `find` on a small table would be a wasted round trip.
382
- const probe = await firstPage;
383
- if (probe.length <= limits.limitSnapshotRows) {
384
- return { pin, image: undefined };
385
- }
386
- return {
387
- pin,
388
- image: await ctx.segments.find({
389
- partition: ctx.partition,
390
- logEpoch,
391
- table: plan.table.name,
392
- schemaVersion: schema.version,
393
- mediaType: 'sqlite',
394
- scopeDigest: await scopeDigest(plan.effective),
395
- asOfCommitSeq: asOf,
396
- }, clockOf(ctx)()),
397
- };
398
- });
399
- return { firstPage, lane };
400
- });
401
- const settled = await Promise.all(reads.map(async (read) => ({
402
- commits: read.commits === undefined ? undefined : await read.commits,
403
- firstPage: await read.firstPage,
404
- lane: await read.lane,
405
- })));
406
- // Validate continuity before committing to an active section. A prune
407
- // during a paged read can otherwise make an incomplete window look empty.
408
- const windowHorizon = settled.some((read) => read.commits !== undefined)
409
- ? Math.max(horizonSeq, await ctx.storage.getHorizonSeq(ctx.partition))
410
- : horizonSeq;
411
- return settled.map((read) => ({
412
- commits: read.commits ?? [],
413
- horizonSeq: read.commits === undefined ? horizonSeq : windowHorizon,
414
- firstPage: read.firstPage,
415
- pin: read.lane?.pin,
416
- image: read.lane?.image,
417
- }));
448
+ return { pages: pageBudget, rowCursor };
418
449
  }
419
450
  /**
420
- * Produce the `SUB_START … SUB_END` section for one subscription (§1.6),
421
- * returning the cursor recorded for the retention watermark (§4.5).
451
+ * Plan one unit's bootstrap and start the reads of every table it can reach
452
+ * in this section: each visited table costs at least one page, so at most
453
+ * `maxSnapshotPages` tables. The reads of every planned unit start before any
454
+ * is awaited, so a storage that batches concurrent page reads (PostgreSQL,
455
+ * D1) answers them in one statement per table.
422
456
  */
423
- export async function* subscriptionSection(ctx, schema, limits, plan, maxSeq, horizonSeq, trace, logEpoch) {
424
- if (logEpoch === undefined || logEpoch.length === 0) {
425
- throw new Error('subscriptionSection requires a non-empty log epoch');
457
+ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq, horizonSeq, logEpoch) {
458
+ // §4.7: resume at the token's pin unless the token is unusable or the pin
459
+ // fell behind the horizon; then restart from a fresh pin.
460
+ const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
461
+ const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
462
+ ? token
463
+ : undefined;
464
+ const tableIndex = resume?.tableIndex ?? 0;
465
+ const rowCursor = resume?.rowCursor ?? null;
466
+ const starts = new Map();
467
+ for (let index = tableIndex; index < unit.tables.length && starts.size < limits.maxSnapshotPages; index++) {
468
+ const table = unit.tables[index];
469
+ const effective = unit.pairs.get(table.name);
470
+ if (effective === undefined)
471
+ continue;
472
+ const start = readTableStart(ctx, schema, limits, unit, { table, effective }, index === tableIndex ? rowCursor : null, maxSeq, horizonSeq, logEpoch);
473
+ // A section that stops early never awaits the later starts.
474
+ start.catch(() => undefined);
475
+ starts.set(index, start);
426
476
  }
427
- const [prepared] = await prepareSections(ctx, schema, limits, [plan], maxSeq, horizonSeq, logEpoch);
428
- if (prepared === undefined)
429
- throw new Error('prepareSections lost a plan');
430
- return yield* preparedSection(ctx, schema, limits, plan, maxSeq, prepared, trace, logEpoch);
477
+ return {
478
+ unit,
479
+ pin: resume?.asOfCommitSeq ?? maxSeq,
480
+ tableIndex,
481
+ rowCursor,
482
+ starts,
483
+ };
431
484
  }
432
- /** The section of one subscription whose reads `prepareSections` ran. */
433
- export async function* preparedSection(ctx, schema, limits, plan, maxSeq, prepared, trace, logEpoch) {
434
- const sub = plan.frame;
435
- if (logEpoch === undefined || logEpoch.length === 0) {
436
- throw new Error('subscriptionSection requires a non-empty log epoch');
437
- }
438
- if (plan.status === 'revoked') {
439
- yield {
440
- type: 'SUB_START',
441
- id: sub.id,
442
- status: 'revoked',
443
- reasonCode: 'sync.scope_revoked',
444
- effectiveScopes: {},
445
- bootstrap: false,
446
- };
447
- yield { type: 'SUB_END', nextCursor: sub.cursor };
448
- return { nextCursor: sub.cursor, active: false };
449
- }
450
- const token = parseBootstrapToken(sub.bootstrapState, sub.table);
451
- const { commits, horizonSeq } = prepared;
452
- // §4.6: a cursor behind the horizon (and not resuming a bootstrap)
453
- // cannot compute deltas — answer `reset` and echo the cursor.
454
- if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
455
- yield {
456
- type: 'SUB_START',
457
- id: sub.id,
458
- status: 'reset',
459
- reasonCode: 'sync.cursor_expired',
460
- effectiveScopes: {},
461
- bootstrap: false,
462
- };
463
- yield { type: 'SUB_END', nextCursor: sub.cursor };
464
- return { nextCursor: sub.cursor, active: false };
465
- }
466
- const bootstrapping = token !== undefined || sub.cursor < 0 || sub.cursor > maxSeq;
467
- if (bootstrapping) {
468
- // §4.7: resume at the pinned point unless the pin fell behind the
469
- // horizon (or the token is unusable) — then restart from a fresh pin.
470
- const resume = token !== undefined && token.asOfCommitSeq >= horizonSeq
471
- ? token
472
- : undefined;
473
- const asOf = resume?.asOfCommitSeq ?? prepared.pin ?? maxSeq;
474
- const startCursor = resume?.rowCursor ?? null;
475
- if (prepared.firstPage === undefined) {
476
- throw new Error('prepareSections did not read the bootstrap page');
477
- }
478
- yield {
479
- type: 'SUB_START',
480
- id: sub.id,
481
- status: 'active',
482
- reasonCode: '',
483
- effectiveScopes: plan.effective,
484
- bootstrap: true,
485
- };
486
- const outcome = yield* bootstrapSegments(ctx, schema, limits, plan, asOf, startCursor, trace, logEpoch, prepared.firstPage, prepared.image);
487
- if (outcome.complete) {
488
- yield { type: 'SUB_END', nextCursor: asOf };
485
+ /**
486
+ * The segments of one bootstrap section (§4.7): tables in bootstrap order,
487
+ * each active pair's rows or image, up to `maxSnapshotPages` pages. Returns
488
+ * the resume token, or `undefined` once every table is exhausted. A pair the
489
+ * wildcard fence revoked receives no segment (§3.3).
490
+ */
491
+ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch) {
492
+ const { unit, pin } = plan;
493
+ let { tableIndex, rowCursor } = plan;
494
+ let pages = 0;
495
+ for (;;) {
496
+ while (tableIndex < unit.tables.length &&
497
+ !unit.pairs.has(unit.tables[tableIndex].name)) {
498
+ tableIndex += 1;
499
+ rowCursor = null;
489
500
  }
490
- else {
491
- const nextToken = {
492
- asOfCommitSeq: asOf,
493
- tables: [sub.table],
494
- tableIndex: 0,
495
- rowCursor: outcome.rowCursor,
496
- };
497
- yield {
498
- type: 'SUB_END',
499
- nextCursor: asOf,
500
- bootstrapState: JSON.stringify(nextToken),
501
+ if (tableIndex === unit.tables.length)
502
+ return undefined;
503
+ if (pages === limits.maxSnapshotPages) {
504
+ const token = {
505
+ asOfCommitSeq: pin,
506
+ tables: unit.tables.map((table) => table.name),
507
+ tableIndex,
508
+ rowCursor,
501
509
  };
510
+ return JSON.stringify(token);
502
511
  }
503
- return { nextCursor: asOf, active: true };
512
+ const table = unit.tables[tableIndex];
513
+ const outcome = yield* tableSegments(ctx, schema, limits, { table, effective: unit.pairs.get(table.name) }, pin, rowCursor, await plan.starts.get(tableIndex), limits.maxSnapshotPages - pages, trace, logEpoch);
514
+ pages += outcome.pages;
515
+ rowCursor = outcome.rowCursor;
516
+ if (rowCursor === null)
517
+ tableIndex += 1;
504
518
  }
505
- // Incremental (§4.5): window cursor < commitSeq <= maxSeq, oldest first,
506
- // cut off at limitCommits total changes, never splitting a commit.
507
- yield {
508
- type: 'SUB_START',
509
- id: sub.id,
510
- status: 'active',
511
- reasonCode: '',
512
- effectiveScopes: plan.effective,
513
- bootstrap: false,
514
- };
515
- let delivered = 0;
516
- let deliveredCommits = 0;
517
- let lastDeliveredSeq = sub.cursor;
518
- for (const commit of commits) {
519
- if (delivered > 0 &&
520
- delivered + commit.changes.length > limits.limitCommits) {
521
- break;
522
- }
523
- yield commitFrame(sub.table, commit, compileSchema(ctx.schema).tables.get(sub.table), plan.table);
524
- delivered += commit.changes.length;
525
- deliveredCommits += 1;
526
- lastDeliveredSeq = commit.commitSeq;
527
- if (delivered >= limits.limitCommits)
528
- break;
529
- }
530
- const totalReturned = commits.reduce((n, c) => n + c.changes.length, 0);
531
- // The window is proven exhausted only when the storage lookahead came
532
- // back under budget (it scanned to `throughSeq`) and every returned
533
- // commit was delivered.
534
- const exhausted = totalReturned <= limits.limitCommits && deliveredCommits === commits.length;
535
- // §4.5: the cursor advances even when no matching changes exist; when
536
- // the change limit truncated the window it stops at the last fully
537
- // delivered commit.
538
- const nextCursor = exhausted
539
- ? Math.max(sub.cursor, maxSeq)
540
- : Math.max(sub.cursor, lastDeliveredSeq);
541
- yield { type: 'SUB_END', nextCursor };
542
- return { nextCursor, active: true };
543
519
  }
544
520
  /**
545
- * Publish the §5.3 sqlite image that a pull for (`table`, `scopes`) looks
546
- * up, from a host with a SQLite engine (a Bun process with the production
521
+ * Publish the §5.3 sqlite image that a pull of (`scopes`, `table`) looks up,
522
+ * from a host with a SQLite engine (a Bun process with the production
547
523
  * storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
548
524
  * reuse key, the build), so the stored identity is exactly the one a
549
525
  * serving host without a builder finds. The image stays current until a
@@ -556,9 +532,16 @@ export async function publishSqliteImage(input) {
556
532
  throw new Error('publishSqliteImage requires config.sqliteImageBuilder');
557
533
  }
558
534
  const schema = compileSchema(config.schema);
559
- const table = schema.tables.get(input.table);
535
+ const table = coveredTables(schema, input.scopes).find((candidate) => candidate.name === input.table);
560
536
  if (table === undefined) {
561
- throw new Error('publishSqliteImage: the schema has no such table');
537
+ throw new Error('publishSqliteImage: the unit does not cover the table');
538
+ }
539
+ const allowed = input.allowed ??
540
+ Object.fromEntries([...schema.declaredVariables].map((v) => [v, ['*']]));
541
+ const unit = planUnit(input.scopes, [table], { ok: true, allowed });
542
+ const effective = unit.pairs.get(table.name);
543
+ if (effective === undefined) {
544
+ throw new Error('publishSqliteImage: the allowed scopes revoke the pair');
562
545
  }
563
546
  await config.storage.ensureSchema(schema, config.checkpoints);
564
547
  const logEpoch = await config.storage.getPartitionLogEpoch(partition);
@@ -570,18 +553,6 @@ export async function publishSqliteImage(input) {
570
553
  partition,
571
554
  actorId: 'syncular.image-publisher',
572
555
  };
573
- const plan = {
574
- frame: {
575
- type: 'SUBSCRIPTION',
576
- id: 'image',
577
- table: input.table,
578
- scopes: input.scopes,
579
- cursor: -1,
580
- },
581
- table,
582
- status: 'active',
583
- effective: input.scopes,
584
- };
585
556
  // One-row pages make every table above one row image-eligible (§5.3).
586
557
  const limits = {
587
558
  limitCommits: 1,
@@ -592,11 +563,9 @@ export async function publishSqliteImage(input) {
592
563
  const maxSeq = await config.storage.getMaxCommitSeq(partition);
593
564
  const horizonSeq = await config.storage.getHorizonSeq(partition);
594
565
  const trace = { segments: [] };
595
- const [prepared] = await prepareSections(ctx, schema, limits, [plan], maxSeq, horizonSeq, logEpoch);
596
- if (prepared === undefined)
597
- throw new Error('prepareSections lost a plan');
566
+ const plan = planBootstrap(ctx, schema, limits, unit, undefined, maxSeq, horizonSeq, logEpoch);
598
567
  let ref;
599
- for await (const frame of preparedSection(ctx, schema, limits, plan, maxSeq, prepared, trace, logEpoch)) {
568
+ for await (const frame of bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch)) {
600
569
  if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
601
570
  ref = frame;
602
571
  }