@syncular/server 0.75.0 → 0.76.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -23,10 +23,11 @@
23
23
  * objects.
24
24
  *
25
25
  * {keyPrefix}find/{sha256Hex(canonical reuse key)}.json
26
- * Whole-table reuse pointer (§5.3): written only when
27
- * `rowCursor === null`, body = the record JSON. The reuse key is the
26
+ * Image reuse pointer (§5.3): written for every sqlite image and every
27
+ * first rows page, body = the record JSON. The reuse key is the
28
28
  * canonical JSON array `[partition, logEpoch, table, schemaVersion,
29
- * mediaType, scopeDigest, asOfCommitSeq]`, so `find` is one GET (plus a
29
+ * mediaType, scopeDigest, asOfCommitSeq]` plus the range start when it is
30
+ * not null, so `find` is one GET (plus a
30
31
  * HEAD to confirm the segment object itself still exists — lifecycle GC
31
32
  * may remove objects independently of pointers).
32
33
  *
@@ -214,6 +214,8 @@ export class BucketSegmentStore {
214
214
  key.mediaType,
215
215
  key.scopeDigest,
216
216
  key.asOfCommitSeq,
217
+ // A first range keeps the pointer key it had before range images.
218
+ ...(key.rowCursor === null ? [] : [key.rowCursor]),
217
219
  ]);
218
220
  return `${this.#prefix}find/${await sha256Hex(canonical)}.json`;
219
221
  }
@@ -300,7 +302,7 @@ export class BucketSegmentStore {
300
302
  sqliteSegments: metadata.mediaType === 'sqlite' ? 1 : 0,
301
303
  });
302
304
  }
303
- if (metadata.rowCursor === null) {
305
+ if (metadata.rowCursor === null || metadata.mediaType === 'sqlite') {
304
306
  await this.#bucket.put(await this.#findKeyFor(metadata), {
305
307
  body: new TextEncoder().encode(recordJson),
306
308
  contentType: 'application/json',
@@ -415,7 +417,7 @@ export class BucketSegmentStore {
415
417
  mediaType: key.mediaType,
416
418
  scopeDigest: key.scopeDigest,
417
419
  asOfCommitSeq: key.asOfCommitSeq,
418
- rowCursor: null,
420
+ rowCursor: key.rowCursor,
419
421
  nowMs,
420
422
  });
421
423
  if (publication === undefined)
package/dist/context.d.ts CHANGED
@@ -86,6 +86,12 @@ export interface ServerLimits {
86
86
  * pass it alone resets the units it reaches, which bootstrap instead.
87
87
  */
88
88
  readonly maxLiveSectionBytes: number;
89
+ /**
90
+ * §5.3 bound on one sqlite image's file size: the server cuts a table into
91
+ * range images in row order, each within this bound. A row larger than the
92
+ * bound gets an image of its own.
93
+ */
94
+ readonly maxImageSegmentBytes: number;
89
95
  }
90
96
  export declare const DEFAULT_LIMITS: ServerLimits;
91
97
  /** Receives applied commits for realtime fanout (§8.2). */
package/dist/context.js CHANGED
@@ -23,6 +23,8 @@ export const DEFAULT_LIMITS = {
23
23
  maxWatchesPerSession: 100,
24
24
  // The live section is buffered whole before the first unit section.
25
25
  maxLiveSectionBytes: 16 * 1024 * 1024,
26
+ // §5.3 DECISION (2026-11-17): a client holds about three images at once.
27
+ maxImageSegmentBytes: 32 * 1024 * 1024,
26
28
  };
27
29
  /** §1.1: `authenticate` returned a context longer than 256 UTF-8 bytes. */
28
30
  export class AuthContextTooLargeError extends Error {
package/dist/index.d.ts CHANGED
@@ -50,7 +50,7 @@ export * from './segment-store.js';
50
50
  export * from './signed-url.js';
51
51
  export * from './sqlite-dialect.js';
52
52
  export * from './sqlite-driver.js';
53
- export { IMAGE_METADATA_TABLE, IMAGE_VERSION_COLUMN, type SqliteImageBuilder, type SqliteImageInput, writeSqliteImage, } from './sqlite-image.js';
53
+ export { IMAGE_METADATA_TABLE, IMAGE_VERSION_COLUMN, type SqliteImage, type SqliteImageBuilder, type SqliteImageInput, writeSqliteImage, } from './sqlite-image.js';
54
54
  export * from './sqlite-blob-store.js';
55
55
  export * from './sqlite-lease-store.js';
56
56
  export * from './sqlite-segment-store.js';
package/dist/pull.d.ts CHANGED
@@ -74,6 +74,8 @@ export interface BootstrapPlan {
74
74
  readonly pin: number;
75
75
  readonly tableIndex: number;
76
76
  readonly rowCursor: string | null;
77
+ /** Whether the position continues the table's range images (§5.3). */
78
+ readonly image: boolean;
77
79
  /** Table index → its start reads. */
78
80
  readonly starts: ReadonlyMap<number, Promise<TableStart>>;
79
81
  }
@@ -108,17 +110,20 @@ export interface PublishSqliteImageInput {
108
110
  readonly allowed?: ScopeMap;
109
111
  }
110
112
  export interface PublishedSqliteImage {
111
- readonly segmentId: string;
112
- /** The §4.7 scope pin the image is stored under. */
113
+ /** The table's range images in row order (§5.3). */
114
+ readonly segmentIds: readonly string[];
115
+ /** The §4.7 scope pin the images are stored under. */
113
116
  readonly asOfCommitSeq: number;
114
117
  readonly scopeDigest: string;
118
+ /** Rows across every range image. */
115
119
  readonly rowCount: number;
120
+ /** Bytes across every range image. */
116
121
  readonly byteLength: number;
117
- /** `reused` when an unexpired image for the same reuse key already existed. */
122
+ /** `reused` when an unexpired image existed for every range's reuse key. */
118
123
  readonly origin: 'built' | 'reused';
119
124
  }
120
125
  /**
121
- * Publish the §5.3 sqlite image that a pull of (`scopes`, `table`) looks up,
126
+ * Publish the §5.3 range images that a pull of (`scopes`, `table`) looks up,
122
127
  * from a host with a SQLite engine (a Bun process with the production
123
128
  * storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
124
129
  * reuse key, the build), so the stored identity is exactly the one a
package/dist/pull.js CHANGED
@@ -144,7 +144,9 @@ function parseBootstrapToken(raw, tables, maxSeq) {
144
144
  parsed.tableIndex === undefined ||
145
145
  parsed.tableIndex < 0 ||
146
146
  parsed.tableIndex >= tables.length ||
147
- (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')) {
147
+ (parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
148
+ (parsed.image !== undefined &&
149
+ (parsed.image !== true || parsed.rowCursor === null))) {
148
150
  return undefined;
149
151
  }
150
152
  return {
@@ -152,6 +154,7 @@ function parseBootstrapToken(raw, tables, maxSeq) {
152
154
  tables: parsed.tables,
153
155
  tableIndex: parsed.tableIndex,
154
156
  rowCursor: parsed.rowCursor ?? null,
157
+ ...(parsed.image === true ? { image: true } : {}),
155
158
  };
156
159
  }
157
160
  catch {
@@ -193,8 +196,12 @@ function segmentRefFrame(record, scopeDigest, extra) {
193
196
  // under different scopes (§5.1), and the record's primary digest is not
194
197
  // necessarily the caller's (§5.5).
195
198
  scopeDigest,
196
- ...(record.rowCursor !== null ? { rowCursor: record.rowCursor } : {}),
197
- ...(record.nextRowCursor !== null
199
+ // §5.3: an image descriptor carries no cursors; the resume token holds
200
+ // the position of a table's next range.
201
+ ...(record.mediaType === 'rows' && record.rowCursor !== null
202
+ ? { rowCursor: record.rowCursor }
203
+ : {}),
204
+ ...(record.mediaType === 'rows' && record.nextRowCursor !== null
198
205
  ? { nextRowCursor: record.nextRowCursor }
199
206
  : {}),
200
207
  ...(extra.url !== undefined ? { url: extra.url } : {}),
@@ -204,26 +211,51 @@ function segmentRefFrame(record, scopeDigest, extra) {
204
211
  };
205
212
  }
206
213
  /**
207
- * §5.3: the sqlite-image lane — whole-table, chosen only at the start of a
208
- * table (a mid-table resume stays on the rows lane, never switching lanes),
214
+ * §5.3: the sqlite-image lane, chosen at the start of a table or continued
215
+ * after a range image (a table on the rows lane never switches to images),
209
216
  * and only when the client advertised bit 2. §5.11: a table with any
210
217
  * encrypted column is image-INELIGIBLE — an image copies ciphertext
211
218
  * wholesale with no per-row decrypt pass, so it MUST be served on the rows
212
219
  * lane (which decrypts per row on the client).
213
220
  */
214
- function sqliteLaneEligible(limits, table, startRowCursor) {
221
+ function sqliteLaneEligible(limits, table, startRowCursor, imageLane) {
215
222
  return ((limits.accept & ACCEPT_SQLITE) !== 0 &&
216
- startRowCursor === null &&
223
+ (startRowCursor === null || imageLane) &&
217
224
  table.encryptedColumnIndices.length === 0);
218
225
  }
219
- async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, maxSeq, horizonSeq, logEpoch) {
226
+ /**
227
+ * §5.3 reuse lookup for the range image that starts after `rowCursor`. Only
228
+ * a position with more than one rows page left can match a stored image, so
229
+ * a smaller remainder skips the lookup. A stored image past the bound (built
230
+ * under a larger one) does not match unless it holds a single row.
231
+ */
232
+ async function findImage(ctx, schema, limits, pair, scopePin, rowCursor, probe, logEpoch) {
233
+ if (probe.length <= limits.limitSnapshotRows)
234
+ return undefined;
235
+ const stored = await ctx.segments.find({
236
+ partition: ctx.partition,
237
+ logEpoch,
238
+ table: pair.table.name,
239
+ schemaVersion: schema.version,
240
+ mediaType: 'sqlite',
241
+ scopeDigest: await scopeDigest(pair.effective),
242
+ asOfCommitSeq: scopePin,
243
+ rowCursor,
244
+ }, clockOf(ctx)());
245
+ return stored !== undefined &&
246
+ (stored.byteLength <= limitsOf(ctx).maxImageSegmentBytes ||
247
+ stored.rowCount === 1)
248
+ ? stored
249
+ : undefined;
250
+ }
251
+ async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, imageLane, maxSeq, horizonSeq, logEpoch) {
220
252
  const scan = ctx.storage.scanRows(ctx.partition, {
221
253
  table: pair.table.name,
222
254
  scopeFilter: pair.effective,
223
255
  afterRowId: startRowCursor,
224
256
  limit: limits.limitSnapshotRows + 1,
225
257
  });
226
- if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
258
+ if (!sqliteLaneEligible(limits, pair.table, startRowCursor, imageLane)) {
227
259
  return { firstPage: await scan };
228
260
  }
229
261
  // §4.7: a sqlite-lane table pins at the newest commit carrying all of the
@@ -238,30 +270,17 @@ async function readTableStart(ctx, schema, limits, unit, pair, startRowCursor, m
238
270
  }),
239
271
  ]);
240
272
  const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
241
- // §5.3 eligibility: a stored image only matches a snapshot larger than one
242
- // rows page, so the probe decides before the reuse lookup; a remote `find`
243
- // on a small table would be a wasted round trip.
244
- const stored = firstPage.length <= limits.limitSnapshotRows
245
- ? undefined
246
- : await ctx.segments.find({
247
- partition: ctx.partition,
248
- logEpoch,
249
- table: pair.table.name,
250
- schemaVersion: schema.version,
251
- mediaType: 'sqlite',
252
- scopeDigest: await scopeDigest(pair.effective),
253
- asOfCommitSeq: scopePin,
254
- }, clockOf(ctx)());
273
+ const stored = await findImage(ctx, schema, limits, pair, scopePin, startRowCursor, firstPage, logEpoch);
255
274
  return { firstPage, image: { scopePin, stored } };
256
275
  }
257
276
  /**
258
- * §5.3 sqlite-image lane: only when the snapshot exceeds one rows page.
259
- * Reuses an unexpired stored image for the same (partition, table,
260
- * schemaVersion, scope digest, scope pin) instead of rebuilding — the
261
- * bootstrap-storm rule. Returns false when no image is served (the rows lane
262
- * takes over).
277
+ * §5.3 sqlite-image lane: one range image starting after `rowCursor`, only
278
+ * when more than one rows page remains there. Reuses an unexpired stored
279
+ * image for the same (partition, table, schemaVersion, scope digest, scope
280
+ * pin, range start) instead of rebuilding — the bootstrap-storm rule.
281
+ * Returns undefined when no image is served (the rows lane takes over).
263
282
  */
264
- async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest, trace, logEpoch, existing, probe) {
283
+ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, rowCursor, digest, trace, logEpoch, existing, probe) {
265
284
  const { storage, segments, partition } = ctx;
266
285
  const now = clockOf(ctx)();
267
286
  if (existing !== undefined) {
@@ -273,15 +292,15 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
273
292
  rows: existing.rowCount,
274
293
  });
275
294
  yield segmentRefFrame(existing, digest, await signedUrlFields(ctx, limits, existing.segmentId, digest, now));
276
- return true;
295
+ return existing;
277
296
  }
278
297
  if (probe.length <= limits.limitSnapshotRows)
279
- return false;
298
+ return undefined;
280
299
  // §5.3: hosts opt into image construction with a runtime-specific builder.
281
300
  // Without one, matching stored images remain usable and cold pulls use rows.
282
301
  const buildImage = ctx.sqliteImageBuilder;
283
302
  if (buildImage === undefined)
284
- return false;
303
+ return undefined;
285
304
  let stores = imageBuilds.get(storage);
286
305
  if (stores === undefined) {
287
306
  stores = new WeakMap();
@@ -300,6 +319,7 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
300
319
  mediaType: 'sqlite',
301
320
  scopeDigest: digest,
302
321
  asOfCommitSeq: scopePin,
322
+ rowCursor,
303
323
  };
304
324
  const current = compileSchema(ctx.schema).tables.get(pair.table.name);
305
325
  const key = JSON.stringify(identity);
@@ -308,43 +328,55 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
308
328
  if (building === undefined) {
309
329
  building = (async () => {
310
330
  // Another request can finish while this one's eligibility probe awaits.
311
- const cached = await segments.find(identity, clockOf(ctx)());
331
+ const cached = await findImage(ctx, schema, limits, pair, scopePin, rowCursor, probe, logEpoch);
312
332
  if (cached !== undefined)
313
333
  return cached;
314
334
  builtHere = true;
315
- let rowCount = 0;
316
- const bytes = await buildImage({
335
+ // The builder stops at the size bound inside the latest batch; these
336
+ // locate the last row it took and whether rows follow it.
337
+ let yielded = 0;
338
+ let batch = [];
339
+ let beforeBatch = rowCursor;
340
+ let exhausted = false;
341
+ const image = await buildImage({
317
342
  table: pair.table,
318
343
  schemaVersion: schema.version,
319
344
  asOfCommitSeq: scopePin,
320
345
  scopeDigest: digest,
346
+ maxBytes: limitsOf(ctx).maxImageSegmentBytes,
321
347
  rowBatches: (async function* () {
322
- rowCount += probe.length;
323
- yield probe.map((row) => ({
324
- ...row,
325
- payload: projectRowPayload(current, pair.table, row.payload),
326
- }));
327
- let afterRowId = probe[probe.length - 1].rowId;
348
+ let rows = probe;
328
349
  for (;;) {
329
- const rows = await storage.scanRows(partition, {
330
- table: pair.table.name,
331
- scopeFilter: pair.effective,
332
- afterRowId,
333
- limit: 5_000,
334
- });
335
- rowCount += rows.length;
350
+ beforeBatch = batch[batch.length - 1]?.rowId ?? beforeBatch;
351
+ batch = rows;
352
+ yielded += rows.length;
336
353
  yield rows.map((row) => ({
337
354
  ...row,
338
355
  payload: projectRowPayload(current, pair.table, row.payload),
339
356
  }));
340
357
  const last = rows[rows.length - 1];
341
- if (rows.length < 5_000 || last === undefined)
358
+ if (last === undefined || (rows !== probe && rows.length < 5_000))
342
359
  break;
343
- afterRowId = last.rowId;
360
+ rows = await storage.scanRows(partition, {
361
+ table: pair.table.name,
362
+ scopeFilter: pair.effective,
363
+ afterRowId: last.rowId,
364
+ limit: 5_000,
365
+ });
344
366
  }
367
+ exhausted = true;
345
368
  })(),
346
369
  });
347
- return segments.put({ ...identity, rowCount, rowCursor: null, nextRowCursor: null }, bytes, clockOf(ctx)());
370
+ const index = image.rowCount - (yielded - batch.length) - 1;
371
+ if (image.rowCount < 1 || image.rowCount > yielded || index < -1) {
372
+ throw new Error('sqliteImageBuilder consumed rows out of order');
373
+ }
374
+ const last = index < 0 ? beforeBatch : batch[index].rowId;
375
+ return segments.put({
376
+ ...identity,
377
+ rowCount: image.rowCount,
378
+ nextRowCursor: image.rowCount < yielded || !exhausted ? last : null,
379
+ }, image.bytes, clockOf(ctx)());
348
380
  })();
349
381
  builds.set(key, building);
350
382
  }
@@ -364,24 +396,22 @@ async function* sqliteImageSegment(ctx, schema, limits, pair, scopePin, digest,
364
396
  rows: record.rowCount,
365
397
  });
366
398
  yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, clockOf(ctx)()));
367
- return true;
399
+ return record;
368
400
  }
369
401
  /**
370
- * One table's segments within the page budget. Returns the pages it used
371
- * and the row cursor to resume at, `null` once the table is exhausted.
402
+ * One table's segments within the page budget: range images while the table
403
+ * is on the image lane, then rows pages. Returns the pages it used, the row
404
+ * cursor to resume at (`null` once the table is exhausted) and whether that
405
+ * position is on the image lane.
372
406
  */
373
407
  async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, start, pageBudget, trace, logEpoch) {
374
408
  const { storage, segments, partition } = ctx;
375
409
  const serverLimits = limitsOf(ctx);
376
410
  const digest = await scopeDigest(pair.effective);
377
411
  const now = clockOf(ctx)();
378
- if (start.image !== undefined) {
379
- const imaged = yield* sqliteImageSegment(ctx, schema, limits, pair, start.image.scopePin, digest, trace, logEpoch, start.image.stored, start.firstPage);
380
- if (imaged)
381
- return { pages: 1, rowCursor: null };
382
- }
383
412
  const current = compileSchema(ctx.schema).tables.get(pair.table.name);
384
413
  let rowCursor = startRowCursor;
414
+ let image = start.image;
385
415
  for (let page = 0; page < pageBudget; page++) {
386
416
  const scanned = page === 0
387
417
  ? start.firstPage
@@ -391,6 +421,20 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
391
421
  afterRowId: rowCursor,
392
422
  limit: limits.limitSnapshotRows + 1,
393
423
  });
424
+ if (image !== undefined) {
425
+ // §5.3: every range of the table shares the scope pin computed at this
426
+ // section's table start.
427
+ const record = yield* sqliteImageSegment(ctx, schema, limits, pair, image.scopePin, rowCursor, digest, trace, logEpoch, page === 0
428
+ ? image.stored
429
+ : await findImage(ctx, schema, limits, pair, image.scopePin, rowCursor, scanned, logEpoch), scanned);
430
+ if (record !== undefined) {
431
+ if (record.nextRowCursor === null)
432
+ return { pages: page + 1, rowCursor: null, image: false };
433
+ rowCursor = record.nextRowCursor;
434
+ continue;
435
+ }
436
+ image = undefined;
437
+ }
394
438
  const pageRows = scanned.slice(0, limits.limitSnapshotRows);
395
439
  const hasMore = scanned.length > limits.limitSnapshotRows;
396
440
  // §5.2: every row record carries the row's current server_version.
@@ -443,10 +487,10 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
443
487
  yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
444
488
  }
445
489
  if (!hasMore)
446
- return { pages: page + 1, rowCursor: null };
490
+ return { pages: page + 1, rowCursor: null, image: false };
447
491
  rowCursor = nextRowCursor;
448
492
  }
449
- return { pages: pageBudget, rowCursor };
493
+ return { pages: pageBudget, rowCursor, image: image !== undefined };
450
494
  }
451
495
  /**
452
496
  * Plan one unit's bootstrap and start the reads of every table it can reach
@@ -464,13 +508,14 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
464
508
  : undefined;
465
509
  const tableIndex = resume?.tableIndex ?? 0;
466
510
  const rowCursor = resume?.rowCursor ?? null;
511
+ const image = resume?.image === true;
467
512
  const starts = new Map();
468
513
  for (let index = tableIndex; index < unit.tables.length && starts.size < limits.maxSnapshotPages; index++) {
469
514
  const table = unit.tables[index];
470
515
  const effective = unit.pairs.get(table.name);
471
516
  if (effective === undefined)
472
517
  continue;
473
- const start = readTableStart(ctx, schema, limits, unit, { table, effective }, index === tableIndex ? rowCursor : null, maxSeq, horizonSeq, logEpoch);
518
+ const start = readTableStart(ctx, schema, limits, unit, { table, effective }, index === tableIndex ? rowCursor : null, index === tableIndex && image, maxSeq, horizonSeq, logEpoch);
474
519
  // A section that stops early never awaits the later starts.
475
520
  start.catch(() => undefined);
476
521
  starts.set(index, start);
@@ -480,6 +525,7 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
480
525
  pin: resume?.asOfCommitSeq ?? maxSeq,
481
526
  tableIndex,
482
527
  rowCursor,
528
+ image,
483
529
  starts,
484
530
  };
485
531
  }
@@ -491,13 +537,14 @@ export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq,
491
537
  */
492
538
  export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch) {
493
539
  const { unit, pin } = plan;
494
- let { tableIndex, rowCursor } = plan;
540
+ let { tableIndex, rowCursor, image } = plan;
495
541
  let pages = 0;
496
542
  for (;;) {
497
543
  while (tableIndex < unit.tables.length &&
498
544
  !unit.pairs.has(unit.tables[tableIndex].name)) {
499
545
  tableIndex += 1;
500
546
  rowCursor = null;
547
+ image = false;
501
548
  }
502
549
  if (tableIndex === unit.tables.length)
503
550
  return undefined;
@@ -507,6 +554,7 @@ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEp
507
554
  tables: unit.tables.map((table) => table.name),
508
555
  tableIndex,
509
556
  rowCursor,
557
+ ...(image ? { image: true } : {}),
510
558
  };
511
559
  return JSON.stringify(token);
512
560
  }
@@ -514,12 +562,13 @@ export async function* bootstrapSegments(ctx, schema, limits, plan, trace, logEp
514
562
  const outcome = yield* tableSegments(ctx, schema, limits, { table, effective: unit.pairs.get(table.name) }, pin, rowCursor, await plan.starts.get(tableIndex), limits.maxSnapshotPages - pages, trace, logEpoch);
515
563
  pages += outcome.pages;
516
564
  rowCursor = outcome.rowCursor;
565
+ image = outcome.image;
517
566
  if (rowCursor === null)
518
567
  tableIndex += 1;
519
568
  }
520
569
  }
521
570
  /**
522
- * Publish the §5.3 sqlite image that a pull of (`scopes`, `table`) looks up,
571
+ * Publish the §5.3 range images that a pull of (`scopes`, `table`) looks up,
523
572
  * from a host with a SQLite engine (a Bun process with the production
524
573
  * storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
525
574
  * reuse key, the build), so the stored identity is exactly the one a
@@ -554,33 +603,36 @@ export async function publishSqliteImage(input) {
554
603
  partition,
555
604
  actorId: 'syncular.image-publisher',
556
605
  };
557
- // One-row pages make every table above one row image-eligible (§5.3).
606
+ // One-row pages make every range above one row image-eligible (§5.3), and
607
+ // an unbounded page budget stores every range under one scope pin.
558
608
  const limits = {
559
609
  limitCommits: 1,
560
610
  limitSnapshotRows: 1,
561
- maxSnapshotPages: 1,
611
+ maxSnapshotPages: Number.MAX_SAFE_INTEGER,
562
612
  accept: ACCEPT_SQLITE,
563
613
  };
564
614
  const maxSeq = await config.storage.getMaxCommitSeq(partition);
565
615
  const horizonSeq = await config.storage.getHorizonSeq(partition);
566
616
  const trace = { segments: [] };
567
617
  const plan = planBootstrap(ctx, schema, limits, unit, undefined, maxSeq, horizonSeq, logEpoch);
568
- let ref;
618
+ const refs = [];
569
619
  for await (const frame of bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch)) {
570
620
  if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
571
- ref = frame;
621
+ refs.push(frame);
572
622
  }
573
623
  }
574
- const image = trace.segments.find((segment) => segment.mediaType === 'sqlite');
575
- if (ref === undefined || image === undefined) {
624
+ const first = refs[0];
625
+ if (first === undefined) {
576
626
  throw new Error('publishSqliteImage: the scope holds at most one row, which pulls serve inline');
577
627
  }
578
628
  return {
579
- segmentId: ref.segmentId,
580
- asOfCommitSeq: ref.asOfCommitSeq,
581
- scopeDigest: ref.scopeDigest,
582
- rowCount: ref.rowCount,
583
- byteLength: ref.byteLength,
584
- origin: image.origin === 'reused' ? 'reused' : 'built',
629
+ segmentIds: refs.map((ref) => ref.segmentId),
630
+ asOfCommitSeq: first.asOfCommitSeq,
631
+ scopeDigest: first.scopeDigest,
632
+ rowCount: refs.reduce((sum, ref) => sum + ref.rowCount, 0),
633
+ byteLength: refs.reduce((sum, ref) => sum + ref.byteLength, 0),
634
+ origin: trace.segments.every((segment) => segment.mediaType !== 'sqlite' || segment.origin === 'reused')
635
+ ? 'reused'
636
+ : 'built',
585
637
  };
586
638
  }
@@ -99,7 +99,8 @@ export declare function mergeSegmentRecord(existing: SegmentRecord | undefined,
99
99
  /**
100
100
  * The §5.3 reuse key: sqlite images are not byte-deterministic, so
101
101
  * cross-client dedup is a metadata lookup, not hash convergence — one
102
- * stored image per (partition, table, schemaVersion, scope digest, pin).
102
+ * stored image per (partition, table, schemaVersion, scope digest, pin, range
103
+ * start).
103
104
  */
104
105
  export interface SegmentFindKey {
105
106
  readonly partition: string;
@@ -109,6 +110,8 @@ export interface SegmentFindKey {
109
110
  readonly mediaType: 'rows' | 'sqlite';
110
111
  readonly scopeDigest: string;
111
112
  readonly asOfCommitSeq: number;
113
+ /** The range start: the row id the image follows, `null` for the first. */
114
+ readonly rowCursor: string | null;
112
115
  }
113
116
  /**
114
117
  * Coarse store-level counters for the admin/console read surface. Optional:
@@ -146,9 +149,8 @@ export interface SegmentStore {
146
149
  bytes: Uint8Array;
147
150
  } | undefined>;
148
151
  /**
149
- * Unexpired record for the §5.3 reuse key (whole-table segments only:
150
- * `rowCursor` null), or undefined. Servers MUST reuse instead of
151
- * rebuilding sqlite images while one exists (§5.3).
152
+ * Unexpired record for the §5.3 reuse key, or undefined. Servers MUST
153
+ * reuse instead of rebuilding sqlite images while one exists (§5.3).
152
154
  *
153
155
  * The digest matches when it is one of the record's `scopeDigests`
154
156
  * (§5.1). That cannot hand a sqlite image to the wrong digest: a sqlite
@@ -188,7 +188,9 @@ export class MemorySegmentStore {
188
188
  };
189
189
  }
190
190
  async find(key, nowMs) {
191
- for (const { record } of this.#entries.values()) {
191
+ // The newest publication wins: a rebuild supersedes an image stored
192
+ // under the same key that the server no longer serves (§5.3 bound).
193
+ for (const { record } of [...this.#entries.values()].reverse()) {
192
194
  const publication = selectSegmentPublication(record, {
193
195
  partition: key.partition,
194
196
  logEpoch: key.logEpoch,
@@ -197,7 +199,7 @@ export class MemorySegmentStore {
197
199
  mediaType: key.mediaType,
198
200
  scopeDigest: key.scopeDigest,
199
201
  asOfCommitSeq: key.asOfCommitSeq,
200
- rowCursor: null,
202
+ rowCursor: key.rowCursor,
201
203
  nowMs,
202
204
  });
203
205
  if (publication !== undefined) {
@@ -30,8 +30,8 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
30
30
  export const buildSqliteImage = async (input) => {
31
31
  const db = new BunSqliteDatabase();
32
32
  try {
33
- await writeSqliteImage(db, input);
34
- return db.serialize();
33
+ const rowCount = await writeSqliteImage(db, input);
34
+ return { bytes: db.serialize(), rowCount };
35
35
  }
36
36
  finally {
37
37
  db.close();
@@ -10,6 +10,13 @@ export interface SqliteImageInput {
10
10
  readonly schemaVersion: number;
11
11
  readonly asOfCommitSeq: number;
12
12
  readonly scopeDigest: string;
13
+ /**
14
+ * The image's file-size bound (`ServerLimits.maxImageSegmentBytes`). The
15
+ * builder stops before the row that could pass it; the first row always
16
+ * goes in.
17
+ */
18
+ readonly maxBytes: number;
19
+ /** The range's rows in row order, from its start to the table's end. */
13
20
  readonly rowBatches: AsyncIterable<readonly StoredRow[]> | Iterable<readonly StoredRow[]>;
14
21
  }
15
22
  /**
@@ -20,6 +27,14 @@ export interface SqliteImageInput {
20
27
  * on the pull path. A Bun or Node host passes `buildSqliteImage`; a Workers
21
28
  * host omits it and serves matching stored images or the rows lane.
22
29
  */
23
- export type SqliteImageBuilder = (input: SqliteImageInput) => Promise<Uint8Array>;
24
- /** Populate a §5.3 image database for a whole-table snapshot. */
25
- export declare function writeSqliteImage(db: SqliteDatabase, input: SqliteImageInput): Promise<void>;
30
+ export type SqliteImageBuilder = (input: SqliteImageInput) => Promise<SqliteImage>;
31
+ /** A built image and the number of leading input rows it holds. */
32
+ export interface SqliteImage {
33
+ readonly bytes: Uint8Array;
34
+ readonly rowCount: number;
35
+ }
36
+ /**
37
+ * Populate a §5.3 image database with the leading input rows that fit
38
+ * `maxBytes`. Returns how many rows it consumed.
39
+ */
40
+ export declare function writeSqliteImage(db: SqliteDatabase, input: SqliteImageInput): Promise<number>;