@syncular/server 0.75.0 → 0.77.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bucket-segment-store.d.ts +4 -3
- package/dist/bucket-segment-store.js +4 -2
- package/dist/context.d.ts +6 -0
- package/dist/context.js +2 -0
- package/dist/index.d.ts +1 -1
- package/dist/pull.d.ts +9 -4
- package/dist/pull.js +128 -76
- package/dist/segment-store.d.ts +6 -4
- package/dist/segment-store.js +4 -2
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +18 -3
- package/dist/sqlite-image.js +27 -5
- package/dist/sqlite-node.js +3 -2
- package/dist/sqlite-segment-store.js +7 -7
- package/package.json +2 -2
- package/src/bucket-segment-store.ts +8 -5
- package/src/context.ts +8 -0
- package/src/index.ts +1 -0
- package/src/pull.ts +199 -94
- package/src/segment-store.ts +10 -6
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +42 -7
- package/src/sqlite-node.ts +3 -2
- package/src/sqlite-segment-store.ts +29 -7
package/src/pull.ts
CHANGED
|
@@ -214,6 +214,8 @@ interface BootstrapToken {
|
|
|
214
214
|
tables: string[];
|
|
215
215
|
tableIndex: number;
|
|
216
216
|
rowCursor: string | null;
|
|
217
|
+
/** Present when the table at `tableIndex` continues on the image lane. */
|
|
218
|
+
image?: true;
|
|
217
219
|
}
|
|
218
220
|
|
|
219
221
|
/**
|
|
@@ -242,7 +244,9 @@ function parseBootstrapToken(
|
|
|
242
244
|
parsed.tableIndex === undefined ||
|
|
243
245
|
parsed.tableIndex < 0 ||
|
|
244
246
|
parsed.tableIndex >= tables.length ||
|
|
245
|
-
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')
|
|
247
|
+
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
|
|
248
|
+
(parsed.image !== undefined &&
|
|
249
|
+
(parsed.image !== true || parsed.rowCursor === null))
|
|
246
250
|
) {
|
|
247
251
|
return undefined;
|
|
248
252
|
}
|
|
@@ -251,6 +255,7 @@ function parseBootstrapToken(
|
|
|
251
255
|
tables: parsed.tables as string[],
|
|
252
256
|
tableIndex: parsed.tableIndex,
|
|
253
257
|
rowCursor: parsed.rowCursor ?? null,
|
|
258
|
+
...(parsed.image === true ? { image: true as const } : {}),
|
|
254
259
|
};
|
|
255
260
|
} catch {
|
|
256
261
|
return undefined;
|
|
@@ -302,8 +307,12 @@ function segmentRefFrame(
|
|
|
302
307
|
// under different scopes (§5.1), and the record's primary digest is not
|
|
303
308
|
// necessarily the caller's (§5.5).
|
|
304
309
|
scopeDigest,
|
|
305
|
-
|
|
306
|
-
|
|
310
|
+
// §5.3: an image descriptor carries no cursors; the resume token holds
|
|
311
|
+
// the position of a table's next range.
|
|
312
|
+
...(record.mediaType === 'rows' && record.rowCursor !== null
|
|
313
|
+
? { rowCursor: record.rowCursor }
|
|
314
|
+
: {}),
|
|
315
|
+
...(record.mediaType === 'rows' && record.nextRowCursor !== null
|
|
307
316
|
? { nextRowCursor: record.nextRowCursor }
|
|
308
317
|
: {}),
|
|
309
318
|
...(extra.url !== undefined ? { url: extra.url } : {}),
|
|
@@ -320,8 +329,8 @@ interface BootstrapPair {
|
|
|
320
329
|
}
|
|
321
330
|
|
|
322
331
|
/**
|
|
323
|
-
* §5.3: the sqlite-image lane
|
|
324
|
-
*
|
|
332
|
+
* §5.3: the sqlite-image lane, chosen at the start of a table or continued
|
|
333
|
+
* after a range image (a table on the rows lane never switches to images),
|
|
325
334
|
* and only when the client advertised bit 2. §5.11: a table with any
|
|
326
335
|
* encrypted column is image-INELIGIBLE — an image copies ciphertext
|
|
327
336
|
* wholesale with no per-row decrypt pass, so it MUST be served on the rows
|
|
@@ -331,14 +340,52 @@ function sqliteLaneEligible(
|
|
|
331
340
|
limits: PullLimits,
|
|
332
341
|
table: CompiledTable,
|
|
333
342
|
startRowCursor: string | null,
|
|
343
|
+
imageLane: boolean,
|
|
334
344
|
): boolean {
|
|
335
345
|
return (
|
|
336
346
|
(limits.accept & ACCEPT_SQLITE) !== 0 &&
|
|
337
|
-
startRowCursor === null &&
|
|
347
|
+
(startRowCursor === null || imageLane) &&
|
|
338
348
|
table.encryptedColumnIndices.length === 0
|
|
339
349
|
);
|
|
340
350
|
}
|
|
341
351
|
|
|
352
|
+
/**
|
|
353
|
+
* §5.3 reuse lookup for the range image that starts after `rowCursor`. Only
|
|
354
|
+
* a position with more than one rows page left can match a stored image, so
|
|
355
|
+
* a smaller remainder skips the lookup. A stored image past the bound (built
|
|
356
|
+
* under a larger one) does not match unless it holds a single row.
|
|
357
|
+
*/
|
|
358
|
+
async function findImage(
|
|
359
|
+
ctx: SyncRequestContext,
|
|
360
|
+
schema: CompiledSchema,
|
|
361
|
+
limits: PullLimits,
|
|
362
|
+
pair: BootstrapPair,
|
|
363
|
+
scopePin: number,
|
|
364
|
+
rowCursor: string | null,
|
|
365
|
+
probe: readonly StoredRow[],
|
|
366
|
+
logEpoch: string,
|
|
367
|
+
): Promise<SegmentRecord | undefined> {
|
|
368
|
+
if (probe.length <= limits.limitSnapshotRows) return undefined;
|
|
369
|
+
const stored = await ctx.segments.find(
|
|
370
|
+
{
|
|
371
|
+
partition: ctx.partition,
|
|
372
|
+
logEpoch,
|
|
373
|
+
table: pair.table.name,
|
|
374
|
+
schemaVersion: schema.version,
|
|
375
|
+
mediaType: 'sqlite',
|
|
376
|
+
scopeDigest: await scopeDigest(pair.effective),
|
|
377
|
+
asOfCommitSeq: scopePin,
|
|
378
|
+
rowCursor,
|
|
379
|
+
},
|
|
380
|
+
clockOf(ctx)(),
|
|
381
|
+
);
|
|
382
|
+
return stored !== undefined &&
|
|
383
|
+
(stored.byteLength <= limitsOf(ctx).maxImageSegmentBytes ||
|
|
384
|
+
stored.rowCount === 1)
|
|
385
|
+
? stored
|
|
386
|
+
: undefined;
|
|
387
|
+
}
|
|
388
|
+
|
|
342
389
|
/** The reads a table's bootstrap needs before its first frame (§4.7, §5.3). */
|
|
343
390
|
interface TableStart {
|
|
344
391
|
/**
|
|
@@ -360,6 +407,7 @@ async function readTableStart(
|
|
|
360
407
|
unit: UnitPlan,
|
|
361
408
|
pair: BootstrapPair,
|
|
362
409
|
startRowCursor: string | null,
|
|
410
|
+
imageLane: boolean,
|
|
363
411
|
maxSeq: number,
|
|
364
412
|
horizonSeq: number,
|
|
365
413
|
logEpoch: string,
|
|
@@ -370,7 +418,7 @@ async function readTableStart(
|
|
|
370
418
|
afterRowId: startRowCursor,
|
|
371
419
|
limit: limits.limitSnapshotRows + 1,
|
|
372
420
|
});
|
|
373
|
-
if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
|
|
421
|
+
if (!sqliteLaneEligible(limits, pair.table, startRowCursor, imageLane)) {
|
|
374
422
|
return { firstPage: await scan };
|
|
375
423
|
}
|
|
376
424
|
// §4.7: a sqlite-lane table pins at the newest commit carrying all of the
|
|
@@ -385,33 +433,25 @@ async function readTableStart(
|
|
|
385
433
|
}),
|
|
386
434
|
]);
|
|
387
435
|
const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
table: pair.table.name,
|
|
399
|
-
schemaVersion: schema.version,
|
|
400
|
-
mediaType: 'sqlite',
|
|
401
|
-
scopeDigest: await scopeDigest(pair.effective),
|
|
402
|
-
asOfCommitSeq: scopePin,
|
|
403
|
-
},
|
|
404
|
-
clockOf(ctx)(),
|
|
405
|
-
);
|
|
436
|
+
const stored = await findImage(
|
|
437
|
+
ctx,
|
|
438
|
+
schema,
|
|
439
|
+
limits,
|
|
440
|
+
pair,
|
|
441
|
+
scopePin,
|
|
442
|
+
startRowCursor,
|
|
443
|
+
firstPage,
|
|
444
|
+
logEpoch,
|
|
445
|
+
);
|
|
406
446
|
return { firstPage, image: { scopePin, stored } };
|
|
407
447
|
}
|
|
408
448
|
|
|
409
449
|
/**
|
|
410
|
-
* §5.3 sqlite-image lane:
|
|
411
|
-
*
|
|
412
|
-
* schemaVersion, scope digest, scope
|
|
413
|
-
*
|
|
414
|
-
* takes over).
|
|
450
|
+
* §5.3 sqlite-image lane: one range image starting after `rowCursor`, only
|
|
451
|
+
* when more than one rows page remains there. Reuses an unexpired stored
|
|
452
|
+
* image for the same (partition, table, schemaVersion, scope digest, scope
|
|
453
|
+
* pin, range start) instead of rebuilding — the bootstrap-storm rule.
|
|
454
|
+
* Returns undefined when no image is served (the rows lane takes over).
|
|
415
455
|
*/
|
|
416
456
|
async function* sqliteImageSegment(
|
|
417
457
|
ctx: SyncRequestContext,
|
|
@@ -419,12 +459,13 @@ async function* sqliteImageSegment(
|
|
|
419
459
|
limits: PullLimits,
|
|
420
460
|
pair: BootstrapPair,
|
|
421
461
|
scopePin: number,
|
|
462
|
+
rowCursor: string | null,
|
|
422
463
|
digest: string,
|
|
423
464
|
trace: PullSectionTrace | undefined,
|
|
424
465
|
logEpoch: string,
|
|
425
466
|
existing: SegmentRecord | undefined,
|
|
426
467
|
probe: StoredRow[],
|
|
427
|
-
): AsyncGenerator<ResponseFrame,
|
|
468
|
+
): AsyncGenerator<ResponseFrame, SegmentRecord | undefined> {
|
|
428
469
|
const { storage, segments, partition } = ctx;
|
|
429
470
|
const now = clockOf(ctx)();
|
|
430
471
|
if (existing !== undefined) {
|
|
@@ -440,13 +481,13 @@ async function* sqliteImageSegment(
|
|
|
440
481
|
digest,
|
|
441
482
|
await signedUrlFields(ctx, limits, existing.segmentId, digest, now),
|
|
442
483
|
);
|
|
443
|
-
return
|
|
484
|
+
return existing;
|
|
444
485
|
}
|
|
445
|
-
if (probe.length <= limits.limitSnapshotRows) return
|
|
486
|
+
if (probe.length <= limits.limitSnapshotRows) return undefined;
|
|
446
487
|
// §5.3: hosts opt into image construction with a runtime-specific builder.
|
|
447
488
|
// Without one, matching stored images remain usable and cold pulls use rows.
|
|
448
489
|
const buildImage = ctx.sqliteImageBuilder;
|
|
449
|
-
if (buildImage === undefined) return
|
|
490
|
+
if (buildImage === undefined) return undefined;
|
|
450
491
|
let stores = imageBuilds.get(storage);
|
|
451
492
|
if (stores === undefined) {
|
|
452
493
|
stores = new WeakMap();
|
|
@@ -465,6 +506,7 @@ async function* sqliteImageSegment(
|
|
|
465
506
|
mediaType: 'sqlite' as const,
|
|
466
507
|
scopeDigest: digest,
|
|
467
508
|
asOfCommitSeq: scopePin,
|
|
509
|
+
rowCursor,
|
|
468
510
|
};
|
|
469
511
|
const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
|
|
470
512
|
const key = JSON.stringify(identity);
|
|
@@ -473,43 +515,65 @@ async function* sqliteImageSegment(
|
|
|
473
515
|
if (building === undefined) {
|
|
474
516
|
building = (async () => {
|
|
475
517
|
// Another request can finish while this one's eligibility probe awaits.
|
|
476
|
-
const cached = await
|
|
518
|
+
const cached = await findImage(
|
|
519
|
+
ctx,
|
|
520
|
+
schema,
|
|
521
|
+
limits,
|
|
522
|
+
pair,
|
|
523
|
+
scopePin,
|
|
524
|
+
rowCursor,
|
|
525
|
+
probe,
|
|
526
|
+
logEpoch,
|
|
527
|
+
);
|
|
477
528
|
if (cached !== undefined) return cached;
|
|
478
529
|
builtHere = true;
|
|
479
|
-
|
|
480
|
-
|
|
530
|
+
// The builder stops at the size bound inside the latest batch; these
|
|
531
|
+
// locate the last row it took and whether rows follow it.
|
|
532
|
+
let yielded = 0;
|
|
533
|
+
let batch: readonly StoredRow[] = [];
|
|
534
|
+
let beforeBatch = rowCursor;
|
|
535
|
+
let exhausted = false;
|
|
536
|
+
const image = await buildImage({
|
|
481
537
|
table: pair.table,
|
|
482
538
|
schemaVersion: schema.version,
|
|
483
539
|
asOfCommitSeq: scopePin,
|
|
484
540
|
scopeDigest: digest,
|
|
541
|
+
maxBytes: limitsOf(ctx).maxImageSegmentBytes,
|
|
485
542
|
rowBatches: (async function* () {
|
|
486
|
-
|
|
487
|
-
yield probe.map((row) => ({
|
|
488
|
-
...row,
|
|
489
|
-
payload: projectRowPayload(current, pair.table, row.payload),
|
|
490
|
-
}));
|
|
491
|
-
let afterRowId = probe[probe.length - 1]!.rowId;
|
|
543
|
+
let rows: readonly StoredRow[] = probe;
|
|
492
544
|
for (;;) {
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
afterRowId,
|
|
497
|
-
limit: 5_000,
|
|
498
|
-
});
|
|
499
|
-
rowCount += rows.length;
|
|
545
|
+
beforeBatch = batch[batch.length - 1]?.rowId ?? beforeBatch;
|
|
546
|
+
batch = rows;
|
|
547
|
+
yielded += rows.length;
|
|
500
548
|
yield rows.map((row) => ({
|
|
501
549
|
...row,
|
|
502
550
|
payload: projectRowPayload(current, pair.table, row.payload),
|
|
503
551
|
}));
|
|
504
552
|
const last = rows[rows.length - 1];
|
|
505
|
-
if (rows.length < 5_000
|
|
506
|
-
|
|
553
|
+
if (last === undefined || (rows !== probe && rows.length < 5_000))
|
|
554
|
+
break;
|
|
555
|
+
rows = await storage.scanRows(partition, {
|
|
556
|
+
table: pair.table.name,
|
|
557
|
+
scopeFilter: pair.effective,
|
|
558
|
+
afterRowId: last.rowId,
|
|
559
|
+
limit: 5_000,
|
|
560
|
+
});
|
|
507
561
|
}
|
|
562
|
+
exhausted = true;
|
|
508
563
|
})(),
|
|
509
564
|
});
|
|
565
|
+
const index = image.rowCount - (yielded - batch.length) - 1;
|
|
566
|
+
if (image.rowCount < 1 || image.rowCount > yielded || index < -1) {
|
|
567
|
+
throw new Error('sqliteImageBuilder consumed rows out of order');
|
|
568
|
+
}
|
|
569
|
+
const last = index < 0 ? beforeBatch : batch[index]!.rowId;
|
|
510
570
|
return segments.put(
|
|
511
|
-
{
|
|
512
|
-
|
|
571
|
+
{
|
|
572
|
+
...identity,
|
|
573
|
+
rowCount: image.rowCount,
|
|
574
|
+
nextRowCursor: image.rowCount < yielded || !exhausted ? last : null,
|
|
575
|
+
},
|
|
576
|
+
image.bytes,
|
|
513
577
|
clockOf(ctx)(),
|
|
514
578
|
);
|
|
515
579
|
})();
|
|
@@ -539,12 +603,14 @@ async function* sqliteImageSegment(
|
|
|
539
603
|
clockOf(ctx)(),
|
|
540
604
|
),
|
|
541
605
|
);
|
|
542
|
-
return
|
|
606
|
+
return record;
|
|
543
607
|
}
|
|
544
608
|
|
|
545
609
|
/**
|
|
546
|
-
* One table's segments within the page budget
|
|
547
|
-
*
|
|
610
|
+
* One table's segments within the page budget: range images while the table
|
|
611
|
+
* is on the image lane, then rows pages. Returns the pages it used, the row
|
|
612
|
+
* cursor to resume at (`null` once the table is exhausted) and whether that
|
|
613
|
+
* position is on the image lane.
|
|
548
614
|
*/
|
|
549
615
|
async function* tableSegments(
|
|
550
616
|
ctx: SyncRequestContext,
|
|
@@ -557,28 +623,17 @@ async function* tableSegments(
|
|
|
557
623
|
pageBudget: number,
|
|
558
624
|
trace: PullSectionTrace | undefined,
|
|
559
625
|
logEpoch: string,
|
|
560
|
-
): AsyncGenerator<
|
|
626
|
+
): AsyncGenerator<
|
|
627
|
+
ResponseFrame,
|
|
628
|
+
{ pages: number; rowCursor: string | null; image: boolean }
|
|
629
|
+
> {
|
|
561
630
|
const { storage, segments, partition } = ctx;
|
|
562
631
|
const serverLimits = limitsOf(ctx);
|
|
563
632
|
const digest = await scopeDigest(pair.effective);
|
|
564
633
|
const now = clockOf(ctx)();
|
|
565
|
-
if (start.image !== undefined) {
|
|
566
|
-
const imaged = yield* sqliteImageSegment(
|
|
567
|
-
ctx,
|
|
568
|
-
schema,
|
|
569
|
-
limits,
|
|
570
|
-
pair,
|
|
571
|
-
start.image.scopePin,
|
|
572
|
-
digest,
|
|
573
|
-
trace,
|
|
574
|
-
logEpoch,
|
|
575
|
-
start.image.stored,
|
|
576
|
-
start.firstPage,
|
|
577
|
-
);
|
|
578
|
-
if (imaged) return { pages: 1, rowCursor: null };
|
|
579
|
-
}
|
|
580
634
|
const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
|
|
581
635
|
let rowCursor = startRowCursor;
|
|
636
|
+
let image = start.image;
|
|
582
637
|
for (let page = 0; page < pageBudget; page++) {
|
|
583
638
|
const scanned =
|
|
584
639
|
page === 0
|
|
@@ -589,6 +644,41 @@ async function* tableSegments(
|
|
|
589
644
|
afterRowId: rowCursor,
|
|
590
645
|
limit: limits.limitSnapshotRows + 1,
|
|
591
646
|
});
|
|
647
|
+
if (image !== undefined) {
|
|
648
|
+
// §5.3: every range of the table shares the scope pin computed at this
|
|
649
|
+
// section's table start.
|
|
650
|
+
const record = yield* sqliteImageSegment(
|
|
651
|
+
ctx,
|
|
652
|
+
schema,
|
|
653
|
+
limits,
|
|
654
|
+
pair,
|
|
655
|
+
image.scopePin,
|
|
656
|
+
rowCursor,
|
|
657
|
+
digest,
|
|
658
|
+
trace,
|
|
659
|
+
logEpoch,
|
|
660
|
+
page === 0
|
|
661
|
+
? image.stored
|
|
662
|
+
: await findImage(
|
|
663
|
+
ctx,
|
|
664
|
+
schema,
|
|
665
|
+
limits,
|
|
666
|
+
pair,
|
|
667
|
+
image.scopePin,
|
|
668
|
+
rowCursor,
|
|
669
|
+
scanned,
|
|
670
|
+
logEpoch,
|
|
671
|
+
),
|
|
672
|
+
scanned,
|
|
673
|
+
);
|
|
674
|
+
if (record !== undefined) {
|
|
675
|
+
if (record.nextRowCursor === null)
|
|
676
|
+
return { pages: page + 1, rowCursor: null, image: false };
|
|
677
|
+
rowCursor = record.nextRowCursor;
|
|
678
|
+
continue;
|
|
679
|
+
}
|
|
680
|
+
image = undefined;
|
|
681
|
+
}
|
|
592
682
|
const pageRows = scanned.slice(0, limits.limitSnapshotRows);
|
|
593
683
|
const hasMore = scanned.length > limits.limitSnapshotRows;
|
|
594
684
|
// §5.2: every row record carries the row's current server_version.
|
|
@@ -650,10 +740,10 @@ async function* tableSegments(
|
|
|
650
740
|
await signedUrlFields(ctx, limits, record.segmentId, digest, now),
|
|
651
741
|
);
|
|
652
742
|
}
|
|
653
|
-
if (!hasMore) return { pages: page + 1, rowCursor: null };
|
|
743
|
+
if (!hasMore) return { pages: page + 1, rowCursor: null, image: false };
|
|
654
744
|
rowCursor = nextRowCursor;
|
|
655
745
|
}
|
|
656
|
-
return { pages: pageBudget, rowCursor };
|
|
746
|
+
return { pages: pageBudget, rowCursor, image: image !== undefined };
|
|
657
747
|
}
|
|
658
748
|
|
|
659
749
|
/** A bootstrapping unit's position and its table starts, read ahead (§4.7). */
|
|
@@ -663,6 +753,8 @@ export interface BootstrapPlan {
|
|
|
663
753
|
readonly pin: number;
|
|
664
754
|
readonly tableIndex: number;
|
|
665
755
|
readonly rowCursor: string | null;
|
|
756
|
+
/** Whether the position continues the table's range images (§5.3). */
|
|
757
|
+
readonly image: boolean;
|
|
666
758
|
/** Table index → its start reads. */
|
|
667
759
|
readonly starts: ReadonlyMap<number, Promise<TableStart>>;
|
|
668
760
|
}
|
|
@@ -693,6 +785,7 @@ export function planBootstrap(
|
|
|
693
785
|
: undefined;
|
|
694
786
|
const tableIndex = resume?.tableIndex ?? 0;
|
|
695
787
|
const rowCursor = resume?.rowCursor ?? null;
|
|
788
|
+
const image = resume?.image === true;
|
|
696
789
|
const starts = new Map<number, Promise<TableStart>>();
|
|
697
790
|
for (
|
|
698
791
|
let index = tableIndex;
|
|
@@ -709,6 +802,7 @@ export function planBootstrap(
|
|
|
709
802
|
unit,
|
|
710
803
|
{ table, effective },
|
|
711
804
|
index === tableIndex ? rowCursor : null,
|
|
805
|
+
index === tableIndex && image,
|
|
712
806
|
maxSeq,
|
|
713
807
|
horizonSeq,
|
|
714
808
|
logEpoch,
|
|
@@ -722,6 +816,7 @@ export function planBootstrap(
|
|
|
722
816
|
pin: resume?.asOfCommitSeq ?? maxSeq,
|
|
723
817
|
tableIndex,
|
|
724
818
|
rowCursor,
|
|
819
|
+
image,
|
|
725
820
|
starts,
|
|
726
821
|
};
|
|
727
822
|
}
|
|
@@ -741,7 +836,7 @@ export async function* bootstrapSegments(
|
|
|
741
836
|
logEpoch: string,
|
|
742
837
|
): AsyncGenerator<ResponseFrame, string | undefined> {
|
|
743
838
|
const { unit, pin } = plan;
|
|
744
|
-
let { tableIndex, rowCursor } = plan;
|
|
839
|
+
let { tableIndex, rowCursor, image } = plan;
|
|
745
840
|
let pages = 0;
|
|
746
841
|
for (;;) {
|
|
747
842
|
while (
|
|
@@ -750,6 +845,7 @@ export async function* bootstrapSegments(
|
|
|
750
845
|
) {
|
|
751
846
|
tableIndex += 1;
|
|
752
847
|
rowCursor = null;
|
|
848
|
+
image = false;
|
|
753
849
|
}
|
|
754
850
|
if (tableIndex === unit.tables.length) return undefined;
|
|
755
851
|
if (pages === limits.maxSnapshotPages) {
|
|
@@ -758,6 +854,7 @@ export async function* bootstrapSegments(
|
|
|
758
854
|
tables: unit.tables.map((table) => table.name),
|
|
759
855
|
tableIndex,
|
|
760
856
|
rowCursor,
|
|
857
|
+
...(image ? { image: true as const } : {}),
|
|
761
858
|
};
|
|
762
859
|
return JSON.stringify(token);
|
|
763
860
|
}
|
|
@@ -776,6 +873,7 @@ export async function* bootstrapSegments(
|
|
|
776
873
|
);
|
|
777
874
|
pages += outcome.pages;
|
|
778
875
|
rowCursor = outcome.rowCursor;
|
|
876
|
+
image = outcome.image;
|
|
779
877
|
if (rowCursor === null) tableIndex += 1;
|
|
780
878
|
}
|
|
781
879
|
}
|
|
@@ -797,18 +895,21 @@ export interface PublishSqliteImageInput {
|
|
|
797
895
|
}
|
|
798
896
|
|
|
799
897
|
export interface PublishedSqliteImage {
|
|
800
|
-
|
|
801
|
-
|
|
898
|
+
/** The table's range images in row order (§5.3). */
|
|
899
|
+
readonly segmentIds: readonly string[];
|
|
900
|
+
/** The §4.7 scope pin the images are stored under. */
|
|
802
901
|
readonly asOfCommitSeq: number;
|
|
803
902
|
readonly scopeDigest: string;
|
|
903
|
+
/** Rows across every range image. */
|
|
804
904
|
readonly rowCount: number;
|
|
905
|
+
/** Bytes across every range image. */
|
|
805
906
|
readonly byteLength: number;
|
|
806
|
-
/** `reused` when an unexpired image for
|
|
907
|
+
/** `reused` when an unexpired image existed for every range's reuse key. */
|
|
807
908
|
readonly origin: 'built' | 'reused';
|
|
808
909
|
}
|
|
809
910
|
|
|
810
911
|
/**
|
|
811
|
-
* Publish the §5.3
|
|
912
|
+
* Publish the §5.3 range images that a pull of (`scopes`, `table`) looks up,
|
|
812
913
|
* from a host with a SQLite engine (a Bun process with the production
|
|
813
914
|
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
814
915
|
* reuse key, the build), so the stored identity is exactly the one a
|
|
@@ -848,11 +949,12 @@ export async function publishSqliteImage(
|
|
|
848
949
|
partition,
|
|
849
950
|
actorId: 'syncular.image-publisher',
|
|
850
951
|
};
|
|
851
|
-
// One-row pages make every
|
|
952
|
+
// One-row pages make every range above one row image-eligible (§5.3), and
|
|
953
|
+
// an unbounded page budget stores every range under one scope pin.
|
|
852
954
|
const limits: PullLimits = {
|
|
853
955
|
limitCommits: 1,
|
|
854
956
|
limitSnapshotRows: 1,
|
|
855
|
-
maxSnapshotPages:
|
|
957
|
+
maxSnapshotPages: Number.MAX_SAFE_INTEGER,
|
|
856
958
|
accept: ACCEPT_SQLITE,
|
|
857
959
|
};
|
|
858
960
|
const maxSeq = await config.storage.getMaxCommitSeq(partition);
|
|
@@ -868,7 +970,7 @@ export async function publishSqliteImage(
|
|
|
868
970
|
horizonSeq,
|
|
869
971
|
logEpoch,
|
|
870
972
|
);
|
|
871
|
-
|
|
973
|
+
const refs: SegmentRefFrame[] = [];
|
|
872
974
|
for await (const frame of bootstrapSegments(
|
|
873
975
|
ctx,
|
|
874
976
|
schema,
|
|
@@ -878,23 +980,26 @@ export async function publishSqliteImage(
|
|
|
878
980
|
logEpoch,
|
|
879
981
|
)) {
|
|
880
982
|
if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
|
|
881
|
-
|
|
983
|
+
refs.push(frame);
|
|
882
984
|
}
|
|
883
985
|
}
|
|
884
|
-
const
|
|
885
|
-
|
|
886
|
-
);
|
|
887
|
-
if (ref === undefined || image === undefined) {
|
|
986
|
+
const first = refs[0];
|
|
987
|
+
if (first === undefined) {
|
|
888
988
|
throw new Error(
|
|
889
989
|
'publishSqliteImage: the scope holds at most one row, which pulls serve inline',
|
|
890
990
|
);
|
|
891
991
|
}
|
|
892
992
|
return {
|
|
893
|
-
|
|
894
|
-
asOfCommitSeq:
|
|
895
|
-
scopeDigest:
|
|
896
|
-
rowCount: ref.rowCount,
|
|
897
|
-
byteLength: ref.byteLength,
|
|
898
|
-
origin:
|
|
993
|
+
segmentIds: refs.map((ref) => ref.segmentId),
|
|
994
|
+
asOfCommitSeq: first.asOfCommitSeq,
|
|
995
|
+
scopeDigest: first.scopeDigest,
|
|
996
|
+
rowCount: refs.reduce((sum, ref) => sum + ref.rowCount, 0),
|
|
997
|
+
byteLength: refs.reduce((sum, ref) => sum + ref.byteLength, 0),
|
|
998
|
+
origin: trace.segments.every(
|
|
999
|
+
(segment) =>
|
|
1000
|
+
segment.mediaType !== 'sqlite' || segment.origin === 'reused',
|
|
1001
|
+
)
|
|
1002
|
+
? 'reused'
|
|
1003
|
+
: 'built',
|
|
899
1004
|
};
|
|
900
1005
|
}
|
package/src/segment-store.ts
CHANGED
|
@@ -250,7 +250,8 @@ export function mergeSegmentRecord(
|
|
|
250
250
|
/**
|
|
251
251
|
* The §5.3 reuse key: sqlite images are not byte-deterministic, so
|
|
252
252
|
* cross-client dedup is a metadata lookup, not hash convergence — one
|
|
253
|
-
* stored image per (partition, table, schemaVersion, scope digest, pin
|
|
253
|
+
* stored image per (partition, table, schemaVersion, scope digest, pin, range
|
|
254
|
+
* start).
|
|
254
255
|
*/
|
|
255
256
|
export interface SegmentFindKey {
|
|
256
257
|
readonly partition: string;
|
|
@@ -260,6 +261,8 @@ export interface SegmentFindKey {
|
|
|
260
261
|
readonly mediaType: 'rows' | 'sqlite';
|
|
261
262
|
readonly scopeDigest: string;
|
|
262
263
|
readonly asOfCommitSeq: number;
|
|
264
|
+
/** The range start: the row id the image follows, `null` for the first. */
|
|
265
|
+
readonly rowCursor: string | null;
|
|
263
266
|
}
|
|
264
267
|
|
|
265
268
|
/**
|
|
@@ -313,9 +316,8 @@ export interface SegmentStore {
|
|
|
313
316
|
segmentId: string,
|
|
314
317
|
): Promise<{ record: SegmentRecord; bytes: Uint8Array } | undefined>;
|
|
315
318
|
/**
|
|
316
|
-
* Unexpired record for the §5.3 reuse key
|
|
317
|
-
*
|
|
318
|
-
* rebuilding sqlite images while one exists (§5.3).
|
|
319
|
+
* Unexpired record for the §5.3 reuse key, or undefined. Servers MUST
|
|
320
|
+
* reuse instead of rebuilding sqlite images while one exists (§5.3).
|
|
319
321
|
*
|
|
320
322
|
* The digest matches when it is one of the record's `scopeDigests`
|
|
321
323
|
* (§5.1). That cannot hand a sqlite image to the wrong digest: a sqlite
|
|
@@ -393,7 +395,9 @@ export class MemorySegmentStore implements SegmentStore {
|
|
|
393
395
|
key: SegmentFindKey,
|
|
394
396
|
nowMs: number,
|
|
395
397
|
): Promise<SegmentRecord | undefined> {
|
|
396
|
-
|
|
398
|
+
// The newest publication wins: a rebuild supersedes an image stored
|
|
399
|
+
// under the same key that the server no longer serves (§5.3 bound).
|
|
400
|
+
for (const { record } of [...this.#entries.values()].reverse()) {
|
|
397
401
|
const publication = selectSegmentPublication(record, {
|
|
398
402
|
partition: key.partition,
|
|
399
403
|
logEpoch: key.logEpoch,
|
|
@@ -402,7 +406,7 @@ export class MemorySegmentStore implements SegmentStore {
|
|
|
402
406
|
mediaType: key.mediaType,
|
|
403
407
|
scopeDigest: key.scopeDigest,
|
|
404
408
|
asOfCommitSeq: key.asOfCommitSeq,
|
|
405
|
-
rowCursor:
|
|
409
|
+
rowCursor: key.rowCursor,
|
|
406
410
|
nowMs,
|
|
407
411
|
});
|
|
408
412
|
if (publication !== undefined) {
|
package/src/sqlite-bun.ts
CHANGED
|
@@ -43,8 +43,8 @@ export class SqliteLeaseStore extends SharedSqliteLeaseStore {
|
|
|
43
43
|
export const buildSqliteImage: SqliteImageBuilder = async (input) => {
|
|
44
44
|
const db = new BunSqliteDatabase();
|
|
45
45
|
try {
|
|
46
|
-
await writeSqliteImage(db, input);
|
|
47
|
-
return db.serialize();
|
|
46
|
+
const rowCount = await writeSqliteImage(db, input);
|
|
47
|
+
return { bytes: db.serialize(), rowCount };
|
|
48
48
|
} finally {
|
|
49
49
|
db.close();
|
|
50
50
|
}
|