@syncular/server 0.74.0 → 0.76.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin.d.ts +6 -3
- package/dist/admin.js +10 -7
- package/dist/blob-handlers.js +2 -0
- package/dist/bucket-segment-store.d.ts +4 -3
- package/dist/bucket-segment-store.js +4 -2
- package/dist/context.d.ts +29 -1
- package/dist/context.js +23 -2
- package/dist/handler.js +7 -2
- package/dist/index.d.ts +1 -1
- package/dist/lease-store.d.ts +10 -3
- package/dist/lease-store.js +0 -0
- package/dist/operations.js +2 -0
- package/dist/pull.d.ts +9 -4
- package/dist/pull.js +129 -76
- package/dist/realtime.d.ts +14 -1
- package/dist/realtime.js +14 -3
- package/dist/segment-download.js +1 -0
- package/dist/segment-store.d.ts +6 -4
- package/dist/segment-store.js +4 -2
- package/dist/signed-url.d.ts +15 -0
- package/dist/signed-url.js +7 -2
- package/dist/sqlite-bun.js +2 -2
- package/dist/sqlite-image.d.ts +18 -3
- package/dist/sqlite-image.js +27 -5
- package/dist/sqlite-lease-store.d.ts +1 -1
- package/dist/sqlite-lease-store.js +19 -6
- package/dist/sqlite-node.js +3 -2
- package/dist/sqlite-segment-store.js +7 -7
- package/package.json +2 -2
- package/src/admin.ts +11 -5
- package/src/blob-handlers.ts +2 -0
- package/src/bucket-segment-store.ts +8 -5
- package/src/context.ts +38 -2
- package/src/handler.ts +9 -1
- package/src/index.ts +1 -0
- package/src/lease-store.ts +0 -0
- package/src/operations.ts +2 -0
- package/src/pull.ts +200 -94
- package/src/realtime.ts +26 -12
- package/src/segment-download.ts +1 -0
- package/src/segment-store.ts +10 -6
- package/src/signed-url.ts +22 -2
- package/src/sqlite-bun.ts +2 -2
- package/src/sqlite-image.ts +42 -7
- package/src/sqlite-lease-store.ts +22 -4
- package/src/sqlite-node.ts +3 -2
- package/src/sqlite-segment-store.ts +29 -7
package/src/pull.ts
CHANGED
|
@@ -214,6 +214,8 @@ interface BootstrapToken {
|
|
|
214
214
|
tables: string[];
|
|
215
215
|
tableIndex: number;
|
|
216
216
|
rowCursor: string | null;
|
|
217
|
+
/** Present when the table at `tableIndex` continues on the image lane. */
|
|
218
|
+
image?: true;
|
|
217
219
|
}
|
|
218
220
|
|
|
219
221
|
/**
|
|
@@ -242,7 +244,9 @@ function parseBootstrapToken(
|
|
|
242
244
|
parsed.tableIndex === undefined ||
|
|
243
245
|
parsed.tableIndex < 0 ||
|
|
244
246
|
parsed.tableIndex >= tables.length ||
|
|
245
|
-
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')
|
|
247
|
+
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string') ||
|
|
248
|
+
(parsed.image !== undefined &&
|
|
249
|
+
(parsed.image !== true || parsed.rowCursor === null))
|
|
246
250
|
) {
|
|
247
251
|
return undefined;
|
|
248
252
|
}
|
|
@@ -251,6 +255,7 @@ function parseBootstrapToken(
|
|
|
251
255
|
tables: parsed.tables as string[],
|
|
252
256
|
tableIndex: parsed.tableIndex,
|
|
253
257
|
rowCursor: parsed.rowCursor ?? null,
|
|
258
|
+
...(parsed.image === true ? { image: true as const } : {}),
|
|
254
259
|
};
|
|
255
260
|
} catch {
|
|
256
261
|
return undefined;
|
|
@@ -280,6 +285,7 @@ async function signedUrlFields(
|
|
|
280
285
|
partition: ctx.partition,
|
|
281
286
|
scopeDigest: digest,
|
|
282
287
|
nowMs: now,
|
|
288
|
+
...(ctx.context !== undefined ? { context: ctx.context } : {}),
|
|
283
289
|
});
|
|
284
290
|
}
|
|
285
291
|
|
|
@@ -301,8 +307,12 @@ function segmentRefFrame(
|
|
|
301
307
|
// under different scopes (§5.1), and the record's primary digest is not
|
|
302
308
|
// necessarily the caller's (§5.5).
|
|
303
309
|
scopeDigest,
|
|
304
|
-
|
|
305
|
-
|
|
310
|
+
// §5.3: an image descriptor carries no cursors; the resume token holds
|
|
311
|
+
// the position of a table's next range.
|
|
312
|
+
...(record.mediaType === 'rows' && record.rowCursor !== null
|
|
313
|
+
? { rowCursor: record.rowCursor }
|
|
314
|
+
: {}),
|
|
315
|
+
...(record.mediaType === 'rows' && record.nextRowCursor !== null
|
|
306
316
|
? { nextRowCursor: record.nextRowCursor }
|
|
307
317
|
: {}),
|
|
308
318
|
...(extra.url !== undefined ? { url: extra.url } : {}),
|
|
@@ -319,8 +329,8 @@ interface BootstrapPair {
|
|
|
319
329
|
}
|
|
320
330
|
|
|
321
331
|
/**
|
|
322
|
-
* §5.3: the sqlite-image lane
|
|
323
|
-
*
|
|
332
|
+
* §5.3: the sqlite-image lane, chosen at the start of a table or continued
|
|
333
|
+
* after a range image (a table on the rows lane never switches to images),
|
|
324
334
|
* and only when the client advertised bit 2. §5.11: a table with any
|
|
325
335
|
* encrypted column is image-INELIGIBLE — an image copies ciphertext
|
|
326
336
|
* wholesale with no per-row decrypt pass, so it MUST be served on the rows
|
|
@@ -330,14 +340,52 @@ function sqliteLaneEligible(
|
|
|
330
340
|
limits: PullLimits,
|
|
331
341
|
table: CompiledTable,
|
|
332
342
|
startRowCursor: string | null,
|
|
343
|
+
imageLane: boolean,
|
|
333
344
|
): boolean {
|
|
334
345
|
return (
|
|
335
346
|
(limits.accept & ACCEPT_SQLITE) !== 0 &&
|
|
336
|
-
startRowCursor === null &&
|
|
347
|
+
(startRowCursor === null || imageLane) &&
|
|
337
348
|
table.encryptedColumnIndices.length === 0
|
|
338
349
|
);
|
|
339
350
|
}
|
|
340
351
|
|
|
352
|
+
/**
|
|
353
|
+
* §5.3 reuse lookup for the range image that starts after `rowCursor`. Only
|
|
354
|
+
* a position with more than one rows page left can match a stored image, so
|
|
355
|
+
* a smaller remainder skips the lookup. A stored image past the bound (built
|
|
356
|
+
* under a larger one) does not match unless it holds a single row.
|
|
357
|
+
*/
|
|
358
|
+
async function findImage(
|
|
359
|
+
ctx: SyncRequestContext,
|
|
360
|
+
schema: CompiledSchema,
|
|
361
|
+
limits: PullLimits,
|
|
362
|
+
pair: BootstrapPair,
|
|
363
|
+
scopePin: number,
|
|
364
|
+
rowCursor: string | null,
|
|
365
|
+
probe: readonly StoredRow[],
|
|
366
|
+
logEpoch: string,
|
|
367
|
+
): Promise<SegmentRecord | undefined> {
|
|
368
|
+
if (probe.length <= limits.limitSnapshotRows) return undefined;
|
|
369
|
+
const stored = await ctx.segments.find(
|
|
370
|
+
{
|
|
371
|
+
partition: ctx.partition,
|
|
372
|
+
logEpoch,
|
|
373
|
+
table: pair.table.name,
|
|
374
|
+
schemaVersion: schema.version,
|
|
375
|
+
mediaType: 'sqlite',
|
|
376
|
+
scopeDigest: await scopeDigest(pair.effective),
|
|
377
|
+
asOfCommitSeq: scopePin,
|
|
378
|
+
rowCursor,
|
|
379
|
+
},
|
|
380
|
+
clockOf(ctx)(),
|
|
381
|
+
);
|
|
382
|
+
return stored !== undefined &&
|
|
383
|
+
(stored.byteLength <= limitsOf(ctx).maxImageSegmentBytes ||
|
|
384
|
+
stored.rowCount === 1)
|
|
385
|
+
? stored
|
|
386
|
+
: undefined;
|
|
387
|
+
}
|
|
388
|
+
|
|
341
389
|
/** The reads a table's bootstrap needs before its first frame (§4.7, §5.3). */
|
|
342
390
|
interface TableStart {
|
|
343
391
|
/**
|
|
@@ -359,6 +407,7 @@ async function readTableStart(
|
|
|
359
407
|
unit: UnitPlan,
|
|
360
408
|
pair: BootstrapPair,
|
|
361
409
|
startRowCursor: string | null,
|
|
410
|
+
imageLane: boolean,
|
|
362
411
|
maxSeq: number,
|
|
363
412
|
horizonSeq: number,
|
|
364
413
|
logEpoch: string,
|
|
@@ -369,7 +418,7 @@ async function readTableStart(
|
|
|
369
418
|
afterRowId: startRowCursor,
|
|
370
419
|
limit: limits.limitSnapshotRows + 1,
|
|
371
420
|
});
|
|
372
|
-
if (!sqliteLaneEligible(limits, pair.table, startRowCursor)) {
|
|
421
|
+
if (!sqliteLaneEligible(limits, pair.table, startRowCursor, imageLane)) {
|
|
373
422
|
return { firstPage: await scan };
|
|
374
423
|
}
|
|
375
424
|
// §4.7: a sqlite-lane table pins at the newest commit carrying all of the
|
|
@@ -384,33 +433,25 @@ async function readTableStart(
|
|
|
384
433
|
}),
|
|
385
434
|
]);
|
|
386
435
|
const scopePin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
table: pair.table.name,
|
|
398
|
-
schemaVersion: schema.version,
|
|
399
|
-
mediaType: 'sqlite',
|
|
400
|
-
scopeDigest: await scopeDigest(pair.effective),
|
|
401
|
-
asOfCommitSeq: scopePin,
|
|
402
|
-
},
|
|
403
|
-
clockOf(ctx)(),
|
|
404
|
-
);
|
|
436
|
+
const stored = await findImage(
|
|
437
|
+
ctx,
|
|
438
|
+
schema,
|
|
439
|
+
limits,
|
|
440
|
+
pair,
|
|
441
|
+
scopePin,
|
|
442
|
+
startRowCursor,
|
|
443
|
+
firstPage,
|
|
444
|
+
logEpoch,
|
|
445
|
+
);
|
|
405
446
|
return { firstPage, image: { scopePin, stored } };
|
|
406
447
|
}
|
|
407
448
|
|
|
408
449
|
/**
|
|
409
|
-
* §5.3 sqlite-image lane:
|
|
410
|
-
*
|
|
411
|
-
* schemaVersion, scope digest, scope
|
|
412
|
-
*
|
|
413
|
-
* takes over).
|
|
450
|
+
* §5.3 sqlite-image lane: one range image starting after `rowCursor`, only
|
|
451
|
+
* when more than one rows page remains there. Reuses an unexpired stored
|
|
452
|
+
* image for the same (partition, table, schemaVersion, scope digest, scope
|
|
453
|
+
* pin, range start) instead of rebuilding — the bootstrap-storm rule.
|
|
454
|
+
* Returns undefined when no image is served (the rows lane takes over).
|
|
414
455
|
*/
|
|
415
456
|
async function* sqliteImageSegment(
|
|
416
457
|
ctx: SyncRequestContext,
|
|
@@ -418,12 +459,13 @@ async function* sqliteImageSegment(
|
|
|
418
459
|
limits: PullLimits,
|
|
419
460
|
pair: BootstrapPair,
|
|
420
461
|
scopePin: number,
|
|
462
|
+
rowCursor: string | null,
|
|
421
463
|
digest: string,
|
|
422
464
|
trace: PullSectionTrace | undefined,
|
|
423
465
|
logEpoch: string,
|
|
424
466
|
existing: SegmentRecord | undefined,
|
|
425
467
|
probe: StoredRow[],
|
|
426
|
-
): AsyncGenerator<ResponseFrame,
|
|
468
|
+
): AsyncGenerator<ResponseFrame, SegmentRecord | undefined> {
|
|
427
469
|
const { storage, segments, partition } = ctx;
|
|
428
470
|
const now = clockOf(ctx)();
|
|
429
471
|
if (existing !== undefined) {
|
|
@@ -439,13 +481,13 @@ async function* sqliteImageSegment(
|
|
|
439
481
|
digest,
|
|
440
482
|
await signedUrlFields(ctx, limits, existing.segmentId, digest, now),
|
|
441
483
|
);
|
|
442
|
-
return
|
|
484
|
+
return existing;
|
|
443
485
|
}
|
|
444
|
-
if (probe.length <= limits.limitSnapshotRows) return
|
|
486
|
+
if (probe.length <= limits.limitSnapshotRows) return undefined;
|
|
445
487
|
// §5.3: hosts opt into image construction with a runtime-specific builder.
|
|
446
488
|
// Without one, matching stored images remain usable and cold pulls use rows.
|
|
447
489
|
const buildImage = ctx.sqliteImageBuilder;
|
|
448
|
-
if (buildImage === undefined) return
|
|
490
|
+
if (buildImage === undefined) return undefined;
|
|
449
491
|
let stores = imageBuilds.get(storage);
|
|
450
492
|
if (stores === undefined) {
|
|
451
493
|
stores = new WeakMap();
|
|
@@ -464,6 +506,7 @@ async function* sqliteImageSegment(
|
|
|
464
506
|
mediaType: 'sqlite' as const,
|
|
465
507
|
scopeDigest: digest,
|
|
466
508
|
asOfCommitSeq: scopePin,
|
|
509
|
+
rowCursor,
|
|
467
510
|
};
|
|
468
511
|
const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
|
|
469
512
|
const key = JSON.stringify(identity);
|
|
@@ -472,43 +515,65 @@ async function* sqliteImageSegment(
|
|
|
472
515
|
if (building === undefined) {
|
|
473
516
|
building = (async () => {
|
|
474
517
|
// Another request can finish while this one's eligibility probe awaits.
|
|
475
|
-
const cached = await
|
|
518
|
+
const cached = await findImage(
|
|
519
|
+
ctx,
|
|
520
|
+
schema,
|
|
521
|
+
limits,
|
|
522
|
+
pair,
|
|
523
|
+
scopePin,
|
|
524
|
+
rowCursor,
|
|
525
|
+
probe,
|
|
526
|
+
logEpoch,
|
|
527
|
+
);
|
|
476
528
|
if (cached !== undefined) return cached;
|
|
477
529
|
builtHere = true;
|
|
478
|
-
|
|
479
|
-
|
|
530
|
+
// The builder stops at the size bound inside the latest batch; these
|
|
531
|
+
// locate the last row it took and whether rows follow it.
|
|
532
|
+
let yielded = 0;
|
|
533
|
+
let batch: readonly StoredRow[] = [];
|
|
534
|
+
let beforeBatch = rowCursor;
|
|
535
|
+
let exhausted = false;
|
|
536
|
+
const image = await buildImage({
|
|
480
537
|
table: pair.table,
|
|
481
538
|
schemaVersion: schema.version,
|
|
482
539
|
asOfCommitSeq: scopePin,
|
|
483
540
|
scopeDigest: digest,
|
|
541
|
+
maxBytes: limitsOf(ctx).maxImageSegmentBytes,
|
|
484
542
|
rowBatches: (async function* () {
|
|
485
|
-
|
|
486
|
-
yield probe.map((row) => ({
|
|
487
|
-
...row,
|
|
488
|
-
payload: projectRowPayload(current, pair.table, row.payload),
|
|
489
|
-
}));
|
|
490
|
-
let afterRowId = probe[probe.length - 1]!.rowId;
|
|
543
|
+
let rows: readonly StoredRow[] = probe;
|
|
491
544
|
for (;;) {
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
afterRowId,
|
|
496
|
-
limit: 5_000,
|
|
497
|
-
});
|
|
498
|
-
rowCount += rows.length;
|
|
545
|
+
beforeBatch = batch[batch.length - 1]?.rowId ?? beforeBatch;
|
|
546
|
+
batch = rows;
|
|
547
|
+
yielded += rows.length;
|
|
499
548
|
yield rows.map((row) => ({
|
|
500
549
|
...row,
|
|
501
550
|
payload: projectRowPayload(current, pair.table, row.payload),
|
|
502
551
|
}));
|
|
503
552
|
const last = rows[rows.length - 1];
|
|
504
|
-
if (rows.length < 5_000
|
|
505
|
-
|
|
553
|
+
if (last === undefined || (rows !== probe && rows.length < 5_000))
|
|
554
|
+
break;
|
|
555
|
+
rows = await storage.scanRows(partition, {
|
|
556
|
+
table: pair.table.name,
|
|
557
|
+
scopeFilter: pair.effective,
|
|
558
|
+
afterRowId: last.rowId,
|
|
559
|
+
limit: 5_000,
|
|
560
|
+
});
|
|
506
561
|
}
|
|
562
|
+
exhausted = true;
|
|
507
563
|
})(),
|
|
508
564
|
});
|
|
565
|
+
const index = image.rowCount - (yielded - batch.length) - 1;
|
|
566
|
+
if (image.rowCount < 1 || image.rowCount > yielded || index < -1) {
|
|
567
|
+
throw new Error('sqliteImageBuilder consumed rows out of order');
|
|
568
|
+
}
|
|
569
|
+
const last = index < 0 ? beforeBatch : batch[index]!.rowId;
|
|
509
570
|
return segments.put(
|
|
510
|
-
{
|
|
511
|
-
|
|
571
|
+
{
|
|
572
|
+
...identity,
|
|
573
|
+
rowCount: image.rowCount,
|
|
574
|
+
nextRowCursor: image.rowCount < yielded || !exhausted ? last : null,
|
|
575
|
+
},
|
|
576
|
+
image.bytes,
|
|
512
577
|
clockOf(ctx)(),
|
|
513
578
|
);
|
|
514
579
|
})();
|
|
@@ -538,12 +603,14 @@ async function* sqliteImageSegment(
|
|
|
538
603
|
clockOf(ctx)(),
|
|
539
604
|
),
|
|
540
605
|
);
|
|
541
|
-
return
|
|
606
|
+
return record;
|
|
542
607
|
}
|
|
543
608
|
|
|
544
609
|
/**
|
|
545
|
-
* One table's segments within the page budget
|
|
546
|
-
*
|
|
610
|
+
* One table's segments within the page budget: range images while the table
|
|
611
|
+
* is on the image lane, then rows pages. Returns the pages it used, the row
|
|
612
|
+
* cursor to resume at (`null` once the table is exhausted) and whether that
|
|
613
|
+
* position is on the image lane.
|
|
547
614
|
*/
|
|
548
615
|
async function* tableSegments(
|
|
549
616
|
ctx: SyncRequestContext,
|
|
@@ -556,28 +623,17 @@ async function* tableSegments(
|
|
|
556
623
|
pageBudget: number,
|
|
557
624
|
trace: PullSectionTrace | undefined,
|
|
558
625
|
logEpoch: string,
|
|
559
|
-
): AsyncGenerator<
|
|
626
|
+
): AsyncGenerator<
|
|
627
|
+
ResponseFrame,
|
|
628
|
+
{ pages: number; rowCursor: string | null; image: boolean }
|
|
629
|
+
> {
|
|
560
630
|
const { storage, segments, partition } = ctx;
|
|
561
631
|
const serverLimits = limitsOf(ctx);
|
|
562
632
|
const digest = await scopeDigest(pair.effective);
|
|
563
633
|
const now = clockOf(ctx)();
|
|
564
|
-
if (start.image !== undefined) {
|
|
565
|
-
const imaged = yield* sqliteImageSegment(
|
|
566
|
-
ctx,
|
|
567
|
-
schema,
|
|
568
|
-
limits,
|
|
569
|
-
pair,
|
|
570
|
-
start.image.scopePin,
|
|
571
|
-
digest,
|
|
572
|
-
trace,
|
|
573
|
-
logEpoch,
|
|
574
|
-
start.image.stored,
|
|
575
|
-
start.firstPage,
|
|
576
|
-
);
|
|
577
|
-
if (imaged) return { pages: 1, rowCursor: null };
|
|
578
|
-
}
|
|
579
634
|
const current = compileSchema(ctx.schema).tables.get(pair.table.name)!;
|
|
580
635
|
let rowCursor = startRowCursor;
|
|
636
|
+
let image = start.image;
|
|
581
637
|
for (let page = 0; page < pageBudget; page++) {
|
|
582
638
|
const scanned =
|
|
583
639
|
page === 0
|
|
@@ -588,6 +644,41 @@ async function* tableSegments(
|
|
|
588
644
|
afterRowId: rowCursor,
|
|
589
645
|
limit: limits.limitSnapshotRows + 1,
|
|
590
646
|
});
|
|
647
|
+
if (image !== undefined) {
|
|
648
|
+
// §5.3: every range of the table shares the scope pin computed at this
|
|
649
|
+
// section's table start.
|
|
650
|
+
const record = yield* sqliteImageSegment(
|
|
651
|
+
ctx,
|
|
652
|
+
schema,
|
|
653
|
+
limits,
|
|
654
|
+
pair,
|
|
655
|
+
image.scopePin,
|
|
656
|
+
rowCursor,
|
|
657
|
+
digest,
|
|
658
|
+
trace,
|
|
659
|
+
logEpoch,
|
|
660
|
+
page === 0
|
|
661
|
+
? image.stored
|
|
662
|
+
: await findImage(
|
|
663
|
+
ctx,
|
|
664
|
+
schema,
|
|
665
|
+
limits,
|
|
666
|
+
pair,
|
|
667
|
+
image.scopePin,
|
|
668
|
+
rowCursor,
|
|
669
|
+
scanned,
|
|
670
|
+
logEpoch,
|
|
671
|
+
),
|
|
672
|
+
scanned,
|
|
673
|
+
);
|
|
674
|
+
if (record !== undefined) {
|
|
675
|
+
if (record.nextRowCursor === null)
|
|
676
|
+
return { pages: page + 1, rowCursor: null, image: false };
|
|
677
|
+
rowCursor = record.nextRowCursor;
|
|
678
|
+
continue;
|
|
679
|
+
}
|
|
680
|
+
image = undefined;
|
|
681
|
+
}
|
|
591
682
|
const pageRows = scanned.slice(0, limits.limitSnapshotRows);
|
|
592
683
|
const hasMore = scanned.length > limits.limitSnapshotRows;
|
|
593
684
|
// §5.2: every row record carries the row's current server_version.
|
|
@@ -649,10 +740,10 @@ async function* tableSegments(
|
|
|
649
740
|
await signedUrlFields(ctx, limits, record.segmentId, digest, now),
|
|
650
741
|
);
|
|
651
742
|
}
|
|
652
|
-
if (!hasMore) return { pages: page + 1, rowCursor: null };
|
|
743
|
+
if (!hasMore) return { pages: page + 1, rowCursor: null, image: false };
|
|
653
744
|
rowCursor = nextRowCursor;
|
|
654
745
|
}
|
|
655
|
-
return { pages: pageBudget, rowCursor };
|
|
746
|
+
return { pages: pageBudget, rowCursor, image: image !== undefined };
|
|
656
747
|
}
|
|
657
748
|
|
|
658
749
|
/** A bootstrapping unit's position and its table starts, read ahead (§4.7). */
|
|
@@ -662,6 +753,8 @@ export interface BootstrapPlan {
|
|
|
662
753
|
readonly pin: number;
|
|
663
754
|
readonly tableIndex: number;
|
|
664
755
|
readonly rowCursor: string | null;
|
|
756
|
+
/** Whether the position continues the table's range images (§5.3). */
|
|
757
|
+
readonly image: boolean;
|
|
665
758
|
/** Table index → its start reads. */
|
|
666
759
|
readonly starts: ReadonlyMap<number, Promise<TableStart>>;
|
|
667
760
|
}
|
|
@@ -692,6 +785,7 @@ export function planBootstrap(
|
|
|
692
785
|
: undefined;
|
|
693
786
|
const tableIndex = resume?.tableIndex ?? 0;
|
|
694
787
|
const rowCursor = resume?.rowCursor ?? null;
|
|
788
|
+
const image = resume?.image === true;
|
|
695
789
|
const starts = new Map<number, Promise<TableStart>>();
|
|
696
790
|
for (
|
|
697
791
|
let index = tableIndex;
|
|
@@ -708,6 +802,7 @@ export function planBootstrap(
|
|
|
708
802
|
unit,
|
|
709
803
|
{ table, effective },
|
|
710
804
|
index === tableIndex ? rowCursor : null,
|
|
805
|
+
index === tableIndex && image,
|
|
711
806
|
maxSeq,
|
|
712
807
|
horizonSeq,
|
|
713
808
|
logEpoch,
|
|
@@ -721,6 +816,7 @@ export function planBootstrap(
|
|
|
721
816
|
pin: resume?.asOfCommitSeq ?? maxSeq,
|
|
722
817
|
tableIndex,
|
|
723
818
|
rowCursor,
|
|
819
|
+
image,
|
|
724
820
|
starts,
|
|
725
821
|
};
|
|
726
822
|
}
|
|
@@ -740,7 +836,7 @@ export async function* bootstrapSegments(
|
|
|
740
836
|
logEpoch: string,
|
|
741
837
|
): AsyncGenerator<ResponseFrame, string | undefined> {
|
|
742
838
|
const { unit, pin } = plan;
|
|
743
|
-
let { tableIndex, rowCursor } = plan;
|
|
839
|
+
let { tableIndex, rowCursor, image } = plan;
|
|
744
840
|
let pages = 0;
|
|
745
841
|
for (;;) {
|
|
746
842
|
while (
|
|
@@ -749,6 +845,7 @@ export async function* bootstrapSegments(
|
|
|
749
845
|
) {
|
|
750
846
|
tableIndex += 1;
|
|
751
847
|
rowCursor = null;
|
|
848
|
+
image = false;
|
|
752
849
|
}
|
|
753
850
|
if (tableIndex === unit.tables.length) return undefined;
|
|
754
851
|
if (pages === limits.maxSnapshotPages) {
|
|
@@ -757,6 +854,7 @@ export async function* bootstrapSegments(
|
|
|
757
854
|
tables: unit.tables.map((table) => table.name),
|
|
758
855
|
tableIndex,
|
|
759
856
|
rowCursor,
|
|
857
|
+
...(image ? { image: true as const } : {}),
|
|
760
858
|
};
|
|
761
859
|
return JSON.stringify(token);
|
|
762
860
|
}
|
|
@@ -775,6 +873,7 @@ export async function* bootstrapSegments(
|
|
|
775
873
|
);
|
|
776
874
|
pages += outcome.pages;
|
|
777
875
|
rowCursor = outcome.rowCursor;
|
|
876
|
+
image = outcome.image;
|
|
778
877
|
if (rowCursor === null) tableIndex += 1;
|
|
779
878
|
}
|
|
780
879
|
}
|
|
@@ -796,18 +895,21 @@ export interface PublishSqliteImageInput {
|
|
|
796
895
|
}
|
|
797
896
|
|
|
798
897
|
export interface PublishedSqliteImage {
|
|
799
|
-
|
|
800
|
-
|
|
898
|
+
/** The table's range images in row order (§5.3). */
|
|
899
|
+
readonly segmentIds: readonly string[];
|
|
900
|
+
/** The §4.7 scope pin the images are stored under. */
|
|
801
901
|
readonly asOfCommitSeq: number;
|
|
802
902
|
readonly scopeDigest: string;
|
|
903
|
+
/** Rows across every range image. */
|
|
803
904
|
readonly rowCount: number;
|
|
905
|
+
/** Bytes across every range image. */
|
|
804
906
|
readonly byteLength: number;
|
|
805
|
-
/** `reused` when an unexpired image for
|
|
907
|
+
/** `reused` when an unexpired image existed for every range's reuse key. */
|
|
806
908
|
readonly origin: 'built' | 'reused';
|
|
807
909
|
}
|
|
808
910
|
|
|
809
911
|
/**
|
|
810
|
-
* Publish the §5.3
|
|
912
|
+
* Publish the §5.3 range images that a pull of (`scopes`, `table`) looks up,
|
|
811
913
|
* from a host with a SQLite engine (a Bun process with the production
|
|
812
914
|
* storage). It runs the pull's own bootstrap path (the §4.7 scope pin, the
|
|
813
915
|
* reuse key, the build), so the stored identity is exactly the one a
|
|
@@ -847,11 +949,12 @@ export async function publishSqliteImage(
|
|
|
847
949
|
partition,
|
|
848
950
|
actorId: 'syncular.image-publisher',
|
|
849
951
|
};
|
|
850
|
-
// One-row pages make every
|
|
952
|
+
// One-row pages make every range above one row image-eligible (§5.3), and
|
|
953
|
+
// an unbounded page budget stores every range under one scope pin.
|
|
851
954
|
const limits: PullLimits = {
|
|
852
955
|
limitCommits: 1,
|
|
853
956
|
limitSnapshotRows: 1,
|
|
854
|
-
maxSnapshotPages:
|
|
957
|
+
maxSnapshotPages: Number.MAX_SAFE_INTEGER,
|
|
855
958
|
accept: ACCEPT_SQLITE,
|
|
856
959
|
};
|
|
857
960
|
const maxSeq = await config.storage.getMaxCommitSeq(partition);
|
|
@@ -867,7 +970,7 @@ export async function publishSqliteImage(
|
|
|
867
970
|
horizonSeq,
|
|
868
971
|
logEpoch,
|
|
869
972
|
);
|
|
870
|
-
|
|
973
|
+
const refs: SegmentRefFrame[] = [];
|
|
871
974
|
for await (const frame of bootstrapSegments(
|
|
872
975
|
ctx,
|
|
873
976
|
schema,
|
|
@@ -877,23 +980,26 @@ export async function publishSqliteImage(
|
|
|
877
980
|
logEpoch,
|
|
878
981
|
)) {
|
|
879
982
|
if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
|
|
880
|
-
|
|
983
|
+
refs.push(frame);
|
|
881
984
|
}
|
|
882
985
|
}
|
|
883
|
-
const
|
|
884
|
-
|
|
885
|
-
);
|
|
886
|
-
if (ref === undefined || image === undefined) {
|
|
986
|
+
const first = refs[0];
|
|
987
|
+
if (first === undefined) {
|
|
887
988
|
throw new Error(
|
|
888
989
|
'publishSqliteImage: the scope holds at most one row, which pulls serve inline',
|
|
889
990
|
);
|
|
890
991
|
}
|
|
891
992
|
return {
|
|
892
|
-
|
|
893
|
-
asOfCommitSeq:
|
|
894
|
-
scopeDigest:
|
|
895
|
-
rowCount: ref.rowCount,
|
|
896
|
-
byteLength: ref.byteLength,
|
|
897
|
-
origin:
|
|
993
|
+
segmentIds: refs.map((ref) => ref.segmentId),
|
|
994
|
+
asOfCommitSeq: first.asOfCommitSeq,
|
|
995
|
+
scopeDigest: first.scopeDigest,
|
|
996
|
+
rowCount: refs.reduce((sum, ref) => sum + ref.rowCount, 0),
|
|
997
|
+
byteLength: refs.reduce((sum, ref) => sum + ref.byteLength, 0),
|
|
998
|
+
origin: trace.segments.every(
|
|
999
|
+
(segment) =>
|
|
1000
|
+
segment.mediaType !== 'sqlite' || segment.origin === 'reused',
|
|
1001
|
+
)
|
|
1002
|
+
? 'reused'
|
|
1003
|
+
: 'built',
|
|
898
1004
|
};
|
|
899
1005
|
}
|
package/src/realtime.ts
CHANGED
|
@@ -101,6 +101,12 @@ export interface RealtimeConnectOptions {
|
|
|
101
101
|
readonly partition: string;
|
|
102
102
|
readonly actorId: string;
|
|
103
103
|
readonly clientId: string;
|
|
104
|
+
/**
|
|
105
|
+
* The authentication context the upgrade authenticated (§1.1, §8.1). The
|
|
106
|
+
* connection resolves scopes under it for its lifetime: registrations,
|
|
107
|
+
* presence grants and socket rounds.
|
|
108
|
+
*/
|
|
109
|
+
readonly context?: string;
|
|
104
110
|
/**
|
|
105
111
|
* Socket send: JSON control messages as text, tagged binary otherwise
|
|
106
112
|
* (§8.7). A host MAY return a promise that resolves when the socket
|
|
@@ -210,6 +216,7 @@ export class RealtimeSession {
|
|
|
210
216
|
readonly partition: string;
|
|
211
217
|
readonly actorId: string;
|
|
212
218
|
readonly clientId: string;
|
|
219
|
+
readonly context: string | undefined;
|
|
213
220
|
readonly logEpoch: string;
|
|
214
221
|
/** Response/delta layout selected by the most recent socket round. */
|
|
215
222
|
wireVersion: number;
|
|
@@ -291,6 +298,7 @@ export class RealtimeSession {
|
|
|
291
298
|
this.partition = options.partition;
|
|
292
299
|
this.actorId = options.actorId;
|
|
293
300
|
this.clientId = options.clientId;
|
|
301
|
+
this.context = options.context;
|
|
294
302
|
this.logEpoch = logEpoch;
|
|
295
303
|
this.wireVersion = wireVersion;
|
|
296
304
|
this.cursor = cursor;
|
|
@@ -825,9 +833,7 @@ export class RealtimeSession {
|
|
|
825
833
|
let registrations: Registration[];
|
|
826
834
|
try {
|
|
827
835
|
registrations = await this.#hub.loadRegistrations(
|
|
828
|
-
this
|
|
829
|
-
this.actorId,
|
|
830
|
-
this.clientId,
|
|
836
|
+
this,
|
|
831
837
|
await this.#storage.getClientRecord(this.partition, this.clientId),
|
|
832
838
|
);
|
|
833
839
|
} catch {
|
|
@@ -1198,17 +1204,22 @@ export class RealtimeHub {
|
|
|
1198
1204
|
* registrations (§8.1 at upgrade; §8.7 at socket-round end).
|
|
1199
1205
|
*/
|
|
1200
1206
|
async loadRegistrations(
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1207
|
+
identity: {
|
|
1208
|
+
readonly partition: string;
|
|
1209
|
+
readonly actorId: string;
|
|
1210
|
+
readonly clientId: string;
|
|
1211
|
+
readonly context?: string | undefined;
|
|
1212
|
+
},
|
|
1204
1213
|
record: ClientRecord | undefined,
|
|
1205
1214
|
): Promise<Registration[]> {
|
|
1215
|
+
const { partition, actorId, clientId, context } = identity;
|
|
1206
1216
|
let resolved: ResolvedScopes;
|
|
1207
1217
|
try {
|
|
1208
1218
|
const allowed = await this.#config.resolveScopes({
|
|
1209
1219
|
partition,
|
|
1210
1220
|
actorId,
|
|
1211
1221
|
clientId,
|
|
1222
|
+
...(context !== undefined ? { context } : {}),
|
|
1212
1223
|
});
|
|
1213
1224
|
// §7.3.3: delta-fanout registration needs live scopes — an outage
|
|
1214
1225
|
// registers nothing (the client's next socket round, §8.7, is
|
|
@@ -1271,6 +1282,7 @@ export class RealtimeHub {
|
|
|
1271
1282
|
requestContextFor(identity: {
|
|
1272
1283
|
readonly partition: string;
|
|
1273
1284
|
readonly actorId: string;
|
|
1285
|
+
readonly context?: string | undefined;
|
|
1274
1286
|
}): SyncRequestContext {
|
|
1275
1287
|
const segments = this.#config.segments;
|
|
1276
1288
|
if (segments === undefined) {
|
|
@@ -1288,6 +1300,7 @@ export class RealtimeHub {
|
|
|
1288
1300
|
segments,
|
|
1289
1301
|
partition: identity.partition,
|
|
1290
1302
|
actorId: identity.actorId,
|
|
1303
|
+
...(identity.context !== undefined ? { context: identity.context } : {}),
|
|
1291
1304
|
realtime: this,
|
|
1292
1305
|
};
|
|
1293
1306
|
}
|
|
@@ -1345,6 +1358,9 @@ export class RealtimeHub {
|
|
|
1345
1358
|
...this.#config,
|
|
1346
1359
|
partition: options.partition,
|
|
1347
1360
|
actorId: options.actorId,
|
|
1361
|
+
...(options.context !== undefined
|
|
1362
|
+
? { context: options.context }
|
|
1363
|
+
: {}),
|
|
1348
1364
|
});
|
|
1349
1365
|
shared = {
|
|
1350
1366
|
logEpoch,
|
|
@@ -1389,6 +1405,9 @@ export class RealtimeHub {
|
|
|
1389
1405
|
...this.#config,
|
|
1390
1406
|
partition: options.partition,
|
|
1391
1407
|
actorId: options.actorId,
|
|
1408
|
+
...(options.context !== undefined
|
|
1409
|
+
? { context: options.context }
|
|
1410
|
+
: {}),
|
|
1392
1411
|
})
|
|
1393
1412
|
).logEpoch;
|
|
1394
1413
|
if (options.clientId.startsWith(REMOTE_COMMAND_CLIENT_ID_PREFIX)) {
|
|
@@ -1407,12 +1426,7 @@ export class RealtimeHub {
|
|
|
1407
1426
|
'clientId is bound to a different actor in this partition (§1.5)',
|
|
1408
1427
|
);
|
|
1409
1428
|
}
|
|
1410
|
-
const registrations = await this.loadRegistrations(
|
|
1411
|
-
options.partition,
|
|
1412
|
-
options.actorId,
|
|
1413
|
-
options.clientId,
|
|
1414
|
-
record,
|
|
1415
|
-
);
|
|
1429
|
+
const registrations = await this.loadRegistrations(options, record);
|
|
1416
1430
|
const latestSeq =
|
|
1417
1431
|
restored?.latestSeq ?? (await storage.getMaxCommitSeq(options.partition));
|
|
1418
1432
|
const cursor = record?.cursor ?? -1;
|
package/src/segment-download.ts
CHANGED
|
@@ -249,6 +249,7 @@ async function authorizePublication(
|
|
|
249
249
|
const allowed = await ctx.resolveScopes({
|
|
250
250
|
partition: ctx.partition,
|
|
251
251
|
actorId: ctx.actorId,
|
|
252
|
+
...(ctx.context !== undefined ? { context: ctx.context } : {}),
|
|
252
253
|
});
|
|
253
254
|
// §7.3.3: leases authorize sync rounds, never downloads — an outage
|
|
254
255
|
// signal here denies (re-authorization needs live scopes, §5.5).
|