@syncular/server 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -22
- package/dist/admin.js +8 -4
- package/dist/blob-handlers.d.ts +25 -31
- package/dist/blob-handlers.js +63 -41
- package/dist/blob-store.d.ts +42 -24
- package/dist/blob-store.js +0 -0
- package/dist/bucket-blob-store.d.ts +12 -8
- package/dist/bucket-blob-store.js +54 -45
- package/dist/bucket-segment-store.js +18 -16
- package/dist/content-encoding.d.ts +5 -2
- package/dist/content-encoding.js +42 -13
- package/dist/context.d.ts +16 -0
- package/dist/context.js +21 -1
- package/dist/d1-storage.d.ts +14 -98
- package/dist/d1-storage.js +223 -1111
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +1 -1
- package/dist/handler.js +135 -103
- package/dist/local-authority.d.ts +2 -0
- package/dist/local-authority.js +17 -3
- package/dist/object-bucket.d.ts +2 -0
- package/dist/operations-realtime.js +69 -50
- package/dist/pg-executor-pglite.d.ts +5 -6
- package/dist/pg-executor-pglite.js +22 -14
- package/dist/pg-executor.d.ts +4 -1
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +182 -105
- package/dist/pull.d.ts +28 -20
- package/dist/pull.js +85 -87
- package/dist/push.js +272 -147
- package/dist/realtime.d.ts +68 -28
- package/dist/realtime.js +297 -136
- package/dist/relational-rows.d.ts +27 -15
- package/dist/relational-rows.js +38 -22
- package/dist/s3-blob-store.d.ts +11 -12
- package/dist/s3-blob-store.js +20 -13
- package/dist/s3-object-bucket.d.ts +8 -1
- package/dist/s3-object-bucket.js +16 -3
- package/dist/schema.d.ts +14 -0
- package/dist/schema.js +23 -0
- package/dist/scopes.d.ts +6 -2
- package/dist/scopes.js +22 -4
- package/dist/seed.js +1 -2
- package/dist/segment-download.js +18 -4
- package/dist/signed-url.d.ts +2 -2
- package/dist/signed-url.js +9 -2
- package/dist/sigv4.d.ts +8 -2
- package/dist/sigv4.js +8 -4
- package/dist/sqlite-blob-store.d.ts +1 -1
- package/dist/sqlite-blob-store.js +23 -22
- package/dist/sqlite-dialect.d.ts +150 -47
- package/dist/sqlite-dialect.js +987 -88
- package/dist/sqlite-lease-store.js +13 -18
- package/dist/sqlite-storage.d.ts +9 -62
- package/dist/sqlite-storage.js +293 -1117
- package/dist/storage.d.ts +32 -12
- package/package.json +2 -2
- package/src/admin.ts +16 -2
- package/src/blob-handlers.ts +91 -72
- package/src/blob-store.ts +0 -0
- package/src/bucket-blob-store.ts +75 -48
- package/src/bucket-segment-store.ts +20 -15
- package/src/content-encoding.ts +48 -16
- package/src/context.ts +32 -1
- package/src/d1-storage.ts +326 -1692
- package/src/errors.ts +8 -0
- package/src/events.ts +1 -1
- package/src/handler.ts +178 -127
- package/src/local-authority.ts +17 -3
- package/src/object-bucket.ts +2 -0
- package/src/operations-realtime.ts +85 -61
- package/src/pg-executor-pglite.ts +27 -23
- package/src/pg-executor.ts +5 -1
- package/src/postgres-storage.ts +231 -138
- package/src/pull.ts +133 -147
- package/src/push.ts +356 -205
- package/src/realtime.ts +380 -179
- package/src/relational-rows.ts +54 -27
- package/src/s3-blob-store.ts +23 -13
- package/src/s3-object-bucket.ts +20 -2
- package/src/schema.ts +31 -0
- package/src/scopes.ts +23 -3
- package/src/seed.ts +1 -2
- package/src/segment-download.ts +25 -4
- package/src/signed-url.ts +10 -4
- package/src/sigv4.ts +17 -4
- package/src/sqlite-blob-store.ts +27 -26
- package/src/sqlite-dialect.ts +1607 -114
- package/src/sqlite-lease-store.ts +12 -23
- package/src/sqlite-storage.ts +360 -1811
- package/src/storage.ts +33 -12
package/src/pull.ts
CHANGED
|
@@ -88,19 +88,31 @@ interface BootstrapToken {
|
|
|
88
88
|
rowCursor: string | null;
|
|
89
89
|
}
|
|
90
90
|
|
|
91
|
+
/**
|
|
92
|
+
* §4.7: a usable resume token for `table`, or `undefined`. A token whose pin
|
|
93
|
+
* is not a commit sequence of this log (`0 ≤ asOfCommitSeq ≤ maxSeq`) or
|
|
94
|
+
* whose position does not name one of its tables is unusable.
|
|
95
|
+
*/
|
|
91
96
|
function parseBootstrapToken(
|
|
92
97
|
raw: string | undefined,
|
|
93
98
|
table: string,
|
|
99
|
+
maxSeq: number,
|
|
94
100
|
): BootstrapToken | undefined {
|
|
95
101
|
if (raw === undefined) return undefined;
|
|
96
102
|
try {
|
|
97
103
|
const parsed = JSON.parse(raw) as Partial<BootstrapToken>;
|
|
98
104
|
if (
|
|
99
|
-
|
|
105
|
+
!Number.isSafeInteger(parsed.asOfCommitSeq) ||
|
|
106
|
+
parsed.asOfCommitSeq === undefined ||
|
|
107
|
+
parsed.asOfCommitSeq < 0 ||
|
|
108
|
+
parsed.asOfCommitSeq > maxSeq ||
|
|
100
109
|
!Array.isArray(parsed.tables) ||
|
|
101
|
-
|
|
102
|
-
(parsed.
|
|
103
|
-
parsed.
|
|
110
|
+
parsed.tables[0] !== table ||
|
|
111
|
+
!Number.isSafeInteger(parsed.tableIndex) ||
|
|
112
|
+
parsed.tableIndex === undefined ||
|
|
113
|
+
parsed.tableIndex < 0 ||
|
|
114
|
+
parsed.tableIndex >= parsed.tables.length ||
|
|
115
|
+
(parsed.rowCursor !== null && typeof parsed.rowCursor !== 'string')
|
|
104
116
|
) {
|
|
105
117
|
return undefined;
|
|
106
118
|
}
|
|
@@ -482,41 +494,35 @@ async function* bootstrapSegments(
|
|
|
482
494
|
}
|
|
483
495
|
|
|
484
496
|
/**
|
|
485
|
-
*
|
|
486
|
-
*
|
|
487
|
-
* concurrent page reads (PostgreSQL, D1) answers the first pages of every
|
|
488
|
-
* subscription in one statement, and the horizon re-check runs once per pull.
|
|
497
|
+
* One subscription's section decision (§4.5–§4.7) with the storage reads it
|
|
498
|
+
* needs before its first frame.
|
|
489
499
|
*/
|
|
490
|
-
export
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
}
|
|
509
|
-
|
|
510
|
-
interface PendingSection {
|
|
511
|
-
readonly commits?: Promise<StoredCommit[]>;
|
|
512
|
-
readonly firstPage?: Promise<StoredRow[]>;
|
|
513
|
-
readonly lane?: Promise<{
|
|
514
|
-
readonly pin: number | undefined;
|
|
515
|
-
readonly image: SegmentRecord | undefined;
|
|
516
|
-
}>;
|
|
517
|
-
}
|
|
500
|
+
export type PreparedSection =
|
|
501
|
+
| { readonly kind: 'revoked' }
|
|
502
|
+
/** §4.6: the cursor is behind the horizon. */
|
|
503
|
+
| { readonly kind: 'reset' }
|
|
504
|
+
/** §4.5 window read for an in-range cursor. */
|
|
505
|
+
| { readonly kind: 'incremental'; readonly commits: StoredCommit[] }
|
|
506
|
+
| {
|
|
507
|
+
readonly kind: 'bootstrap';
|
|
508
|
+
/** §4.7 pin: a usable resume token's, the sqlite lane's scope pin, or `maxCommitSeq`. */
|
|
509
|
+
readonly asOf: number;
|
|
510
|
+
readonly startCursor: string | null;
|
|
511
|
+
/**
|
|
512
|
+
* The first snapshot page (`limitSnapshotRows + 1` rows from
|
|
513
|
+
* `startCursor`), which is also the §5.3 image eligibility probe.
|
|
514
|
+
*/
|
|
515
|
+
readonly firstPage: StoredRow[];
|
|
516
|
+
/** §5.3 stored image under the reuse key, read only for an eligible probe. */
|
|
517
|
+
readonly image: SegmentRecord | undefined;
|
|
518
|
+
};
|
|
518
519
|
|
|
519
|
-
/**
|
|
520
|
+
/**
|
|
521
|
+
* Decide the section of every plan and run its pre-frame storage reads. The
|
|
522
|
+
* reads of all plans start before any is awaited, so a storage that batches
|
|
523
|
+
* concurrent page reads (PostgreSQL, D1) answers the first pages of every
|
|
524
|
+
* plan in one statement per table, and the horizon re-check runs once.
|
|
525
|
+
*/
|
|
520
526
|
export async function prepareSections(
|
|
521
527
|
ctx: SyncRequestContext,
|
|
522
528
|
schema: CompiledSchema,
|
|
@@ -526,103 +532,98 @@ export async function prepareSections(
|
|
|
526
532
|
horizonSeq: number,
|
|
527
533
|
logEpoch: string,
|
|
528
534
|
): Promise<PreparedSection[]> {
|
|
529
|
-
const
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
535
|
+
const prepared = await Promise.all(
|
|
536
|
+
plans.map(async (plan): Promise<PreparedSection> => {
|
|
537
|
+
const sub = plan.frame;
|
|
538
|
+
if (plan.status === 'revoked') return { kind: 'revoked' };
|
|
539
|
+
// §4.7: a present resume token keeps the subscription bootstrapping,
|
|
540
|
+
// whether or not it is usable.
|
|
541
|
+
if (
|
|
542
|
+
sub.bootstrapState === undefined &&
|
|
543
|
+
sub.cursor >= 0 &&
|
|
544
|
+
sub.cursor <= maxSeq
|
|
545
|
+
) {
|
|
546
|
+
if (sub.cursor < horizonSeq) return { kind: 'reset' };
|
|
547
|
+
return {
|
|
548
|
+
kind: 'incremental',
|
|
549
|
+
commits: await ctx.storage.readCommitWindow(ctx.partition, {
|
|
550
|
+
table: sub.table,
|
|
551
|
+
scopeFilter: plan.effective,
|
|
552
|
+
afterSeq: sub.cursor,
|
|
553
|
+
throughSeq: maxSeq,
|
|
554
|
+
limitChanges: limits.limitCommits + 1,
|
|
555
|
+
}),
|
|
556
|
+
};
|
|
557
|
+
}
|
|
558
|
+
// §4.7: resume at the token's pin unless the token is unusable or the
|
|
559
|
+
// pin fell behind the horizon; then restart from a fresh pin.
|
|
560
|
+
const token = parseBootstrapToken(sub.bootstrapState, sub.table, maxSeq);
|
|
561
|
+
const resume =
|
|
562
|
+
token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
563
|
+
? token
|
|
564
|
+
: undefined;
|
|
565
|
+
const startCursor = resume?.rowCursor ?? null;
|
|
566
|
+
const scan = ctx.storage.scanRows(ctx.partition, {
|
|
567
|
+
table: plan.table.name,
|
|
568
|
+
scopeFilter: plan.effective,
|
|
569
|
+
afterRowId: startCursor,
|
|
570
|
+
limit: limits.limitSnapshotRows + 1,
|
|
571
|
+
});
|
|
572
|
+
if (!sqliteLaneEligible(limits, plan, startCursor)) {
|
|
573
|
+
return {
|
|
574
|
+
kind: 'bootstrap',
|
|
575
|
+
asOf: resume?.asOfCommitSeq ?? maxSeq,
|
|
576
|
+
startCursor,
|
|
577
|
+
firstPage: await scan,
|
|
578
|
+
image: undefined,
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
// §4.7: a sqlite-lane bootstrap starts at the table start and pins at
|
|
582
|
+
// the newest change in its scope (never below the horizon, never above
|
|
583
|
+
// maxCommitSeq), so a stored image stays the reuse key's match until
|
|
584
|
+
// this scope changes (§5.3). The server computes the pin; a client
|
|
585
|
+
// token never supplies the reuse key.
|
|
586
|
+
const [firstPage, latest] = await Promise.all([
|
|
587
|
+
scan,
|
|
588
|
+
ctx.storage.latestScopedChangeSeq(ctx.partition, {
|
|
589
|
+
table: plan.table.name,
|
|
542
590
|
scopeFilter: plan.effective,
|
|
543
|
-
afterSeq: sub.cursor,
|
|
544
|
-
throughSeq: maxSeq,
|
|
545
|
-
limitChanges: limits.limitCommits + 1,
|
|
546
591
|
}),
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
// §4.6 reset: no reads.
|
|
550
|
-
if (token === undefined && sub.cursor >= 0 && sub.cursor < horizonSeq) {
|
|
551
|
-
return {};
|
|
552
|
-
}
|
|
553
|
-
// Bootstrapping: the same resume decision `preparedSection` makes (§4.7).
|
|
554
|
-
const resume =
|
|
555
|
-
token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
556
|
-
? token
|
|
557
|
-
: undefined;
|
|
558
|
-
const startCursor = resume?.rowCursor ?? null;
|
|
559
|
-
const firstPage = ctx.storage.scanRows(ctx.partition, {
|
|
560
|
-
table: plan.table.name,
|
|
561
|
-
scopeFilter: plan.effective,
|
|
562
|
-
afterRowId: startCursor,
|
|
563
|
-
limit: limits.limitSnapshotRows + 1,
|
|
564
|
-
});
|
|
565
|
-
if (!sqliteLaneEligible(limits, plan, startCursor)) return { firstPage };
|
|
566
|
-
// §4.7: a fresh sqlite-lane bootstrap pins at the newest change in its
|
|
567
|
-
// scope (never below the horizon, never above maxCommitSeq), so a stored
|
|
568
|
-
// image stays the reuse key's match until this scope changes (§5.3).
|
|
569
|
-
const pinned =
|
|
570
|
-
resume !== undefined
|
|
571
|
-
? Promise.resolve({ asOf: resume.asOfCommitSeq, pin: undefined })
|
|
572
|
-
: ctx.storage
|
|
573
|
-
.latestScopedChangeSeq(ctx.partition, {
|
|
574
|
-
table: plan.table.name,
|
|
575
|
-
scopeFilter: plan.effective,
|
|
576
|
-
})
|
|
577
|
-
.then((latest) => {
|
|
578
|
-
const pin = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
579
|
-
return { asOf: pin, pin };
|
|
580
|
-
});
|
|
581
|
-
const lane = pinned.then(async ({ asOf, pin }) => {
|
|
592
|
+
]);
|
|
593
|
+
const asOf = Math.min(maxSeq, Math.max(latest, horizonSeq));
|
|
582
594
|
// §5.3 eligibility: a stored image only matches a snapshot larger than
|
|
583
595
|
// one rows page, so the probe decides before the reuse lookup; a remote
|
|
584
596
|
// `find` on a small table would be a wasted round trip.
|
|
585
|
-
const
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
),
|
|
603
|
-
};
|
|
604
|
-
});
|
|
605
|
-
return { firstPage, lane };
|
|
606
|
-
});
|
|
607
|
-
const settled = await Promise.all(
|
|
608
|
-
reads.map(async (read) => ({
|
|
609
|
-
commits: read.commits === undefined ? undefined : await read.commits,
|
|
610
|
-
firstPage: await read.firstPage,
|
|
611
|
-
lane: await read.lane,
|
|
612
|
-
})),
|
|
597
|
+
const image =
|
|
598
|
+
firstPage.length <= limits.limitSnapshotRows
|
|
599
|
+
? undefined
|
|
600
|
+
: await ctx.segments.find(
|
|
601
|
+
{
|
|
602
|
+
partition: ctx.partition,
|
|
603
|
+
logEpoch,
|
|
604
|
+
table: plan.table.name,
|
|
605
|
+
schemaVersion: schema.version,
|
|
606
|
+
mediaType: 'sqlite',
|
|
607
|
+
scopeDigest: await scopeDigest(plan.effective),
|
|
608
|
+
asOfCommitSeq: asOf,
|
|
609
|
+
},
|
|
610
|
+
clockOf(ctx)(),
|
|
611
|
+
);
|
|
612
|
+
return { kind: 'bootstrap', asOf, startCursor, firstPage, image };
|
|
613
|
+
}),
|
|
613
614
|
);
|
|
614
615
|
// Validate continuity before committing to an active section. A prune
|
|
615
616
|
// during a paged read can otherwise make an incomplete window look empty.
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
617
|
+
if (!prepared.some((section) => section.kind === 'incremental')) {
|
|
618
|
+
return prepared;
|
|
619
|
+
}
|
|
620
|
+
const windowHorizon = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
621
|
+
return prepared.map((section, index) =>
|
|
622
|
+
section.kind === 'incremental' &&
|
|
623
|
+
(plans[index]?.frame.cursor ?? -1) < windowHorizon
|
|
624
|
+
? { kind: 'reset' }
|
|
625
|
+
: section,
|
|
626
|
+
);
|
|
626
627
|
}
|
|
627
628
|
|
|
628
629
|
/**
|
|
@@ -680,7 +681,7 @@ export async function* preparedSection(
|
|
|
680
681
|
throw new Error('subscriptionSection requires a non-empty log epoch');
|
|
681
682
|
}
|
|
682
683
|
|
|
683
|
-
if (
|
|
684
|
+
if (prepared.kind === 'revoked') {
|
|
684
685
|
yield {
|
|
685
686
|
type: 'SUB_START',
|
|
686
687
|
id: sub.id,
|
|
@@ -693,12 +694,9 @@ export async function* preparedSection(
|
|
|
693
694
|
return { nextCursor: sub.cursor, active: false };
|
|
694
695
|
}
|
|
695
696
|
|
|
696
|
-
const token = parseBootstrapToken(sub.bootstrapState, sub.table);
|
|
697
|
-
const { commits, horizonSeq } = prepared;
|
|
698
|
-
|
|
699
697
|
// §4.6: a cursor behind the horizon (and not resuming a bootstrap)
|
|
700
698
|
// cannot compute deltas — answer `reset` and echo the cursor.
|
|
701
|
-
if (
|
|
699
|
+
if (prepared.kind === 'reset') {
|
|
702
700
|
yield {
|
|
703
701
|
type: 'SUB_START',
|
|
704
702
|
id: sub.id,
|
|
@@ -711,21 +709,8 @@ export async function* preparedSection(
|
|
|
711
709
|
return { nextCursor: sub.cursor, active: false };
|
|
712
710
|
}
|
|
713
711
|
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
if (bootstrapping) {
|
|
718
|
-
// §4.7: resume at the pinned point unless the pin fell behind the
|
|
719
|
-
// horizon (or the token is unusable) — then restart from a fresh pin.
|
|
720
|
-
const resume =
|
|
721
|
-
token !== undefined && token.asOfCommitSeq >= horizonSeq
|
|
722
|
-
? token
|
|
723
|
-
: undefined;
|
|
724
|
-
const asOf = resume?.asOfCommitSeq ?? prepared.pin ?? maxSeq;
|
|
725
|
-
const startCursor = resume?.rowCursor ?? null;
|
|
726
|
-
if (prepared.firstPage === undefined) {
|
|
727
|
-
throw new Error('prepareSections did not read the bootstrap page');
|
|
728
|
-
}
|
|
712
|
+
if (prepared.kind === 'bootstrap') {
|
|
713
|
+
const { asOf } = prepared;
|
|
729
714
|
yield {
|
|
730
715
|
type: 'SUB_START',
|
|
731
716
|
id: sub.id,
|
|
@@ -740,7 +725,7 @@ export async function* preparedSection(
|
|
|
740
725
|
limits,
|
|
741
726
|
plan,
|
|
742
727
|
asOf,
|
|
743
|
-
startCursor,
|
|
728
|
+
prepared.startCursor,
|
|
744
729
|
trace,
|
|
745
730
|
logEpoch,
|
|
746
731
|
prepared.firstPage,
|
|
@@ -777,6 +762,7 @@ export async function* preparedSection(
|
|
|
777
762
|
let delivered = 0;
|
|
778
763
|
let deliveredCommits = 0;
|
|
779
764
|
let lastDeliveredSeq = sub.cursor;
|
|
765
|
+
const { commits } = prepared;
|
|
780
766
|
for (const commit of commits) {
|
|
781
767
|
if (
|
|
782
768
|
delivered > 0 &&
|