@syncular/server 0.94.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context.d.ts +6 -0
- package/dist/context.js +3 -0
- package/dist/handler.js +38 -2
- package/dist/pull.d.ts +9 -1
- package/dist/pull.js +17 -11
- package/package.json +2 -2
- package/src/context.ts +9 -0
- package/src/handler.ts +38 -4
- package/src/pull.ts +41 -26
package/dist/context.d.ts
CHANGED
|
@@ -102,6 +102,12 @@ export interface ServerLimits {
|
|
|
102
102
|
* bound gets an image of its own.
|
|
103
103
|
*/
|
|
104
104
|
readonly maxImageSegmentBytes: number;
|
|
105
|
+
/**
|
|
106
|
+
* Segment puts one pull keeps in flight. The pull scans and encodes its
|
|
107
|
+
* pages in order and stores up to this many at once, so it holds this
|
|
108
|
+
* many encoded segments in memory.
|
|
109
|
+
*/
|
|
110
|
+
readonly maxConcurrentSegmentPuts: number;
|
|
105
111
|
}
|
|
106
112
|
export declare const DEFAULT_LIMITS: ServerLimits;
|
|
107
113
|
/** Receives applied commits for realtime fanout (§8.2). */
|
package/dist/context.js
CHANGED
|
@@ -25,6 +25,9 @@ export const DEFAULT_LIMITS = {
|
|
|
25
25
|
maxLiveSectionBytes: 16 * 1024 * 1024,
|
|
26
26
|
// §5.3 DECISION (2026-11-17): a client holds about three images at once.
|
|
27
27
|
maxImageSegmentBytes: 32 * 1024 * 1024,
|
|
28
|
+
// A Workers invocation holds six connections open at once, and an R2 put
|
|
29
|
+
// uses one; past six the runtime queues the put.
|
|
30
|
+
maxConcurrentSegmentPuts: 6,
|
|
28
31
|
};
|
|
29
32
|
/** §1.1: `authenticate` returned a context longer than 256 UTF-8 bytes. */
|
|
30
33
|
export class AuthContextTooLargeError extends Error {
|
package/dist/handler.js
CHANGED
|
@@ -615,6 +615,12 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
615
615
|
// follows the configured limits, never the unit count.
|
|
616
616
|
const bootstrapping = states.flatMap((state, index) => state.kind === 'bootstrap' ? [index] : []);
|
|
617
617
|
const firstPages = Math.max(1, Math.floor(100_000 / (limits.limitSnapshotRows + 1)));
|
|
618
|
+
// The segment puts of every bootstrap section, in frame order. A
|
|
619
|
+
// pull holds the encoded bytes of the puts in flight.
|
|
620
|
+
const puts = [];
|
|
621
|
+
const inFlight = new Set();
|
|
622
|
+
const maxPuts = limitsOf(ctx).maxConcurrentSegmentPuts;
|
|
623
|
+
let failed = false;
|
|
618
624
|
for (let next = 0; next < bootstrapping.length;) {
|
|
619
625
|
const window = [];
|
|
620
626
|
let pages = 0;
|
|
@@ -647,7 +653,30 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
647
653
|
const segments = bootstrapSegments(ctx, schema, limits, bootstrap, traces[index], plan.logEpoch);
|
|
648
654
|
let step = await segments.next();
|
|
649
655
|
while (!step.done) {
|
|
650
|
-
|
|
656
|
+
const frame = step.value;
|
|
657
|
+
if (frame.type === 'PENDING_REF') {
|
|
658
|
+
// A rows segment's put runs while later pages are scanned;
|
|
659
|
+
// the next page waits while `maxPuts` puts are in flight.
|
|
660
|
+
const settled = frame.ref.then(() => void inFlight.delete(settled), () => {
|
|
661
|
+
failed = true;
|
|
662
|
+
inFlight.delete(settled);
|
|
663
|
+
});
|
|
664
|
+
inFlight.add(settled);
|
|
665
|
+
puts.push(frame.ref);
|
|
666
|
+
frames.push(frame.ref);
|
|
667
|
+
while (inFlight.size >= maxPuts) {
|
|
668
|
+
await Promise.race(inFlight);
|
|
669
|
+
if (failed)
|
|
670
|
+
break;
|
|
671
|
+
}
|
|
672
|
+
// A failed put ends the pull before it scans another page.
|
|
673
|
+
if (failed)
|
|
674
|
+
for (const put of puts)
|
|
675
|
+
await put;
|
|
676
|
+
}
|
|
677
|
+
else {
|
|
678
|
+
frames.push(frame);
|
|
679
|
+
}
|
|
651
680
|
step = await segments.next();
|
|
652
681
|
}
|
|
653
682
|
const restrict = units[index].restrict;
|
|
@@ -660,6 +689,13 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
660
689
|
};
|
|
661
690
|
}
|
|
662
691
|
}
|
|
692
|
+
// Every put completes before the live read, as in a pull with one put
|
|
693
|
+
// in flight. A failed put fails the pull with the error of the first
|
|
694
|
+
// failed page, and the puts still in flight finish unobserved: each
|
|
695
|
+
// stores its bytes before its record, so no record points at missing
|
|
696
|
+
// bytes.
|
|
697
|
+
for (const put of puts)
|
|
698
|
+
await put;
|
|
663
699
|
const bound = (state) => state.kind === 'live'
|
|
664
700
|
? R
|
|
665
701
|
: state.kind === 'post'
|
|
@@ -757,7 +793,7 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
757
793
|
}, plan.wireVersion);
|
|
758
794
|
if (state.kind === 'bootstrap') {
|
|
759
795
|
for (const frame of state.frames) {
|
|
760
|
-
yield encodeResponseFrame(frame, plan.wireVersion);
|
|
796
|
+
yield encodeResponseFrame(await frame, plan.wireVersion);
|
|
761
797
|
}
|
|
762
798
|
}
|
|
763
799
|
yield encodeResponseFrame(state.kind === 'bootstrap'
|
package/dist/pull.d.ts
CHANGED
|
@@ -67,6 +67,14 @@ interface TableStart {
|
|
|
67
67
|
readonly stored: SegmentRecord | undefined;
|
|
68
68
|
};
|
|
69
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* A section frame: a response frame, or a rows segment's `SEGMENT_REF`
|
|
72
|
+
* whose put is in flight. The consumer emits `ref` in its frame order.
|
|
73
|
+
*/
|
|
74
|
+
type SectionFrame = ResponseFrame | {
|
|
75
|
+
readonly type: 'PENDING_REF';
|
|
76
|
+
readonly ref: Promise<ResponseFrame>;
|
|
77
|
+
};
|
|
70
78
|
/** A bootstrapping unit's position and its table starts, read ahead (§4.7). */
|
|
71
79
|
export interface BootstrapPlan {
|
|
72
80
|
readonly unit: UnitPlan;
|
|
@@ -98,7 +106,7 @@ export declare function planBootstrap(ctx: SyncRequestContext, schema: CompiledS
|
|
|
98
106
|
* the resume token, or `undefined` once every table is exhausted. A pair the
|
|
99
107
|
* wildcard fence revoked receives no segment (§3.3).
|
|
100
108
|
*/
|
|
101
|
-
export declare function bootstrapSegments(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, plan: BootstrapPlan, trace: PullSectionTrace | undefined, logEpoch: string): AsyncGenerator<
|
|
109
|
+
export declare function bootstrapSegments(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, plan: BootstrapPlan, trace: PullSectionTrace | undefined, logEpoch: string): AsyncGenerator<SectionFrame, string | undefined>;
|
|
102
110
|
export interface PublishSqliteImageInput {
|
|
103
111
|
/** The sync config the serving host runs, plus a `sqliteImageBuilder`. */
|
|
104
112
|
readonly config: SyncServerConfig;
|
package/dist/pull.js
CHANGED
|
@@ -473,7 +473,17 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
|
|
|
473
473
|
yield { type: 'SEGMENT_INLINE', payload: bytes };
|
|
474
474
|
}
|
|
475
475
|
else {
|
|
476
|
-
|
|
476
|
+
// The page's put runs while the next page is scanned; the caller keeps
|
|
477
|
+
// `maxConcurrentSegmentPuts` in flight and emits the frames in order.
|
|
478
|
+
trace?.segments.push({
|
|
479
|
+
mediaType: 'rows',
|
|
480
|
+
delivery: 'ref',
|
|
481
|
+
origin: 'built',
|
|
482
|
+
bytes: bytes.length,
|
|
483
|
+
rows: pageRows.length,
|
|
484
|
+
});
|
|
485
|
+
const ref = segments
|
|
486
|
+
.put({
|
|
477
487
|
partition,
|
|
478
488
|
logEpoch,
|
|
479
489
|
table: pair.table.name,
|
|
@@ -484,15 +494,9 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
|
|
|
484
494
|
rowCount: pageRows.length,
|
|
485
495
|
rowCursor,
|
|
486
496
|
nextRowCursor,
|
|
487
|
-
}, bytes, now)
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
delivery: 'ref',
|
|
491
|
-
origin: 'built',
|
|
492
|
-
bytes: record.byteLength,
|
|
493
|
-
rows: record.rowCount,
|
|
494
|
-
});
|
|
495
|
-
yield segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now));
|
|
497
|
+
}, bytes, now)
|
|
498
|
+
.then(async (record) => segmentRefFrame(record, digest, await signedUrlFields(ctx, limits, record.segmentId, digest, now)));
|
|
499
|
+
yield { type: 'PENDING_REF', ref };
|
|
496
500
|
}
|
|
497
501
|
if (!hasMore)
|
|
498
502
|
return { pages: page + 1, rows, rowCursor: null, image: false };
|
|
@@ -650,7 +654,9 @@ export async function publishSqliteImage(input) {
|
|
|
650
654
|
const plan = planBootstrap(ctx, schema, limits, unit, undefined, maxSeq, horizonSeq, logEpoch);
|
|
651
655
|
const refs = [];
|
|
652
656
|
for await (const frame of bootstrapSegments(ctx, schema, limits, plan, trace, logEpoch)) {
|
|
653
|
-
if (frame.type === '
|
|
657
|
+
if (frame.type === 'PENDING_REF')
|
|
658
|
+
await frame.ref;
|
|
659
|
+
else if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
|
|
654
660
|
refs.push(frame);
|
|
655
661
|
}
|
|
656
662
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.95.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.95.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/context.ts
CHANGED
|
@@ -124,6 +124,12 @@ export interface ServerLimits {
|
|
|
124
124
|
* bound gets an image of its own.
|
|
125
125
|
*/
|
|
126
126
|
readonly maxImageSegmentBytes: number;
|
|
127
|
+
/**
|
|
128
|
+
* Segment puts one pull keeps in flight. The pull scans and encodes its
|
|
129
|
+
* pages in order and stores up to this many at once, so it holds this
|
|
130
|
+
* many encoded segments in memory.
|
|
131
|
+
*/
|
|
132
|
+
readonly maxConcurrentSegmentPuts: number;
|
|
127
133
|
}
|
|
128
134
|
|
|
129
135
|
export const DEFAULT_LIMITS: ServerLimits = {
|
|
@@ -140,6 +146,9 @@ export const DEFAULT_LIMITS: ServerLimits = {
|
|
|
140
146
|
maxLiveSectionBytes: 16 * 1024 * 1024,
|
|
141
147
|
// §5.3 DECISION (2026-11-17): a client holds about three images at once.
|
|
142
148
|
maxImageSegmentBytes: 32 * 1024 * 1024,
|
|
149
|
+
// A Workers invocation holds six connections open at once, and an R2 put
|
|
150
|
+
// uses one; past six the runtime queues the put.
|
|
151
|
+
maxConcurrentSegmentPuts: 6,
|
|
143
152
|
};
|
|
144
153
|
|
|
145
154
|
/** Receives applied commits for realtime fanout (§8.2). */
|
package/src/handler.ts
CHANGED
|
@@ -886,7 +886,7 @@ async function* streamResponse(
|
|
|
886
886
|
/** The join cursor (§4.7): the pin, or the kept cursor. */
|
|
887
887
|
pin: number;
|
|
888
888
|
restricted: boolean;
|
|
889
|
-
frames: ResponseFrame[];
|
|
889
|
+
frames: (ResponseFrame | Promise<ResponseFrame>)[];
|
|
890
890
|
token: string | undefined;
|
|
891
891
|
};
|
|
892
892
|
const states: UnitState[] = units.map(
|
|
@@ -941,6 +941,12 @@ async function* streamResponse(
|
|
|
941
941
|
1,
|
|
942
942
|
Math.floor(100_000 / (limits.limitSnapshotRows + 1)),
|
|
943
943
|
);
|
|
944
|
+
// The segment puts of every bootstrap section, in frame order. A
|
|
945
|
+
// pull holds the encoded bytes of the puts in flight.
|
|
946
|
+
const puts: Promise<ResponseFrame>[] = [];
|
|
947
|
+
const inFlight = new Set<Promise<void>>();
|
|
948
|
+
const maxPuts = limitsOf(ctx).maxConcurrentSegmentPuts;
|
|
949
|
+
let failed = false;
|
|
944
950
|
for (let next = 0; next < bootstrapping.length;) {
|
|
945
951
|
const window: number[] = [];
|
|
946
952
|
let pages = 0;
|
|
@@ -986,7 +992,7 @@ async function* streamResponse(
|
|
|
986
992
|
};
|
|
987
993
|
});
|
|
988
994
|
for (const { index, plan: bootstrap } of batch) {
|
|
989
|
-
const frames: ResponseFrame[] = [];
|
|
995
|
+
const frames: (ResponseFrame | Promise<ResponseFrame>)[] = [];
|
|
990
996
|
const segments = bootstrapSegments(
|
|
991
997
|
ctx,
|
|
992
998
|
schema,
|
|
@@ -997,7 +1003,29 @@ async function* streamResponse(
|
|
|
997
1003
|
);
|
|
998
1004
|
let step = await segments.next();
|
|
999
1005
|
while (!step.done) {
|
|
1000
|
-
|
|
1006
|
+
const frame = step.value;
|
|
1007
|
+
if (frame.type === 'PENDING_REF') {
|
|
1008
|
+
// A rows segment's put runs while later pages are scanned;
|
|
1009
|
+
// the next page waits while `maxPuts` puts are in flight.
|
|
1010
|
+
const settled: Promise<void> = frame.ref.then(
|
|
1011
|
+
() => void inFlight.delete(settled),
|
|
1012
|
+
() => {
|
|
1013
|
+
failed = true;
|
|
1014
|
+
inFlight.delete(settled);
|
|
1015
|
+
},
|
|
1016
|
+
);
|
|
1017
|
+
inFlight.add(settled);
|
|
1018
|
+
puts.push(frame.ref);
|
|
1019
|
+
frames.push(frame.ref);
|
|
1020
|
+
while (inFlight.size >= maxPuts) {
|
|
1021
|
+
await Promise.race(inFlight);
|
|
1022
|
+
if (failed) break;
|
|
1023
|
+
}
|
|
1024
|
+
// A failed put ends the pull before it scans another page.
|
|
1025
|
+
if (failed) for (const put of puts) await put;
|
|
1026
|
+
} else {
|
|
1027
|
+
frames.push(frame);
|
|
1028
|
+
}
|
|
1001
1029
|
step = await segments.next();
|
|
1002
1030
|
}
|
|
1003
1031
|
const restrict = units[index]!.restrict;
|
|
@@ -1010,6 +1038,12 @@ async function* streamResponse(
|
|
|
1010
1038
|
};
|
|
1011
1039
|
}
|
|
1012
1040
|
}
|
|
1041
|
+
// Every put completes before the live read, as in a pull with one put
|
|
1042
|
+
// in flight. A failed put fails the pull with the error of the first
|
|
1043
|
+
// failed page, and the puts still in flight finish unobserved: each
|
|
1044
|
+
// stores its bytes before its record, so no record points at missing
|
|
1045
|
+
// bytes.
|
|
1046
|
+
for (const put of puts) await put;
|
|
1013
1047
|
const bound = (state: UnitState): number | undefined =>
|
|
1014
1048
|
state.kind === 'live'
|
|
1015
1049
|
? R
|
|
@@ -1107,7 +1141,7 @@ async function* streamResponse(
|
|
|
1107
1141
|
);
|
|
1108
1142
|
if (state.kind === 'bootstrap') {
|
|
1109
1143
|
for (const frame of state.frames) {
|
|
1110
|
-
yield encodeResponseFrame(frame, plan.wireVersion);
|
|
1144
|
+
yield encodeResponseFrame(await frame, plan.wireVersion);
|
|
1111
1145
|
}
|
|
1112
1146
|
}
|
|
1113
1147
|
yield encodeResponseFrame(
|
package/src/pull.ts
CHANGED
|
@@ -617,6 +617,14 @@ async function* sqliteImageSegment(
|
|
|
617
617
|
return record;
|
|
618
618
|
}
|
|
619
619
|
|
|
620
|
+
/**
|
|
621
|
+
* A section frame: a response frame, or a rows segment's `SEGMENT_REF`
|
|
622
|
+
* whose put is in flight. The consumer emits `ref` in its frame order.
|
|
623
|
+
*/
|
|
624
|
+
type SectionFrame =
|
|
625
|
+
| ResponseFrame
|
|
626
|
+
| { readonly type: 'PENDING_REF'; readonly ref: Promise<ResponseFrame> };
|
|
627
|
+
|
|
620
628
|
/**
|
|
621
629
|
* One table's segments within the page budget: range images while the table
|
|
622
630
|
* is on the image lane, then rows pages. Returns the pages it used, the row
|
|
@@ -635,7 +643,7 @@ async function* tableSegments(
|
|
|
635
643
|
trace: PullSectionTrace | undefined,
|
|
636
644
|
logEpoch: string,
|
|
637
645
|
): AsyncGenerator<
|
|
638
|
-
|
|
646
|
+
SectionFrame,
|
|
639
647
|
{ pages: number; rows: number; rowCursor: string | null; image: boolean }
|
|
640
648
|
> {
|
|
641
649
|
const { storage, segments, partition } = ctx;
|
|
@@ -725,34 +733,40 @@ async function* tableSegments(
|
|
|
725
733
|
});
|
|
726
734
|
yield { type: 'SEGMENT_INLINE', payload: bytes };
|
|
727
735
|
} else {
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
partition,
|
|
731
|
-
logEpoch,
|
|
732
|
-
table: pair.table.name,
|
|
733
|
-
schemaVersion: schema.version,
|
|
734
|
-
mediaType: 'rows',
|
|
735
|
-
scopeDigest: digest,
|
|
736
|
-
asOfCommitSeq: pin,
|
|
737
|
-
rowCount: pageRows.length,
|
|
738
|
-
rowCursor,
|
|
739
|
-
nextRowCursor,
|
|
740
|
-
},
|
|
741
|
-
bytes,
|
|
742
|
-
now,
|
|
743
|
-
);
|
|
736
|
+
// The page's put runs while the next page is scanned; the caller keeps
|
|
737
|
+
// `maxConcurrentSegmentPuts` in flight and emits the frames in order.
|
|
744
738
|
trace?.segments.push({
|
|
745
739
|
mediaType: 'rows',
|
|
746
740
|
delivery: 'ref',
|
|
747
741
|
origin: 'built',
|
|
748
|
-
bytes:
|
|
749
|
-
rows:
|
|
742
|
+
bytes: bytes.length,
|
|
743
|
+
rows: pageRows.length,
|
|
750
744
|
});
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
745
|
+
const ref = segments
|
|
746
|
+
.put(
|
|
747
|
+
{
|
|
748
|
+
partition,
|
|
749
|
+
logEpoch,
|
|
750
|
+
table: pair.table.name,
|
|
751
|
+
schemaVersion: schema.version,
|
|
752
|
+
mediaType: 'rows',
|
|
753
|
+
scopeDigest: digest,
|
|
754
|
+
asOfCommitSeq: pin,
|
|
755
|
+
rowCount: pageRows.length,
|
|
756
|
+
rowCursor,
|
|
757
|
+
nextRowCursor,
|
|
758
|
+
},
|
|
759
|
+
bytes,
|
|
760
|
+
now,
|
|
761
|
+
)
|
|
762
|
+
.then(async (record) =>
|
|
763
|
+
segmentRefFrame(
|
|
764
|
+
record,
|
|
765
|
+
digest,
|
|
766
|
+
await signedUrlFields(ctx, limits, record.segmentId, digest, now),
|
|
767
|
+
),
|
|
768
|
+
);
|
|
769
|
+
yield { type: 'PENDING_REF', ref };
|
|
756
770
|
}
|
|
757
771
|
if (!hasMore)
|
|
758
772
|
return { pages: page + 1, rows, rowCursor: null, image: false };
|
|
@@ -876,7 +890,7 @@ export async function* bootstrapSegments(
|
|
|
876
890
|
plan: BootstrapPlan,
|
|
877
891
|
trace: PullSectionTrace | undefined,
|
|
878
892
|
logEpoch: string,
|
|
879
|
-
): AsyncGenerator<
|
|
893
|
+
): AsyncGenerator<SectionFrame, string | undefined> {
|
|
880
894
|
const { unit, pin } = plan;
|
|
881
895
|
let { tableIndex, rowCursor, image } = plan;
|
|
882
896
|
let pages = 0;
|
|
@@ -1028,7 +1042,8 @@ export async function publishSqliteImage(
|
|
|
1028
1042
|
trace,
|
|
1029
1043
|
logEpoch,
|
|
1030
1044
|
)) {
|
|
1031
|
-
if (frame.type === '
|
|
1045
|
+
if (frame.type === 'PENDING_REF') await frame.ref;
|
|
1046
|
+
else if (frame.type === 'SEGMENT_REF' && frame.mediaType === 'sqlite') {
|
|
1032
1047
|
refs.push(frame);
|
|
1033
1048
|
}
|
|
1034
1049
|
}
|