@syncular/server 0.86.0 → 0.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/handler.js +59 -9
- package/dist/postgres-storage.js +7 -3
- package/dist/pull.d.ts +1 -1
- package/dist/pull.js +6 -4
- package/dist/sqlite-dialect.js +7 -3
- package/dist/storage-query.d.ts +11 -4
- package/dist/storage-query.js +16 -0
- package/package.json +2 -2
- package/src/handler.ts +104 -32
- package/src/postgres-storage.ts +8 -3
- package/src/pull.ts +6 -3
- package/src/sqlite-dialect.ts +8 -3
- package/src/storage-query.ts +21 -4
package/dist/handler.js
CHANGED
|
@@ -117,7 +117,11 @@ async function planRequest(request, ctx, schema, registry) {
|
|
|
117
117
|
let pullHeader;
|
|
118
118
|
const unitFrames = [];
|
|
119
119
|
const progressFrames = [];
|
|
120
|
+
// §4.3: a RESTRICT belongs to the PROGRESS it immediately follows.
|
|
121
|
+
const restricts = new Map();
|
|
120
122
|
for (const frame of request.frames) {
|
|
123
|
+
if (frame.type === 'RESTRICT')
|
|
124
|
+
restricts.set(progressFrames.at(-1).id, frame);
|
|
121
125
|
if (frame.type === 'ROUND_FAILURE')
|
|
122
126
|
roundFailure = frame;
|
|
123
127
|
else if (frame.type === 'PUSH_COMMIT')
|
|
@@ -234,6 +238,14 @@ async function planRequest(request, ctx, schema, registry) {
|
|
|
234
238
|
}
|
|
235
239
|
progressed.add(frame.id);
|
|
236
240
|
}
|
|
241
|
+
for (const { id, tables } of list) {
|
|
242
|
+
const restrict = restricts.get(id);
|
|
243
|
+
if (restrict !== undefined &&
|
|
244
|
+
(restrict.tables.length >= tables.length ||
|
|
245
|
+
restrict.tables.some((name) => !tables.some((table) => table.name === name)))) {
|
|
246
|
+
throw syncError('sync.invalid_subscription', 'a RESTRICT frame names a table the unit does not cover, or every covered table (§4.3)', JSON.stringify({ reason: 'restrict_tables', id }));
|
|
247
|
+
}
|
|
248
|
+
}
|
|
237
249
|
}
|
|
238
250
|
}
|
|
239
251
|
const { resolved, leaseToEmit } = await resolveOnce(ctx, header.clientId, compileSchema(ctx.schema));
|
|
@@ -253,7 +265,11 @@ async function planRequest(request, ctx, schema, registry) {
|
|
|
253
265
|
registration,
|
|
254
266
|
units: list?.map(({ scopes, tables, id }) => {
|
|
255
267
|
const plan = planUnit(scopes, tables, resolved, id);
|
|
256
|
-
return {
|
|
268
|
+
return {
|
|
269
|
+
plan,
|
|
270
|
+
progress: progress.get(plan.id),
|
|
271
|
+
restrict: restricts.get(plan.id),
|
|
272
|
+
};
|
|
257
273
|
}),
|
|
258
274
|
},
|
|
259
275
|
record,
|
|
@@ -551,11 +567,22 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
551
567
|
const liveUnits = units.some((unit) => unit.progress === undefined && unit.plan.status === 'active');
|
|
552
568
|
// §4.6 replica reset: a live unit with R outside [horizon, max].
|
|
553
569
|
let liveReset = liveUnits && (R < horizonSeq || R > maxSeq);
|
|
554
|
-
const states = units.map(({ plan: unit, progress }) => {
|
|
570
|
+
const states = units.map(({ plan: unit, progress, restrict }) => {
|
|
555
571
|
if (unit.status === 'revoked')
|
|
556
572
|
return { kind: 'revoked' };
|
|
557
573
|
if (progress === undefined)
|
|
558
574
|
return { kind: 'live' };
|
|
575
|
+
// §4.6: a kept cursor outside [horizon, max] cannot join.
|
|
576
|
+
if (restrict !== undefined)
|
|
577
|
+
return restrict.cursor < horizonSeq || restrict.cursor > maxSeq
|
|
578
|
+
? { kind: 'reset' }
|
|
579
|
+
: {
|
|
580
|
+
kind: 'bootstrap',
|
|
581
|
+
pin: -1,
|
|
582
|
+
restricted: true,
|
|
583
|
+
frames: [],
|
|
584
|
+
token: '',
|
|
585
|
+
};
|
|
559
586
|
// §4.7: a token, cursor -1 or a cursor from the future bootstraps.
|
|
560
587
|
if (progress.bootstrapState === undefined &&
|
|
561
588
|
progress.cursor >= 0 &&
|
|
@@ -565,7 +592,13 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
565
592
|
: { kind: 'post', cursor: progress.cursor };
|
|
566
593
|
}
|
|
567
594
|
// Replaced by the section the bootstrap below produces.
|
|
568
|
-
return {
|
|
595
|
+
return {
|
|
596
|
+
kind: 'bootstrap',
|
|
597
|
+
pin: -1,
|
|
598
|
+
restricted: false,
|
|
599
|
+
frames: [],
|
|
600
|
+
token: '',
|
|
601
|
+
};
|
|
569
602
|
});
|
|
570
603
|
const traces = units.map(() => events !== undefined ? { segments: [] } : undefined);
|
|
571
604
|
// Bootstrap sections run before the live read: a bootstrap that
|
|
@@ -589,10 +622,21 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
589
622
|
pages += reads;
|
|
590
623
|
next += 1;
|
|
591
624
|
}
|
|
592
|
-
const batch = window.map((index) =>
|
|
593
|
-
index
|
|
594
|
-
|
|
595
|
-
|
|
625
|
+
const batch = window.map((index) => {
|
|
626
|
+
const { plan: unit, progress, restrict } = units[index];
|
|
627
|
+
// §4.7: a restricted bootstrap serves only the RESTRICT tables.
|
|
628
|
+
const only = restrict === undefined ? undefined : new Set(restrict.tables);
|
|
629
|
+
return {
|
|
630
|
+
index,
|
|
631
|
+
plan: planBootstrap(ctx, schema, limits, only === undefined
|
|
632
|
+
? unit
|
|
633
|
+
: {
|
|
634
|
+
...unit,
|
|
635
|
+
tables: unit.tables.filter((table) => only.has(table.name)),
|
|
636
|
+
pairs: new Map([...unit.pairs].filter(([name]) => only.has(name))),
|
|
637
|
+
}, progress?.bootstrapState, maxSeq, horizonSeq, plan.logEpoch, restrict?.cursor),
|
|
638
|
+
};
|
|
639
|
+
});
|
|
596
640
|
for (const { index, plan: bootstrap } of batch) {
|
|
597
641
|
const frames = [];
|
|
598
642
|
const segments = bootstrapSegments(ctx, schema, limits, bootstrap, traces[index], plan.logEpoch);
|
|
@@ -601,9 +645,11 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
601
645
|
frames.push(step.value);
|
|
602
646
|
step = await segments.next();
|
|
603
647
|
}
|
|
648
|
+
const restrict = units[index].restrict;
|
|
604
649
|
states[index] = {
|
|
605
650
|
kind: 'bootstrap',
|
|
606
|
-
pin: bootstrap.pin,
|
|
651
|
+
pin: restrict?.cursor ?? bootstrap.pin,
|
|
652
|
+
restricted: restrict !== undefined,
|
|
607
653
|
frames,
|
|
608
654
|
token: step.value,
|
|
609
655
|
};
|
|
@@ -651,7 +697,11 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
651
697
|
const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
652
698
|
let expired = false;
|
|
653
699
|
for (const [index, state] of states.entries()) {
|
|
654
|
-
|
|
700
|
+
// §4.6: a restricted bootstrap's kept cursor is rechecked too.
|
|
701
|
+
const afterSeq = bound(state) ??
|
|
702
|
+
(state.kind === 'bootstrap' && state.restricted
|
|
703
|
+
? state.pin
|
|
704
|
+
: undefined);
|
|
655
705
|
if (afterSeq === undefined || afterSeq >= recheck)
|
|
656
706
|
continue;
|
|
657
707
|
expired = true;
|
package/dist/postgres-storage.js
CHANGED
|
@@ -2242,10 +2242,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2242
2242
|
ORDER BY commit_seq LIMIT $4) s
|
|
2243
2243
|
ORDER BY s.commit_seq LIMIT $4)
|
|
2244
2244
|
SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
2245
|
-
z.key AS key, z.bytes AS bytes
|
|
2245
|
+
z.key AS key, z.bytes AS bytes, z.changes AS changes
|
|
2246
2246
|
FROM c
|
|
2247
2247
|
LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
2248
|
-
LEFT JOIN LATERAL (SELECT k.i AS key,
|
|
2248
|
+
LEFT JOIN LATERAL (SELECT k.i AS key, count(*) AS changes,
|
|
2249
2249
|
sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
|
|
2250
2250
|
FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value
|
|
2251
2251
|
WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
|
|
@@ -2263,7 +2263,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2263
2263
|
}),
|
|
2264
2264
|
...(record.key === null
|
|
2265
2265
|
? {}
|
|
2266
|
-
: {
|
|
2266
|
+
: {
|
|
2267
|
+
key: asNumber(record.key),
|
|
2268
|
+
bytes: asNumber(record.bytes),
|
|
2269
|
+
changes: asNumber(record.changes),
|
|
2270
|
+
}),
|
|
2267
2271
|
}));
|
|
2268
2272
|
}, async (keys, commitSeqs) => {
|
|
2269
2273
|
const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
|
package/dist/pull.d.ts
CHANGED
|
@@ -91,7 +91,7 @@ export interface BootstrapPlan {
|
|
|
91
91
|
* is awaited, so a storage that batches concurrent page reads (PostgreSQL,
|
|
92
92
|
* D1) answers them in one statement per table.
|
|
93
93
|
*/
|
|
94
|
-
export declare function planBootstrap(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, unit: UnitPlan, bootstrapState: string | undefined, maxSeq: number, horizonSeq: number, logEpoch: string): BootstrapPlan;
|
|
94
|
+
export declare function planBootstrap(ctx: SyncRequestContext, schema: CompiledSchema, limits: PullLimits, unit: UnitPlan, bootstrapState: string | undefined, maxSeq: number, horizonSeq: number, logEpoch: string, keptCursor?: number): BootstrapPlan;
|
|
95
95
|
/**
|
|
96
96
|
* The segments of one bootstrap section (§4.7): tables in bootstrap order,
|
|
97
97
|
* each active pair's rows or image, up to `maxSnapshotPages` pages. Returns
|
package/dist/pull.js
CHANGED
|
@@ -507,11 +507,13 @@ async function* tableSegments(ctx, schema, limits, pair, pin, startRowCursor, st
|
|
|
507
507
|
* is awaited, so a storage that batches concurrent page reads (PostgreSQL,
|
|
508
508
|
* D1) answers them in one statement per table.
|
|
509
509
|
*/
|
|
510
|
-
export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq, horizonSeq, logEpoch) {
|
|
511
|
-
// §4.7: resume at the token's pin unless the token is unusable or the pin
|
|
512
|
-
// fell behind the horizon
|
|
510
|
+
export function planBootstrap(ctx, schema, limits, unit, bootstrapState, maxSeq, horizonSeq, logEpoch, keptCursor = 0) {
|
|
511
|
+
// §4.7: resume at the token's pin unless the token is unusable, or the pin
|
|
512
|
+
// fell behind the horizon or a restricted bootstrap's kept cursor; then
|
|
513
|
+
// restart from a fresh pin.
|
|
513
514
|
const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
|
|
514
|
-
const resume = token !== undefined &&
|
|
515
|
+
const resume = token !== undefined &&
|
|
516
|
+
token.asOfCommitSeq >= Math.max(horizonSeq, keptCursor)
|
|
515
517
|
? token
|
|
516
518
|
: undefined;
|
|
517
519
|
const tableIndex = resume?.tableIndex ?? 0;
|
package/dist/sqlite-dialect.js
CHANGED
|
@@ -623,10 +623,10 @@ export class SqliteFamilyStorage {
|
|
|
623
623
|
AND s.commit_seq>k.after AND s.commit_seq<=?3
|
|
624
624
|
ORDER BY s.commit_seq LIMIT ?4)
|
|
625
625
|
SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
626
|
-
z.i AS key, z.bytes AS bytes
|
|
626
|
+
z.i AS key, z.bytes AS bytes, z.changes AS changes
|
|
627
627
|
FROM c
|
|
628
628
|
LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
|
|
629
|
-
LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
|
|
629
|
+
LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i, count(*) AS changes,
|
|
630
630
|
sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
|
|
631
631
|
FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
|
|
632
632
|
CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
|
|
@@ -642,7 +642,11 @@ export class SqliteFamilyStorage {
|
|
|
642
642
|
}
|
|
643
643
|
: {}),
|
|
644
644
|
...(record.key !== null
|
|
645
|
-
? {
|
|
645
|
+
? {
|
|
646
|
+
key: record.key,
|
|
647
|
+
bytes: record.bytes ?? 0,
|
|
648
|
+
changes: record.changes ?? 0,
|
|
649
|
+
}
|
|
646
650
|
: {}),
|
|
647
651
|
})), async (keys, commitSeqs) => (await this.guardedAll(`WITH k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
|
|
648
652
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
package/dist/storage-query.d.ts
CHANGED
|
@@ -54,10 +54,11 @@ export interface LiveKey {
|
|
|
54
54
|
}
|
|
55
55
|
/**
|
|
56
56
|
* One row of a live-window candidate page: a candidate commit joined to its
|
|
57
|
-
* metadata (absent when the commit vanished) and, per key, the
|
|
58
|
-
* (payload, row id, scopes) of its changes that key reads: a
|
|
59
|
-
* and the key's value in the stored scopes. A commit with no
|
|
60
|
-
* comes back once without `key`. Rows are in (commitSeq, key)
|
|
57
|
+
* metadata (absent when the commit vanished) and, per key, the count and
|
|
58
|
+
* stored bytes (payload, row id, scopes) of its changes that key reads: a
|
|
59
|
+
* covered table and the key's value in the stored scopes. A commit with no
|
|
60
|
+
* such change comes back once without `key`. Rows are in (commitSeq, key)
|
|
61
|
+
* order.
|
|
61
62
|
*/
|
|
62
63
|
export interface LiveCandidateRow {
|
|
63
64
|
readonly commitSeq: number;
|
|
@@ -67,6 +68,7 @@ export interface LiveCandidateRow {
|
|
|
67
68
|
};
|
|
68
69
|
readonly key?: number;
|
|
69
70
|
readonly bytes?: number;
|
|
71
|
+
readonly changes?: number;
|
|
70
72
|
}
|
|
71
73
|
/**
|
|
72
74
|
* The §4.5 live scan over a backend's two reads. `readCandidates` returns
|
|
@@ -81,6 +83,11 @@ export interface LiveCandidateRow {
|
|
|
81
83
|
* bound. Every candidate advances the scan, so a page of non-matching
|
|
82
84
|
* candidates never repeats.
|
|
83
85
|
*
|
|
86
|
+
* The counts bound the delivered changes from above, because every change
|
|
87
|
+
* a unit keeps is one its key reads: the scan loads pending candidates once
|
|
88
|
+
* their counts could reach the rest of `limitChanges`, so a page loads
|
|
89
|
+
* about the changes it delivers and never a candidate past the cut.
|
|
90
|
+
*
|
|
84
91
|
* The sizes bound the loaded bytes by `limitBytes` before any payload is
|
|
85
92
|
* read: the section stops before the commit that would pass it, and a
|
|
86
93
|
* commit that passes it alone ends the read as `oversized`, naming the
|
package/dist/storage-query.js
CHANGED
|
@@ -110,6 +110,11 @@ export async function transactionQuery(tx, query) {
|
|
|
110
110
|
* bound. Every candidate advances the scan, so a page of non-matching
|
|
111
111
|
* candidates never repeats.
|
|
112
112
|
*
|
|
113
|
+
* The counts bound the delivered changes from above, because every change
|
|
114
|
+
* a unit keeps is one its key reads: the scan loads pending candidates once
|
|
115
|
+
* their counts could reach the rest of `limitChanges`, so a page loads
|
|
116
|
+
* about the changes it delivers and never a candidate past the cut.
|
|
117
|
+
*
|
|
113
118
|
* The sizes bound the loaded bytes by `limitBytes` before any payload is
|
|
114
119
|
* read: the section stops before the commit that would pass it, and a
|
|
115
120
|
* commit that passes it alone ends the read as `oversized`, naming the
|
|
@@ -150,6 +155,8 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
|
|
|
150
155
|
const limit = Math.max(64, query.limitChanges);
|
|
151
156
|
let delivered = 0;
|
|
152
157
|
let loaded = 0;
|
|
158
|
+
// Upper bound of the changes the pending candidates deliver.
|
|
159
|
+
let bound = 0;
|
|
153
160
|
let scanned = -1;
|
|
154
161
|
// Candidates sized but not loaded yet: (scanned, pending.at(-1)].
|
|
155
162
|
let pending = [];
|
|
@@ -162,6 +169,7 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
|
|
|
162
169
|
const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
|
|
163
170
|
const rows = await readChanges(scanKeys(), pending.map((c) => c.commitSeq));
|
|
164
171
|
pending = [];
|
|
172
|
+
bound = 0;
|
|
165
173
|
scanned = last.commitSeq;
|
|
166
174
|
for (let i = 0; i < rows.length;) {
|
|
167
175
|
const seq = rows[i].commitSeq;
|
|
@@ -194,12 +202,14 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
|
|
|
194
202
|
const head = rows[i];
|
|
195
203
|
candidates += 1;
|
|
196
204
|
let bytes = 0;
|
|
205
|
+
let changes = 0;
|
|
197
206
|
const sizes = [];
|
|
198
207
|
for (; rows[i]?.commitSeq === head.commitSeq; i++) {
|
|
199
208
|
const row = rows[i];
|
|
200
209
|
if (row.key === undefined)
|
|
201
210
|
continue;
|
|
202
211
|
bytes += row.bytes ?? 0;
|
|
212
|
+
changes += row.changes ?? 0;
|
|
203
213
|
sizes.push([row.key, row.bytes ?? 0]);
|
|
204
214
|
}
|
|
205
215
|
if (bytes > query.limitBytes) {
|
|
@@ -233,7 +243,13 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
|
|
|
233
243
|
return { commits, throughSeq: cut ?? scanned };
|
|
234
244
|
}
|
|
235
245
|
loaded += bytes;
|
|
246
|
+
bound += changes;
|
|
236
247
|
pending.push({ commitSeq: head.commitSeq, meta: head.meta });
|
|
248
|
+
if (delivered + bound >= query.limitChanges) {
|
|
249
|
+
const cut = await load();
|
|
250
|
+
if (cut !== undefined)
|
|
251
|
+
return { commits, throughSeq: cut };
|
|
252
|
+
}
|
|
237
253
|
}
|
|
238
254
|
const cut = await load();
|
|
239
255
|
if (cut !== undefined)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.91.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.91.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/handler.ts
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
decodeRequest,
|
|
15
15
|
echoDigest,
|
|
16
16
|
type ProgressFrame,
|
|
17
|
+
type RestrictFrame,
|
|
17
18
|
type PullHeaderFrame,
|
|
18
19
|
type PushCommitFrame,
|
|
19
20
|
type PushResultDetailsFrame,
|
|
@@ -82,6 +83,8 @@ interface PlannedUnit {
|
|
|
82
83
|
readonly plan: UnitPlan;
|
|
83
84
|
/** Absent for a live unit. */
|
|
84
85
|
readonly progress: ProgressFrame | undefined;
|
|
86
|
+
/** §4.3: the restriction that follows the unit's `PROGRESS`. */
|
|
87
|
+
readonly restrict: RestrictFrame | undefined;
|
|
85
88
|
}
|
|
86
89
|
|
|
87
90
|
interface PullPlan {
|
|
@@ -259,7 +262,11 @@ async function planRequest(
|
|
|
259
262
|
let pullHeader: PullHeaderFrame | undefined;
|
|
260
263
|
const unitFrames: UnitFrame[] = [];
|
|
261
264
|
const progressFrames: ProgressFrame[] = [];
|
|
265
|
+
// §4.3: a RESTRICT belongs to the PROGRESS it immediately follows.
|
|
266
|
+
const restricts = new Map<string, RestrictFrame>();
|
|
262
267
|
for (const frame of request.frames) {
|
|
268
|
+
if (frame.type === 'RESTRICT')
|
|
269
|
+
restricts.set(progressFrames.at(-1)!.id, frame);
|
|
263
270
|
if (frame.type === 'ROUND_FAILURE') roundFailure = frame;
|
|
264
271
|
else if (frame.type === 'PUSH_COMMIT') pushes.push(frame);
|
|
265
272
|
else if (frame.type === 'PULL_HEADER') pullHeader = frame;
|
|
@@ -425,6 +432,22 @@ async function planRequest(
|
|
|
425
432
|
}
|
|
426
433
|
progressed.add(frame.id);
|
|
427
434
|
}
|
|
435
|
+
for (const { id, tables } of list) {
|
|
436
|
+
const restrict = restricts.get(id);
|
|
437
|
+
if (
|
|
438
|
+
restrict !== undefined &&
|
|
439
|
+
(restrict.tables.length >= tables.length ||
|
|
440
|
+
restrict.tables.some(
|
|
441
|
+
(name) => !tables.some((table) => table.name === name),
|
|
442
|
+
))
|
|
443
|
+
) {
|
|
444
|
+
throw syncError(
|
|
445
|
+
'sync.invalid_subscription',
|
|
446
|
+
'a RESTRICT frame names a table the unit does not cover, or every covered table (§4.3)',
|
|
447
|
+
JSON.stringify({ reason: 'restrict_tables', id }),
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
428
451
|
}
|
|
429
452
|
}
|
|
430
453
|
|
|
@@ -450,7 +473,11 @@ async function planRequest(
|
|
|
450
473
|
registration,
|
|
451
474
|
units: list?.map(({ scopes, tables, id }) => {
|
|
452
475
|
const plan = planUnit(scopes, tables, resolved, id);
|
|
453
|
-
return {
|
|
476
|
+
return {
|
|
477
|
+
plan,
|
|
478
|
+
progress: progress.get(plan.id),
|
|
479
|
+
restrict: restricts.get(plan.id),
|
|
480
|
+
};
|
|
454
481
|
}),
|
|
455
482
|
},
|
|
456
483
|
record,
|
|
@@ -848,26 +875,47 @@ async function* streamResponse(
|
|
|
848
875
|
| { kind: 'post'; cursor: number }
|
|
849
876
|
| {
|
|
850
877
|
kind: 'bootstrap';
|
|
878
|
+
/** The join cursor (§4.7): the pin, or the kept cursor. */
|
|
851
879
|
pin: number;
|
|
880
|
+
restricted: boolean;
|
|
852
881
|
frames: ResponseFrame[];
|
|
853
882
|
token: string | undefined;
|
|
854
883
|
};
|
|
855
|
-
const states: UnitState[] = units.map(
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
884
|
+
const states: UnitState[] = units.map(
|
|
885
|
+
({ plan: unit, progress, restrict }): UnitState => {
|
|
886
|
+
if (unit.status === 'revoked') return { kind: 'revoked' };
|
|
887
|
+
if (progress === undefined) return { kind: 'live' };
|
|
888
|
+
// §4.6: a kept cursor outside [horizon, max] cannot join.
|
|
889
|
+
if (restrict !== undefined)
|
|
890
|
+
return restrict.cursor < horizonSeq || restrict.cursor > maxSeq
|
|
891
|
+
? { kind: 'reset' }
|
|
892
|
+
: {
|
|
893
|
+
kind: 'bootstrap',
|
|
894
|
+
pin: -1,
|
|
895
|
+
restricted: true,
|
|
896
|
+
frames: [],
|
|
897
|
+
token: '',
|
|
898
|
+
};
|
|
899
|
+
// §4.7: a token, cursor -1 or a cursor from the future bootstraps.
|
|
900
|
+
if (
|
|
901
|
+
progress.bootstrapState === undefined &&
|
|
902
|
+
progress.cursor >= 0 &&
|
|
903
|
+
progress.cursor <= maxSeq
|
|
904
|
+
) {
|
|
905
|
+
return progress.cursor < horizonSeq
|
|
906
|
+
? { kind: 'reset' }
|
|
907
|
+
: { kind: 'post', cursor: progress.cursor };
|
|
908
|
+
}
|
|
909
|
+
// Replaced by the section the bootstrap below produces.
|
|
910
|
+
return {
|
|
911
|
+
kind: 'bootstrap',
|
|
912
|
+
pin: -1,
|
|
913
|
+
restricted: false,
|
|
914
|
+
frames: [],
|
|
915
|
+
token: '',
|
|
916
|
+
};
|
|
917
|
+
},
|
|
918
|
+
);
|
|
871
919
|
const traces = units.map((): PullSectionTrace | undefined =>
|
|
872
920
|
events !== undefined ? { segments: [] } : undefined,
|
|
873
921
|
);
|
|
@@ -899,19 +947,36 @@ async function* streamResponse(
|
|
|
899
947
|
pages += reads;
|
|
900
948
|
next += 1;
|
|
901
949
|
}
|
|
902
|
-
const batch = window.map((index) =>
|
|
903
|
-
index
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
950
|
+
const batch = window.map((index) => {
|
|
951
|
+
const { plan: unit, progress, restrict } = units[index]!;
|
|
952
|
+
// §4.7: a restricted bootstrap serves only the RESTRICT tables.
|
|
953
|
+
const only =
|
|
954
|
+
restrict === undefined ? undefined : new Set(restrict.tables);
|
|
955
|
+
return {
|
|
956
|
+
index,
|
|
957
|
+
plan: planBootstrap(
|
|
958
|
+
ctx,
|
|
959
|
+
schema,
|
|
960
|
+
limits,
|
|
961
|
+
only === undefined
|
|
962
|
+
? unit
|
|
963
|
+
: {
|
|
964
|
+
...unit,
|
|
965
|
+
tables: unit.tables.filter((table) =>
|
|
966
|
+
only.has(table.name),
|
|
967
|
+
),
|
|
968
|
+
pairs: new Map(
|
|
969
|
+
[...unit.pairs].filter(([name]) => only.has(name)),
|
|
970
|
+
),
|
|
971
|
+
},
|
|
972
|
+
progress?.bootstrapState,
|
|
973
|
+
maxSeq,
|
|
974
|
+
horizonSeq,
|
|
975
|
+
plan.logEpoch,
|
|
976
|
+
restrict?.cursor,
|
|
977
|
+
),
|
|
978
|
+
};
|
|
979
|
+
});
|
|
915
980
|
for (const { index, plan: bootstrap } of batch) {
|
|
916
981
|
const frames: ResponseFrame[] = [];
|
|
917
982
|
const segments = bootstrapSegments(
|
|
@@ -927,9 +992,11 @@ async function* streamResponse(
|
|
|
927
992
|
frames.push(step.value);
|
|
928
993
|
step = await segments.next();
|
|
929
994
|
}
|
|
995
|
+
const restrict = units[index]!.restrict;
|
|
930
996
|
states[index] = {
|
|
931
997
|
kind: 'bootstrap',
|
|
932
|
-
pin: bootstrap.pin,
|
|
998
|
+
pin: restrict?.cursor ?? bootstrap.pin,
|
|
999
|
+
restricted: restrict !== undefined,
|
|
933
1000
|
frames,
|
|
934
1001
|
token: step.value,
|
|
935
1002
|
};
|
|
@@ -975,7 +1042,12 @@ async function* streamResponse(
|
|
|
975
1042
|
const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
976
1043
|
let expired = false;
|
|
977
1044
|
for (const [index, state] of states.entries()) {
|
|
978
|
-
|
|
1045
|
+
// §4.6: a restricted bootstrap's kept cursor is rechecked too.
|
|
1046
|
+
const afterSeq =
|
|
1047
|
+
bound(state) ??
|
|
1048
|
+
(state.kind === 'bootstrap' && state.restricted
|
|
1049
|
+
? state.pin
|
|
1050
|
+
: undefined);
|
|
979
1051
|
if (afterSeq === undefined || afterSeq >= recheck) continue;
|
|
980
1052
|
expired = true;
|
|
981
1053
|
if (state.kind === 'live') liveReset = true;
|
package/src/postgres-storage.ts
CHANGED
|
@@ -3302,6 +3302,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3302
3302
|
created_at_ms: unknown;
|
|
3303
3303
|
key: unknown;
|
|
3304
3304
|
bytes: unknown;
|
|
3305
|
+
changes: unknown;
|
|
3305
3306
|
}>(
|
|
3306
3307
|
`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
|
|
3307
3308
|
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
|
|
@@ -3311,10 +3312,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3311
3312
|
ORDER BY commit_seq LIMIT $4) s
|
|
3312
3313
|
ORDER BY s.commit_seq LIMIT $4)
|
|
3313
3314
|
SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
3314
|
-
z.key AS key, z.bytes AS bytes
|
|
3315
|
+
z.key AS key, z.bytes AS bytes, z.changes AS changes
|
|
3315
3316
|
FROM c
|
|
3316
3317
|
LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
3317
|
-
LEFT JOIN LATERAL (SELECT k.i AS key,
|
|
3318
|
+
LEFT JOIN LATERAL (SELECT k.i AS key, count(*) AS changes,
|
|
3318
3319
|
sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
|
|
3319
3320
|
FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value
|
|
3320
3321
|
WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
|
|
@@ -3334,7 +3335,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3334
3335
|
}),
|
|
3335
3336
|
...(record.key === null
|
|
3336
3337
|
? {}
|
|
3337
|
-
: {
|
|
3338
|
+
: {
|
|
3339
|
+
key: asNumber(record.key),
|
|
3340
|
+
bytes: asNumber(record.bytes),
|
|
3341
|
+
changes: asNumber(record.changes),
|
|
3342
|
+
}),
|
|
3338
3343
|
}));
|
|
3339
3344
|
},
|
|
3340
3345
|
async (keys, commitSeqs) => {
|
package/src/pull.ts
CHANGED
|
@@ -794,12 +794,15 @@ export function planBootstrap(
|
|
|
794
794
|
maxSeq: number,
|
|
795
795
|
horizonSeq: number,
|
|
796
796
|
logEpoch: string,
|
|
797
|
+
keptCursor = 0,
|
|
797
798
|
): BootstrapPlan {
|
|
798
|
-
// §4.7: resume at the token's pin unless the token is unusable or the pin
|
|
799
|
-
// fell behind the horizon
|
|
799
|
+
// §4.7: resume at the token's pin unless the token is unusable, or the pin
|
|
800
|
+
// fell behind the horizon or a restricted bootstrap's kept cursor; then
|
|
801
|
+
// restart from a fresh pin.
|
|
800
802
|
const token = parseBootstrapToken(bootstrapState, unit.tables, maxSeq);
|
|
801
803
|
const resume =
|
|
802
|
-
token !== undefined &&
|
|
804
|
+
token !== undefined &&
|
|
805
|
+
token.asOfCommitSeq >= Math.max(horizonSeq, keptCursor)
|
|
803
806
|
? token
|
|
804
807
|
: undefined;
|
|
805
808
|
const tableIndex = resume?.tableIndex ?? 0;
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -983,6 +983,7 @@ export abstract class SqliteFamilyStorage {
|
|
|
983
983
|
created_at_ms: number | null;
|
|
984
984
|
key: number | null;
|
|
985
985
|
bytes: number | null;
|
|
986
|
+
changes: number | null;
|
|
986
987
|
}>(
|
|
987
988
|
`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
|
|
988
989
|
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
@@ -991,10 +992,10 @@ export abstract class SqliteFamilyStorage {
|
|
|
991
992
|
AND s.commit_seq>k.after AND s.commit_seq<=?3
|
|
992
993
|
ORDER BY s.commit_seq LIMIT ?4)
|
|
993
994
|
SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
994
|
-
z.i AS key, z.bytes AS bytes
|
|
995
|
+
z.i AS key, z.bytes AS bytes, z.changes AS changes
|
|
995
996
|
FROM c
|
|
996
997
|
LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
|
|
997
|
-
LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
|
|
998
|
+
LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i, count(*) AS changes,
|
|
998
999
|
sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
|
|
999
1000
|
FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
|
|
1000
1001
|
CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
|
|
@@ -1013,7 +1014,11 @@ export abstract class SqliteFamilyStorage {
|
|
|
1013
1014
|
}
|
|
1014
1015
|
: {}),
|
|
1015
1016
|
...(record.key !== null
|
|
1016
|
-
? {
|
|
1017
|
+
? {
|
|
1018
|
+
key: record.key,
|
|
1019
|
+
bytes: record.bytes ?? 0,
|
|
1020
|
+
changes: record.changes ?? 0,
|
|
1021
|
+
}
|
|
1017
1022
|
: {}),
|
|
1018
1023
|
})),
|
|
1019
1024
|
async (keys, commitSeqs) =>
|
package/src/storage-query.ts
CHANGED
|
@@ -159,16 +159,18 @@ export interface LiveKey {
|
|
|
159
159
|
|
|
160
160
|
/**
|
|
161
161
|
* One row of a live-window candidate page: a candidate commit joined to its
|
|
162
|
-
* metadata (absent when the commit vanished) and, per key, the
|
|
163
|
-
* (payload, row id, scopes) of its changes that key reads: a
|
|
164
|
-
* and the key's value in the stored scopes. A commit with no
|
|
165
|
-
* comes back once without `key`. Rows are in (commitSeq, key)
|
|
162
|
+
* metadata (absent when the commit vanished) and, per key, the count and
|
|
163
|
+
* stored bytes (payload, row id, scopes) of its changes that key reads: a
|
|
164
|
+
* covered table and the key's value in the stored scopes. A commit with no
|
|
165
|
+
* such change comes back once without `key`. Rows are in (commitSeq, key)
|
|
166
|
+
* order.
|
|
166
167
|
*/
|
|
167
168
|
export interface LiveCandidateRow {
|
|
168
169
|
readonly commitSeq: number;
|
|
169
170
|
readonly meta?: { readonly actorId: string; readonly createdAtMs: number };
|
|
170
171
|
readonly key?: number;
|
|
171
172
|
readonly bytes?: number;
|
|
173
|
+
readonly changes?: number;
|
|
172
174
|
}
|
|
173
175
|
|
|
174
176
|
/**
|
|
@@ -184,6 +186,11 @@ export interface LiveCandidateRow {
|
|
|
184
186
|
* bound. Every candidate advances the scan, so a page of non-matching
|
|
185
187
|
* candidates never repeats.
|
|
186
188
|
*
|
|
189
|
+
* The counts bound the delivered changes from above, because every change
|
|
190
|
+
* a unit keeps is one its key reads: the scan loads pending candidates once
|
|
191
|
+
* their counts could reach the rest of `limitChanges`, so a page loads
|
|
192
|
+
* about the changes it delivers and never a candidate past the cut.
|
|
193
|
+
*
|
|
187
194
|
* The sizes bound the loaded bytes by `limitBytes` before any payload is
|
|
188
195
|
* read: the section stops before the commit that would pass it, and a
|
|
189
196
|
* commit that passes it alone ends the read as `oversized`, naming the
|
|
@@ -236,6 +243,8 @@ export async function readLiveWindowPages(
|
|
|
236
243
|
const limit = Math.max(64, query.limitChanges);
|
|
237
244
|
let delivered = 0;
|
|
238
245
|
let loaded = 0;
|
|
246
|
+
// Upper bound of the changes the pending candidates deliver.
|
|
247
|
+
let bound = 0;
|
|
239
248
|
let scanned = -1;
|
|
240
249
|
// Candidates sized but not loaded yet: (scanned, pending.at(-1)].
|
|
241
250
|
let pending: { commitSeq: number; meta?: LiveCandidateRow['meta'] }[] = [];
|
|
@@ -251,6 +260,7 @@ export async function readLiveWindowPages(
|
|
|
251
260
|
pending.map((c) => c.commitSeq),
|
|
252
261
|
);
|
|
253
262
|
pending = [];
|
|
263
|
+
bound = 0;
|
|
254
264
|
scanned = last.commitSeq;
|
|
255
265
|
for (let i = 0; i < rows.length;) {
|
|
256
266
|
const seq = rows[i]!.commitSeq;
|
|
@@ -287,11 +297,13 @@ export async function readLiveWindowPages(
|
|
|
287
297
|
const head = rows[i]!;
|
|
288
298
|
candidates += 1;
|
|
289
299
|
let bytes = 0;
|
|
300
|
+
let changes = 0;
|
|
290
301
|
const sizes: [key: number, bytes: number][] = [];
|
|
291
302
|
for (; rows[i]?.commitSeq === head.commitSeq; i++) {
|
|
292
303
|
const row = rows[i]!;
|
|
293
304
|
if (row.key === undefined) continue;
|
|
294
305
|
bytes += row.bytes ?? 0;
|
|
306
|
+
changes += row.changes ?? 0;
|
|
295
307
|
sizes.push([row.key, row.bytes ?? 0]);
|
|
296
308
|
}
|
|
297
309
|
if (bytes > query.limitBytes) {
|
|
@@ -324,7 +336,12 @@ export async function readLiveWindowPages(
|
|
|
324
336
|
return { commits, throughSeq: cut ?? scanned };
|
|
325
337
|
}
|
|
326
338
|
loaded += bytes;
|
|
339
|
+
bound += changes;
|
|
327
340
|
pending.push({ commitSeq: head.commitSeq, meta: head.meta });
|
|
341
|
+
if (delivered + bound >= query.limitChanges) {
|
|
342
|
+
const cut = await load();
|
|
343
|
+
if (cut !== undefined) return { commits, throughSeq: cut };
|
|
344
|
+
}
|
|
328
345
|
}
|
|
329
346
|
const cut = await load();
|
|
330
347
|
if (cut !== undefined) return { commits, throughSeq: cut };
|