@syncular/server 0.97.0 → 0.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/events.d.ts +3 -2
- package/dist/handler.js +3 -4
- package/dist/postgres-storage.js +57 -26
- package/dist/sqlite-dialect.js +45 -20
- package/dist/storage-query.d.ts +14 -7
- package/dist/storage-query.js +18 -8
- package/package.json +2 -2
- package/src/events.ts +3 -2
- package/src/handler.ts +3 -4
- package/src/postgres-storage.ts +58 -27
- package/src/sqlite-dialect.ts +47 -22
- package/src/storage-query.ts +19 -8
package/dist/events.d.ts
CHANGED
|
@@ -290,8 +290,9 @@ export interface PruneCompletedEvent {
|
|
|
290
290
|
readonly advanced: boolean;
|
|
291
291
|
readonly removedCommits: number;
|
|
292
292
|
}
|
|
293
|
-
/** The host `resolveScopes` callback threw
|
|
294
|
-
*
|
|
293
|
+
/** The host `resolveScopes` callback threw (§3.2 rule 3): a sync request
|
|
294
|
+
* fails with `sync.internal_error`, a download or realtime registration is
|
|
295
|
+
* denied. */
|
|
295
296
|
export interface ScopesResolveFailedEvent {
|
|
296
297
|
readonly type: 'scopes.resolve_failed';
|
|
297
298
|
readonly atMs: number;
|
package/dist/handler.js
CHANGED
|
@@ -76,9 +76,8 @@ async function resolveOnce(ctx, clientId, schema) {
|
|
|
76
76
|
catch (error) {
|
|
77
77
|
if (error instanceof SyncError)
|
|
78
78
|
throw error;
|
|
79
|
-
// §3.2 rule
|
|
80
|
-
//
|
|
81
|
-
// outage (§7.3.3): it never reaches the lease.
|
|
79
|
+
// §3.2 rule 3: a throw fails the whole request; nothing is revoked,
|
|
80
|
+
// rejected or leased. A throw is NOT an outage (§7.3.3).
|
|
82
81
|
const events = ctx.events;
|
|
83
82
|
if (events !== undefined) {
|
|
84
83
|
emitEvent(events, {
|
|
@@ -90,7 +89,7 @@ async function resolveOnce(ctx, clientId, schema) {
|
|
|
90
89
|
message: error instanceof Error ? error.message : String(error),
|
|
91
90
|
});
|
|
92
91
|
}
|
|
93
|
-
|
|
92
|
+
throw error;
|
|
94
93
|
}
|
|
95
94
|
if (allowed === RESOLVER_OUTAGE) {
|
|
96
95
|
// §7.3.3: the host opted this request into lease authorization. No
|
package/dist/postgres-storage.js
CHANGED
|
@@ -2254,29 +2254,56 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2254
2254
|
readLiveWindow(partition, query) {
|
|
2255
2255
|
// Two statements per page. The candidate read takes one LATERAL range of
|
|
2256
2256
|
// `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
|
|
2257
|
-
// the index, merged in commit order and LIMITed again
|
|
2258
|
-
// the
|
|
2259
|
-
// the
|
|
2260
|
-
//
|
|
2261
|
-
//
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2268
|
-
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
FROM
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2257
|
+
// the index, merged in commit order and LIMITed again into one array.
|
|
2258
|
+
// `walk` then sizes the candidates one at a time and stops after the one
|
|
2259
|
+
// that reaches the budget: per key, the count and byte size of the
|
|
2260
|
+
// commit's changes the key reads. Each change's scopes expand once and
|
|
2261
|
+
// hash-join the keys on (var, value); `octet_length` of a TOASTed payload
|
|
2262
|
+
// reads its pointer, never the value. Every walked commit LEFT JOINs its
|
|
2263
|
+
// meta (NULL when the commit vanished, which still advances the scan).
|
|
2264
|
+
// The change read then seeks only the commits the byte budget admits,
|
|
2265
|
+
// matching each change to the keys by the same join.
|
|
2266
|
+
return readLiveWindowPages(query, async (keys, limit, budget) => {
|
|
2267
|
+
const { rows } = await this.#exec.query(`WITH RECURSIVE k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
|
|
2268
|
+
c AS MATERIALIZED (SELECT coalesce(array_agg(d.commit_seq ORDER BY d.commit_seq), '{}') AS seqs
|
|
2269
|
+
FROM (SELECT DISTINCT s.commit_seq
|
|
2270
|
+
FROM k CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
2271
|
+
WHERE partition=$1 AND var=k.var AND value=k.value
|
|
2272
|
+
AND commit_seq>k.after AND commit_seq<=$3
|
|
2273
|
+
ORDER BY commit_seq LIMIT $4) s
|
|
2274
|
+
ORDER BY s.commit_seq LIMIT $4) d),
|
|
2275
|
+
km AS MATERIALIZED (SELECT jsonb_object_agg(x.var, x.vals) AS m
|
|
2276
|
+
FROM (SELECT var, jsonb_object_agg(value, jsonb_build_array(i, after, tables)) AS vals FROM k GROUP BY var) x),
|
|
2277
|
+
walk(n, commit_seq, changes, bytes, sizes) AS (
|
|
2278
|
+
SELECT 0, NULL::bigint, 0::numeric, 0::numeric, NULL::jsonb
|
|
2279
|
+
UNION ALL
|
|
2280
|
+
SELECT w.n+1, c.seqs[w.n+1], w.changes+z.changes, w.bytes+z.bytes, z.sizes
|
|
2281
|
+
FROM walk w CROSS JOIN c CROSS JOIN km CROSS JOIN LATERAL (
|
|
2282
|
+
SELECT coalesce(sum(g.changes), 0) AS changes, coalesce(sum(g.bytes), 0) AS bytes,
|
|
2283
|
+
jsonb_agg(jsonb_build_array(g.key, g.bytes, g.changes)) AS sizes
|
|
2284
|
+
FROM (SELECT (x.kv->>0)::int AS key, count(*) AS changes,
|
|
2285
|
+
sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
|
|
2286
|
+
FROM sync_changes ch
|
|
2287
|
+
CROSS JOIN LATERAL jsonb_each_text(CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END) e
|
|
2288
|
+
CROSS JOIN LATERAL (SELECT km.m->e.key->e.value AS kv) x
|
|
2289
|
+
WHERE ch.partition=$1 AND ch.commit_seq=c.seqs[w.n+1]
|
|
2290
|
+
AND ch.commit_seq>(x.kv->>1)::bigint AND x.kv->2 @> to_jsonb(ch.tbl)
|
|
2291
|
+
GROUP BY 1) g) z
|
|
2292
|
+
WHERE w.n<cardinality(c.seqs) AND w.changes<$5 AND w.bytes<=$6)
|
|
2293
|
+
SELECT w.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
2294
|
+
(z->>0)::int AS key, (z->>1)::bigint AS bytes, (z->>2)::bigint AS changes
|
|
2295
|
+
FROM walk w
|
|
2296
|
+
LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=w.commit_seq
|
|
2297
|
+
LEFT JOIN LATERAL jsonb_array_elements(w.sizes) z ON true
|
|
2298
|
+
WHERE w.n>0
|
|
2299
|
+
ORDER BY w.commit_seq, key`, [
|
|
2300
|
+
partition,
|
|
2301
|
+
JSON.stringify(keys),
|
|
2302
|
+
query.throughSeq,
|
|
2303
|
+
limit,
|
|
2304
|
+
budget.changes,
|
|
2305
|
+
budget.bytes,
|
|
2306
|
+
]);
|
|
2280
2307
|
return rows.map((record) => ({
|
|
2281
2308
|
commitSeq: asNumber(record.commit_seq),
|
|
2282
2309
|
...(record.actor_id === null
|
|
@@ -2296,12 +2323,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2296
2323
|
}),
|
|
2297
2324
|
}));
|
|
2298
2325
|
}, async (keys, commitSeqs) => {
|
|
2299
|
-
const { rows } = await this.#exec.query(`WITH
|
|
2326
|
+
const { rows } = await this.#exec.query(`WITH km AS MATERIALIZED (SELECT jsonb_object_agg(x.var, x.vals) AS m
|
|
2327
|
+
FROM (SELECT var, jsonb_object_agg(value, jsonb_build_array(after, tables)) AS vals
|
|
2328
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS k(var text, value text, after bigint, tables jsonb) GROUP BY var) x)
|
|
2300
2329
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
2301
2330
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
2302
|
-
FROM sync_changes ch
|
|
2331
|
+
FROM sync_changes ch CROSS JOIN km
|
|
2303
2332
|
WHERE ch.partition=$1 AND ch.commit_seq=ANY(ARRAY(SELECT jsonb_array_elements_text($3::text::jsonb)::bigint))
|
|
2304
|
-
AND EXISTS (SELECT 1 FROM
|
|
2333
|
+
AND EXISTS (SELECT 1 FROM jsonb_each_text(CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END) e
|
|
2334
|
+
CROSS JOIN LATERAL (SELECT km.m->e.key->e.value AS kv) x
|
|
2335
|
+
WHERE ch.commit_seq>(x.kv->>0)::bigint AND x.kv->1 @> to_jsonb(ch.tbl))
|
|
2305
2336
|
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)]);
|
|
2306
2337
|
return rows.map((record) => ({
|
|
2307
2338
|
commitSeq: asNumber(record.commit_seq),
|
package/dist/sqlite-dialect.js
CHANGED
|
@@ -646,28 +646,52 @@ export class SqliteFamilyStorage {
|
|
|
646
646
|
// Two statements per page. The candidate read binds the unit keys as one
|
|
647
647
|
// JSON array; the CROSS JOIN keeps them the outer loop, so each key seeks
|
|
648
648
|
// one range of `sync_change_scopes_by_key` (INDEXED BY fails loudly when
|
|
649
|
-
// the index is missing).
|
|
650
|
-
//
|
|
651
|
-
// the byte size of the changes the key reads
|
|
652
|
-
//
|
|
653
|
-
//
|
|
654
|
-
//
|
|
655
|
-
|
|
656
|
-
|
|
649
|
+
// the index is missing). `walk` then sizes the candidates one at a time,
|
|
650
|
+
// in commit order, and stops after the one that reaches the budget: per
|
|
651
|
+
// key, the count and byte size of the commit's changes the key reads.
|
|
652
|
+
// Each change's scopes expand once and join the keys on (var, value)
|
|
653
|
+
// through an automatic index; `length()` of a blob reads the record
|
|
654
|
+
// header, never the payload. Every walked commit LEFT JOINs its meta
|
|
655
|
+
// (NULL when the commit vanished, which still advances the scan). The
|
|
656
|
+
// change read then seeks only the commits the byte budget admits,
|
|
657
|
+
// through the primary key, matching each change to the keys by the same
|
|
658
|
+
// join.
|
|
659
|
+
return readLiveWindowPages(query, async (keys, limit, budget) => (await this.guardedAll(`WITH RECURSIVE k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
|
|
660
|
+
c AS MATERIALIZED (SELECT row_number() OVER (ORDER BY commit_seq) AS n, commit_seq FROM (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
657
661
|
FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
|
|
658
662
|
ON s.partition=?1 AND s.var=k.var AND s.value=k.value
|
|
659
663
|
AND s.commit_seq>k.after AND s.commit_seq<=?3
|
|
660
|
-
ORDER BY s.commit_seq LIMIT ?4)
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
664
|
+
ORDER BY s.commit_seq LIMIT ?4)),
|
|
665
|
+
walk(n, commit_seq, sizes, changes, bytes) AS (
|
|
666
|
+
SELECT 0, NULL, '[]', 0, 0
|
|
667
|
+
UNION ALL
|
|
668
|
+
SELECT c.n, c.commit_seq,
|
|
669
|
+
(SELECT json_group_array(json_array(g.i, g.bytes, g.changes)) FROM (SELECT k.i AS i, count(*) AS changes,
|
|
670
|
+
sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
|
|
671
|
+
FROM sync_changes AS ch CROSS JOIN json_each(ch.scopes) AS e
|
|
672
|
+
CROSS JOIN k ON k.var=e.key AND k.value=e.value
|
|
673
|
+
WHERE ch.partition=?1 AND ch.commit_seq=c.commit_seq
|
|
674
|
+
AND ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables))
|
|
675
|
+
GROUP BY k.i) AS g),
|
|
676
|
+
w.changes + (SELECT total(value ->> 2) FROM json_each(w.sizes)),
|
|
677
|
+
w.bytes + (SELECT total(value ->> 1) FROM json_each(w.sizes))
|
|
678
|
+
FROM walk AS w JOIN c ON c.n=w.n+1
|
|
679
|
+
WHERE w.changes + (SELECT total(value ->> 2) FROM json_each(w.sizes)) < ?5
|
|
680
|
+
AND w.bytes + (SELECT total(value ->> 1) FROM json_each(w.sizes)) <= ?6)
|
|
681
|
+
SELECT w.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
682
|
+
z.value ->> 0 AS key, z.value ->> 1 AS bytes, z.value ->> 2 AS changes
|
|
683
|
+
FROM walk AS w
|
|
684
|
+
LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=w.commit_seq
|
|
685
|
+
LEFT JOIN json_each(w.sizes) AS z
|
|
686
|
+
WHERE w.n>0
|
|
687
|
+
ORDER BY w.commit_seq, key`, [
|
|
688
|
+
partition,
|
|
689
|
+
JSON.stringify(keys),
|
|
690
|
+
query.throughSeq,
|
|
691
|
+
limit,
|
|
692
|
+
budget.changes,
|
|
693
|
+
budget.bytes,
|
|
694
|
+
])).map((record) => ({
|
|
671
695
|
commitSeq: record.commit_seq,
|
|
672
696
|
...(record.actor_id !== null && record.created_at_ms !== null
|
|
673
697
|
? {
|
|
@@ -688,7 +712,8 @@ export class SqliteFamilyStorage {
|
|
|
688
712
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
689
713
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
690
714
|
FROM json_each(?3) AS c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.value
|
|
691
|
-
WHERE EXISTS (SELECT 1 FROM
|
|
715
|
+
WHERE EXISTS (SELECT 1 FROM json_each(ch.scopes) AS e CROSS JOIN k ON k.var=e.key AND k.value=e.value
|
|
716
|
+
WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)))
|
|
692
717
|
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)])).map((record) => ({
|
|
693
718
|
commitSeq: record.commit_seq,
|
|
694
719
|
change: toStoredChange(record),
|
package/dist/storage-query.d.ts
CHANGED
|
@@ -74,12 +74,16 @@ export interface LiveCandidateRow {
|
|
|
74
74
|
readonly changes?: number;
|
|
75
75
|
}
|
|
76
76
|
/**
|
|
77
|
-
* The §4.5 live scan over a backend's two reads. `readCandidates`
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
77
|
+
* The §4.5 live scan over a backend's two reads. `readCandidates` takes up
|
|
78
|
+
* to `limit` candidate commits above every key range's bound and sizes them
|
|
79
|
+
* in commit order, without payload, until their change count reaches
|
|
80
|
+
* `budget.changes` or their byte size passes `budget.bytes`. It returns the
|
|
81
|
+
* sized candidates with their per-key sizes, the one that reached the budget
|
|
82
|
+
* last, so a page sizes the commits it delivers and one more. `readChanges`
|
|
83
|
+
* loads the changes the keys read in the given candidate commits, in
|
|
84
|
+
* (commitSeq, change index) order; it never visits a commit outside that
|
|
85
|
+
* list, so the cost of a page follows its candidates and not the length of
|
|
86
|
+
* the change log below them. Each distinct unit key (the first variable, keys sorted)
|
|
83
87
|
* becomes one index range starting at the lowest bound of the units that
|
|
84
88
|
* share it. A change is kept when a unit covers its table, the pair's
|
|
85
89
|
* effective scopes match it, and its commit lies above that unit's own
|
|
@@ -97,7 +101,10 @@ export interface LiveCandidateRow {
|
|
|
97
101
|
* units of the keys that read the most of it, largest first, until the rest
|
|
98
102
|
* fits (§4.6).
|
|
99
103
|
*/
|
|
100
|
-
export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number
|
|
104
|
+
export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number, budget: {
|
|
105
|
+
readonly changes: number;
|
|
106
|
+
readonly bytes: number;
|
|
107
|
+
}) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[], commitSeqs: readonly number[]) => Promise<readonly {
|
|
101
108
|
readonly commitSeq: number;
|
|
102
109
|
readonly change: StoredChange;
|
|
103
110
|
}[]>): Promise<LiveWindow>;
|
package/dist/storage-query.js
CHANGED
|
@@ -101,12 +101,16 @@ export async function transactionQuery(tx, query) {
|
|
|
101
101
|
return tx.queryAuthoritative(query);
|
|
102
102
|
}
|
|
103
103
|
/**
|
|
104
|
-
* The §4.5 live scan over a backend's two reads. `readCandidates`
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
104
|
+
* The §4.5 live scan over a backend's two reads. `readCandidates` takes up
|
|
105
|
+
* to `limit` candidate commits above every key range's bound and sizes them
|
|
106
|
+
* in commit order, without payload, until their change count reaches
|
|
107
|
+
* `budget.changes` or their byte size passes `budget.bytes`. It returns the
|
|
108
|
+
* sized candidates with their per-key sizes, the one that reached the budget
|
|
109
|
+
* last, so a page sizes the commits it delivers and one more. `readChanges`
|
|
110
|
+
* loads the changes the keys read in the given candidate commits, in
|
|
111
|
+
* (commitSeq, change index) order; it never visits a commit outside that
|
|
112
|
+
* list, so the cost of a page follows its candidates and not the length of
|
|
113
|
+
* the change log below them. Each distinct unit key (the first variable, keys sorted)
|
|
110
114
|
* becomes one index range starting at the lowest bound of the units that
|
|
111
115
|
* share it. A change is kept when a unit covers its table, the pair's
|
|
112
116
|
* effective scopes match it, and its commit lies above that unit's own
|
|
@@ -199,8 +203,13 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
|
|
|
199
203
|
return undefined;
|
|
200
204
|
};
|
|
201
205
|
while (keys.length > 0) {
|
|
202
|
-
const rows = await readCandidates(scanKeys(), limit
|
|
206
|
+
const rows = await readCandidates(scanKeys(), limit, {
|
|
207
|
+
changes: query.limitChanges - delivered,
|
|
208
|
+
bytes: query.limitBytes - loaded,
|
|
209
|
+
});
|
|
203
210
|
let candidates = 0;
|
|
211
|
+
// The change budget ended the read, possibly before `limit` candidates.
|
|
212
|
+
let reached = false;
|
|
204
213
|
for (let i = 0; i < rows.length;) {
|
|
205
214
|
const head = rows[i];
|
|
206
215
|
candidates += 1;
|
|
@@ -249,6 +258,7 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
|
|
|
249
258
|
bound += changes;
|
|
250
259
|
pending.push({ commitSeq: head.commitSeq, meta: head.meta });
|
|
251
260
|
if (delivered + bound >= query.limitChanges) {
|
|
261
|
+
reached = true;
|
|
252
262
|
const cut = await load();
|
|
253
263
|
if (cut !== undefined)
|
|
254
264
|
return { commits, throughSeq: cut };
|
|
@@ -257,7 +267,7 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
|
|
|
257
267
|
const cut = await load();
|
|
258
268
|
if (cut !== undefined)
|
|
259
269
|
return { commits, throughSeq: cut };
|
|
260
|
-
if (candidates < limit)
|
|
270
|
+
if (candidates < limit && !reached)
|
|
261
271
|
break;
|
|
262
272
|
}
|
|
263
273
|
return { commits, throughSeq: query.throughSeq };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.99.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.99.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/events.ts
CHANGED
|
@@ -324,8 +324,9 @@ export interface PruneCompletedEvent {
|
|
|
324
324
|
readonly removedCommits: number;
|
|
325
325
|
}
|
|
326
326
|
|
|
327
|
-
/** The host `resolveScopes` callback threw
|
|
328
|
-
*
|
|
327
|
+
/** The host `resolveScopes` callback threw (§3.2 rule 3): a sync request
|
|
328
|
+
* fails with `sync.internal_error`, a download or realtime registration is
|
|
329
|
+
* denied. */
|
|
329
330
|
export interface ScopesResolveFailedEvent {
|
|
330
331
|
readonly type: 'scopes.resolve_failed';
|
|
331
332
|
readonly atMs: number;
|
package/src/handler.ts
CHANGED
|
@@ -205,9 +205,8 @@ async function resolveOnce(
|
|
|
205
205
|
});
|
|
206
206
|
} catch (error) {
|
|
207
207
|
if (error instanceof SyncError) throw error;
|
|
208
|
-
// §3.2 rule
|
|
209
|
-
//
|
|
210
|
-
// outage (§7.3.3): it never reaches the lease.
|
|
208
|
+
// §3.2 rule 3: a throw fails the whole request; nothing is revoked,
|
|
209
|
+
// rejected or leased. A throw is NOT an outage (§7.3.3).
|
|
211
210
|
const events = ctx.events;
|
|
212
211
|
if (events !== undefined) {
|
|
213
212
|
emitEvent(events, {
|
|
@@ -219,7 +218,7 @@ async function resolveOnce(
|
|
|
219
218
|
message: error instanceof Error ? error.message : String(error),
|
|
220
219
|
});
|
|
221
220
|
}
|
|
222
|
-
|
|
221
|
+
throw error;
|
|
223
222
|
}
|
|
224
223
|
|
|
225
224
|
if (allowed === RESOLVER_OUTAGE) {
|
package/src/postgres-storage.ts
CHANGED
|
@@ -3328,14 +3328,18 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3328
3328
|
): Promise<LiveWindow> {
|
|
3329
3329
|
// Two statements per page. The candidate read takes one LATERAL range of
|
|
3330
3330
|
// `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
|
|
3331
|
-
// the index, merged in commit order and LIMITed again
|
|
3332
|
-
// the
|
|
3333
|
-
// the
|
|
3334
|
-
//
|
|
3335
|
-
//
|
|
3331
|
+
// the index, merged in commit order and LIMITed again into one array.
|
|
3332
|
+
// `walk` then sizes the candidates one at a time and stops after the one
|
|
3333
|
+
// that reaches the budget: per key, the count and byte size of the
|
|
3334
|
+
// commit's changes the key reads. Each change's scopes expand once and
|
|
3335
|
+
// hash-join the keys on (var, value); `octet_length` of a TOASTed payload
|
|
3336
|
+
// reads its pointer, never the value. Every walked commit LEFT JOINs its
|
|
3337
|
+
// meta (NULL when the commit vanished, which still advances the scan).
|
|
3338
|
+
// The change read then seeks only the commits the byte budget admits,
|
|
3339
|
+
// matching each change to the keys by the same join.
|
|
3336
3340
|
return readLiveWindowPages(
|
|
3337
3341
|
query,
|
|
3338
|
-
async (keys, limit) => {
|
|
3342
|
+
async (keys, limit, budget) => {
|
|
3339
3343
|
const { rows } = await this.#exec.query<{
|
|
3340
3344
|
commit_seq: unknown;
|
|
3341
3345
|
actor_id: string | null;
|
|
@@ -3344,24 +3348,47 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3344
3348
|
bytes: unknown;
|
|
3345
3349
|
changes: unknown;
|
|
3346
3350
|
}>(
|
|
3347
|
-
`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
|
|
3348
|
-
c AS MATERIALIZED (SELECT
|
|
3349
|
-
FROM
|
|
3350
|
-
|
|
3351
|
-
AND
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3351
|
+
`WITH RECURSIVE k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
|
|
3352
|
+
c AS MATERIALIZED (SELECT coalesce(array_agg(d.commit_seq ORDER BY d.commit_seq), '{}') AS seqs
|
|
3353
|
+
FROM (SELECT DISTINCT s.commit_seq
|
|
3354
|
+
FROM k CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
3355
|
+
WHERE partition=$1 AND var=k.var AND value=k.value
|
|
3356
|
+
AND commit_seq>k.after AND commit_seq<=$3
|
|
3357
|
+
ORDER BY commit_seq LIMIT $4) s
|
|
3358
|
+
ORDER BY s.commit_seq LIMIT $4) d),
|
|
3359
|
+
km AS MATERIALIZED (SELECT jsonb_object_agg(x.var, x.vals) AS m
|
|
3360
|
+
FROM (SELECT var, jsonb_object_agg(value, jsonb_build_array(i, after, tables)) AS vals FROM k GROUP BY var) x),
|
|
3361
|
+
walk(n, commit_seq, changes, bytes, sizes) AS (
|
|
3362
|
+
SELECT 0, NULL::bigint, 0::numeric, 0::numeric, NULL::jsonb
|
|
3363
|
+
UNION ALL
|
|
3364
|
+
SELECT w.n+1, c.seqs[w.n+1], w.changes+z.changes, w.bytes+z.bytes, z.sizes
|
|
3365
|
+
FROM walk w CROSS JOIN c CROSS JOIN km CROSS JOIN LATERAL (
|
|
3366
|
+
SELECT coalesce(sum(g.changes), 0) AS changes, coalesce(sum(g.bytes), 0) AS bytes,
|
|
3367
|
+
jsonb_agg(jsonb_build_array(g.key, g.bytes, g.changes)) AS sizes
|
|
3368
|
+
FROM (SELECT (x.kv->>0)::int AS key, count(*) AS changes,
|
|
3369
|
+
sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
|
|
3370
|
+
FROM sync_changes ch
|
|
3371
|
+
CROSS JOIN LATERAL jsonb_each_text(CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END) e
|
|
3372
|
+
CROSS JOIN LATERAL (SELECT km.m->e.key->e.value AS kv) x
|
|
3373
|
+
WHERE ch.partition=$1 AND ch.commit_seq=c.seqs[w.n+1]
|
|
3374
|
+
AND ch.commit_seq>(x.kv->>1)::bigint AND x.kv->2 @> to_jsonb(ch.tbl)
|
|
3375
|
+
GROUP BY 1) g) z
|
|
3376
|
+
WHERE w.n<cardinality(c.seqs) AND w.changes<$5 AND w.bytes<=$6)
|
|
3377
|
+
SELECT w.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
3378
|
+
(z->>0)::int AS key, (z->>1)::bigint AS bytes, (z->>2)::bigint AS changes
|
|
3379
|
+
FROM walk w
|
|
3380
|
+
LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=w.commit_seq
|
|
3381
|
+
LEFT JOIN LATERAL jsonb_array_elements(w.sizes) z ON true
|
|
3382
|
+
WHERE w.n>0
|
|
3383
|
+
ORDER BY w.commit_seq, key`,
|
|
3384
|
+
[
|
|
3385
|
+
partition,
|
|
3386
|
+
JSON.stringify(keys),
|
|
3387
|
+
query.throughSeq,
|
|
3388
|
+
limit,
|
|
3389
|
+
budget.changes,
|
|
3390
|
+
budget.bytes,
|
|
3391
|
+
],
|
|
3365
3392
|
);
|
|
3366
3393
|
return rows.map((record) => ({
|
|
3367
3394
|
commitSeq: asNumber(record.commit_seq),
|
|
@@ -3386,12 +3413,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3386
3413
|
const { rows } = await this.#exec.query<
|
|
3387
3414
|
StoredChangeRecord & { commit_seq: unknown }
|
|
3388
3415
|
>(
|
|
3389
|
-
`WITH
|
|
3416
|
+
`WITH km AS MATERIALIZED (SELECT jsonb_object_agg(x.var, x.vals) AS m
|
|
3417
|
+
FROM (SELECT var, jsonb_object_agg(value, jsonb_build_array(after, tables)) AS vals
|
|
3418
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS k(var text, value text, after bigint, tables jsonb) GROUP BY var) x)
|
|
3390
3419
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
3391
3420
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
3392
|
-
FROM sync_changes ch
|
|
3421
|
+
FROM sync_changes ch CROSS JOIN km
|
|
3393
3422
|
WHERE ch.partition=$1 AND ch.commit_seq=ANY(ARRAY(SELECT jsonb_array_elements_text($3::text::jsonb)::bigint))
|
|
3394
|
-
AND EXISTS (SELECT 1 FROM
|
|
3423
|
+
AND EXISTS (SELECT 1 FROM jsonb_each_text(CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END) e
|
|
3424
|
+
CROSS JOIN LATERAL (SELECT km.m->e.key->e.value AS kv) x
|
|
3425
|
+
WHERE ch.commit_seq>(x.kv->>0)::bigint AND x.kv->1 @> to_jsonb(ch.tbl))
|
|
3395
3426
|
ORDER BY ch.commit_seq, ch.idx`,
|
|
3396
3427
|
[partition, JSON.stringify(keys), JSON.stringify(commitSeqs)],
|
|
3397
3428
|
);
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -1023,15 +1023,19 @@ export abstract class SqliteFamilyStorage {
|
|
|
1023
1023
|
// Two statements per page. The candidate read binds the unit keys as one
|
|
1024
1024
|
// JSON array; the CROSS JOIN keeps them the outer loop, so each key seeks
|
|
1025
1025
|
// one range of `sync_change_scopes_by_key` (INDEXED BY fails loudly when
|
|
1026
|
-
// the index is missing).
|
|
1027
|
-
//
|
|
1028
|
-
// the byte size of the changes the key reads
|
|
1029
|
-
//
|
|
1030
|
-
//
|
|
1031
|
-
//
|
|
1026
|
+
// the index is missing). `walk` then sizes the candidates one at a time,
|
|
1027
|
+
// in commit order, and stops after the one that reaches the budget: per
|
|
1028
|
+
// key, the count and byte size of the commit's changes the key reads.
|
|
1029
|
+
// Each change's scopes expand once and join the keys on (var, value)
|
|
1030
|
+
// through an automatic index; `length()` of a blob reads the record
|
|
1031
|
+
// header, never the payload. Every walked commit LEFT JOINs its meta
|
|
1032
|
+
// (NULL when the commit vanished, which still advances the scan). The
|
|
1033
|
+
// change read then seeks only the commits the byte budget admits,
|
|
1034
|
+
// through the primary key, matching each change to the keys by the same
|
|
1035
|
+
// join.
|
|
1032
1036
|
return readLiveWindowPages(
|
|
1033
1037
|
query,
|
|
1034
|
-
async (keys, limit) =>
|
|
1038
|
+
async (keys, limit, budget) =>
|
|
1035
1039
|
(
|
|
1036
1040
|
await this.guardedAll<{
|
|
1037
1041
|
commit_seq: number;
|
|
@@ -1041,23 +1045,43 @@ export abstract class SqliteFamilyStorage {
|
|
|
1041
1045
|
bytes: number | null;
|
|
1042
1046
|
changes: number | null;
|
|
1043
1047
|
}>(
|
|
1044
|
-
`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
|
|
1045
|
-
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
1048
|
+
`WITH RECURSIVE k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
|
|
1049
|
+
c AS MATERIALIZED (SELECT row_number() OVER (ORDER BY commit_seq) AS n, commit_seq FROM (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
1046
1050
|
FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
|
|
1047
1051
|
ON s.partition=?1 AND s.var=k.var AND s.value=k.value
|
|
1048
1052
|
AND s.commit_seq>k.after AND s.commit_seq<=?3
|
|
1049
|
-
ORDER BY s.commit_seq LIMIT ?4)
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1053
|
+
ORDER BY s.commit_seq LIMIT ?4)),
|
|
1054
|
+
walk(n, commit_seq, sizes, changes, bytes) AS (
|
|
1055
|
+
SELECT 0, NULL, '[]', 0, 0
|
|
1056
|
+
UNION ALL
|
|
1057
|
+
SELECT c.n, c.commit_seq,
|
|
1058
|
+
(SELECT json_group_array(json_array(g.i, g.bytes, g.changes)) FROM (SELECT k.i AS i, count(*) AS changes,
|
|
1059
|
+
sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
|
|
1060
|
+
FROM sync_changes AS ch CROSS JOIN json_each(ch.scopes) AS e
|
|
1061
|
+
CROSS JOIN k ON k.var=e.key AND k.value=e.value
|
|
1062
|
+
WHERE ch.partition=?1 AND ch.commit_seq=c.commit_seq
|
|
1063
|
+
AND ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables))
|
|
1064
|
+
GROUP BY k.i) AS g),
|
|
1065
|
+
w.changes + (SELECT total(value ->> 2) FROM json_each(w.sizes)),
|
|
1066
|
+
w.bytes + (SELECT total(value ->> 1) FROM json_each(w.sizes))
|
|
1067
|
+
FROM walk AS w JOIN c ON c.n=w.n+1
|
|
1068
|
+
WHERE w.changes + (SELECT total(value ->> 2) FROM json_each(w.sizes)) < ?5
|
|
1069
|
+
AND w.bytes + (SELECT total(value ->> 1) FROM json_each(w.sizes)) <= ?6)
|
|
1070
|
+
SELECT w.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
1071
|
+
z.value ->> 0 AS key, z.value ->> 1 AS bytes, z.value ->> 2 AS changes
|
|
1072
|
+
FROM walk AS w
|
|
1073
|
+
LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=w.commit_seq
|
|
1074
|
+
LEFT JOIN json_each(w.sizes) AS z
|
|
1075
|
+
WHERE w.n>0
|
|
1076
|
+
ORDER BY w.commit_seq, key`,
|
|
1077
|
+
[
|
|
1078
|
+
partition,
|
|
1079
|
+
JSON.stringify(keys),
|
|
1080
|
+
query.throughSeq,
|
|
1081
|
+
limit,
|
|
1082
|
+
budget.changes,
|
|
1083
|
+
budget.bytes,
|
|
1084
|
+
],
|
|
1061
1085
|
)
|
|
1062
1086
|
).map((record) => ({
|
|
1063
1087
|
commitSeq: record.commit_seq,
|
|
@@ -1084,7 +1108,8 @@ export abstract class SqliteFamilyStorage {
|
|
|
1084
1108
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
1085
1109
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
1086
1110
|
FROM json_each(?3) AS c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.value
|
|
1087
|
-
WHERE EXISTS (SELECT 1 FROM
|
|
1111
|
+
WHERE EXISTS (SELECT 1 FROM json_each(ch.scopes) AS e CROSS JOIN k ON k.var=e.key AND k.value=e.value
|
|
1112
|
+
WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)))
|
|
1088
1113
|
ORDER BY ch.commit_seq, ch.idx`,
|
|
1089
1114
|
[partition, JSON.stringify(keys), JSON.stringify(commitSeqs)],
|
|
1090
1115
|
)
|
package/src/storage-query.ts
CHANGED
|
@@ -177,12 +177,16 @@ export interface LiveCandidateRow {
|
|
|
177
177
|
}
|
|
178
178
|
|
|
179
179
|
/**
|
|
180
|
-
* The §4.5 live scan over a backend's two reads. `readCandidates`
|
|
181
|
-
*
|
|
182
|
-
*
|
|
183
|
-
*
|
|
184
|
-
*
|
|
185
|
-
*
|
|
180
|
+
* The §4.5 live scan over a backend's two reads. `readCandidates` takes up
|
|
181
|
+
* to `limit` candidate commits above every key range's bound and sizes them
|
|
182
|
+
* in commit order, without payload, until their change count reaches
|
|
183
|
+
* `budget.changes` or their byte size passes `budget.bytes`. It returns the
|
|
184
|
+
* sized candidates with their per-key sizes, the one that reached the budget
|
|
185
|
+
* last, so a page sizes the commits it delivers and one more. `readChanges`
|
|
186
|
+
* loads the changes the keys read in the given candidate commits, in
|
|
187
|
+
* (commitSeq, change index) order; it never visits a commit outside that
|
|
188
|
+
* list, so the cost of a page follows its candidates and not the length of
|
|
189
|
+
* the change log below them. Each distinct unit key (the first variable, keys sorted)
|
|
186
190
|
* becomes one index range starting at the lowest bound of the units that
|
|
187
191
|
* share it. A change is kept when a unit covers its table, the pair's
|
|
188
192
|
* effective scopes match it, and its commit lies above that unit's own
|
|
@@ -205,6 +209,7 @@ export async function readLiveWindowPages(
|
|
|
205
209
|
readCandidates: (
|
|
206
210
|
keys: readonly LiveKey[],
|
|
207
211
|
limit: number,
|
|
212
|
+
budget: { readonly changes: number; readonly bytes: number },
|
|
208
213
|
) => Promise<readonly LiveCandidateRow[]>,
|
|
209
214
|
readChanges: (
|
|
210
215
|
keys: readonly LiveKey[],
|
|
@@ -294,8 +299,13 @@ export async function readLiveWindowPages(
|
|
|
294
299
|
return undefined;
|
|
295
300
|
};
|
|
296
301
|
while (keys.length > 0) {
|
|
297
|
-
const rows = await readCandidates(scanKeys(), limit
|
|
302
|
+
const rows = await readCandidates(scanKeys(), limit, {
|
|
303
|
+
changes: query.limitChanges - delivered,
|
|
304
|
+
bytes: query.limitBytes - loaded,
|
|
305
|
+
});
|
|
298
306
|
let candidates = 0;
|
|
307
|
+
// The change budget ended the read, possibly before `limit` candidates.
|
|
308
|
+
let reached = false;
|
|
299
309
|
for (let i = 0; i < rows.length;) {
|
|
300
310
|
const head = rows[i]!;
|
|
301
311
|
candidates += 1;
|
|
@@ -342,13 +352,14 @@ export async function readLiveWindowPages(
|
|
|
342
352
|
bound += changes;
|
|
343
353
|
pending.push({ commitSeq: head.commitSeq, meta: head.meta });
|
|
344
354
|
if (delivered + bound >= query.limitChanges) {
|
|
355
|
+
reached = true;
|
|
345
356
|
const cut = await load();
|
|
346
357
|
if (cut !== undefined) return { commits, throughSeq: cut };
|
|
347
358
|
}
|
|
348
359
|
}
|
|
349
360
|
const cut = await load();
|
|
350
361
|
if (cut !== undefined) return { commits, throughSeq: cut };
|
|
351
|
-
if (candidates < limit) break;
|
|
362
|
+
if (candidates < limit && !reached) break;
|
|
352
363
|
}
|
|
353
364
|
return { commits, throughSeq: query.throughSeq };
|
|
354
365
|
}
|