@syncular/server 0.68.0 → 0.70.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context.d.ts +7 -0
- package/dist/context.js +2 -0
- package/dist/handler.js +18 -4
- package/dist/local-authority.js +4 -0
- package/dist/postgres-storage.js +35 -15
- package/dist/realtime.js +10 -0
- package/dist/sqlite-dialect.js +35 -20
- package/dist/storage-query.d.ts +34 -14
- package/dist/storage-query.js +114 -36
- package/dist/storage.d.ts +22 -5
- package/package.json +2 -2
- package/src/context.ts +9 -0
- package/src/handler.ts +16 -4
- package/src/local-authority.ts +4 -0
- package/src/postgres-storage.ts +67 -42
- package/src/realtime.ts +23 -0
- package/src/sqlite-dialect.ts +67 -47
- package/src/storage-query.ts +129 -41
- package/src/storage.ts +22 -5
package/dist/context.d.ts
CHANGED
|
@@ -73,6 +73,13 @@ export interface ServerLimits {
|
|
|
73
73
|
readonly maxSubscriptionsPerRequest: number;
|
|
74
74
|
/** Cap on concurrent registered-query watches per watch session. */
|
|
75
75
|
readonly maxWatchesPerSession: number;
|
|
76
|
+
/**
|
|
77
|
+
* §4.6 byte budget of one live section: the stored bytes (payload, row id,
|
|
78
|
+
* scopes) of the changes a pull loads from the log. A section stops before
|
|
79
|
+
* the commit that would pass it; a commit whose changes for the scan set
|
|
80
|
+
* pass it alone resets the units it reaches, which bootstrap instead.
|
|
81
|
+
*/
|
|
82
|
+
readonly maxLiveSectionBytes: number;
|
|
76
83
|
}
|
|
77
84
|
export declare const DEFAULT_LIMITS: ServerLimits;
|
|
78
85
|
/** Receives applied commits for realtime fanout (§8.2). */
|
package/dist/context.js
CHANGED
|
@@ -21,6 +21,8 @@ export const DEFAULT_LIMITS = {
|
|
|
21
21
|
maxSubscriptionsPerRequest: 500,
|
|
22
22
|
// Every commit reruns a session's affected watches one after another.
|
|
23
23
|
maxWatchesPerSession: 100,
|
|
24
|
+
// The live section is buffered whole before the first unit section.
|
|
25
|
+
maxLiveSectionBytes: 16 * 1024 * 1024,
|
|
24
26
|
};
|
|
25
27
|
export function clockOf(ctx) {
|
|
26
28
|
return ctx.clock ?? Date.now;
|
package/dist/handler.js
CHANGED
|
@@ -597,19 +597,33 @@ async function* streamResponse(plan, ctx, schema, report) {
|
|
|
597
597
|
// resets the live section), and the section is read again.
|
|
598
598
|
let live;
|
|
599
599
|
for (;;) {
|
|
600
|
-
const
|
|
600
|
+
const scanned = units.flatMap(({ plan: unit }, index) => {
|
|
601
601
|
const afterSeq = bound(states[index]);
|
|
602
602
|
return afterSeq === undefined
|
|
603
603
|
? []
|
|
604
|
-
: [{ scopes: unit.scopes, afterSeq, tables: unit.pairs }];
|
|
604
|
+
: [{ index, scopes: unit.scopes, afterSeq, tables: unit.pairs }];
|
|
605
605
|
});
|
|
606
|
-
if (liveReset ||
|
|
606
|
+
if (liveReset || scanned.length === 0)
|
|
607
607
|
break;
|
|
608
608
|
live = await ctx.storage.readLiveWindow(ctx.partition, {
|
|
609
|
-
units:
|
|
609
|
+
units: scanned,
|
|
610
610
|
throughSeq: maxSeq,
|
|
611
611
|
limitChanges: limits.limitCommits,
|
|
612
|
+
limitBytes: limitsOf(ctx).maxLiveSectionBytes,
|
|
612
613
|
});
|
|
614
|
+
// §4.6: a commit past the byte budget on its own cannot be served
|
|
615
|
+
// from the log; the units it reaches reset and bootstrap.
|
|
616
|
+
for (const at of live.oversized?.units ?? []) {
|
|
617
|
+
const index = scanned[at].index;
|
|
618
|
+
if (states[index].kind === 'live')
|
|
619
|
+
liveReset = true;
|
|
620
|
+
else
|
|
621
|
+
states[index] = { kind: 'reset' };
|
|
622
|
+
}
|
|
623
|
+
if (live.oversized !== undefined) {
|
|
624
|
+
live = undefined;
|
|
625
|
+
continue;
|
|
626
|
+
}
|
|
613
627
|
const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
614
628
|
let expired = false;
|
|
615
629
|
for (const [index, state] of states.entries()) {
|
package/dist/local-authority.js
CHANGED
|
@@ -86,6 +86,10 @@ export async function openLocalAuthority(options) {
|
|
|
86
86
|
segments: new MemorySegmentStore(),
|
|
87
87
|
resolveScopes: () => allowed,
|
|
88
88
|
clock,
|
|
89
|
+
// §12.2: the client's own process serves its own commits; the
|
|
90
|
+
// live byte budget (§4.6) bounds a shared server's heap and does
|
|
91
|
+
// not apply here, as in the native authority.
|
|
92
|
+
limits: { maxLiveSectionBytes: Number.MAX_SAFE_INTEGER },
|
|
89
93
|
...(options.crdtMergers !== undefined
|
|
90
94
|
? { crdtMergers: options.crdtMergers }
|
|
91
95
|
: {}),
|
package/dist/postgres-storage.js
CHANGED
|
@@ -2253,24 +2253,32 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2253
2253
|
return row === undefined || row.seq === null ? 0 : asNumber(row.seq);
|
|
2254
2254
|
}
|
|
2255
2255
|
readLiveWindow(partition, query) {
|
|
2256
|
-
//
|
|
2256
|
+
// Two statements per page. The candidate read takes one LATERAL range of
|
|
2257
2257
|
// `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
|
|
2258
|
-
// the index, merged in commit order and LIMITed again, then LEFT
|
|
2259
|
-
//
|
|
2260
|
-
//
|
|
2258
|
+
// the index, merged in commit order and LIMITed again, then LEFT JOINs
|
|
2259
|
+
// the commit meta (NULL when the commit vanished, which still advances
|
|
2260
|
+
// the scan) and, per key, the byte size of the changes the key reads;
|
|
2261
|
+
// `octet_length` of a TOASTed payload reads its pointer, never the value.
|
|
2262
|
+
// The change read then loads only those changes of the commits the byte
|
|
2263
|
+
// budget admits.
|
|
2261
2264
|
return readLiveWindowPages(query, async (keys, limit) => {
|
|
2262
|
-
const { rows } = await this.#exec.query(`SELECT
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS k(var text, value text, after bigint)
|
|
2266
|
-
CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
2265
|
+
const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
|
|
2266
|
+
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
|
|
2267
|
+
FROM k CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
2267
2268
|
WHERE partition=$1 AND var=k.var AND value=k.value
|
|
2268
2269
|
AND commit_seq>k.after AND commit_seq<=$3
|
|
2269
2270
|
ORDER BY commit_seq LIMIT $4) s
|
|
2270
|
-
ORDER BY s.commit_seq LIMIT $4)
|
|
2271
|
-
|
|
2272
|
-
|
|
2273
|
-
|
|
2271
|
+
ORDER BY s.commit_seq LIMIT $4)
|
|
2272
|
+
SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
2273
|
+
z.key AS key, z.bytes AS bytes
|
|
2274
|
+
FROM c
|
|
2275
|
+
LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
2276
|
+
LEFT JOIN LATERAL (SELECT k.i AS key,
|
|
2277
|
+
sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
|
|
2278
|
+
FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND ch.scopes->>k.var=k.value
|
|
2279
|
+
WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
|
|
2280
|
+
GROUP BY k.i) z ON true
|
|
2281
|
+
ORDER BY c.commit_seq, z.key`, [partition, JSON.stringify(keys), query.throughSeq, limit]);
|
|
2274
2282
|
return rows.map((record) => ({
|
|
2275
2283
|
commitSeq: asNumber(record.commit_seq),
|
|
2276
2284
|
...(record.actor_id === null
|
|
@@ -2281,9 +2289,21 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2281
2289
|
createdAtMs: asNumber(record.created_at_ms),
|
|
2282
2290
|
},
|
|
2283
2291
|
}),
|
|
2284
|
-
...(record.
|
|
2292
|
+
...(record.key === null
|
|
2285
2293
|
? {}
|
|
2286
|
-
: {
|
|
2294
|
+
: { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
|
|
2295
|
+
}));
|
|
2296
|
+
}, async (keys, after, through) => {
|
|
2297
|
+
const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
|
|
2298
|
+
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
2299
|
+
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
2300
|
+
FROM sync_changes ch
|
|
2301
|
+
WHERE ch.partition=$1 AND ch.commit_seq>$3 AND ch.commit_seq<=$4
|
|
2302
|
+
AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND ch.scopes->>k.var=k.value)
|
|
2303
|
+
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), after, through]);
|
|
2304
|
+
return rows.map((record) => ({
|
|
2305
|
+
commitSeq: asNumber(record.commit_seq),
|
|
2306
|
+
change: toStoredChange(record),
|
|
2287
2307
|
}));
|
|
2288
2308
|
});
|
|
2289
2309
|
}
|
package/dist/realtime.js
CHANGED
|
@@ -642,9 +642,19 @@ export class RealtimeSession {
|
|
|
642
642
|
// the keys before, reconcile after (leaves on lost keys, snapshots on
|
|
643
643
|
// gained keys).
|
|
644
644
|
const previousKeys = this.scopeKeys;
|
|
645
|
+
const previous = this.registrations;
|
|
645
646
|
this.registrations = registrations;
|
|
646
647
|
this.scopeKeys = this.#hub.scopeKeysOf(registrations);
|
|
647
648
|
this.reconcilePresence(previousKeys);
|
|
649
|
+
// §8.7: a host refresh that removes or narrows a registration wakes the
|
|
650
|
+
// session, so its client runs the round whose §3.3 revocation purge
|
|
651
|
+
// drops the rows it lost. A registration survives when the same unit
|
|
652
|
+
// and table come back with every previous scope value still allowed.
|
|
653
|
+
if (failClosed &&
|
|
654
|
+
previous.some((old) => !registrations.some((next) => next.id === old.id &&
|
|
655
|
+
next.table === old.table &&
|
|
656
|
+
Object.entries(next.effective).every(([variable, values]) => old.effective[variable]?.every((value) => values.includes(value)) ?? false))))
|
|
657
|
+
this.sendWake('catchup-required');
|
|
648
658
|
}
|
|
649
659
|
/** §8.7 protocol violation: drop the connection, fail loud. The
|
|
650
660
|
* message is intentionally unsent — there is no error channel for an
|
package/dist/sqlite-dialect.js
CHANGED
|
@@ -788,24 +788,30 @@ export class SqliteFamilyStorage {
|
|
|
788
788
|
return row?.seq ?? 0;
|
|
789
789
|
}
|
|
790
790
|
readLiveWindow(partition, query) {
|
|
791
|
-
//
|
|
792
|
-
// CROSS JOIN keeps them the outer loop, so each key seeks
|
|
793
|
-
// `sync_change_scopes_by_key` (INDEXED BY fails loudly when
|
|
794
|
-
// missing).
|
|
795
|
-
// the commit
|
|
796
|
-
//
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
|
|
802
|
-
ON s.partition=?1 AND s.var=
|
|
803
|
-
AND s.
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
791
|
+
// Two statements per page. The candidate read binds the unit keys as one
|
|
792
|
+
// JSON array; the CROSS JOIN keeps them the outer loop, so each key seeks
|
|
793
|
+
// one range of `sync_change_scopes_by_key` (INDEXED BY fails loudly when
|
|
794
|
+
// the index is missing). Each candidate LEFT JOINs its commit meta (NULL
|
|
795
|
+
// when the commit vanished, which still advances the scan) and, per key,
|
|
796
|
+
// the byte size of the changes the key reads; `length()` of a blob reads
|
|
797
|
+
// the record header, never the payload. The change read then loads only
|
|
798
|
+
// those changes of the commits the byte budget admits.
|
|
799
|
+
return readLiveWindowPages(query, async (keys, limit) => (await this.guardedAll(`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
|
|
800
|
+
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
801
|
+
FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
|
|
802
|
+
ON s.partition=?1 AND s.var=k.var AND s.value=k.value
|
|
803
|
+
AND s.commit_seq>k.after AND s.commit_seq<=?3
|
|
804
|
+
ORDER BY s.commit_seq LIMIT ?4)
|
|
805
|
+
SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
806
|
+
z.i AS key, z.bytes AS bytes
|
|
807
|
+
FROM c
|
|
808
|
+
LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
|
|
809
|
+
LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
|
|
810
|
+
sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
|
|
811
|
+
FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
|
|
812
|
+
CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
|
|
813
|
+
GROUP BY ch.commit_seq, k.i) AS z ON z.commit_seq=c.commit_seq
|
|
814
|
+
ORDER BY c.commit_seq, z.i`, [partition, JSON.stringify(keys), query.throughSeq, limit])).map((record) => ({
|
|
809
815
|
commitSeq: record.commit_seq,
|
|
810
816
|
...(record.actor_id !== null && record.created_at_ms !== null
|
|
811
817
|
? {
|
|
@@ -815,9 +821,18 @@ export class SqliteFamilyStorage {
|
|
|
815
821
|
},
|
|
816
822
|
}
|
|
817
823
|
: {}),
|
|
818
|
-
...(record.
|
|
819
|
-
? {
|
|
824
|
+
...(record.key !== null
|
|
825
|
+
? { key: record.key, bytes: record.bytes ?? 0 }
|
|
820
826
|
: {}),
|
|
827
|
+
})), async (keys, after, through) => (await this.guardedAll(`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
|
|
828
|
+
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
829
|
+
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
830
|
+
FROM sync_changes AS ch
|
|
831
|
+
WHERE ch.partition=?1 AND ch.commit_seq>?3 AND ch.commit_seq<=?4
|
|
832
|
+
AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
|
|
833
|
+
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), after, through])).map((record) => ({
|
|
834
|
+
commitSeq: record.commit_seq,
|
|
835
|
+
change: toStoredChange(record),
|
|
821
836
|
})));
|
|
822
837
|
}
|
|
823
838
|
async scanRows(partition, query) {
|
package/dist/storage-query.d.ts
CHANGED
|
@@ -15,32 +15,52 @@ export declare function resolveIndexRowScan(table: CompiledTable, query: IndexRo
|
|
|
15
15
|
* which reads outside the transaction.
|
|
16
16
|
*/
|
|
17
17
|
export declare function transactionQuery(tx: StorageTransaction, query: AuthoritativeQueryRequest): Promise<AuthoritativeQueryResult>;
|
|
18
|
-
/**
|
|
18
|
+
/**
|
|
19
|
+
* One index range of the live scan: a unit key, its exclusive lower bound,
|
|
20
|
+
* and the tables its units cover. `i` is the key's position in the list.
|
|
21
|
+
*/
|
|
19
22
|
export interface LiveKey {
|
|
23
|
+
readonly i: number;
|
|
20
24
|
readonly var: string;
|
|
21
25
|
readonly value: string;
|
|
22
26
|
readonly after: number;
|
|
27
|
+
readonly tables: readonly string[];
|
|
23
28
|
}
|
|
24
29
|
/**
|
|
25
|
-
* One row of a live-window page: a candidate commit joined to its
|
|
26
|
-
* (absent when the commit vanished) and
|
|
27
|
-
*
|
|
30
|
+
* One row of a live-window candidate page: a candidate commit joined to its
|
|
31
|
+
* metadata (absent when the commit vanished) and, per key, the stored bytes
|
|
32
|
+
* (payload, row id, scopes) of its changes that key reads: a covered table
|
|
33
|
+
* and the key's value in the stored scopes. A commit with no such change
|
|
34
|
+
* comes back once without `key`. Rows are in (commitSeq, key) order.
|
|
28
35
|
*/
|
|
29
|
-
export interface
|
|
36
|
+
export interface LiveCandidateRow {
|
|
30
37
|
readonly commitSeq: number;
|
|
31
38
|
readonly meta?: {
|
|
32
39
|
readonly actorId: string;
|
|
33
40
|
readonly createdAtMs: number;
|
|
34
41
|
};
|
|
35
|
-
readonly
|
|
42
|
+
readonly key?: number;
|
|
43
|
+
readonly bytes?: number;
|
|
36
44
|
}
|
|
37
45
|
/**
|
|
38
|
-
* The §4.5 live scan over a backend's
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
46
|
+
* The §4.5 live scan over a backend's two reads. `readCandidates` returns
|
|
47
|
+
* up to `limit` candidate commits above every key range's bound with their
|
|
48
|
+
* per-key byte sizes, and no payload. `readChanges` loads the changes the
|
|
49
|
+
* keys read in a commit range `(after, through]`, in (commitSeq, change
|
|
50
|
+
* index) order. Each distinct unit key (the first variable, keys sorted)
|
|
51
|
+
* becomes one index range starting at the lowest bound of the units that
|
|
52
|
+
* share it. A change is kept when a unit covers its table, the pair's
|
|
53
|
+
* effective scopes match it, and its commit lies above that unit's own
|
|
54
|
+
* bound. Every candidate advances the scan, so a page of non-matching
|
|
55
|
+
* candidates never repeats.
|
|
56
|
+
*
|
|
57
|
+
* The sizes bound the loaded bytes by `limitBytes` before any payload is
|
|
58
|
+
* read: the section stops before the commit that would pass it, and a
|
|
59
|
+
* commit that passes it alone ends the read as `oversized`, naming the
|
|
60
|
+
* units of the keys that read the most of it, largest first, until the rest
|
|
61
|
+
* fits (§4.6).
|
|
45
62
|
*/
|
|
46
|
-
export declare function readLiveWindowPages(query: LiveWindowQuery,
|
|
63
|
+
export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[], after: number, through: number) => Promise<readonly {
|
|
64
|
+
readonly commitSeq: number;
|
|
65
|
+
readonly change: StoredChange;
|
|
66
|
+
}[]>): Promise<LiveWindow>;
|
package/dist/storage-query.js
CHANGED
|
@@ -48,19 +48,31 @@ export async function transactionQuery(tx, query) {
|
|
|
48
48
|
return tx.queryAuthoritative(query);
|
|
49
49
|
}
|
|
50
50
|
/**
|
|
51
|
-
* The §4.5 live scan over a backend's
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
51
|
+
* The §4.5 live scan over a backend's two reads. `readCandidates` returns
|
|
52
|
+
* up to `limit` candidate commits above every key range's bound with their
|
|
53
|
+
* per-key byte sizes, and no payload. `readChanges` loads the changes the
|
|
54
|
+
* keys read in a commit range `(after, through]`, in (commitSeq, change
|
|
55
|
+
* index) order. Each distinct unit key (the first variable, keys sorted)
|
|
56
|
+
* becomes one index range starting at the lowest bound of the units that
|
|
57
|
+
* share it. A change is kept when a unit covers its table, the pair's
|
|
58
|
+
* effective scopes match it, and its commit lies above that unit's own
|
|
59
|
+
* bound. Every candidate advances the scan, so a page of non-matching
|
|
60
|
+
* candidates never repeats.
|
|
61
|
+
*
|
|
62
|
+
* The sizes bound the loaded bytes by `limitBytes` before any payload is
|
|
63
|
+
* read: the section stops before the commit that would pass it, and a
|
|
64
|
+
* commit that passes it alone ends the read as `oversized`, naming the
|
|
65
|
+
* units of the keys that read the most of it, largest first, until the rest
|
|
66
|
+
* fits (§4.6).
|
|
58
67
|
*/
|
|
59
|
-
export async function readLiveWindowPages(query,
|
|
68
|
+
export async function readLiveWindowPages(query, readCandidates, readChanges) {
|
|
60
69
|
const bounds = new Map();
|
|
70
|
+
const unitKeys = [];
|
|
61
71
|
for (const unit of query.units) {
|
|
62
|
-
if (unit.tables.size === 0)
|
|
72
|
+
if (unit.tables.size === 0) {
|
|
73
|
+
unitKeys.push(undefined);
|
|
63
74
|
continue;
|
|
75
|
+
}
|
|
64
76
|
const variable = Object.keys(unit.scopes).sort()[0];
|
|
65
77
|
const value = variable === undefined ? undefined : unit.scopes[variable];
|
|
66
78
|
if (variable === undefined || value === undefined) {
|
|
@@ -68,47 +80,113 @@ export async function readLiveWindowPages(query, readPage) {
|
|
|
68
80
|
}
|
|
69
81
|
const id = JSON.stringify([variable, value]);
|
|
70
82
|
const prior = bounds.get(id);
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
83
|
+
const tables = new Set([...(prior?.tables ?? []), ...unit.tables.keys()]);
|
|
84
|
+
const i = prior?.key.i ?? bounds.size;
|
|
85
|
+
bounds.set(id, {
|
|
86
|
+
key: {
|
|
87
|
+
i,
|
|
88
|
+
var: variable,
|
|
89
|
+
value,
|
|
90
|
+
after: Math.min(prior?.key.after ?? unit.afterSeq, unit.afterSeq),
|
|
91
|
+
tables: [...tables],
|
|
92
|
+
},
|
|
93
|
+
tables,
|
|
94
|
+
});
|
|
95
|
+
unitKeys.push(i);
|
|
74
96
|
}
|
|
97
|
+
const keys = [...bounds.values()].map(({ key }) => key);
|
|
75
98
|
const commits = [];
|
|
76
99
|
const limit = Math.max(64, query.limitChanges);
|
|
77
100
|
let delivered = 0;
|
|
101
|
+
let loaded = 0;
|
|
78
102
|
let scanned = -1;
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
103
|
+
// Candidates sized but not loaded yet: (scanned, pending.at(-1)].
|
|
104
|
+
let pending = [];
|
|
105
|
+
const scanKeys = () => keys.map((key) => ({ ...key, after: Math.max(key.after, scanned) }));
|
|
106
|
+
// Loads the pending candidates; returns T when `limitChanges` cut there.
|
|
107
|
+
const load = async () => {
|
|
108
|
+
const last = pending.at(-1);
|
|
109
|
+
if (last === undefined)
|
|
110
|
+
return undefined;
|
|
111
|
+
const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
|
|
112
|
+
const rows = await readChanges(scanKeys(), scanned, last.commitSeq);
|
|
113
|
+
pending = [];
|
|
114
|
+
scanned = last.commitSeq;
|
|
85
115
|
for (let i = 0; i < rows.length;) {
|
|
86
|
-
const
|
|
87
|
-
if (head === undefined)
|
|
88
|
-
break;
|
|
89
|
-
candidates += 1;
|
|
90
|
-
scanned = head.commitSeq;
|
|
116
|
+
const seq = rows[i].commitSeq;
|
|
91
117
|
const changes = [];
|
|
92
|
-
for (; rows[i]?.commitSeq ===
|
|
93
|
-
const change = rows[i]
|
|
94
|
-
if (
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
})) {
|
|
118
|
+
for (; rows[i]?.commitSeq === seq; i++) {
|
|
119
|
+
const change = rows[i].change;
|
|
120
|
+
if (query.units.some((unit) => {
|
|
121
|
+
const effective = unit.tables.get(change.table);
|
|
122
|
+
return (seq > unit.afterSeq &&
|
|
123
|
+
effective !== undefined &&
|
|
124
|
+
matchesEffective(change.scopes, effective));
|
|
125
|
+
})) {
|
|
101
126
|
changes.push(change);
|
|
102
127
|
}
|
|
103
128
|
}
|
|
104
|
-
|
|
129
|
+
const meta = metas.get(seq);
|
|
130
|
+
if (meta === undefined || changes.length === 0)
|
|
105
131
|
continue;
|
|
106
|
-
commits.push({ commitSeq:
|
|
132
|
+
commits.push({ commitSeq: seq, ...meta, changes });
|
|
107
133
|
delivered += changes.length;
|
|
108
|
-
if (delivered >= query.limitChanges)
|
|
109
|
-
return
|
|
134
|
+
if (delivered >= query.limitChanges)
|
|
135
|
+
return seq;
|
|
136
|
+
}
|
|
137
|
+
return undefined;
|
|
138
|
+
};
|
|
139
|
+
while (keys.length > 0) {
|
|
140
|
+
const rows = await readCandidates(scanKeys(), limit);
|
|
141
|
+
let candidates = 0;
|
|
142
|
+
for (let i = 0; i < rows.length;) {
|
|
143
|
+
const head = rows[i];
|
|
144
|
+
candidates += 1;
|
|
145
|
+
let bytes = 0;
|
|
146
|
+
const sizes = [];
|
|
147
|
+
for (; rows[i]?.commitSeq === head.commitSeq; i++) {
|
|
148
|
+
const row = rows[i];
|
|
149
|
+
if (row.key === undefined)
|
|
150
|
+
continue;
|
|
151
|
+
bytes += row.bytes ?? 0;
|
|
152
|
+
sizes.push([row.key, row.bytes ?? 0]);
|
|
153
|
+
}
|
|
154
|
+
if (bytes > query.limitBytes) {
|
|
155
|
+
// The keys that read the most of it leave until the rest fits.
|
|
156
|
+
const reached = new Set();
|
|
157
|
+
let rest = bytes;
|
|
158
|
+
for (const [key, size] of sizes.sort((x, y) => y[1] - x[1])) {
|
|
159
|
+
if (rest <= query.limitBytes)
|
|
160
|
+
break;
|
|
161
|
+
reached.add(key);
|
|
162
|
+
rest -= size;
|
|
163
|
+
}
|
|
164
|
+
return {
|
|
165
|
+
commits: [],
|
|
166
|
+
throughSeq: query.throughSeq,
|
|
167
|
+
oversized: {
|
|
168
|
+
commitSeq: head.commitSeq,
|
|
169
|
+
units: query.units.flatMap((unit, index) => {
|
|
170
|
+
const key = unitKeys[index];
|
|
171
|
+
return key !== undefined &&
|
|
172
|
+
reached.has(key) &&
|
|
173
|
+
head.commitSeq > unit.afterSeq
|
|
174
|
+
? [index]
|
|
175
|
+
: [];
|
|
176
|
+
}),
|
|
177
|
+
},
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
if (loaded + bytes > query.limitBytes) {
|
|
181
|
+
const cut = await load();
|
|
182
|
+
return { commits, throughSeq: cut ?? scanned };
|
|
110
183
|
}
|
|
184
|
+
loaded += bytes;
|
|
185
|
+
pending.push({ commitSeq: head.commitSeq, meta: head.meta });
|
|
111
186
|
}
|
|
187
|
+
const cut = await load();
|
|
188
|
+
if (cut !== undefined)
|
|
189
|
+
return { commits, throughSeq: cut };
|
|
112
190
|
if (candidates < limit)
|
|
113
191
|
break;
|
|
114
192
|
}
|
package/dist/storage.d.ts
CHANGED
|
@@ -366,16 +366,31 @@ export interface LiveWindowQuery {
|
|
|
366
366
|
readonly throughSeq: number;
|
|
367
367
|
/** The `limitCommits` budget: stop after the commit that reaches it. */
|
|
368
368
|
readonly limitChanges: number;
|
|
369
|
+
/**
|
|
370
|
+
* `ServerLimits.maxLiveSectionBytes`: the stored bytes (payload, row id,
|
|
371
|
+
* scopes) of the changes a section loads. The section stops before the
|
|
372
|
+
* commit that would pass it; a commit that passes it alone is oversized.
|
|
373
|
+
*/
|
|
374
|
+
readonly limitBytes: number;
|
|
369
375
|
}
|
|
370
376
|
/** The commits of one live section (§4.5). */
|
|
371
377
|
export interface LiveWindow {
|
|
372
378
|
/** Oldest first, each with its matching changes in change order. */
|
|
373
379
|
readonly commits: readonly StoredCommit[];
|
|
374
380
|
/**
|
|
375
|
-
* `T`: `query.throughSeq`, or the last
|
|
376
|
-
*
|
|
381
|
+
* `T`: `query.throughSeq`, or the last scanned `commitSeq` when a budget
|
|
382
|
+
* cut the section.
|
|
377
383
|
*/
|
|
378
384
|
readonly throughSeq: number;
|
|
385
|
+
/**
|
|
386
|
+
* A commit whose changes for the scan set pass `limitBytes` on their own
|
|
387
|
+
* (§4.6): `units` are the indexes into `query.units` it reaches. The
|
|
388
|
+
* section is unusable; the caller resets those units and reads again.
|
|
389
|
+
*/
|
|
390
|
+
readonly oversized?: {
|
|
391
|
+
readonly commitSeq: number;
|
|
392
|
+
readonly units: readonly number[];
|
|
393
|
+
};
|
|
379
394
|
}
|
|
380
395
|
export interface RowScanQuery {
|
|
381
396
|
readonly table: string;
|
|
@@ -806,9 +821,11 @@ export interface ServerStorage {
|
|
|
806
821
|
* The §4.5 live section: every commit in `(min afterSeq, throughSeq]`
|
|
807
822
|
* carrying a change that matches a unit of the scan set, oldest first,
|
|
808
823
|
* each with its matching changes once, in change order, across tables.
|
|
809
|
-
* Stops after the commit whose matching changes reach `limitChanges
|
|
810
|
-
*
|
|
811
|
-
*
|
|
824
|
+
* Stops after the commit whose matching changes reach `limitChanges`, or
|
|
825
|
+
* before the commit that would pass `limitBytes` (never splitting a
|
|
826
|
+
* commit). Candidates come from the change-scope key index, one index
|
|
827
|
+
* range per distinct unit key, never from a log scan, and only changes of
|
|
828
|
+
* a covered table carrying a unit key's value are read.
|
|
812
829
|
*/
|
|
813
830
|
readLiveWindow(partition: string, query: LiveWindowQuery): Promise<LiveWindow>;
|
|
814
831
|
/**
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.70.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.70.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/context.ts
CHANGED
|
@@ -91,6 +91,13 @@ export interface ServerLimits {
|
|
|
91
91
|
readonly maxSubscriptionsPerRequest: number;
|
|
92
92
|
/** Cap on concurrent registered-query watches per watch session. */
|
|
93
93
|
readonly maxWatchesPerSession: number;
|
|
94
|
+
/**
|
|
95
|
+
* §4.6 byte budget of one live section: the stored bytes (payload, row id,
|
|
96
|
+
* scopes) of the changes a pull loads from the log. A section stops before
|
|
97
|
+
* the commit that would pass it; a commit whose changes for the scan set
|
|
98
|
+
* pass it alone resets the units it reaches, which bootstrap instead.
|
|
99
|
+
*/
|
|
100
|
+
readonly maxLiveSectionBytes: number;
|
|
94
101
|
}
|
|
95
102
|
|
|
96
103
|
export const DEFAULT_LIMITS: ServerLimits = {
|
|
@@ -103,6 +110,8 @@ export const DEFAULT_LIMITS: ServerLimits = {
|
|
|
103
110
|
maxSubscriptionsPerRequest: 500,
|
|
104
111
|
// Every commit reruns a session's affected watches one after another.
|
|
105
112
|
maxWatchesPerSession: 100,
|
|
113
|
+
// The live section is buffered whole before the first unit section.
|
|
114
|
+
maxLiveSectionBytes: 16 * 1024 * 1024,
|
|
106
115
|
};
|
|
107
116
|
|
|
108
117
|
/** Receives applied commits for realtime fanout (§8.2). */
|
package/src/handler.ts
CHANGED
|
@@ -918,18 +918,30 @@ async function* streamResponse(
|
|
|
918
918
|
// resets the live section), and the section is read again.
|
|
919
919
|
let live: LiveWindow | undefined;
|
|
920
920
|
for (;;) {
|
|
921
|
-
const
|
|
921
|
+
const scanned = units.flatMap(({ plan: unit }, index) => {
|
|
922
922
|
const afterSeq = bound(states[index]!);
|
|
923
923
|
return afterSeq === undefined
|
|
924
924
|
? []
|
|
925
|
-
: [{ scopes: unit.scopes, afterSeq, tables: unit.pairs }];
|
|
925
|
+
: [{ index, scopes: unit.scopes, afterSeq, tables: unit.pairs }];
|
|
926
926
|
});
|
|
927
|
-
if (liveReset ||
|
|
927
|
+
if (liveReset || scanned.length === 0) break;
|
|
928
928
|
live = await ctx.storage.readLiveWindow(ctx.partition, {
|
|
929
|
-
units:
|
|
929
|
+
units: scanned,
|
|
930
930
|
throughSeq: maxSeq,
|
|
931
931
|
limitChanges: limits.limitCommits,
|
|
932
|
+
limitBytes: limitsOf(ctx).maxLiveSectionBytes,
|
|
932
933
|
});
|
|
934
|
+
// §4.6: a commit past the byte budget on its own cannot be served
|
|
935
|
+
// from the log; the units it reaches reset and bootstrap.
|
|
936
|
+
for (const at of live.oversized?.units ?? []) {
|
|
937
|
+
const index = scanned[at]!.index;
|
|
938
|
+
if (states[index]!.kind === 'live') liveReset = true;
|
|
939
|
+
else states[index] = { kind: 'reset' };
|
|
940
|
+
}
|
|
941
|
+
if (live.oversized !== undefined) {
|
|
942
|
+
live = undefined;
|
|
943
|
+
continue;
|
|
944
|
+
}
|
|
933
945
|
const recheck = await ctx.storage.getHorizonSeq(ctx.partition);
|
|
934
946
|
let expired = false;
|
|
935
947
|
for (const [index, state] of states.entries()) {
|
package/src/local-authority.ts
CHANGED
|
@@ -215,6 +215,10 @@ export async function openLocalAuthority(
|
|
|
215
215
|
segments: new MemorySegmentStore(),
|
|
216
216
|
resolveScopes: () => allowed,
|
|
217
217
|
clock,
|
|
218
|
+
// §12.2: the client's own process serves its own commits; the
|
|
219
|
+
// live byte budget (§4.6) bounds a shared server's heap and does
|
|
220
|
+
// not apply here, as in the native authority.
|
|
221
|
+
limits: { maxLiveSectionBytes: Number.MAX_SAFE_INTEGER },
|
|
218
222
|
...(options.crdtMergers !== undefined
|
|
219
223
|
? { crdtMergers: options.crdtMergers }
|
|
220
224
|
: {}),
|
package/src/postgres-storage.ts
CHANGED
|
@@ -3314,52 +3314,77 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3314
3314
|
partition: string,
|
|
3315
3315
|
query: LiveWindowQuery,
|
|
3316
3316
|
): Promise<LiveWindow> {
|
|
3317
|
-
//
|
|
3317
|
+
// Two statements per page. The candidate read takes one LATERAL range of
|
|
3318
3318
|
// `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
|
|
3319
|
-
// the index, merged in commit order and LIMITed again, then LEFT
|
|
3320
|
-
//
|
|
3321
|
-
//
|
|
3322
|
-
|
|
3323
|
-
|
|
3324
|
-
|
|
3325
|
-
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3330
|
-
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
FROM
|
|
3338
|
-
CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
3319
|
+
// the index, merged in commit order and LIMITed again, then LEFT JOINs
|
|
3320
|
+
// the commit meta (NULL when the commit vanished, which still advances
|
|
3321
|
+
// the scan) and, per key, the byte size of the changes the key reads;
|
|
3322
|
+
// `octet_length` of a TOASTed payload reads its pointer, never the value.
|
|
3323
|
+
// The change read then loads only those changes of the commits the byte
|
|
3324
|
+
// budget admits.
|
|
3325
|
+
return readLiveWindowPages(
|
|
3326
|
+
query,
|
|
3327
|
+
async (keys, limit) => {
|
|
3328
|
+
const { rows } = await this.#exec.query<{
|
|
3329
|
+
commit_seq: unknown;
|
|
3330
|
+
actor_id: string | null;
|
|
3331
|
+
created_at_ms: unknown;
|
|
3332
|
+
key: unknown;
|
|
3333
|
+
bytes: unknown;
|
|
3334
|
+
}>(
|
|
3335
|
+
`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
|
|
3336
|
+
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
|
|
3337
|
+
FROM k CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
3339
3338
|
WHERE partition=$1 AND var=k.var AND value=k.value
|
|
3340
3339
|
AND commit_seq>k.after AND commit_seq<=$3
|
|
3341
3340
|
ORDER BY commit_seq LIMIT $4) s
|
|
3342
|
-
ORDER BY s.commit_seq LIMIT $4)
|
|
3343
|
-
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3347
|
-
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3341
|
+
ORDER BY s.commit_seq LIMIT $4)
|
|
3342
|
+
SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
3343
|
+
z.key AS key, z.bytes AS bytes
|
|
3344
|
+
FROM c
|
|
3345
|
+
LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
3346
|
+
LEFT JOIN LATERAL (SELECT k.i AS key,
|
|
3347
|
+
sum(coalesce(octet_length(ch.payload), 0) + octet_length(ch.row_id) + octet_length(ch.scopes::text)) AS bytes
|
|
3348
|
+
FROM sync_changes ch JOIN k ON ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND ch.scopes->>k.var=k.value
|
|
3349
|
+
WHERE ch.partition=$1 AND ch.commit_seq=c.commit_seq
|
|
3350
|
+
GROUP BY k.i) z ON true
|
|
3351
|
+
ORDER BY c.commit_seq, z.key`,
|
|
3352
|
+
[partition, JSON.stringify(keys), query.throughSeq, limit],
|
|
3353
|
+
);
|
|
3354
|
+
return rows.map((record) => ({
|
|
3355
|
+
commitSeq: asNumber(record.commit_seq),
|
|
3356
|
+
...(record.actor_id === null
|
|
3357
|
+
? {}
|
|
3358
|
+
: {
|
|
3359
|
+
meta: {
|
|
3360
|
+
actorId: record.actor_id,
|
|
3361
|
+
createdAtMs: asNumber(record.created_at_ms),
|
|
3362
|
+
},
|
|
3363
|
+
}),
|
|
3364
|
+
...(record.key === null
|
|
3365
|
+
? {}
|
|
3366
|
+
: { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
|
|
3367
|
+
}));
|
|
3368
|
+
},
|
|
3369
|
+
async (keys, after, through) => {
|
|
3370
|
+
const { rows } = await this.#exec.query<
|
|
3371
|
+
ChangeRecord & { commit_seq: unknown }
|
|
3372
|
+
>(
|
|
3373
|
+
`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
|
|
3374
|
+
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
3375
|
+
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
3376
|
+
FROM sync_changes ch
|
|
3377
|
+
WHERE ch.partition=$1 AND ch.commit_seq>$3 AND ch.commit_seq<=$4
|
|
3378
|
+
AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND ch.scopes->>k.var=k.value)
|
|
3379
|
+
ORDER BY ch.commit_seq, ch.idx`,
|
|
3380
|
+
[partition, JSON.stringify(keys), after, through],
|
|
3381
|
+
);
|
|
3382
|
+
return rows.map((record) => ({
|
|
3383
|
+
commitSeq: asNumber(record.commit_seq),
|
|
3384
|
+
change: toStoredChange(record),
|
|
3385
|
+
}));
|
|
3386
|
+
},
|
|
3387
|
+
);
|
|
3363
3388
|
}
|
|
3364
3389
|
|
|
3365
3390
|
async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
|
package/src/realtime.ts
CHANGED
|
@@ -846,9 +846,32 @@ export class RealtimeSession {
|
|
|
846
846
|
// the keys before, reconcile after (leaves on lost keys, snapshots on
|
|
847
847
|
// gained keys).
|
|
848
848
|
const previousKeys = this.scopeKeys;
|
|
849
|
+
const previous = this.registrations;
|
|
849
850
|
this.registrations = registrations;
|
|
850
851
|
this.scopeKeys = this.#hub.scopeKeysOf(registrations);
|
|
851
852
|
this.reconcilePresence(previousKeys);
|
|
853
|
+
// §8.7: a host refresh that removes or narrows a registration wakes the
|
|
854
|
+
// session, so its client runs the round whose §3.3 revocation purge
|
|
855
|
+
// drops the rows it lost. A registration survives when the same unit
|
|
856
|
+
// and table come back with every previous scope value still allowed.
|
|
857
|
+
if (
|
|
858
|
+
failClosed &&
|
|
859
|
+
previous.some(
|
|
860
|
+
(old) =>
|
|
861
|
+
!registrations.some(
|
|
862
|
+
(next) =>
|
|
863
|
+
next.id === old.id &&
|
|
864
|
+
next.table === old.table &&
|
|
865
|
+
Object.entries(next.effective).every(
|
|
866
|
+
([variable, values]) =>
|
|
867
|
+
old.effective[variable]?.every((value) =>
|
|
868
|
+
values.includes(value),
|
|
869
|
+
) ?? false,
|
|
870
|
+
),
|
|
871
|
+
),
|
|
872
|
+
)
|
|
873
|
+
)
|
|
874
|
+
this.sendWake('catchup-required');
|
|
852
875
|
}
|
|
853
876
|
|
|
854
877
|
/** §8.7 protocol violation: drop the connection, fail loud. The
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -1199,53 +1199,73 @@ export abstract class SqliteFamilyStorage {
|
|
|
1199
1199
|
partition: string,
|
|
1200
1200
|
query: LiveWindowQuery,
|
|
1201
1201
|
): Promise<LiveWindow> {
|
|
1202
|
-
//
|
|
1203
|
-
// CROSS JOIN keeps them the outer loop, so each key seeks
|
|
1204
|
-
// `sync_change_scopes_by_key` (INDEXED BY fails loudly when
|
|
1205
|
-
// missing).
|
|
1206
|
-
// the commit
|
|
1207
|
-
//
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1202
|
+
// Two statements per page. The candidate read binds the unit keys as one
|
|
1203
|
+
// JSON array; the CROSS JOIN keeps them the outer loop, so each key seeks
|
|
1204
|
+
// one range of `sync_change_scopes_by_key` (INDEXED BY fails loudly when
|
|
1205
|
+
// the index is missing). Each candidate LEFT JOINs its commit meta (NULL
|
|
1206
|
+
// when the commit vanished, which still advances the scan) and, per key,
|
|
1207
|
+
// the byte size of the changes the key reads; `length()` of a blob reads
|
|
1208
|
+
// the record header, never the payload. The change read then loads only
|
|
1209
|
+
// those changes of the commits the byte budget admits.
|
|
1210
|
+
return readLiveWindowPages(
|
|
1211
|
+
query,
|
|
1212
|
+
async (keys, limit) =>
|
|
1213
|
+
(
|
|
1214
|
+
await this.guardedAll<{
|
|
1215
|
+
commit_seq: number;
|
|
1216
|
+
actor_id: string | null;
|
|
1217
|
+
created_at_ms: number | null;
|
|
1218
|
+
key: number | null;
|
|
1219
|
+
bytes: number | null;
|
|
1220
|
+
}>(
|
|
1221
|
+
`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
|
|
1222
|
+
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
1223
|
+
FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
|
|
1224
|
+
ON s.partition=?1 AND s.var=k.var AND s.value=k.value
|
|
1225
|
+
AND s.commit_seq>k.after AND s.commit_seq<=?3
|
|
1226
|
+
ORDER BY s.commit_seq LIMIT ?4)
|
|
1227
|
+
SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
1228
|
+
z.i AS key, z.bytes AS bytes
|
|
1229
|
+
FROM c
|
|
1230
|
+
LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
|
|
1231
|
+
LEFT JOIN (SELECT ch.commit_seq AS commit_seq, k.i AS i,
|
|
1232
|
+
sum(coalesce(length(ch.payload), 0) + length(CAST(ch.row_id AS BLOB)) + length(CAST(ch.scopes AS BLOB))) AS bytes
|
|
1233
|
+
FROM c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
|
|
1234
|
+
CROSS JOIN k ON ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value
|
|
1235
|
+
GROUP BY ch.commit_seq, k.i) AS z ON z.commit_seq=c.commit_seq
|
|
1236
|
+
ORDER BY c.commit_seq, z.i`,
|
|
1237
|
+
[partition, JSON.stringify(keys), query.throughSeq, limit],
|
|
1238
|
+
)
|
|
1239
|
+
).map((record) => ({
|
|
1240
|
+
commitSeq: record.commit_seq,
|
|
1241
|
+
...(record.actor_id !== null && record.created_at_ms !== null
|
|
1242
|
+
? {
|
|
1243
|
+
meta: {
|
|
1244
|
+
actorId: record.actor_id,
|
|
1245
|
+
createdAtMs: record.created_at_ms,
|
|
1246
|
+
},
|
|
1247
|
+
}
|
|
1248
|
+
: {}),
|
|
1249
|
+
...(record.key !== null
|
|
1250
|
+
? { key: record.key, bytes: record.bytes ?? 0 }
|
|
1251
|
+
: {}),
|
|
1252
|
+
})),
|
|
1253
|
+
async (keys, after, through) =>
|
|
1254
|
+
(
|
|
1255
|
+
await this.guardedAll<SqliteChangeRecord & { commit_seq: number }>(
|
|
1256
|
+
`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
|
|
1257
|
+
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
1258
|
+
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
1259
|
+
FROM sync_changes AS ch
|
|
1260
|
+
WHERE ch.partition=?1 AND ch.commit_seq>?3 AND ch.commit_seq<=?4
|
|
1261
|
+
AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
|
|
1262
|
+
ORDER BY ch.commit_seq, ch.idx`,
|
|
1263
|
+
[partition, JSON.stringify(keys), after, through],
|
|
1264
|
+
)
|
|
1265
|
+
).map((record) => ({
|
|
1266
|
+
commitSeq: record.commit_seq,
|
|
1267
|
+
change: toStoredChange(record),
|
|
1268
|
+
})),
|
|
1249
1269
|
);
|
|
1250
1270
|
}
|
|
1251
1271
|
|
package/src/storage-query.ts
CHANGED
|
@@ -79,43 +79,71 @@ export async function transactionQuery(
|
|
|
79
79
|
return tx.queryAuthoritative(query);
|
|
80
80
|
}
|
|
81
81
|
|
|
82
|
-
/**
|
|
82
|
+
/**
|
|
83
|
+
* One index range of the live scan: a unit key, its exclusive lower bound,
|
|
84
|
+
* and the tables its units cover. `i` is the key's position in the list.
|
|
85
|
+
*/
|
|
83
86
|
export interface LiveKey {
|
|
87
|
+
readonly i: number;
|
|
84
88
|
readonly var: string;
|
|
85
89
|
readonly value: string;
|
|
86
90
|
readonly after: number;
|
|
91
|
+
readonly tables: readonly string[];
|
|
87
92
|
}
|
|
88
93
|
|
|
89
94
|
/**
|
|
90
|
-
* One row of a live-window page: a candidate commit joined to its
|
|
91
|
-
* (absent when the commit vanished) and
|
|
92
|
-
*
|
|
95
|
+
* One row of a live-window candidate page: a candidate commit joined to its
|
|
96
|
+
* metadata (absent when the commit vanished) and, per key, the stored bytes
|
|
97
|
+
* (payload, row id, scopes) of its changes that key reads: a covered table
|
|
98
|
+
* and the key's value in the stored scopes. A commit with no such change
|
|
99
|
+
* comes back once without `key`. Rows are in (commitSeq, key) order.
|
|
93
100
|
*/
|
|
94
|
-
export interface
|
|
101
|
+
export interface LiveCandidateRow {
|
|
95
102
|
readonly commitSeq: number;
|
|
96
103
|
readonly meta?: { readonly actorId: string; readonly createdAtMs: number };
|
|
97
|
-
readonly
|
|
104
|
+
readonly key?: number;
|
|
105
|
+
readonly bytes?: number;
|
|
98
106
|
}
|
|
99
107
|
|
|
100
108
|
/**
|
|
101
|
-
* The §4.5 live scan over a backend's
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
*
|
|
107
|
-
*
|
|
109
|
+
* The §4.5 live scan over a backend's two reads. `readCandidates` returns
|
|
110
|
+
* up to `limit` candidate commits above every key range's bound with their
|
|
111
|
+
* per-key byte sizes, and no payload. `readChanges` loads the changes the
|
|
112
|
+
* keys read in a commit range `(after, through]`, in (commitSeq, change
|
|
113
|
+
* index) order. Each distinct unit key (the first variable, keys sorted)
|
|
114
|
+
* becomes one index range starting at the lowest bound of the units that
|
|
115
|
+
* share it. A change is kept when a unit covers its table, the pair's
|
|
116
|
+
* effective scopes match it, and its commit lies above that unit's own
|
|
117
|
+
* bound. Every candidate advances the scan, so a page of non-matching
|
|
118
|
+
* candidates never repeats.
|
|
119
|
+
*
|
|
120
|
+
* The sizes bound the loaded bytes by `limitBytes` before any payload is
|
|
121
|
+
* read: the section stops before the commit that would pass it, and a
|
|
122
|
+
* commit that passes it alone ends the read as `oversized`, naming the
|
|
123
|
+
* units of the keys that read the most of it, largest first, until the rest
|
|
124
|
+
* fits (§4.6).
|
|
108
125
|
*/
|
|
109
126
|
export async function readLiveWindowPages(
|
|
110
127
|
query: LiveWindowQuery,
|
|
111
|
-
|
|
128
|
+
readCandidates: (
|
|
112
129
|
keys: readonly LiveKey[],
|
|
113
130
|
limit: number,
|
|
114
|
-
) => Promise<readonly
|
|
131
|
+
) => Promise<readonly LiveCandidateRow[]>,
|
|
132
|
+
readChanges: (
|
|
133
|
+
keys: readonly LiveKey[],
|
|
134
|
+
after: number,
|
|
135
|
+
through: number,
|
|
136
|
+
) => Promise<
|
|
137
|
+
readonly { readonly commitSeq: number; readonly change: StoredChange }[]
|
|
138
|
+
>,
|
|
115
139
|
): Promise<LiveWindow> {
|
|
116
|
-
const bounds = new Map<string, LiveKey>();
|
|
140
|
+
const bounds = new Map<string, { key: LiveKey; tables: Set<string> }>();
|
|
141
|
+
const unitKeys: (number | undefined)[] = [];
|
|
117
142
|
for (const unit of query.units) {
|
|
118
|
-
if (unit.tables.size === 0)
|
|
143
|
+
if (unit.tables.size === 0) {
|
|
144
|
+
unitKeys.push(undefined);
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
119
147
|
const variable = Object.keys(unit.scopes).sort()[0];
|
|
120
148
|
const value = variable === undefined ? undefined : unit.scopes[variable];
|
|
121
149
|
if (variable === undefined || value === undefined) {
|
|
@@ -123,39 +151,50 @@ export async function readLiveWindowPages(
|
|
|
123
151
|
}
|
|
124
152
|
const id = JSON.stringify([variable, value]);
|
|
125
153
|
const prior = bounds.get(id);
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
154
|
+
const tables = new Set([...(prior?.tables ?? []), ...unit.tables.keys()]);
|
|
155
|
+
const i = prior?.key.i ?? bounds.size;
|
|
156
|
+
bounds.set(id, {
|
|
157
|
+
key: {
|
|
158
|
+
i,
|
|
159
|
+
var: variable,
|
|
160
|
+
value,
|
|
161
|
+
after: Math.min(prior?.key.after ?? unit.afterSeq, unit.afterSeq),
|
|
162
|
+
tables: [...tables],
|
|
163
|
+
},
|
|
164
|
+
tables,
|
|
165
|
+
});
|
|
166
|
+
unitKeys.push(i);
|
|
129
167
|
}
|
|
168
|
+
const keys = [...bounds.values()].map(({ key }) => key);
|
|
130
169
|
const commits: StoredCommit[] = [];
|
|
131
170
|
const limit = Math.max(64, query.limitChanges);
|
|
132
171
|
let delivered = 0;
|
|
172
|
+
let loaded = 0;
|
|
133
173
|
let scanned = -1;
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
);
|
|
142
|
-
|
|
174
|
+
// Candidates sized but not loaded yet: (scanned, pending.at(-1)].
|
|
175
|
+
let pending: { commitSeq: number; meta?: LiveCandidateRow['meta'] }[] = [];
|
|
176
|
+
const scanKeys = () =>
|
|
177
|
+
keys.map((key) => ({ ...key, after: Math.max(key.after, scanned) }));
|
|
178
|
+
// Loads the pending candidates; returns T when `limitChanges` cut there.
|
|
179
|
+
const load = async (): Promise<number | undefined> => {
|
|
180
|
+
const last = pending.at(-1);
|
|
181
|
+
if (last === undefined) return undefined;
|
|
182
|
+
const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
|
|
183
|
+
const rows = await readChanges(scanKeys(), scanned, last.commitSeq);
|
|
184
|
+
pending = [];
|
|
185
|
+
scanned = last.commitSeq;
|
|
143
186
|
for (let i = 0; i < rows.length;) {
|
|
144
|
-
const
|
|
145
|
-
if (head === undefined) break;
|
|
146
|
-
candidates += 1;
|
|
147
|
-
scanned = head.commitSeq;
|
|
187
|
+
const seq = rows[i]!.commitSeq;
|
|
148
188
|
const changes: StoredChange[] = [];
|
|
149
|
-
for (; rows[i]?.commitSeq ===
|
|
150
|
-
const change = rows[i]
|
|
189
|
+
for (; rows[i]?.commitSeq === seq; i++) {
|
|
190
|
+
const change = rows[i]!.change;
|
|
151
191
|
if (
|
|
152
|
-
change !== undefined &&
|
|
153
192
|
query.units.some((unit) => {
|
|
154
193
|
const effective: ScopeMap | undefined = unit.tables.get(
|
|
155
194
|
change.table,
|
|
156
195
|
);
|
|
157
196
|
return (
|
|
158
|
-
|
|
197
|
+
seq > unit.afterSeq &&
|
|
159
198
|
effective !== undefined &&
|
|
160
199
|
matchesEffective(change.scopes, effective)
|
|
161
200
|
);
|
|
@@ -164,13 +203,62 @@ export async function readLiveWindowPages(
|
|
|
164
203
|
changes.push(change);
|
|
165
204
|
}
|
|
166
205
|
}
|
|
167
|
-
|
|
168
|
-
|
|
206
|
+
const meta = metas.get(seq);
|
|
207
|
+
if (meta === undefined || changes.length === 0) continue;
|
|
208
|
+
commits.push({ commitSeq: seq, ...meta, changes });
|
|
169
209
|
delivered += changes.length;
|
|
170
|
-
if (delivered >= query.limitChanges)
|
|
171
|
-
|
|
210
|
+
if (delivered >= query.limitChanges) return seq;
|
|
211
|
+
}
|
|
212
|
+
return undefined;
|
|
213
|
+
};
|
|
214
|
+
while (keys.length > 0) {
|
|
215
|
+
const rows = await readCandidates(scanKeys(), limit);
|
|
216
|
+
let candidates = 0;
|
|
217
|
+
for (let i = 0; i < rows.length;) {
|
|
218
|
+
const head = rows[i]!;
|
|
219
|
+
candidates += 1;
|
|
220
|
+
let bytes = 0;
|
|
221
|
+
const sizes: [key: number, bytes: number][] = [];
|
|
222
|
+
for (; rows[i]?.commitSeq === head.commitSeq; i++) {
|
|
223
|
+
const row = rows[i]!;
|
|
224
|
+
if (row.key === undefined) continue;
|
|
225
|
+
bytes += row.bytes ?? 0;
|
|
226
|
+
sizes.push([row.key, row.bytes ?? 0]);
|
|
227
|
+
}
|
|
228
|
+
if (bytes > query.limitBytes) {
|
|
229
|
+
// The keys that read the most of it leave until the rest fits.
|
|
230
|
+
const reached = new Set<number>();
|
|
231
|
+
let rest = bytes;
|
|
232
|
+
for (const [key, size] of sizes.sort((x, y) => y[1] - x[1])) {
|
|
233
|
+
if (rest <= query.limitBytes) break;
|
|
234
|
+
reached.add(key);
|
|
235
|
+
rest -= size;
|
|
236
|
+
}
|
|
237
|
+
return {
|
|
238
|
+
commits: [],
|
|
239
|
+
throughSeq: query.throughSeq,
|
|
240
|
+
oversized: {
|
|
241
|
+
commitSeq: head.commitSeq,
|
|
242
|
+
units: query.units.flatMap((unit, index) => {
|
|
243
|
+
const key = unitKeys[index];
|
|
244
|
+
return key !== undefined &&
|
|
245
|
+
reached.has(key) &&
|
|
246
|
+
head.commitSeq > unit.afterSeq
|
|
247
|
+
? [index]
|
|
248
|
+
: [];
|
|
249
|
+
}),
|
|
250
|
+
},
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
if (loaded + bytes > query.limitBytes) {
|
|
254
|
+
const cut = await load();
|
|
255
|
+
return { commits, throughSeq: cut ?? scanned };
|
|
172
256
|
}
|
|
257
|
+
loaded += bytes;
|
|
258
|
+
pending.push({ commitSeq: head.commitSeq, meta: head.meta });
|
|
173
259
|
}
|
|
260
|
+
const cut = await load();
|
|
261
|
+
if (cut !== undefined) return { commits, throughSeq: cut };
|
|
174
262
|
if (candidates < limit) break;
|
|
175
263
|
}
|
|
176
264
|
return { commits, throughSeq: query.throughSeq };
|
package/src/storage.ts
CHANGED
|
@@ -496,6 +496,12 @@ export interface LiveWindowQuery {
|
|
|
496
496
|
readonly throughSeq: number;
|
|
497
497
|
/** The `limitCommits` budget: stop after the commit that reaches it. */
|
|
498
498
|
readonly limitChanges: number;
|
|
499
|
+
/**
|
|
500
|
+
* `ServerLimits.maxLiveSectionBytes`: the stored bytes (payload, row id,
|
|
501
|
+
* scopes) of the changes a section loads. The section stops before the
|
|
502
|
+
* commit that would pass it; a commit that passes it alone is oversized.
|
|
503
|
+
*/
|
|
504
|
+
readonly limitBytes: number;
|
|
499
505
|
}
|
|
500
506
|
|
|
501
507
|
/** The commits of one live section (§4.5). */
|
|
@@ -503,10 +509,19 @@ export interface LiveWindow {
|
|
|
503
509
|
/** Oldest first, each with its matching changes in change order. */
|
|
504
510
|
readonly commits: readonly StoredCommit[];
|
|
505
511
|
/**
|
|
506
|
-
* `T`: `query.throughSeq`, or the last
|
|
507
|
-
*
|
|
512
|
+
* `T`: `query.throughSeq`, or the last scanned `commitSeq` when a budget
|
|
513
|
+
* cut the section.
|
|
508
514
|
*/
|
|
509
515
|
readonly throughSeq: number;
|
|
516
|
+
/**
|
|
517
|
+
* A commit whose changes for the scan set pass `limitBytes` on their own
|
|
518
|
+
* (§4.6): `units` are the indexes into `query.units` it reaches. The
|
|
519
|
+
* section is unusable; the caller resets those units and reads again.
|
|
520
|
+
*/
|
|
521
|
+
readonly oversized?: {
|
|
522
|
+
readonly commitSeq: number;
|
|
523
|
+
readonly units: readonly number[];
|
|
524
|
+
};
|
|
510
525
|
}
|
|
511
526
|
|
|
512
527
|
export interface RowScanQuery {
|
|
@@ -1111,9 +1126,11 @@ export interface ServerStorage {
|
|
|
1111
1126
|
* The §4.5 live section: every commit in `(min afterSeq, throughSeq]`
|
|
1112
1127
|
* carrying a change that matches a unit of the scan set, oldest first,
|
|
1113
1128
|
* each with its matching changes once, in change order, across tables.
|
|
1114
|
-
* Stops after the commit whose matching changes reach `limitChanges
|
|
1115
|
-
*
|
|
1116
|
-
*
|
|
1129
|
+
* Stops after the commit whose matching changes reach `limitChanges`, or
|
|
1130
|
+
* before the commit that would pass `limitBytes` (never splitting a
|
|
1131
|
+
* commit). Candidates come from the change-scope key index, one index
|
|
1132
|
+
* range per distinct unit key, never from a log scan, and only changes of
|
|
1133
|
+
* a covered table carrying a unit key's value are read.
|
|
1117
1134
|
*/
|
|
1118
1135
|
readLiveWindow(
|
|
1119
1136
|
partition: string,
|