@syncular/server 0.82.2 → 0.82.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/postgres-storage.js
CHANGED
|
@@ -2404,8 +2404,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2404
2404
|
// the commit meta (NULL when the commit vanished, which still advances
|
|
2405
2405
|
// the scan) and, per key, the byte size of the changes the key reads;
|
|
2406
2406
|
// `octet_length` of a TOASTed payload reads its pointer, never the value.
|
|
2407
|
-
// The change read then
|
|
2408
|
-
// budget admits.
|
|
2407
|
+
// The change read then seeks only the commits the byte budget admits.
|
|
2409
2408
|
return readLiveWindowPages(query, async (keys, limit) => {
|
|
2410
2409
|
const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
|
|
2411
2410
|
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
|
|
@@ -2438,14 +2437,14 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2438
2437
|
? {}
|
|
2439
2438
|
: { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
|
|
2440
2439
|
}));
|
|
2441
|
-
}, async (keys,
|
|
2440
|
+
}, async (keys, commitSeqs) => {
|
|
2442
2441
|
const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
|
|
2443
2442
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
2444
2443
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
2445
2444
|
FROM sync_changes ch
|
|
2446
|
-
WHERE ch.partition=$1 AND ch.commit_seq
|
|
2445
|
+
WHERE ch.partition=$1 AND ch.commit_seq=ANY(ARRAY(SELECT jsonb_array_elements_text($3::text::jsonb)::bigint))
|
|
2447
2446
|
AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value)
|
|
2448
|
-
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys),
|
|
2447
|
+
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)]);
|
|
2449
2448
|
return rows.map((record) => ({
|
|
2450
2449
|
commitSeq: asNumber(record.commit_seq),
|
|
2451
2450
|
change: toStoredChange(record),
|
package/dist/sqlite-dialect.js
CHANGED
|
@@ -796,8 +796,9 @@ export class SqliteFamilyStorage {
|
|
|
796
796
|
// the index is missing). Each candidate LEFT JOINs its commit meta (NULL
|
|
797
797
|
// when the commit vanished, which still advances the scan) and, per key,
|
|
798
798
|
// the byte size of the changes the key reads; `length()` of a blob reads
|
|
799
|
-
// the record header, never the payload. The change read then
|
|
800
|
-
//
|
|
799
|
+
// the record header, never the payload. The change read then seeks only
|
|
800
|
+
// the commits the byte budget admits, through the primary key, with the
|
|
801
|
+
// key list materialized once per statement.
|
|
801
802
|
return readLiveWindowPages(query, async (keys, limit) => (await this.guardedAll(`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
|
|
802
803
|
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
803
804
|
FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
|
|
@@ -826,13 +827,12 @@ export class SqliteFamilyStorage {
|
|
|
826
827
|
...(record.key !== null
|
|
827
828
|
? { key: record.key, bytes: record.bytes ?? 0 }
|
|
828
829
|
: {}),
|
|
829
|
-
})), async (keys,
|
|
830
|
+
})), async (keys, commitSeqs) => (await this.guardedAll(`WITH k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
|
|
830
831
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
831
832
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
832
|
-
FROM sync_changes AS ch
|
|
833
|
-
WHERE ch.
|
|
834
|
-
|
|
835
|
-
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), after, through])).map((record) => ({
|
|
833
|
+
FROM json_each(?3) AS c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.value
|
|
834
|
+
WHERE EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
|
|
835
|
+
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)])).map((record) => ({
|
|
836
836
|
commitSeq: record.commit_seq,
|
|
837
837
|
change: toStoredChange(record),
|
|
838
838
|
})));
|
package/dist/storage-query.d.ts
CHANGED
|
@@ -72,8 +72,9 @@ export interface LiveCandidateRow {
|
|
|
72
72
|
* The §4.5 live scan over a backend's two reads. `readCandidates` returns
|
|
73
73
|
* up to `limit` candidate commits above every key range's bound with their
|
|
74
74
|
* per-key byte sizes, and no payload. `readChanges` loads the changes the
|
|
75
|
-
* keys read in
|
|
76
|
-
*
|
|
75
|
+
* keys read in the given candidate commits, in (commitSeq, change index)
|
|
76
|
+
* order; it never visits a commit outside that list, so the cost of a page
|
|
77
|
+
* follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
|
|
77
78
|
* becomes one index range starting at the lowest bound of the units that
|
|
78
79
|
* share it. A change is kept when a unit covers its table, the pair's
|
|
79
80
|
* effective scopes match it, and its commit lies above that unit's own
|
|
@@ -86,7 +87,7 @@ export interface LiveCandidateRow {
|
|
|
86
87
|
* units of the keys that read the most of it, largest first, until the rest
|
|
87
88
|
* fits (§4.6).
|
|
88
89
|
*/
|
|
89
|
-
export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[],
|
|
90
|
+
export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[], commitSeqs: readonly number[]) => Promise<readonly {
|
|
90
91
|
readonly commitSeq: number;
|
|
91
92
|
readonly change: StoredChange;
|
|
92
93
|
}[]>): Promise<LiveWindow>;
|
package/dist/storage-query.js
CHANGED
|
@@ -101,8 +101,9 @@ export async function transactionQuery(tx, query) {
|
|
|
101
101
|
* The §4.5 live scan over a backend's two reads. `readCandidates` returns
|
|
102
102
|
* up to `limit` candidate commits above every key range's bound with their
|
|
103
103
|
* per-key byte sizes, and no payload. `readChanges` loads the changes the
|
|
104
|
-
* keys read in
|
|
105
|
-
*
|
|
104
|
+
* keys read in the given candidate commits, in (commitSeq, change index)
|
|
105
|
+
* order; it never visits a commit outside that list, so the cost of a page
|
|
106
|
+
* follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
|
|
106
107
|
* becomes one index range starting at the lowest bound of the units that
|
|
107
108
|
* share it. A change is kept when a unit covers its table, the pair's
|
|
108
109
|
* effective scopes match it, and its commit lies above that unit's own
|
|
@@ -159,7 +160,7 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
|
|
|
159
160
|
if (last === undefined)
|
|
160
161
|
return undefined;
|
|
161
162
|
const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
|
|
162
|
-
const rows = await readChanges(scanKeys(),
|
|
163
|
+
const rows = await readChanges(scanKeys(), pending.map((c) => c.commitSeq));
|
|
163
164
|
pending = [];
|
|
164
165
|
scanned = last.commitSeq;
|
|
165
166
|
for (let i = 0; i < rows.length;) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.82.
|
|
3
|
+
"version": "0.82.3",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.82.
|
|
71
|
+
"@syncular/core": "0.82.3"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/postgres-storage.ts
CHANGED
|
@@ -3551,8 +3551,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3551
3551
|
// the commit meta (NULL when the commit vanished, which still advances
|
|
3552
3552
|
// the scan) and, per key, the byte size of the changes the key reads;
|
|
3553
3553
|
// `octet_length` of a TOASTed payload reads its pointer, never the value.
|
|
3554
|
-
// The change read then
|
|
3555
|
-
// budget admits.
|
|
3554
|
+
// The change read then seeks only the commits the byte budget admits.
|
|
3556
3555
|
return readLiveWindowPages(
|
|
3557
3556
|
query,
|
|
3558
3557
|
async (keys, limit) => {
|
|
@@ -3597,7 +3596,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3597
3596
|
: { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
|
|
3598
3597
|
}));
|
|
3599
3598
|
},
|
|
3600
|
-
async (keys,
|
|
3599
|
+
async (keys, commitSeqs) => {
|
|
3601
3600
|
const { rows } = await this.#exec.query<
|
|
3602
3601
|
ChangeRecord & { commit_seq: unknown }
|
|
3603
3602
|
>(
|
|
@@ -3605,10 +3604,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3605
3604
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
3606
3605
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
3607
3606
|
FROM sync_changes ch
|
|
3608
|
-
WHERE ch.partition=$1 AND ch.commit_seq
|
|
3607
|
+
WHERE ch.partition=$1 AND ch.commit_seq=ANY(ARRAY(SELECT jsonb_array_elements_text($3::text::jsonb)::bigint))
|
|
3609
3608
|
AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value)
|
|
3610
3609
|
ORDER BY ch.commit_seq, ch.idx`,
|
|
3611
|
-
[partition, JSON.stringify(keys),
|
|
3610
|
+
[partition, JSON.stringify(keys), JSON.stringify(commitSeqs)],
|
|
3612
3611
|
);
|
|
3613
3612
|
return rows.map((record) => ({
|
|
3614
3613
|
commitSeq: asNumber(record.commit_seq),
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -1208,8 +1208,9 @@ export abstract class SqliteFamilyStorage {
|
|
|
1208
1208
|
// the index is missing). Each candidate LEFT JOINs its commit meta (NULL
|
|
1209
1209
|
// when the commit vanished, which still advances the scan) and, per key,
|
|
1210
1210
|
// the byte size of the changes the key reads; `length()` of a blob reads
|
|
1211
|
-
// the record header, never the payload. The change read then
|
|
1212
|
-
//
|
|
1211
|
+
// the record header, never the payload. The change read then seeks only
|
|
1212
|
+
// the commits the byte budget admits, through the primary key, with the
|
|
1213
|
+
// key list materialized once per statement.
|
|
1213
1214
|
return readLiveWindowPages(
|
|
1214
1215
|
query,
|
|
1215
1216
|
async (keys, limit) =>
|
|
@@ -1253,17 +1254,16 @@ export abstract class SqliteFamilyStorage {
|
|
|
1253
1254
|
? { key: record.key, bytes: record.bytes ?? 0 }
|
|
1254
1255
|
: {}),
|
|
1255
1256
|
})),
|
|
1256
|
-
async (keys,
|
|
1257
|
+
async (keys, commitSeqs) =>
|
|
1257
1258
|
(
|
|
1258
1259
|
await this.guardedAll<SqliteChangeRecord & { commit_seq: number }>(
|
|
1259
|
-
`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
|
|
1260
|
+
`WITH k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
|
|
1260
1261
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
1261
1262
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
1262
|
-
FROM sync_changes AS ch
|
|
1263
|
-
WHERE ch.
|
|
1264
|
-
AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
|
|
1263
|
+
FROM json_each(?3) AS c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.value
|
|
1264
|
+
WHERE EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
|
|
1265
1265
|
ORDER BY ch.commit_seq, ch.idx`,
|
|
1266
|
-
[partition, JSON.stringify(keys),
|
|
1266
|
+
[partition, JSON.stringify(keys), JSON.stringify(commitSeqs)],
|
|
1267
1267
|
)
|
|
1268
1268
|
).map((record) => ({
|
|
1269
1269
|
commitSeq: record.commit_seq,
|
package/src/storage-query.ts
CHANGED
|
@@ -175,8 +175,9 @@ export interface LiveCandidateRow {
|
|
|
175
175
|
* The §4.5 live scan over a backend's two reads. `readCandidates` returns
|
|
176
176
|
* up to `limit` candidate commits above every key range's bound with their
|
|
177
177
|
* per-key byte sizes, and no payload. `readChanges` loads the changes the
|
|
178
|
-
* keys read in
|
|
179
|
-
*
|
|
178
|
+
* keys read in the given candidate commits, in (commitSeq, change index)
|
|
179
|
+
* order; it never visits a commit outside that list, so the cost of a page
|
|
180
|
+
* follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
|
|
180
181
|
* becomes one index range starting at the lowest bound of the units that
|
|
181
182
|
* share it. A change is kept when a unit covers its table, the pair's
|
|
182
183
|
* effective scopes match it, and its commit lies above that unit's own
|
|
@@ -197,8 +198,7 @@ export async function readLiveWindowPages(
|
|
|
197
198
|
) => Promise<readonly LiveCandidateRow[]>,
|
|
198
199
|
readChanges: (
|
|
199
200
|
keys: readonly LiveKey[],
|
|
200
|
-
|
|
201
|
-
through: number,
|
|
201
|
+
commitSeqs: readonly number[],
|
|
202
202
|
) => Promise<
|
|
203
203
|
readonly { readonly commitSeq: number; readonly change: StoredChange }[]
|
|
204
204
|
>,
|
|
@@ -246,7 +246,10 @@ export async function readLiveWindowPages(
|
|
|
246
246
|
const last = pending.at(-1);
|
|
247
247
|
if (last === undefined) return undefined;
|
|
248
248
|
const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
|
|
249
|
-
const rows = await readChanges(
|
|
249
|
+
const rows = await readChanges(
|
|
250
|
+
scanKeys(),
|
|
251
|
+
pending.map((c) => c.commitSeq),
|
|
252
|
+
);
|
|
250
253
|
pending = [];
|
|
251
254
|
scanned = last.commitSeq;
|
|
252
255
|
for (let i = 0; i < rows.length;) {
|