@syncular/server 0.82.1 → 0.82.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/postgres-storage.js +39 -13
- package/dist/sqlite-dialect.js +7 -7
- package/dist/storage-query.d.ts +4 -3
- package/dist/storage-query.js +4 -3
- package/package.json +2 -2
- package/src/postgres-storage.ts +50 -19
- package/src/sqlite-dialect.ts +8 -8
- package/src/storage-query.ts +8 -5
package/dist/postgres-storage.js
CHANGED
|
@@ -1153,6 +1153,26 @@ class PostgresTransaction {
|
|
|
1153
1153
|
})));
|
|
1154
1154
|
return cursors.find((cursor) => cursor !== undefined);
|
|
1155
1155
|
}
|
|
1156
|
+
/**
|
|
1157
|
+
* The executor of a read over `tables`: it writes the pending writes first
|
|
1158
|
+
* only when one of them is a row of those tables or a change of the commit
|
|
1159
|
+
* log to them (§6.4). A read of other tables runs without them, so it takes
|
|
1160
|
+
* no savepoint and moves no write ahead of itself.
|
|
1161
|
+
*/
|
|
1162
|
+
#readOf(tables) {
|
|
1163
|
+
return {
|
|
1164
|
+
query: (text, params) => {
|
|
1165
|
+
const appends = this.#stagedAppend === undefined
|
|
1166
|
+
? this.#closedAppends
|
|
1167
|
+
: [...this.#closedAppends, this.#stagedAppend];
|
|
1168
|
+
return tables.some((table) => this.#staged.has(table) ||
|
|
1169
|
+
this.#closedRows.has(table) ||
|
|
1170
|
+
appends.some(({ commit }) => commit.changes.some((change) => change.table === table)))
|
|
1171
|
+
? this.#client.query(text, params)
|
|
1172
|
+
: this.#serial.query(text, params);
|
|
1173
|
+
},
|
|
1174
|
+
};
|
|
1175
|
+
}
|
|
1156
1176
|
/** The candidate row this transaction staged and has not yet written. */
|
|
1157
1177
|
#stagedRow(table, rowId) {
|
|
1158
1178
|
return (this.#staged.get(table)?.get(rowId)?.row ??
|
|
@@ -1211,7 +1231,10 @@ class PostgresTransaction {
|
|
|
1211
1231
|
}
|
|
1212
1232
|
if (specs.length === 0)
|
|
1213
1233
|
return;
|
|
1214
|
-
const { rows } = await this.#
|
|
1234
|
+
const { rows } = await this.#readOf(specs.map((spec) => spec.tbl)).query(prefetchRowsBatchSql(tables), [
|
|
1235
|
+
this.#partition,
|
|
1236
|
+
JSON.stringify(specs),
|
|
1237
|
+
]);
|
|
1215
1238
|
this.#recordPrefetched(rows);
|
|
1216
1239
|
}
|
|
1217
1240
|
/** Answer the following `getRow`/`getTombstone` calls from a read snapshot. */
|
|
@@ -1229,7 +1252,7 @@ class PostgresTransaction {
|
|
|
1229
1252
|
const prefetched = this.#prefetchedTombstones.get(`${table}\u0000${rowId}`);
|
|
1230
1253
|
if (prefetched !== undefined)
|
|
1231
1254
|
return prefetched ?? undefined;
|
|
1232
|
-
const { rows } = await this.#
|
|
1255
|
+
const { rows } = await this.#readOf([table]).query('SELECT commit_seq, server_version, scopes FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId]);
|
|
1233
1256
|
const row = rows[0];
|
|
1234
1257
|
return (toTombstone(row?.commit_seq, row?.server_version, row?.scopes) ??
|
|
1235
1258
|
undefined);
|
|
@@ -1241,7 +1264,8 @@ class PostgresTransaction {
|
|
|
1241
1264
|
}
|
|
1242
1265
|
async scanRows(query) {
|
|
1243
1266
|
this.#assertOpen();
|
|
1244
|
-
const
|
|
1267
|
+
const client = this.#readOf([query.table]);
|
|
1268
|
+
const variable = await chooseScanVariable(this.#partition, query, this.#scanVariables, (probes, cap) => client
|
|
1245
1269
|
.query(scanProbeSql('postgres'), [
|
|
1246
1270
|
this.#partition,
|
|
1247
1271
|
JSON.stringify(probes.map((probe) => ({ ...probe, lim: cap }))),
|
|
@@ -1255,7 +1279,7 @@ class PostgresTransaction {
|
|
|
1255
1279
|
let afterRowId = query.afterRowId ?? '';
|
|
1256
1280
|
const batchSize = Math.max(64, query.limit);
|
|
1257
1281
|
while (rows.length < query.limit) {
|
|
1258
|
-
const { rows: records } = await
|
|
1282
|
+
const { rows: records } = await client.query(sql, [
|
|
1259
1283
|
this.#partition,
|
|
1260
1284
|
query.table,
|
|
1261
1285
|
variable,
|
|
@@ -1283,16 +1307,19 @@ class PostgresTransaction {
|
|
|
1283
1307
|
}
|
|
1284
1308
|
scanRowsByIndex(query) {
|
|
1285
1309
|
this.#assertOpen();
|
|
1286
|
-
return scanRowsByIndexOn(this.#
|
|
1310
|
+
return scanRowsByIndexOn(this.#readOf([query.table]), this.#resolveTable(query.table), this.#partition, query);
|
|
1287
1311
|
}
|
|
1288
1312
|
async queryAuthoritative(query) {
|
|
1289
1313
|
this.#assertOpen();
|
|
1290
1314
|
// Runs on this transaction's pinned client, so the read observes every
|
|
1291
|
-
// staged row and never waits for a second pool connection.
|
|
1315
|
+
// staged row and never waits for a second pool connection. The prepared
|
|
1316
|
+
// statement reads exactly `query.tables`: every relation of the plan is
|
|
1317
|
+
// one of them, rewritten to a derived relation over its own table.
|
|
1292
1318
|
const prepared = this.#prepareQuery(query);
|
|
1293
|
-
const result = await this.#
|
|
1319
|
+
const result = await this.#readOf(query.tables).query(postgresPlaceholders(prepared.sql), prepared.params);
|
|
1294
1320
|
// Under the partition lock the cursor is the locked `max_commit_seq` plus
|
|
1295
|
-
// this transaction's appends
|
|
1321
|
+
// this transaction's appends: no read. An append still held back changed
|
|
1322
|
+
// no table the query read, so the rows are those of that cursor.
|
|
1296
1323
|
if (this.#nextCommitSeq !== undefined) {
|
|
1297
1324
|
return { rows: result.rows, maxCommitSeq: this.#nextCommitSeq - 1 };
|
|
1298
1325
|
}
|
|
@@ -2377,8 +2404,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2377
2404
|
// the commit meta (NULL when the commit vanished, which still advances
|
|
2378
2405
|
// the scan) and, per key, the byte size of the changes the key reads;
|
|
2379
2406
|
// `octet_length` of a TOASTed payload reads its pointer, never the value.
|
|
2380
|
-
// The change read then
|
|
2381
|
-
// budget admits.
|
|
2407
|
+
// The change read then seeks only the commits the byte budget admits.
|
|
2382
2408
|
return readLiveWindowPages(query, async (keys, limit) => {
|
|
2383
2409
|
const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb)),
|
|
2384
2410
|
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq
|
|
@@ -2411,14 +2437,14 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2411
2437
|
? {}
|
|
2412
2438
|
: { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
|
|
2413
2439
|
}));
|
|
2414
|
-
}, async (keys,
|
|
2440
|
+
}, async (keys, commitSeqs) => {
|
|
2415
2441
|
const { rows } = await this.#exec.query(`WITH k AS MATERIALIZED (SELECT * FROM jsonb_to_recordset($2::text::jsonb) AS k(i int, var text, value text, after bigint, tables jsonb))
|
|
2416
2442
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
2417
2443
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
2418
2444
|
FROM sync_changes ch
|
|
2419
|
-
WHERE ch.partition=$1 AND ch.commit_seq
|
|
2445
|
+
WHERE ch.partition=$1 AND ch.commit_seq=ANY(ARRAY(SELECT jsonb_array_elements_text($3::text::jsonb)::bigint))
|
|
2420
2446
|
AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value)
|
|
2421
|
-
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys),
|
|
2447
|
+
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)]);
|
|
2422
2448
|
return rows.map((record) => ({
|
|
2423
2449
|
commitSeq: asNumber(record.commit_seq),
|
|
2424
2450
|
change: toStoredChange(record),
|
package/dist/sqlite-dialect.js
CHANGED
|
@@ -796,8 +796,9 @@ export class SqliteFamilyStorage {
|
|
|
796
796
|
// the index is missing). Each candidate LEFT JOINs its commit meta (NULL
|
|
797
797
|
// when the commit vanished, which still advances the scan) and, per key,
|
|
798
798
|
// the byte size of the changes the key reads; `length()` of a blob reads
|
|
799
|
-
// the record header, never the payload. The change read then
|
|
800
|
-
//
|
|
799
|
+
// the record header, never the payload. The change read then seeks only
|
|
800
|
+
// the commits the byte budget admits, through the primary key, with the
|
|
801
|
+
// key list materialized once per statement.
|
|
801
802
|
return readLiveWindowPages(query, async (keys, limit) => (await this.guardedAll(`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2)),
|
|
802
803
|
c AS MATERIALIZED (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
803
804
|
FROM k CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
|
|
@@ -826,13 +827,12 @@ export class SqliteFamilyStorage {
|
|
|
826
827
|
...(record.key !== null
|
|
827
828
|
? { key: record.key, bytes: record.bytes ?? 0 }
|
|
828
829
|
: {}),
|
|
829
|
-
})), async (keys,
|
|
830
|
+
})), async (keys, commitSeqs) => (await this.guardedAll(`WITH k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
|
|
830
831
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
831
832
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
832
|
-
FROM sync_changes AS ch
|
|
833
|
-
WHERE ch.
|
|
834
|
-
|
|
835
|
-
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), after, through])).map((record) => ({
|
|
833
|
+
FROM json_each(?3) AS c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.value
|
|
834
|
+
WHERE EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
|
|
835
|
+
ORDER BY ch.commit_seq, ch.idx`, [partition, JSON.stringify(keys), JSON.stringify(commitSeqs)])).map((record) => ({
|
|
836
836
|
commitSeq: record.commit_seq,
|
|
837
837
|
change: toStoredChange(record),
|
|
838
838
|
})));
|
package/dist/storage-query.d.ts
CHANGED
|
@@ -72,8 +72,9 @@ export interface LiveCandidateRow {
|
|
|
72
72
|
* The §4.5 live scan over a backend's two reads. `readCandidates` returns
|
|
73
73
|
* up to `limit` candidate commits above every key range's bound with their
|
|
74
74
|
* per-key byte sizes, and no payload. `readChanges` loads the changes the
|
|
75
|
-
* keys read in
|
|
76
|
-
*
|
|
75
|
+
* keys read in the given candidate commits, in (commitSeq, change index)
|
|
76
|
+
* order; it never visits a commit outside that list, so the cost of a page
|
|
77
|
+
* follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
|
|
77
78
|
* becomes one index range starting at the lowest bound of the units that
|
|
78
79
|
* share it. A change is kept when a unit covers its table, the pair's
|
|
79
80
|
* effective scopes match it, and its commit lies above that unit's own
|
|
@@ -86,7 +87,7 @@ export interface LiveCandidateRow {
|
|
|
86
87
|
* units of the keys that read the most of it, largest first, until the rest
|
|
87
88
|
* fits (§4.6).
|
|
88
89
|
*/
|
|
89
|
-
export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[],
|
|
90
|
+
export declare function readLiveWindowPages(query: LiveWindowQuery, readCandidates: (keys: readonly LiveKey[], limit: number) => Promise<readonly LiveCandidateRow[]>, readChanges: (keys: readonly LiveKey[], commitSeqs: readonly number[]) => Promise<readonly {
|
|
90
91
|
readonly commitSeq: number;
|
|
91
92
|
readonly change: StoredChange;
|
|
92
93
|
}[]>): Promise<LiveWindow>;
|
package/dist/storage-query.js
CHANGED
|
@@ -101,8 +101,9 @@ export async function transactionQuery(tx, query) {
|
|
|
101
101
|
* The §4.5 live scan over a backend's two reads. `readCandidates` returns
|
|
102
102
|
* up to `limit` candidate commits above every key range's bound with their
|
|
103
103
|
* per-key byte sizes, and no payload. `readChanges` loads the changes the
|
|
104
|
-
* keys read in
|
|
105
|
-
*
|
|
104
|
+
* keys read in the given candidate commits, in (commitSeq, change index)
|
|
105
|
+
* order; it never visits a commit outside that list, so the cost of a page
|
|
106
|
+
* follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
|
|
106
107
|
* becomes one index range starting at the lowest bound of the units that
|
|
107
108
|
* share it. A change is kept when a unit covers its table, the pair's
|
|
108
109
|
* effective scopes match it, and its commit lies above that unit's own
|
|
@@ -159,7 +160,7 @@ export async function readLiveWindowPages(query, readCandidates, readChanges) {
|
|
|
159
160
|
if (last === undefined)
|
|
160
161
|
return undefined;
|
|
161
162
|
const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
|
|
162
|
-
const rows = await readChanges(scanKeys(),
|
|
163
|
+
const rows = await readChanges(scanKeys(), pending.map((c) => c.commitSeq));
|
|
163
164
|
pending = [];
|
|
164
165
|
scanned = last.commitSeq;
|
|
165
166
|
for (let i = 0; i < rows.length;) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.82.
|
|
3
|
+
"version": "0.82.3",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.82.
|
|
71
|
+
"@syncular/core": "0.82.3"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/postgres-storage.ts
CHANGED
|
@@ -1748,6 +1748,33 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1748
1748
|
return cursors.find((cursor) => cursor !== undefined);
|
|
1749
1749
|
}
|
|
1750
1750
|
|
|
1751
|
+
/**
|
|
1752
|
+
* The executor of a read over `tables`: it writes the pending writes first
|
|
1753
|
+
* only when one of them is a row of those tables or a change of the commit
|
|
1754
|
+
* log to them (§6.4). A read of other tables runs without them, so it takes
|
|
1755
|
+
* no savepoint and moves no write ahead of itself.
|
|
1756
|
+
*/
|
|
1757
|
+
#readOf(tables: readonly string[]): PgQueryable {
|
|
1758
|
+
return {
|
|
1759
|
+
query: (text, params) => {
|
|
1760
|
+
const appends =
|
|
1761
|
+
this.#stagedAppend === undefined
|
|
1762
|
+
? this.#closedAppends
|
|
1763
|
+
: [...this.#closedAppends, this.#stagedAppend];
|
|
1764
|
+
return tables.some(
|
|
1765
|
+
(table) =>
|
|
1766
|
+
this.#staged.has(table) ||
|
|
1767
|
+
this.#closedRows.has(table) ||
|
|
1768
|
+
appends.some(({ commit }) =>
|
|
1769
|
+
commit.changes.some((change) => change.table === table),
|
|
1770
|
+
),
|
|
1771
|
+
)
|
|
1772
|
+
? this.#client.query(text, params)
|
|
1773
|
+
: this.#serial.query(text, params);
|
|
1774
|
+
},
|
|
1775
|
+
};
|
|
1776
|
+
}
|
|
1777
|
+
|
|
1751
1778
|
/** The candidate row this transaction staged and has not yet written. */
|
|
1752
1779
|
#stagedRow(table: string, rowId: string): StoredRow | undefined {
|
|
1753
1780
|
return (
|
|
@@ -1822,10 +1849,12 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1822
1849
|
specs.push({ tbl: target.table, ids });
|
|
1823
1850
|
}
|
|
1824
1851
|
if (specs.length === 0) return;
|
|
1825
|
-
const { rows } = await this.#
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1852
|
+
const { rows } = await this.#readOf(
|
|
1853
|
+
specs.map((spec) => spec.tbl),
|
|
1854
|
+
).query<PrefetchRow>(prefetchRowsBatchSql(tables), [
|
|
1855
|
+
this.#partition,
|
|
1856
|
+
JSON.stringify(specs),
|
|
1857
|
+
]);
|
|
1829
1858
|
this.#recordPrefetched(rows);
|
|
1830
1859
|
}
|
|
1831
1860
|
|
|
@@ -1857,7 +1886,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1857
1886
|
this.#assertOpen();
|
|
1858
1887
|
const prefetched = this.#prefetchedTombstones.get(`${table}\u0000${rowId}`);
|
|
1859
1888
|
if (prefetched !== undefined) return prefetched ?? undefined;
|
|
1860
|
-
const { rows } = await this.#
|
|
1889
|
+
const { rows } = await this.#readOf([table]).query<{
|
|
1861
1890
|
commit_seq: unknown;
|
|
1862
1891
|
server_version: unknown;
|
|
1863
1892
|
scopes: unknown;
|
|
@@ -1884,12 +1913,13 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1884
1913
|
|
|
1885
1914
|
async scanRows(query: RowScanQuery): Promise<StoredRow[]> {
|
|
1886
1915
|
this.#assertOpen();
|
|
1916
|
+
const client = this.#readOf([query.table]);
|
|
1887
1917
|
const variable = await chooseScanVariable(
|
|
1888
1918
|
this.#partition,
|
|
1889
1919
|
query,
|
|
1890
1920
|
this.#scanVariables,
|
|
1891
1921
|
(probes, cap) =>
|
|
1892
|
-
|
|
1922
|
+
client
|
|
1893
1923
|
.query<{ sub: unknown; n: unknown }>(scanProbeSql('postgres'), [
|
|
1894
1924
|
this.#partition,
|
|
1895
1925
|
JSON.stringify(probes.map((probe) => ({ ...probe, lim: cap }))),
|
|
@@ -1907,7 +1937,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1907
1937
|
let afterRowId = query.afterRowId ?? '';
|
|
1908
1938
|
const batchSize = Math.max(64, query.limit);
|
|
1909
1939
|
while (rows.length < query.limit) {
|
|
1910
|
-
const { rows: records } = await
|
|
1940
|
+
const { rows: records } = await client.query<RowRecord>(sql, [
|
|
1911
1941
|
this.#partition,
|
|
1912
1942
|
query.table,
|
|
1913
1943
|
variable,
|
|
@@ -1932,7 +1962,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1932
1962
|
scanRowsByIndex(query: IndexRowScanQuery): Promise<StoredRow[]> {
|
|
1933
1963
|
this.#assertOpen();
|
|
1934
1964
|
return scanRowsByIndexOn(
|
|
1935
|
-
this.#
|
|
1965
|
+
this.#readOf([query.table]),
|
|
1936
1966
|
this.#resolveTable(query.table),
|
|
1937
1967
|
this.#partition,
|
|
1938
1968
|
query,
|
|
@@ -1944,14 +1974,16 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1944
1974
|
): Promise<AuthoritativeQueryResult> {
|
|
1945
1975
|
this.#assertOpen();
|
|
1946
1976
|
// Runs on this transaction's pinned client, so the read observes every
|
|
1947
|
-
// staged row and never waits for a second pool connection.
|
|
1977
|
+
// staged row and never waits for a second pool connection. The prepared
|
|
1978
|
+
// statement reads exactly `query.tables`: every relation of the plan is
|
|
1979
|
+
// one of them, rewritten to a derived relation over its own table.
|
|
1948
1980
|
const prepared = this.#prepareQuery(query);
|
|
1949
|
-
const result = await this.#
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
);
|
|
1981
|
+
const result = await this.#readOf(query.tables).query<
|
|
1982
|
+
Readonly<Record<string, unknown>>
|
|
1983
|
+
>(postgresPlaceholders(prepared.sql), prepared.params);
|
|
1953
1984
|
// Under the partition lock the cursor is the locked `max_commit_seq` plus
|
|
1954
|
-
// this transaction's appends
|
|
1985
|
+
// this transaction's appends: no read. An append still held back changed
|
|
1986
|
+
// no table the query read, so the rows are those of that cursor.
|
|
1955
1987
|
if (this.#nextCommitSeq !== undefined) {
|
|
1956
1988
|
return { rows: result.rows, maxCommitSeq: this.#nextCommitSeq - 1 };
|
|
1957
1989
|
}
|
|
@@ -3519,8 +3551,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3519
3551
|
// the commit meta (NULL when the commit vanished, which still advances
|
|
3520
3552
|
// the scan) and, per key, the byte size of the changes the key reads;
|
|
3521
3553
|
// `octet_length` of a TOASTed payload reads its pointer, never the value.
|
|
3522
|
-
// The change read then
|
|
3523
|
-
// budget admits.
|
|
3554
|
+
// The change read then seeks only the commits the byte budget admits.
|
|
3524
3555
|
return readLiveWindowPages(
|
|
3525
3556
|
query,
|
|
3526
3557
|
async (keys, limit) => {
|
|
@@ -3565,7 +3596,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3565
3596
|
: { key: asNumber(record.key), bytes: asNumber(record.bytes) }),
|
|
3566
3597
|
}));
|
|
3567
3598
|
},
|
|
3568
|
-
async (keys,
|
|
3599
|
+
async (keys, commitSeqs) => {
|
|
3569
3600
|
const { rows } = await this.#exec.query<
|
|
3570
3601
|
ChangeRecord & { commit_seq: unknown }
|
|
3571
3602
|
>(
|
|
@@ -3573,10 +3604,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3573
3604
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
3574
3605
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
3575
3606
|
FROM sync_changes ch
|
|
3576
|
-
WHERE ch.partition=$1 AND ch.commit_seq
|
|
3607
|
+
WHERE ch.partition=$1 AND ch.commit_seq=ANY(ARRAY(SELECT jsonb_array_elements_text($3::text::jsonb)::bigint))
|
|
3577
3608
|
AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND k.tables @> to_jsonb(ch.tbl) AND (CASE jsonb_typeof(ch.scopes) WHEN 'string' THEN (ch.scopes#>>'{}')::jsonb ELSE ch.scopes END)->>k.var=k.value)
|
|
3578
3609
|
ORDER BY ch.commit_seq, ch.idx`,
|
|
3579
|
-
[partition, JSON.stringify(keys),
|
|
3610
|
+
[partition, JSON.stringify(keys), JSON.stringify(commitSeqs)],
|
|
3580
3611
|
);
|
|
3581
3612
|
return rows.map((record) => ({
|
|
3582
3613
|
commitSeq: asNumber(record.commit_seq),
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -1208,8 +1208,9 @@ export abstract class SqliteFamilyStorage {
|
|
|
1208
1208
|
// the index is missing). Each candidate LEFT JOINs its commit meta (NULL
|
|
1209
1209
|
// when the commit vanished, which still advances the scan) and, per key,
|
|
1210
1210
|
// the byte size of the changes the key reads; `length()` of a blob reads
|
|
1211
|
-
// the record header, never the payload. The change read then
|
|
1212
|
-
//
|
|
1211
|
+
// the record header, never the payload. The change read then seeks only
|
|
1212
|
+
// the commits the byte budget admits, through the primary key, with the
|
|
1213
|
+
// key list materialized once per statement.
|
|
1213
1214
|
return readLiveWindowPages(
|
|
1214
1215
|
query,
|
|
1215
1216
|
async (keys, limit) =>
|
|
@@ -1253,17 +1254,16 @@ export abstract class SqliteFamilyStorage {
|
|
|
1253
1254
|
? { key: record.key, bytes: record.bytes ?? 0 }
|
|
1254
1255
|
: {}),
|
|
1255
1256
|
})),
|
|
1256
|
-
async (keys,
|
|
1257
|
+
async (keys, commitSeqs) =>
|
|
1257
1258
|
(
|
|
1258
1259
|
await this.guardedAll<SqliteChangeRecord & { commit_seq: number }>(
|
|
1259
|
-
`WITH k AS (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
|
|
1260
|
+
`WITH k AS MATERIALIZED (SELECT json_extract(value, '$.i') AS i, json_extract(value, '$.var') AS var, json_extract(value, '$.value') AS value, json_extract(value, '$.after') AS after, json_extract(value, '$.tables') AS tables FROM json_each(?2))
|
|
1260
1261
|
SELECT ch.commit_seq AS commit_seq, ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op,
|
|
1261
1262
|
ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
1262
|
-
FROM sync_changes AS ch
|
|
1263
|
-
WHERE ch.
|
|
1264
|
-
AND EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
|
|
1263
|
+
FROM json_each(?3) AS c CROSS JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.value
|
|
1264
|
+
WHERE EXISTS (SELECT 1 FROM k WHERE ch.commit_seq>k.after AND ch.tbl IN (SELECT value FROM json_each(k.tables)) AND (ch.scopes ->> k.var)=k.value)
|
|
1265
1265
|
ORDER BY ch.commit_seq, ch.idx`,
|
|
1266
|
-
[partition, JSON.stringify(keys),
|
|
1266
|
+
[partition, JSON.stringify(keys), JSON.stringify(commitSeqs)],
|
|
1267
1267
|
)
|
|
1268
1268
|
).map((record) => ({
|
|
1269
1269
|
commitSeq: record.commit_seq,
|
package/src/storage-query.ts
CHANGED
|
@@ -175,8 +175,9 @@ export interface LiveCandidateRow {
|
|
|
175
175
|
* The §4.5 live scan over a backend's two reads. `readCandidates` returns
|
|
176
176
|
* up to `limit` candidate commits above every key range's bound with their
|
|
177
177
|
* per-key byte sizes, and no payload. `readChanges` loads the changes the
|
|
178
|
-
* keys read in
|
|
179
|
-
*
|
|
178
|
+
* keys read in the given candidate commits, in (commitSeq, change index)
|
|
179
|
+
* order; it never visits a commit outside that list, so the cost of a page
|
|
180
|
+
* follows its candidates and not the length of the change log below them. Each distinct unit key (the first variable, keys sorted)
|
|
180
181
|
* becomes one index range starting at the lowest bound of the units that
|
|
181
182
|
* share it. A change is kept when a unit covers its table, the pair's
|
|
182
183
|
* effective scopes match it, and its commit lies above that unit's own
|
|
@@ -197,8 +198,7 @@ export async function readLiveWindowPages(
|
|
|
197
198
|
) => Promise<readonly LiveCandidateRow[]>,
|
|
198
199
|
readChanges: (
|
|
199
200
|
keys: readonly LiveKey[],
|
|
200
|
-
|
|
201
|
-
through: number,
|
|
201
|
+
commitSeqs: readonly number[],
|
|
202
202
|
) => Promise<
|
|
203
203
|
readonly { readonly commitSeq: number; readonly change: StoredChange }[]
|
|
204
204
|
>,
|
|
@@ -246,7 +246,10 @@ export async function readLiveWindowPages(
|
|
|
246
246
|
const last = pending.at(-1);
|
|
247
247
|
if (last === undefined) return undefined;
|
|
248
248
|
const metas = new Map(pending.map((c) => [c.commitSeq, c.meta]));
|
|
249
|
-
const rows = await readChanges(
|
|
249
|
+
const rows = await readChanges(
|
|
250
|
+
scanKeys(),
|
|
251
|
+
pending.map((c) => c.commitSeq),
|
|
252
|
+
);
|
|
250
253
|
pending = [];
|
|
251
254
|
scanned = last.commitSeq;
|
|
252
255
|
for (let i = 0; i < rows.length;) {
|