@syncular/server 0.77.1 → 0.77.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/d1-storage.js +17 -11
- package/dist/postgres-storage.js +28 -15
- package/dist/relational-rows.d.ts +13 -0
- package/dist/relational-rows.js +22 -0
- package/dist/sqlite-dialect.d.ts +2 -0
- package/dist/sqlite-dialect.js +14 -8
- package/dist/storage-query.d.ts +30 -4
- package/dist/storage-query.js +59 -9
- package/package.json +2 -2
- package/src/d1-storage.ts +24 -9
- package/src/postgres-storage.ts +47 -12
- package/src/relational-rows.ts +23 -0
- package/src/sqlite-dialect.ts +20 -7
- package/src/storage-query.ts +75 -9
package/dist/d1-storage.js
CHANGED
|
@@ -40,12 +40,12 @@ var _a;
|
|
|
40
40
|
* rather than a lock D1 does not expose.
|
|
41
41
|
*/
|
|
42
42
|
import { decodeRow } from '@syncular/core';
|
|
43
|
-
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
|
|
43
|
+
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanProbeSql, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
|
|
44
44
|
import { matchesEffective } from './scopes.js';
|
|
45
45
|
import { asUint8Array, deserializePushResult, placeholders, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteDdlStatements, SQLITE_UPGRADE_PROBE, sqliteUpgradeDdl, toStoredRow, toTombstone, } from './sqlite-dialect.js';
|
|
46
46
|
import { commitDeleteFloor } from './storage.js';
|
|
47
47
|
import { isD1ConstraintError, StorageConstraintError, StorageQueryError, } from './storage-errors.js';
|
|
48
|
-
import {
|
|
48
|
+
import { chooseScanVariable, resolveIndexRowScan } from './storage-query.js';
|
|
49
49
|
/** The rows of one `db.batch` result entry (`{ results }` per statement). */
|
|
50
50
|
function batchRows(result) {
|
|
51
51
|
if (typeof result !== 'object' ||
|
|
@@ -88,6 +88,7 @@ class D1Transaction extends SqliteFamilyTransaction {
|
|
|
88
88
|
/** Storage-owned relation rewrite, bound to this transaction's partition. */
|
|
89
89
|
#prepareQuery;
|
|
90
90
|
#pushApplySerialized;
|
|
91
|
+
#scanVariables;
|
|
91
92
|
#buffer = [];
|
|
92
93
|
/** Live snapshot of `max_commit_seq`, advanced within this transaction. */
|
|
93
94
|
#maxCommitSeq;
|
|
@@ -117,9 +118,10 @@ class D1Transaction extends SqliteFamilyTransaction {
|
|
|
117
118
|
* this transaction wrote.
|
|
118
119
|
*/
|
|
119
120
|
#prefetched = new Map();
|
|
120
|
-
constructor(db, partition, resolveTable, prepareQuery, pushApplySerialized) {
|
|
121
|
+
constructor(db, partition, resolveTable, prepareQuery, pushApplySerialized, scanVariables) {
|
|
121
122
|
super(partition);
|
|
122
123
|
this.#db = db;
|
|
124
|
+
this.#scanVariables = scanVariables;
|
|
123
125
|
this.#resolveTable = resolveTable;
|
|
124
126
|
this.#prepareQuery = prepareQuery;
|
|
125
127
|
this.#pushApplySerialized = pushApplySerialized;
|
|
@@ -222,20 +224,24 @@ class D1Transaction extends SqliteFamilyTransaction {
|
|
|
222
224
|
}
|
|
223
225
|
async scanRows(query) {
|
|
224
226
|
this.assertOpen();
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
227
|
+
// A probe that finds a variable without persisted entries leaves only
|
|
228
|
+
// this transaction's pending rows to merge.
|
|
229
|
+
const variable = await chooseScanVariable(this.partition, query, this.#scanVariables, (probes, cap) => this.all(scanProbeSql('sqlite'), [
|
|
230
|
+
this.partition,
|
|
231
|
+
JSON.stringify(probes),
|
|
232
|
+
cap,
|
|
233
|
+
]));
|
|
229
234
|
const pendingForTable = [...this.#pending.entries()].filter(([key]) => key.startsWith(`${query.table}\u0000`));
|
|
230
235
|
const persistedLimit = query.limit + pendingForTable.length;
|
|
231
|
-
const
|
|
236
|
+
const values = variable === undefined ? [] : query.scopeFilter[variable];
|
|
237
|
+
const sql = scanRowPageSql(this.table(query.table), values.length, 'sqlite');
|
|
232
238
|
const persisted = [];
|
|
233
239
|
let afterRowId = query.afterRowId ?? '';
|
|
234
240
|
const batchSize = Math.max(64, persistedLimit);
|
|
235
|
-
while (persisted.length < persistedLimit) {
|
|
241
|
+
while (values.length > 0 && persisted.length < persistedLimit) {
|
|
236
242
|
const { results: records } = await this.#db
|
|
237
243
|
.prepare(sql)
|
|
238
|
-
.bind(this.partition, query.table,
|
|
244
|
+
.bind(this.partition, query.table, variable, JSON.stringify(values), afterRowId, batchSize, this.partition)
|
|
239
245
|
.all();
|
|
240
246
|
if (records.length === 0)
|
|
241
247
|
break;
|
|
@@ -1101,7 +1107,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
1101
1107
|
if (table === undefined)
|
|
1102
1108
|
throw new StorageQueryError('sync.storage.schema_changed');
|
|
1103
1109
|
return table;
|
|
1104
|
-
}, (query) => this.prepareAuthoritativeQuery(partition, query), this.#pushApplySerialized);
|
|
1110
|
+
}, (query) => this.prepareAuthoritativeQuery(partition, query), this.#pushApplySerialized, this.scanVariables);
|
|
1105
1111
|
}
|
|
1106
1112
|
async queryAuthoritative(partition, query, _checkpoints) {
|
|
1107
1113
|
// D1 declares no checkpoints. `#schemaDatabase` prepends a guard that
|
package/dist/postgres-storage.js
CHANGED
|
@@ -43,10 +43,10 @@ import { decodeRow, } from '@syncular/core';
|
|
|
43
43
|
import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
44
44
|
import { syncError } from './errors.js';
|
|
45
45
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
46
|
-
import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
46
|
+
import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanProbeSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
47
47
|
import { matchesEffective, sha256Hex } from './scopes.js';
|
|
48
48
|
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
49
|
-
import {
|
|
49
|
+
import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
|
|
50
50
|
/**
|
|
51
51
|
* Schema DDL. Applied by `PostgresServerStorage.migrate()` (idempotent).
|
|
52
52
|
*
|
|
@@ -837,6 +837,8 @@ class PostgresTransaction {
|
|
|
837
837
|
/** Schema version this transaction writes; rides every appended commit row. */
|
|
838
838
|
#writerVersion;
|
|
839
839
|
#resolveTable;
|
|
840
|
+
/** The storage's `chooseScanVariable` picks. */
|
|
841
|
+
#scanVariables;
|
|
840
842
|
#open = true;
|
|
841
843
|
#pushApplySavepoint = false;
|
|
842
844
|
/**
|
|
@@ -872,7 +874,8 @@ class PostgresTransaction {
|
|
|
872
874
|
/** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
|
|
873
875
|
#resolve;
|
|
874
876
|
#reject;
|
|
875
|
-
constructor(client, partition, writerVersion, resolveTable, prepareQuery, resolve, reject) {
|
|
877
|
+
constructor(client, partition, writerVersion, resolveTable, prepareQuery, resolve, reject, scanVariables) {
|
|
878
|
+
this.#scanVariables = scanVariables;
|
|
876
879
|
// One pinned connection carries one statement at a time: node-postgres
|
|
877
880
|
// warns about a concurrently issued second query today and rejects it in
|
|
878
881
|
// node-postgres 9. A whole-commit validator that fans independent reads
|
|
@@ -1038,11 +1041,16 @@ class PostgresTransaction {
|
|
|
1038
1041
|
}
|
|
1039
1042
|
async scanRows(query) {
|
|
1040
1043
|
this.#assertOpen();
|
|
1041
|
-
const
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
+
const variable = await chooseScanVariable(this.#partition, query, this.#scanVariables, (probes, cap) => this.#client
|
|
1045
|
+
.query(scanProbeSql('postgres'), [
|
|
1046
|
+
this.#partition,
|
|
1047
|
+
JSON.stringify(probes.map((probe) => ({ ...probe, lim: cap }))),
|
|
1048
|
+
])
|
|
1049
|
+
.then(({ rows }) => rows));
|
|
1050
|
+
if (variable === undefined)
|
|
1044
1051
|
return [];
|
|
1045
|
-
const
|
|
1052
|
+
const values = query.scopeFilter[variable];
|
|
1053
|
+
const sql = scanRowPageSql(this.#resolveTable(query.table), values.length, 'postgres');
|
|
1046
1054
|
const rows = [];
|
|
1047
1055
|
let afterRowId = query.afterRowId ?? '';
|
|
1048
1056
|
const batchSize = Math.max(64, query.limit);
|
|
@@ -1050,8 +1058,8 @@ class PostgresTransaction {
|
|
|
1050
1058
|
const { rows: records } = await this.#client.query(sql, [
|
|
1051
1059
|
this.#partition,
|
|
1052
1060
|
query.table,
|
|
1053
|
-
|
|
1054
|
-
...
|
|
1061
|
+
variable,
|
|
1062
|
+
...values,
|
|
1055
1063
|
afterRowId,
|
|
1056
1064
|
batchSize,
|
|
1057
1065
|
]);
|
|
@@ -1501,6 +1509,8 @@ export class PostgresServerStorage {
|
|
|
1501
1509
|
* (SYNCULAR-PULL-ROUNDTRIPS-001).
|
|
1502
1510
|
*/
|
|
1503
1511
|
#pageBatches = new Map();
|
|
1512
|
+
/** `chooseScanVariable` picks, shared with this storage's transactions. */
|
|
1513
|
+
#scanVariables = new Map();
|
|
1504
1514
|
constructor(exec) {
|
|
1505
1515
|
this.#exec = exec;
|
|
1506
1516
|
}
|
|
@@ -1891,7 +1901,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1891
1901
|
});
|
|
1892
1902
|
const done = this.#exec
|
|
1893
1903
|
.transaction(async (client) => {
|
|
1894
|
-
const tx = new PostgresTransaction(client, partition, this.#schemaVersion ?? 0, (name) => this.table(name), (query) => this.#prepareAuthoritativeQuery(partition, query), resolveScope, rejectScope);
|
|
1904
|
+
const tx = new PostgresTransaction(client, partition, this.#schemaVersion ?? 0, (name) => this.table(name), (query) => this.#prepareAuthoritativeQuery(partition, query), resolveScope, rejectScope, this.#scanVariables);
|
|
1895
1905
|
opened = true;
|
|
1896
1906
|
resolveReady(tx);
|
|
1897
1907
|
// Hold the transaction open until commit()/rollback() settles `scope`.
|
|
@@ -2308,9 +2318,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2308
2318
|
});
|
|
2309
2319
|
}
|
|
2310
2320
|
async scanRows(partition, query) {
|
|
2311
|
-
|
|
2312
|
-
const
|
|
2313
|
-
|
|
2321
|
+
// The probes of concurrent scans leave as one statement, like their pages.
|
|
2322
|
+
const variable = await chooseScanVariable(partition, query, this.#scanVariables, (probes, cap) => Promise.all(probes.map(async (probe) => {
|
|
2323
|
+
const [row] = await this.#batchedPage('scanProbe', scanProbeSql('postgres'), partition, { tbl: probe.tbl, var: probe.var, vals: probe.vals, lim: cap });
|
|
2324
|
+
return { sub: probe.i, n: row?.n };
|
|
2325
|
+
})));
|
|
2326
|
+
if (variable === undefined)
|
|
2314
2327
|
return [];
|
|
2315
2328
|
// Candidates via the inverted index (ordered + LIMITed at the covering
|
|
2316
2329
|
// PK) LEFT JOINed to the row table — one round trip per page, never one
|
|
@@ -2323,8 +2336,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2323
2336
|
while (rows.length < query.limit) {
|
|
2324
2337
|
const records = await this.#batchedPage('scanRows', (specs) => scanRowBatchSql([...new Set(specs.map((spec) => String(spec.tbl)))].map((table) => this.table(table))), partition, {
|
|
2325
2338
|
tbl: query.table,
|
|
2326
|
-
var:
|
|
2327
|
-
vals:
|
|
2339
|
+
var: variable,
|
|
2340
|
+
vals: query.scopeFilter[variable],
|
|
2328
2341
|
after: afterRowId,
|
|
2329
2342
|
lim: batchSize,
|
|
2330
2343
|
});
|
|
@@ -230,6 +230,19 @@ export declare function indexRowPageStatement(table: CompiledTable, index: Index
|
|
|
230
230
|
* cap is unreachable through scope values.
|
|
231
231
|
*/
|
|
232
232
|
export declare function scanRowPageSql(table: CompiledTable, valueCount: number, dialect: RelationalDialect): string;
|
|
233
|
+
/**
|
|
234
|
+
* Selectivity probe for `chooseScanVariable`: per `{i, tbl, var, vals}`
|
|
235
|
+
* record of the JSON array, the count of its `sync_row_scopes` entries up
|
|
236
|
+
* to a cap, read from the covering primary key. Rows come back as
|
|
237
|
+
* `(sub, n)`, `sub` being the record's `i`.
|
|
238
|
+
*
|
|
239
|
+
* Bind order:
|
|
240
|
+
* - postgres: [partition, records as JSON text]; each record carries its
|
|
241
|
+
* own cap as `lim`, so probes of concurrent scans share one statement;
|
|
242
|
+
* - sqlite: [partition, records as JSON text, cap]. SQLite rejects a
|
|
243
|
+
* correlated LIMIT, so one cap binds for every record.
|
|
244
|
+
*/
|
|
245
|
+
export declare function scanProbeSql(dialect: RelationalDialect): string;
|
|
233
246
|
/**
|
|
234
247
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
235
248
|
* page of every subscription scanning any table in a pull. `$2` is a JSON
|
package/dist/relational-rows.js
CHANGED
|
@@ -574,6 +574,28 @@ export function scanRowPageSql(table, valueCount, dialect) {
|
|
|
574
574
|
ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=${joinPartition} AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=c.row_id
|
|
575
575
|
ORDER BY c.row_id`;
|
|
576
576
|
}
|
|
577
|
+
/**
|
|
578
|
+
* Selectivity probe for `chooseScanVariable`: per `{i, tbl, var, vals}`
|
|
579
|
+
* record of the JSON array, the count of its `sync_row_scopes` entries up
|
|
580
|
+
* to a cap, read from the covering primary key. Rows come back as
|
|
581
|
+
* `(sub, n)`, `sub` being the record's `i`.
|
|
582
|
+
*
|
|
583
|
+
* Bind order:
|
|
584
|
+
* - postgres: [partition, records as JSON text]; each record carries its
|
|
585
|
+
* own cap as `lim`, so probes of concurrent scans share one statement;
|
|
586
|
+
* - sqlite: [partition, records as JSON text, cap]. SQLite rejects a
|
|
587
|
+
* correlated LIMIT, so one cap binds for every record.
|
|
588
|
+
*/
|
|
589
|
+
export function scanProbeSql(dialect) {
|
|
590
|
+
return dialect === 'sqlite'
|
|
591
|
+
? `SELECT json_extract(p.value, '$.i') AS sub, (SELECT count(*) FROM (SELECT 1 FROM sync_row_scopes
|
|
592
|
+
WHERE partition=?1 AND tbl=json_extract(p.value, '$.tbl') AND var=json_extract(p.value, '$.var')
|
|
593
|
+
AND value IN (SELECT value FROM json_each(p.value, '$.vals')) LIMIT ?3)) AS n
|
|
594
|
+
FROM json_each(?2) AS p`
|
|
595
|
+
: `SELECT p.i AS sub, (SELECT count(*) FROM (SELECT 1 FROM sync_row_scopes
|
|
596
|
+
WHERE partition=$1 AND tbl=p.tbl AND var=p.var AND value=ANY(p.vals) LIMIT p.lim) x) AS n
|
|
597
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS p(i int, tbl text, var text, vals text[], lim int)`;
|
|
598
|
+
}
|
|
577
599
|
/**
|
|
578
600
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
579
601
|
* page of every subscription scanning any table in a pull. `$2` is a JSON
|
package/dist/sqlite-dialect.d.ts
CHANGED
|
@@ -103,6 +103,8 @@ export declare abstract class SqliteFamilyStorage {
|
|
|
103
103
|
protected schemaVersion: number | undefined;
|
|
104
104
|
/** Bound parameters one statement accepts. */
|
|
105
105
|
protected abstract readonly maxBindParams: number;
|
|
106
|
+
/** `chooseScanVariable` picks, shared with this storage's transactions. */
|
|
107
|
+
protected readonly scanVariables: Map<string, string>;
|
|
106
108
|
/** Run one statement and return its rows. */
|
|
107
109
|
protected abstract all<T>(sql: string, params: readonly SqliteValue[]): Promise<T[]>;
|
|
108
110
|
/**
|
package/dist/sqlite-dialect.js
CHANGED
|
@@ -2,10 +2,10 @@ import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './author
|
|
|
2
2
|
import { syncError } from './errors.js';
|
|
3
3
|
import { validateCommitPruneQuery } from './prune.js';
|
|
4
4
|
import { StorageQueryError } from './storage-errors.js';
|
|
5
|
-
import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
|
|
5
|
+
import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, scanProbeSql, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
|
|
6
6
|
import { matchesEffective } from './scopes.js';
|
|
7
7
|
import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, rowsLiveAfterDelete, } from './storage.js';
|
|
8
|
-
import {
|
|
8
|
+
import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
|
|
9
9
|
/**
|
|
10
10
|
* Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
|
|
11
11
|
* the whole string via `db.exec(SQLITE_DDL)`; D1 applies each statement
|
|
@@ -446,6 +446,8 @@ export class SqliteFamilyStorage {
|
|
|
446
446
|
/** Set by `ensureSchema`: app-table lookup for the relational row store. */
|
|
447
447
|
tables;
|
|
448
448
|
schemaVersion;
|
|
449
|
+
/** `chooseScanVariable` picks, shared with this storage's transactions. */
|
|
450
|
+
scanVariables = new Map();
|
|
449
451
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
450
452
|
table(name) {
|
|
451
453
|
const table = this.tables?.get(name);
|
|
@@ -836,14 +838,18 @@ export class SqliteFamilyStorage {
|
|
|
836
838
|
})));
|
|
837
839
|
}
|
|
838
840
|
async scanRows(partition, query) {
|
|
839
|
-
const
|
|
840
|
-
|
|
841
|
-
|
|
841
|
+
const variable = await chooseScanVariable(partition, query, this.scanVariables, (probes, cap) => this.guardedAll(scanProbeSql('sqlite'), [
|
|
842
|
+
partition,
|
|
843
|
+
JSON.stringify(probes),
|
|
844
|
+
cap,
|
|
845
|
+
]));
|
|
846
|
+
if (variable === undefined)
|
|
842
847
|
return [];
|
|
848
|
+
const values = query.scopeFilter[variable];
|
|
843
849
|
// Candidates via the inverted index LEFT JOINed to the row table — one
|
|
844
850
|
// statement per page, never one per row (see `scanRowPageSql`). Exact
|
|
845
851
|
// multi-variable verification against the stored scope map below.
|
|
846
|
-
const sql = scanRowPageSql(this.table(query.table),
|
|
852
|
+
const sql = scanRowPageSql(this.table(query.table), values.length, 'sqlite');
|
|
847
853
|
const rows = [];
|
|
848
854
|
let afterRowId = query.afterRowId ?? '';
|
|
849
855
|
const batchSize = Math.max(64, query.limit);
|
|
@@ -851,8 +857,8 @@ export class SqliteFamilyStorage {
|
|
|
851
857
|
const records = await this.guardedAll(sql, [
|
|
852
858
|
partition,
|
|
853
859
|
query.table,
|
|
854
|
-
|
|
855
|
-
JSON.stringify(
|
|
860
|
+
variable,
|
|
861
|
+
JSON.stringify(values),
|
|
856
862
|
afterRowId,
|
|
857
863
|
batchSize,
|
|
858
864
|
partition,
|
package/dist/storage-query.d.ts
CHANGED
|
@@ -1,11 +1,37 @@
|
|
|
1
1
|
import type { CompiledTable, IndexSchema } from './schema.js';
|
|
2
2
|
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, IndexRowScanQuery, LiveWindow, LiveWindowQuery, RowScanQuery, StorageTransaction, StoredChange } from './storage.js';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* drives its inverted-index candidate selection with.
|
|
4
|
+
* One variable of a scan filter whose index entries a probe counts; `i` is
|
|
5
|
+
* the row's `sub` in the probe result (see `scanProbeSql`).
|
|
7
6
|
*/
|
|
8
|
-
export
|
|
7
|
+
export interface ScanProbe {
|
|
8
|
+
readonly i: number;
|
|
9
|
+
readonly tbl: string;
|
|
10
|
+
readonly var: string;
|
|
11
|
+
readonly vals: readonly string[];
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* The scope variable a `scanRows` call seeks the row-scope index with: the
|
|
15
|
+
* one whose values have the fewest index entries, ties broken alphabetically.
|
|
16
|
+
* `undefined` means the intersection is empty: a variable without values or
|
|
17
|
+
* without entries, so the scan returns no persisted row. An empty or omitted
|
|
18
|
+
* filter fails loudly instead of making an unsupported unscoped scan look
|
|
19
|
+
* empty.
|
|
20
|
+
*
|
|
21
|
+
* A single-variable filter needs no probe. Otherwise `count` counts each
|
|
22
|
+
* variable's entries in one statement, up to `cap`. The cap starts at
|
|
23
|
+
* max(64, limit) and grows fourfold, to at most 65,536, while the fewest
|
|
24
|
+
* count is shared by several variables at the cap.
|
|
25
|
+
*
|
|
26
|
+
* `chosen` memoizes the pick per (partition, table, filter), at most 1,024
|
|
27
|
+
* entries, so the page loop of a bootstrap probes once. A stale pick only
|
|
28
|
+
* costs index reads: every scan still verifies all variables against the
|
|
29
|
+
* stored scopes, so the choice never changes a result.
|
|
30
|
+
*/
|
|
31
|
+
export declare function chooseScanVariable(partition: string, query: RowScanQuery, chosen: Map<string, string>, count: (probes: readonly ScanProbe[], cap: number) => Promise<readonly {
|
|
32
|
+
readonly sub: unknown;
|
|
33
|
+
readonly n: unknown;
|
|
34
|
+
}[]>): Promise<string | undefined>;
|
|
9
35
|
/** Validate and resolve one exact trusted-host relational index lookup. */
|
|
10
36
|
export declare function resolveIndexRowScan(table: CompiledTable, query: IndexRowScanQuery): IndexSchema;
|
|
11
37
|
/**
|
package/dist/storage-query.js
CHANGED
|
@@ -1,19 +1,69 @@
|
|
|
1
1
|
import { matchesEffective } from './scopes.js';
|
|
2
2
|
import { StorageQueryError } from './storage-errors.js';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* The scope variable a `scanRows` call seeks the row-scope index with: the
|
|
5
|
+
* one whose values have the fewest index entries, ties broken alphabetically.
|
|
6
|
+
* `undefined` means the intersection is empty: a variable without values or
|
|
7
|
+
* without entries, so the scan returns no persisted row. An empty or omitted
|
|
8
|
+
* filter fails loudly instead of making an unsupported unscoped scan look
|
|
9
|
+
* empty.
|
|
10
|
+
*
|
|
11
|
+
* A single-variable filter needs no probe. Otherwise `count` counts each
|
|
12
|
+
* variable's entries in one statement, up to `cap`. The cap starts at
|
|
13
|
+
* max(64, limit) and grows fourfold, to at most 65,536, while the fewest
|
|
14
|
+
* count is shared by several variables at the cap.
|
|
15
|
+
*
|
|
16
|
+
* `chosen` memoizes the pick per (partition, table, filter), at most 1,024
|
|
17
|
+
* entries, so the page loop of a bootstrap probes once. A stale pick only
|
|
18
|
+
* costs index reads: every scan still verifies all variables against the
|
|
19
|
+
* stored scopes, so the choice never changes a result.
|
|
7
20
|
*/
|
|
8
|
-
export function
|
|
21
|
+
export async function chooseScanVariable(partition, query, chosen, count) {
|
|
9
22
|
const scopeFilter = query.scopeFilter;
|
|
10
|
-
const
|
|
11
|
-
?
|
|
12
|
-
: Object.keys(scopeFilter).sort()
|
|
13
|
-
if (
|
|
23
|
+
const variables = scopeFilter === undefined || scopeFilter === null
|
|
24
|
+
? []
|
|
25
|
+
: Object.keys(scopeFilter).sort();
|
|
26
|
+
if (scopeFilter === undefined ||
|
|
27
|
+
scopeFilter === null ||
|
|
28
|
+
variables[0] === undefined) {
|
|
14
29
|
throw new StorageQueryError('sync.storage.scan_requires_scope');
|
|
15
30
|
}
|
|
16
|
-
|
|
31
|
+
if (variables.some((variable) => scopeFilter[variable]?.length === 0)) {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
if (variables.length === 1)
|
|
35
|
+
return variables[0];
|
|
36
|
+
const key = JSON.stringify([
|
|
37
|
+
partition,
|
|
38
|
+
query.table,
|
|
39
|
+
variables.map((variable) => [variable, [...scopeFilter[variable]].sort()]),
|
|
40
|
+
]);
|
|
41
|
+
const cached = chosen.get(key);
|
|
42
|
+
if (cached !== undefined)
|
|
43
|
+
return cached;
|
|
44
|
+
let tied = variables;
|
|
45
|
+
for (let cap = Math.max(64, query.limit);; cap = Math.min(cap * 4, 65_536)) {
|
|
46
|
+
const counts = tied.map(() => 0);
|
|
47
|
+
const probes = tied.map((variable, i) => ({
|
|
48
|
+
i,
|
|
49
|
+
tbl: query.table,
|
|
50
|
+
var: variable,
|
|
51
|
+
vals: scopeFilter[variable],
|
|
52
|
+
}));
|
|
53
|
+
for (const row of await count(probes, cap)) {
|
|
54
|
+
counts[Number(row.sub)] = Number(row.n);
|
|
55
|
+
}
|
|
56
|
+
const fewest = Math.min(...counts);
|
|
57
|
+
if (fewest === 0)
|
|
58
|
+
return undefined;
|
|
59
|
+
tied = tied.filter((_, i) => counts[i] === fewest);
|
|
60
|
+
if (fewest < cap || tied.length === 1 || cap >= 65_536)
|
|
61
|
+
break;
|
|
62
|
+
}
|
|
63
|
+
if (chosen.size >= 1_024)
|
|
64
|
+
chosen.delete(chosen.keys().next().value);
|
|
65
|
+
chosen.set(key, tied[0]);
|
|
66
|
+
return tied[0];
|
|
17
67
|
}
|
|
18
68
|
/** Validate and resolve one exact trusted-host relational index lookup. */
|
|
19
69
|
export function resolveIndexRowScan(table, query) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.77.
|
|
3
|
+
"version": "0.77.3",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.77.
|
|
71
|
+
"@syncular/core": "0.77.3"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/d1-storage.ts
CHANGED
|
@@ -61,6 +61,7 @@ import {
|
|
|
61
61
|
rewriteValues,
|
|
62
62
|
SCHEMA_META_DDL_SQLITE,
|
|
63
63
|
type StoredColumnLayout,
|
|
64
|
+
scanProbeSql,
|
|
64
65
|
scanRowPageSql,
|
|
65
66
|
schemaDdl,
|
|
66
67
|
serverIndexDdl,
|
|
@@ -118,7 +119,7 @@ import {
|
|
|
118
119
|
StorageConstraintError,
|
|
119
120
|
StorageQueryError,
|
|
120
121
|
} from './storage-errors';
|
|
121
|
-
import {
|
|
122
|
+
import { chooseScanVariable, resolveIndexRowScan } from './storage-query';
|
|
122
123
|
|
|
123
124
|
// -- The subset of the D1 API this storage uses (structural typing) ---------
|
|
124
125
|
// Declared locally so the package takes no `@cloudflare/workers-types`
|
|
@@ -206,6 +207,7 @@ class D1Transaction
|
|
|
206
207
|
query: AuthoritativeQueryRequest,
|
|
207
208
|
) => BoundAuthoritativeQuery;
|
|
208
209
|
readonly #pushApplySerialized: boolean;
|
|
210
|
+
readonly #scanVariables: Map<string, string>;
|
|
209
211
|
readonly #buffer: SqliteStatementSpec[] = [];
|
|
210
212
|
/** Live snapshot of `max_commit_seq`, advanced within this transaction. */
|
|
211
213
|
#maxCommitSeq: number | undefined;
|
|
@@ -248,9 +250,11 @@ class D1Transaction
|
|
|
248
250
|
resolveTable: (name: string) => CompiledTable,
|
|
249
251
|
prepareQuery: (query: AuthoritativeQueryRequest) => BoundAuthoritativeQuery,
|
|
250
252
|
pushApplySerialized: boolean,
|
|
253
|
+
scanVariables: Map<string, string>,
|
|
251
254
|
) {
|
|
252
255
|
super(partition);
|
|
253
256
|
this.#db = db;
|
|
257
|
+
this.#scanVariables = scanVariables;
|
|
254
258
|
this.#resolveTable = resolveTable;
|
|
255
259
|
this.#prepareQuery = prepareQuery;
|
|
256
260
|
this.#pushApplySerialized = pushApplySerialized;
|
|
@@ -393,30 +397,40 @@ class D1Transaction
|
|
|
393
397
|
|
|
394
398
|
async scanRows(query: RowScanQuery): Promise<StoredRow[]> {
|
|
395
399
|
this.assertOpen();
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
+
// A probe that finds a variable without persisted entries leaves only
|
|
401
|
+
// this transaction's pending rows to merge.
|
|
402
|
+
const variable = await chooseScanVariable(
|
|
403
|
+
this.partition,
|
|
404
|
+
query,
|
|
405
|
+
this.#scanVariables,
|
|
406
|
+
(probes, cap) =>
|
|
407
|
+
this.all<{ sub: number; n: number }>(scanProbeSql('sqlite'), [
|
|
408
|
+
this.partition,
|
|
409
|
+
JSON.stringify(probes),
|
|
410
|
+
cap,
|
|
411
|
+
]),
|
|
412
|
+
);
|
|
400
413
|
const pendingForTable = [...this.#pending.entries()].filter(([key]) =>
|
|
401
414
|
key.startsWith(`${query.table}\u0000`),
|
|
402
415
|
);
|
|
403
416
|
const persistedLimit = query.limit + pendingForTable.length;
|
|
417
|
+
const values = variable === undefined ? [] : query.scopeFilter[variable]!;
|
|
404
418
|
const sql = scanRowPageSql(
|
|
405
419
|
this.table(query.table),
|
|
406
|
-
|
|
420
|
+
values.length,
|
|
407
421
|
'sqlite',
|
|
408
422
|
);
|
|
409
423
|
const persisted: StoredRow[] = [];
|
|
410
424
|
let afterRowId = query.afterRowId ?? '';
|
|
411
425
|
const batchSize = Math.max(64, persistedLimit);
|
|
412
|
-
while (persisted.length < persistedLimit) {
|
|
426
|
+
while (values.length > 0 && persisted.length < persistedLimit) {
|
|
413
427
|
const { results: records } = await this.#db
|
|
414
428
|
.prepare(sql)
|
|
415
429
|
.bind(
|
|
416
430
|
this.partition,
|
|
417
431
|
query.table,
|
|
418
|
-
|
|
419
|
-
JSON.stringify(
|
|
432
|
+
variable,
|
|
433
|
+
JSON.stringify(values),
|
|
420
434
|
afterRowId,
|
|
421
435
|
batchSize,
|
|
422
436
|
this.partition,
|
|
@@ -1610,6 +1624,7 @@ export class D1ServerStorage
|
|
|
1610
1624
|
},
|
|
1611
1625
|
(query) => this.prepareAuthoritativeQuery(partition, query),
|
|
1612
1626
|
this.#pushApplySerialized,
|
|
1627
|
+
this.scanVariables,
|
|
1613
1628
|
);
|
|
1614
1629
|
}
|
|
1615
1630
|
|
package/src/postgres-storage.ts
CHANGED
|
@@ -94,6 +94,7 @@ import {
|
|
|
94
94
|
SYNC_VERSION_COLUMN,
|
|
95
95
|
type StoredColumnLayout,
|
|
96
96
|
scanRowBatchSql,
|
|
97
|
+
scanProbeSql,
|
|
97
98
|
scanRowPageSql,
|
|
98
99
|
indexSignature,
|
|
99
100
|
schemaDdl,
|
|
@@ -161,7 +162,7 @@ import {
|
|
|
161
162
|
StorageConstraintError,
|
|
162
163
|
} from './storage-errors';
|
|
163
164
|
import {
|
|
164
|
-
|
|
165
|
+
chooseScanVariable,
|
|
165
166
|
readLiveWindowPages,
|
|
166
167
|
resolveIndexRowScan,
|
|
167
168
|
} from './storage-query';
|
|
@@ -1270,6 +1271,8 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1270
1271
|
/** Schema version this transaction writes; rides every appended commit row. */
|
|
1271
1272
|
#writerVersion: number;
|
|
1272
1273
|
#resolveTable: (name: string) => CompiledTable;
|
|
1274
|
+
/** The storage's `chooseScanVariable` picks. */
|
|
1275
|
+
readonly #scanVariables: Map<string, string>;
|
|
1273
1276
|
#open = true;
|
|
1274
1277
|
#pushApplySavepoint = false;
|
|
1275
1278
|
/**
|
|
@@ -1324,7 +1327,9 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1324
1327
|
prepareQuery: (query: AuthoritativeQueryRequest) => BoundAuthoritativeQuery,
|
|
1325
1328
|
resolve: () => void,
|
|
1326
1329
|
reject: (error: unknown) => void,
|
|
1330
|
+
scanVariables: Map<string, string>,
|
|
1327
1331
|
) {
|
|
1332
|
+
this.#scanVariables = scanVariables;
|
|
1328
1333
|
// One pinned connection carries one statement at a time: node-postgres
|
|
1329
1334
|
// warns about a concurrently issued second query today and rejects it in
|
|
1330
1335
|
// node-postgres 9. A whole-commit validator that fans independent reads
|
|
@@ -1546,12 +1551,23 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1546
1551
|
|
|
1547
1552
|
async scanRows(query: RowScanQuery): Promise<StoredRow[]> {
|
|
1548
1553
|
this.#assertOpen();
|
|
1549
|
-
const
|
|
1550
|
-
|
|
1551
|
-
|
|
1554
|
+
const variable = await chooseScanVariable(
|
|
1555
|
+
this.#partition,
|
|
1556
|
+
query,
|
|
1557
|
+
this.#scanVariables,
|
|
1558
|
+
(probes, cap) =>
|
|
1559
|
+
this.#client
|
|
1560
|
+
.query<{ sub: unknown; n: unknown }>(scanProbeSql('postgres'), [
|
|
1561
|
+
this.#partition,
|
|
1562
|
+
JSON.stringify(probes.map((probe) => ({ ...probe, lim: cap }))),
|
|
1563
|
+
])
|
|
1564
|
+
.then(({ rows }) => rows),
|
|
1565
|
+
);
|
|
1566
|
+
if (variable === undefined) return [];
|
|
1567
|
+
const values = query.scopeFilter[variable]!;
|
|
1552
1568
|
const sql = scanRowPageSql(
|
|
1553
1569
|
this.#resolveTable(query.table),
|
|
1554
|
-
|
|
1570
|
+
values.length,
|
|
1555
1571
|
'postgres',
|
|
1556
1572
|
);
|
|
1557
1573
|
const rows: StoredRow[] = [];
|
|
@@ -1561,8 +1577,8 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1561
1577
|
const { rows: records } = await this.#client.query<RowRecord>(sql, [
|
|
1562
1578
|
this.#partition,
|
|
1563
1579
|
query.table,
|
|
1564
|
-
|
|
1565
|
-
...
|
|
1580
|
+
variable,
|
|
1581
|
+
...values,
|
|
1566
1582
|
afterRowId,
|
|
1567
1583
|
batchSize,
|
|
1568
1584
|
]);
|
|
@@ -2170,6 +2186,8 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
2170
2186
|
* (SYNCULAR-PULL-ROUNDTRIPS-001).
|
|
2171
2187
|
*/
|
|
2172
2188
|
readonly #pageBatches = new Map<string, PendingPage[]>();
|
|
2189
|
+
/** `chooseScanVariable` picks, shared with this storage's transactions. */
|
|
2190
|
+
readonly #scanVariables = new Map<string, string>();
|
|
2173
2191
|
|
|
2174
2192
|
constructor(exec: PgExecutor) {
|
|
2175
2193
|
this.#exec = exec;
|
|
@@ -2720,6 +2738,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2720
2738
|
(query) => this.#prepareAuthoritativeQuery(partition, query),
|
|
2721
2739
|
resolveScope,
|
|
2722
2740
|
rejectScope,
|
|
2741
|
+
this.#scanVariables,
|
|
2723
2742
|
);
|
|
2724
2743
|
opened = true;
|
|
2725
2744
|
resolveReady(tx);
|
|
@@ -3388,9 +3407,25 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3388
3407
|
}
|
|
3389
3408
|
|
|
3390
3409
|
async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
|
|
3391
|
-
|
|
3392
|
-
const
|
|
3393
|
-
|
|
3410
|
+
// The probes of concurrent scans leave as one statement, like their pages.
|
|
3411
|
+
const variable = await chooseScanVariable(
|
|
3412
|
+
partition,
|
|
3413
|
+
query,
|
|
3414
|
+
this.#scanVariables,
|
|
3415
|
+
(probes, cap) =>
|
|
3416
|
+
Promise.all(
|
|
3417
|
+
probes.map(async (probe) => {
|
|
3418
|
+
const [row] = await this.#batchedPage<{ n: unknown }>(
|
|
3419
|
+
'scanProbe',
|
|
3420
|
+
scanProbeSql('postgres'),
|
|
3421
|
+
partition,
|
|
3422
|
+
{ tbl: probe.tbl, var: probe.var, vals: probe.vals, lim: cap },
|
|
3423
|
+
);
|
|
3424
|
+
return { sub: probe.i, n: row?.n };
|
|
3425
|
+
}),
|
|
3426
|
+
),
|
|
3427
|
+
);
|
|
3428
|
+
if (variable === undefined) return [];
|
|
3394
3429
|
// Candidates via the inverted index (ordered + LIMITed at the covering
|
|
3395
3430
|
// PK) LEFT JOINed to the row table — one round trip per page, never one
|
|
3396
3431
|
// per row, and concurrent scans across tables share one statement (see
|
|
@@ -3411,8 +3446,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
3411
3446
|
partition,
|
|
3412
3447
|
{
|
|
3413
3448
|
tbl: query.table,
|
|
3414
|
-
var:
|
|
3415
|
-
vals:
|
|
3449
|
+
var: variable,
|
|
3450
|
+
vals: query.scopeFilter[variable],
|
|
3416
3451
|
after: afterRowId,
|
|
3417
3452
|
lim: batchSize,
|
|
3418
3453
|
},
|
package/src/relational-rows.ts
CHANGED
|
@@ -716,6 +716,29 @@ export function scanRowPageSql(
|
|
|
716
716
|
ORDER BY c.row_id`;
|
|
717
717
|
}
|
|
718
718
|
|
|
719
|
+
/**
|
|
720
|
+
* Selectivity probe for `chooseScanVariable`: per `{i, tbl, var, vals}`
|
|
721
|
+
* record of the JSON array, the count of its `sync_row_scopes` entries up
|
|
722
|
+
* to a cap, read from the covering primary key. Rows come back as
|
|
723
|
+
* `(sub, n)`, `sub` being the record's `i`.
|
|
724
|
+
*
|
|
725
|
+
* Bind order:
|
|
726
|
+
* - postgres: [partition, records as JSON text]; each record carries its
|
|
727
|
+
* own cap as `lim`, so probes of concurrent scans share one statement;
|
|
728
|
+
* - sqlite: [partition, records as JSON text, cap]. SQLite rejects a
|
|
729
|
+
* correlated LIMIT, so one cap binds for every record.
|
|
730
|
+
*/
|
|
731
|
+
export function scanProbeSql(dialect: RelationalDialect): string {
|
|
732
|
+
return dialect === 'sqlite'
|
|
733
|
+
? `SELECT json_extract(p.value, '$.i') AS sub, (SELECT count(*) FROM (SELECT 1 FROM sync_row_scopes
|
|
734
|
+
WHERE partition=?1 AND tbl=json_extract(p.value, '$.tbl') AND var=json_extract(p.value, '$.var')
|
|
735
|
+
AND value IN (SELECT value FROM json_each(p.value, '$.vals')) LIMIT ?3)) AS n
|
|
736
|
+
FROM json_each(?2) AS p`
|
|
737
|
+
: `SELECT p.i AS sub, (SELECT count(*) FROM (SELECT 1 FROM sync_row_scopes
|
|
738
|
+
WHERE partition=$1 AND tbl=p.tbl AND var=p.var AND value=ANY(p.vals) LIMIT p.lim) x) AS n
|
|
739
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS p(i int, tbl text, var text, vals text[], lim int)`;
|
|
740
|
+
}
|
|
741
|
+
|
|
719
742
|
/**
|
|
720
743
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
721
744
|
* page of every subscription scanning any table in a pull. `$2` is a JSON
|
package/src/sqlite-dialect.ts
CHANGED
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
deleteSqliteRowScopesSql,
|
|
29
29
|
indexRowPageStatement,
|
|
30
30
|
quoteIdent,
|
|
31
|
+
scanProbeSql,
|
|
31
32
|
scanRowPageSql,
|
|
32
33
|
rowsReferencingBlobSql,
|
|
33
34
|
selectRowSql,
|
|
@@ -91,7 +92,7 @@ import type {
|
|
|
91
92
|
Tombstone,
|
|
92
93
|
} from './storage';
|
|
93
94
|
import {
|
|
94
|
-
|
|
95
|
+
chooseScanVariable,
|
|
95
96
|
readLiveWindowPages,
|
|
96
97
|
resolveIndexRowScan,
|
|
97
98
|
} from './storage-query';
|
|
@@ -670,6 +671,8 @@ export abstract class SqliteFamilyStorage {
|
|
|
670
671
|
protected schemaVersion: number | undefined;
|
|
671
672
|
/** Bound parameters one statement accepts. */
|
|
672
673
|
protected abstract readonly maxBindParams: number;
|
|
674
|
+
/** `chooseScanVariable` picks, shared with this storage's transactions. */
|
|
675
|
+
protected readonly scanVariables = new Map<string, string>();
|
|
673
676
|
|
|
674
677
|
/** Run one statement and return its rows. */
|
|
675
678
|
protected abstract all<T>(
|
|
@@ -1270,15 +1273,25 @@ export abstract class SqliteFamilyStorage {
|
|
|
1270
1273
|
}
|
|
1271
1274
|
|
|
1272
1275
|
async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
|
|
1273
|
-
const
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
+
const variable = await chooseScanVariable(
|
|
1277
|
+
partition,
|
|
1278
|
+
query,
|
|
1279
|
+
this.scanVariables,
|
|
1280
|
+
(probes, cap) =>
|
|
1281
|
+
this.guardedAll<{ sub: number; n: number }>(scanProbeSql('sqlite'), [
|
|
1282
|
+
partition,
|
|
1283
|
+
JSON.stringify(probes),
|
|
1284
|
+
cap,
|
|
1285
|
+
]),
|
|
1286
|
+
);
|
|
1287
|
+
if (variable === undefined) return [];
|
|
1288
|
+
const values = query.scopeFilter[variable]!;
|
|
1276
1289
|
// Candidates via the inverted index LEFT JOINed to the row table — one
|
|
1277
1290
|
// statement per page, never one per row (see `scanRowPageSql`). Exact
|
|
1278
1291
|
// multi-variable verification against the stored scope map below.
|
|
1279
1292
|
const sql = scanRowPageSql(
|
|
1280
1293
|
this.table(query.table),
|
|
1281
|
-
|
|
1294
|
+
values.length,
|
|
1282
1295
|
'sqlite',
|
|
1283
1296
|
);
|
|
1284
1297
|
const rows: StoredRow[] = [];
|
|
@@ -1290,8 +1303,8 @@ export abstract class SqliteFamilyStorage {
|
|
|
1290
1303
|
>(sql, [
|
|
1291
1304
|
partition,
|
|
1292
1305
|
query.table,
|
|
1293
|
-
|
|
1294
|
-
JSON.stringify(
|
|
1306
|
+
variable,
|
|
1307
|
+
JSON.stringify(values),
|
|
1295
1308
|
afterRowId,
|
|
1296
1309
|
batchSize,
|
|
1297
1310
|
partition,
|
package/src/storage-query.ts
CHANGED
|
@@ -15,22 +15,88 @@ import type {
|
|
|
15
15
|
import { StorageQueryError } from './storage-errors';
|
|
16
16
|
|
|
17
17
|
/**
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
* drives its inverted-index candidate selection with.
|
|
18
|
+
* One variable of a scan filter whose index entries a probe counts; `i` is
|
|
19
|
+
* the row's `sub` in the probe result (see `scanProbeSql`).
|
|
21
20
|
*/
|
|
22
|
-
export
|
|
21
|
+
export interface ScanProbe {
|
|
22
|
+
readonly i: number;
|
|
23
|
+
readonly tbl: string;
|
|
24
|
+
readonly var: string;
|
|
25
|
+
readonly vals: readonly string[];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* The scope variable a `scanRows` call seeks the row-scope index with: the
|
|
30
|
+
* one whose values have the fewest index entries, ties broken alphabetically.
|
|
31
|
+
* `undefined` means the intersection is empty: a variable without values or
|
|
32
|
+
* without entries, so the scan returns no persisted row. An empty or omitted
|
|
33
|
+
* filter fails loudly instead of making an unsupported unscoped scan look
|
|
34
|
+
* empty.
|
|
35
|
+
*
|
|
36
|
+
* A single-variable filter needs no probe. Otherwise `count` counts each
|
|
37
|
+
* variable's entries in one statement, up to `cap`. The cap starts at
|
|
38
|
+
* max(64, limit) and grows fourfold, to at most 65,536, while the fewest
|
|
39
|
+
* count is shared by several variables at the cap.
|
|
40
|
+
*
|
|
41
|
+
* `chosen` memoizes the pick per (partition, table, filter), at most 1,024
|
|
42
|
+
* entries, so the page loop of a bootstrap probes once. A stale pick only
|
|
43
|
+
* costs index reads: every scan still verifies all variables against the
|
|
44
|
+
* stored scopes, so the choice never changes a result.
|
|
45
|
+
*/
|
|
46
|
+
export async function chooseScanVariable(
|
|
47
|
+
partition: string,
|
|
48
|
+
query: RowScanQuery,
|
|
49
|
+
chosen: Map<string, string>,
|
|
50
|
+
count: (
|
|
51
|
+
probes: readonly ScanProbe[],
|
|
52
|
+
cap: number,
|
|
53
|
+
) => Promise<readonly { readonly sub: unknown; readonly n: unknown }[]>,
|
|
54
|
+
): Promise<string | undefined> {
|
|
23
55
|
const scopeFilter = (
|
|
24
56
|
query as RowScanQuery & { readonly scopeFilter?: ScopeMap | null }
|
|
25
57
|
).scopeFilter;
|
|
26
|
-
const
|
|
58
|
+
const variables =
|
|
27
59
|
scopeFilter === undefined || scopeFilter === null
|
|
28
|
-
?
|
|
29
|
-
: Object.keys(scopeFilter).sort()
|
|
30
|
-
if (
|
|
60
|
+
? []
|
|
61
|
+
: Object.keys(scopeFilter).sort();
|
|
62
|
+
if (
|
|
63
|
+
scopeFilter === undefined ||
|
|
64
|
+
scopeFilter === null ||
|
|
65
|
+
variables[0] === undefined
|
|
66
|
+
) {
|
|
31
67
|
throw new StorageQueryError('sync.storage.scan_requires_scope');
|
|
32
68
|
}
|
|
33
|
-
|
|
69
|
+
if (variables.some((variable) => scopeFilter[variable]?.length === 0)) {
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
if (variables.length === 1) return variables[0];
|
|
73
|
+
const key = JSON.stringify([
|
|
74
|
+
partition,
|
|
75
|
+
query.table,
|
|
76
|
+
variables.map((variable) => [variable, [...scopeFilter[variable]!].sort()]),
|
|
77
|
+
]);
|
|
78
|
+
const cached = chosen.get(key);
|
|
79
|
+
if (cached !== undefined) return cached;
|
|
80
|
+
let tied = variables;
|
|
81
|
+
for (let cap = Math.max(64, query.limit); ; cap = Math.min(cap * 4, 65_536)) {
|
|
82
|
+
const counts = tied.map(() => 0);
|
|
83
|
+
const probes = tied.map((variable, i) => ({
|
|
84
|
+
i,
|
|
85
|
+
tbl: query.table,
|
|
86
|
+
var: variable,
|
|
87
|
+
vals: scopeFilter[variable]!,
|
|
88
|
+
}));
|
|
89
|
+
for (const row of await count(probes, cap)) {
|
|
90
|
+
counts[Number(row.sub)] = Number(row.n);
|
|
91
|
+
}
|
|
92
|
+
const fewest = Math.min(...counts);
|
|
93
|
+
if (fewest === 0) return undefined;
|
|
94
|
+
tied = tied.filter((_, i) => counts[i] === fewest);
|
|
95
|
+
if (fewest < cap || tied.length === 1 || cap >= 65_536) break;
|
|
96
|
+
}
|
|
97
|
+
if (chosen.size >= 1_024) chosen.delete(chosen.keys().next().value!);
|
|
98
|
+
chosen.set(key, tied[0]!);
|
|
99
|
+
return tied[0];
|
|
34
100
|
}
|
|
35
101
|
|
|
36
102
|
/** Validate and resolve one exact trusted-host relational index lookup. */
|