@spooky-sync/core 0.0.1-canary.133 → 0.0.1-canary.134
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +115 -190
- package/dist/sqlite-plan-sql.js +194 -0
- package/dist/sqlite-worker.js +97 -4
- package/dist/types.d.ts +7 -0
- package/package.json +3 -3
- package/src/services/database/plan-render.test.ts +26 -0
- package/src/services/database/plan-render.ts +9 -1
- package/src/services/database/sqlite-cache-engine.test.ts +3 -0
- package/src/services/database/sqlite-cache-engine.ts +163 -95
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +106 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +279 -0
- package/src/services/database/sqlite-select.ts +113 -0
- package/src/services/database/sqlite-worker.ts +36 -5
- package/src/types.ts +7 -0
package/dist/sqlite-worker.js
CHANGED
|
@@ -1,5 +1,77 @@
|
|
|
1
|
+
import { i as reviveRow, n as renderOrderSql, o as resolveRelations, r as renderWhereSql, s as stableKey, t as project } from "./sqlite-plan-sql.js";
|
|
1
2
|
import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
|
|
2
3
|
|
|
4
|
+
//#region src/services/database/sqlite-select.ts
|
|
5
|
+
function ensureTable(db, table) {
|
|
6
|
+
if (db.knownTables.has(table)) return;
|
|
7
|
+
db.run(`CREATE TABLE IF NOT EXISTS "${table}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)`);
|
|
8
|
+
db.knownTables.add(table);
|
|
9
|
+
}
|
|
10
|
+
function execRows(db, sql, bind) {
|
|
11
|
+
return db.exec(sql, bind).map((r) => reviveRow(r.data));
|
|
12
|
+
}
|
|
13
|
+
/** Mirrors the engine's `selectByIds`: fetch by primary id, preserving `ids`
|
|
14
|
+
* order unless an ORDER BY overrides it. */
|
|
15
|
+
function selectByIds(db, table, ids, opts) {
|
|
16
|
+
if (ids.length === 0) return [];
|
|
17
|
+
ensureTable(db, table);
|
|
18
|
+
const keys = ids.map(stableKey);
|
|
19
|
+
let sql = `SELECT data FROM "${table}" WHERE id IN (${keys.map(() => "?").join(", ")})`;
|
|
20
|
+
if (opts?.orderBy && opts.orderBy.length > 0) sql += renderOrderSql(opts.orderBy);
|
|
21
|
+
let rows = execRows(db, sql, keys);
|
|
22
|
+
if (!opts?.orderBy || opts.orderBy.length === 0) {
|
|
23
|
+
const pos = new Map(keys.map((k, i) => [k, i]));
|
|
24
|
+
rows = rows.sort((a, b) => (pos.get(stableKey(a.id)) ?? 0) - (pos.get(stableKey(b.id)) ?? 0));
|
|
25
|
+
}
|
|
26
|
+
return opts?.select ? rows.map((r) => project(r, opts.select)) : rows;
|
|
27
|
+
}
|
|
28
|
+
/** Mirrors the engine's `fetchRelation` SQL exactly. */
|
|
29
|
+
function fetchRelation(db, req) {
|
|
30
|
+
ensureTable(db, req.table);
|
|
31
|
+
const keys = req.keys.map(stableKey);
|
|
32
|
+
const placeholders = keys.map(() => "?").join(", ");
|
|
33
|
+
const bind = [...keys];
|
|
34
|
+
const lhs = req.matchField === "id" ? "id" : `json_extract(data, '$.${req.matchField}')`;
|
|
35
|
+
let sql = `SELECT data FROM "${req.table}" WHERE ${lhs} IN (${placeholders})`;
|
|
36
|
+
if (req.where && req.where.length > 0) sql += ` AND ${renderWhereSql(req.where, bind, {})}`;
|
|
37
|
+
if (req.orderBy && req.orderBy.length > 0) sql += renderOrderSql(req.orderBy);
|
|
38
|
+
const rows = execRows(db, sql, bind);
|
|
39
|
+
return req.select ? rows.map((r) => project(r, req.select)) : rows;
|
|
40
|
+
}
|
|
41
|
+
async function executeSelect(plan, params, db) {
|
|
42
|
+
const counter = { n: 0 };
|
|
43
|
+
const fetcher = { fetchRelation: (req) => {
|
|
44
|
+
counter.n++;
|
|
45
|
+
return Promise.resolve(fetchRelation(db, req));
|
|
46
|
+
} };
|
|
47
|
+
if (plan.ids) {
|
|
48
|
+
const rows = selectByIds(db, plan.table, plan.ids, {
|
|
49
|
+
select: plan.select,
|
|
50
|
+
orderBy: plan.orderBy
|
|
51
|
+
});
|
|
52
|
+
await resolveRelations(rows, plan.relations, fetcher);
|
|
53
|
+
return {
|
|
54
|
+
rows,
|
|
55
|
+
relationFetches: counter.n
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
ensureTable(db, plan.table);
|
|
59
|
+
const bind = [];
|
|
60
|
+
let sql = `SELECT data FROM "${plan.table}"`;
|
|
61
|
+
if (plan.where && plan.where.length > 0) sql += ` WHERE ${renderWhereSql(plan.where, bind, params)}`;
|
|
62
|
+
if (plan.orderBy && plan.orderBy.length > 0) sql += renderOrderSql(plan.orderBy);
|
|
63
|
+
if (plan.limit !== void 0) sql += ` LIMIT ${Number(plan.limit)}`;
|
|
64
|
+
if (plan.offset !== void 0) sql += ` OFFSET ${Number(plan.offset)}`;
|
|
65
|
+
const rows = execRows(db, sql, bind);
|
|
66
|
+
const projected = plan.select ? rows.map((r) => project(r, plan.select)) : rows;
|
|
67
|
+
await resolveRelations(projected, plan.relations, fetcher);
|
|
68
|
+
return {
|
|
69
|
+
rows: projected,
|
|
70
|
+
relationFetches: counter.n
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
//#endregion
|
|
3
75
|
//#region src/services/database/sqlite-worker.ts
|
|
4
76
|
/**
|
|
5
77
|
* Dedicated Web Worker that owns the SQLite-WASM handle. All DB access is
|
|
@@ -11,11 +83,17 @@ import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
|
|
|
11
83
|
* to an in-memory DB when OPFS is unavailable.
|
|
12
84
|
*
|
|
13
85
|
* Message protocol (request/response keyed by `id`):
|
|
14
|
-
* { id, type: 'open',
|
|
15
|
-
* { id, type: 'exec',
|
|
16
|
-
* { id, type: 'run',
|
|
17
|
-
* { id, type: 'batch',
|
|
86
|
+
* { id, type: 'open', payload: { dbName, useOpfs } }
|
|
87
|
+
* { id, type: 'exec', payload: { sql, bind } } -> { id, ok, rows }
|
|
88
|
+
* { id, type: 'run', payload: { sql, bind } } -> { id, ok }
|
|
89
|
+
* { id, type: 'batch', payload: [{ sql, bind }] } (atomic BEGIN/COMMIT)
|
|
90
|
+
* { id, type: 'select', payload: { plan, params } } -> { id, ok, rows, relationFetches }
|
|
18
91
|
* { id, type: 'close' }
|
|
92
|
+
*
|
|
93
|
+
* `select` executes a whole QueryPlan — table creation, base select, the full
|
|
94
|
+
* `.related()` tree (shared `resolveRelations`), and JSON row parsing — in ONE
|
|
95
|
+
* round-trip, returning structured-clone row objects. This is the first-load
|
|
96
|
+
* hot path; the per-statement ops remain for the write/shim paths.
|
|
19
97
|
*/
|
|
20
98
|
let db = null;
|
|
21
99
|
async function open(dbName, useOpfs) {
|
|
@@ -63,14 +141,27 @@ function batch(stmts) {
|
|
|
63
141
|
throw e;
|
|
64
142
|
}
|
|
65
143
|
}
|
|
144
|
+
/** DB handle for `executeSelect` (see `sqlite-select.ts`, the unit-testable
|
|
145
|
+
* plan executor). `knownTables` is cleared on open/close — a bucket switch
|
|
146
|
+
* spawns a fresh worker anyway. */
|
|
147
|
+
const selectDb = {
|
|
148
|
+
exec: (sql, bind) => exec(sql, bind),
|
|
149
|
+
run,
|
|
150
|
+
knownTables: /* @__PURE__ */ new Set()
|
|
151
|
+
};
|
|
66
152
|
self.onmessage = async (ev) => {
|
|
67
153
|
const { id, type, payload } = ev.data ?? {};
|
|
154
|
+
const t0 = performance.now();
|
|
68
155
|
try {
|
|
69
156
|
let result;
|
|
70
157
|
switch (type) {
|
|
71
158
|
case "open":
|
|
159
|
+
selectDb.knownTables.clear();
|
|
72
160
|
result = await open(payload.dbName, payload.useOpfs);
|
|
73
161
|
break;
|
|
162
|
+
case "select":
|
|
163
|
+
result = await executeSelect(payload.plan, payload.params ?? {}, selectDb);
|
|
164
|
+
break;
|
|
74
165
|
case "exec":
|
|
75
166
|
result = { rows: exec(payload.sql, payload.bind) };
|
|
76
167
|
break;
|
|
@@ -85,6 +176,7 @@ self.onmessage = async (ev) => {
|
|
|
85
176
|
case "close":
|
|
86
177
|
db?.close();
|
|
87
178
|
db = null;
|
|
179
|
+
selectDb.knownTables.clear();
|
|
88
180
|
result = {};
|
|
89
181
|
break;
|
|
90
182
|
default: throw new Error(`sqlite worker: unknown message ${type}`);
|
|
@@ -92,6 +184,7 @@ self.onmessage = async (ev) => {
|
|
|
92
184
|
self.postMessage({
|
|
93
185
|
id,
|
|
94
186
|
ok: true,
|
|
187
|
+
wt: performance.now() - t0,
|
|
95
188
|
...result
|
|
96
189
|
});
|
|
97
190
|
} catch (err) {
|
package/dist/types.d.ts
CHANGED
|
@@ -404,6 +404,13 @@ interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
404
404
|
store?: StoreType;
|
|
405
405
|
/** Authentication token. */
|
|
406
406
|
token?: string;
|
|
407
|
+
/**
|
|
408
|
+
* SQLite engine only: execute `select` plans (base rows + relation tree +
|
|
409
|
+
* row parsing) inside the worker as ONE round-trip instead of one hop per
|
|
410
|
+
* table/relation level. Defaults to true; set false to force the legacy
|
|
411
|
+
* multi-hop path (escape hatch while the worker-side path beds in).
|
|
412
|
+
*/
|
|
413
|
+
workerSelect?: boolean;
|
|
407
414
|
};
|
|
408
415
|
/** The schema definition. */
|
|
409
416
|
schema: S;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.134",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"main": "./dist/index.js",
|
|
6
6
|
"types": "./dist/index.d.ts",
|
|
@@ -59,8 +59,8 @@
|
|
|
59
59
|
}
|
|
60
60
|
},
|
|
61
61
|
"dependencies": {
|
|
62
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
63
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
62
|
+
"@spooky-sync/query-builder": "0.0.1-canary.134",
|
|
63
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.134",
|
|
64
64
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
65
65
|
"@surrealdb/wasm": "^3.0.3",
|
|
66
66
|
"fast-json-patch": "^3.1.1",
|
|
@@ -73,6 +73,32 @@ describe('record-id coercion (authors/comments loading regression)', () => {
|
|
|
73
73
|
expect(sql).toBe('SELECT * FROM thread WHERE title = $__p0 AND published = $__p1;');
|
|
74
74
|
});
|
|
75
75
|
|
|
76
|
+
// Regression guard for the thread-detail "crossed results → 404" bug: a
|
|
77
|
+
// slave-mode node carries BOTH a baked `value` and a `paramRef` (= the field
|
|
78
|
+
// name). Materialization MUST bind `params[paramRef]` (the query's identity),
|
|
79
|
+
// NOT the baked value — otherwise a stale/other plan whose baked id ≠ the
|
|
80
|
+
// query's params would surface a different record's row. So rendering a plan
|
|
81
|
+
// baked to id-A with params for id-B must filter by B.
|
|
82
|
+
it('slave-mode node binds params over the baked value (records follow identity)', () => {
|
|
83
|
+
const planBakedToA: QueryPlan = {
|
|
84
|
+
table: 'thread',
|
|
85
|
+
where: [{ field: 'id', op: '=', value: 'thread:A', paramRef: 'id' }],
|
|
86
|
+
};
|
|
87
|
+
const { sql, vars } = renderBaseSelectSurql(planBakedToA, { id: 'thread:B' });
|
|
88
|
+
expect(sql).toBe('SELECT * FROM thread WHERE id = type::record(<string> $id);');
|
|
89
|
+
expect(vars).toEqual({ id: 'thread:B' }); // B wins — the baked A is ignored
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
it('slave-mode node falls back to the baked value when the param is absent', () => {
|
|
93
|
+
const plan: QueryPlan = {
|
|
94
|
+
table: 'thread',
|
|
95
|
+
where: [{ field: 'id', op: '=', value: 'thread:A', paramRef: 'id' }],
|
|
96
|
+
};
|
|
97
|
+
const { sql, vars } = renderBaseSelectSurql(plan, {}); // no params.id
|
|
98
|
+
expect(sql).toBe('SELECT * FROM thread WHERE id = type::record(<string> $__p0);');
|
|
99
|
+
expect(vars).toEqual({ __p0: 'thread:A' });
|
|
100
|
+
});
|
|
101
|
+
|
|
76
102
|
it('coerces `id` inside an OR group too', () => {
|
|
77
103
|
const plan: QueryPlan = {
|
|
78
104
|
table: 'thread',
|
|
@@ -26,7 +26,15 @@ function bind(ctx: RenderCtx, value: unknown): string {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
function renderComparisonSurql(c: WhereComparison, ctx: RenderCtx): string {
|
|
29
|
-
|
|
29
|
+
// Prefer the query's own param (`paramRef`) so a filter materializes from
|
|
30
|
+
// `params` (the query's identity), not a baked literal — this is what slaves
|
|
31
|
+
// a query's rows to its id. A pure `$`-ref node has no `value`, so it always
|
|
32
|
+
// uses the param. A slave-mode node also carries `value` as a fallback for
|
|
33
|
+
// when the param is absent (e.g. a non-column field stripped by parseParams).
|
|
34
|
+
const useParam =
|
|
35
|
+
c.paramRef !== undefined &&
|
|
36
|
+
(c.value === undefined || Object.prototype.hasOwnProperty.call(ctx.vars, c.paramRef));
|
|
37
|
+
const rawRight = useParam ? `$${c.paramRef}` : bind(ctx, c.value);
|
|
30
38
|
// `id` is a RecordId, but correlation/filter values arrive as record-id
|
|
31
39
|
// STRINGS (e.g. "thread:abc"). On SurrealDB `id = "thread:abc"` never matches
|
|
32
40
|
// (string ≠ record), so a base select filtered by id (e.g. the ThreadDetail
|
|
@@ -39,6 +39,9 @@ class FakeWorker {
|
|
|
39
39
|
} else if (msg.type === 'exec') {
|
|
40
40
|
if (!this.dbOpen) { ok = false; error = 'sqlite: DB not open'; }
|
|
41
41
|
else rest = { rows: [] };
|
|
42
|
+
} else if (msg.type === 'select') {
|
|
43
|
+
if (!this.dbOpen) { ok = false; error = 'sqlite: DB not open'; }
|
|
44
|
+
else rest = { rows: [], relationFetches: 0 };
|
|
42
45
|
} else if (msg.type === 'run' || msg.type === 'batch') {
|
|
43
46
|
if (!this.dbOpen) { ok = false; error = 'sqlite: DB not open'; }
|
|
44
47
|
}
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
import { applyPatch, type Operation } from 'fast-json-patch';
|
|
2
|
-
import type { QueryPlan,
|
|
2
|
+
import type { QueryPlan, RelationPlan, WhereNode } from '@spooky-sync/query-builder';
|
|
3
|
+
import {
|
|
4
|
+
renderOrderSql,
|
|
5
|
+
renderWhereSql,
|
|
6
|
+
reviveRow,
|
|
7
|
+
serializeRow,
|
|
8
|
+
project,
|
|
9
|
+
} from './sqlite-plan-sql';
|
|
3
10
|
import type { Logger } from '../logger/index';
|
|
4
11
|
import type { Sp00kyConfig } from '../../types';
|
|
5
12
|
import type { SealedQuery } from '../../utils/surql';
|
|
@@ -60,6 +67,10 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
60
67
|
private storeEpoch = 0;
|
|
61
68
|
private knownTables = new Set<string>();
|
|
62
69
|
private useOpfs: boolean;
|
|
70
|
+
/** Whether `select` runs as one worker round-trip (plan executed in-worker).
|
|
71
|
+
* Flipped off at runtime if the worker script predates the `select` op
|
|
72
|
+
* (stale cached bundle) — degrade to the legacy multi-hop path, don't break. */
|
|
73
|
+
private workerSelect: boolean;
|
|
63
74
|
private events: DatabaseEventSystem = createDatabaseEventSystem();
|
|
64
75
|
private bucketId = 'anon';
|
|
65
76
|
/** Schemaless — tables are created lazily on first write; no migrator. */
|
|
@@ -68,9 +79,10 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
68
79
|
constructor(
|
|
69
80
|
private config: Sp00kyConfig<any>['database'],
|
|
70
81
|
private logger: Logger,
|
|
71
|
-
opts: { useOpfs?: boolean } = {}
|
|
82
|
+
opts: { useOpfs?: boolean; workerSelect?: boolean } = {}
|
|
72
83
|
) {
|
|
73
84
|
this.useOpfs = opts.useOpfs ?? true;
|
|
85
|
+
this.workerSelect = opts.workerSelect ?? config.workerSelect ?? true;
|
|
74
86
|
}
|
|
75
87
|
|
|
76
88
|
get epoch(): number {
|
|
@@ -144,7 +156,12 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
144
156
|
private opQueue: Promise<unknown> = Promise.resolve();
|
|
145
157
|
|
|
146
158
|
private call<T = any>(type: string, payload?: unknown): Promise<T> {
|
|
147
|
-
const
|
|
159
|
+
const enqueuedAt = performance.now();
|
|
160
|
+
const run = () => {
|
|
161
|
+
// Time spent waiting behind other ops in the queue, not doing work.
|
|
162
|
+
getStats().queueWaitMs += performance.now() - enqueuedAt;
|
|
163
|
+
return this.rawCall<T>(type, payload);
|
|
164
|
+
};
|
|
148
165
|
const result = this.opQueue.then(run, run);
|
|
149
166
|
// Keep the chain alive regardless of individual failures.
|
|
150
167
|
this.opQueue = result.then(
|
|
@@ -170,10 +187,18 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
170
187
|
const done = () => {
|
|
171
188
|
s.inFlight--;
|
|
172
189
|
};
|
|
190
|
+
const sentAt = performance.now();
|
|
173
191
|
return new Promise<T>((resolve, reject) => {
|
|
174
192
|
this.pending.set(id, {
|
|
175
193
|
resolve: (v: T) => {
|
|
176
194
|
done();
|
|
195
|
+
// Split the round-trip: `wt` is time inside the worker's handler,
|
|
196
|
+
// the remainder is postMessage + scheduling overhead.
|
|
197
|
+
const wt = (v as { wt?: unknown } | null)?.wt;
|
|
198
|
+
if (typeof wt === 'number') {
|
|
199
|
+
s.workerMs += wt;
|
|
200
|
+
s.rpcOverheadMs += Math.max(0, performance.now() - sentAt - wt);
|
|
201
|
+
}
|
|
177
202
|
resolve(v);
|
|
178
203
|
},
|
|
179
204
|
reject: (e: unknown) => {
|
|
@@ -264,12 +289,68 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
264
289
|
|
|
265
290
|
private async execRows(sql: string, bind: unknown[]): Promise<Row[]> {
|
|
266
291
|
const { rows } = await this.call<{ rows: { data: string }[] }>('exec', { sql, bind });
|
|
267
|
-
|
|
292
|
+
const s = getStats();
|
|
293
|
+
const t0 = performance.now();
|
|
294
|
+
const out = (rows ?? []).map((r) => {
|
|
295
|
+
s.bytesParsed += r.data.length;
|
|
296
|
+
return reviveRow(r.data);
|
|
297
|
+
});
|
|
298
|
+
s.parseMs += performance.now() - t0;
|
|
299
|
+
s.rowsParsed += out.length;
|
|
300
|
+
return out;
|
|
268
301
|
}
|
|
269
302
|
|
|
270
303
|
// ---- reads ---------------------------------------------------------------
|
|
271
304
|
|
|
272
305
|
async select(plan: QueryPlan, params: Record<string, unknown> = {}): Promise<Row[]> {
|
|
306
|
+
if (this.workerSelect) {
|
|
307
|
+
// ONE round-trip: the worker executes the whole plan (table creation,
|
|
308
|
+
// base select, relation tree, JSON parse) and returns structured-clone
|
|
309
|
+
// rows. The legacy path below pays a postMessage hop per table/relation
|
|
310
|
+
// level plus main-thread parsing — the dominant first-load cost.
|
|
311
|
+
//
|
|
312
|
+
// Normalize before postMessage: class-instance VALUES (RecordId & co) →
|
|
313
|
+
// their `stableKey` string. structuredClone strips a class instance to a
|
|
314
|
+
// bare plain object — and surrealdb's RecordId keeps its data behind
|
|
315
|
+
// getters (no own properties), so it clones to `{}`: the worker would
|
|
316
|
+
// filter on garbage. Applies to params AND to values baked inside the
|
|
317
|
+
// plan (where nodes, relation sub-wheres, window ids).
|
|
318
|
+
// Param KEYS must pass through untouched — `comparisonSql` resolves
|
|
319
|
+
// `paramRef` via hasOwnProperty, and a dropped key silently falls back to
|
|
320
|
+
// the baked literal (the crossed-results class fixed in aa4af79b).
|
|
321
|
+
const normPlan = normalizePlanForClone(plan);
|
|
322
|
+
const normParams: Record<string, unknown> = {};
|
|
323
|
+
for (const [k, v] of Object.entries(params)) normParams[k] = toCloneSafe(v);
|
|
324
|
+
try {
|
|
325
|
+
const res = await this.call<{ rows: Row[]; relationFetches?: number }>('select', {
|
|
326
|
+
plan: normPlan,
|
|
327
|
+
params: normParams,
|
|
328
|
+
});
|
|
329
|
+
getStats().relationFetches += res.relationFetches ?? 0;
|
|
330
|
+
return res.rows ?? [];
|
|
331
|
+
} catch (err) {
|
|
332
|
+
// Stale worker script without the 'select' op: fall back for good.
|
|
333
|
+
if (err instanceof Error && err.message.includes('unknown message')) {
|
|
334
|
+
this.workerSelect = false;
|
|
335
|
+
this.logger.warn(
|
|
336
|
+
{ err, Category: 'sp00ky-client::SqliteCacheEngine::select' },
|
|
337
|
+
'Worker lacks select op (stale script?) — falling back to multi-hop select'
|
|
338
|
+
);
|
|
339
|
+
} else {
|
|
340
|
+
throw err;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
return this.selectLegacy(plan, params);
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** Pre-worker-select path: one worker round-trip per table/relation level,
|
|
348
|
+
* rows parsed on the main thread. Kept as the `workerSelect:false` escape
|
|
349
|
+
* hatch and the stale-worker fallback. */
|
|
350
|
+
private async selectLegacy(
|
|
351
|
+
plan: QueryPlan,
|
|
352
|
+
params: Record<string, unknown> = {}
|
|
353
|
+
): Promise<Row[]> {
|
|
273
354
|
// Window materialization: base rows are exactly `plan.ids`, ordered.
|
|
274
355
|
if (plan.ids) {
|
|
275
356
|
const result = await this.selectByIds(plan.table, plan.ids, {
|
|
@@ -297,6 +378,7 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
297
378
|
}
|
|
298
379
|
|
|
299
380
|
async fetchRelation(req: RelationFetch): Promise<Row[]> {
|
|
381
|
+
getStats().relationFetches++;
|
|
300
382
|
await this.ensureTable(req.table);
|
|
301
383
|
const keys = req.keys.map(stableKey);
|
|
302
384
|
const placeholders = keys.map(() => '?').join(', ');
|
|
@@ -617,113 +699,99 @@ interface SqliteStats {
|
|
|
617
699
|
inFlight: number;
|
|
618
700
|
maxInFlight: number;
|
|
619
701
|
byType: Record<string, number>;
|
|
702
|
+
/** Time ops spent waiting behind the opQueue before dispatch. */
|
|
703
|
+
queueWaitMs: number;
|
|
704
|
+
/** Time inside the worker's message handler (actual SQLite work). */
|
|
705
|
+
workerMs: number;
|
|
706
|
+
/** Round-trip time minus workerMs: postMessage + scheduling overhead. */
|
|
707
|
+
rpcOverheadMs: number;
|
|
708
|
+
/** Main-thread JSON parse/revive of returned rows. */
|
|
709
|
+
parseMs: number;
|
|
710
|
+
rowsParsed: number;
|
|
711
|
+
bytesParsed: number;
|
|
712
|
+
/** Relation-resolver fan-out fetches (one worker round-trip each). */
|
|
713
|
+
relationFetches: number;
|
|
620
714
|
}
|
|
621
715
|
|
|
716
|
+
const EMPTY_STATS: SqliteStats = {
|
|
717
|
+
roundTrips: 0,
|
|
718
|
+
batchStatements: 0,
|
|
719
|
+
maxBatch: 0,
|
|
720
|
+
inFlight: 0,
|
|
721
|
+
maxInFlight: 0,
|
|
722
|
+
byType: {},
|
|
723
|
+
queueWaitMs: 0,
|
|
724
|
+
workerMs: 0,
|
|
725
|
+
rpcOverheadMs: 0,
|
|
726
|
+
parseMs: 0,
|
|
727
|
+
rowsParsed: 0,
|
|
728
|
+
bytesParsed: 0,
|
|
729
|
+
relationFetches: 0,
|
|
730
|
+
};
|
|
731
|
+
|
|
622
732
|
/** Live stats, inspectable in the browser console via `__sqliteStats`. Counts
|
|
623
|
-
* worker round-trips (the sync-down cost driver), batch sizes,
|
|
624
|
-
* (`maxInFlight`)
|
|
733
|
+
* worker round-trips (the sync-down cost driver), batch sizes, queue depth
|
|
734
|
+
* (`maxInFlight`), and the latency split of each round-trip (queue wait vs
|
|
735
|
+
* worker time vs RPC overhead vs main-thread row parsing) so first-load cost
|
|
736
|
+
* can be measured rather than guessed. */
|
|
625
737
|
function getStats(): SqliteStats {
|
|
626
738
|
const g = globalThis as unknown as { __sqliteStats?: SqliteStats };
|
|
627
739
|
if (!g.__sqliteStats) {
|
|
628
|
-
g.__sqliteStats = {
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
byType: {},
|
|
635
|
-
};
|
|
740
|
+
g.__sqliteStats = { ...EMPTY_STATS, byType: {} };
|
|
741
|
+
} else {
|
|
742
|
+
// Backfill fields added since the object was created (HMR / older bundle).
|
|
743
|
+
for (const [k, v] of Object.entries(EMPTY_STATS)) {
|
|
744
|
+
if ((g.__sqliteStats as any)[k] === undefined) (g.__sqliteStats as any)[k] = v;
|
|
745
|
+
}
|
|
636
746
|
}
|
|
637
747
|
return g.__sqliteStats;
|
|
638
748
|
}
|
|
639
749
|
|
|
640
|
-
//
|
|
750
|
+
// SQL rendering + row (de)serialization live in `sqlite-plan-sql.ts`, shared
|
|
751
|
+
// with the worker (worker-side plan execution renders the same SQL).
|
|
641
752
|
|
|
642
|
-
|
|
643
|
-
return ` ORDER BY ${orderBy
|
|
644
|
-
.map(([f, d]) => `json_extract(data, '$.${f}') ${d === 'desc' ? 'DESC' : 'ASC'}`)
|
|
645
|
-
.join(', ')}`;
|
|
646
|
-
}
|
|
647
|
-
|
|
648
|
-
function comparisonSql(
|
|
649
|
-
c: WhereComparison,
|
|
650
|
-
bind: unknown[],
|
|
651
|
-
params: Record<string, unknown>
|
|
652
|
-
): string {
|
|
653
|
-
const lhs = c.field === 'id' ? 'id' : `json_extract(data, '$.${c.field}')`;
|
|
654
|
-
const value = c.paramRef ? params[c.paramRef] : c.value;
|
|
655
|
-
bind.push(scalar(value));
|
|
656
|
-
const op = c.op === '!=' ? '!=' : c.op;
|
|
657
|
-
return c.swap ? `? ${op} ${lhs}` : `${lhs} ${op} ?`;
|
|
658
|
-
}
|
|
659
|
-
|
|
660
|
-
function renderWhereSql(
|
|
661
|
-
nodes: WhereNode[],
|
|
662
|
-
bind: unknown[],
|
|
663
|
-
params: Record<string, unknown>
|
|
664
|
-
): string {
|
|
665
|
-
return nodes
|
|
666
|
-
.map((node) => {
|
|
667
|
-
if ('or' in node) {
|
|
668
|
-
return `(${node.or.map((c) => comparisonSql(c, bind, params)).join(' OR ')})`;
|
|
669
|
-
}
|
|
670
|
-
return comparisonSql(node, bind, params);
|
|
671
|
-
})
|
|
672
|
-
.join(' AND ');
|
|
673
|
-
}
|
|
674
|
-
|
|
675
|
-
// ==================== value (de)serialization ====================
|
|
676
|
-
|
|
677
|
-
/** A comparable scalar for SQL binding: record links → `table:id`, everything
|
|
678
|
-
* else passed through (numbers/strings/bools). */
|
|
679
|
-
function scalar(value: unknown): unknown {
|
|
680
|
-
if (value == null) return null;
|
|
681
|
-
if (typeof value === 'object') return stableKey(value);
|
|
682
|
-
return value;
|
|
683
|
-
}
|
|
753
|
+
// ==================== structured-clone normalization ====================
|
|
684
754
|
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
// invoking a JS callback for every key of every row on the read hot path.
|
|
700
|
-
if (json.indexOf('"__u8"') === -1) return JSON.parse(json);
|
|
701
|
-
return JSON.parse(json, (_k, v) => {
|
|
702
|
-
if (v && typeof v === 'object' && typeof (v as any).__u8 === 'string') {
|
|
703
|
-
return fromBase64((v as any).__u8);
|
|
704
|
-
}
|
|
705
|
-
return v;
|
|
706
|
-
});
|
|
755
|
+
/**
|
|
756
|
+
* Make a bind/param value safe to cross the worker boundary. structuredClone
|
|
757
|
+
* keeps plain data intact but strips a CLASS instance to a bare object — and
|
|
758
|
+
* surrealdb's `RecordId` stores its fields behind getters (zero own
|
|
759
|
+
* properties), so it clones to `{}`. Convert such instances to their
|
|
760
|
+
* `stableKey` string (the exact value `scalar()` would bind on the main
|
|
761
|
+
* thread), leave everything clone-representable untouched.
|
|
762
|
+
*/
|
|
763
|
+
function toCloneSafe(v: unknown): unknown {
|
|
764
|
+
if (v === null || typeof v !== 'object') return v;
|
|
765
|
+
if (Array.isArray(v) || v instanceof Uint8Array || v instanceof Date) return v;
|
|
766
|
+
const proto = Object.getPrototypeOf(v);
|
|
767
|
+
if (proto === Object.prototype || proto === null) return v;
|
|
768
|
+
return stableKey(v);
|
|
707
769
|
}
|
|
708
770
|
|
|
709
|
-
function
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
771
|
+
function normalizeWhereForClone(nodes: WhereNode[] | undefined): WhereNode[] | undefined {
|
|
772
|
+
if (!nodes) return nodes;
|
|
773
|
+
return nodes.map((n) =>
|
|
774
|
+
'or' in n
|
|
775
|
+
? { or: n.or.map((c) => ({ ...c, value: toCloneSafe(c.value) })) }
|
|
776
|
+
: { ...n, value: toCloneSafe(n.value) }
|
|
777
|
+
);
|
|
713
778
|
}
|
|
714
779
|
|
|
715
|
-
function
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
780
|
+
function normalizeRelationForClone(r: RelationPlan): RelationPlan {
|
|
781
|
+
return {
|
|
782
|
+
...r,
|
|
783
|
+
where: normalizeWhereForClone(r.where),
|
|
784
|
+
relations: r.relations?.map(normalizeRelationForClone),
|
|
785
|
+
};
|
|
719
786
|
}
|
|
720
787
|
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
788
|
+
/** Normalize every baked value a plan carries (where trees, window ids) for
|
|
789
|
+
* the postMessage to the worker's `select` op. */
|
|
790
|
+
function normalizePlanForClone(plan: QueryPlan): QueryPlan {
|
|
791
|
+
return {
|
|
792
|
+
...plan,
|
|
793
|
+
ids: plan.ids?.map(stableKey),
|
|
794
|
+
where: normalizeWhereForClone(plan.where),
|
|
795
|
+
relations: plan.relations?.map(normalizeRelationForClone),
|
|
796
|
+
};
|
|
729
797
|
}
|