@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +57 -0
- package/dist/index.d.ts +2514 -58
- package/dist/index.js +12561 -2449
- package/dist/otel/index.d.ts +2 -2
- package/dist/otel/index.js +6 -6
- package/dist/sqlite-open.js +303 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +439 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +472 -0
- package/dist/types.d.ts +751 -11
- package/package.json +11 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
- package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
- package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
- package/src/bucket-blurhash.test.ts +148 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +36 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/auth.local-first.test.ts +101 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +127 -24
- package/src/modules/cache/cache.relay.test.ts +95 -0
- package/src/modules/cache/index.ts +163 -43
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +210 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +463 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.membership.test.ts +523 -0
- package/src/modules/data/data.notify-table.test.ts +41 -0
- package/src/modules/data/data.pending-ids.test.ts +199 -0
- package/src/modules/data/data.rebind.test.ts +170 -0
- package/src/modules/data/data.rematerialize.test.ts +114 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.settled-writes.test.ts +206 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +1815 -151
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +194 -0
- package/src/modules/devtools/flags.ts +349 -0
- package/src/modules/devtools/index.ts +450 -46
- package/src/modules/devtools/notify-throttle.test.ts +154 -0
- package/src/modules/devtools/state-shape.test.ts +146 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +168 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +110 -0
- package/src/modules/feature-flag/index.test.ts +251 -0
- package/src/modules/feature-flag/index.ts +308 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +164 -82
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.test.ts +180 -0
- package/src/modules/sync/queue/queue-down.ts +80 -13
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
- package/src/modules/sync/queue/queue-up.ts +241 -57
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.retry.test.ts +237 -0
- package/src/modules/sync/scheduler.ts +215 -13
- package/src/modules/sync/sync.cleanup.test.ts +116 -0
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.heartbeat.test.ts +80 -0
- package/src/modules/sync/sync.live-removal.test.ts +175 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.tabs.test.ts +249 -0
- package/src/modules/sync/sync.ts +1726 -99
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +201 -17
- package/src/otel/index.ts +13 -10
- package/src/services/blobs/blob-cache.test.ts +359 -0
- package/src/services/blobs/blob-cache.ts +603 -0
- package/src/services/blobs/blob-manifest.ts +227 -0
- package/src/services/blobs/blob-store.test.ts +77 -0
- package/src/services/blobs/blob-store.ts +359 -0
- package/src/services/blobs/blob.fixture.ts +90 -0
- package/src/services/blobs/index.ts +70 -0
- package/src/services/database/cache-engine.ts +193 -0
- package/src/services/database/connection-supervisor.test.ts +289 -0
- package/src/services/database/connection-supervisor.ts +415 -0
- package/src/services/database/database.query-timeout.test.ts +83 -0
- package/src/services/database/database.ts +41 -12
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/errors.ts +34 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +7 -0
- package/src/services/database/local-migrator.ts +30 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +484 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +110 -14
- package/src/services/database/sqlite-cache-engine.test.ts +616 -0
- package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
- package/src/services/database/sqlite-cache-engine.ts +1358 -0
- package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
- package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
- package/src/services/database/sqlite-lock-verify.test.ts +33 -0
- package/src/services/database/sqlite-lock-verify.ts +45 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +138 -0
- package/src/services/database/sqlite-projection.test.ts +99 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +246 -0
- package/src/services/database/sqlite-select.ts +131 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +224 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +416 -0
- package/src/services/database/surreal-cache-engine.ts +161 -0
- package/src/services/logger/index.ts +3 -2
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +796 -84
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
- package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +59 -3
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +327 -0
- package/src/services/tabs/coordinator.test.ts +365 -0
- package/src/services/tabs/coordinator.ts +633 -0
- package/src/services/tabs/fake-ports.fixture.ts +112 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +258 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +640 -0
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.local-first.test.ts +60 -0
- package/src/sp00ky.ts +1693 -0
- package/src/types.ts +528 -13
- package/src/utils/blurhash.ts +90 -0
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +79 -13
- package/src/utils/parser.test.ts +49 -120
- package/src/utils/parser.ts +32 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +30 -18
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +86 -1
- package/src/spooky.ts +0 -395
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import { describe, it, expect, beforeAll, afterAll } from 'vitest';
|
|
2
|
+
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
|
3
|
+
import type { QueryPlan } from '@spooky-sync/query-builder';
|
|
4
|
+
import { executeSelect, type SelectDb } from './sqlite-select';
|
|
5
|
+
import { serializeRow } from './sqlite-plan-sql';
|
|
6
|
+
import type { Row } from './cache-engine';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Integration: `executeSelect` (the worker-side plan executor) against a REAL
|
|
10
|
+
* in-memory SQLite (the same @sqlite.org/sqlite-wasm build the worker loads).
|
|
11
|
+
* The scripted-responder tests in `sqlite-select.test.ts` prove parity of the
|
|
12
|
+
* statement sequence; these prove the SQL itself is correct — json_extract
|
|
13
|
+
* filtering, ORDER BY, LIMIT/OFFSET, IN matching, per-parent relation
|
|
14
|
+
* order/limit, and the JSON round-trip of row bodies.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
let db: any;
|
|
18
|
+
let selectDb: SelectDb;
|
|
19
|
+
|
|
20
|
+
function seed(table: string, rows: Row[]): void {
|
|
21
|
+
db.exec({ sql: `CREATE TABLE IF NOT EXISTS "${table}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)` });
|
|
22
|
+
selectDb.knownTables.add(table);
|
|
23
|
+
for (const row of rows) {
|
|
24
|
+
db.exec({
|
|
25
|
+
sql: `INSERT INTO "${table}"(id, data) VALUES(?, ?)`,
|
|
26
|
+
bind: [row.id, serializeRow(row)],
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
beforeAll(async () => {
|
|
32
|
+
const sqlite3: any = await sqlite3InitModule();
|
|
33
|
+
db = new sqlite3.oo1.DB(':memory:', 'c');
|
|
34
|
+
selectDb = {
|
|
35
|
+
exec: (sql, bind) =>
|
|
36
|
+
db.exec({ sql, bind, rowMode: 'object', returnValue: 'resultRows' }) as { data: string }[],
|
|
37
|
+
run: (sql, bind) => {
|
|
38
|
+
db.exec({ sql, bind });
|
|
39
|
+
},
|
|
40
|
+
knownTables: new Set<string>(),
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
seed('thread', [
|
|
44
|
+
{ id: 'thread:A', title: 'alpha', createdAt: 3, author: 'user:1' },
|
|
45
|
+
{ id: 'thread:B', title: 'beta', createdAt: 1, author: 'user:2' },
|
|
46
|
+
{ id: 'thread:C', title: 'gamma', createdAt: 2, author: 'user:1' },
|
|
47
|
+
]);
|
|
48
|
+
seed('comment', [
|
|
49
|
+
{ id: 'comment:1', thread: 'thread:B', body: 'b-old', votes: 1, author: 'user:1' },
|
|
50
|
+
{ id: 'comment:2', thread: 'thread:B', body: 'b-new', votes: 5, author: 'user:2' },
|
|
51
|
+
{ id: 'comment:3', thread: 'thread:B', body: 'b-mid', votes: 3, author: 'user:1' },
|
|
52
|
+
{ id: 'comment:4', thread: 'thread:A', body: 'a-only', votes: 2, author: 'user:2' },
|
|
53
|
+
]);
|
|
54
|
+
seed('user', [
|
|
55
|
+
{ id: 'user:1', name: 'ada' },
|
|
56
|
+
{ id: 'user:2', name: 'grace' },
|
|
57
|
+
]);
|
|
58
|
+
seed('snapshot', [{ id: 'snapshot:1', blob: new Uint8Array([1, 2, 250]), kind: 'crdt' }]);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
afterAll(() => {
|
|
62
|
+
db?.close();
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
describe('executeSelect against real SQLite', () => {
|
|
66
|
+
it('filters via json_extract, slaved to params over the baked literal (aa4af79b)', async () => {
|
|
67
|
+
const plan: QueryPlan = {
|
|
68
|
+
table: 'thread',
|
|
69
|
+
where: [{ field: 'id', op: '=', value: 'thread:A', paramRef: 'id' }],
|
|
70
|
+
};
|
|
71
|
+
const withParam = await executeSelect(plan, { id: 'thread:B' }, selectDb);
|
|
72
|
+
expect(withParam.rows.map((r) => r.id)).toEqual(['thread:B']);
|
|
73
|
+
|
|
74
|
+
const withoutParam = await executeSelect(plan, {}, selectDb);
|
|
75
|
+
expect(withoutParam.rows.map((r) => r.id)).toEqual(['thread:A']);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it('filters on a non-id field and binds a RecordId-shaped value as table:id', async () => {
|
|
79
|
+
const plan: QueryPlan = {
|
|
80
|
+
table: 'thread',
|
|
81
|
+
where: [{ field: 'author', op: '=', value: { tb: 'user', id: '1' } }],
|
|
82
|
+
orderBy: [['createdAt', 'asc']],
|
|
83
|
+
};
|
|
84
|
+
const { rows } = await executeSelect(plan, {}, selectDb);
|
|
85
|
+
expect(rows.map((r) => r.id)).toEqual(['thread:C', 'thread:A']);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('applies ORDER BY / LIMIT / OFFSET in SQL', async () => {
|
|
89
|
+
const plan: QueryPlan = {
|
|
90
|
+
table: 'thread',
|
|
91
|
+
orderBy: [['createdAt', 'desc']],
|
|
92
|
+
limit: 2,
|
|
93
|
+
offset: 1,
|
|
94
|
+
};
|
|
95
|
+
const { rows } = await executeSelect(plan, {}, selectDb);
|
|
96
|
+
expect(rows.map((r) => r.id)).toEqual(['thread:C', 'thread:B']);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it('supports OR groups', async () => {
|
|
100
|
+
const plan: QueryPlan = {
|
|
101
|
+
table: 'thread',
|
|
102
|
+
where: [
|
|
103
|
+
{
|
|
104
|
+
or: [
|
|
105
|
+
{ field: 'title', op: '=', value: 'alpha' },
|
|
106
|
+
{ field: 'title', op: '=', value: 'beta' },
|
|
107
|
+
],
|
|
108
|
+
},
|
|
109
|
+
],
|
|
110
|
+
orderBy: [['title', 'asc']],
|
|
111
|
+
};
|
|
112
|
+
const { rows } = await executeSelect(plan, {}, selectDb);
|
|
113
|
+
expect(rows.map((r) => r.id)).toEqual(['thread:A', 'thread:B']);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('resolves a nested relation tree with per-parent order/limit, one batch per level', async () => {
|
|
117
|
+
const plan: QueryPlan = {
|
|
118
|
+
table: 'thread',
|
|
119
|
+
where: [{ field: 'id', op: '=', value: 'thread:B' }],
|
|
120
|
+
relations: [
|
|
121
|
+
{
|
|
122
|
+
alias: 'comments',
|
|
123
|
+
table: 'comment',
|
|
124
|
+
cardinality: 'many',
|
|
125
|
+
foreignKeyField: 'thread',
|
|
126
|
+
orderBy: [['votes', 'desc']],
|
|
127
|
+
limit: 2,
|
|
128
|
+
relations: [
|
|
129
|
+
{ alias: 'author', table: 'user', cardinality: 'one', foreignKeyField: 'author' },
|
|
130
|
+
],
|
|
131
|
+
},
|
|
132
|
+
],
|
|
133
|
+
};
|
|
134
|
+
const { rows, relationFetches } = await executeSelect(plan, {}, selectDb);
|
|
135
|
+
expect(rows).toHaveLength(1);
|
|
136
|
+
const comments = rows[0].comments as Row[];
|
|
137
|
+
// Top-2 by votes, per parent.
|
|
138
|
+
expect(comments.map((c) => c.id)).toEqual(['comment:2', 'comment:3']);
|
|
139
|
+
// Nested `one` relation attached on each child.
|
|
140
|
+
expect(comments.map((c) => (c.author as Row).name)).toEqual(['grace', 'ada']);
|
|
141
|
+
// Level-ordered batching: one fetch for comments + one for users.
|
|
142
|
+
expect(relationFetches).toBe(2);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
it('materializes an ids window preserving ids order, or ORDER BY when given', async () => {
|
|
146
|
+
const windowPlan: QueryPlan = { table: 'comment', ids: ['comment:3', 'comment:1'] };
|
|
147
|
+
const inIdsOrder = await executeSelect(windowPlan, {}, selectDb);
|
|
148
|
+
expect(inIdsOrder.rows.map((r) => r.id)).toEqual(['comment:3', 'comment:1']);
|
|
149
|
+
|
|
150
|
+
const ordered = await executeSelect(
|
|
151
|
+
{ ...windowPlan, orderBy: [['votes', 'asc']] },
|
|
152
|
+
{},
|
|
153
|
+
selectDb
|
|
154
|
+
);
|
|
155
|
+
expect(ordered.rows.map((r) => r.id)).toEqual(['comment:1', 'comment:3']);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
it('trims to the projection while keeping id', async () => {
|
|
159
|
+
const plan: QueryPlan = {
|
|
160
|
+
table: 'thread',
|
|
161
|
+
where: [{ field: 'id', op: '=', value: 'thread:A' }],
|
|
162
|
+
select: ['title'],
|
|
163
|
+
};
|
|
164
|
+
const { rows } = await executeSelect(plan, {}, selectDb);
|
|
165
|
+
expect(rows).toEqual([{ id: 'thread:A', title: 'alpha' }]);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it('revives Uint8Array bodies through the {__u8} tag round-trip', async () => {
|
|
169
|
+
const plan: QueryPlan = {
|
|
170
|
+
table: 'snapshot',
|
|
171
|
+
where: [{ field: 'kind', op: '=', value: 'crdt' }],
|
|
172
|
+
};
|
|
173
|
+
const { rows } = await executeSelect(plan, {}, selectDb);
|
|
174
|
+
expect(rows).toHaveLength(1);
|
|
175
|
+
expect(rows[0].blob).toBeInstanceOf(Uint8Array);
|
|
176
|
+
expect([...(rows[0].blob as Uint8Array)]).toEqual([1, 2, 250]);
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
it('creates a missing table on first touch instead of failing', async () => {
|
|
180
|
+
const plan: QueryPlan = { table: 'never_written', where: [{ field: 'x', op: '=', value: 1 }] };
|
|
181
|
+
const { rows } = await executeSelect(plan, {}, selectDb);
|
|
182
|
+
expect(rows).toEqual([]);
|
|
183
|
+
expect(selectDb.knownTables.has('never_written')).toBe(true);
|
|
184
|
+
});
|
|
185
|
+
});
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { RecordId } from 'surrealdb';
|
|
3
|
+
import type { QueryPlan } from '@spooky-sync/query-builder';
|
|
4
|
+
import { executeSelect, type SelectDb } from './sqlite-select';
|
|
5
|
+
import { SqliteCacheEngine } from './sqlite-cache-engine';
|
|
6
|
+
import { stubTransport } from './sqlite-transport.fixture';
|
|
7
|
+
import { stableKey } from './relation-resolver';
|
|
8
|
+
import type { Row } from './cache-engine';
|
|
9
|
+
|
|
10
|
+
function makeLogger(): any {
|
|
11
|
+
const noop = () => {};
|
|
12
|
+
const l: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
13
|
+
l.child = () => l;
|
|
14
|
+
return l;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* A tiny scripted "SQL responder" for the EXACT statement shapes the SQLite
|
|
19
|
+
* renderers emit (base select by id, id-window IN, relation json_extract IN).
|
|
20
|
+
* Not a SQL engine — just enough to make legacy-vs-worker parity observable.
|
|
21
|
+
*/
|
|
22
|
+
function makeResponder(tables: Record<string, Row[]>) {
|
|
23
|
+
return (sql: string, bind: unknown[] = []): { data: string }[] => {
|
|
24
|
+
const table = /FROM "([^"]+)"/.exec(sql)?.[1];
|
|
25
|
+
let rows = table ? (tables[table] ?? []) : [];
|
|
26
|
+
const rel = /WHERE json_extract\(data, '\$\.([A-Za-z0-9_.]+)'\) IN/.exec(sql);
|
|
27
|
+
if (rel) {
|
|
28
|
+
rows = rows.filter((r) => bind.includes(r[rel[1]]));
|
|
29
|
+
} else if (/WHERE id IN/.test(sql)) {
|
|
30
|
+
rows = rows.filter((r) => bind.includes(r.id));
|
|
31
|
+
} else if (/WHERE id = \?/.test(sql)) {
|
|
32
|
+
rows = rows.filter((r) => r.id === bind[0]);
|
|
33
|
+
}
|
|
34
|
+
return rows.map((r) => ({ data: JSON.stringify(r) }));
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const FIXTURE: Record<string, Row[]> = {
|
|
39
|
+
thread: [
|
|
40
|
+
{ id: 'thread:A', title: 'thread A', author: 'user:1' },
|
|
41
|
+
{ id: 'thread:B', title: 'thread B', author: 'user:2' },
|
|
42
|
+
],
|
|
43
|
+
comment: [
|
|
44
|
+
{ id: 'comment:1', thread: 'thread:B', body: 'first' },
|
|
45
|
+
{ id: 'comment:2', thread: 'thread:B', body: 'second' },
|
|
46
|
+
{ id: 'comment:3', thread: 'thread:A', body: 'other thread' },
|
|
47
|
+
],
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
/** The `.one()` detail-view plan shape: id baked as a literal AND slaved to
|
|
51
|
+
* the `id` param (aa4af79b), plus a `many` relation. */
|
|
52
|
+
const DETAIL_PLAN: QueryPlan = {
|
|
53
|
+
table: 'thread',
|
|
54
|
+
where: [{ field: 'id', op: '=', value: 'thread:A', paramRef: 'id' }],
|
|
55
|
+
relations: [
|
|
56
|
+
{ alias: 'comments', table: 'comment', cardinality: 'many', foreignKeyField: 'thread' },
|
|
57
|
+
],
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
function makeSelectDb(
|
|
61
|
+
tables: Record<string, Row[]>,
|
|
62
|
+
calls: { sql: string; bind?: unknown[] }[]
|
|
63
|
+
): SelectDb {
|
|
64
|
+
const respond = makeResponder(tables);
|
|
65
|
+
return {
|
|
66
|
+
exec: (sql, bind) => {
|
|
67
|
+
calls.push({ sql, bind });
|
|
68
|
+
return respond(sql, bind ?? []);
|
|
69
|
+
},
|
|
70
|
+
run: (sql, bind) => {
|
|
71
|
+
calls.push({ sql, bind });
|
|
72
|
+
},
|
|
73
|
+
knownTables: new Set<string>(),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** An engine whose worker answers exec/run against the fixture (legacy path)
|
|
78
|
+
* and optionally the one-hop `select` op, recording every message. */
|
|
79
|
+
function makeEngine(opts: {
|
|
80
|
+
workerSelect: boolean;
|
|
81
|
+
answerSelect?: boolean;
|
|
82
|
+
calls?: { type: string; sql?: string; bind?: unknown[]; payload?: any }[];
|
|
83
|
+
}) {
|
|
84
|
+
const respond = makeResponder(FIXTURE);
|
|
85
|
+
const calls = opts.calls ?? [];
|
|
86
|
+
const engine = new SqliteCacheEngine(
|
|
87
|
+
{ namespace: 'n', database: 'd' } as any,
|
|
88
|
+
makeLogger(),
|
|
89
|
+
{ useOpfs: false, workerSelect: opts.workerSelect }
|
|
90
|
+
);
|
|
91
|
+
stubTransport(engine, async (type, payload) => {
|
|
92
|
+
calls.push({ type, sql: payload?.sql, bind: payload?.bind, payload });
|
|
93
|
+
if (type === 'open') return { persisted: false };
|
|
94
|
+
if (type === 'exec') return { rows: respond(payload.sql, payload.bind ?? []) };
|
|
95
|
+
if (type === 'run' || type === 'batch' || type === 'close') return {};
|
|
96
|
+
if (type === 'select') {
|
|
97
|
+
if (!opts.answerSelect) throw new Error(`sqlite worker: unknown message ${type}`);
|
|
98
|
+
const dbCalls: { sql: string; bind?: unknown[] }[] = [];
|
|
99
|
+
return executeSelect(payload.plan, payload.params ?? {}, makeSelectDb(FIXTURE, dbCalls));
|
|
100
|
+
}
|
|
101
|
+
throw new Error(`sqlite worker: unknown message ${type}`);
|
|
102
|
+
});
|
|
103
|
+
return { engine, calls };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
describe('executeSelect (worker-side plan execution)', () => {
|
|
107
|
+
it('slaves the base filter to params over the baked literal (aa4af79b guard)', async () => {
|
|
108
|
+
const calls: { sql: string; bind?: unknown[] }[] = [];
|
|
109
|
+
const { rows } = await executeSelect(DETAIL_PLAN, { id: 'thread:B' }, makeSelectDb(FIXTURE, calls));
|
|
110
|
+
// Filtered by the PARAM (thread:B), not the baked literal (thread:A).
|
|
111
|
+
expect(rows).toHaveLength(1);
|
|
112
|
+
expect(rows[0].id).toBe('thread:B');
|
|
113
|
+
expect(rows[0].comments).toEqual([
|
|
114
|
+
{ id: 'comment:1', thread: 'thread:B', body: 'first' },
|
|
115
|
+
{ id: 'comment:2', thread: 'thread:B', body: 'second' },
|
|
116
|
+
]);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it('falls back to the baked literal when the param key is absent', async () => {
|
|
120
|
+
const calls: { sql: string; bind?: unknown[] }[] = [];
|
|
121
|
+
const { rows } = await executeSelect(DETAIL_PLAN, {}, makeSelectDb(FIXTURE, calls));
|
|
122
|
+
expect(rows).toHaveLength(1);
|
|
123
|
+
expect(rows[0].id).toBe('thread:A');
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
it('counts relation fetches and honors the ids window ordering', async () => {
|
|
127
|
+
const calls: { sql: string; bind?: unknown[] }[] = [];
|
|
128
|
+
const plan: QueryPlan = {
|
|
129
|
+
table: 'comment',
|
|
130
|
+
ids: ['comment:2', 'comment:1'],
|
|
131
|
+
};
|
|
132
|
+
const { rows, relationFetches } = await executeSelect(plan, {}, makeSelectDb(FIXTURE, calls));
|
|
133
|
+
expect(relationFetches).toBe(0);
|
|
134
|
+
// ids order preserved (no ORDER BY).
|
|
135
|
+
expect(rows.map((r) => r.id)).toEqual(['comment:2', 'comment:1']);
|
|
136
|
+
});
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
describe('worker-select parity with the legacy multi-hop path', () => {
|
|
140
|
+
it('produces identical rows AND an identical SQL statement sequence', async () => {
|
|
141
|
+
// Legacy: engine drives one worker round-trip per statement.
|
|
142
|
+
const legacy = makeEngine({ workerSelect: false });
|
|
143
|
+
await legacy.engine.connect('anon');
|
|
144
|
+
const legacyRows = await legacy.engine.select(DETAIL_PLAN, { id: 'thread:B' });
|
|
145
|
+
const legacySql = legacy.calls
|
|
146
|
+
.filter((c) => c.type === 'exec' || c.type === 'run')
|
|
147
|
+
.map((c) => ({ sql: c.sql, bind: c.bind ?? [] }));
|
|
148
|
+
|
|
149
|
+
// Worker path: same plan executed in one hop via executeSelect.
|
|
150
|
+
const workerCalls: { sql: string; bind?: unknown[] }[] = [];
|
|
151
|
+
const { rows: workerRows } = await executeSelect(
|
|
152
|
+
DETAIL_PLAN,
|
|
153
|
+
{ id: 'thread:B' },
|
|
154
|
+
makeSelectDb(FIXTURE, workerCalls)
|
|
155
|
+
);
|
|
156
|
+
|
|
157
|
+
expect(workerRows).toEqual(legacyRows);
|
|
158
|
+
expect(workerCalls.map((c) => ({ sql: c.sql, bind: c.bind ?? [] }))).toEqual(legacySql);
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
describe('SqliteCacheEngine.select via worker (one hop)', () => {
|
|
163
|
+
it('sends exactly one select message, zero exec/run', async () => {
|
|
164
|
+
const { engine, calls } = makeEngine({ workerSelect: true, answerSelect: true });
|
|
165
|
+
await engine.connect('anon');
|
|
166
|
+
const rows = await engine.select(DETAIL_PLAN, { id: 'thread:B' });
|
|
167
|
+
expect(rows).toHaveLength(1);
|
|
168
|
+
expect(rows[0].id).toBe('thread:B');
|
|
169
|
+
const afterOpen = calls.filter((c) => c.type !== 'open');
|
|
170
|
+
expect(afterOpen.map((c) => c.type)).toEqual(['select']);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it('normalizes RecordId param VALUES to strings but never drops param KEYS', async () => {
|
|
174
|
+
const { engine, calls } = makeEngine({ workerSelect: true, answerSelect: true });
|
|
175
|
+
await engine.connect('anon');
|
|
176
|
+
await engine.select(DETAIL_PLAN, { id: new RecordId('thread', 'B'), untouched: 7 });
|
|
177
|
+
const sel = calls.find((c) => c.type === 'select')!;
|
|
178
|
+
// VALUE normalized (a class instance would lose its prototype in the real
|
|
179
|
+
// structured clone); KEY present — comparisonSql's paramRef resolution
|
|
180
|
+
// checks hasOwnProperty, and a dropped key silently falls back to the
|
|
181
|
+
// baked literal (the aa4af79b crossed-results class).
|
|
182
|
+
expect(sel.payload.params).toEqual({ id: 'thread:B', untouched: 7 });
|
|
183
|
+
expect(Object.prototype.hasOwnProperty.call(sel.payload.params, 'id')).toBe(true);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
it('normalizes a window plan\'s RecordId ids to the same strings the legacy path binds', async () => {
|
|
187
|
+
const rid = new RecordId('comment', '2');
|
|
188
|
+
const { engine, calls } = makeEngine({ workerSelect: true, answerSelect: true });
|
|
189
|
+
await engine.connect('anon');
|
|
190
|
+
await engine.select({ table: 'comment', ids: [rid, 'comment:1'] }, {});
|
|
191
|
+
const sel = calls.find((c) => c.type === 'select')!;
|
|
192
|
+
// stableKey output (surrealdb may escape the id part, e.g. comment:⟨2⟩) —
|
|
193
|
+
// identical to what the legacy `selectByIds` binds, so stored rows match.
|
|
194
|
+
expect(sel.payload.plan.ids).toEqual([stableKey(rid), 'comment:1']);
|
|
195
|
+
expect(sel.payload.plan.ids.every((i: unknown) => typeof i === 'string')).toBe(true);
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
it('normalizes RecordId values baked inside the plan\'s where tree', async () => {
|
|
199
|
+
const { engine, calls } = makeEngine({ workerSelect: true, answerSelect: true });
|
|
200
|
+
await engine.connect('anon');
|
|
201
|
+
const plan: QueryPlan = {
|
|
202
|
+
table: 'thread',
|
|
203
|
+
where: [{ field: 'author', op: '=', value: new RecordId('user', 'u1') }],
|
|
204
|
+
relations: [
|
|
205
|
+
{
|
|
206
|
+
alias: 'comments',
|
|
207
|
+
table: 'comment',
|
|
208
|
+
cardinality: 'many',
|
|
209
|
+
foreignKeyField: 'thread',
|
|
210
|
+
where: [{ field: 'author', op: '=', value: new RecordId('user', 'u1') }],
|
|
211
|
+
},
|
|
212
|
+
],
|
|
213
|
+
};
|
|
214
|
+
await engine.select(plan, {});
|
|
215
|
+
const sel = calls.find((c) => c.type === 'select')!;
|
|
216
|
+
// Both the top-level where and the relation sub-where cross the boundary
|
|
217
|
+
// as strings — a RecordId instance would structured-clone to `{}`.
|
|
218
|
+
expect(sel.payload.plan.where[0].value).toBe('user:u1');
|
|
219
|
+
expect(sel.payload.plan.relations[0].where[0].value).toBe('user:u1');
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
it('folds the worker-reported relationFetches into __sqliteStats', async () => {
|
|
223
|
+
const { engine } = makeEngine({ workerSelect: true, answerSelect: true });
|
|
224
|
+
await engine.connect('anon');
|
|
225
|
+
const before = (globalThis as any).__sqliteStats?.relationFetches ?? 0;
|
|
226
|
+
await engine.select(DETAIL_PLAN, { id: 'thread:B' });
|
|
227
|
+
// DETAIL_PLAN has one relation level → executeSelect reports 1 fetch.
|
|
228
|
+
expect((globalThis as any).__sqliteStats.relationFetches).toBe(before + 1);
|
|
229
|
+
});
|
|
230
|
+
|
|
231
|
+
it('degrades to the legacy path when the worker lacks the select op', async () => {
|
|
232
|
+
const { engine, calls } = makeEngine({ workerSelect: true, answerSelect: false });
|
|
233
|
+
await engine.connect('anon');
|
|
234
|
+
const rows = await engine.select(DETAIL_PLAN, { id: 'thread:B' });
|
|
235
|
+
expect(rows).toHaveLength(1);
|
|
236
|
+
expect(rows[0].id).toBe('thread:B');
|
|
237
|
+
// First attempt was 'select'; the rejection flipped the flag and the
|
|
238
|
+
// legacy path completed with exec/run hops.
|
|
239
|
+
expect(calls.some((c) => c.type === 'select')).toBe(true);
|
|
240
|
+
expect(calls.some((c) => c.type === 'exec')).toBe(true);
|
|
241
|
+
// The flag stays off: a second select never retries the worker op.
|
|
242
|
+
const before = calls.filter((c) => c.type === 'select').length;
|
|
243
|
+
await engine.select(DETAIL_PLAN, { id: 'thread:A' });
|
|
244
|
+
expect(calls.filter((c) => c.type === 'select').length).toBe(before);
|
|
245
|
+
});
|
|
246
|
+
});
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
import type { QueryPlan } from '@spooky-sync/query-builder';
|
|
2
|
+
import { resolveRelations, stableKey } from './relation-resolver';
|
|
3
|
+
import {
|
|
4
|
+
renderOrderSql,
|
|
5
|
+
renderWhereSql,
|
|
6
|
+
reviveRow,
|
|
7
|
+
project,
|
|
8
|
+
projectedDataSql,
|
|
9
|
+
} from './sqlite-plan-sql';
|
|
10
|
+
import type { OrderBy, RelationFetch, Row, RowFetcher } from './cache-engine';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Worker-side execution of a whole {@link QueryPlan} — table creation, base
|
|
14
|
+
* select (either the `ids` window or where/order/limit), projection, and the
|
|
15
|
+
* full `.related()` tree via the SHARED `resolveRelations` — against an
|
|
16
|
+
* injected DB handle. Runs inside `sqlite-worker.ts` so the engine pays ONE
|
|
17
|
+
* postMessage round-trip per select instead of one per table/relation level;
|
|
18
|
+
* extracted into its own module so the logic is unit-testable off-worker
|
|
19
|
+
* (parity with the engine's legacy multi-hop path).
|
|
20
|
+
*
|
|
21
|
+
* Semantics mirror `SqliteCacheEngine.selectLegacy` exactly: same SQL, same
|
|
22
|
+
* ordering rules, same projection, same resolver.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/** The slice of the worker's DB surface `executeSelect` needs. */
|
|
26
|
+
export interface SelectDb {
|
|
27
|
+
/** Run a row-returning statement; rows come back as `{ data: <json> }`. */
|
|
28
|
+
exec(sql: string, bind?: unknown[]): { data: string }[];
|
|
29
|
+
/** Run a statement for effect only (CREATE TABLE). */
|
|
30
|
+
run(sql: string, bind?: unknown[]): void;
|
|
31
|
+
/** Tables already CREATEd on this handle (caller owns the lifecycle). */
|
|
32
|
+
knownTables: Set<string>;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function ensureTable(db: SelectDb, table: string): void {
|
|
36
|
+
if (db.knownTables.has(table)) return;
|
|
37
|
+
db.run(`CREATE TABLE IF NOT EXISTS "${table}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)`);
|
|
38
|
+
db.knownTables.add(table);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function execRows(db: SelectDb, sql: string, bind: unknown[]): Row[] {
|
|
42
|
+
return db.exec(sql, bind).map((r) => reviveRow(r.data));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** Mirrors the engine's `selectByIds`: fetch by primary id, preserving `ids`
|
|
46
|
+
* order unless an ORDER BY overrides it. */
|
|
47
|
+
function selectByIds(
|
|
48
|
+
db: SelectDb,
|
|
49
|
+
table: string,
|
|
50
|
+
ids: unknown[],
|
|
51
|
+
opts?: { select?: string[]; orderBy?: OrderBy }
|
|
52
|
+
): Row[] {
|
|
53
|
+
if (ids.length === 0) return [];
|
|
54
|
+
ensureTable(db, table);
|
|
55
|
+
const keys = ids.map(stableKey);
|
|
56
|
+
const placeholders = keys.map(() => '?').join(', ');
|
|
57
|
+
// Projection binds land in the SELECT clause, so they go first.
|
|
58
|
+
const bind: unknown[] = [];
|
|
59
|
+
const dataCol = opts?.select ? projectedDataSql(opts.select, bind) : 'data';
|
|
60
|
+
bind.push(...keys);
|
|
61
|
+
let sql = `SELECT ${dataCol} FROM "${table}" WHERE id IN (${placeholders})`;
|
|
62
|
+
if (opts?.orderBy && opts.orderBy.length > 0) sql += renderOrderSql(opts.orderBy);
|
|
63
|
+
let rows = execRows(db, sql, bind);
|
|
64
|
+
if (!opts?.orderBy || opts.orderBy.length === 0) {
|
|
65
|
+
const pos = new Map(keys.map((k, i) => [k, i]));
|
|
66
|
+
rows = rows.sort((a, b) => (pos.get(stableKey(a.id)) ?? 0) - (pos.get(stableKey(b.id)) ?? 0));
|
|
67
|
+
}
|
|
68
|
+
return rows;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** Mirrors the engine's `fetchRelation` SQL exactly. */
|
|
72
|
+
function fetchRelation(db: SelectDb, req: RelationFetch): Row[] {
|
|
73
|
+
ensureTable(db, req.table);
|
|
74
|
+
const keys = req.keys.map(stableKey);
|
|
75
|
+
const placeholders = keys.map(() => '?').join(', ');
|
|
76
|
+
const bind: unknown[] = [...keys];
|
|
77
|
+
const lhs = req.matchField === 'id' ? 'id' : `json_extract(data, '$.${req.matchField}')`;
|
|
78
|
+
let sql = `SELECT data FROM "${req.table}" WHERE ${lhs} IN (${placeholders})`;
|
|
79
|
+
if (req.where && req.where.length > 0) {
|
|
80
|
+
sql += ` AND ${renderWhereSql(req.where, bind, {})}`;
|
|
81
|
+
}
|
|
82
|
+
if (req.orderBy && req.orderBy.length > 0) sql += renderOrderSql(req.orderBy);
|
|
83
|
+
const rows = execRows(db, sql, bind);
|
|
84
|
+
return req.select ? rows.map((r) => project(r, req.select!)) : rows;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export async function executeSelect(
|
|
88
|
+
plan: QueryPlan,
|
|
89
|
+
params: Record<string, unknown>,
|
|
90
|
+
db: SelectDb
|
|
91
|
+
): Promise<{ rows: Row[]; relationFetches: number }> {
|
|
92
|
+
// Per-call fetch counter (the engine folds it into `__sqliteStats`).
|
|
93
|
+
const counter = { n: 0 };
|
|
94
|
+
const fetcher: RowFetcher = {
|
|
95
|
+
fetchRelation: (req) => {
|
|
96
|
+
counter.n++;
|
|
97
|
+
return Promise.resolve(fetchRelation(db, req));
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
// Window materialization: base rows are exactly `plan.ids`, ordered.
|
|
101
|
+
if (plan.ids) {
|
|
102
|
+
const rows = selectByIds(db, plan.table, plan.ids, {
|
|
103
|
+
select: plan.select,
|
|
104
|
+
orderBy: plan.orderBy,
|
|
105
|
+
});
|
|
106
|
+
await resolveRelations(rows, plan.relations, fetcher);
|
|
107
|
+
return { rows, relationFetches: counter.n };
|
|
108
|
+
}
|
|
109
|
+
ensureTable(db, plan.table);
|
|
110
|
+
const bind: unknown[] = [];
|
|
111
|
+
// Projection binds land in the SELECT clause, ahead of any WHERE binds.
|
|
112
|
+
const dataCol = plan.select ? projectedDataSql(plan.select, bind) : 'data';
|
|
113
|
+
let sql = `SELECT ${dataCol} FROM "${plan.table}"`;
|
|
114
|
+
if (plan.where && plan.where.length > 0) {
|
|
115
|
+
sql += ` WHERE ${renderWhereSql(plan.where, bind, params)}`;
|
|
116
|
+
}
|
|
117
|
+
if (plan.orderBy && plan.orderBy.length > 0) sql += renderOrderSql(plan.orderBy);
|
|
118
|
+
// A query with no ORDER BY still has to render in SOME order, and "whatever
|
|
119
|
+
// SQLite hands back" is insertion order — which disagrees with the order the
|
|
120
|
+
// same query gets once it renders from server membership, and disagrees with
|
|
121
|
+
// SurrealDB, whose natural order is by id. That mismatch is visible: the
|
|
122
|
+
// first paint comes from this scan and the second from membership, so an
|
|
123
|
+
// unordered list visibly reshuffled about a second after load. Ordering by
|
|
124
|
+
// id here makes the two agree and makes the result stable across reloads.
|
|
125
|
+
else sql += ` ORDER BY id`;
|
|
126
|
+
if (plan.limit !== undefined) sql += ` LIMIT ${Number(plan.limit)}`;
|
|
127
|
+
if (plan.offset !== undefined) sql += ` OFFSET ${Number(plan.offset)}`;
|
|
128
|
+
const rows = execRows(db, sql, bind);
|
|
129
|
+
await resolveRelations(rows, plan.relations, fetcher);
|
|
130
|
+
return { rows, relationFetches: counter.n };
|
|
131
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test-only helper: stubs `SqliteCacheEngine.createTransport` with a fake
|
|
3
|
+
* transport driven by a plain handler function, replacing the old pattern of
|
|
4
|
+
* faking a whole Worker plus the engine's pending-map wiring. Not exported
|
|
5
|
+
* from the package; imported only by *.test.ts files.
|
|
6
|
+
*/
|
|
7
|
+
import type { SqliteTransport } from './sqlite-transport';
|
|
8
|
+
|
|
9
|
+
export type FakeTransportHandler = (type: string, payload: any) => unknown | Promise<unknown>;
|
|
10
|
+
|
|
11
|
+
/** Replace the engine's transport factory. Each open spawns a fresh fake, like
|
|
12
|
+
* the real factory spawns a fresh Worker. The handler returns the reply rest
|
|
13
|
+
* (without id/ok/wt) or throws to produce an error reply. */
|
|
14
|
+
export function stubTransport(engine: unknown, handler: FakeTransportHandler): void {
|
|
15
|
+
(engine as { createTransport: () => SqliteTransport }).createTransport = () => {
|
|
16
|
+
let closed = false;
|
|
17
|
+
return {
|
|
18
|
+
kind: 'worker',
|
|
19
|
+
get connected() {
|
|
20
|
+
return !closed;
|
|
21
|
+
},
|
|
22
|
+
call: <T>(type: string, payload?: unknown) =>
|
|
23
|
+
Promise.resolve().then(() => handler(type, payload)) as Promise<T>,
|
|
24
|
+
failAll() {},
|
|
25
|
+
close() {
|
|
26
|
+
closed = true;
|
|
27
|
+
},
|
|
28
|
+
} as SqliteTransport;
|
|
29
|
+
};
|
|
30
|
+
}
|