@spooky-sync/core 0.0.1-canary.20 → 0.0.1-canary.201
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +57 -0
- package/dist/index.d.ts +2184 -54
- package/dist/index.js +11515 -2399
- package/dist/otel/index.d.ts +2 -2
- package/dist/otel/index.js +6 -6
- package/dist/sqlite-open.js +276 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +421 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +434 -0
- package/dist/types.d.ts +688 -11
- package/package.json +11 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
- package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
- package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
- package/src/bucket-blurhash.test.ts +148 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +35 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +59 -20
- package/src/modules/cache/index.ts +112 -32
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +210 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +463 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.membership.test.ts +462 -0
- package/src/modules/data/data.rebind.test.ts +147 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.settled-writes.test.ts +206 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +1580 -130
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +194 -0
- package/src/modules/devtools/flags.ts +349 -0
- package/src/modules/devtools/index.ts +386 -37
- package/src/modules/devtools/notify-throttle.test.ts +149 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +168 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +110 -0
- package/src/modules/feature-flag/index.test.ts +251 -0
- package/src/modules/feature-flag/index.ts +308 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +101 -37
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.test.ts +107 -0
- package/src/modules/sync/queue/queue-down.ts +35 -6
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
- package/src/modules/sync/queue/queue-up.ts +241 -57
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.retry.test.ts +156 -0
- package/src/modules/sync/scheduler.ts +158 -11
- package/src/modules/sync/sync.cleanup.test.ts +116 -0
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.heartbeat.test.ts +80 -0
- package/src/modules/sync/sync.live-removal.test.ts +134 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.ts +1558 -99
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +201 -17
- package/src/otel/index.ts +13 -10
- package/src/services/blobs/blob-cache.test.ts +359 -0
- package/src/services/blobs/blob-cache.ts +603 -0
- package/src/services/blobs/blob-manifest.ts +227 -0
- package/src/services/blobs/blob-store.test.ts +77 -0
- package/src/services/blobs/blob-store.ts +359 -0
- package/src/services/blobs/blob.fixture.ts +90 -0
- package/src/services/blobs/index.ts +70 -0
- package/src/services/database/cache-engine.ts +160 -0
- package/src/services/database/connection-supervisor.test.ts +289 -0
- package/src/services/database/connection-supervisor.ts +415 -0
- package/src/services/database/database.query-timeout.test.ts +83 -0
- package/src/services/database/database.ts +32 -12
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +7 -0
- package/src/services/database/local-migrator.ts +30 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +478 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +110 -14
- package/src/services/database/sqlite-cache-engine.test.ts +558 -0
- package/src/services/database/sqlite-cache-engine.ts +1257 -0
- package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
- package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +106 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +246 -0
- package/src/services/database/sqlite-select.ts +121 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +221 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +416 -0
- package/src/services/database/surreal-cache-engine.ts +141 -0
- package/src/services/logger/index.ts +3 -2
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +444 -52
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +216 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +23 -2
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +278 -0
- package/src/services/tabs/coordinator.test.ts +244 -0
- package/src/services/tabs/coordinator.ts +576 -0
- package/src/services/tabs/fake-ports.fixture.ts +112 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +242 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +586 -0
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.ts +1543 -0
- package/src/types.ts +496 -13
- package/src/utils/blurhash.ts +90 -0
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +73 -13
- package/src/utils/parser.ts +3 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +30 -18
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +86 -1
- package/src/spooky.ts +0 -395
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import { describe, it, expect, beforeAll } from 'vitest';
|
|
2
|
+
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
|
3
|
+
import { SqliteCacheEngine } from './sqlite-cache-engine';
|
|
4
|
+
import { stubTransport } from './sqlite-transport.fixture';
|
|
5
|
+
import { serializeRow } from './sqlite-plan-sql';
|
|
6
|
+
import type { Row } from './cache-engine';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Integration: the DevTools Database explorer's statements run end-to-end
|
|
10
|
+
* (translate → engine SQL → REAL in-memory SQLite, the same
|
|
11
|
+
* @sqlite.org/sqlite-wasm build the worker loads). The unit test asserts the
|
|
12
|
+
* translation; this asserts the SQL actually executes and returns the right
|
|
13
|
+
* rows — the paging window in particular is easy to render into invalid SQLite
|
|
14
|
+
* (there is no bare OFFSET).
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
let db: any;
|
|
18
|
+
|
|
19
|
+
function makeLogger(): any {
|
|
20
|
+
const noop = () => {};
|
|
21
|
+
const l: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
22
|
+
l.child = () => l;
|
|
23
|
+
return l;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** An engine whose transport is a real SQLite database. */
|
|
27
|
+
function realEngine(): SqliteCacheEngine {
|
|
28
|
+
const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger());
|
|
29
|
+
stubTransport(engine, (type, payload: any) => {
|
|
30
|
+
switch (type) {
|
|
31
|
+
case 'open':
|
|
32
|
+
for (const t of payload.systemTables ?? []) {
|
|
33
|
+
db.exec({
|
|
34
|
+
sql: `CREATE TABLE IF NOT EXISTS "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)`,
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
return { persisted: true };
|
|
38
|
+
case 'exec':
|
|
39
|
+
return {
|
|
40
|
+
rows: db.exec({
|
|
41
|
+
sql: payload.sql,
|
|
42
|
+
bind: payload.bind,
|
|
43
|
+
rowMode: 'object',
|
|
44
|
+
returnValue: 'resultRows',
|
|
45
|
+
}),
|
|
46
|
+
};
|
|
47
|
+
case 'run':
|
|
48
|
+
db.exec({ sql: payload.sql, bind: payload.bind });
|
|
49
|
+
return {};
|
|
50
|
+
case 'batch':
|
|
51
|
+
for (const stmt of payload as { sql: string; bind?: unknown[] }[]) {
|
|
52
|
+
db.exec({ sql: stmt.sql, bind: stmt.bind });
|
|
53
|
+
}
|
|
54
|
+
return {};
|
|
55
|
+
default:
|
|
56
|
+
return {};
|
|
57
|
+
}
|
|
58
|
+
});
|
|
59
|
+
return engine;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function seed(table: string, rows: Row[]): void {
|
|
63
|
+
db.exec({
|
|
64
|
+
sql: `CREATE TABLE IF NOT EXISTS "${table}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)`,
|
|
65
|
+
});
|
|
66
|
+
for (const row of rows) {
|
|
67
|
+
db.exec({
|
|
68
|
+
sql: `INSERT INTO "${table}"(id, data) VALUES(?, ?)`,
|
|
69
|
+
bind: [row.id, serializeRow(row)],
|
|
70
|
+
});
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
beforeAll(async () => {
|
|
75
|
+
const sqlite3: any = await sqlite3InitModule();
|
|
76
|
+
db = new sqlite3.oo1.DB(':memory:', 'c');
|
|
77
|
+
seed(
|
|
78
|
+
'game_insight',
|
|
79
|
+
Array.from({ length: 5 }, (_, i) => ({
|
|
80
|
+
id: `game_insight:${i}`,
|
|
81
|
+
n: i,
|
|
82
|
+
tag: i % 2 ? 'odd' : 'even',
|
|
83
|
+
}))
|
|
84
|
+
);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
describe('DevTools explorer against real SQLite', () => {
|
|
88
|
+
it('pages a table with LIMIT/START', async () => {
|
|
89
|
+
const engine = realEngine();
|
|
90
|
+
await engine.connect('anon');
|
|
91
|
+
|
|
92
|
+
const [page1] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 2 START 0');
|
|
93
|
+
const [page2] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 2 START 2');
|
|
94
|
+
const [tail] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 20 START 4');
|
|
95
|
+
|
|
96
|
+
expect(page1.map((r) => r.id)).toEqual(['game_insight:0', 'game_insight:1']);
|
|
97
|
+
expect(page2.map((r) => r.id)).toEqual(['game_insight:2', 'game_insight:3']);
|
|
98
|
+
expect(tail.map((r) => r.id)).toEqual(['game_insight:4']);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
it('counts rows, with and without a filter', async () => {
|
|
102
|
+
const engine = realEngine();
|
|
103
|
+
await engine.connect('anon');
|
|
104
|
+
|
|
105
|
+
const [all] = await engine.query<[{ count: number }[]]>(
|
|
106
|
+
'SELECT count() FROM game_insight GROUP ALL'
|
|
107
|
+
);
|
|
108
|
+
const [odd] = await engine.query<[{ count: number }[]]>(
|
|
109
|
+
"SELECT count() FROM game_insight WHERE tag = 'odd' GROUP ALL"
|
|
110
|
+
);
|
|
111
|
+
|
|
112
|
+
expect(all).toEqual([{ count: 5 }]);
|
|
113
|
+
expect(odd).toEqual([{ count: 2 }]);
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('lists tables via INFO FOR DB', async () => {
|
|
117
|
+
const engine = realEngine();
|
|
118
|
+
await engine.connect('anon');
|
|
119
|
+
|
|
120
|
+
const [info] = await engine.query<[{ tables: Record<string, string> }]>('INFO FOR DB');
|
|
121
|
+
|
|
122
|
+
expect(Object.keys(info.tables)).toContain('game_insight');
|
|
123
|
+
// Seeded system tables show up too — the panel's "internal" toggle filters
|
|
124
|
+
// them, the engine must not pre-filter.
|
|
125
|
+
expect(Object.keys(info.tables)).toContain('_00_query');
|
|
126
|
+
expect(Object.keys(info.tables).some((t) => t.startsWith('sqlite_'))).toBe(false);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
it('edits and deletes a row by literal record id', async () => {
|
|
130
|
+
const engine = realEngine();
|
|
131
|
+
await engine.connect('anon');
|
|
132
|
+
|
|
133
|
+
await engine.query('UPDATE game_insight:1 MERGE $updates', { updates: { tag: 'edited' } });
|
|
134
|
+
const [edited] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 20 START 1');
|
|
135
|
+
expect(edited[0]).toMatchObject({ id: 'game_insight:1', n: 1, tag: 'edited' });
|
|
136
|
+
|
|
137
|
+
await engine.query('DELETE game_insight:1');
|
|
138
|
+
const [after] = await engine.query<[{ count: number }[]]>(
|
|
139
|
+
'SELECT count() FROM game_insight GROUP ALL'
|
|
140
|
+
);
|
|
141
|
+
expect(after).toEqual([{ count: 4 }]);
|
|
142
|
+
});
|
|
143
|
+
});
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { describe, it, expect } from 'vitest';
|
|
2
|
+
import { SqliteCacheEngine } from './sqlite-cache-engine';
|
|
3
|
+
import { stubTransport } from './sqlite-transport.fixture';
|
|
4
|
+
import { translateSurql } from './surql-translate';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* The statements the DevTools Database explorer emits against the LOCAL store.
|
|
8
|
+
* None of them come from `surql`, so they only ever exercised the SurrealDB
|
|
9
|
+
* engine: on SQLite the paging read threw "unsupported SurrealQL for
|
|
10
|
+
* translation: SELECT * FROM game_insight LIMIT 20 START 0", the table list came
|
|
11
|
+
* back empty (`INFO FOR DB` lowered to a noop) and row edit/delete never
|
|
12
|
+
* matched (they inline a literal record id).
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
function makeLogger(): any {
|
|
16
|
+
const noop = () => {};
|
|
17
|
+
const l: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
|
|
18
|
+
l.child = () => l;
|
|
19
|
+
return l;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Records every exec/run SQL so the paging window can be asserted. */
|
|
23
|
+
function recordingEngine(rows: Record<string, unknown[]> = {}) {
|
|
24
|
+
const sql: string[] = [];
|
|
25
|
+
const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger());
|
|
26
|
+
stubTransport(engine, (type, payload: any) => {
|
|
27
|
+
if (type === 'open') return { persisted: true };
|
|
28
|
+
if (type === 'exec' || type === 'run') {
|
|
29
|
+
sql.push(payload.sql);
|
|
30
|
+
for (const [needle, out] of Object.entries(rows)) {
|
|
31
|
+
if (payload.sql.includes(needle)) return { rows: out };
|
|
32
|
+
}
|
|
33
|
+
return { rows: [] };
|
|
34
|
+
}
|
|
35
|
+
return {};
|
|
36
|
+
});
|
|
37
|
+
return { engine, sql };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
describe('DevTools table paging (LIMIT/START)', () => {
|
|
41
|
+
it('translates the window instead of throwing', () => {
|
|
42
|
+
const { ops } = translateSurql('SELECT * FROM game_insight LIMIT 20 START 0', {});
|
|
43
|
+
expect(ops).toEqual([
|
|
44
|
+
{
|
|
45
|
+
kind: 'selectTable',
|
|
46
|
+
table: 'game_insight',
|
|
47
|
+
where: undefined,
|
|
48
|
+
orderBy: undefined,
|
|
49
|
+
select: undefined,
|
|
50
|
+
value: undefined,
|
|
51
|
+
limit: 20,
|
|
52
|
+
start: 0,
|
|
53
|
+
},
|
|
54
|
+
]);
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
it('renders LIMIT/OFFSET in SQL, page 3 of 20', async () => {
|
|
58
|
+
const { engine, sql } = recordingEngine();
|
|
59
|
+
await engine.connect('anon');
|
|
60
|
+
await engine.query('SELECT * FROM game_insight LIMIT 20 START 40');
|
|
61
|
+
expect(sql.some((s) => /SELECT data FROM "game_insight" LIMIT 20 OFFSET 40$/.test(s))).toBe(
|
|
62
|
+
true
|
|
63
|
+
);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('keeps WHERE and ORDER BY alongside the window', () => {
|
|
67
|
+
const { ops } = translateSurql(
|
|
68
|
+
'SELECT * FROM game WHERE done = true ORDER BY date desc LIMIT 10 START 30',
|
|
69
|
+
{}
|
|
70
|
+
);
|
|
71
|
+
expect(ops[0]).toMatchObject({
|
|
72
|
+
kind: 'selectTable',
|
|
73
|
+
table: 'game',
|
|
74
|
+
where: [{ field: 'done', op: '=', value: true }],
|
|
75
|
+
orderBy: [['date', 'desc']],
|
|
76
|
+
limit: 10,
|
|
77
|
+
start: 30,
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('leaves a LIMIT that is part of a string literal alone', () => {
|
|
82
|
+
const { ops } = translateSurql("SELECT * FROM game WHERE note = 'LIMIT 5'", {});
|
|
83
|
+
expect(ops[0]).toMatchObject({
|
|
84
|
+
kind: 'selectTable',
|
|
85
|
+
where: [{ field: 'note', op: '=', value: 'LIMIT 5' }],
|
|
86
|
+
limit: undefined,
|
|
87
|
+
start: undefined,
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
it('a START with no LIMIT still renders valid SQLite', async () => {
|
|
92
|
+
const { engine, sql } = recordingEngine();
|
|
93
|
+
await engine.connect('anon');
|
|
94
|
+
await engine.query('SELECT * FROM game START 5');
|
|
95
|
+
// SQLite has no bare OFFSET — `LIMIT -1` is its "everything" sentinel.
|
|
96
|
+
expect(sql.some((s) => s.endsWith('LIMIT -1 OFFSET 5'))).toBe(true);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
describe('DevTools row count', () => {
|
|
101
|
+
it("answers `SELECT count() FROM t GROUP ALL` in SurrealDB's shape", async () => {
|
|
102
|
+
const { engine } = recordingEngine({ 'COUNT(*)': [{ n: 137 }] });
|
|
103
|
+
await engine.connect('anon');
|
|
104
|
+
const res = await engine.query<[{ count: number }[]]>(
|
|
105
|
+
'SELECT count() FROM game_insight GROUP ALL'
|
|
106
|
+
);
|
|
107
|
+
expect(res[0]).toEqual([{ count: 137 }]);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
it('counts zero rows as 0, not undefined', async () => {
|
|
111
|
+
const { engine } = recordingEngine();
|
|
112
|
+
await engine.connect('anon');
|
|
113
|
+
const res = await engine.query<[{ count: number }[]]>('SELECT count() FROM empty GROUP ALL');
|
|
114
|
+
expect(res[0]).toEqual([{ count: 0 }]);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
117
|
+
|
|
118
|
+
describe('DevTools table list', () => {
|
|
119
|
+
it('answers INFO FOR DB from sqlite_master, minus SQLite internals', async () => {
|
|
120
|
+
const { engine } = recordingEngine({
|
|
121
|
+
sqlite_master: [{ name: 'game' }, { name: '_00_query' }],
|
|
122
|
+
});
|
|
123
|
+
await engine.connect('anon');
|
|
124
|
+
const [info] = await engine.query<[{ tables: Record<string, string> }]>('INFO FOR DB');
|
|
125
|
+
expect(Object.keys(info.tables)).toEqual(['game', '_00_query']);
|
|
126
|
+
});
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
describe('DevTools row edit / delete (literal record ids)', () => {
|
|
130
|
+
it('UPDATE <table>:<id> MERGE $updates writes that row', async () => {
|
|
131
|
+
const { engine, sql } = recordingEngine();
|
|
132
|
+
await engine.connect('anon');
|
|
133
|
+
await engine.query('UPDATE game:abc MERGE $updates', { updates: { white: 'hikaru' } });
|
|
134
|
+
const write = sql.find((s) => s.startsWith('INSERT INTO "game"'));
|
|
135
|
+
expect(write).toBeDefined();
|
|
136
|
+
expect(write).toContain('json_patch');
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
it('DELETE <table>:<id> deletes one row, DELETE <table> still clears the table', () => {
|
|
140
|
+
expect(translateSurql('DELETE game:abc', {}).ops[0]).toEqual({
|
|
141
|
+
kind: 'delete',
|
|
142
|
+
id: 'game:abc',
|
|
143
|
+
});
|
|
144
|
+
expect(translateSurql('DELETE game', {}).ops[0]).toEqual({ kind: 'deleteAll', table: 'game' });
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
it('refuses a table-wide MERGE rather than writing a row named after the table', () => {
|
|
148
|
+
// `UPDATE game MERGE $x` means every row of `game` in SurrealQL. Unsupported
|
|
149
|
+
// here — and it must SAY so, not silently create `game:undefined`.
|
|
150
|
+
expect(() => translateSurql('UPDATE game MERGE $x', { x: {} })).toThrow(
|
|
151
|
+
/unsupported SurrealQL/
|
|
152
|
+
);
|
|
153
|
+
});
|
|
154
|
+
});
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { openDb } from './sqlite-open';
|
|
3
|
+
|
|
4
|
+
class FakeDb {
|
|
5
|
+
constructor(public arg: unknown) {}
|
|
6
|
+
exec() {
|
|
7
|
+
return [];
|
|
8
|
+
}
|
|
9
|
+
close() {}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** Minimal stand-in for the initialized sqlite-wasm module. `install` is the
|
|
13
|
+
* `installOpfsSAHPoolVfs` behavior under test; omit it to model a build that
|
|
14
|
+
* lacks the SAHPool VFS entirely. */
|
|
15
|
+
function makeSqlite3(install?: (opts: any) => Promise<unknown>) {
|
|
16
|
+
const sqlite3: any = { oo1: { DB: FakeDb } };
|
|
17
|
+
if (install) sqlite3.installOpfsSAHPoolVfs = vi.fn(install);
|
|
18
|
+
return sqlite3;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const pool = { OpfsSAHPoolDb: FakeDb };
|
|
22
|
+
/** What a pool locked by another tab of the app actually throws. */
|
|
23
|
+
function lockedError(): Error {
|
|
24
|
+
const e = new Error('Access Handles cannot be acquired');
|
|
25
|
+
e.name = 'NoModificationAllowedError';
|
|
26
|
+
return e;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const noSleep = () => Promise.resolve();
|
|
30
|
+
|
|
31
|
+
let errSpy: ReturnType<typeof vi.spyOn>;
|
|
32
|
+
beforeEach(() => {
|
|
33
|
+
errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
34
|
+
});
|
|
35
|
+
afterEach(() => {
|
|
36
|
+
errSpy.mockRestore();
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
// The bare `catch {}` this replaces turned every OPFS failure into a silent
|
|
40
|
+
// full-RAM database: no log, no reason, no way for the app to know its writes
|
|
41
|
+
// die on reload. Each case below pins one half of the fix: keep trying when
|
|
42
|
+
// retrying can plausibly work, and when it can't, say so loudly and hand the
|
|
43
|
+
// reason back.
|
|
44
|
+
describe('openDb', () => {
|
|
45
|
+
it('opens the OPFS pool on the first attempt', async () => {
|
|
46
|
+
const sqlite3 = makeSqlite3(async () => pool);
|
|
47
|
+
const res = await openDb(sqlite3, 'user:abc', true, { sleep: noSleep });
|
|
48
|
+
|
|
49
|
+
expect(res.persisted).toBe(true);
|
|
50
|
+
expect(res.opfsError).toBeUndefined();
|
|
51
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
|
|
52
|
+
// The pool is named per bucket, and the first try must NOT force a re-init
|
|
53
|
+
// (that would throw away a pool another attempt is legitimately using).
|
|
54
|
+
expect(sqlite3.installOpfsSAHPoolVfs.mock.calls[0][0]).toEqual({ name: 'sp00ky-user:abc' });
|
|
55
|
+
expect(errSpy).not.toHaveBeenCalled();
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
// The tab-closing race: the old tab still holds the sync access handles when
|
|
59
|
+
// the new one boots. Without a retry that tab is stuck in RAM for its whole
|
|
60
|
+
// lifetime, even though the lock frees milliseconds later.
|
|
61
|
+
it('retries a locked pool and succeeds, forcing re-init after the first failure', async () => {
|
|
62
|
+
let calls = 0;
|
|
63
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
64
|
+
if (++calls < 3) throw lockedError();
|
|
65
|
+
return pool;
|
|
66
|
+
});
|
|
67
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
68
|
+
|
|
69
|
+
expect(res.persisted).toBe(true);
|
|
70
|
+
expect(res.opfsError).toBeUndefined();
|
|
71
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
|
|
72
|
+
// sqlite-wasm caches the first rejection against the VFS name, so retries
|
|
73
|
+
// that don't ask for a real re-init just replay it.
|
|
74
|
+
const [first, second, third] = sqlite3.installOpfsSAHPoolVfs.mock.calls.map((c: any[]) => c[0]);
|
|
75
|
+
expect(first.forceReinitIfPreviouslyFailed).toBeUndefined();
|
|
76
|
+
expect(second.forceReinitIfPreviouslyFailed).toBe(true);
|
|
77
|
+
expect(third.forceReinitIfPreviouslyFailed).toBe(true);
|
|
78
|
+
expect(errSpy).not.toHaveBeenCalled();
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('falls back loudly after exhausting the retries, keeping the reason', async () => {
|
|
82
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
83
|
+
throw lockedError();
|
|
84
|
+
});
|
|
85
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
86
|
+
|
|
87
|
+
expect(res.persisted).toBe(false);
|
|
88
|
+
// The DOMException name is the diagnostic part: it names the lock.
|
|
89
|
+
expect(res.opfsError).toContain('NoModificationAllowedError');
|
|
90
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
|
|
91
|
+
expect(errSpy).toHaveBeenCalledTimes(1);
|
|
92
|
+
expect(String(errSpy.mock.calls[0][0])).toContain('IN MEMORY');
|
|
93
|
+
// Still a usable handle: losing durability must not break the app.
|
|
94
|
+
expect(res.db).toBeDefined();
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
// An insecure context has no sync access handles at all, so retrying just
|
|
98
|
+
// adds boot latency to a foregone conclusion.
|
|
99
|
+
it('does not retry when the OPFS APIs are missing entirely', async () => {
|
|
100
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
101
|
+
throw new Error('Missing required OPFS APIs.');
|
|
102
|
+
});
|
|
103
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
104
|
+
|
|
105
|
+
expect(res.persisted).toBe(false);
|
|
106
|
+
expect(res.opfsError).toContain('Missing required OPFS APIs');
|
|
107
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
|
|
108
|
+
expect(errSpy).toHaveBeenCalledTimes(1);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('reports a build without the SAHPool VFS without calling anything', async () => {
|
|
112
|
+
const sqlite3 = makeSqlite3();
|
|
113
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
114
|
+
|
|
115
|
+
expect(res.persisted).toBe(false);
|
|
116
|
+
expect(res.opfsError).toContain('installOpfsSAHPoolVfs');
|
|
117
|
+
expect(errSpy).toHaveBeenCalledTimes(1);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
// `store: 'memory'` is a configuration choice, not a degradation, so it must
|
|
121
|
+
// stay silent and carry no error for the UI to warn about.
|
|
122
|
+
it('opens in memory quietly when persistence was not requested', async () => {
|
|
123
|
+
const sqlite3 = makeSqlite3(async () => pool);
|
|
124
|
+
const res = await openDb(sqlite3, 'main', false, { sleep: noSleep });
|
|
125
|
+
|
|
126
|
+
expect(res.persisted).toBe(false);
|
|
127
|
+
expect(res.opfsError).toBeUndefined();
|
|
128
|
+
expect(sqlite3.installOpfsSAHPoolVfs).not.toHaveBeenCalled();
|
|
129
|
+
expect(errSpy).not.toHaveBeenCalled();
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('honors maxAttempts and waits the configured backoff between tries', async () => {
|
|
133
|
+
const slept: number[] = [];
|
|
134
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
135
|
+
throw lockedError();
|
|
136
|
+
});
|
|
137
|
+
const res = await openDb(sqlite3, 'main', true, {
|
|
138
|
+
maxAttempts: 4,
|
|
139
|
+
backoffMs: [10, 20],
|
|
140
|
+
sleep: async (ms) => {
|
|
141
|
+
slept.push(ms);
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
expect(res.persisted).toBe(false);
|
|
146
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(4);
|
|
147
|
+
// One sleep per gap (never after the last attempt), last delay repeating.
|
|
148
|
+
expect(slept).toEqual([10, 20, 20]);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Opening the worker's SQLite handle: the OPFS SAHPool VFS when durable
|
|
3
|
+
* storage was asked for, an in-memory DB only as a last resort. Extracted from
|
|
4
|
+
* `sqlite-worker.ts` (which imports the wasm module at module scope and so
|
|
5
|
+
* can't be loaded in a unit test) to keep the retry/fallback policy testable
|
|
6
|
+
* off-worker, the same split as `sqlite-select.ts`.
|
|
7
|
+
*
|
|
8
|
+
* Why retry: SAHPool holds an EXCLUSIVE sync access handle on every file in
|
|
9
|
+
* its pool, so only one client per pool name can have it open. A second tab of
|
|
10
|
+
* the same app therefore fails init, and `installOpfsSAHPoolVfs` CACHES that
|
|
11
|
+
* rejection per VFS name, so a later call only gets a real second chance when
|
|
12
|
+
* it passes `forceReinitIfPreviouslyFailed`. Retrying with that flag turns the
|
|
13
|
+
* common "the other tab is still closing" race into a success instead of a
|
|
14
|
+
* permanent in-memory session.
|
|
15
|
+
*
|
|
16
|
+
* Why the noise: `:memory:` holds the whole dataset in RAM (the
|
|
17
|
+
* OOM-on-wasm-heavy-pages failure mode the OPFS store exists to avoid) and
|
|
18
|
+
* drops every local write on reload. Host apps run pino at their own level,
|
|
19
|
+
* some at `fatal`, so the fallback ALSO writes to `console.error` from inside
|
|
20
|
+
* the worker, and the reason travels back to the engine as `opfsError` for the
|
|
21
|
+
* app to surface.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** The DB surface the worker uses (a `sqlite3.oo1.DB` or an `OpfsSAHPoolDb`). */
|
|
25
|
+
export interface SqliteDbHandle {
|
|
26
|
+
exec: (opts: { sql: string; bind?: unknown[]; rowMode?: string; returnValue?: string }) => unknown;
|
|
27
|
+
close: () => void;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** The slice of the OpfsSAHPoolUtil the worker needs for teardown. */
|
|
31
|
+
export interface SqlitePoolHandle {
|
|
32
|
+
/** Unregisters the VFS and releases every sync access handle, leaving the
|
|
33
|
+
* files intact, so another worker can open the pool without waiting for
|
|
34
|
+
* this worker to be garbage collected. Throws while files are open. */
|
|
35
|
+
pauseVfs?: () => unknown;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface OpenDbResult {
|
|
39
|
+
db: SqliteDbHandle;
|
|
40
|
+
/** True only when the handle is backed by OPFS and survives a reload. */
|
|
41
|
+
persisted: boolean;
|
|
42
|
+
/** Why persistence failed. Set only when OPFS was requested and fell back. */
|
|
43
|
+
opfsError?: string;
|
|
44
|
+
/** Pool util, present only for an OPFS-backed handle. */
|
|
45
|
+
pool?: SqlitePoolHandle;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface OpenDbOptions {
|
|
49
|
+
/** Total OPFS init attempts, including the first. Default 3. */
|
|
50
|
+
maxAttempts?: number;
|
|
51
|
+
/** Delay before each retry; the last entry repeats. Default [250, 500]. */
|
|
52
|
+
backoffMs?: number[];
|
|
53
|
+
/**
|
|
54
|
+
* Throw (`opfs-unavailable: <reason>`) instead of falling back to memory.
|
|
55
|
+
* Used by shared-tabs leader promotion: a silently-in-memory LEADER would
|
|
56
|
+
* put every tab's data in RAM, so promotion prefers failing the election
|
|
57
|
+
* (the broker retries, possibly on another tab) over degrading. The broker
|
|
58
|
+
* grants an explicit memory fallback only after repeated failed cycles.
|
|
59
|
+
*/
|
|
60
|
+
disallowMemoryFallback?: boolean;
|
|
61
|
+
/** Injectable for tests. */
|
|
62
|
+
sleep?: (ms: number) => Promise<void>;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
66
|
+
/** Bounded on purpose: this runs on the boot path, before the first query. */
|
|
67
|
+
const DEFAULT_BACKOFF_MS = [250, 500];
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Leader-promotion profile (~5.8s worst case). A dead leader's sync access
|
|
71
|
+
* handles release when the browser garbage-collects its worker, typically
|
|
72
|
+
* well under a second but not synchronously with the Web Lock release the
|
|
73
|
+
* election observed, so promotion retries longer than a cold boot.
|
|
74
|
+
*/
|
|
75
|
+
export const PROMOTION_OPEN_OPTIONS: Pick<OpenDbOptions, 'maxAttempts' | 'backoffMs'> = {
|
|
76
|
+
maxAttempts: 10,
|
|
77
|
+
backoffMs: [50, 100, 200, 400, 800, 1000],
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
/** Failures no retry can fix: the APIs aren't there at all (insecure context,
|
|
81
|
+
* or a browser without sync access handles). Fall back immediately. */
|
|
82
|
+
const UNRETRYABLE = ['Missing required OPFS APIs'];
|
|
83
|
+
|
|
84
|
+
const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
85
|
+
|
|
86
|
+
/** Keep the DOMException name (e.g. `NoModificationAllowedError` for a pool
|
|
87
|
+
* locked by another tab): it is the most diagnostic part of the failure. */
|
|
88
|
+
function errMessage(e: unknown): string {
|
|
89
|
+
if (e instanceof Error) return e.name && e.name !== 'Error' ? `${e.name}: ${e.message}` : e.message;
|
|
90
|
+
return String(e);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function fallbackToMemory(
|
|
94
|
+
sqlite3: any,
|
|
95
|
+
dbName: string,
|
|
96
|
+
reason: string,
|
|
97
|
+
attempts: number
|
|
98
|
+
): OpenDbResult {
|
|
99
|
+
const tried = attempts > 0 ? ` after ${attempts} attempt${attempts === 1 ? '' : 's'}` : '';
|
|
100
|
+
// Deliberately console, not the logger: host apps configure pino's level (some
|
|
101
|
+
// run `fatal`), and losing durability must never be filtered into silence.
|
|
102
|
+
// oxlint-disable-next-line no-console
|
|
103
|
+
console.error(
|
|
104
|
+
`[sp00ky] OPFS persistence unavailable for "${dbName}"${tried}: ${reason}. The local SQLite ` +
|
|
105
|
+
'cache is running IN MEMORY, which keeps the whole dataset in RAM and loses every local ' +
|
|
106
|
+
'write on reload. The usual cause is another tab of this app holding the storage lock, so ' +
|
|
107
|
+
'closing the other tabs and reloading restores persistence.'
|
|
108
|
+
);
|
|
109
|
+
return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false, opfsError: reason };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Open `dbName`'s handle. Never throws for a storage problem: a caller that
|
|
114
|
+
* asked for persistence and can't have it gets a working in-memory handle plus
|
|
115
|
+
* `persisted: false` and an `opfsError` to report.
|
|
116
|
+
*/
|
|
117
|
+
export async function openDb(
|
|
118
|
+
sqlite3: any,
|
|
119
|
+
dbName: string,
|
|
120
|
+
useOpfs: boolean,
|
|
121
|
+
opts: OpenDbOptions = {}
|
|
122
|
+
): Promise<OpenDbResult> {
|
|
123
|
+
// Memory was the configured choice (`store: 'memory'`), not a failure, so no
|
|
124
|
+
// error and no noise.
|
|
125
|
+
if (!useOpfs) return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false };
|
|
126
|
+
|
|
127
|
+
if (!sqlite3.installOpfsSAHPoolVfs) {
|
|
128
|
+
if (opts.disallowMemoryFallback) {
|
|
129
|
+
throw new Error('opfs-unavailable: sqlite-wasm build has no installOpfsSAHPoolVfs');
|
|
130
|
+
}
|
|
131
|
+
return fallbackToMemory(sqlite3, dbName, 'sqlite-wasm build has no installOpfsSAHPoolVfs', 0);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const maxAttempts = Math.max(1, opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
|
|
135
|
+
const backoffMs = opts.backoffMs ?? DEFAULT_BACKOFF_MS;
|
|
136
|
+
const sleep = opts.sleep ?? defaultSleep;
|
|
137
|
+
|
|
138
|
+
let lastError = 'unknown error';
|
|
139
|
+
let attempts = 0;
|
|
140
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
141
|
+
attempts = attempt;
|
|
142
|
+
try {
|
|
143
|
+
// `initialCapacity` stays at the sqlite-wasm default (6 files): one pool
|
|
144
|
+
// per bucket holds a single DB plus its journals, so preallocating more
|
|
145
|
+
// OPFS files would only be waste. A "SAH pool is full" error still
|
|
146
|
+
// reaches the caller verbatim via `opfsError`.
|
|
147
|
+
const pool = await sqlite3.installOpfsSAHPoolVfs({
|
|
148
|
+
name: `sp00ky-${dbName}`,
|
|
149
|
+
// The first failure is cached against the VFS name, so a retry that
|
|
150
|
+
// doesn't ask for a real re-init just replays the same rejection.
|
|
151
|
+
...(attempt > 1 ? { forceReinitIfPreviouslyFailed: true } : {}),
|
|
152
|
+
});
|
|
153
|
+
return { db: new pool.OpfsSAHPoolDb(`/${dbName}.sqlite3`), persisted: true, pool };
|
|
154
|
+
} catch (e) {
|
|
155
|
+
lastError = errMessage(e);
|
|
156
|
+
if (attempt === maxAttempts || UNRETRYABLE.some((m) => lastError.includes(m))) break;
|
|
157
|
+
await sleep(backoffMs[Math.min(attempt - 1, backoffMs.length - 1)] ?? 0);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
if (opts.disallowMemoryFallback) {
|
|
161
|
+
throw new Error(`opfs-unavailable: ${lastError} (after ${attempts} attempts)`);
|
|
162
|
+
}
|
|
163
|
+
return fallbackToMemory(sqlite3, dbName, lastError, attempts);
|
|
164
|
+
}
|