@spooky-sync/core 0.0.1-canary.20 → 0.0.1-canary.201

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/AGENTS.md +57 -0
  2. package/dist/index.d.ts +2184 -54
  3. package/dist/index.js +11515 -2399
  4. package/dist/otel/index.d.ts +2 -2
  5. package/dist/otel/index.js +6 -6
  6. package/dist/sqlite-open.js +276 -0
  7. package/dist/sqlite-worker.d.ts +1 -0
  8. package/dist/sqlite-worker.js +421 -0
  9. package/dist/tabs-broker-worker.d.ts +8 -0
  10. package/dist/tabs-broker-worker.js +434 -0
  11. package/dist/types.d.ts +688 -11
  12. package/package.json +11 -7
  13. package/scripts/check-broker-bundle.mjs +33 -0
  14. package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
  15. package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
  16. package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
  17. package/src/bucket-blurhash.test.ts +148 -0
  18. package/src/build-globals.d.ts +12 -0
  19. package/src/events/events.test.ts +2 -1
  20. package/src/events/index.ts +3 -0
  21. package/src/index.ts +35 -2
  22. package/src/modules/app-release/index.test.ts +125 -0
  23. package/src/modules/app-release/index.ts +201 -0
  24. package/src/modules/auth/events/index.ts +2 -1
  25. package/src/modules/auth/index.ts +59 -20
  26. package/src/modules/cache/index.ts +112 -32
  27. package/src/modules/cache/types.ts +2 -2
  28. package/src/modules/crdt/crdt-field.ts +294 -0
  29. package/src/modules/crdt/crdt-hydration.test.ts +210 -0
  30. package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
  31. package/src/modules/crdt/index.ts +463 -0
  32. package/src/modules/crdt/loro-loader.ts +25 -0
  33. package/src/modules/data/data.hydration.test.ts +142 -0
  34. package/src/modules/data/data.membership.test.ts +462 -0
  35. package/src/modules/data/data.rebind.test.ts +147 -0
  36. package/src/modules/data/data.run.test.ts +113 -0
  37. package/src/modules/data/data.settled-writes.test.ts +206 -0
  38. package/src/modules/data/data.status.test.ts +249 -0
  39. package/src/modules/data/id-set-plan.test.ts +122 -0
  40. package/src/modules/data/index.ts +1580 -130
  41. package/src/modules/data/mutation-id.test.ts +25 -0
  42. package/src/modules/data/mutation-id.ts +35 -0
  43. package/src/modules/data/window-query.test.ts +52 -0
  44. package/src/modules/data/window-query.ts +194 -0
  45. package/src/modules/devtools/flags.ts +349 -0
  46. package/src/modules/devtools/index.ts +386 -37
  47. package/src/modules/devtools/notify-throttle.test.ts +149 -0
  48. package/src/modules/devtools/storage-info.test.ts +79 -0
  49. package/src/modules/devtools/storage-info.ts +168 -0
  50. package/src/modules/devtools/versions.test.ts +74 -0
  51. package/src/modules/devtools/versions.ts +110 -0
  52. package/src/modules/feature-flag/index.test.ts +251 -0
  53. package/src/modules/feature-flag/index.ts +308 -0
  54. package/src/modules/ref-tables.test.ts +91 -0
  55. package/src/modules/ref-tables.ts +88 -0
  56. package/src/modules/sync/engine.ts +101 -37
  57. package/src/modules/sync/events/index.ts +9 -2
  58. package/src/modules/sync/queue/queue-down.test.ts +107 -0
  59. package/src/modules/sync/queue/queue-down.ts +35 -6
  60. package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
  61. package/src/modules/sync/queue/queue-up.ts +241 -57
  62. package/src/modules/sync/scheduler.pause.test.ts +109 -0
  63. package/src/modules/sync/scheduler.retry.test.ts +156 -0
  64. package/src/modules/sync/scheduler.ts +158 -11
  65. package/src/modules/sync/sync.cleanup.test.ts +116 -0
  66. package/src/modules/sync/sync.health.test.ts +149 -0
  67. package/src/modules/sync/sync.heartbeat.test.ts +80 -0
  68. package/src/modules/sync/sync.live-removal.test.ts +134 -0
  69. package/src/modules/sync/sync.reconnect.test.ts +145 -0
  70. package/src/modules/sync/sync.subquery.test.ts +82 -0
  71. package/src/modules/sync/sync.ts +1558 -99
  72. package/src/modules/sync/utils.test.ts +269 -2
  73. package/src/modules/sync/utils.ts +201 -17
  74. package/src/otel/index.ts +13 -10
  75. package/src/services/blobs/blob-cache.test.ts +359 -0
  76. package/src/services/blobs/blob-cache.ts +603 -0
  77. package/src/services/blobs/blob-manifest.ts +227 -0
  78. package/src/services/blobs/blob-store.test.ts +77 -0
  79. package/src/services/blobs/blob-store.ts +359 -0
  80. package/src/services/blobs/blob.fixture.ts +90 -0
  81. package/src/services/blobs/index.ts +70 -0
  82. package/src/services/database/cache-engine.ts +160 -0
  83. package/src/services/database/connection-supervisor.test.ts +289 -0
  84. package/src/services/database/connection-supervisor.ts +415 -0
  85. package/src/services/database/database.query-timeout.test.ts +83 -0
  86. package/src/services/database/database.ts +32 -12
  87. package/src/services/database/engine-factory.ts +33 -0
  88. package/src/services/database/events/index.ts +2 -1
  89. package/src/services/database/index.ts +7 -0
  90. package/src/services/database/local-migrator.ts +30 -27
  91. package/src/services/database/local.test.ts +64 -0
  92. package/src/services/database/local.ts +478 -67
  93. package/src/services/database/plan-render.test.ts +159 -0
  94. package/src/services/database/plan-render.ts +108 -0
  95. package/src/services/database/relation-resolver.test.ts +413 -0
  96. package/src/services/database/relation-resolver.ts +0 -0
  97. package/src/services/database/remote.ts +110 -14
  98. package/src/services/database/sqlite-cache-engine.test.ts +558 -0
  99. package/src/services/database/sqlite-cache-engine.ts +1257 -0
  100. package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
  101. package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
  102. package/src/services/database/sqlite-open.test.ts +150 -0
  103. package/src/services/database/sqlite-open.ts +164 -0
  104. package/src/services/database/sqlite-plan-sql.test.ts +104 -0
  105. package/src/services/database/sqlite-plan-sql.ts +106 -0
  106. package/src/services/database/sqlite-select.integration.test.ts +185 -0
  107. package/src/services/database/sqlite-select.test.ts +246 -0
  108. package/src/services/database/sqlite-select.ts +121 -0
  109. package/src/services/database/sqlite-transport.fixture.ts +30 -0
  110. package/src/services/database/sqlite-transport.ts +221 -0
  111. package/src/services/database/sqlite-worker.ts +437 -0
  112. package/src/services/database/surql-translate.ts +416 -0
  113. package/src/services/database/surreal-cache-engine.ts +141 -0
  114. package/src/services/logger/index.ts +3 -2
  115. package/src/services/persistence/localstorage.ts +2 -2
  116. package/src/services/persistence/resilient.ts +11 -4
  117. package/src/services/persistence/surrealdb.ts +10 -10
  118. package/src/services/stream-processor/index.ts +444 -52
  119. package/src/services/stream-processor/permissions.test.ts +47 -0
  120. package/src/services/stream-processor/permissions.ts +53 -0
  121. package/src/services/stream-processor/stream-processor.batch.test.ts +136 -0
  122. package/src/services/stream-processor/stream-processor.reset.test.ts +216 -0
  123. package/src/services/stream-processor/stream-processor.test.ts +1 -1
  124. package/src/services/stream-processor/wasm-types.ts +23 -2
  125. package/src/services/tabs/broker-client.ts +283 -0
  126. package/src/services/tabs/broker.test.ts +278 -0
  127. package/src/services/tabs/coordinator.test.ts +244 -0
  128. package/src/services/tabs/coordinator.ts +576 -0
  129. package/src/services/tabs/fake-ports.fixture.ts +112 -0
  130. package/src/services/tabs/leader-locks.ts +75 -0
  131. package/src/services/tabs/protocol.ts +242 -0
  132. package/src/services/tabs/support.ts +36 -0
  133. package/src/services/tabs/tabs-broker-worker.ts +586 -0
  134. package/src/sp00ky.auth-order.test.ts +92 -0
  135. package/src/sp00ky.init-query.test.ts +183 -0
  136. package/src/sp00ky.ts +1543 -0
  137. package/src/types.ts +496 -13
  138. package/src/utils/blurhash.ts +90 -0
  139. package/src/utils/error-classification.test.ts +44 -0
  140. package/src/utils/error-classification.ts +7 -0
  141. package/src/utils/index.ts +73 -13
  142. package/src/utils/parser.ts +3 -2
  143. package/src/utils/semver.test.ts +32 -0
  144. package/src/utils/semver.ts +30 -0
  145. package/src/utils/surql.ts +30 -18
  146. package/src/utils/withRetry.test.ts +1 -1
  147. package/tsdown.config.ts +86 -1
  148. package/src/spooky.ts +0 -395
@@ -0,0 +1,143 @@
1
+ import { describe, it, expect, beforeAll } from 'vitest';
2
+ import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
3
+ import { SqliteCacheEngine } from './sqlite-cache-engine';
4
+ import { stubTransport } from './sqlite-transport.fixture';
5
+ import { serializeRow } from './sqlite-plan-sql';
6
+ import type { Row } from './cache-engine';
7
+
8
+ /**
9
+ * Integration: the DevTools Database explorer's statements run end-to-end
10
+ * (translate → engine SQL → REAL in-memory SQLite, the same
11
+ * @sqlite.org/sqlite-wasm build the worker loads). The unit test asserts the
12
+ * translation; this asserts the SQL actually executes and returns the right
13
+ * rows — the paging window in particular is easy to render into invalid SQLite
14
+ * (there is no bare OFFSET).
15
+ */
16
+
17
+ let db: any;
18
+
19
+ function makeLogger(): any {
20
+ const noop = () => {};
21
+ const l: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
22
+ l.child = () => l;
23
+ return l;
24
+ }
25
+
26
+ /** An engine whose transport is a real SQLite database. */
27
+ function realEngine(): SqliteCacheEngine {
28
+ const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger());
29
+ stubTransport(engine, (type, payload: any) => {
30
+ switch (type) {
31
+ case 'open':
32
+ for (const t of payload.systemTables ?? []) {
33
+ db.exec({
34
+ sql: `CREATE TABLE IF NOT EXISTS "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)`,
35
+ });
36
+ }
37
+ return { persisted: true };
38
+ case 'exec':
39
+ return {
40
+ rows: db.exec({
41
+ sql: payload.sql,
42
+ bind: payload.bind,
43
+ rowMode: 'object',
44
+ returnValue: 'resultRows',
45
+ }),
46
+ };
47
+ case 'run':
48
+ db.exec({ sql: payload.sql, bind: payload.bind });
49
+ return {};
50
+ case 'batch':
51
+ for (const stmt of payload as { sql: string; bind?: unknown[] }[]) {
52
+ db.exec({ sql: stmt.sql, bind: stmt.bind });
53
+ }
54
+ return {};
55
+ default:
56
+ return {};
57
+ }
58
+ });
59
+ return engine;
60
+ }
61
+
62
+ function seed(table: string, rows: Row[]): void {
63
+ db.exec({
64
+ sql: `CREATE TABLE IF NOT EXISTS "${table}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)`,
65
+ });
66
+ for (const row of rows) {
67
+ db.exec({
68
+ sql: `INSERT INTO "${table}"(id, data) VALUES(?, ?)`,
69
+ bind: [row.id, serializeRow(row)],
70
+ });
71
+ }
72
+ }
73
+
74
+ beforeAll(async () => {
75
+ const sqlite3: any = await sqlite3InitModule();
76
+ db = new sqlite3.oo1.DB(':memory:', 'c');
77
+ seed(
78
+ 'game_insight',
79
+ Array.from({ length: 5 }, (_, i) => ({
80
+ id: `game_insight:${i}`,
81
+ n: i,
82
+ tag: i % 2 ? 'odd' : 'even',
83
+ }))
84
+ );
85
+ });
86
+
87
+ describe('DevTools explorer against real SQLite', () => {
88
+ it('pages a table with LIMIT/START', async () => {
89
+ const engine = realEngine();
90
+ await engine.connect('anon');
91
+
92
+ const [page1] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 2 START 0');
93
+ const [page2] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 2 START 2');
94
+ const [tail] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 20 START 4');
95
+
96
+ expect(page1.map((r) => r.id)).toEqual(['game_insight:0', 'game_insight:1']);
97
+ expect(page2.map((r) => r.id)).toEqual(['game_insight:2', 'game_insight:3']);
98
+ expect(tail.map((r) => r.id)).toEqual(['game_insight:4']);
99
+ });
100
+
101
+ it('counts rows, with and without a filter', async () => {
102
+ const engine = realEngine();
103
+ await engine.connect('anon');
104
+
105
+ const [all] = await engine.query<[{ count: number }[]]>(
106
+ 'SELECT count() FROM game_insight GROUP ALL'
107
+ );
108
+ const [odd] = await engine.query<[{ count: number }[]]>(
109
+ "SELECT count() FROM game_insight WHERE tag = 'odd' GROUP ALL"
110
+ );
111
+
112
+ expect(all).toEqual([{ count: 5 }]);
113
+ expect(odd).toEqual([{ count: 2 }]);
114
+ });
115
+
116
+ it('lists tables via INFO FOR DB', async () => {
117
+ const engine = realEngine();
118
+ await engine.connect('anon');
119
+
120
+ const [info] = await engine.query<[{ tables: Record<string, string> }]>('INFO FOR DB');
121
+
122
+ expect(Object.keys(info.tables)).toContain('game_insight');
123
+ // Seeded system tables show up too — the panel's "internal" toggle filters
124
+ // them, the engine must not pre-filter.
125
+ expect(Object.keys(info.tables)).toContain('_00_query');
126
+ expect(Object.keys(info.tables).some((t) => t.startsWith('sqlite_'))).toBe(false);
127
+ });
128
+
129
+ it('edits and deletes a row by literal record id', async () => {
130
+ const engine = realEngine();
131
+ await engine.connect('anon');
132
+
133
+ await engine.query('UPDATE game_insight:1 MERGE $updates', { updates: { tag: 'edited' } });
134
+ const [edited] = await engine.query<[Row[]]>('SELECT * FROM game_insight LIMIT 20 START 1');
135
+ expect(edited[0]).toMatchObject({ id: 'game_insight:1', n: 1, tag: 'edited' });
136
+
137
+ await engine.query('DELETE game_insight:1');
138
+ const [after] = await engine.query<[{ count: number }[]]>(
139
+ 'SELECT count() FROM game_insight GROUP ALL'
140
+ );
141
+ expect(after).toEqual([{ count: 4 }]);
142
+ });
143
+ });
@@ -0,0 +1,154 @@
1
+ import { describe, it, expect } from 'vitest';
2
+ import { SqliteCacheEngine } from './sqlite-cache-engine';
3
+ import { stubTransport } from './sqlite-transport.fixture';
4
+ import { translateSurql } from './surql-translate';
5
+
6
+ /**
7
+ * The statements the DevTools Database explorer emits against the LOCAL store.
8
+ * None of them come from `surql`, so they only ever exercised the SurrealDB
9
+ * engine: on SQLite the paging read threw "unsupported SurrealQL for
10
+ * translation: SELECT * FROM game_insight LIMIT 20 START 0", the table list came
11
+ * back empty (`INFO FOR DB` lowered to a noop) and row edit/delete never
12
+ * matched (they inline a literal record id).
13
+ */
14
+
15
+ function makeLogger(): any {
16
+ const noop = () => {};
17
+ const l: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
18
+ l.child = () => l;
19
+ return l;
20
+ }
21
+
22
+ /** Records every exec/run SQL so the paging window can be asserted. */
23
+ function recordingEngine(rows: Record<string, unknown[]> = {}) {
24
+ const sql: string[] = [];
25
+ const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger());
26
+ stubTransport(engine, (type, payload: any) => {
27
+ if (type === 'open') return { persisted: true };
28
+ if (type === 'exec' || type === 'run') {
29
+ sql.push(payload.sql);
30
+ for (const [needle, out] of Object.entries(rows)) {
31
+ if (payload.sql.includes(needle)) return { rows: out };
32
+ }
33
+ return { rows: [] };
34
+ }
35
+ return {};
36
+ });
37
+ return { engine, sql };
38
+ }
39
+
40
+ describe('DevTools table paging (LIMIT/START)', () => {
41
+ it('translates the window instead of throwing', () => {
42
+ const { ops } = translateSurql('SELECT * FROM game_insight LIMIT 20 START 0', {});
43
+ expect(ops).toEqual([
44
+ {
45
+ kind: 'selectTable',
46
+ table: 'game_insight',
47
+ where: undefined,
48
+ orderBy: undefined,
49
+ select: undefined,
50
+ value: undefined,
51
+ limit: 20,
52
+ start: 0,
53
+ },
54
+ ]);
55
+ });
56
+
57
+ it('renders LIMIT/OFFSET in SQL, page 3 of 20', async () => {
58
+ const { engine, sql } = recordingEngine();
59
+ await engine.connect('anon');
60
+ await engine.query('SELECT * FROM game_insight LIMIT 20 START 40');
61
+ expect(sql.some((s) => /SELECT data FROM "game_insight" LIMIT 20 OFFSET 40$/.test(s))).toBe(
62
+ true
63
+ );
64
+ });
65
+
66
+ it('keeps WHERE and ORDER BY alongside the window', () => {
67
+ const { ops } = translateSurql(
68
+ 'SELECT * FROM game WHERE done = true ORDER BY date desc LIMIT 10 START 30',
69
+ {}
70
+ );
71
+ expect(ops[0]).toMatchObject({
72
+ kind: 'selectTable',
73
+ table: 'game',
74
+ where: [{ field: 'done', op: '=', value: true }],
75
+ orderBy: [['date', 'desc']],
76
+ limit: 10,
77
+ start: 30,
78
+ });
79
+ });
80
+
81
+ it('leaves a LIMIT that is part of a string literal alone', () => {
82
+ const { ops } = translateSurql("SELECT * FROM game WHERE note = 'LIMIT 5'", {});
83
+ expect(ops[0]).toMatchObject({
84
+ kind: 'selectTable',
85
+ where: [{ field: 'note', op: '=', value: 'LIMIT 5' }],
86
+ limit: undefined,
87
+ start: undefined,
88
+ });
89
+ });
90
+
91
+ it('a START with no LIMIT still renders valid SQLite', async () => {
92
+ const { engine, sql } = recordingEngine();
93
+ await engine.connect('anon');
94
+ await engine.query('SELECT * FROM game START 5');
95
+ // SQLite has no bare OFFSET — `LIMIT -1` is its "everything" sentinel.
96
+ expect(sql.some((s) => s.endsWith('LIMIT -1 OFFSET 5'))).toBe(true);
97
+ });
98
+ });
99
+
100
+ describe('DevTools row count', () => {
101
+ it("answers `SELECT count() FROM t GROUP ALL` in SurrealDB's shape", async () => {
102
+ const { engine } = recordingEngine({ 'COUNT(*)': [{ n: 137 }] });
103
+ await engine.connect('anon');
104
+ const res = await engine.query<[{ count: number }[]]>(
105
+ 'SELECT count() FROM game_insight GROUP ALL'
106
+ );
107
+ expect(res[0]).toEqual([{ count: 137 }]);
108
+ });
109
+
110
+ it('counts zero rows as 0, not undefined', async () => {
111
+ const { engine } = recordingEngine();
112
+ await engine.connect('anon');
113
+ const res = await engine.query<[{ count: number }[]]>('SELECT count() FROM empty GROUP ALL');
114
+ expect(res[0]).toEqual([{ count: 0 }]);
115
+ });
116
+ });
117
+
118
+ describe('DevTools table list', () => {
119
+ it('answers INFO FOR DB from sqlite_master, minus SQLite internals', async () => {
120
+ const { engine } = recordingEngine({
121
+ sqlite_master: [{ name: 'game' }, { name: '_00_query' }],
122
+ });
123
+ await engine.connect('anon');
124
+ const [info] = await engine.query<[{ tables: Record<string, string> }]>('INFO FOR DB');
125
+ expect(Object.keys(info.tables)).toEqual(['game', '_00_query']);
126
+ });
127
+ });
128
+
129
+ describe('DevTools row edit / delete (literal record ids)', () => {
130
+ it('UPDATE <table>:<id> MERGE $updates writes that row', async () => {
131
+ const { engine, sql } = recordingEngine();
132
+ await engine.connect('anon');
133
+ await engine.query('UPDATE game:abc MERGE $updates', { updates: { white: 'hikaru' } });
134
+ const write = sql.find((s) => s.startsWith('INSERT INTO "game"'));
135
+ expect(write).toBeDefined();
136
+ expect(write).toContain('json_patch');
137
+ });
138
+
139
+ it('DELETE <table>:<id> deletes one row, DELETE <table> still clears the table', () => {
140
+ expect(translateSurql('DELETE game:abc', {}).ops[0]).toEqual({
141
+ kind: 'delete',
142
+ id: 'game:abc',
143
+ });
144
+ expect(translateSurql('DELETE game', {}).ops[0]).toEqual({ kind: 'deleteAll', table: 'game' });
145
+ });
146
+
147
+ it('refuses a table-wide MERGE rather than writing a row named after the table', () => {
148
+ // `UPDATE game MERGE $x` means every row of `game` in SurrealQL. Unsupported
149
+ // here — and it must SAY so, not silently create `game:undefined`.
150
+ expect(() => translateSurql('UPDATE game MERGE $x', { x: {} })).toThrow(
151
+ /unsupported SurrealQL/
152
+ );
153
+ });
154
+ });
@@ -0,0 +1,150 @@
1
+ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
2
+ import { openDb } from './sqlite-open';
3
+
4
+ class FakeDb {
5
+ constructor(public arg: unknown) {}
6
+ exec() {
7
+ return [];
8
+ }
9
+ close() {}
10
+ }
11
+
12
+ /** Minimal stand-in for the initialized sqlite-wasm module. `install` is the
13
+ * `installOpfsSAHPoolVfs` behavior under test; omit it to model a build that
14
+ * lacks the SAHPool VFS entirely. */
15
+ function makeSqlite3(install?: (opts: any) => Promise<unknown>) {
16
+ const sqlite3: any = { oo1: { DB: FakeDb } };
17
+ if (install) sqlite3.installOpfsSAHPoolVfs = vi.fn(install);
18
+ return sqlite3;
19
+ }
20
+
21
+ const pool = { OpfsSAHPoolDb: FakeDb };
22
+ /** What a pool locked by another tab of the app actually throws. */
23
+ function lockedError(): Error {
24
+ const e = new Error('Access Handles cannot be acquired');
25
+ e.name = 'NoModificationAllowedError';
26
+ return e;
27
+ }
28
+
29
+ const noSleep = () => Promise.resolve();
30
+
31
+ let errSpy: ReturnType<typeof vi.spyOn>;
32
+ beforeEach(() => {
33
+ errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
34
+ });
35
+ afterEach(() => {
36
+ errSpy.mockRestore();
37
+ });
38
+
39
+ // The bare `catch {}` this replaces turned every OPFS failure into a silent
40
+ // full-RAM database: no log, no reason, no way for the app to know its writes
41
+ // die on reload. Each case below pins one half of the fix: keep trying when
42
+ // retrying can plausibly work, and when it can't, say so loudly and hand the
43
+ // reason back.
44
+ describe('openDb', () => {
45
+ it('opens the OPFS pool on the first attempt', async () => {
46
+ const sqlite3 = makeSqlite3(async () => pool);
47
+ const res = await openDb(sqlite3, 'user:abc', true, { sleep: noSleep });
48
+
49
+ expect(res.persisted).toBe(true);
50
+ expect(res.opfsError).toBeUndefined();
51
+ expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
52
+ // The pool is named per bucket, and the first try must NOT force a re-init
53
+ // (that would throw away a pool another attempt is legitimately using).
54
+ expect(sqlite3.installOpfsSAHPoolVfs.mock.calls[0][0]).toEqual({ name: 'sp00ky-user:abc' });
55
+ expect(errSpy).not.toHaveBeenCalled();
56
+ });
57
+
58
+ // The tab-closing race: the old tab still holds the sync access handles when
59
+ // the new one boots. Without a retry that tab is stuck in RAM for its whole
60
+ // lifetime, even though the lock frees milliseconds later.
61
+ it('retries a locked pool and succeeds, forcing re-init after the first failure', async () => {
62
+ let calls = 0;
63
+ const sqlite3 = makeSqlite3(async () => {
64
+ if (++calls < 3) throw lockedError();
65
+ return pool;
66
+ });
67
+ const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
68
+
69
+ expect(res.persisted).toBe(true);
70
+ expect(res.opfsError).toBeUndefined();
71
+ expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
72
+ // sqlite-wasm caches the first rejection against the VFS name, so retries
73
+ // that don't ask for a real re-init just replay it.
74
+ const [first, second, third] = sqlite3.installOpfsSAHPoolVfs.mock.calls.map((c: any[]) => c[0]);
75
+ expect(first.forceReinitIfPreviouslyFailed).toBeUndefined();
76
+ expect(second.forceReinitIfPreviouslyFailed).toBe(true);
77
+ expect(third.forceReinitIfPreviouslyFailed).toBe(true);
78
+ expect(errSpy).not.toHaveBeenCalled();
79
+ });
80
+
81
+ it('falls back loudly after exhausting the retries, keeping the reason', async () => {
82
+ const sqlite3 = makeSqlite3(async () => {
83
+ throw lockedError();
84
+ });
85
+ const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
86
+
87
+ expect(res.persisted).toBe(false);
88
+ // The DOMException name is the diagnostic part: it names the lock.
89
+ expect(res.opfsError).toContain('NoModificationAllowedError');
90
+ expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
91
+ expect(errSpy).toHaveBeenCalledTimes(1);
92
+ expect(String(errSpy.mock.calls[0][0])).toContain('IN MEMORY');
93
+ // Still a usable handle: losing durability must not break the app.
94
+ expect(res.db).toBeDefined();
95
+ });
96
+
97
+ // An insecure context has no sync access handles at all, so retrying just
98
+ // adds boot latency to a foregone conclusion.
99
+ it('does not retry when the OPFS APIs are missing entirely', async () => {
100
+ const sqlite3 = makeSqlite3(async () => {
101
+ throw new Error('Missing required OPFS APIs.');
102
+ });
103
+ const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
104
+
105
+ expect(res.persisted).toBe(false);
106
+ expect(res.opfsError).toContain('Missing required OPFS APIs');
107
+ expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
108
+ expect(errSpy).toHaveBeenCalledTimes(1);
109
+ });
110
+
111
+ it('reports a build without the SAHPool VFS without calling anything', async () => {
112
+ const sqlite3 = makeSqlite3();
113
+ const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
114
+
115
+ expect(res.persisted).toBe(false);
116
+ expect(res.opfsError).toContain('installOpfsSAHPoolVfs');
117
+ expect(errSpy).toHaveBeenCalledTimes(1);
118
+ });
119
+
120
+ // `store: 'memory'` is a configuration choice, not a degradation, so it must
121
+ // stay silent and carry no error for the UI to warn about.
122
+ it('opens in memory quietly when persistence was not requested', async () => {
123
+ const sqlite3 = makeSqlite3(async () => pool);
124
+ const res = await openDb(sqlite3, 'main', false, { sleep: noSleep });
125
+
126
+ expect(res.persisted).toBe(false);
127
+ expect(res.opfsError).toBeUndefined();
128
+ expect(sqlite3.installOpfsSAHPoolVfs).not.toHaveBeenCalled();
129
+ expect(errSpy).not.toHaveBeenCalled();
130
+ });
131
+
132
+ it('honors maxAttempts and waits the configured backoff between tries', async () => {
133
+ const slept: number[] = [];
134
+ const sqlite3 = makeSqlite3(async () => {
135
+ throw lockedError();
136
+ });
137
+ const res = await openDb(sqlite3, 'main', true, {
138
+ maxAttempts: 4,
139
+ backoffMs: [10, 20],
140
+ sleep: async (ms) => {
141
+ slept.push(ms);
142
+ },
143
+ });
144
+
145
+ expect(res.persisted).toBe(false);
146
+ expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(4);
147
+ // One sleep per gap (never after the last attempt), last delay repeating.
148
+ expect(slept).toEqual([10, 20, 20]);
149
+ });
150
+ });
@@ -0,0 +1,164 @@
1
+ /**
2
+ * Opening the worker's SQLite handle: the OPFS SAHPool VFS when durable
3
+ * storage was asked for, an in-memory DB only as a last resort. Extracted from
4
+ * `sqlite-worker.ts` (which imports the wasm module at module scope and so
5
+ * can't be loaded in a unit test) to keep the retry/fallback policy testable
6
+ * off-worker, the same split as `sqlite-select.ts`.
7
+ *
8
+ * Why retry: SAHPool holds an EXCLUSIVE sync access handle on every file in
9
+ * its pool, so only one client per pool name can have it open. A second tab of
10
+ * the same app therefore fails init, and `installOpfsSAHPoolVfs` CACHES that
11
+ * rejection per VFS name, so a later call only gets a real second chance when
12
+ * it passes `forceReinitIfPreviouslyFailed`. Retrying with that flag turns the
13
+ * common "the other tab is still closing" race into a success instead of a
14
+ * permanent in-memory session.
15
+ *
16
+ * Why the noise: `:memory:` holds the whole dataset in RAM (the
17
+ * OOM-on-wasm-heavy-pages failure mode the OPFS store exists to avoid) and
18
+ * drops every local write on reload. Host apps run pino at their own level,
19
+ * some at `fatal`, so the fallback ALSO writes to `console.error` from inside
20
+ * the worker, and the reason travels back to the engine as `opfsError` for the
21
+ * app to surface.
22
+ */
23
+
24
+ /** The DB surface the worker uses (a `sqlite3.oo1.DB` or an `OpfsSAHPoolDb`). */
25
+ export interface SqliteDbHandle {
26
+ exec: (opts: { sql: string; bind?: unknown[]; rowMode?: string; returnValue?: string }) => unknown;
27
+ close: () => void;
28
+ }
29
+
30
+ /** The slice of the OpfsSAHPoolUtil the worker needs for teardown. */
31
+ export interface SqlitePoolHandle {
32
+ /** Unregisters the VFS and releases every sync access handle, leaving the
33
+ * files intact, so another worker can open the pool without waiting for
34
+ * this worker to be garbage collected. Throws while files are open. */
35
+ pauseVfs?: () => unknown;
36
+ }
37
+
38
+ export interface OpenDbResult {
39
+ db: SqliteDbHandle;
40
+ /** True only when the handle is backed by OPFS and survives a reload. */
41
+ persisted: boolean;
42
+ /** Why persistence failed. Set only when OPFS was requested and fell back. */
43
+ opfsError?: string;
44
+ /** Pool util, present only for an OPFS-backed handle. */
45
+ pool?: SqlitePoolHandle;
46
+ }
47
+
48
+ export interface OpenDbOptions {
49
+ /** Total OPFS init attempts, including the first. Default 3. */
50
+ maxAttempts?: number;
51
+ /** Delay before each retry; the last entry repeats. Default [250, 500]. */
52
+ backoffMs?: number[];
53
+ /**
54
+ * Throw (`opfs-unavailable: <reason>`) instead of falling back to memory.
55
+ * Used by shared-tabs leader promotion: a silently-in-memory LEADER would
56
+ * put every tab's data in RAM, so promotion prefers failing the election
57
+ * (the broker retries, possibly on another tab) over degrading. The broker
58
+ * grants an explicit memory fallback only after repeated failed cycles.
59
+ */
60
+ disallowMemoryFallback?: boolean;
61
+ /** Injectable for tests. */
62
+ sleep?: (ms: number) => Promise<void>;
63
+ }
64
+
65
+ const DEFAULT_MAX_ATTEMPTS = 3;
66
+ /** Bounded on purpose: this runs on the boot path, before the first query. */
67
+ const DEFAULT_BACKOFF_MS = [250, 500];
68
+
69
+ /**
70
+ * Leader-promotion profile (~5.8s worst case). A dead leader's sync access
71
+ * handles release when the browser garbage-collects its worker, typically
72
+ * well under a second but not synchronously with the Web Lock release the
73
+ * election observed, so promotion retries longer than a cold boot.
74
+ */
75
+ export const PROMOTION_OPEN_OPTIONS: Pick<OpenDbOptions, 'maxAttempts' | 'backoffMs'> = {
76
+ maxAttempts: 10,
77
+ backoffMs: [50, 100, 200, 400, 800, 1000],
78
+ };
79
+
80
+ /** Failures no retry can fix: the APIs aren't there at all (insecure context,
81
+ * or a browser without sync access handles). Fall back immediately. */
82
+ const UNRETRYABLE = ['Missing required OPFS APIs'];
83
+
84
+ const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
85
+
86
+ /** Keep the DOMException name (e.g. `NoModificationAllowedError` for a pool
87
+ * locked by another tab): it is the most diagnostic part of the failure. */
88
+ function errMessage(e: unknown): string {
89
+ if (e instanceof Error) return e.name && e.name !== 'Error' ? `${e.name}: ${e.message}` : e.message;
90
+ return String(e);
91
+ }
92
+
93
+ function fallbackToMemory(
94
+ sqlite3: any,
95
+ dbName: string,
96
+ reason: string,
97
+ attempts: number
98
+ ): OpenDbResult {
99
+ const tried = attempts > 0 ? ` after ${attempts} attempt${attempts === 1 ? '' : 's'}` : '';
100
+ // Deliberately console, not the logger: host apps configure pino's level (some
101
+ // run `fatal`), and losing durability must never be filtered into silence.
102
+ // oxlint-disable-next-line no-console
103
+ console.error(
104
+ `[sp00ky] OPFS persistence unavailable for "${dbName}"${tried}: ${reason}. The local SQLite ` +
105
+ 'cache is running IN MEMORY, which keeps the whole dataset in RAM and loses every local ' +
106
+ 'write on reload. The usual cause is another tab of this app holding the storage lock, so ' +
107
+ 'closing the other tabs and reloading restores persistence.'
108
+ );
109
+ return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false, opfsError: reason };
110
+ }
111
+
112
+ /**
113
+ * Open `dbName`'s handle. Never throws for a storage problem: a caller that
114
+ * asked for persistence and can't have it gets a working in-memory handle plus
115
+ * `persisted: false` and an `opfsError` to report.
116
+ */
117
+ export async function openDb(
118
+ sqlite3: any,
119
+ dbName: string,
120
+ useOpfs: boolean,
121
+ opts: OpenDbOptions = {}
122
+ ): Promise<OpenDbResult> {
123
+ // Memory was the configured choice (`store: 'memory'`), not a failure, so no
124
+ // error and no noise.
125
+ if (!useOpfs) return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false };
126
+
127
+ if (!sqlite3.installOpfsSAHPoolVfs) {
128
+ if (opts.disallowMemoryFallback) {
129
+ throw new Error('opfs-unavailable: sqlite-wasm build has no installOpfsSAHPoolVfs');
130
+ }
131
+ return fallbackToMemory(sqlite3, dbName, 'sqlite-wasm build has no installOpfsSAHPoolVfs', 0);
132
+ }
133
+
134
+ const maxAttempts = Math.max(1, opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
135
+ const backoffMs = opts.backoffMs ?? DEFAULT_BACKOFF_MS;
136
+ const sleep = opts.sleep ?? defaultSleep;
137
+
138
+ let lastError = 'unknown error';
139
+ let attempts = 0;
140
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
141
+ attempts = attempt;
142
+ try {
143
+ // `initialCapacity` stays at the sqlite-wasm default (6 files): one pool
144
+ // per bucket holds a single DB plus its journals, so preallocating more
145
+ // OPFS files would only be waste. A "SAH pool is full" error still
146
+ // reaches the caller verbatim via `opfsError`.
147
+ const pool = await sqlite3.installOpfsSAHPoolVfs({
148
+ name: `sp00ky-${dbName}`,
149
+ // The first failure is cached against the VFS name, so a retry that
150
+ // doesn't ask for a real re-init just replays the same rejection.
151
+ ...(attempt > 1 ? { forceReinitIfPreviouslyFailed: true } : {}),
152
+ });
153
+ return { db: new pool.OpfsSAHPoolDb(`/${dbName}.sqlite3`), persisted: true, pool };
154
+ } catch (e) {
155
+ lastError = errMessage(e);
156
+ if (attempt === maxAttempts || UNRETRYABLE.some((m) => lastError.includes(m))) break;
157
+ await sleep(backoffMs[Math.min(attempt - 1, backoffMs.length - 1)] ?? 0);
158
+ }
159
+ }
160
+ if (opts.disallowMemoryFallback) {
161
+ throw new Error(`opfs-unavailable: ${lastError} (after ${attempts} attempts)`);
162
+ }
163
+ return fallbackToMemory(sqlite3, dbName, lastError, attempts);
164
+ }