@spooky-sync/core 0.0.1-canary.152 → 0.0.1-canary.154
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +11 -2
- package/dist/index.js +251 -7
- package/dist/sqlite-worker.js +69 -11
- package/dist/types.d.ts +65 -1
- package/package.json +3 -3
- package/src/modules/devtools/index.ts +81 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +116 -0
- package/src/services/database/cache-engine.ts +18 -1
- package/src/services/database/sqlite-cache-engine.test.ts +159 -0
- package/src/services/database/sqlite-cache-engine.ts +118 -6
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +129 -0
- package/src/services/database/sqlite-worker.ts +12 -20
- package/src/services/database/surreal-cache-engine.ts +2 -0
- package/src/sp00ky.ts +28 -1
- package/src/types.ts +27 -0
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
project,
|
|
9
9
|
} from './sqlite-plan-sql';
|
|
10
10
|
import type { Logger } from '../logger/index';
|
|
11
|
-
import type { Sp00kyConfig } from '../../types';
|
|
11
|
+
import type { Sp00kyConfig, StorageHealth } from '../../types';
|
|
12
12
|
import type { SealedQuery } from '../../utils/surql';
|
|
13
13
|
import { resolveRelations, stableKey } from './relation-resolver';
|
|
14
14
|
import {
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
import { StaleEpochError } from './local';
|
|
20
20
|
import { translateSurql, tableOf, setPath, getPath, type SqlOp } from './surql-translate';
|
|
21
21
|
import type { EngineTx, Id, LocalStore, OrderBy, RelationFetch, Row } from './cache-engine';
|
|
22
|
+
import type { EngineStorageDiagnostics } from '../../modules/devtools/storage-info';
|
|
22
23
|
|
|
23
24
|
/**
|
|
24
25
|
* The statement result a pure-write op contributes to a query's results array.
|
|
@@ -92,11 +93,21 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
92
93
|
* Flipped off at runtime if the worker script predates the `select` op
|
|
93
94
|
* (stale cached bundle) — degrade to the legacy multi-hop path, don't break. */
|
|
94
95
|
private workerSelect: boolean;
|
|
96
|
+
/** What `workerSelect` was at construction, so DevTools can tell a runtime
|
|
97
|
+
* downgrade (configured true, effective false) from a configured-off. */
|
|
98
|
+
private workerSelectConfigured: boolean;
|
|
95
99
|
private events: DatabaseEventSystem = createDatabaseEventSystem();
|
|
96
100
|
private bucketId = 'anon';
|
|
101
|
+
/** Durability of the local store, set on every open. A plain Set of callbacks
|
|
102
|
+
* rather than a `DatabaseEventSystem` event: this changes at most once per
|
|
103
|
+
* open, and the typed event map is about query traffic. */
|
|
104
|
+
private storageHealthValue: StorageHealth = { status: 'unknown', fallback: false };
|
|
105
|
+
private storageHealthSubs = new Set<(health: StorageHealth) => void>();
|
|
97
106
|
/** Schemaless — tables are created lazily on first write; no migrator. */
|
|
98
107
|
readonly usesSurqlSchema = false;
|
|
99
108
|
|
|
109
|
+
readonly engineKind = 'sqlite' as const;
|
|
110
|
+
|
|
100
111
|
constructor(
|
|
101
112
|
private config: Sp00kyConfig<any>['database'],
|
|
102
113
|
private logger: Logger,
|
|
@@ -104,6 +115,7 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
104
115
|
) {
|
|
105
116
|
this.useOpfs = opts.useOpfs ?? true;
|
|
106
117
|
this.workerSelect = opts.workerSelect ?? config.workerSelect ?? true;
|
|
118
|
+
this.workerSelectConfigured = this.workerSelect;
|
|
107
119
|
}
|
|
108
120
|
|
|
109
121
|
get epoch(): number {
|
|
@@ -114,10 +126,75 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
114
126
|
return this.bucketId;
|
|
115
127
|
}
|
|
116
128
|
|
|
129
|
+
get storageHealth(): StorageHealth {
|
|
130
|
+
return this.storageHealthValue;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Fires immediately with the current snapshot (the store opens during
|
|
134
|
+
* `connect()`, before app components mount, so a late subscriber must still
|
|
135
|
+
* learn a fallback happened), then on every change. */
|
|
136
|
+
subscribeToStorageHealth(cb: (health: StorageHealth) => void): () => void {
|
|
137
|
+
cb(this.storageHealthValue);
|
|
138
|
+
this.storageHealthSubs.add(cb);
|
|
139
|
+
return () => {
|
|
140
|
+
this.storageHealthSubs.delete(cb);
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
private setStorageHealth(health: StorageHealth): void {
|
|
145
|
+
this.storageHealthValue = health;
|
|
146
|
+
for (const cb of this.storageHealthSubs) cb(health);
|
|
147
|
+
}
|
|
148
|
+
|
|
117
149
|
getConfig(): Sp00kyConfig<any>['database'] {
|
|
118
150
|
return this.config;
|
|
119
151
|
}
|
|
120
152
|
|
|
153
|
+
/**
|
|
154
|
+
* Storage numbers for the DevTools Storage tab. Uses {@link call} so the
|
|
155
|
+
* reads serialize with regular traffic (no SQLITE_BUSY). Never throws — the
|
|
156
|
+
* worker may be mid bucket-switch; a failure lands in `error` instead.
|
|
157
|
+
*/
|
|
158
|
+
async getStorageDiagnostics(opts?: { tableCounts?: boolean }): Promise<EngineStorageDiagnostics> {
|
|
159
|
+
const diag: EngineStorageDiagnostics = {
|
|
160
|
+
engine: 'sqlite',
|
|
161
|
+
bucketId: this.bucketId,
|
|
162
|
+
useOpfs: this.useOpfs,
|
|
163
|
+
workerSelectConfigured: this.workerSelectConfigured,
|
|
164
|
+
workerSelectEffective: this.workerSelect,
|
|
165
|
+
};
|
|
166
|
+
try {
|
|
167
|
+
const { rows } = await this.call<{ rows: { bytes: number; freelist: number }[] }>('exec', {
|
|
168
|
+
sql:
|
|
169
|
+
'SELECT (SELECT * FROM pragma_page_count()) * (SELECT * FROM pragma_page_size()) AS bytes, ' +
|
|
170
|
+
'(SELECT * FROM pragma_freelist_count()) * (SELECT * FROM pragma_page_size()) AS freelist',
|
|
171
|
+
});
|
|
172
|
+
diag.dbSizeBytes = rows?.[0]?.bytes;
|
|
173
|
+
diag.freelistBytes = rows?.[0]?.freelist;
|
|
174
|
+
if (opts?.tableCounts) {
|
|
175
|
+
const { rows: tables } = await this.call<{ rows: { name: string }[] }>('exec', {
|
|
176
|
+
sql: "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name",
|
|
177
|
+
});
|
|
178
|
+
const names = (tables ?? []).map((r) => r.name);
|
|
179
|
+
if (names.length) {
|
|
180
|
+
// Names come from sqlite_master itself; double-quoting is enough.
|
|
181
|
+
const sql = names
|
|
182
|
+
.map((n) => `SELECT '${n.replace(/'/g, "''")}' AS t, COUNT(*) AS n FROM "${n.replace(/"/g, '""')}"`)
|
|
183
|
+
.join(' UNION ALL ');
|
|
184
|
+
const { rows: counts } = await this.call<{ rows: { t: string; n: number }[] }>('exec', {
|
|
185
|
+
sql,
|
|
186
|
+
});
|
|
187
|
+
diag.tableCounts = (counts ?? []).map((r) => ({ table: r.t, rows: r.n }));
|
|
188
|
+
} else {
|
|
189
|
+
diag.tableCounts = [];
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
} catch (e) {
|
|
193
|
+
diag.error = e instanceof Error ? e.message : String(e);
|
|
194
|
+
}
|
|
195
|
+
return diag;
|
|
196
|
+
}
|
|
197
|
+
|
|
121
198
|
getEvents(): DatabaseEventSystem {
|
|
122
199
|
return this.events;
|
|
123
200
|
}
|
|
@@ -247,7 +324,12 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
247
324
|
// "no such table: _00_query" and the client wedged on "Loading database".
|
|
248
325
|
// Creating them inside `open` guarantees any access order is safe without
|
|
249
326
|
// adding ops to the engine's queue.
|
|
250
|
-
|
|
327
|
+
// `opfsError` is absent from a worker bundle older than this field, which
|
|
328
|
+
// just reads as "no reason given" rather than breaking the open.
|
|
329
|
+
const { persisted, opfsError } = await this.rawCall<{
|
|
330
|
+
persisted: boolean;
|
|
331
|
+
opfsError?: string;
|
|
332
|
+
}>('open', {
|
|
251
333
|
dbName: bucketId,
|
|
252
334
|
useOpfs: this.useOpfs,
|
|
253
335
|
systemTables: SYSTEM_TABLES,
|
|
@@ -255,10 +337,34 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
255
337
|
this.knownTables.clear();
|
|
256
338
|
for (const t of SYSTEM_TABLES) this.knownTables.add(t);
|
|
257
339
|
this.bucketId = bucketId;
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
340
|
+
// Durability was requested but could not be had: the store is in RAM, so it
|
|
341
|
+
// loses local writes on reload and can OOM a wasm-heavy renderer. Report it
|
|
342
|
+
// (the worker also console.errors, since host apps may run pino at `fatal`)
|
|
343
|
+
// and publish it so the app can warn the user.
|
|
344
|
+
const fellBack = this.useOpfs && !persisted;
|
|
345
|
+
// Omit `error` rather than setting it to `undefined`: the devtools
|
|
346
|
+
// serializer renders an undefined value as the STRING 'undefined'.
|
|
347
|
+
const health: StorageHealth = {
|
|
348
|
+
status: persisted ? 'persistent' : 'memory',
|
|
349
|
+
fallback: fellBack,
|
|
350
|
+
};
|
|
351
|
+
if (fellBack && opfsError) health.error = opfsError;
|
|
352
|
+
this.setStorageHealth(health);
|
|
353
|
+
const stats = getStats();
|
|
354
|
+
stats.persisted = persisted;
|
|
355
|
+
if (fellBack && opfsError) stats.opfsError = opfsError;
|
|
356
|
+
else delete stats.opfsError;
|
|
357
|
+
if (fellBack) {
|
|
358
|
+
this.logger.error(
|
|
359
|
+
{ bucketId, opfsError, Category: 'sp00ky-client::SqliteCacheEngine::connect' },
|
|
360
|
+
'SQLite OPFS persistence failed; store is IN MEMORY and will not survive reload'
|
|
361
|
+
);
|
|
362
|
+
} else {
|
|
363
|
+
this.logger.info(
|
|
364
|
+
{ bucketId, persisted, Category: 'sp00ky-client::SqliteCacheEngine::connect' },
|
|
365
|
+
persisted ? 'SQLite OPFS store opened' : 'SQLite in-memory store opened (as configured)'
|
|
366
|
+
);
|
|
367
|
+
}
|
|
262
368
|
}
|
|
263
369
|
|
|
264
370
|
/** Enqueue `fn` as a single serialized opQueue entry (mirrors {@link call}'s
|
|
@@ -743,6 +849,12 @@ interface SqliteStats {
|
|
|
743
849
|
bytesParsed: number;
|
|
744
850
|
/** Relation-resolver fan-out fetches (one worker round-trip each). */
|
|
745
851
|
relationFetches: number;
|
|
852
|
+
/** Whether the open store is OPFS-backed. `false` here with an `opfsError`
|
|
853
|
+
* means the whole dataset is sitting in RAM. Optional so it stays absent
|
|
854
|
+
* until the first open (and is skipped by the backfill loop below). */
|
|
855
|
+
persisted?: boolean;
|
|
856
|
+
/** Why OPFS persistence failed, when it did. */
|
|
857
|
+
opfsError?: string;
|
|
746
858
|
}
|
|
747
859
|
|
|
748
860
|
const EMPTY_STATS: SqliteStats = {
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { openDb } from './sqlite-open';
|
|
3
|
+
|
|
4
|
+
class FakeDb {
|
|
5
|
+
constructor(public arg: unknown) {}
|
|
6
|
+
exec() {
|
|
7
|
+
return [];
|
|
8
|
+
}
|
|
9
|
+
close() {}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** Minimal stand-in for the initialized sqlite-wasm module. `install` is the
|
|
13
|
+
* `installOpfsSAHPoolVfs` behavior under test; omit it to model a build that
|
|
14
|
+
* lacks the SAHPool VFS entirely. */
|
|
15
|
+
function makeSqlite3(install?: (opts: any) => Promise<unknown>) {
|
|
16
|
+
const sqlite3: any = { oo1: { DB: FakeDb } };
|
|
17
|
+
if (install) sqlite3.installOpfsSAHPoolVfs = vi.fn(install);
|
|
18
|
+
return sqlite3;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const pool = { OpfsSAHPoolDb: FakeDb };
|
|
22
|
+
/** What a pool locked by another tab of the app actually throws. */
|
|
23
|
+
function lockedError(): Error {
|
|
24
|
+
const e = new Error('Access Handles cannot be acquired');
|
|
25
|
+
e.name = 'NoModificationAllowedError';
|
|
26
|
+
return e;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const noSleep = () => Promise.resolve();
|
|
30
|
+
|
|
31
|
+
let errSpy: ReturnType<typeof vi.spyOn>;
|
|
32
|
+
beforeEach(() => {
|
|
33
|
+
errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
34
|
+
});
|
|
35
|
+
afterEach(() => {
|
|
36
|
+
errSpy.mockRestore();
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
// The bare `catch {}` this replaces turned every OPFS failure into a silent
|
|
40
|
+
// full-RAM database: no log, no reason, no way for the app to know its writes
|
|
41
|
+
// die on reload. Each case below pins one half of the fix: keep trying when
|
|
42
|
+
// retrying can plausibly work, and when it can't, say so loudly and hand the
|
|
43
|
+
// reason back.
|
|
44
|
+
describe('openDb', () => {
|
|
45
|
+
it('opens the OPFS pool on the first attempt', async () => {
|
|
46
|
+
const sqlite3 = makeSqlite3(async () => pool);
|
|
47
|
+
const res = await openDb(sqlite3, 'user:abc', true, { sleep: noSleep });
|
|
48
|
+
|
|
49
|
+
expect(res.persisted).toBe(true);
|
|
50
|
+
expect(res.opfsError).toBeUndefined();
|
|
51
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
|
|
52
|
+
// The pool is named per bucket, and the first try must NOT force a re-init
|
|
53
|
+
// (that would throw away a pool another attempt is legitimately using).
|
|
54
|
+
expect(sqlite3.installOpfsSAHPoolVfs.mock.calls[0][0]).toEqual({ name: 'sp00ky-user:abc' });
|
|
55
|
+
expect(errSpy).not.toHaveBeenCalled();
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
// The tab-closing race: the old tab still holds the sync access handles when
|
|
59
|
+
// the new one boots. Without a retry that tab is stuck in RAM for its whole
|
|
60
|
+
// lifetime, even though the lock frees milliseconds later.
|
|
61
|
+
it('retries a locked pool and succeeds, forcing re-init after the first failure', async () => {
|
|
62
|
+
let calls = 0;
|
|
63
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
64
|
+
if (++calls < 3) throw lockedError();
|
|
65
|
+
return pool;
|
|
66
|
+
});
|
|
67
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
68
|
+
|
|
69
|
+
expect(res.persisted).toBe(true);
|
|
70
|
+
expect(res.opfsError).toBeUndefined();
|
|
71
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
|
|
72
|
+
// sqlite-wasm caches the first rejection against the VFS name, so retries
|
|
73
|
+
// that don't ask for a real re-init just replay it.
|
|
74
|
+
const [first, second, third] = sqlite3.installOpfsSAHPoolVfs.mock.calls.map((c: any[]) => c[0]);
|
|
75
|
+
expect(first.forceReinitIfPreviouslyFailed).toBeUndefined();
|
|
76
|
+
expect(second.forceReinitIfPreviouslyFailed).toBe(true);
|
|
77
|
+
expect(third.forceReinitIfPreviouslyFailed).toBe(true);
|
|
78
|
+
expect(errSpy).not.toHaveBeenCalled();
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('falls back loudly after exhausting the retries, keeping the reason', async () => {
|
|
82
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
83
|
+
throw lockedError();
|
|
84
|
+
});
|
|
85
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
86
|
+
|
|
87
|
+
expect(res.persisted).toBe(false);
|
|
88
|
+
// The DOMException name is the diagnostic part: it names the lock.
|
|
89
|
+
expect(res.opfsError).toContain('NoModificationAllowedError');
|
|
90
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
|
|
91
|
+
expect(errSpy).toHaveBeenCalledTimes(1);
|
|
92
|
+
expect(String(errSpy.mock.calls[0][0])).toContain('IN MEMORY');
|
|
93
|
+
// Still a usable handle: losing durability must not break the app.
|
|
94
|
+
expect(res.db).toBeDefined();
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
// An insecure context has no sync access handles at all, so retrying just
|
|
98
|
+
// adds boot latency to a foregone conclusion.
|
|
99
|
+
it('does not retry when the OPFS APIs are missing entirely', async () => {
|
|
100
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
101
|
+
throw new Error('Missing required OPFS APIs.');
|
|
102
|
+
});
|
|
103
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
104
|
+
|
|
105
|
+
expect(res.persisted).toBe(false);
|
|
106
|
+
expect(res.opfsError).toContain('Missing required OPFS APIs');
|
|
107
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
|
|
108
|
+
expect(errSpy).toHaveBeenCalledTimes(1);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('reports a build without the SAHPool VFS without calling anything', async () => {
|
|
112
|
+
const sqlite3 = makeSqlite3();
|
|
113
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
114
|
+
|
|
115
|
+
expect(res.persisted).toBe(false);
|
|
116
|
+
expect(res.opfsError).toContain('installOpfsSAHPoolVfs');
|
|
117
|
+
expect(errSpy).toHaveBeenCalledTimes(1);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
// `store: 'memory'` is a configuration choice, not a degradation, so it must
|
|
121
|
+
// stay silent and carry no error for the UI to warn about.
|
|
122
|
+
it('opens in memory quietly when persistence was not requested', async () => {
|
|
123
|
+
const sqlite3 = makeSqlite3(async () => pool);
|
|
124
|
+
const res = await openDb(sqlite3, 'main', false, { sleep: noSleep });
|
|
125
|
+
|
|
126
|
+
expect(res.persisted).toBe(false);
|
|
127
|
+
expect(res.opfsError).toBeUndefined();
|
|
128
|
+
expect(sqlite3.installOpfsSAHPoolVfs).not.toHaveBeenCalled();
|
|
129
|
+
expect(errSpy).not.toHaveBeenCalled();
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('honors maxAttempts and waits the configured backoff between tries', async () => {
|
|
133
|
+
const slept: number[] = [];
|
|
134
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
135
|
+
throw lockedError();
|
|
136
|
+
});
|
|
137
|
+
const res = await openDb(sqlite3, 'main', true, {
|
|
138
|
+
maxAttempts: 4,
|
|
139
|
+
backoffMs: [10, 20],
|
|
140
|
+
sleep: async (ms) => {
|
|
141
|
+
slept.push(ms);
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
expect(res.persisted).toBe(false);
|
|
146
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(4);
|
|
147
|
+
// One sleep per gap (never after the last attempt), last delay repeating.
|
|
148
|
+
expect(slept).toEqual([10, 20, 20]);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Opening the worker's SQLite handle: the OPFS SAHPool VFS when durable
|
|
3
|
+
* storage was asked for, an in-memory DB only as a last resort. Extracted from
|
|
4
|
+
* `sqlite-worker.ts` (which imports the wasm module at module scope and so
|
|
5
|
+
* can't be loaded in a unit test) to keep the retry/fallback policy testable
|
|
6
|
+
* off-worker, the same split as `sqlite-select.ts`.
|
|
7
|
+
*
|
|
8
|
+
* Why retry: SAHPool holds an EXCLUSIVE sync access handle on every file in
|
|
9
|
+
* its pool, so only one client per pool name can have it open. A second tab of
|
|
10
|
+
* the same app therefore fails init, and `installOpfsSAHPoolVfs` CACHES that
|
|
11
|
+
* rejection per VFS name, so a later call only gets a real second chance when
|
|
12
|
+
* it passes `forceReinitIfPreviouslyFailed`. Retrying with that flag turns the
|
|
13
|
+
* common "the other tab is still closing" race into a success instead of a
|
|
14
|
+
* permanent in-memory session.
|
|
15
|
+
*
|
|
16
|
+
* Why the noise: `:memory:` holds the whole dataset in RAM (the
|
|
17
|
+
* OOM-on-wasm-heavy-pages failure mode the OPFS store exists to avoid) and
|
|
18
|
+
* drops every local write on reload. Host apps run pino at their own level,
|
|
19
|
+
* some at `fatal`, so the fallback ALSO writes to `console.error` from inside
|
|
20
|
+
* the worker, and the reason travels back to the engine as `opfsError` for the
|
|
21
|
+
* app to surface.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** The DB surface the worker uses (a `sqlite3.oo1.DB` or an `OpfsSAHPoolDb`). */
|
|
25
|
+
export interface SqliteDbHandle {
|
|
26
|
+
exec: (opts: { sql: string; bind?: unknown[]; rowMode?: string; returnValue?: string }) => unknown;
|
|
27
|
+
close: () => void;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface OpenDbResult {
|
|
31
|
+
db: SqliteDbHandle;
|
|
32
|
+
/** True only when the handle is backed by OPFS and survives a reload. */
|
|
33
|
+
persisted: boolean;
|
|
34
|
+
/** Why persistence failed. Set only when OPFS was requested and fell back. */
|
|
35
|
+
opfsError?: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface OpenDbOptions {
|
|
39
|
+
/** Total OPFS init attempts, including the first. Default 3. */
|
|
40
|
+
maxAttempts?: number;
|
|
41
|
+
/** Delay before each retry; the last entry repeats. Default [250, 500]. */
|
|
42
|
+
backoffMs?: number[];
|
|
43
|
+
/** Injectable for tests. */
|
|
44
|
+
sleep?: (ms: number) => Promise<void>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
48
|
+
/** Bounded on purpose: this runs on the boot path, before the first query. */
|
|
49
|
+
const DEFAULT_BACKOFF_MS = [250, 500];
|
|
50
|
+
|
|
51
|
+
/** Failures no retry can fix: the APIs aren't there at all (insecure context,
|
|
52
|
+
* or a browser without sync access handles). Fall back immediately. */
|
|
53
|
+
const UNRETRYABLE = ['Missing required OPFS APIs'];
|
|
54
|
+
|
|
55
|
+
const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
56
|
+
|
|
57
|
+
/** Keep the DOMException name (e.g. `NoModificationAllowedError` for a pool
|
|
58
|
+
* locked by another tab): it is the most diagnostic part of the failure. */
|
|
59
|
+
function errMessage(e: unknown): string {
|
|
60
|
+
if (e instanceof Error) return e.name && e.name !== 'Error' ? `${e.name}: ${e.message}` : e.message;
|
|
61
|
+
return String(e);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function fallbackToMemory(
|
|
65
|
+
sqlite3: any,
|
|
66
|
+
dbName: string,
|
|
67
|
+
reason: string,
|
|
68
|
+
attempts: number
|
|
69
|
+
): OpenDbResult {
|
|
70
|
+
const tried = attempts > 0 ? ` after ${attempts} attempt${attempts === 1 ? '' : 's'}` : '';
|
|
71
|
+
// Deliberately console, not the logger: host apps configure pino's level (some
|
|
72
|
+
// run `fatal`), and losing durability must never be filtered into silence.
|
|
73
|
+
// oxlint-disable-next-line no-console
|
|
74
|
+
console.error(
|
|
75
|
+
`[sp00ky] OPFS persistence unavailable for "${dbName}"${tried}: ${reason}. The local SQLite ` +
|
|
76
|
+
'cache is running IN MEMORY, which keeps the whole dataset in RAM and loses every local ' +
|
|
77
|
+
'write on reload. The usual cause is another tab of this app holding the storage lock, so ' +
|
|
78
|
+
'closing the other tabs and reloading restores persistence.'
|
|
79
|
+
);
|
|
80
|
+
return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false, opfsError: reason };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Open `dbName`'s handle. Never throws for a storage problem: a caller that
|
|
85
|
+
* asked for persistence and can't have it gets a working in-memory handle plus
|
|
86
|
+
* `persisted: false` and an `opfsError` to report.
|
|
87
|
+
*/
|
|
88
|
+
export async function openDb(
|
|
89
|
+
sqlite3: any,
|
|
90
|
+
dbName: string,
|
|
91
|
+
useOpfs: boolean,
|
|
92
|
+
opts: OpenDbOptions = {}
|
|
93
|
+
): Promise<OpenDbResult> {
|
|
94
|
+
// Memory was the configured choice (`store: 'memory'`), not a failure, so no
|
|
95
|
+
// error and no noise.
|
|
96
|
+
if (!useOpfs) return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false };
|
|
97
|
+
|
|
98
|
+
if (!sqlite3.installOpfsSAHPoolVfs) {
|
|
99
|
+
return fallbackToMemory(sqlite3, dbName, 'sqlite-wasm build has no installOpfsSAHPoolVfs', 0);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const maxAttempts = Math.max(1, opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
|
|
103
|
+
const backoffMs = opts.backoffMs ?? DEFAULT_BACKOFF_MS;
|
|
104
|
+
const sleep = opts.sleep ?? defaultSleep;
|
|
105
|
+
|
|
106
|
+
let lastError = 'unknown error';
|
|
107
|
+
let attempts = 0;
|
|
108
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
109
|
+
attempts = attempt;
|
|
110
|
+
try {
|
|
111
|
+
// `initialCapacity` stays at the sqlite-wasm default (6 files): one pool
|
|
112
|
+
// per bucket holds a single DB plus its journals, so preallocating more
|
|
113
|
+
// OPFS files would only be waste. A "SAH pool is full" error still
|
|
114
|
+
// reaches the caller verbatim via `opfsError`.
|
|
115
|
+
const pool = await sqlite3.installOpfsSAHPoolVfs({
|
|
116
|
+
name: `sp00ky-${dbName}`,
|
|
117
|
+
// The first failure is cached against the VFS name, so a retry that
|
|
118
|
+
// doesn't ask for a real re-init just replays the same rejection.
|
|
119
|
+
...(attempt > 1 ? { forceReinitIfPreviouslyFailed: true } : {}),
|
|
120
|
+
});
|
|
121
|
+
return { db: new pool.OpfsSAHPoolDb(`/${dbName}.sqlite3`), persisted: true };
|
|
122
|
+
} catch (e) {
|
|
123
|
+
lastError = errMessage(e);
|
|
124
|
+
if (attempt === maxAttempts || UNRETRYABLE.some((m) => lastError.includes(m))) break;
|
|
125
|
+
await sleep(backoffMs[Math.min(attempt - 1, backoffMs.length - 1)] ?? 0);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return fallbackToMemory(sqlite3, dbName, lastError, attempts);
|
|
129
|
+
}
|
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
* is also what the OPFS VFS requires — file access must happen off the main
|
|
6
6
|
* thread. Persistence uses the **OPFS SAHPool VFS**: durable, and (unlike the
|
|
7
7
|
* classic OPFS VFS) it does NOT require COOP/COEP cross-origin isolation
|
|
8
|
-
* headers, so host apps embedding the client need no server changes.
|
|
9
|
-
* to an in-memory DB
|
|
8
|
+
* headers, so host apps embedding the client need no server changes. When OPFS
|
|
9
|
+
* is unavailable it retries, then falls back to an in-memory DB and REPORTS the
|
|
10
|
+
* loss of durability (see `sqlite-open.ts`) instead of degrading silently.
|
|
10
11
|
*
|
|
11
12
|
* Message protocol (request/response keyed by `id`):
|
|
12
|
-
* { id, type: 'open', payload: { dbName, useOpfs } }
|
|
13
|
+
* { id, type: 'open', payload: { dbName, useOpfs } } -> { id, ok, persisted, opfsError? }
|
|
13
14
|
* { id, type: 'exec', payload: { sql, bind } } -> { id, ok, rows }
|
|
14
15
|
* { id, type: 'run', payload: { sql, bind } } -> { id, ok }
|
|
15
16
|
* { id, type: 'batch', payload: [{ sql, bind }] } (atomic BEGIN/COMMIT)
|
|
@@ -22,6 +23,7 @@
|
|
|
22
23
|
* hot path; the per-statement ops remain for the write/shim paths.
|
|
23
24
|
*/
|
|
24
25
|
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
|
26
|
+
import { openDb, type SqliteDbHandle } from './sqlite-open';
|
|
25
27
|
import { executeSelect, type SelectDb } from './sqlite-select';
|
|
26
28
|
|
|
27
29
|
interface Stmt {
|
|
@@ -29,28 +31,18 @@ interface Stmt {
|
|
|
29
31
|
bind?: unknown[];
|
|
30
32
|
}
|
|
31
33
|
|
|
32
|
-
let db:
|
|
33
|
-
exec: (opts: { sql: string; bind?: unknown[]; rowMode?: string; returnValue?: string }) => unknown;
|
|
34
|
-
close: () => void;
|
|
35
|
-
} | null = null;
|
|
34
|
+
let db: SqliteDbHandle | null = null;
|
|
36
35
|
|
|
37
36
|
async function open(
|
|
38
37
|
dbName: string,
|
|
39
38
|
useOpfs: boolean,
|
|
40
39
|
systemTables: readonly string[] = []
|
|
41
|
-
): Promise<{ persisted: boolean }> {
|
|
40
|
+
): Promise<{ persisted: boolean; opfsError?: string }> {
|
|
42
41
|
const sqlite3: any = await sqlite3InitModule();
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
db = new pool.OpfsSAHPoolDb(`/${dbName}.sqlite3`);
|
|
48
|
-
persisted = true;
|
|
49
|
-
} catch {
|
|
50
|
-
// fall through to in-memory
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
if (!db) db = new sqlite3.oo1.DB(':memory:', 'c');
|
|
42
|
+
// Retry/fallback policy (and the loud report when persistence is lost) lives
|
|
43
|
+
// in `sqlite-open.ts` so it can be unit tested off-worker.
|
|
44
|
+
const { db: handle, persisted, opfsError } = await openDb(sqlite3, dbName, useOpfs);
|
|
45
|
+
db = handle;
|
|
54
46
|
// Physically create the internal `_00_*` tables the client reads before any
|
|
55
47
|
// write (DEFINE is a noop on this engine, so the migrator can't). Prevents
|
|
56
48
|
// "no such table: _00_query" on a fresh bucket right after signup.
|
|
@@ -66,7 +58,7 @@ async function open(
|
|
|
66
58
|
} catch {
|
|
67
59
|
/* pragma best-effort */
|
|
68
60
|
}
|
|
69
|
-
return { persisted };
|
|
61
|
+
return { persisted, opfsError };
|
|
70
62
|
}
|
|
71
63
|
|
|
72
64
|
function exec(sql: string, bind?: unknown[]): unknown[] {
|
|
@@ -31,6 +31,8 @@ export class SurrealCacheEngine extends LocalDatabaseService implements LocalCac
|
|
|
31
31
|
/** SurrealDB needs its SurrealQL schema provisioned locally. */
|
|
32
32
|
readonly usesSurqlSchema = true;
|
|
33
33
|
|
|
34
|
+
readonly engineKind = 'surrealdb' as const;
|
|
35
|
+
|
|
34
36
|
/** {@link LocalCacheEngine} alias for {@link LocalDatabaseService.switchStore}. */
|
|
35
37
|
switchBucket(bucketId: string): Promise<void> {
|
|
36
38
|
return this.switchStore(bucketId);
|
package/src/sp00ky.ts
CHANGED
|
@@ -8,7 +8,8 @@ import type {
|
|
|
8
8
|
PreloadOptions,
|
|
9
9
|
UpdateOptions,
|
|
10
10
|
RunOptions,
|
|
11
|
-
SyncHealth
|
|
11
|
+
SyncHealth,
|
|
12
|
+
StorageHealth} from './types';
|
|
12
13
|
import {
|
|
13
14
|
LocalMigrator,
|
|
14
15
|
RemoteDatabaseService,
|
|
@@ -106,6 +107,13 @@ export class BucketHandle {
|
|
|
106
107
|
*/
|
|
107
108
|
const LAST_BUCKET_KEY = 'sp00ky:last_bucket';
|
|
108
109
|
|
|
110
|
+
/** Reported for engines that don't track local-store durability. Frozen so a
|
|
111
|
+
* subscriber can't mutate the shared snapshot. */
|
|
112
|
+
const UNKNOWN_STORAGE_HEALTH: StorageHealth = Object.freeze({
|
|
113
|
+
status: 'unknown',
|
|
114
|
+
fallback: false,
|
|
115
|
+
});
|
|
116
|
+
|
|
109
117
|
function readBootBucketHint(): string | null {
|
|
110
118
|
try {
|
|
111
119
|
return typeof localStorage !== 'undefined' ? localStorage.getItem(LAST_BUCKET_KEY) : null;
|
|
@@ -188,6 +196,25 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
188
196
|
return this.sync.subscribeToSyncHealth(cb);
|
|
189
197
|
}
|
|
190
198
|
|
|
199
|
+
/** Durability of the local cache. See {@link StorageHealth}. `'unknown'` for
|
|
200
|
+
* engines that don't report it. */
|
|
201
|
+
get storageHealth(): StorageHealth {
|
|
202
|
+
return this.local.storageHealth ?? UNKNOWN_STORAGE_HEALTH;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Observe local-store durability. Fires immediately with the current snapshot
|
|
207
|
+
* and again on every change (at most once per bucket open in practice).
|
|
208
|
+
* Returns an unsubscribe.
|
|
209
|
+
*/
|
|
210
|
+
subscribeToStorageHealth(cb: (health: StorageHealth) => void): () => void {
|
|
211
|
+
if (this.local.subscribeToStorageHealth) {
|
|
212
|
+
return this.local.subscribeToStorageHealth(cb);
|
|
213
|
+
}
|
|
214
|
+
cb(UNKNOWN_STORAGE_HEALTH);
|
|
215
|
+
return () => {};
|
|
216
|
+
}
|
|
217
|
+
|
|
191
218
|
constructor(private config: Sp00kyConfig<S>) {
|
|
192
219
|
const logger = createLogger(config.logLevel ?? 'info', config.otelTransmit);
|
|
193
220
|
this.logger = logger.child({ service: 'Sp00kyClient' });
|
package/src/types.ts
CHANGED
|
@@ -247,6 +247,33 @@ export interface SyncHealth {
|
|
|
247
247
|
everConnected: boolean;
|
|
248
248
|
}
|
|
249
249
|
|
|
250
|
+
export type StorageHealthStatus = 'unknown' | 'persistent' | 'memory';
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Durability of the LOCAL cache, delivered to `subscribeToStorageHealth`
|
|
254
|
+
* subscribers. Separate from {@link SyncHealth}: that one is about reaching the
|
|
255
|
+
* server, this one is about whether the local store survives a reload.
|
|
256
|
+
*
|
|
257
|
+
* Under `localEngine: 'sqlite'` the durable store is the OPFS SAHPool VFS,
|
|
258
|
+
* which only one client per bucket can hold open. When it can't be opened (a
|
|
259
|
+
* second tab of the app already has it, an insecure context, a full pool) the
|
|
260
|
+
* engine keeps working against an in-memory DB, which holds the whole dataset
|
|
261
|
+
* in RAM and loses local writes on reload. `fallback` marks exactly that case,
|
|
262
|
+
* so a UI can warn about it.
|
|
263
|
+
*/
|
|
264
|
+
export interface StorageHealth {
|
|
265
|
+
/** `'unknown'` until the local cache has opened, or for engines that don't report. */
|
|
266
|
+
status: StorageHealthStatus;
|
|
267
|
+
/**
|
|
268
|
+
* `true` only when durable storage was REQUESTED and could not be opened.
|
|
269
|
+
* Stays `false` for a configured-in-memory store (`store: 'memory'`), which
|
|
270
|
+
* is a choice rather than a failure, so a UI can key off this alone.
|
|
271
|
+
*/
|
|
272
|
+
fallback: boolean;
|
|
273
|
+
/** Reason durable storage failed (only set while `fallback` is `true`). */
|
|
274
|
+
error?: string;
|
|
275
|
+
}
|
|
276
|
+
|
|
250
277
|
export type QueryHash = string;
|
|
251
278
|
|
|
252
279
|
// Flat array format: [[record-id, version], [record-id, version], ...]
|