@spooky-sync/core 0.0.1-canary.151 → 0.0.1-canary.153
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +11 -2
- package/dist/index.js +71 -6
- package/dist/sqlite-worker.js +69 -11
- package/dist/types.d.ts +35 -1
- package/package.json +3 -3
- package/src/modules/devtools/index.ts +8 -0
- package/src/services/database/cache-engine.ts +10 -1
- package/src/services/database/sqlite-cache-engine.test.ts +87 -0
- package/src/services/database/sqlite-cache-engine.ts +62 -6
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +129 -0
- package/src/services/database/sqlite-worker.ts +12 -20
- package/src/sp00ky.ts +28 -1
- package/src/types.ts +27 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { A as
|
|
1
|
+
import { A as StorageHealthStatus, B as DatabaseEventSystem, C as RecordVersionDiff, D as Sp00kyQueryResult, E as Sp00kyConfig, F as TimingPhase, G as EventSystem, H as Logger$1, I as UpdateOptions, L as UpEvent, M as SyncHealth, N as SyncHealthConfig, O as Sp00kyQueryResultPromise, P as SyncHealthStatus, R as LocalStore, S as RecordVersionArray, T as RunOptions, U as SyncEventSystem, V as DatabaseEventTypes, W as EventDefinition, _ as QueryStatus, a as MutationCallback, b as QueryTimings, c as PersistenceClient, d as PreloadOptions, f as PreloadRefresh, g as QueryState, h as QueryHash, i as MATERIALIZATION_SAMPLE_WINDOW, j as StoreType, k as StorageHealth, l as PhaseStat, m as QueryConfigRecord, n as EventSubscriptionOptions, o as MutationEvent, p as QueryConfig, r as Level, s as MutationEventType, t as DebounceOptions, u as PinoTransmit, v as QueryStatusCallback, w as RegistrationTimings, x as QueryUpdateCallback, y as QueryTimeToLive, z as SealedQuery } from "./types.js";
|
|
2
2
|
import * as surrealdb0 from "surrealdb";
|
|
3
3
|
import { Duration, RecordId, Surreal as Surreal$1, SurrealTransaction } from "surrealdb";
|
|
4
4
|
import { AccessDefinition, BackendNames, BackendRoutes, BucketNames, ColumnSchema, FinalQuery, GetTable, QueryBuilder, QueryOptions, QueryPlan, RoutePayload, SchemaStructure, TableModel, TableNames, TypeNameToTypeMap } from "@spooky-sync/query-builder";
|
|
@@ -1265,6 +1265,15 @@ declare class Sp00kyClient<S extends SchemaStructure> {
|
|
|
1265
1265
|
* on every healthy↔degraded transition. Returns an unsubscribe.
|
|
1266
1266
|
*/
|
|
1267
1267
|
subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void;
|
|
1268
|
+
/** Durability of the local cache. See {@link StorageHealth}. `'unknown'` for
|
|
1269
|
+
* engines that don't report it. */
|
|
1270
|
+
get storageHealth(): StorageHealth;
|
|
1271
|
+
/**
|
|
1272
|
+
* Observe local-store durability. Fires immediately with the current snapshot
|
|
1273
|
+
* and again on every change (at most once per bucket open in practice).
|
|
1274
|
+
* Returns an unsubscribe.
|
|
1275
|
+
*/
|
|
1276
|
+
subscribeToStorageHealth(cb: (health: StorageHealth) => void): () => void;
|
|
1268
1277
|
constructor(config: Sp00kyConfig<S>);
|
|
1269
1278
|
/**
|
|
1270
1279
|
* Setup direct callbacks instead of event subscriptions
|
|
@@ -1428,4 +1437,4 @@ declare function textToHtml(text: string): string;
|
|
|
1428
1437
|
*/
|
|
1429
1438
|
|
|
1430
1439
|
//#endregion
|
|
1431
|
-
export { AppReleaseHandle, AppReleaseModule, type AppReleaseOptions, type AppReleaseSnapshot, AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, FeatureFlagHandle, FeatureFlagModule, type FeatureFlagOptions, type FeatureFlagSnapshot, Level, MATERIALIZATION_SAMPLE_WINDOW, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PhaseStat, PinoTransmit, PreloadOptions, PreloadRefresh, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryStatus, QueryStatusCallback, QueryTimeToLive, QueryTimings, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RegistrationTimings, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StoreType, SyncHealth, SyncHealthConfig, SyncHealthStatus, TimingPhase, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, semverGt, textToHtml };
|
|
1440
|
+
export { AppReleaseHandle, AppReleaseModule, type AppReleaseOptions, type AppReleaseSnapshot, AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, FeatureFlagHandle, FeatureFlagModule, type FeatureFlagOptions, type FeatureFlagSnapshot, Level, MATERIALIZATION_SAMPLE_WINDOW, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PhaseStat, PinoTransmit, PreloadOptions, PreloadRefresh, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryStatus, QueryStatusCallback, QueryTimeToLive, QueryTimings, QueryUpdateCallback, RecordVersionArray, RecordVersionDiff, RegistrationTimings, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StorageHealth, StorageHealthStatus, StoreType, SyncHealth, SyncHealthConfig, SyncHealthStatus, TimingPhase, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, semverGt, textToHtml };
|
package/dist/index.js
CHANGED
|
@@ -1511,6 +1511,14 @@ var SqliteCacheEngine = class {
|
|
|
1511
1511
|
workerSelect;
|
|
1512
1512
|
events = createDatabaseEventSystem();
|
|
1513
1513
|
bucketId = "anon";
|
|
1514
|
+
/** Durability of the local store, set on every open. A plain Set of callbacks
|
|
1515
|
+
* rather than a `DatabaseEventSystem` event: this changes at most once per
|
|
1516
|
+
* open, and the typed event map is about query traffic. */
|
|
1517
|
+
storageHealthValue = {
|
|
1518
|
+
status: "unknown",
|
|
1519
|
+
fallback: false
|
|
1520
|
+
};
|
|
1521
|
+
storageHealthSubs = /* @__PURE__ */ new Set();
|
|
1514
1522
|
/** Schemaless — tables are created lazily on first write; no migrator. */
|
|
1515
1523
|
usesSurqlSchema = false;
|
|
1516
1524
|
constructor(config, logger, opts = {}) {
|
|
@@ -1525,6 +1533,23 @@ var SqliteCacheEngine = class {
|
|
|
1525
1533
|
get currentBucketId() {
|
|
1526
1534
|
return this.bucketId;
|
|
1527
1535
|
}
|
|
1536
|
+
get storageHealth() {
|
|
1537
|
+
return this.storageHealthValue;
|
|
1538
|
+
}
|
|
1539
|
+
/** Fires immediately with the current snapshot (the store opens during
|
|
1540
|
+
* `connect()`, before app components mount, so a late subscriber must still
|
|
1541
|
+
* learn a fallback happened), then on every change. */
|
|
1542
|
+
subscribeToStorageHealth(cb) {
|
|
1543
|
+
cb(this.storageHealthValue);
|
|
1544
|
+
this.storageHealthSubs.add(cb);
|
|
1545
|
+
return () => {
|
|
1546
|
+
this.storageHealthSubs.delete(cb);
|
|
1547
|
+
};
|
|
1548
|
+
}
|
|
1549
|
+
setStorageHealth(health) {
|
|
1550
|
+
this.storageHealthValue = health;
|
|
1551
|
+
for (const cb of this.storageHealthSubs) cb(health);
|
|
1552
|
+
}
|
|
1528
1553
|
getConfig() {
|
|
1529
1554
|
return this.config;
|
|
1530
1555
|
}
|
|
@@ -1625,7 +1650,7 @@ var SqliteCacheEngine = class {
|
|
|
1625
1650
|
*/
|
|
1626
1651
|
async openInternal(bucketId) {
|
|
1627
1652
|
this.worker = this.spawnWorker();
|
|
1628
|
-
const { persisted } = await this.rawCall("open", {
|
|
1653
|
+
const { persisted, opfsError } = await this.rawCall("open", {
|
|
1629
1654
|
dbName: bucketId,
|
|
1630
1655
|
useOpfs: this.useOpfs,
|
|
1631
1656
|
systemTables: SYSTEM_TABLES
|
|
@@ -1633,11 +1658,25 @@ var SqliteCacheEngine = class {
|
|
|
1633
1658
|
this.knownTables.clear();
|
|
1634
1659
|
for (const t of SYSTEM_TABLES) this.knownTables.add(t);
|
|
1635
1660
|
this.bucketId = bucketId;
|
|
1636
|
-
this.
|
|
1661
|
+
const fellBack = this.useOpfs && !persisted;
|
|
1662
|
+
this.setStorageHealth({
|
|
1663
|
+
status: persisted ? "persistent" : "memory",
|
|
1664
|
+
fallback: fellBack,
|
|
1665
|
+
error: fellBack ? opfsError : void 0
|
|
1666
|
+
});
|
|
1667
|
+
const stats = getStats();
|
|
1668
|
+
stats.persisted = persisted;
|
|
1669
|
+
stats.opfsError = fellBack ? opfsError : void 0;
|
|
1670
|
+
if (fellBack) this.logger.error({
|
|
1671
|
+
bucketId,
|
|
1672
|
+
opfsError,
|
|
1673
|
+
Category: "sp00ky-client::SqliteCacheEngine::connect"
|
|
1674
|
+
}, "SQLite OPFS persistence failed; store is IN MEMORY and will not survive reload");
|
|
1675
|
+
else this.logger.info({
|
|
1637
1676
|
bucketId,
|
|
1638
1677
|
persisted,
|
|
1639
1678
|
Category: "sp00ky-client::SqliteCacheEngine::connect"
|
|
1640
|
-
}, persisted ? "SQLite OPFS store opened" : "SQLite in-memory store opened (
|
|
1679
|
+
}, persisted ? "SQLite OPFS store opened" : "SQLite in-memory store opened (as configured)");
|
|
1641
1680
|
}
|
|
1642
1681
|
/** Enqueue `fn` as a single serialized opQueue entry (mirrors {@link call}'s
|
|
1643
1682
|
* chaining) so it can't interleave with reads/writes at the worker. */
|
|
@@ -5051,8 +5090,8 @@ function parseBackendInfo(raw) {
|
|
|
5051
5090
|
|
|
5052
5091
|
//#endregion
|
|
5053
5092
|
//#region src/modules/devtools/index.ts
|
|
5054
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
5055
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
5093
|
+
const CORE_VERSION = "0.0.1-canary.153";
|
|
5094
|
+
const WASM_VERSION = "0.0.1-canary.153";
|
|
5056
5095
|
const SURREAL_VERSION = "3.0.3";
|
|
5057
5096
|
var DevToolsService = class {
|
|
5058
5097
|
eventsHistory = [];
|
|
@@ -5083,6 +5122,7 @@ var DevToolsService = class {
|
|
|
5083
5122
|
if (this.authService.isAuthenticated && this.backendInfo.versions.ssp === UNAVAILABLE) this.refreshBackendVersions();
|
|
5084
5123
|
else this.notifyDevTools();
|
|
5085
5124
|
});
|
|
5125
|
+
this.databaseService.subscribeToStorageHealth?.(() => this.notifyDevTools());
|
|
5086
5126
|
this.refreshBackendVersions();
|
|
5087
5127
|
this.logger.debug({ Category: "sp00ky-client::DevToolsService::init" }, "Service initialized");
|
|
5088
5128
|
}
|
|
@@ -5248,7 +5288,11 @@ var DevToolsService = class {
|
|
|
5248
5288
|
},
|
|
5249
5289
|
database: {
|
|
5250
5290
|
tables: this.localTables.length ? this.localTables : this.schema.tables.map((t) => t.name),
|
|
5251
|
-
tableData: {}
|
|
5291
|
+
tableData: {},
|
|
5292
|
+
storage: this.databaseService.storageHealth ?? {
|
|
5293
|
+
status: "unknown",
|
|
5294
|
+
fallback: false
|
|
5295
|
+
}
|
|
5252
5296
|
}
|
|
5253
5297
|
});
|
|
5254
5298
|
}
|
|
@@ -7159,6 +7203,12 @@ var BucketHandle = class {
|
|
|
7159
7203
|
* callback switches to the user's bucket (cache + outbox intact).
|
|
7160
7204
|
*/
|
|
7161
7205
|
const LAST_BUCKET_KEY = "sp00ky:last_bucket";
|
|
7206
|
+
/** Reported for engines that don't track local-store durability. Frozen so a
|
|
7207
|
+
* subscriber can't mutate the shared snapshot. */
|
|
7208
|
+
const UNKNOWN_STORAGE_HEALTH = Object.freeze({
|
|
7209
|
+
status: "unknown",
|
|
7210
|
+
fallback: false
|
|
7211
|
+
});
|
|
7162
7212
|
function readBootBucketHint() {
|
|
7163
7213
|
try {
|
|
7164
7214
|
return typeof localStorage !== "undefined" ? localStorage.getItem(LAST_BUCKET_KEY) : null;
|
|
@@ -7219,6 +7269,21 @@ var Sp00kyClient = class {
|
|
|
7219
7269
|
subscribeToSyncHealth(cb) {
|
|
7220
7270
|
return this.sync.subscribeToSyncHealth(cb);
|
|
7221
7271
|
}
|
|
7272
|
+
/** Durability of the local cache. See {@link StorageHealth}. `'unknown'` for
|
|
7273
|
+
* engines that don't report it. */
|
|
7274
|
+
get storageHealth() {
|
|
7275
|
+
return this.local.storageHealth ?? UNKNOWN_STORAGE_HEALTH;
|
|
7276
|
+
}
|
|
7277
|
+
/**
|
|
7278
|
+
* Observe local-store durability. Fires immediately with the current snapshot
|
|
7279
|
+
* and again on every change (at most once per bucket open in practice).
|
|
7280
|
+
* Returns an unsubscribe.
|
|
7281
|
+
*/
|
|
7282
|
+
subscribeToStorageHealth(cb) {
|
|
7283
|
+
if (this.local.subscribeToStorageHealth) return this.local.subscribeToStorageHealth(cb);
|
|
7284
|
+
cb(UNKNOWN_STORAGE_HEALTH);
|
|
7285
|
+
return () => {};
|
|
7286
|
+
}
|
|
7222
7287
|
constructor(config) {
|
|
7223
7288
|
this.config = config;
|
|
7224
7289
|
const logger = createLogger(config.logLevel ?? "info", config.otelTransmit);
|
package/dist/sqlite-worker.js
CHANGED
|
@@ -1,6 +1,65 @@
|
|
|
1
1
|
import { i as reviveRow, n as renderOrderSql, o as resolveRelations, r as renderWhereSql, s as stableKey, t as project } from "./sqlite-plan-sql.js";
|
|
2
2
|
import sqlite3InitModule from "@sqlite.org/sqlite-wasm";
|
|
3
3
|
|
|
4
|
+
//#region src/services/database/sqlite-open.ts
|
|
5
|
+
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
6
|
+
/** Bounded on purpose: this runs on the boot path, before the first query. */
|
|
7
|
+
const DEFAULT_BACKOFF_MS = [250, 500];
|
|
8
|
+
/** Failures no retry can fix: the APIs aren't there at all (insecure context,
|
|
9
|
+
* or a browser without sync access handles). Fall back immediately. */
|
|
10
|
+
const UNRETRYABLE = ["Missing required OPFS APIs"];
|
|
11
|
+
const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
12
|
+
/** Keep the DOMException name (e.g. `NoModificationAllowedError` for a pool
|
|
13
|
+
* locked by another tab): it is the most diagnostic part of the failure. */
|
|
14
|
+
function errMessage(e) {
|
|
15
|
+
if (e instanceof Error) return e.name && e.name !== "Error" ? `${e.name}: ${e.message}` : e.message;
|
|
16
|
+
return String(e);
|
|
17
|
+
}
|
|
18
|
+
function fallbackToMemory(sqlite3, dbName, reason, attempts) {
|
|
19
|
+
const tried = attempts > 0 ? ` after ${attempts} attempt${attempts === 1 ? "" : "s"}` : "";
|
|
20
|
+
console.error(`[sp00ky] OPFS persistence unavailable for "${dbName}"${tried}: ${reason}. The local SQLite cache is running IN MEMORY, which keeps the whole dataset in RAM and loses every local write on reload. The usual cause is another tab of this app holding the storage lock, so closing the other tabs and reloading restores persistence.`);
|
|
21
|
+
return {
|
|
22
|
+
db: new sqlite3.oo1.DB(":memory:", "c"),
|
|
23
|
+
persisted: false,
|
|
24
|
+
opfsError: reason
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Open `dbName`'s handle. Never throws for a storage problem: a caller that
|
|
29
|
+
* asked for persistence and can't have it gets a working in-memory handle plus
|
|
30
|
+
* `persisted: false` and an `opfsError` to report.
|
|
31
|
+
*/
|
|
32
|
+
async function openDb(sqlite3, dbName, useOpfs, opts = {}) {
|
|
33
|
+
if (!useOpfs) return {
|
|
34
|
+
db: new sqlite3.oo1.DB(":memory:", "c"),
|
|
35
|
+
persisted: false
|
|
36
|
+
};
|
|
37
|
+
if (!sqlite3.installOpfsSAHPoolVfs) return fallbackToMemory(sqlite3, dbName, "sqlite-wasm build has no installOpfsSAHPoolVfs", 0);
|
|
38
|
+
const maxAttempts = Math.max(1, opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
|
|
39
|
+
const backoffMs = opts.backoffMs ?? DEFAULT_BACKOFF_MS;
|
|
40
|
+
const sleep = opts.sleep ?? defaultSleep;
|
|
41
|
+
let lastError = "unknown error";
|
|
42
|
+
let attempts = 0;
|
|
43
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
44
|
+
attempts = attempt;
|
|
45
|
+
try {
|
|
46
|
+
return {
|
|
47
|
+
db: new (await (sqlite3.installOpfsSAHPoolVfs({
|
|
48
|
+
name: `sp00ky-${dbName}`,
|
|
49
|
+
...attempt > 1 ? { forceReinitIfPreviouslyFailed: true } : {}
|
|
50
|
+
}))).OpfsSAHPoolDb(`/${dbName}.sqlite3`),
|
|
51
|
+
persisted: true
|
|
52
|
+
};
|
|
53
|
+
} catch (e) {
|
|
54
|
+
lastError = errMessage(e);
|
|
55
|
+
if (attempt === maxAttempts || UNRETRYABLE.some((m) => lastError.includes(m))) break;
|
|
56
|
+
await sleep(backoffMs[Math.min(attempt - 1, backoffMs.length - 1)] ?? 0);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return fallbackToMemory(sqlite3, dbName, lastError, attempts);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
//#endregion
|
|
4
63
|
//#region src/services/database/sqlite-select.ts
|
|
5
64
|
function ensureTable(db, table) {
|
|
6
65
|
if (db.knownTables.has(table)) return;
|
|
@@ -79,11 +138,12 @@ async function executeSelect(plan, params, db) {
|
|
|
79
138
|
* is also what the OPFS VFS requires — file access must happen off the main
|
|
80
139
|
* thread. Persistence uses the **OPFS SAHPool VFS**: durable, and (unlike the
|
|
81
140
|
* classic OPFS VFS) it does NOT require COOP/COEP cross-origin isolation
|
|
82
|
-
* headers, so host apps embedding the client need no server changes.
|
|
83
|
-
* to an in-memory DB
|
|
141
|
+
* headers, so host apps embedding the client need no server changes. When OPFS
|
|
142
|
+
* is unavailable it retries, then falls back to an in-memory DB and REPORTS the
|
|
143
|
+
* loss of durability (see `sqlite-open.ts`) instead of degrading silently.
|
|
84
144
|
*
|
|
85
145
|
* Message protocol (request/response keyed by `id`):
|
|
86
|
-
* { id, type: 'open', payload: { dbName, useOpfs } }
|
|
146
|
+
* { id, type: 'open', payload: { dbName, useOpfs } } -> { id, ok, persisted, opfsError? }
|
|
87
147
|
* { id, type: 'exec', payload: { sql, bind } } -> { id, ok, rows }
|
|
88
148
|
* { id, type: 'run', payload: { sql, bind } } -> { id, ok }
|
|
89
149
|
* { id, type: 'batch', payload: [{ sql, bind }] } (atomic BEGIN/COMMIT)
|
|
@@ -97,18 +157,16 @@ async function executeSelect(plan, params, db) {
|
|
|
97
157
|
*/
|
|
98
158
|
let db = null;
|
|
99
159
|
async function open(dbName, useOpfs, systemTables = []) {
|
|
100
|
-
const
|
|
101
|
-
|
|
102
|
-
if (useOpfs && sqlite3.installOpfsSAHPoolVfs) try {
|
|
103
|
-
db = new (await (sqlite3.installOpfsSAHPoolVfs({ name: `sp00ky-${dbName}` }))).OpfsSAHPoolDb(`/${dbName}.sqlite3`);
|
|
104
|
-
persisted = true;
|
|
105
|
-
} catch {}
|
|
106
|
-
if (!db) db = new sqlite3.oo1.DB(":memory:", "c");
|
|
160
|
+
const { db: handle, persisted, opfsError } = await openDb(await sqlite3InitModule(), dbName, useOpfs);
|
|
161
|
+
db = handle;
|
|
107
162
|
for (const t of systemTables) db.exec({ sql: `CREATE TABLE IF NOT EXISTS "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)` });
|
|
108
163
|
try {
|
|
109
164
|
db.exec({ sql: "PRAGMA busy_timeout = 5000; PRAGMA cache_size = -32000;" });
|
|
110
165
|
} catch {}
|
|
111
|
-
return {
|
|
166
|
+
return {
|
|
167
|
+
persisted,
|
|
168
|
+
opfsError
|
|
169
|
+
};
|
|
112
170
|
}
|
|
113
171
|
function exec(sql, bind) {
|
|
114
172
|
if (!db) throw new Error("sqlite: DB not open");
|
package/dist/types.d.ts
CHANGED
|
@@ -288,6 +288,15 @@ interface LocalStore extends LocalCacheEngine {
|
|
|
288
288
|
getClient(): unknown;
|
|
289
289
|
getConfig(): Sp00kyConfig<any>['database'];
|
|
290
290
|
readonly currentBucketId: string;
|
|
291
|
+
/**
|
|
292
|
+
* Durability of this engine's local store. OPTIONAL: engines that don't
|
|
293
|
+
* report it (SurrealDB, custom engines) are treated as `'unknown'` by the
|
|
294
|
+
* client facade, so adding this needs no change on their side.
|
|
295
|
+
*/
|
|
296
|
+
readonly storageHealth?: StorageHealth;
|
|
297
|
+
/** Fires immediately with the current snapshot, then on every change.
|
|
298
|
+
* Returns an unsubscribe function. */
|
|
299
|
+
subscribeToStorageHealth?(cb: (health: StorageHealth) => void): () => void;
|
|
291
300
|
}
|
|
292
301
|
/** Selected local cache backend. Mirrors the `persistenceClient` config pattern. */
|
|
293
302
|
type LocalEngineChoice = 'surrealdb' | 'sqlite' | LocalStore;
|
|
@@ -530,6 +539,31 @@ interface SyncHealth {
|
|
|
530
539
|
*/
|
|
531
540
|
everConnected: boolean;
|
|
532
541
|
}
|
|
542
|
+
type StorageHealthStatus = 'unknown' | 'persistent' | 'memory';
|
|
543
|
+
/**
|
|
544
|
+
* Durability of the LOCAL cache, delivered to `subscribeToStorageHealth`
|
|
545
|
+
* subscribers. Separate from {@link SyncHealth}: that one is about reaching the
|
|
546
|
+
* server, this one is about whether the local store survives a reload.
|
|
547
|
+
*
|
|
548
|
+
* Under `localEngine: 'sqlite'` the durable store is the OPFS SAHPool VFS,
|
|
549
|
+
* which only one client per bucket can hold open. When it can't be opened (a
|
|
550
|
+
* second tab of the app already has it, an insecure context, a full pool) the
|
|
551
|
+
* engine keeps working against an in-memory DB, which holds the whole dataset
|
|
552
|
+
* in RAM and loses local writes on reload. `fallback` marks exactly that case,
|
|
553
|
+
* so a UI can warn about it.
|
|
554
|
+
*/
|
|
555
|
+
interface StorageHealth {
|
|
556
|
+
/** `'unknown'` until the local cache has opened, or for engines that don't report. */
|
|
557
|
+
status: StorageHealthStatus;
|
|
558
|
+
/**
|
|
559
|
+
* `true` only when durable storage was REQUESTED and could not be opened.
|
|
560
|
+
* Stays `false` for a configured-in-memory store (`store: 'memory'`), which
|
|
561
|
+
* is a choice rather than a failure, so a UI can key off this alone.
|
|
562
|
+
*/
|
|
563
|
+
fallback: boolean;
|
|
564
|
+
/** Reason durable storage failed (only set while `fallback` is `true`). */
|
|
565
|
+
error?: string;
|
|
566
|
+
}
|
|
533
567
|
type QueryHash = string;
|
|
534
568
|
type RecordVersionArray = Array<[string, number]>;
|
|
535
569
|
/**
|
|
@@ -752,4 +786,4 @@ interface DebounceOptions {
|
|
|
752
786
|
delay?: number;
|
|
753
787
|
}
|
|
754
788
|
//#endregion
|
|
755
|
-
export {
|
|
789
|
+
export { StorageHealthStatus as A, DatabaseEventSystem as B, RecordVersionDiff as C, Sp00kyQueryResult as D, Sp00kyConfig as E, TimingPhase as F, EventSystem as G, Logger$1 as H, UpdateOptions as I, UpEvent as L, SyncHealth as M, SyncHealthConfig as N, Sp00kyQueryResultPromise as O, SyncHealthStatus as P, LocalStore as R, RecordVersionArray as S, RunOptions as T, SyncEventSystem as U, DatabaseEventTypes as V, EventDefinition as W, QueryStatus as _, MutationCallback as a, QueryTimings as b, PersistenceClient as c, PreloadOptions as d, PreloadRefresh as f, QueryState as g, QueryHash as h, MATERIALIZATION_SAMPLE_WINDOW as i, StoreType as j, StorageHealth as k, PhaseStat as l, QueryConfigRecord as m, EventSubscriptionOptions as n, MutationEvent as o, QueryConfig as p, Level$1 as r, MutationEventType as s, DebounceOptions as t, PinoTransmit as u, QueryStatusCallback as v, RegistrationTimings as w, QueryUpdateCallback as x, QueryTimeToLive as y, SealedQuery as z };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.153",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -60,8 +60,8 @@
|
|
|
60
60
|
}
|
|
61
61
|
},
|
|
62
62
|
"dependencies": {
|
|
63
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
64
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
63
|
+
"@spooky-sync/query-builder": "0.0.1-canary.153",
|
|
64
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.153",
|
|
65
65
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
66
66
|
"@surrealdb/wasm": "^3.0.3",
|
|
67
67
|
"fast-json-patch": "^3.1.1",
|
|
@@ -96,6 +96,11 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
96
96
|
}
|
|
97
97
|
});
|
|
98
98
|
|
|
99
|
+
// Push state when the local store reports its durability (the open happens
|
|
100
|
+
// during connect, typically before a panel attaches, so this mostly matters
|
|
101
|
+
// for a later bucket switch that loses OPFS).
|
|
102
|
+
this.databaseService.subscribeToStorageHealth?.(() => this.notifyDevTools());
|
|
103
|
+
|
|
99
104
|
// Fire-and-forget backend version discovery; re-push state when it lands.
|
|
100
105
|
void this.refreshBackendVersions();
|
|
101
106
|
|
|
@@ -318,6 +323,9 @@ export class DevToolsService implements StreamUpdateReceiver {
|
|
|
318
323
|
? this.localTables
|
|
319
324
|
: this.schema.tables.map((t) => t.name),
|
|
320
325
|
tableData: {},
|
|
326
|
+
// Durability of the local store. `fallback: true` means persistence was
|
|
327
|
+
// requested but the dataset is actually sitting in RAM.
|
|
328
|
+
storage: this.databaseService.storageHealth ?? { status: 'unknown', fallback: false },
|
|
321
329
|
},
|
|
322
330
|
});
|
|
323
331
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { QueryPlan, RelationPlan, WhereNode } from '@spooky-sync/query-builder';
|
|
2
2
|
import type { SealedQuery } from '../../utils/surql';
|
|
3
3
|
import type { DatabaseEventSystem } from './events/index';
|
|
4
|
-
import type { Sp00kyConfig } from '../../types';
|
|
4
|
+
import type { Sp00kyConfig, StorageHealth } from '../../types';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
7
|
* A materialized row. Keys are field names; values are already decoded to the
|
|
@@ -121,6 +121,15 @@ export interface LocalStore extends LocalCacheEngine {
|
|
|
121
121
|
getClient(): unknown;
|
|
122
122
|
getConfig(): Sp00kyConfig<any>['database'];
|
|
123
123
|
readonly currentBucketId: string;
|
|
124
|
+
/**
|
|
125
|
+
* Durability of this engine's local store. OPTIONAL: engines that don't
|
|
126
|
+
* report it (SurrealDB, custom engines) are treated as `'unknown'` by the
|
|
127
|
+
* client facade, so adding this needs no change on their side.
|
|
128
|
+
*/
|
|
129
|
+
readonly storageHealth?: StorageHealth;
|
|
130
|
+
/** Fires immediately with the current snapshot, then on every change.
|
|
131
|
+
* Returns an unsubscribe function. */
|
|
132
|
+
subscribeToStorageHealth?(cb: (health: StorageHealth) => void): () => void;
|
|
124
133
|
}
|
|
125
134
|
|
|
126
135
|
/** Selected local cache backend. Mirrors the `persistenceClient` config pattern. */
|
|
@@ -135,6 +135,93 @@ describe('SqliteCacheEngine system-table seeding', () => {
|
|
|
135
135
|
});
|
|
136
136
|
});
|
|
137
137
|
|
|
138
|
+
// The worker's `persisted`/`opfsError` reply used to die in a `logger.info`
|
|
139
|
+
// line, so a host app running pino at `fatal` (whitepawn does) could not tell a
|
|
140
|
+
// disk-backed store from a full-RAM one. It now lands on the engine as
|
|
141
|
+
// observable state the app can render.
|
|
142
|
+
describe('SqliteCacheEngine storage health', () => {
|
|
143
|
+
/** Engine wired to a worker whose `open` replies with `openReply`. */
|
|
144
|
+
function makeEngine(openReply: Record<string, unknown>, opts?: { useOpfs?: boolean }) {
|
|
145
|
+
const logs: { level: string; msg: string; meta: any }[] = [];
|
|
146
|
+
const logger: any = {};
|
|
147
|
+
for (const level of ['debug', 'info', 'warn', 'error', 'trace']) {
|
|
148
|
+
logger[level] = (meta: any, msg: string) => logs.push({ level, msg, meta });
|
|
149
|
+
}
|
|
150
|
+
logger.child = () => logger;
|
|
151
|
+
|
|
152
|
+
const engine = new SqliteCacheEngine(
|
|
153
|
+
{ namespace: 'n', database: 'd' } as any,
|
|
154
|
+
logger,
|
|
155
|
+
opts ?? {}
|
|
156
|
+
);
|
|
157
|
+
(engine as any).spawnWorker = () => {
|
|
158
|
+
const w: any = { onmessage: null, onerror: null, onmessageerror: null, terminate() {} };
|
|
159
|
+
w.postMessage = (msg: any) => {
|
|
160
|
+
Promise.resolve().then(() => {
|
|
161
|
+
const rest = msg.type === 'open' ? openReply : {};
|
|
162
|
+
const { id, ok, error, ...payload } = { id: msg.id, ok: true, error: undefined, ...rest };
|
|
163
|
+
const p = (engine as any).pending.get(id);
|
|
164
|
+
if (!p) return;
|
|
165
|
+
(engine as any).pending.delete(id);
|
|
166
|
+
if (ok) p.resolve(payload);
|
|
167
|
+
else p.reject(new Error(error));
|
|
168
|
+
});
|
|
169
|
+
};
|
|
170
|
+
return w as unknown as Worker;
|
|
171
|
+
};
|
|
172
|
+
return { engine, logs };
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
it('publishes a persistent store and logs no error', async () => {
|
|
176
|
+
const { engine, logs } = makeEngine({ persisted: true });
|
|
177
|
+
await engine.connect('user:abc');
|
|
178
|
+
|
|
179
|
+
expect(engine.storageHealth).toEqual({
|
|
180
|
+
status: 'persistent',
|
|
181
|
+
fallback: false,
|
|
182
|
+
error: undefined,
|
|
183
|
+
});
|
|
184
|
+
expect(logs.some((l) => l.level === 'error')).toBe(false);
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
it('publishes the fallback, its reason, and an error log when OPFS is lost', async () => {
|
|
188
|
+
const { engine, logs } = makeEngine({ persisted: false, opfsError: 'NoModificationAllowedError: locked' });
|
|
189
|
+
await engine.connect('user:abc');
|
|
190
|
+
|
|
191
|
+
expect(engine.storageHealth).toEqual({
|
|
192
|
+
status: 'memory',
|
|
193
|
+
fallback: true,
|
|
194
|
+
error: 'NoModificationAllowedError: locked',
|
|
195
|
+
});
|
|
196
|
+
const err = logs.find((l) => l.level === 'error');
|
|
197
|
+
expect(err?.msg).toContain('IN MEMORY');
|
|
198
|
+
expect(err?.meta.opfsError).toBe('NoModificationAllowedError: locked');
|
|
199
|
+
// Inspectable from the console without any logging configured.
|
|
200
|
+
expect((globalThis as any).__sqliteStats.persisted).toBe(false);
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
// A subscriber almost always attaches AFTER connect() (components mount
|
|
204
|
+
// later), so an immediate fire is the only way it learns about a fallback.
|
|
205
|
+
it('fires a late subscriber with the current snapshot', async () => {
|
|
206
|
+
const { engine } = makeEngine({ persisted: false, opfsError: 'boom' });
|
|
207
|
+
await engine.connect('user:abc');
|
|
208
|
+
|
|
209
|
+
const seen: any[] = [];
|
|
210
|
+
const unsub = engine.subscribeToStorageHealth((h) => seen.push(h));
|
|
211
|
+
expect(seen).toEqual([{ status: 'memory', fallback: true, error: 'boom' }]);
|
|
212
|
+
unsub();
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
// `store: 'memory'` asked for RAM, so it is not a fallback and must not warn.
|
|
216
|
+
it('does not flag a configured in-memory store as a fallback', async () => {
|
|
217
|
+
const { engine, logs } = makeEngine({ persisted: false }, { useOpfs: false });
|
|
218
|
+
await engine.connect('user:abc');
|
|
219
|
+
|
|
220
|
+
expect(engine.storageHealth).toEqual({ status: 'memory', fallback: false, error: undefined });
|
|
221
|
+
expect(logs.some((l) => l.level === 'error')).toBe(false);
|
|
222
|
+
});
|
|
223
|
+
});
|
|
224
|
+
|
|
138
225
|
// `pureWriteOpResult` is the single source of truth for what a pure-write op
|
|
139
226
|
// contributes to a query's per-statement results. The batched fast path in
|
|
140
227
|
// `query()` and the per-op `execOp` path BOTH route through it, so a caller that
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
project,
|
|
9
9
|
} from './sqlite-plan-sql';
|
|
10
10
|
import type { Logger } from '../logger/index';
|
|
11
|
-
import type { Sp00kyConfig } from '../../types';
|
|
11
|
+
import type { Sp00kyConfig, StorageHealth } from '../../types';
|
|
12
12
|
import type { SealedQuery } from '../../utils/surql';
|
|
13
13
|
import { resolveRelations, stableKey } from './relation-resolver';
|
|
14
14
|
import {
|
|
@@ -94,6 +94,11 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
94
94
|
private workerSelect: boolean;
|
|
95
95
|
private events: DatabaseEventSystem = createDatabaseEventSystem();
|
|
96
96
|
private bucketId = 'anon';
|
|
97
|
+
/** Durability of the local store, set on every open. A plain Set of callbacks
|
|
98
|
+
* rather than a `DatabaseEventSystem` event: this changes at most once per
|
|
99
|
+
* open, and the typed event map is about query traffic. */
|
|
100
|
+
private storageHealthValue: StorageHealth = { status: 'unknown', fallback: false };
|
|
101
|
+
private storageHealthSubs = new Set<(health: StorageHealth) => void>();
|
|
97
102
|
/** Schemaless — tables are created lazily on first write; no migrator. */
|
|
98
103
|
readonly usesSurqlSchema = false;
|
|
99
104
|
|
|
@@ -114,6 +119,26 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
114
119
|
return this.bucketId;
|
|
115
120
|
}
|
|
116
121
|
|
|
122
|
+
get storageHealth(): StorageHealth {
|
|
123
|
+
return this.storageHealthValue;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Fires immediately with the current snapshot (the store opens during
|
|
127
|
+
* `connect()`, before app components mount, so a late subscriber must still
|
|
128
|
+
* learn a fallback happened), then on every change. */
|
|
129
|
+
subscribeToStorageHealth(cb: (health: StorageHealth) => void): () => void {
|
|
130
|
+
cb(this.storageHealthValue);
|
|
131
|
+
this.storageHealthSubs.add(cb);
|
|
132
|
+
return () => {
|
|
133
|
+
this.storageHealthSubs.delete(cb);
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
private setStorageHealth(health: StorageHealth): void {
|
|
138
|
+
this.storageHealthValue = health;
|
|
139
|
+
for (const cb of this.storageHealthSubs) cb(health);
|
|
140
|
+
}
|
|
141
|
+
|
|
117
142
|
getConfig(): Sp00kyConfig<any>['database'] {
|
|
118
143
|
return this.config;
|
|
119
144
|
}
|
|
@@ -247,7 +272,12 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
247
272
|
// "no such table: _00_query" and the client wedged on "Loading database".
|
|
248
273
|
// Creating them inside `open` guarantees any access order is safe without
|
|
249
274
|
// adding ops to the engine's queue.
|
|
250
|
-
|
|
275
|
+
// `opfsError` is absent from a worker bundle older than this field, which
|
|
276
|
+
// just reads as "no reason given" rather than breaking the open.
|
|
277
|
+
const { persisted, opfsError } = await this.rawCall<{
|
|
278
|
+
persisted: boolean;
|
|
279
|
+
opfsError?: string;
|
|
280
|
+
}>('open', {
|
|
251
281
|
dbName: bucketId,
|
|
252
282
|
useOpfs: this.useOpfs,
|
|
253
283
|
systemTables: SYSTEM_TABLES,
|
|
@@ -255,10 +285,30 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
255
285
|
this.knownTables.clear();
|
|
256
286
|
for (const t of SYSTEM_TABLES) this.knownTables.add(t);
|
|
257
287
|
this.bucketId = bucketId;
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
288
|
+
// Durability was requested but could not be had: the store is in RAM, so it
|
|
289
|
+
// loses local writes on reload and can OOM a wasm-heavy renderer. Report it
|
|
290
|
+
// (the worker also console.errors, since host apps may run pino at `fatal`)
|
|
291
|
+
// and publish it so the app can warn the user.
|
|
292
|
+
const fellBack = this.useOpfs && !persisted;
|
|
293
|
+
this.setStorageHealth({
|
|
294
|
+
status: persisted ? 'persistent' : 'memory',
|
|
295
|
+
fallback: fellBack,
|
|
296
|
+
error: fellBack ? opfsError : undefined,
|
|
297
|
+
});
|
|
298
|
+
const stats = getStats();
|
|
299
|
+
stats.persisted = persisted;
|
|
300
|
+
stats.opfsError = fellBack ? opfsError : undefined;
|
|
301
|
+
if (fellBack) {
|
|
302
|
+
this.logger.error(
|
|
303
|
+
{ bucketId, opfsError, Category: 'sp00ky-client::SqliteCacheEngine::connect' },
|
|
304
|
+
'SQLite OPFS persistence failed; store is IN MEMORY and will not survive reload'
|
|
305
|
+
);
|
|
306
|
+
} else {
|
|
307
|
+
this.logger.info(
|
|
308
|
+
{ bucketId, persisted, Category: 'sp00ky-client::SqliteCacheEngine::connect' },
|
|
309
|
+
persisted ? 'SQLite OPFS store opened' : 'SQLite in-memory store opened (as configured)'
|
|
310
|
+
);
|
|
311
|
+
}
|
|
262
312
|
}
|
|
263
313
|
|
|
264
314
|
/** Enqueue `fn` as a single serialized opQueue entry (mirrors {@link call}'s
|
|
@@ -743,6 +793,12 @@ interface SqliteStats {
|
|
|
743
793
|
bytesParsed: number;
|
|
744
794
|
/** Relation-resolver fan-out fetches (one worker round-trip each). */
|
|
745
795
|
relationFetches: number;
|
|
796
|
+
/** Whether the open store is OPFS-backed. `false` here with an `opfsError`
|
|
797
|
+
* means the whole dataset is sitting in RAM. Optional so it stays absent
|
|
798
|
+
* until the first open (and is skipped by the backfill loop below). */
|
|
799
|
+
persisted?: boolean;
|
|
800
|
+
/** Why OPFS persistence failed, when it did. */
|
|
801
|
+
opfsError?: string;
|
|
746
802
|
}
|
|
747
803
|
|
|
748
804
|
const EMPTY_STATS: SqliteStats = {
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import { openDb } from './sqlite-open';
|
|
3
|
+
|
|
4
|
+
class FakeDb {
|
|
5
|
+
constructor(public arg: unknown) {}
|
|
6
|
+
exec() {
|
|
7
|
+
return [];
|
|
8
|
+
}
|
|
9
|
+
close() {}
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** Minimal stand-in for the initialized sqlite-wasm module. `install` is the
|
|
13
|
+
* `installOpfsSAHPoolVfs` behavior under test; omit it to model a build that
|
|
14
|
+
* lacks the SAHPool VFS entirely. */
|
|
15
|
+
function makeSqlite3(install?: (opts: any) => Promise<unknown>) {
|
|
16
|
+
const sqlite3: any = { oo1: { DB: FakeDb } };
|
|
17
|
+
if (install) sqlite3.installOpfsSAHPoolVfs = vi.fn(install);
|
|
18
|
+
return sqlite3;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const pool = { OpfsSAHPoolDb: FakeDb };
|
|
22
|
+
/** What a pool locked by another tab of the app actually throws. */
|
|
23
|
+
function lockedError(): Error {
|
|
24
|
+
const e = new Error('Access Handles cannot be acquired');
|
|
25
|
+
e.name = 'NoModificationAllowedError';
|
|
26
|
+
return e;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const noSleep = () => Promise.resolve();
|
|
30
|
+
|
|
31
|
+
let errSpy: ReturnType<typeof vi.spyOn>;
|
|
32
|
+
beforeEach(() => {
|
|
33
|
+
errSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
34
|
+
});
|
|
35
|
+
afterEach(() => {
|
|
36
|
+
errSpy.mockRestore();
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
// The bare `catch {}` this replaces turned every OPFS failure into a silent
|
|
40
|
+
// full-RAM database: no log, no reason, no way for the app to know its writes
|
|
41
|
+
// die on reload. Each case below pins one half of the fix: keep trying when
|
|
42
|
+
// retrying can plausibly work, and when it can't, say so loudly and hand the
|
|
43
|
+
// reason back.
|
|
44
|
+
describe('openDb', () => {
|
|
45
|
+
it('opens the OPFS pool on the first attempt', async () => {
|
|
46
|
+
const sqlite3 = makeSqlite3(async () => pool);
|
|
47
|
+
const res = await openDb(sqlite3, 'user:abc', true, { sleep: noSleep });
|
|
48
|
+
|
|
49
|
+
expect(res.persisted).toBe(true);
|
|
50
|
+
expect(res.opfsError).toBeUndefined();
|
|
51
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
|
|
52
|
+
// The pool is named per bucket, and the first try must NOT force a re-init
|
|
53
|
+
// (that would throw away a pool another attempt is legitimately using).
|
|
54
|
+
expect(sqlite3.installOpfsSAHPoolVfs.mock.calls[0][0]).toEqual({ name: 'sp00ky-user:abc' });
|
|
55
|
+
expect(errSpy).not.toHaveBeenCalled();
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
// The tab-closing race: the old tab still holds the sync access handles when
|
|
59
|
+
// the new one boots. Without a retry that tab is stuck in RAM for its whole
|
|
60
|
+
// lifetime, even though the lock frees milliseconds later.
|
|
61
|
+
it('retries a locked pool and succeeds, forcing re-init after the first failure', async () => {
|
|
62
|
+
let calls = 0;
|
|
63
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
64
|
+
if (++calls < 3) throw lockedError();
|
|
65
|
+
return pool;
|
|
66
|
+
});
|
|
67
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
68
|
+
|
|
69
|
+
expect(res.persisted).toBe(true);
|
|
70
|
+
expect(res.opfsError).toBeUndefined();
|
|
71
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
|
|
72
|
+
// sqlite-wasm caches the first rejection against the VFS name, so retries
|
|
73
|
+
// that don't ask for a real re-init just replay it.
|
|
74
|
+
const [first, second, third] = sqlite3.installOpfsSAHPoolVfs.mock.calls.map((c: any[]) => c[0]);
|
|
75
|
+
expect(first.forceReinitIfPreviouslyFailed).toBeUndefined();
|
|
76
|
+
expect(second.forceReinitIfPreviouslyFailed).toBe(true);
|
|
77
|
+
expect(third.forceReinitIfPreviouslyFailed).toBe(true);
|
|
78
|
+
expect(errSpy).not.toHaveBeenCalled();
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('falls back loudly after exhausting the retries, keeping the reason', async () => {
|
|
82
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
83
|
+
throw lockedError();
|
|
84
|
+
});
|
|
85
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
86
|
+
|
|
87
|
+
expect(res.persisted).toBe(false);
|
|
88
|
+
// The DOMException name is the diagnostic part: it names the lock.
|
|
89
|
+
expect(res.opfsError).toContain('NoModificationAllowedError');
|
|
90
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(3);
|
|
91
|
+
expect(errSpy).toHaveBeenCalledTimes(1);
|
|
92
|
+
expect(String(errSpy.mock.calls[0][0])).toContain('IN MEMORY');
|
|
93
|
+
// Still a usable handle: losing durability must not break the app.
|
|
94
|
+
expect(res.db).toBeDefined();
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
// An insecure context has no sync access handles at all, so retrying just
|
|
98
|
+
// adds boot latency to a foregone conclusion.
|
|
99
|
+
it('does not retry when the OPFS APIs are missing entirely', async () => {
|
|
100
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
101
|
+
throw new Error('Missing required OPFS APIs.');
|
|
102
|
+
});
|
|
103
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
104
|
+
|
|
105
|
+
expect(res.persisted).toBe(false);
|
|
106
|
+
expect(res.opfsError).toContain('Missing required OPFS APIs');
|
|
107
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(1);
|
|
108
|
+
expect(errSpy).toHaveBeenCalledTimes(1);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('reports a build without the SAHPool VFS without calling anything', async () => {
|
|
112
|
+
const sqlite3 = makeSqlite3();
|
|
113
|
+
const res = await openDb(sqlite3, 'main', true, { sleep: noSleep });
|
|
114
|
+
|
|
115
|
+
expect(res.persisted).toBe(false);
|
|
116
|
+
expect(res.opfsError).toContain('installOpfsSAHPoolVfs');
|
|
117
|
+
expect(errSpy).toHaveBeenCalledTimes(1);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
// `store: 'memory'` is a configuration choice, not a degradation, so it must
|
|
121
|
+
// stay silent and carry no error for the UI to warn about.
|
|
122
|
+
it('opens in memory quietly when persistence was not requested', async () => {
|
|
123
|
+
const sqlite3 = makeSqlite3(async () => pool);
|
|
124
|
+
const res = await openDb(sqlite3, 'main', false, { sleep: noSleep });
|
|
125
|
+
|
|
126
|
+
expect(res.persisted).toBe(false);
|
|
127
|
+
expect(res.opfsError).toBeUndefined();
|
|
128
|
+
expect(sqlite3.installOpfsSAHPoolVfs).not.toHaveBeenCalled();
|
|
129
|
+
expect(errSpy).not.toHaveBeenCalled();
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('honors maxAttempts and waits the configured backoff between tries', async () => {
|
|
133
|
+
const slept: number[] = [];
|
|
134
|
+
const sqlite3 = makeSqlite3(async () => {
|
|
135
|
+
throw lockedError();
|
|
136
|
+
});
|
|
137
|
+
const res = await openDb(sqlite3, 'main', true, {
|
|
138
|
+
maxAttempts: 4,
|
|
139
|
+
backoffMs: [10, 20],
|
|
140
|
+
sleep: async (ms) => {
|
|
141
|
+
slept.push(ms);
|
|
142
|
+
},
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
expect(res.persisted).toBe(false);
|
|
146
|
+
expect(sqlite3.installOpfsSAHPoolVfs).toHaveBeenCalledTimes(4);
|
|
147
|
+
// One sleep per gap (never after the last attempt), last delay repeating.
|
|
148
|
+
expect(slept).toEqual([10, 20, 20]);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Opening the worker's SQLite handle: the OPFS SAHPool VFS when durable
|
|
3
|
+
* storage was asked for, an in-memory DB only as a last resort. Extracted from
|
|
4
|
+
* `sqlite-worker.ts` (which imports the wasm module at module scope and so
|
|
5
|
+
* can't be loaded in a unit test) to keep the retry/fallback policy testable
|
|
6
|
+
* off-worker, the same split as `sqlite-select.ts`.
|
|
7
|
+
*
|
|
8
|
+
* Why retry: SAHPool holds an EXCLUSIVE sync access handle on every file in
|
|
9
|
+
* its pool, so only one client per pool name can have it open. A second tab of
|
|
10
|
+
* the same app therefore fails init, and `installOpfsSAHPoolVfs` CACHES that
|
|
11
|
+
* rejection per VFS name, so a later call only gets a real second chance when
|
|
12
|
+
* it passes `forceReinitIfPreviouslyFailed`. Retrying with that flag turns the
|
|
13
|
+
* common "the other tab is still closing" race into a success instead of a
|
|
14
|
+
* permanent in-memory session.
|
|
15
|
+
*
|
|
16
|
+
* Why the noise: `:memory:` holds the whole dataset in RAM (the
|
|
17
|
+
* OOM-on-wasm-heavy-pages failure mode the OPFS store exists to avoid) and
|
|
18
|
+
* drops every local write on reload. Host apps run pino at their own level,
|
|
19
|
+
* some at `fatal`, so the fallback ALSO writes to `console.error` from inside
|
|
20
|
+
* the worker, and the reason travels back to the engine as `opfsError` for the
|
|
21
|
+
* app to surface.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** The DB surface the worker uses (a `sqlite3.oo1.DB` or an `OpfsSAHPoolDb`). */
|
|
25
|
+
export interface SqliteDbHandle {
|
|
26
|
+
exec: (opts: { sql: string; bind?: unknown[]; rowMode?: string; returnValue?: string }) => unknown;
|
|
27
|
+
close: () => void;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface OpenDbResult {
|
|
31
|
+
db: SqliteDbHandle;
|
|
32
|
+
/** True only when the handle is backed by OPFS and survives a reload. */
|
|
33
|
+
persisted: boolean;
|
|
34
|
+
/** Why persistence failed. Set only when OPFS was requested and fell back. */
|
|
35
|
+
opfsError?: string;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface OpenDbOptions {
|
|
39
|
+
/** Total OPFS init attempts, including the first. Default 3. */
|
|
40
|
+
maxAttempts?: number;
|
|
41
|
+
/** Delay before each retry; the last entry repeats. Default [250, 500]. */
|
|
42
|
+
backoffMs?: number[];
|
|
43
|
+
/** Injectable for tests. */
|
|
44
|
+
sleep?: (ms: number) => Promise<void>;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const DEFAULT_MAX_ATTEMPTS = 3;
|
|
48
|
+
/** Bounded on purpose: this runs on the boot path, before the first query. */
|
|
49
|
+
const DEFAULT_BACKOFF_MS = [250, 500];
|
|
50
|
+
|
|
51
|
+
/** Failures no retry can fix: the APIs aren't there at all (insecure context,
|
|
52
|
+
* or a browser without sync access handles). Fall back immediately. */
|
|
53
|
+
const UNRETRYABLE = ['Missing required OPFS APIs'];
|
|
54
|
+
|
|
55
|
+
const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
56
|
+
|
|
57
|
+
/** Keep the DOMException name (e.g. `NoModificationAllowedError` for a pool
|
|
58
|
+
* locked by another tab): it is the most diagnostic part of the failure. */
|
|
59
|
+
function errMessage(e: unknown): string {
|
|
60
|
+
if (e instanceof Error) return e.name && e.name !== 'Error' ? `${e.name}: ${e.message}` : e.message;
|
|
61
|
+
return String(e);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function fallbackToMemory(
|
|
65
|
+
sqlite3: any,
|
|
66
|
+
dbName: string,
|
|
67
|
+
reason: string,
|
|
68
|
+
attempts: number
|
|
69
|
+
): OpenDbResult {
|
|
70
|
+
const tried = attempts > 0 ? ` after ${attempts} attempt${attempts === 1 ? '' : 's'}` : '';
|
|
71
|
+
// Deliberately console, not the logger: host apps configure pino's level (some
|
|
72
|
+
// run `fatal`), and losing durability must never be filtered into silence.
|
|
73
|
+
// oxlint-disable-next-line no-console
|
|
74
|
+
console.error(
|
|
75
|
+
`[sp00ky] OPFS persistence unavailable for "${dbName}"${tried}: ${reason}. The local SQLite ` +
|
|
76
|
+
'cache is running IN MEMORY, which keeps the whole dataset in RAM and loses every local ' +
|
|
77
|
+
'write on reload. The usual cause is another tab of this app holding the storage lock, so ' +
|
|
78
|
+
'closing the other tabs and reloading restores persistence.'
|
|
79
|
+
);
|
|
80
|
+
return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false, opfsError: reason };
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Open `dbName`'s handle. Never throws for a storage problem: a caller that
|
|
85
|
+
* asked for persistence and can't have it gets a working in-memory handle plus
|
|
86
|
+
* `persisted: false` and an `opfsError` to report.
|
|
87
|
+
*/
|
|
88
|
+
export async function openDb(
|
|
89
|
+
sqlite3: any,
|
|
90
|
+
dbName: string,
|
|
91
|
+
useOpfs: boolean,
|
|
92
|
+
opts: OpenDbOptions = {}
|
|
93
|
+
): Promise<OpenDbResult> {
|
|
94
|
+
// Memory was the configured choice (`store: 'memory'`), not a failure, so no
|
|
95
|
+
// error and no noise.
|
|
96
|
+
if (!useOpfs) return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false };
|
|
97
|
+
|
|
98
|
+
if (!sqlite3.installOpfsSAHPoolVfs) {
|
|
99
|
+
return fallbackToMemory(sqlite3, dbName, 'sqlite-wasm build has no installOpfsSAHPoolVfs', 0);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const maxAttempts = Math.max(1, opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS);
|
|
103
|
+
const backoffMs = opts.backoffMs ?? DEFAULT_BACKOFF_MS;
|
|
104
|
+
const sleep = opts.sleep ?? defaultSleep;
|
|
105
|
+
|
|
106
|
+
let lastError = 'unknown error';
|
|
107
|
+
let attempts = 0;
|
|
108
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
109
|
+
attempts = attempt;
|
|
110
|
+
try {
|
|
111
|
+
// `initialCapacity` stays at the sqlite-wasm default (6 files): one pool
|
|
112
|
+
// per bucket holds a single DB plus its journals, so preallocating more
|
|
113
|
+
// OPFS files would only be waste. A "SAH pool is full" error still
|
|
114
|
+
// reaches the caller verbatim via `opfsError`.
|
|
115
|
+
const pool = await sqlite3.installOpfsSAHPoolVfs({
|
|
116
|
+
name: `sp00ky-${dbName}`,
|
|
117
|
+
// The first failure is cached against the VFS name, so a retry that
|
|
118
|
+
// doesn't ask for a real re-init just replays the same rejection.
|
|
119
|
+
...(attempt > 1 ? { forceReinitIfPreviouslyFailed: true } : {}),
|
|
120
|
+
});
|
|
121
|
+
return { db: new pool.OpfsSAHPoolDb(`/${dbName}.sqlite3`), persisted: true };
|
|
122
|
+
} catch (e) {
|
|
123
|
+
lastError = errMessage(e);
|
|
124
|
+
if (attempt === maxAttempts || UNRETRYABLE.some((m) => lastError.includes(m))) break;
|
|
125
|
+
await sleep(backoffMs[Math.min(attempt - 1, backoffMs.length - 1)] ?? 0);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return fallbackToMemory(sqlite3, dbName, lastError, attempts);
|
|
129
|
+
}
|
|
@@ -5,11 +5,12 @@
|
|
|
5
5
|
* is also what the OPFS VFS requires — file access must happen off the main
|
|
6
6
|
* thread. Persistence uses the **OPFS SAHPool VFS**: durable, and (unlike the
|
|
7
7
|
* classic OPFS VFS) it does NOT require COOP/COEP cross-origin isolation
|
|
8
|
-
* headers, so host apps embedding the client need no server changes.
|
|
9
|
-
* to an in-memory DB
|
|
8
|
+
* headers, so host apps embedding the client need no server changes. When OPFS
|
|
9
|
+
* is unavailable it retries, then falls back to an in-memory DB and REPORTS the
|
|
10
|
+
* loss of durability (see `sqlite-open.ts`) instead of degrading silently.
|
|
10
11
|
*
|
|
11
12
|
* Message protocol (request/response keyed by `id`):
|
|
12
|
-
* { id, type: 'open', payload: { dbName, useOpfs } }
|
|
13
|
+
* { id, type: 'open', payload: { dbName, useOpfs } } -> { id, ok, persisted, opfsError? }
|
|
13
14
|
* { id, type: 'exec', payload: { sql, bind } } -> { id, ok, rows }
|
|
14
15
|
* { id, type: 'run', payload: { sql, bind } } -> { id, ok }
|
|
15
16
|
* { id, type: 'batch', payload: [{ sql, bind }] } (atomic BEGIN/COMMIT)
|
|
@@ -22,6 +23,7 @@
|
|
|
22
23
|
* hot path; the per-statement ops remain for the write/shim paths.
|
|
23
24
|
*/
|
|
24
25
|
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
|
26
|
+
import { openDb, type SqliteDbHandle } from './sqlite-open';
|
|
25
27
|
import { executeSelect, type SelectDb } from './sqlite-select';
|
|
26
28
|
|
|
27
29
|
interface Stmt {
|
|
@@ -29,28 +31,18 @@ interface Stmt {
|
|
|
29
31
|
bind?: unknown[];
|
|
30
32
|
}
|
|
31
33
|
|
|
32
|
-
let db:
|
|
33
|
-
exec: (opts: { sql: string; bind?: unknown[]; rowMode?: string; returnValue?: string }) => unknown;
|
|
34
|
-
close: () => void;
|
|
35
|
-
} | null = null;
|
|
34
|
+
let db: SqliteDbHandle | null = null;
|
|
36
35
|
|
|
37
36
|
async function open(
|
|
38
37
|
dbName: string,
|
|
39
38
|
useOpfs: boolean,
|
|
40
39
|
systemTables: readonly string[] = []
|
|
41
|
-
): Promise<{ persisted: boolean }> {
|
|
40
|
+
): Promise<{ persisted: boolean; opfsError?: string }> {
|
|
42
41
|
const sqlite3: any = await sqlite3InitModule();
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
db = new pool.OpfsSAHPoolDb(`/${dbName}.sqlite3`);
|
|
48
|
-
persisted = true;
|
|
49
|
-
} catch {
|
|
50
|
-
// fall through to in-memory
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
if (!db) db = new sqlite3.oo1.DB(':memory:', 'c');
|
|
42
|
+
// Retry/fallback policy (and the loud report when persistence is lost) lives
|
|
43
|
+
// in `sqlite-open.ts` so it can be unit tested off-worker.
|
|
44
|
+
const { db: handle, persisted, opfsError } = await openDb(sqlite3, dbName, useOpfs);
|
|
45
|
+
db = handle;
|
|
54
46
|
// Physically create the internal `_00_*` tables the client reads before any
|
|
55
47
|
// write (DEFINE is a noop on this engine, so the migrator can't). Prevents
|
|
56
48
|
// "no such table: _00_query" on a fresh bucket right after signup.
|
|
@@ -66,7 +58,7 @@ async function open(
|
|
|
66
58
|
} catch {
|
|
67
59
|
/* pragma best-effort */
|
|
68
60
|
}
|
|
69
|
-
return { persisted };
|
|
61
|
+
return { persisted, opfsError };
|
|
70
62
|
}
|
|
71
63
|
|
|
72
64
|
function exec(sql: string, bind?: unknown[]): unknown[] {
|
package/src/sp00ky.ts
CHANGED
|
@@ -8,7 +8,8 @@ import type {
|
|
|
8
8
|
PreloadOptions,
|
|
9
9
|
UpdateOptions,
|
|
10
10
|
RunOptions,
|
|
11
|
-
SyncHealth
|
|
11
|
+
SyncHealth,
|
|
12
|
+
StorageHealth} from './types';
|
|
12
13
|
import {
|
|
13
14
|
LocalMigrator,
|
|
14
15
|
RemoteDatabaseService,
|
|
@@ -106,6 +107,13 @@ export class BucketHandle {
|
|
|
106
107
|
*/
|
|
107
108
|
const LAST_BUCKET_KEY = 'sp00ky:last_bucket';
|
|
108
109
|
|
|
110
|
+
/** Reported for engines that don't track local-store durability. Frozen so a
|
|
111
|
+
* subscriber can't mutate the shared snapshot. */
|
|
112
|
+
const UNKNOWN_STORAGE_HEALTH: StorageHealth = Object.freeze({
|
|
113
|
+
status: 'unknown',
|
|
114
|
+
fallback: false,
|
|
115
|
+
});
|
|
116
|
+
|
|
109
117
|
function readBootBucketHint(): string | null {
|
|
110
118
|
try {
|
|
111
119
|
return typeof localStorage !== 'undefined' ? localStorage.getItem(LAST_BUCKET_KEY) : null;
|
|
@@ -188,6 +196,25 @@ export class Sp00kyClient<S extends SchemaStructure> {
|
|
|
188
196
|
return this.sync.subscribeToSyncHealth(cb);
|
|
189
197
|
}
|
|
190
198
|
|
|
199
|
+
/** Durability of the local cache. See {@link StorageHealth}. `'unknown'` for
|
|
200
|
+
* engines that don't report it. */
|
|
201
|
+
get storageHealth(): StorageHealth {
|
|
202
|
+
return this.local.storageHealth ?? UNKNOWN_STORAGE_HEALTH;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Observe local-store durability. Fires immediately with the current snapshot
|
|
207
|
+
* and again on every change (at most once per bucket open in practice).
|
|
208
|
+
* Returns an unsubscribe.
|
|
209
|
+
*/
|
|
210
|
+
subscribeToStorageHealth(cb: (health: StorageHealth) => void): () => void {
|
|
211
|
+
if (this.local.subscribeToStorageHealth) {
|
|
212
|
+
return this.local.subscribeToStorageHealth(cb);
|
|
213
|
+
}
|
|
214
|
+
cb(UNKNOWN_STORAGE_HEALTH);
|
|
215
|
+
return () => {};
|
|
216
|
+
}
|
|
217
|
+
|
|
191
218
|
constructor(private config: Sp00kyConfig<S>) {
|
|
192
219
|
const logger = createLogger(config.logLevel ?? 'info', config.otelTransmit);
|
|
193
220
|
this.logger = logger.child({ service: 'Sp00kyClient' });
|
package/src/types.ts
CHANGED
|
@@ -247,6 +247,33 @@ export interface SyncHealth {
|
|
|
247
247
|
everConnected: boolean;
|
|
248
248
|
}
|
|
249
249
|
|
|
250
|
+
export type StorageHealthStatus = 'unknown' | 'persistent' | 'memory';
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Durability of the LOCAL cache, delivered to `subscribeToStorageHealth`
|
|
254
|
+
* subscribers. Separate from {@link SyncHealth}: that one is about reaching the
|
|
255
|
+
* server, this one is about whether the local store survives a reload.
|
|
256
|
+
*
|
|
257
|
+
* Under `localEngine: 'sqlite'` the durable store is the OPFS SAHPool VFS,
|
|
258
|
+
* which only one client per bucket can hold open. When it can't be opened (a
|
|
259
|
+
* second tab of the app already has it, an insecure context, a full pool) the
|
|
260
|
+
* engine keeps working against an in-memory DB, which holds the whole dataset
|
|
261
|
+
* in RAM and loses local writes on reload. `fallback` marks exactly that case,
|
|
262
|
+
* so a UI can warn about it.
|
|
263
|
+
*/
|
|
264
|
+
export interface StorageHealth {
|
|
265
|
+
/** `'unknown'` until the local cache has opened, or for engines that don't report. */
|
|
266
|
+
status: StorageHealthStatus;
|
|
267
|
+
/**
|
|
268
|
+
* `true` only when durable storage was REQUESTED and could not be opened.
|
|
269
|
+
* Stays `false` for a configured-in-memory store (`store: 'memory'`), which
|
|
270
|
+
* is a choice rather than a failure, so a UI can key off this alone.
|
|
271
|
+
*/
|
|
272
|
+
fallback: boolean;
|
|
273
|
+
/** Reason durable storage failed (only set while `fallback` is `true`). */
|
|
274
|
+
error?: string;
|
|
275
|
+
}
|
|
276
|
+
|
|
250
277
|
export type QueryHash = string;
|
|
251
278
|
|
|
252
279
|
// Flat array format: [[record-id, version], [record-id, version], ...]
|