@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +57 -0
- package/dist/index.d.ts +2514 -58
- package/dist/index.js +12561 -2449
- package/dist/otel/index.d.ts +2 -2
- package/dist/otel/index.js +6 -6
- package/dist/sqlite-open.js +303 -0
- package/dist/sqlite-worker.d.ts +1 -0
- package/dist/sqlite-worker.js +439 -0
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +472 -0
- package/dist/types.d.ts +751 -11
- package/package.json +11 -7
- package/scripts/check-broker-bundle.mjs +33 -0
- package/skills/{spooky-core → sp00ky-core}/SKILL.md +12 -12
- package/skills/{spooky-core → sp00ky-core}/references/auth.md +1 -1
- package/skills/{spooky-core → sp00ky-core}/references/config.md +2 -2
- package/src/bucket-blurhash.test.ts +148 -0
- package/src/build-globals.d.ts +12 -0
- package/src/events/events.test.ts +2 -1
- package/src/events/index.ts +3 -0
- package/src/index.ts +36 -2
- package/src/modules/app-release/index.test.ts +125 -0
- package/src/modules/app-release/index.ts +201 -0
- package/src/modules/auth/auth.local-first.test.ts +101 -0
- package/src/modules/auth/events/index.ts +2 -1
- package/src/modules/auth/index.ts +127 -24
- package/src/modules/cache/cache.relay.test.ts +95 -0
- package/src/modules/cache/index.ts +163 -43
- package/src/modules/cache/types.ts +2 -2
- package/src/modules/crdt/crdt-field.ts +294 -0
- package/src/modules/crdt/crdt-hydration.test.ts +210 -0
- package/src/modules/crdt/crdt-reconnect.test.ts +195 -0
- package/src/modules/crdt/index.ts +463 -0
- package/src/modules/crdt/loro-loader.ts +25 -0
- package/src/modules/data/data.hydration.test.ts +142 -0
- package/src/modules/data/data.membership.test.ts +523 -0
- package/src/modules/data/data.notify-table.test.ts +41 -0
- package/src/modules/data/data.pending-ids.test.ts +199 -0
- package/src/modules/data/data.rebind.test.ts +170 -0
- package/src/modules/data/data.rematerialize.test.ts +114 -0
- package/src/modules/data/data.run.test.ts +113 -0
- package/src/modules/data/data.settled-writes.test.ts +206 -0
- package/src/modules/data/data.status.test.ts +249 -0
- package/src/modules/data/id-set-plan.test.ts +122 -0
- package/src/modules/data/index.ts +1815 -151
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/data/window-query.test.ts +52 -0
- package/src/modules/data/window-query.ts +194 -0
- package/src/modules/devtools/flags.ts +349 -0
- package/src/modules/devtools/index.ts +450 -46
- package/src/modules/devtools/notify-throttle.test.ts +154 -0
- package/src/modules/devtools/state-shape.test.ts +146 -0
- package/src/modules/devtools/storage-info.test.ts +79 -0
- package/src/modules/devtools/storage-info.ts +168 -0
- package/src/modules/devtools/versions.test.ts +74 -0
- package/src/modules/devtools/versions.ts +110 -0
- package/src/modules/feature-flag/index.test.ts +251 -0
- package/src/modules/feature-flag/index.ts +308 -0
- package/src/modules/ref-tables.test.ts +91 -0
- package/src/modules/ref-tables.ts +88 -0
- package/src/modules/sync/engine.ts +164 -82
- package/src/modules/sync/events/index.ts +9 -2
- package/src/modules/sync/queue/queue-down.test.ts +180 -0
- package/src/modules/sync/queue/queue-down.ts +80 -13
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +164 -0
- package/src/modules/sync/queue/queue-up.ts +241 -57
- package/src/modules/sync/scheduler.pause.test.ts +109 -0
- package/src/modules/sync/scheduler.retry.test.ts +237 -0
- package/src/modules/sync/scheduler.ts +215 -13
- package/src/modules/sync/sync.cleanup.test.ts +116 -0
- package/src/modules/sync/sync.health.test.ts +149 -0
- package/src/modules/sync/sync.heartbeat.test.ts +80 -0
- package/src/modules/sync/sync.live-removal.test.ts +175 -0
- package/src/modules/sync/sync.reconnect.test.ts +145 -0
- package/src/modules/sync/sync.subquery.test.ts +82 -0
- package/src/modules/sync/sync.tabs.test.ts +249 -0
- package/src/modules/sync/sync.ts +1726 -99
- package/src/modules/sync/utils.test.ts +269 -2
- package/src/modules/sync/utils.ts +201 -17
- package/src/otel/index.ts +13 -10
- package/src/services/blobs/blob-cache.test.ts +359 -0
- package/src/services/blobs/blob-cache.ts +603 -0
- package/src/services/blobs/blob-manifest.ts +227 -0
- package/src/services/blobs/blob-store.test.ts +77 -0
- package/src/services/blobs/blob-store.ts +359 -0
- package/src/services/blobs/blob.fixture.ts +90 -0
- package/src/services/blobs/index.ts +70 -0
- package/src/services/database/cache-engine.ts +193 -0
- package/src/services/database/connection-supervisor.test.ts +289 -0
- package/src/services/database/connection-supervisor.ts +415 -0
- package/src/services/database/database.query-timeout.test.ts +83 -0
- package/src/services/database/database.ts +41 -12
- package/src/services/database/engine-factory.ts +33 -0
- package/src/services/database/errors.ts +34 -0
- package/src/services/database/events/index.ts +2 -1
- package/src/services/database/index.ts +7 -0
- package/src/services/database/local-migrator.ts +30 -27
- package/src/services/database/local.test.ts +64 -0
- package/src/services/database/local.ts +484 -67
- package/src/services/database/plan-render.test.ts +159 -0
- package/src/services/database/plan-render.ts +108 -0
- package/src/services/database/relation-resolver.test.ts +413 -0
- package/src/services/database/relation-resolver.ts +0 -0
- package/src/services/database/remote.ts +110 -14
- package/src/services/database/sqlite-cache-engine.test.ts +616 -0
- package/src/services/database/sqlite-cache-engine.timeout.test.ts +61 -0
- package/src/services/database/sqlite-cache-engine.ts +1358 -0
- package/src/services/database/sqlite-devtools-queries.integration.test.ts +143 -0
- package/src/services/database/sqlite-devtools-queries.test.ts +154 -0
- package/src/services/database/sqlite-lock-verify.test.ts +33 -0
- package/src/services/database/sqlite-lock-verify.ts +45 -0
- package/src/services/database/sqlite-open.test.ts +150 -0
- package/src/services/database/sqlite-open.ts +164 -0
- package/src/services/database/sqlite-plan-sql.test.ts +104 -0
- package/src/services/database/sqlite-plan-sql.ts +138 -0
- package/src/services/database/sqlite-projection.test.ts +99 -0
- package/src/services/database/sqlite-select.integration.test.ts +185 -0
- package/src/services/database/sqlite-select.test.ts +246 -0
- package/src/services/database/sqlite-select.ts +131 -0
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +224 -0
- package/src/services/database/sqlite-worker.ts +437 -0
- package/src/services/database/surql-translate.ts +416 -0
- package/src/services/database/surreal-cache-engine.ts +161 -0
- package/src/services/logger/index.ts +3 -2
- package/src/services/persistence/localstorage.ts +2 -2
- package/src/services/persistence/resilient.ts +11 -4
- package/src/services/persistence/surrealdb.ts +10 -10
- package/src/services/stream-processor/index.ts +796 -84
- package/src/services/stream-processor/permissions.test.ts +47 -0
- package/src/services/stream-processor/permissions.ts +53 -0
- package/src/services/stream-processor/stream-processor.batch.test.ts +186 -0
- package/src/services/stream-processor/stream-processor.prime.test.ts +198 -0
- package/src/services/stream-processor/stream-processor.reset.test.ts +226 -0
- package/src/services/stream-processor/stream-processor.test.ts +1 -1
- package/src/services/stream-processor/wasm-types.ts +59 -3
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +327 -0
- package/src/services/tabs/coordinator.test.ts +365 -0
- package/src/services/tabs/coordinator.ts +633 -0
- package/src/services/tabs/fake-ports.fixture.ts +112 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +258 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +640 -0
- package/src/sp00ky.auth-order.test.ts +92 -0
- package/src/sp00ky.init-query.test.ts +183 -0
- package/src/sp00ky.local-first.test.ts +60 -0
- package/src/sp00ky.ts +1693 -0
- package/src/types.ts +528 -13
- package/src/utils/blurhash.ts +90 -0
- package/src/utils/error-classification.test.ts +44 -0
- package/src/utils/error-classification.ts +7 -0
- package/src/utils/index.ts +79 -13
- package/src/utils/parser.test.ts +49 -120
- package/src/utils/parser.ts +32 -2
- package/src/utils/semver.test.ts +32 -0
- package/src/utils/semver.ts +30 -0
- package/src/utils/surql.ts +30 -18
- package/src/utils/withRetry.test.ts +1 -1
- package/tsdown.config.ts +86 -1
- package/src/spooky.ts +0 -395
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The wire between `SqliteCacheEngine` and the SQLite worker, extracted so the
|
|
3
|
+
* engine can swap it at runtime without touching any SQL logic. Two shapes:
|
|
4
|
+
*
|
|
5
|
+
* - {@link WorkerSqliteTransport}: owns a dedicated Worker (solo mode, and the
|
|
6
|
+
* leader tab in shared-tabs mode). Controls the worker's lifecycle and can
|
|
7
|
+
* forward follower MessagePorts into it (`add-client`).
|
|
8
|
+
* - {@link PortSqliteTransport}: speaks the same request/response protocol over
|
|
9
|
+
* a MessagePort handed out by the leader (follower tabs). No lifecycle
|
|
10
|
+
* control; the port dying surfaces through `onPortDead`.
|
|
11
|
+
*
|
|
12
|
+
* Both keep the pending-map semantics the engine had inline before: requests
|
|
13
|
+
* keyed by a locally-minted numeric id, one resolve/reject per id, `failAll`
|
|
14
|
+
* rejects everything in flight (worker crash, port loss, role change).
|
|
15
|
+
*/
|
|
16
|
+
import type { Logger } from '../logger/index';
|
|
17
|
+
|
|
18
|
+
export interface SqliteTransport {
|
|
19
|
+
readonly kind: 'worker' | 'port';
|
|
20
|
+
/** True while the transport can carry a request. */
|
|
21
|
+
readonly connected: boolean;
|
|
22
|
+
call<T = unknown>(type: string, payload?: unknown): Promise<T>;
|
|
23
|
+
/** Reject every pending request with `reason`. Safe to call repeatedly. */
|
|
24
|
+
failAll(reason: string): void;
|
|
25
|
+
/** failAll + release the underlying channel. Terminal. `err` overrides the
|
|
26
|
+
* transport's own error shape — used for a deliberate teardown (role change)
|
|
27
|
+
* so callers see a retryable "transport lost", not "the worker crashed". */
|
|
28
|
+
close(reason?: string, err?: Error): void;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Thrown into pending follower calls when the leader (or its port) goes away.
|
|
32
|
+
* Callers treat it like a transient error: the op may be retried once a new
|
|
33
|
+
* leader is attached; nothing was necessarily executed. */
|
|
34
|
+
export class BrokerPortClosedError extends Error {
|
|
35
|
+
readonly retryable = true;
|
|
36
|
+
constructor(reason: string) {
|
|
37
|
+
super(`sqlite transport lost: ${reason}`);
|
|
38
|
+
this.name = 'BrokerPortClosedError';
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
interface Pending {
|
|
43
|
+
resolve: (v: any) => void;
|
|
44
|
+
reject: (e: unknown) => void;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/** Shared request/response bookkeeping over any postMessage-shaped channel. */
|
|
48
|
+
abstract class BaseTransport implements SqliteTransport {
|
|
49
|
+
abstract readonly kind: 'worker' | 'port';
|
|
50
|
+
protected pending = new Map<number, Pending>();
|
|
51
|
+
protected seq = 0;
|
|
52
|
+
protected closed = false;
|
|
53
|
+
|
|
54
|
+
constructor(protected logger: Logger) {}
|
|
55
|
+
|
|
56
|
+
get connected(): boolean {
|
|
57
|
+
return !this.closed;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
protected abstract post(msg: unknown, transfer?: Transferable[]): void;
|
|
61
|
+
protected abstract makeError(reason: string): Error;
|
|
62
|
+
|
|
63
|
+
protected handleMessage(data: any): void {
|
|
64
|
+
const { id, ok, error, ...rest } = data ?? {};
|
|
65
|
+
const p = this.pending.get(id);
|
|
66
|
+
if (!p) return;
|
|
67
|
+
this.pending.delete(id);
|
|
68
|
+
if (ok) p.resolve(rest);
|
|
69
|
+
else p.reject(new Error(error));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
call<T = unknown>(type: string, payload?: unknown): Promise<T> {
|
|
73
|
+
if (this.closed) return Promise.reject(this.makeError('transport closed'));
|
|
74
|
+
const id = ++this.seq;
|
|
75
|
+
return new Promise<T>((resolve, reject) => {
|
|
76
|
+
this.pending.set(id, { resolve, reject });
|
|
77
|
+
try {
|
|
78
|
+
this.post({ id, type, payload });
|
|
79
|
+
} catch (e) {
|
|
80
|
+
this.pending.delete(id);
|
|
81
|
+
reject(e);
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
failAll(reason: string, err?: Error): void {
|
|
87
|
+
if (this.pending.size === 0) return;
|
|
88
|
+
const e = err ?? this.makeError(reason);
|
|
89
|
+
for (const [, p] of this.pending) p.reject(e);
|
|
90
|
+
this.pending.clear();
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
close(reason = 'closed', err?: Error): void {
|
|
94
|
+
if (this.closed) return;
|
|
95
|
+
this.closed = true;
|
|
96
|
+
this.failAll(reason, err);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export class WorkerSqliteTransport extends BaseTransport {
|
|
101
|
+
readonly kind = 'worker' as const;
|
|
102
|
+
private worker: Worker;
|
|
103
|
+
/** Fired when the worker reports it fenced itself after a leadership steal
|
|
104
|
+
* (see sqlite-worker.ts). The engine must treat the store as gone. */
|
|
105
|
+
onLockLost: ((reason: string) => void) | null = null;
|
|
106
|
+
|
|
107
|
+
constructor(logger: Logger) {
|
|
108
|
+
super(logger);
|
|
109
|
+
// Source references the `.ts` so the monorepo's src-bundling consumers
|
|
110
|
+
// (e.g. the example app, which aliases `@spooky-sync/core` to `src`)
|
|
111
|
+
// resolve it — Vite handles `.ts` workers. For the published package, the
|
|
112
|
+
// tsdown build rewrites this to `./sqlite-worker.js` (the top-level
|
|
113
|
+
// emitted entry; see tsdown.config.ts), which the flat `dist/index.js`
|
|
114
|
+
// resolves. The worker (+ `@sqlite.org/sqlite-wasm`) still loads lazily —
|
|
115
|
+
// only when `localEngine: 'sqlite'` is used.
|
|
116
|
+
this.worker = new Worker(new URL('./sqlite-worker.ts', import.meta.url), { type: 'module' });
|
|
117
|
+
this.worker.onmessage = (ev: MessageEvent) => {
|
|
118
|
+
// Unsolicited worker-initiated notification, not a reply.
|
|
119
|
+
if (ev.data?.type === 'lock-lost') {
|
|
120
|
+
this.logger.error(
|
|
121
|
+
{ reason: ev.data.reason, Category: 'sp00ky-client::SqliteCacheEngine::worker' },
|
|
122
|
+
'SQLite worker fenced after leadership loss'
|
|
123
|
+
);
|
|
124
|
+
this.failAll('worker fenced');
|
|
125
|
+
this.onLockLost?.(String(ev.data.reason ?? 'lock-lost'));
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
this.handleMessage(ev.data);
|
|
129
|
+
};
|
|
130
|
+
// Surface a worker crash (wasm abort / OOM) instead of leaving every
|
|
131
|
+
// pending call hung forever — reject them all with a clear error, AND
|
|
132
|
+
// close: `failAll` alone left `connected` true, so every later call was
|
|
133
|
+
// posted to the dead worker and parked in `pending` for good. Closed, the
|
|
134
|
+
// engine sees `!connected` and spawns a fresh worker on its next open.
|
|
135
|
+
const crash = (msg: string) => {
|
|
136
|
+
this.logger.error(
|
|
137
|
+
{ err: this.makeError(msg), Category: 'sp00ky-client::SqliteCacheEngine::worker' },
|
|
138
|
+
'Worker error'
|
|
139
|
+
);
|
|
140
|
+
this.close(msg);
|
|
141
|
+
};
|
|
142
|
+
this.worker.onerror = (e: ErrorEvent) => crash(e.message || 'onerror');
|
|
143
|
+
this.worker.onmessageerror = () => crash('messageerror');
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
protected post(msg: unknown, transfer?: Transferable[]): void {
|
|
147
|
+
if (transfer) this.worker.postMessage(msg, transfer);
|
|
148
|
+
else this.worker.postMessage(msg);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
protected makeError(reason: string): Error {
|
|
152
|
+
return new Error(`SQLite worker crashed: ${reason}`);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/** Forward a follower's MessagePort into the worker as an extra client. */
|
|
156
|
+
addClientPort(clientId: string, port: MessagePort): Promise<void> {
|
|
157
|
+
if (this.closed) return Promise.reject(this.makeError('transport closed'));
|
|
158
|
+
const id = ++this.seq;
|
|
159
|
+
return new Promise<void>((resolve, reject) => {
|
|
160
|
+
this.pending.set(id, { resolve: () => resolve(), reject });
|
|
161
|
+
this.worker.postMessage({ id, type: 'add-client', payload: { clientId } }, [port]);
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
removeClientPort(clientId: string): Promise<void> {
|
|
166
|
+
return this.call('remove-client', { clientId }).then(() => undefined);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/** Ask the worker to close + pauseVfs + self-close (graceful pagehide). */
|
|
170
|
+
shutdown(): Promise<void> {
|
|
171
|
+
return this.call('shutdown').then(() => undefined);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
close(reason = 'closed', err?: Error): void {
|
|
175
|
+
if (this.closed) return;
|
|
176
|
+
super.close(reason, err);
|
|
177
|
+
this.worker.terminate();
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
export class PortSqliteTransport extends BaseTransport {
|
|
182
|
+
readonly kind = 'port' as const;
|
|
183
|
+
|
|
184
|
+
constructor(
|
|
185
|
+
private port: MessagePort,
|
|
186
|
+
private onPortDead: (reason: string) => void,
|
|
187
|
+
logger: Logger
|
|
188
|
+
) {
|
|
189
|
+
super(logger);
|
|
190
|
+
port.onmessage = (ev: MessageEvent) => this.handleMessage(ev.data);
|
|
191
|
+
port.onmessageerror = () => this.dead('messageerror');
|
|
192
|
+
port.start?.();
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** The broker/coordinator learned the leader is gone; the port itself has no
|
|
196
|
+
* close event, so the coordinator calls this explicitly. */
|
|
197
|
+
markDead(reason: string): void {
|
|
198
|
+
this.dead(reason);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
private dead(reason: string, err?: Error): void {
|
|
202
|
+
if (this.closed) return;
|
|
203
|
+
this.closed = true;
|
|
204
|
+
this.failAll(reason, err);
|
|
205
|
+
try {
|
|
206
|
+
this.port.close();
|
|
207
|
+
} catch {
|
|
208
|
+
/* ignore */
|
|
209
|
+
}
|
|
210
|
+
this.onPortDead(reason);
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
protected post(msg: unknown): void {
|
|
214
|
+
this.port.postMessage(msg);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
protected makeError(reason: string): Error {
|
|
218
|
+
return new BrokerPortClosedError(reason);
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
close(reason = 'closed', err?: Error): void {
|
|
222
|
+
this.dead(reason, err);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
@@ -0,0 +1,437 @@
|
|
|
1
|
+
/// <reference lib="webworker" />
|
|
2
|
+
/**
|
|
3
|
+
* Dedicated Web Worker that owns the SQLite-WASM handle. All DB access is
|
|
4
|
+
* funnelled through here (the main thread never touches the wasm module), which
|
|
5
|
+
* is also what the OPFS VFS requires — file access must happen off the main
|
|
6
|
+
* thread. Persistence uses the **OPFS SAHPool VFS**: durable, and (unlike the
|
|
7
|
+
* classic OPFS VFS) it does NOT require COOP/COEP cross-origin isolation
|
|
8
|
+
* headers, so host apps embedding the client need no server changes. When OPFS
|
|
9
|
+
* is unavailable it retries, then falls back to an in-memory DB and REPORTS the
|
|
10
|
+
* loss of durability (see `sqlite-open.ts`) instead of degrading silently.
|
|
11
|
+
*
|
|
12
|
+
* Message protocol (request/response keyed by `id`; replies go to the channel
|
|
13
|
+
* the request arrived on):
|
|
14
|
+
* { id, type: 'open', payload: { dbName, useOpfs, workerLockName? } }
|
|
15
|
+
* -> { id, ok, persisted, opfsError? }
|
|
16
|
+
* { id, type: 'exec', payload: { sql, bind } } -> { id, ok, rows }
|
|
17
|
+
* { id, type: 'run', payload: { sql, bind } } -> { id, ok }
|
|
18
|
+
* { id, type: 'batch', payload: [{ sql, bind }] } (atomic BEGIN/COMMIT)
|
|
19
|
+
* { id, type: 'select', payload: { plan, params } } -> { id, ok, rows, relationFetches }
|
|
20
|
+
* { id, type: 'close' }
|
|
21
|
+
* { id, type: 'shutdown' } (owner only: close + pauseVfs + self.close)
|
|
22
|
+
* { id, type: 'relock', payload: { workerLockName } } (owner only)
|
|
23
|
+
* { id, type: 'add-client', payload: { clientId } } + ev.ports[0] (owner only)
|
|
24
|
+
* { id, type: 'remove-client', payload: { clientId } } (owner only)
|
|
25
|
+
*
|
|
26
|
+
* Multi-client (shared-tabs mode): the OWNER (the leader tab's engine) speaks
|
|
27
|
+
* on the worker's own channel and controls the lifecycle. Follower tabs get a
|
|
28
|
+
* MessagePort each (`add-client`) and may only issue data ops
|
|
29
|
+
* (exec/run/batch/select). Every op, from any channel, runs through ONE op
|
|
30
|
+
* chain so an async `select` from one client can never interleave with another
|
|
31
|
+
* client's `batch` at the VFS layer.
|
|
32
|
+
*
|
|
33
|
+
* Leader fencing (shared-tabs mode): `open` may carry `workerLockName`, a Web
|
|
34
|
+
* Lock this worker acquires BEFORE opening the pool and holds for the DB's
|
|
35
|
+
* lifetime. The name is unique per leadership, so the lock being gone has
|
|
36
|
+
* exactly one meaning: the broker stole it because this tab was presumed dead
|
|
37
|
+
* (usually frozen). When that happens the worker FENCES itself: closes the DB,
|
|
38
|
+
* releases the OPFS sync access handles (pauseVfs), tells the owner, and
|
|
39
|
+
* refuses every further op. A frozen-then-thawed ex-leader therefore can never
|
|
40
|
+
* write the durable file after a new leader took over. The thaw gate below
|
|
41
|
+
* closes the wake-up ordering race where a queued write could run before the
|
|
42
|
+
* lock-loss callback fires.
|
|
43
|
+
*
|
|
44
|
+
* `select` executes a whole QueryPlan — table creation, base select, the full
|
|
45
|
+
* `.related()` tree (shared `resolveRelations`), and JSON row parsing — in ONE
|
|
46
|
+
* round-trip, returning structured-clone row objects. This is the first-load
|
|
47
|
+
* hot path; the per-statement ops remain for the write/shim paths.
|
|
48
|
+
*/
|
|
49
|
+
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
|
50
|
+
import { verifyLockStillHeld as verifyLockStillHeldBounded } from './sqlite-lock-verify';
|
|
51
|
+
import {
|
|
52
|
+
openDb,
|
|
53
|
+
type OpenDbOptions,
|
|
54
|
+
type SqliteDbHandle,
|
|
55
|
+
type SqlitePoolHandle,
|
|
56
|
+
} from './sqlite-open';
|
|
57
|
+
import { executeSelect, type SelectDb } from './sqlite-select';
|
|
58
|
+
|
|
59
|
+
interface Stmt {
|
|
60
|
+
sql: string;
|
|
61
|
+
bind?: unknown[];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
interface PostTarget {
|
|
65
|
+
postMessage: (msg: unknown) => void;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
let db: SqliteDbHandle | null = null;
|
|
69
|
+
let pool: SqlitePoolHandle | null = null;
|
|
70
|
+
|
|
71
|
+
// ==================== leader fencing (worker web lock) ====================
|
|
72
|
+
|
|
73
|
+
/** Once fenced, every op is refused and the DB stays closed. Terminal. */
|
|
74
|
+
let fenced = false;
|
|
75
|
+
|
|
76
|
+
interface HeldLock {
|
|
77
|
+
name: string;
|
|
78
|
+
/** Release on purpose; the closure-local flag keeps the steal handler from
|
|
79
|
+
* mistaking it for a takeover. Idempotent. */
|
|
80
|
+
release: () => void;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The per-leadership lock currently held (null = no fencing active). A
|
|
84
|
+
* relock (same-leader re-promotion after a broker restart) briefly holds two
|
|
85
|
+
* locks: the new one is installed here first, then the old one's own release
|
|
86
|
+
* is called, so there is no window where neither is held. */
|
|
87
|
+
let currentLock: HeldLock | null = null;
|
|
88
|
+
|
|
89
|
+
function workerLocks(): LockManager | null {
|
|
90
|
+
const nav = (globalThis as { navigator?: { locks?: LockManager } }).navigator;
|
|
91
|
+
return nav?.locks && typeof nav.locks.request === 'function' ? nav.locks : null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Acquire `name` exclusively without waiting. Resolves with a handle when
|
|
96
|
+
* granted, null when someone else holds it. If the request settles while the
|
|
97
|
+
* holder still believes it owns the lock, the broker stole it: fence.
|
|
98
|
+
*/
|
|
99
|
+
function acquireWorkerLock(name: string): Promise<HeldLock | null> {
|
|
100
|
+
const locks = workerLocks();
|
|
101
|
+
// No Web Locks (non-browser test env): no fencing, behave as before.
|
|
102
|
+
if (!locks) return Promise.resolve({ name, release() {} });
|
|
103
|
+
return new Promise<HeldLock | null>((resolve) => {
|
|
104
|
+
let granted = false;
|
|
105
|
+
let intentional = false;
|
|
106
|
+
let releaseGrant: (() => void) | null = null;
|
|
107
|
+
locks
|
|
108
|
+
.request(name, { mode: 'exclusive', ifAvailable: true }, (lock) => {
|
|
109
|
+
if (!lock) {
|
|
110
|
+
resolve(null);
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
granted = true;
|
|
114
|
+
resolve({
|
|
115
|
+
name,
|
|
116
|
+
release() {
|
|
117
|
+
intentional = true;
|
|
118
|
+
releaseGrant?.();
|
|
119
|
+
},
|
|
120
|
+
});
|
|
121
|
+
return new Promise<void>((release) => {
|
|
122
|
+
releaseGrant = release;
|
|
123
|
+
});
|
|
124
|
+
})
|
|
125
|
+
.then(
|
|
126
|
+
() => {
|
|
127
|
+
if (granted && !intentional) void fence('worker lock stolen');
|
|
128
|
+
},
|
|
129
|
+
() => {
|
|
130
|
+
if (granted && !intentional) void fence('worker lock request failed');
|
|
131
|
+
}
|
|
132
|
+
);
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function releaseHeldLock(): void {
|
|
137
|
+
currentLock?.release();
|
|
138
|
+
currentLock = null;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Terminal teardown after leadership loss. Closing the DB first makes
|
|
143
|
+
* pauseVfs legal (it throws while files are open); pausing releases the sync
|
|
144
|
+
* access handles so the NEW leader's open succeeds without waiting for this
|
|
145
|
+
* worker to be garbage collected.
|
|
146
|
+
*/
|
|
147
|
+
async function fence(reason: string): Promise<void> {
|
|
148
|
+
if (fenced) return;
|
|
149
|
+
fenced = true;
|
|
150
|
+
// Deliberately console, not a logger: this must be visible in any host app
|
|
151
|
+
// regardless of its configured log level.
|
|
152
|
+
// oxlint-disable-next-line no-console
|
|
153
|
+
console.error(
|
|
154
|
+
`[sp00ky] sqlite worker fenced (${reason}): leadership was taken over, ` +
|
|
155
|
+
'closing the database and refusing further ops.'
|
|
156
|
+
);
|
|
157
|
+
try {
|
|
158
|
+
db?.close();
|
|
159
|
+
} catch {
|
|
160
|
+
/* already closed */
|
|
161
|
+
}
|
|
162
|
+
db = null;
|
|
163
|
+
try {
|
|
164
|
+
pool?.pauseVfs?.();
|
|
165
|
+
} catch {
|
|
166
|
+
/* pool already torn down or files still open; new leader retries anyway */
|
|
167
|
+
}
|
|
168
|
+
pool = null;
|
|
169
|
+
try {
|
|
170
|
+
(self as unknown as Worker).postMessage({ type: 'lock-lost', reason });
|
|
171
|
+
} catch {
|
|
172
|
+
/* owner gone */
|
|
173
|
+
}
|
|
174
|
+
releaseHeldLock();
|
|
175
|
+
self.close();
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// ==================== thaw gate ====================
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* A frozen tab's worker resumes with its message queue intact, and a queued
|
|
182
|
+
* write could run BEFORE the lock-steal callback fires. Both the 1s interval
|
|
183
|
+
* and every op call `noteTick()`; whichever runs first after a long gap kicks
|
|
184
|
+
* off a lock verification, and ops await it before touching the DB. The gap
|
|
185
|
+
* threshold must stay BELOW the broker's pong timeout: a freeze shorter than
|
|
186
|
+
* the threshold cannot have triggered a steal yet, so skipping the check for
|
|
187
|
+
* small gaps is safe.
|
|
188
|
+
*/
|
|
189
|
+
const FREEZE_SUSPECT_MS = 10_000;
|
|
190
|
+
let lastTickAt = performance.now();
|
|
191
|
+
let thawVerification: Promise<void> | null = null;
|
|
192
|
+
|
|
193
|
+
function noteTick(): void {
|
|
194
|
+
const now = performance.now();
|
|
195
|
+
const gap = now - lastTickAt;
|
|
196
|
+
lastTickAt = now;
|
|
197
|
+
if (gap > FREEZE_SUSPECT_MS && currentLock && !fenced && !thawVerification) {
|
|
198
|
+
thawVerification = verifyLockStillHeld().finally(() => {
|
|
199
|
+
thawVerification = null;
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
setInterval(noteTick, 1000);
|
|
204
|
+
|
|
205
|
+
// Bounded (see sqlite-lock-verify.ts): every op on the chain awaits this, and
|
|
206
|
+
// a `locks.query()` that never answered stopped every reply to every client.
|
|
207
|
+
function verifyLockStillHeld(): Promise<void> {
|
|
208
|
+
return verifyLockStillHeldBounded(
|
|
209
|
+
workerLocks() as any,
|
|
210
|
+
currentLock?.name,
|
|
211
|
+
(reason) => fence(reason),
|
|
212
|
+
undefined,
|
|
213
|
+
(msg) => console.warn(`[sqlite-worker] ${msg}`)
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// ==================== db ops ====================
|
|
218
|
+
|
|
219
|
+
async function open(
|
|
220
|
+
dbName: string,
|
|
221
|
+
useOpfs: boolean,
|
|
222
|
+
systemTables: readonly string[] = [],
|
|
223
|
+
workerLockName?: string,
|
|
224
|
+
openOptions?: OpenDbOptions
|
|
225
|
+
): Promise<{ persisted: boolean; opfsError?: string }> {
|
|
226
|
+
if (workerLockName) {
|
|
227
|
+
const handle = await acquireWorkerLock(workerLockName);
|
|
228
|
+
if (!handle) throw new Error('worker-lock-unavailable');
|
|
229
|
+
releaseHeldLock();
|
|
230
|
+
currentLock = handle;
|
|
231
|
+
}
|
|
232
|
+
const sqlite3: any = await sqlite3InitModule();
|
|
233
|
+
// Retry/fallback policy (and the loud report when persistence is lost) lives
|
|
234
|
+
// in `sqlite-open.ts` so it can be unit tested off-worker.
|
|
235
|
+
const result = await openDb(sqlite3, dbName, useOpfs, openOptions ?? {});
|
|
236
|
+
db = result.db;
|
|
237
|
+
pool = result.pool ?? null;
|
|
238
|
+
// Physically create the internal `_00_*` tables the client reads before any
|
|
239
|
+
// write (DEFINE is a noop on this engine, so the migrator can't). Prevents
|
|
240
|
+
// "no such table: _00_query" on a fresh bucket right after signup.
|
|
241
|
+
for (const t of systemTables) {
|
|
242
|
+
db!.exec({ sql: `CREATE TABLE IF NOT EXISTS "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)` });
|
|
243
|
+
}
|
|
244
|
+
// Retry (rather than instantly failing with SQLITE_BUSY=5) if a lock is held.
|
|
245
|
+
// Combined with the single-flight op chain, overlap is avoided.
|
|
246
|
+
// `cache_size` is negated → KiB (here 32 MiB) so SQLite's page cache can't
|
|
247
|
+
// grow unbounded and starve a wasm-heavy renderer.
|
|
248
|
+
try {
|
|
249
|
+
db!.exec({ sql: 'PRAGMA busy_timeout = 5000; PRAGMA cache_size = -32000;' });
|
|
250
|
+
} catch {
|
|
251
|
+
/* pragma best-effort */
|
|
252
|
+
}
|
|
253
|
+
return { persisted: result.persisted, opfsError: result.opfsError };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function closeDb(): void {
|
|
257
|
+
db?.close();
|
|
258
|
+
db = null;
|
|
259
|
+
pool = null;
|
|
260
|
+
selectDb.knownTables.clear();
|
|
261
|
+
// The lock is scoped to (bucket, leadership); a same-worker bucket switch
|
|
262
|
+
// closes then reopens under a NEW name, so the old one must go now.
|
|
263
|
+
releaseHeldLock();
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
function exec(sql: string, bind?: unknown[]): unknown[] {
|
|
267
|
+
if (!db) throw new Error('sqlite: DB not open');
|
|
268
|
+
return db.exec({ sql, bind, rowMode: 'object', returnValue: 'resultRows' }) as unknown[];
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function run(sql: string, bind?: unknown[]): void {
|
|
272
|
+
if (!db) throw new Error('sqlite: DB not open');
|
|
273
|
+
db.exec({ sql, bind });
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function batch(stmts: Stmt[]): void {
|
|
277
|
+
if (!db) throw new Error('sqlite: DB not open');
|
|
278
|
+
db.exec({ sql: 'BEGIN' });
|
|
279
|
+
try {
|
|
280
|
+
for (const s of stmts) db.exec({ sql: s.sql, bind: s.bind });
|
|
281
|
+
db.exec({ sql: 'COMMIT' });
|
|
282
|
+
} catch (e) {
|
|
283
|
+
try {
|
|
284
|
+
db.exec({ sql: 'ROLLBACK' });
|
|
285
|
+
} catch {
|
|
286
|
+
/* ignore */
|
|
287
|
+
}
|
|
288
|
+
throw e;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
// ==================== worker-side plan execution ('select') ====================
|
|
293
|
+
|
|
294
|
+
/** DB handle for `executeSelect` (see `sqlite-select.ts`, the unit-testable
|
|
295
|
+
* plan executor). `knownTables` is cleared on open/close, which covers both
|
|
296
|
+
* the solo fresh-worker path and the shared-mode same-worker bucket switch. */
|
|
297
|
+
const selectDb: SelectDb = {
|
|
298
|
+
exec: (sql, bind) => exec(sql, bind) as { data: string }[],
|
|
299
|
+
run,
|
|
300
|
+
knownTables: new Set<string>(),
|
|
301
|
+
};
|
|
302
|
+
|
|
303
|
+
// ==================== dispatch ====================
|
|
304
|
+
|
|
305
|
+
/** Follower client ports, keyed by the clientId the owner assigned. */
|
|
306
|
+
const clients = new Map<string, MessagePort>();
|
|
307
|
+
|
|
308
|
+
/** Data ops a follower client port may issue; lifecycle stays owner-only. */
|
|
309
|
+
const CLIENT_OPS = new Set(['exec', 'run', 'batch', 'select']);
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* ONE chain for every op from every channel. The handlers for `open` and
|
|
313
|
+
* `select` are async; without this, a follower's select could interleave with
|
|
314
|
+
* the owner's batch mid-transaction at the VFS layer. (Solo mode kept this
|
|
315
|
+
* invariant on the main thread via the engine's opQueue; with multiple client
|
|
316
|
+
* ports only the worker can.)
|
|
317
|
+
*/
|
|
318
|
+
let opChain: Promise<void> = Promise.resolve();
|
|
319
|
+
|
|
320
|
+
async function handle(type: string, payload: any, source: 'owner' | 'client'): Promise<unknown> {
|
|
321
|
+
if (fenced) throw new Error('sqlite: fenced, leadership was taken over');
|
|
322
|
+
if (source === 'client' && !CLIENT_OPS.has(type)) {
|
|
323
|
+
throw new Error(`sqlite worker: op ${type} is not allowed on a client port`);
|
|
324
|
+
}
|
|
325
|
+
switch (type) {
|
|
326
|
+
case 'open':
|
|
327
|
+
selectDb.knownTables.clear();
|
|
328
|
+
{
|
|
329
|
+
const result = await open(
|
|
330
|
+
payload.dbName,
|
|
331
|
+
payload.useOpfs,
|
|
332
|
+
payload.systemTables,
|
|
333
|
+
payload.workerLockName,
|
|
334
|
+
payload.openOptions
|
|
335
|
+
);
|
|
336
|
+
// The freshly seeded system tables exist — record them so the select
|
|
337
|
+
// path doesn't redundantly re-issue CREATE TABLE for each.
|
|
338
|
+
for (const t of (payload.systemTables ?? []) as string[]) selectDb.knownTables.add(t);
|
|
339
|
+
return result;
|
|
340
|
+
}
|
|
341
|
+
case 'relock': {
|
|
342
|
+
// Same-leader re-promotion after a broker restart: the DB stays open,
|
|
343
|
+
// only the per-leadership lock name rolls forward. Acquire the new lock
|
|
344
|
+
// BEFORE releasing the old one so there is no unfenced window.
|
|
345
|
+
const handle = await acquireWorkerLock(payload.workerLockName);
|
|
346
|
+
if (!handle) throw new Error('worker-lock-unavailable');
|
|
347
|
+
releaseHeldLock();
|
|
348
|
+
currentLock = handle;
|
|
349
|
+
return {};
|
|
350
|
+
}
|
|
351
|
+
case 'select':
|
|
352
|
+
return executeSelect(payload.plan, payload.params ?? {}, selectDb);
|
|
353
|
+
case 'exec':
|
|
354
|
+
return { rows: exec(payload.sql, payload.bind) };
|
|
355
|
+
case 'run':
|
|
356
|
+
run(payload.sql, payload.bind);
|
|
357
|
+
return {};
|
|
358
|
+
case 'batch':
|
|
359
|
+
batch(payload as Stmt[]);
|
|
360
|
+
return {};
|
|
361
|
+
case 'close':
|
|
362
|
+
closeDb();
|
|
363
|
+
return {};
|
|
364
|
+
case 'shutdown':
|
|
365
|
+
// Graceful pagehide path: pausing the VFS releases the sync access
|
|
366
|
+
// handles NOW instead of whenever this worker gets garbage collected,
|
|
367
|
+
// so the next leader's open does not race the browser's GC.
|
|
368
|
+
try {
|
|
369
|
+
db?.close();
|
|
370
|
+
} catch {
|
|
371
|
+
/* ignore */
|
|
372
|
+
}
|
|
373
|
+
db = null;
|
|
374
|
+
try {
|
|
375
|
+
pool?.pauseVfs?.();
|
|
376
|
+
} catch {
|
|
377
|
+
/* ignore */
|
|
378
|
+
}
|
|
379
|
+
pool = null;
|
|
380
|
+
releaseHeldLock();
|
|
381
|
+
queueMicrotask(() => self.close());
|
|
382
|
+
return {};
|
|
383
|
+
case 'remove-client': {
|
|
384
|
+
const port = clients.get(payload.clientId);
|
|
385
|
+
port?.close();
|
|
386
|
+
clients.delete(payload.clientId);
|
|
387
|
+
return {};
|
|
388
|
+
}
|
|
389
|
+
default:
|
|
390
|
+
throw new Error(`sqlite worker: unknown message ${type}`);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function dispatch(target: PostTarget, data: any, source: 'owner' | 'client'): void {
|
|
395
|
+
const { id, type, payload } = data ?? {};
|
|
396
|
+
noteTick();
|
|
397
|
+
const t0 = performance.now();
|
|
398
|
+
const runOp = async () => {
|
|
399
|
+
try {
|
|
400
|
+
if (thawVerification) await thawVerification;
|
|
401
|
+
const result = await handle(type, payload, source);
|
|
402
|
+
// `wt` (worker time) lets the main thread split a round-trip into actual
|
|
403
|
+
// DB work vs postMessage/queue overhead (see `__sqliteStats`).
|
|
404
|
+
target.postMessage({ id, ok: true, wt: performance.now() - t0, ...(result as object) });
|
|
405
|
+
} catch (err) {
|
|
406
|
+
target.postMessage({ id, ok: false, error: err instanceof Error ? err.message : String(err) });
|
|
407
|
+
}
|
|
408
|
+
};
|
|
409
|
+
opChain = opChain.then(runOp, runOp);
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
self.onmessage = (ev: MessageEvent) => {
|
|
413
|
+
const data = ev.data ?? {};
|
|
414
|
+
// `add-client` is handled inline (not on the op chain): it only registers a
|
|
415
|
+
// port and must not wait behind a long select, or the follower's first ops
|
|
416
|
+
// (already queued on that port by the time the ack arrives) would deadlock
|
|
417
|
+
// the attach handshake in the engine.
|
|
418
|
+
if (data.type === 'add-client') {
|
|
419
|
+
const port = ev.ports?.[0];
|
|
420
|
+
const clientId = data.payload?.clientId as string | undefined;
|
|
421
|
+
if (!port || !clientId) {
|
|
422
|
+
(self as unknown as Worker).postMessage({
|
|
423
|
+
id: data.id,
|
|
424
|
+
ok: false,
|
|
425
|
+
error: 'sqlite worker: add-client needs a clientId and a transferred port',
|
|
426
|
+
});
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
clients.get(clientId)?.close();
|
|
430
|
+
clients.set(clientId, port);
|
|
431
|
+
port.onmessage = (pe: MessageEvent) => dispatch(port, pe.data, 'client');
|
|
432
|
+
port.start?.();
|
|
433
|
+
(self as unknown as Worker).postMessage({ id: data.id, ok: true, wt: 0 });
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
dispatch(self as unknown as Worker, data, 'owner');
|
|
437
|
+
};
|