@spooky-sync/core 0.0.1-canary.154 → 0.0.1-canary.155
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +151 -0
- package/dist/index.js +1509 -120
- package/dist/{sqlite-plan-sql.js → sqlite-open.js} +83 -1
- package/dist/sqlite-worker.js +268 -108
- package/dist/tabs-broker-worker.d.ts +8 -0
- package/dist/tabs-broker-worker.js +434 -0
- package/dist/types.d.ts +25 -0
- package/package.json +4 -4
- package/scripts/check-broker-bundle.mjs +33 -0
- package/src/modules/cache/index.ts +35 -0
- package/src/modules/data/index.ts +13 -3
- package/src/modules/data/mutation-id.test.ts +25 -0
- package/src/modules/data/mutation-id.ts +35 -0
- package/src/modules/devtools/index.ts +9 -0
- package/src/modules/sync/queue/queue-up.forwarded.test.ts +51 -0
- package/src/modules/sync/queue/queue-up.ts +74 -38
- package/src/modules/sync/scheduler.ts +4 -2
- package/src/modules/sync/sync.ts +186 -8
- package/src/services/database/engine-factory.ts +3 -2
- package/src/services/database/sqlite-cache-engine.test.ts +160 -103
- package/src/services/database/sqlite-cache-engine.ts +285 -72
- package/src/services/database/sqlite-open.ts +36 -1
- package/src/services/database/sqlite-select.test.ts +13 -46
- package/src/services/database/sqlite-transport.fixture.ts +30 -0
- package/src/services/database/sqlite-transport.ts +219 -0
- package/src/services/database/sqlite-worker.ts +340 -55
- package/src/services/stream-processor/index.ts +11 -2
- package/src/services/tabs/broker-client.ts +283 -0
- package/src/services/tabs/broker.test.ts +278 -0
- package/src/services/tabs/coordinator.test.ts +192 -0
- package/src/services/tabs/coordinator.ts +567 -0
- package/src/services/tabs/fake-ports.fixture.ts +70 -0
- package/src/services/tabs/leader-locks.ts +75 -0
- package/src/services/tabs/protocol.ts +239 -0
- package/src/services/tabs/support.ts +36 -0
- package/src/services/tabs/tabs-broker-worker.ts +581 -0
- package/src/sp00ky.ts +164 -6
- package/src/types.ts +25 -0
- package/tsdown.config.ts +19 -10
|
@@ -18,6 +18,13 @@ import {
|
|
|
18
18
|
} from './events/index';
|
|
19
19
|
import { StaleEpochError } from './local';
|
|
20
20
|
import { translateSurql, tableOf, setPath, getPath, type SqlOp } from './surql-translate';
|
|
21
|
+
import {
|
|
22
|
+
BrokerPortClosedError,
|
|
23
|
+
PortSqliteTransport,
|
|
24
|
+
WorkerSqliteTransport,
|
|
25
|
+
type SqliteTransport,
|
|
26
|
+
} from './sqlite-transport';
|
|
27
|
+
import { PROMOTION_OPEN_OPTIONS } from './sqlite-open';
|
|
21
28
|
import type { EngineTx, Id, LocalStore, OrderBy, RelationFetch, Row } from './cache-engine';
|
|
22
29
|
import type { EngineStorageDiagnostics } from '../../modules/devtools/storage-info';
|
|
23
30
|
|
|
@@ -83,9 +90,9 @@ export function pureWriteOpResult(op: SqlOp): unknown {
|
|
|
83
90
|
* oracle E2E in the browser.
|
|
84
91
|
*/
|
|
85
92
|
export class SqliteCacheEngine implements LocalStore {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
private
|
|
93
|
+
/** The wire to the SQLite worker: an owned dedicated Worker (solo/leader) or
|
|
94
|
+
* a MessagePort into another tab's worker (follower). See sqlite-transport. */
|
|
95
|
+
private transport: SqliteTransport | null = null;
|
|
89
96
|
private storeEpoch = 0;
|
|
90
97
|
private knownTables = new Set<string>();
|
|
91
98
|
private useOpfs: boolean;
|
|
@@ -108,14 +115,23 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
108
115
|
|
|
109
116
|
readonly engineKind = 'sqlite' as const;
|
|
110
117
|
|
|
118
|
+
/** Shared-tabs mode: the engine's transport is swapped at runtime by the
|
|
119
|
+
* TabsCoordinator (owner worker as leader, leader's port as follower). */
|
|
120
|
+
private shared: boolean;
|
|
121
|
+
/** Leaderless parking (shared mode): ops entering the opQueue await this
|
|
122
|
+
* gate until a new role lands or the timeout rejects them. */
|
|
123
|
+
private roleGate: { promise: Promise<void>; release: () => void } | null = null;
|
|
124
|
+
private roleGateTimer: ReturnType<typeof setTimeout> | null = null;
|
|
125
|
+
|
|
111
126
|
constructor(
|
|
112
127
|
private config: Sp00kyConfig<any>['database'],
|
|
113
128
|
private logger: Logger,
|
|
114
|
-
opts: { useOpfs?: boolean; workerSelect?: boolean } = {}
|
|
129
|
+
opts: { useOpfs?: boolean; workerSelect?: boolean; shared?: boolean } = {}
|
|
115
130
|
) {
|
|
116
131
|
this.useOpfs = opts.useOpfs ?? true;
|
|
117
132
|
this.workerSelect = opts.workerSelect ?? config.workerSelect ?? true;
|
|
118
133
|
this.workerSelectConfigured = this.workerSelect;
|
|
134
|
+
this.shared = opts.shared ?? false;
|
|
119
135
|
}
|
|
120
136
|
|
|
121
137
|
get epoch(): number {
|
|
@@ -215,47 +231,19 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
215
231
|
|
|
216
232
|
// ---- worker plumbing -----------------------------------------------------
|
|
217
233
|
|
|
218
|
-
private spawnWorker(): Worker {
|
|
219
|
-
// Source references the `.ts` so the monorepo's src-bundling consumers
|
|
220
|
-
// (e.g. the example app, which aliases `@spooky-sync/core` to `src`) resolve
|
|
221
|
-
// it — Vite handles `.ts` workers. For the published package, the tsdown
|
|
222
|
-
// build rewrites this to `./sqlite-worker.js` (the top-level emitted entry;
|
|
223
|
-
// see tsdown.config.ts), which the flat `dist/index.js` resolves. The worker
|
|
224
|
-
// (+ `@sqlite.org/sqlite-wasm`) still loads lazily — only when `localEngine:
|
|
225
|
-
// 'sqlite'` is used.
|
|
226
|
-
const worker = new Worker(new URL('./sqlite-worker.ts', import.meta.url), { type: 'module' });
|
|
227
|
-
worker.onmessage = (ev: MessageEvent) => {
|
|
228
|
-
const { id, ok, error, ...rest } = ev.data ?? {};
|
|
229
|
-
const p = this.pending.get(id);
|
|
230
|
-
if (!p) return;
|
|
231
|
-
this.pending.delete(id);
|
|
232
|
-
if (ok) p.resolve(rest);
|
|
233
|
-
else p.reject(new Error(error));
|
|
234
|
-
};
|
|
235
|
-
// Surface a worker crash (wasm abort / OOM) instead of leaving every pending
|
|
236
|
-
// call hung forever — reject them all with a clear error.
|
|
237
|
-
const failAll = (msg: string) => {
|
|
238
|
-
const err = new Error(`SQLite worker crashed: ${msg}`);
|
|
239
|
-
this.logger.error(
|
|
240
|
-
{ err, Category: 'sp00ky-client::SqliteCacheEngine::worker' },
|
|
241
|
-
'Worker error'
|
|
242
|
-
);
|
|
243
|
-
for (const [, p] of this.pending) p.reject(err);
|
|
244
|
-
this.pending.clear();
|
|
245
|
-
};
|
|
246
|
-
worker.onerror = (e: ErrorEvent) => failAll(e.message || 'onerror');
|
|
247
|
-
worker.onmessageerror = () => failAll('messageerror');
|
|
248
|
-
return worker;
|
|
249
|
-
}
|
|
250
|
-
|
|
251
234
|
/** Serializes every worker op so reads/writes never overlap at the VFS layer
|
|
252
235
|
* (overlapping ops trip SQLITE_BUSY). Mirrors the SurrealDB engine's
|
|
253
|
-
* single-flight query queue.
|
|
236
|
+
* single-flight query queue. (The worker keeps its own chain too, for
|
|
237
|
+
* multi-client mode; this one additionally provides the queue-wait stat and
|
|
238
|
+
* the boot/switch atomicity below.) */
|
|
254
239
|
private opQueue: Promise<unknown> = Promise.resolve();
|
|
255
240
|
|
|
256
241
|
private call<T = any>(type: string, payload?: unknown): Promise<T> {
|
|
257
242
|
const enqueuedAt = performance.now();
|
|
258
|
-
const run = () => {
|
|
243
|
+
const run = async () => {
|
|
244
|
+
// Leaderless window (shared mode): park behind the role gate instead of
|
|
245
|
+
// failing; a new leader releases it, the timeout rejects it.
|
|
246
|
+
if (this.roleGate) await this.roleGate.promise;
|
|
259
247
|
// Time spent waiting behind other ops in the queue, not doing work.
|
|
260
248
|
getStats().queueWaitMs += performance.now() - enqueuedAt;
|
|
261
249
|
return this.rawCall<T>(type, payload);
|
|
@@ -270,8 +258,7 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
270
258
|
}
|
|
271
259
|
|
|
272
260
|
private rawCall<T = any>(type: string, payload?: unknown): Promise<T> {
|
|
273
|
-
if (!this.
|
|
274
|
-
const id = ++this.seq;
|
|
261
|
+
if (!this.transport) throw new Error('SqliteCacheEngine: not connected');
|
|
275
262
|
// --- instrumentation: live, inspectable via `globalThis.__sqliteStats` ---
|
|
276
263
|
const s = getStats();
|
|
277
264
|
s.roundTrips++;
|
|
@@ -282,30 +269,25 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
282
269
|
}
|
|
283
270
|
s.inFlight++;
|
|
284
271
|
s.maxInFlight = Math.max(s.maxInFlight, s.inFlight);
|
|
285
|
-
|
|
286
|
-
s.inFlight--;
|
|
287
|
-
};
|
|
272
|
+
if (this.transport.kind === 'port') s.proxiedOps = (s.proxiedOps ?? 0) + 1;
|
|
288
273
|
const sentAt = performance.now();
|
|
289
|
-
return
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
});
|
|
307
|
-
this.worker!.postMessage({ id, type, payload });
|
|
308
|
-
});
|
|
274
|
+
return this.transport.call<T>(type, payload).then(
|
|
275
|
+
(v: T) => {
|
|
276
|
+
s.inFlight--;
|
|
277
|
+
// Split the round-trip: `wt` is time inside the worker's handler,
|
|
278
|
+
// the remainder is postMessage + scheduling overhead.
|
|
279
|
+
const wt = (v as { wt?: unknown } | null)?.wt;
|
|
280
|
+
if (typeof wt === 'number') {
|
|
281
|
+
s.workerMs += wt;
|
|
282
|
+
s.rpcOverheadMs += Math.max(0, performance.now() - sentAt - wt);
|
|
283
|
+
}
|
|
284
|
+
return v;
|
|
285
|
+
},
|
|
286
|
+
(e: unknown) => {
|
|
287
|
+
s.inFlight--;
|
|
288
|
+
throw e;
|
|
289
|
+
}
|
|
290
|
+
);
|
|
309
291
|
}
|
|
310
292
|
|
|
311
293
|
/**
|
|
@@ -313,8 +295,30 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
313
295
|
* {@link call}) so it can run as the body of an already-queued opQueue entry
|
|
314
296
|
* without re-queuing onto itself. Callers must run it through the opQueue.
|
|
315
297
|
*/
|
|
316
|
-
|
|
317
|
-
|
|
298
|
+
/** Seam for tests: swap in a fake transport instead of a real Worker. */
|
|
299
|
+
protected createTransport(): SqliteTransport {
|
|
300
|
+
return new WorkerSqliteTransport(this.logger);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
private async openInternal(
|
|
304
|
+
bucketId: string,
|
|
305
|
+
extras?: {
|
|
306
|
+
workerLockName?: string;
|
|
307
|
+
openOptions?: { maxAttempts?: number; backoffMs?: number[]; disallowMemoryFallback?: boolean };
|
|
308
|
+
}
|
|
309
|
+
): Promise<void> {
|
|
310
|
+
if (!this.transport || this.transport.kind !== 'worker' || !this.transport.connected) {
|
|
311
|
+
this.transport = this.createTransport();
|
|
312
|
+
if (this.transport instanceof WorkerSqliteTransport) {
|
|
313
|
+
// The worker fencing itself (leadership stolen from a frozen tab that
|
|
314
|
+
// thawed) is a leader-loss: park ops until the broker re-adopts us.
|
|
315
|
+
this.transport.onLockLost = (reason) => {
|
|
316
|
+
this.transport = null;
|
|
317
|
+
this.storeEpoch++;
|
|
318
|
+
this.openRoleGate(`worker fenced: ${reason}`);
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
}
|
|
318
322
|
// Seed the `_00_*` system tables as part of `open` (worker-side, one round
|
|
319
323
|
// trip). The LocalMigrator DEFINEs them, but `translateSurql` lowers every
|
|
320
324
|
// DEFINE to a noop on this engine (SQLite has no DDL vocabulary), so
|
|
@@ -333,6 +337,8 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
333
337
|
dbName: bucketId,
|
|
334
338
|
useOpfs: this.useOpfs,
|
|
335
339
|
systemTables: SYSTEM_TABLES,
|
|
340
|
+
...(extras?.workerLockName ? { workerLockName: extras.workerLockName } : {}),
|
|
341
|
+
...(extras?.openOptions ? { openOptions: extras.openOptions } : {}),
|
|
336
342
|
});
|
|
337
343
|
this.knownTables.clear();
|
|
338
344
|
for (const t of SYSTEM_TABLES) this.knownTables.add(t);
|
|
@@ -349,6 +355,7 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
349
355
|
fallback: fellBack,
|
|
350
356
|
};
|
|
351
357
|
if (fellBack && opfsError) health.error = opfsError;
|
|
358
|
+
if (this.shared) health.role = this.roleLabel;
|
|
352
359
|
this.setStorageHealth(health);
|
|
353
360
|
const stats = getStats();
|
|
354
361
|
stats.persisted = persisted;
|
|
@@ -393,28 +400,230 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
393
400
|
// As a single entry, every other op runs fully before the close or after
|
|
394
401
|
// the reopen — never against a closed DB.
|
|
395
402
|
await this.enqueue(async () => {
|
|
396
|
-
if (this.
|
|
403
|
+
if (this.transport) {
|
|
397
404
|
try {
|
|
398
405
|
await this.rawCall('close');
|
|
399
406
|
} catch {
|
|
400
407
|
/* ignore */
|
|
401
408
|
}
|
|
402
|
-
this.
|
|
403
|
-
this.
|
|
409
|
+
this.transport.close('bucket switch');
|
|
410
|
+
this.transport = null;
|
|
404
411
|
}
|
|
405
412
|
await this.openInternal(bucketId);
|
|
406
413
|
});
|
|
407
414
|
}
|
|
408
415
|
|
|
409
416
|
async close(): Promise<void> {
|
|
410
|
-
if (!this.
|
|
417
|
+
if (!this.transport) return;
|
|
411
418
|
try {
|
|
412
419
|
await this.call('close');
|
|
413
420
|
} catch {
|
|
414
421
|
/* ignore */
|
|
415
422
|
}
|
|
416
|
-
this.
|
|
417
|
-
this.
|
|
423
|
+
this.transport.close('engine closed');
|
|
424
|
+
this.transport = null;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// ---- shared-tabs role modes ------------------------------------------------
|
|
428
|
+
// The TabsCoordinator drives these; solo mode never touches them. The engine
|
|
429
|
+
// object is never replaced across role changes, so `storeEpoch` stays one
|
|
430
|
+
// monotonic per-tab counter and every existing fencing consumer keeps
|
|
431
|
+
// working unchanged.
|
|
432
|
+
|
|
433
|
+
/** Which role the current transport represents, for StorageHealth. */
|
|
434
|
+
private roleLabel: 'leader' | 'follower' | 'solo' = 'solo';
|
|
435
|
+
/** True once this engine has had a usable store at least once; role changes
|
|
436
|
+
* after that point invalidate in-flight reads and must bump the epoch. */
|
|
437
|
+
private hadStore = false;
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* Role transitions run on their OWN chain, never on the opQueue: parked ops
|
|
441
|
+
* sit INSIDE opQueue entries waiting for the role gate, so a transition
|
|
442
|
+
* queued behind them could never run to release them (deadlock). Transitions
|
|
443
|
+
* are safe off-queue because in-flight ops on a dead transport were already
|
|
444
|
+
* rejected, parked ops only resume after the transition completes, and the
|
|
445
|
+
* worker serializes everything worker-side anyway.
|
|
446
|
+
*/
|
|
447
|
+
private transitionChain: Promise<unknown> = Promise.resolve();
|
|
448
|
+
|
|
449
|
+
private chainTransition<T>(fn: () => Promise<T>): Promise<T> {
|
|
450
|
+
const result = this.transitionChain.then(fn, fn);
|
|
451
|
+
this.transitionChain = result.then(
|
|
452
|
+
() => undefined,
|
|
453
|
+
() => undefined
|
|
454
|
+
);
|
|
455
|
+
return result;
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
private openRoleGate(reason: string): void {
|
|
459
|
+
if (this.roleGate) return;
|
|
460
|
+
let release!: () => void;
|
|
461
|
+
let rejectFn!: (e: Error) => void;
|
|
462
|
+
const promise = new Promise<void>((res, rej) => {
|
|
463
|
+
release = res;
|
|
464
|
+
rejectFn = rej;
|
|
465
|
+
});
|
|
466
|
+
// Swallow the timeout rejection for waiters that already resolved.
|
|
467
|
+
promise.catch(() => {});
|
|
468
|
+
this.roleGate = { promise, release };
|
|
469
|
+
this.roleGateTimer = setTimeout(() => {
|
|
470
|
+
// Nothing adopted us in time: reject the parked ops (retryable) and drop
|
|
471
|
+
// the gate so later calls fail fast with 'not connected'.
|
|
472
|
+
rejectFn(new BrokerPortClosedError(`no leader adopted this tab: ${reason}`));
|
|
473
|
+
this.roleGate = null;
|
|
474
|
+
this.roleGateTimer = null;
|
|
475
|
+
}, 20_000);
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
private closeRoleGate(): void {
|
|
479
|
+
if (this.roleGateTimer) clearTimeout(this.roleGateTimer);
|
|
480
|
+
this.roleGateTimer = null;
|
|
481
|
+
this.roleGate?.release();
|
|
482
|
+
this.roleGate = null;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* Become the store owner (leader). Boot and failover share this path; a
|
|
487
|
+
* failover (a previous transport existed) bumps the epoch FIRST so every
|
|
488
|
+
* in-flight chain that captured the old epoch fences itself. `resumeHeld`
|
|
489
|
+
* keeps the live worker after a broker restart and only rolls the
|
|
490
|
+
* per-leadership lock forward.
|
|
491
|
+
*/
|
|
492
|
+
async adoptOwner(
|
|
493
|
+
bucketId: string,
|
|
494
|
+
opts: {
|
|
495
|
+
workerLockName: string;
|
|
496
|
+
allowMemoryFallback: boolean;
|
|
497
|
+
resumeHeld: boolean;
|
|
498
|
+
}
|
|
499
|
+
): Promise<StorageHealth> {
|
|
500
|
+
this.roleLabel = 'leader';
|
|
501
|
+
return this.chainTransition(async () => {
|
|
502
|
+
if (
|
|
503
|
+
opts.resumeHeld &&
|
|
504
|
+
this.transport?.kind === 'worker' &&
|
|
505
|
+
this.transport.connected &&
|
|
506
|
+
this.bucketId === bucketId
|
|
507
|
+
) {
|
|
508
|
+
await this.rawCall('relock', { workerLockName: opts.workerLockName });
|
|
509
|
+
this.closeRoleGate();
|
|
510
|
+
return this.storageHealthValue;
|
|
511
|
+
}
|
|
512
|
+
if (this.hadStore) this.storeEpoch++;
|
|
513
|
+
if (this.transport) {
|
|
514
|
+
try {
|
|
515
|
+
if (this.transport.kind === 'worker') await this.rawCall('close');
|
|
516
|
+
} catch {
|
|
517
|
+
/* ignore */
|
|
518
|
+
}
|
|
519
|
+
this.transport.close('adopting ownership');
|
|
520
|
+
this.transport = null;
|
|
521
|
+
}
|
|
522
|
+
await this.openInternal(bucketId, {
|
|
523
|
+
workerLockName: opts.workerLockName,
|
|
524
|
+
openOptions: {
|
|
525
|
+
...PROMOTION_OPEN_OPTIONS,
|
|
526
|
+
disallowMemoryFallback: !opts.allowMemoryFallback,
|
|
527
|
+
},
|
|
528
|
+
});
|
|
529
|
+
// Leader wipe-on-pool-open: local `_00_query` rows from earlier sessions
|
|
530
|
+
// are dead (query hashes are session-salted) and this is the one moment
|
|
531
|
+
// no other tab is attached, so clearing here replaces the per-switch
|
|
532
|
+
// DELETE that solo mode does in doSwitchBucket. Followers never wipe.
|
|
533
|
+
try {
|
|
534
|
+
await this.rawCall('run', { sql: 'DELETE FROM "_00_query"' });
|
|
535
|
+
} catch {
|
|
536
|
+
/* fresh bucket: table just seeded, nothing to wipe */
|
|
537
|
+
}
|
|
538
|
+
getStats().roleChanges = (getStats().roleChanges ?? 0) + 1;
|
|
539
|
+
this.hadStore = true;
|
|
540
|
+
this.closeRoleGate();
|
|
541
|
+
return this.storageHealthValue;
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/** Attach to a leader's worker through `dbPort` (follower). */
|
|
546
|
+
async adoptAttached(
|
|
547
|
+
dbPort: MessagePort,
|
|
548
|
+
snapshot: { bucketId: string; storageHealth: StorageHealth },
|
|
549
|
+
onPortDead: (reason: string) => void
|
|
550
|
+
): Promise<void> {
|
|
551
|
+
this.roleLabel = 'follower';
|
|
552
|
+
return this.chainTransition(async () => {
|
|
553
|
+
if (this.hadStore) this.storeEpoch++;
|
|
554
|
+
this.transport?.close('adopting leader port');
|
|
555
|
+
this.transport = new PortSqliteTransport(dbPort, onPortDead, this.logger);
|
|
556
|
+
this.bucketId = snapshot.bucketId;
|
|
557
|
+
// The shared store exists and is seeded; mirror the owner's bookkeeping.
|
|
558
|
+
this.knownTables.clear();
|
|
559
|
+
for (const t of SYSTEM_TABLES) this.knownTables.add(t);
|
|
560
|
+
const health: StorageHealth = { ...snapshot.storageHealth, role: 'follower' };
|
|
561
|
+
this.setStorageHealth(health);
|
|
562
|
+
const stats = getStats();
|
|
563
|
+
stats.persisted = snapshot.storageHealth.status === 'persistent';
|
|
564
|
+
stats.roleChanges = (stats.roleChanges ?? 0) + 1;
|
|
565
|
+
delete stats.opfsError;
|
|
566
|
+
this.hadStore = true;
|
|
567
|
+
this.closeRoleGate();
|
|
568
|
+
});
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
/** Demoted while owning the store (zombie thaw, stale promotion): tear the
|
|
572
|
+
* worker down so its OPFS handles free up, then park until re-adopted. */
|
|
573
|
+
async releaseOwnership(): Promise<void> {
|
|
574
|
+
await this.chainTransition(async () => {
|
|
575
|
+
if (this.transport) {
|
|
576
|
+
try {
|
|
577
|
+
if (this.transport.kind === 'worker') {
|
|
578
|
+
await (this.transport as WorkerSqliteTransport).shutdown();
|
|
579
|
+
}
|
|
580
|
+
} catch {
|
|
581
|
+
/* worker may already be fenced/dead */
|
|
582
|
+
}
|
|
583
|
+
this.transport.close('ownership released');
|
|
584
|
+
this.transport = null;
|
|
585
|
+
}
|
|
586
|
+
this.storeEpoch++;
|
|
587
|
+
this.openRoleGate('ownership released');
|
|
588
|
+
});
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
/** The leader (or its port) died. Called from the port-dead callback and the
|
|
592
|
+
* coordinator; NOT enqueued, so in-flight ops reject immediately instead of
|
|
593
|
+
* waiting behind whatever is stuck. */
|
|
594
|
+
onLeaderLost(reason: string): void {
|
|
595
|
+
if (this.transport?.kind === 'port') {
|
|
596
|
+
this.transport.close(reason);
|
|
597
|
+
this.transport = null;
|
|
598
|
+
}
|
|
599
|
+
this.storeEpoch++;
|
|
600
|
+
this.openRoleGate(reason);
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/** Leader side: forward a follower's dbPort into the owned worker. */
|
|
604
|
+
async exposeClientPort(clientId: string, port: MessagePort): Promise<void> {
|
|
605
|
+
if (this.transport?.kind !== 'worker') {
|
|
606
|
+
throw new Error('SqliteCacheEngine: not the store owner');
|
|
607
|
+
}
|
|
608
|
+
await (this.transport as WorkerSqliteTransport).addClientPort(clientId, port);
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
async removeClientPort(clientId: string): Promise<void> {
|
|
612
|
+
if (this.transport?.kind !== 'worker') return;
|
|
613
|
+
await (this.transport as WorkerSqliteTransport).removeClientPort(clientId);
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
/** Graceful pagehide as owner: release OPFS handles NOW so the next leader
|
|
617
|
+
* does not race the browser's worker GC. */
|
|
618
|
+
async shutdownOwnedWorker(): Promise<void> {
|
|
619
|
+
if (this.transport?.kind !== 'worker') return;
|
|
620
|
+
try {
|
|
621
|
+
await (this.transport as WorkerSqliteTransport).shutdown();
|
|
622
|
+
} catch {
|
|
623
|
+
/* ignore */
|
|
624
|
+
}
|
|
625
|
+
this.transport.close('shutdown');
|
|
626
|
+
this.transport = null;
|
|
418
627
|
}
|
|
419
628
|
|
|
420
629
|
private async ensureTable(table: string): Promise<void> {
|
|
@@ -855,6 +1064,10 @@ interface SqliteStats {
|
|
|
855
1064
|
persisted?: boolean;
|
|
856
1065
|
/** Why OPFS persistence failed, when it did. */
|
|
857
1066
|
opfsError?: string;
|
|
1067
|
+
/** Shared-tabs follower: ops that crossed the leader's MessagePort. */
|
|
1068
|
+
proxiedOps?: number;
|
|
1069
|
+
/** Times this tab's engine changed hands (promotions + attachments). */
|
|
1070
|
+
roleChanges?: number;
|
|
858
1071
|
}
|
|
859
1072
|
|
|
860
1073
|
const EMPTY_STATS: SqliteStats = {
|
|
@@ -27,12 +27,22 @@ export interface SqliteDbHandle {
|
|
|
27
27
|
close: () => void;
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
/** The slice of the OpfsSAHPoolUtil the worker needs for teardown. */
|
|
31
|
+
export interface SqlitePoolHandle {
|
|
32
|
+
/** Unregisters the VFS and releases every sync access handle, leaving the
|
|
33
|
+
* files intact, so another worker can open the pool without waiting for
|
|
34
|
+
* this worker to be garbage collected. Throws while files are open. */
|
|
35
|
+
pauseVfs?: () => unknown;
|
|
36
|
+
}
|
|
37
|
+
|
|
30
38
|
export interface OpenDbResult {
|
|
31
39
|
db: SqliteDbHandle;
|
|
32
40
|
/** True only when the handle is backed by OPFS and survives a reload. */
|
|
33
41
|
persisted: boolean;
|
|
34
42
|
/** Why persistence failed. Set only when OPFS was requested and fell back. */
|
|
35
43
|
opfsError?: string;
|
|
44
|
+
/** Pool util, present only for an OPFS-backed handle. */
|
|
45
|
+
pool?: SqlitePoolHandle;
|
|
36
46
|
}
|
|
37
47
|
|
|
38
48
|
export interface OpenDbOptions {
|
|
@@ -40,6 +50,14 @@ export interface OpenDbOptions {
|
|
|
40
50
|
maxAttempts?: number;
|
|
41
51
|
/** Delay before each retry; the last entry repeats. Default [250, 500]. */
|
|
42
52
|
backoffMs?: number[];
|
|
53
|
+
/**
|
|
54
|
+
* Throw (`opfs-unavailable: <reason>`) instead of falling back to memory.
|
|
55
|
+
* Used by shared-tabs leader promotion: a silently-in-memory LEADER would
|
|
56
|
+
* put every tab's data in RAM, so promotion prefers failing the election
|
|
57
|
+
* (the broker retries, possibly on another tab) over degrading. The broker
|
|
58
|
+
* grants an explicit memory fallback only after repeated failed cycles.
|
|
59
|
+
*/
|
|
60
|
+
disallowMemoryFallback?: boolean;
|
|
43
61
|
/** Injectable for tests. */
|
|
44
62
|
sleep?: (ms: number) => Promise<void>;
|
|
45
63
|
}
|
|
@@ -48,6 +66,17 @@ const DEFAULT_MAX_ATTEMPTS = 3;
|
|
|
48
66
|
/** Bounded on purpose: this runs on the boot path, before the first query. */
|
|
49
67
|
const DEFAULT_BACKOFF_MS = [250, 500];
|
|
50
68
|
|
|
69
|
+
/**
|
|
70
|
+
* Leader-promotion profile (~5.8s worst case). A dead leader's sync access
|
|
71
|
+
* handles release when the browser garbage-collects its worker, typically
|
|
72
|
+
* well under a second but not synchronously with the Web Lock release the
|
|
73
|
+
* election observed, so promotion retries longer than a cold boot.
|
|
74
|
+
*/
|
|
75
|
+
export const PROMOTION_OPEN_OPTIONS: Pick<OpenDbOptions, 'maxAttempts' | 'backoffMs'> = {
|
|
76
|
+
maxAttempts: 10,
|
|
77
|
+
backoffMs: [50, 100, 200, 400, 800, 1000],
|
|
78
|
+
};
|
|
79
|
+
|
|
51
80
|
/** Failures no retry can fix: the APIs aren't there at all (insecure context,
|
|
52
81
|
* or a browser without sync access handles). Fall back immediately. */
|
|
53
82
|
const UNRETRYABLE = ['Missing required OPFS APIs'];
|
|
@@ -96,6 +125,9 @@ export async function openDb(
|
|
|
96
125
|
if (!useOpfs) return { db: new sqlite3.oo1.DB(':memory:', 'c'), persisted: false };
|
|
97
126
|
|
|
98
127
|
if (!sqlite3.installOpfsSAHPoolVfs) {
|
|
128
|
+
if (opts.disallowMemoryFallback) {
|
|
129
|
+
throw new Error('opfs-unavailable: sqlite-wasm build has no installOpfsSAHPoolVfs');
|
|
130
|
+
}
|
|
99
131
|
return fallbackToMemory(sqlite3, dbName, 'sqlite-wasm build has no installOpfsSAHPoolVfs', 0);
|
|
100
132
|
}
|
|
101
133
|
|
|
@@ -118,12 +150,15 @@ export async function openDb(
|
|
|
118
150
|
// doesn't ask for a real re-init just replays the same rejection.
|
|
119
151
|
...(attempt > 1 ? { forceReinitIfPreviouslyFailed: true } : {}),
|
|
120
152
|
});
|
|
121
|
-
return { db: new pool.OpfsSAHPoolDb(`/${dbName}.sqlite3`), persisted: true };
|
|
153
|
+
return { db: new pool.OpfsSAHPoolDb(`/${dbName}.sqlite3`), persisted: true, pool };
|
|
122
154
|
} catch (e) {
|
|
123
155
|
lastError = errMessage(e);
|
|
124
156
|
if (attempt === maxAttempts || UNRETRYABLE.some((m) => lastError.includes(m))) break;
|
|
125
157
|
await sleep(backoffMs[Math.min(attempt - 1, backoffMs.length - 1)] ?? 0);
|
|
126
158
|
}
|
|
127
159
|
}
|
|
160
|
+
if (opts.disallowMemoryFallback) {
|
|
161
|
+
throw new Error(`opfs-unavailable: ${lastError} (after ${attempts} attempts)`);
|
|
162
|
+
}
|
|
128
163
|
return fallbackToMemory(sqlite3, dbName, lastError, attempts);
|
|
129
164
|
}
|
|
@@ -3,6 +3,7 @@ import { RecordId } from 'surrealdb';
|
|
|
3
3
|
import type { QueryPlan } from '@spooky-sync/query-builder';
|
|
4
4
|
import { executeSelect, type SelectDb } from './sqlite-select';
|
|
5
5
|
import { SqliteCacheEngine } from './sqlite-cache-engine';
|
|
6
|
+
import { stubTransport } from './sqlite-transport.fixture';
|
|
6
7
|
import { stableKey } from './relation-resolver';
|
|
7
8
|
import type { Row } from './cache-engine';
|
|
8
9
|
|
|
@@ -87,52 +88,18 @@ function makeEngine(opts: {
|
|
|
87
88
|
makeLogger(),
|
|
88
89
|
{ useOpfs: false, workerSelect: opts.workerSelect }
|
|
89
90
|
);
|
|
90
|
-
(engine
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
Promise.resolve().then(async () => {
|
|
103
|
-
let ok = true;
|
|
104
|
-
let error: string | undefined;
|
|
105
|
-
let rest: Record<string, unknown> = {};
|
|
106
|
-
try {
|
|
107
|
-
if (msg.type === 'open') rest = { persisted: false };
|
|
108
|
-
else if (msg.type === 'exec') rest = { rows: respond(msg.payload.sql, msg.payload.bind ?? []) };
|
|
109
|
-
else if (msg.type === 'run' || msg.type === 'batch' || msg.type === 'close') rest = {};
|
|
110
|
-
else if (msg.type === 'select') {
|
|
111
|
-
if (!opts.answerSelect) throw new Error(`sqlite worker: unknown message ${msg.type}`);
|
|
112
|
-
const dbCalls: { sql: string; bind?: unknown[] }[] = [];
|
|
113
|
-
rest = await executeSelect(
|
|
114
|
-
msg.payload.plan,
|
|
115
|
-
msg.payload.params ?? {},
|
|
116
|
-
makeSelectDb(FIXTURE, dbCalls)
|
|
117
|
-
);
|
|
118
|
-
} else throw new Error(`sqlite worker: unknown message ${msg.type}`);
|
|
119
|
-
} catch (e) {
|
|
120
|
-
ok = false;
|
|
121
|
-
error = e instanceof Error ? e.message : String(e);
|
|
122
|
-
}
|
|
123
|
-
(engine as any).worker &&
|
|
124
|
-
(engine as any).pending.get(msg.id) &&
|
|
125
|
-
((): void => {
|
|
126
|
-
const p = (engine as any).pending.get(msg.id);
|
|
127
|
-
(engine as any).pending.delete(msg.id);
|
|
128
|
-
if (ok) p.resolve(rest);
|
|
129
|
-
else p.reject(new Error(error));
|
|
130
|
-
})();
|
|
131
|
-
});
|
|
132
|
-
},
|
|
133
|
-
};
|
|
134
|
-
return w;
|
|
135
|
-
};
|
|
91
|
+
stubTransport(engine, async (type, payload) => {
|
|
92
|
+
calls.push({ type, sql: payload?.sql, bind: payload?.bind, payload });
|
|
93
|
+
if (type === 'open') return { persisted: false };
|
|
94
|
+
if (type === 'exec') return { rows: respond(payload.sql, payload.bind ?? []) };
|
|
95
|
+
if (type === 'run' || type === 'batch' || type === 'close') return {};
|
|
96
|
+
if (type === 'select') {
|
|
97
|
+
if (!opts.answerSelect) throw new Error(`sqlite worker: unknown message ${type}`);
|
|
98
|
+
const dbCalls: { sql: string; bind?: unknown[] }[] = [];
|
|
99
|
+
return executeSelect(payload.plan, payload.params ?? {}, makeSelectDb(FIXTURE, dbCalls));
|
|
100
|
+
}
|
|
101
|
+
throw new Error(`sqlite worker: unknown message ${type}`);
|
|
102
|
+
});
|
|
136
103
|
return { engine, calls };
|
|
137
104
|
}
|
|
138
105
|
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Test-only helper: stubs `SqliteCacheEngine.createTransport` with a fake
|
|
3
|
+
* transport driven by a plain handler function, replacing the old pattern of
|
|
4
|
+
* faking a whole Worker plus the engine's pending-map wiring. Not exported
|
|
5
|
+
* from the package; imported only by *.test.ts files.
|
|
6
|
+
*/
|
|
7
|
+
import type { SqliteTransport } from './sqlite-transport';
|
|
8
|
+
|
|
9
|
+
export type FakeTransportHandler = (type: string, payload: any) => unknown | Promise<unknown>;
|
|
10
|
+
|
|
11
|
+
/** Replace the engine's transport factory. Each open spawns a fresh fake, like
|
|
12
|
+
* the real factory spawns a fresh Worker. The handler returns the reply rest
|
|
13
|
+
* (without id/ok/wt) or throws to produce an error reply. */
|
|
14
|
+
export function stubTransport(engine: unknown, handler: FakeTransportHandler): void {
|
|
15
|
+
(engine as { createTransport: () => SqliteTransport }).createTransport = () => {
|
|
16
|
+
let closed = false;
|
|
17
|
+
return {
|
|
18
|
+
kind: 'worker',
|
|
19
|
+
get connected() {
|
|
20
|
+
return !closed;
|
|
21
|
+
},
|
|
22
|
+
call: <T>(type: string, payload?: unknown) =>
|
|
23
|
+
Promise.resolve().then(() => handler(type, payload)) as Promise<T>,
|
|
24
|
+
failAll() {},
|
|
25
|
+
close() {
|
|
26
|
+
closed = true;
|
|
27
|
+
},
|
|
28
|
+
} as SqliteTransport;
|
|
29
|
+
};
|
|
30
|
+
}
|