@spooky-sync/core 0.0.1-canary.176 → 0.0.1-canary.178

package/dist/index.js

@@ -1998,16 +1998,16 @@ var BaseTransport = class {
 			}
 		});
 	}
-	failAll(reason) {
+	failAll(reason, err) {
 		if (this.pending.size === 0) return;
-		const err = this.makeError(reason);
-		for (const [, p] of this.pending) p.reject(err);
+		const e = err ?? this.makeError(reason);
+		for (const [, p] of this.pending) p.reject(e);
 		this.pending.clear();
 	}
-	close(reason = "closed") {
+	close(reason = "closed", err) {
 		if (this.closed) return;
 		this.closed = true;
-		this.failAll(reason);
+		this.failAll(reason, err);
 	}
 };
 var WorkerSqliteTransport = class extends BaseTransport {
@@ -2071,9 +2071,9 @@ var WorkerSqliteTransport = class extends BaseTransport {
 	shutdown() {
 		return this.call("shutdown").then(() => void 0);
 	}
-	close(reason = "closed") {
+	close(reason = "closed", err) {
 		if (this.closed) return;
-		super.close(reason);
+		super.close(reason, err);
 		this.worker.terminate();
 	}
 };
@@ -2092,10 +2092,10 @@ var PortSqliteTransport = class extends BaseTransport {
 	markDead(reason) {
 		this.dead(reason);
 	}
-	dead(reason) {
+	dead(reason, err) {
 		if (this.closed) return;
 		this.closed = true;
-		this.failAll(reason);
+		this.failAll(reason, err);
 		try {
 			this.port.close();
 		} catch {}
@@ -2107,8 +2107,8 @@ var PortSqliteTransport = class extends BaseTransport {
 	makeError(reason) {
 		return new BrokerPortClosedError(reason);
 	}
-	close(reason = "closed") {
-		this.dead(reason);
+	close(reason = "closed", err) {
+		this.dead(reason, err);
 	}
 };
 
@@ -2304,6 +2304,28 @@ var SqliteCacheEngine = class {
 		this.opQueue = result.then(() => void 0, () => void 0);
 		return result;
 	}
+	/**
+	* Ops dispatched to the worker and not yet answered. Role transitions run on
+	* their own chain (see {@link transitionChain}), so they can start while the
+	* opQueue still has an op at the worker; tearing the transport down under it
+	* would reject that op — and its caller may be a query whose only fetch this
+	* was. {@link drainInFlight} lets a deliberate teardown wait them out.
+	*/
+	inFlightCalls = /* @__PURE__ */ new Set();
+	/** Wait for dispatched ops to answer before a deliberate transport teardown.
+	*  Bounded: a wedged worker must not block the role change forever. */
+	async drainInFlight(timeoutMs = 2e3) {
+		if (this.inFlightCalls.size === 0) return;
+		const settled = Promise.all([...this.inFlightCalls].map((p) => p.catch(() => void 0)));
+		let timer;
+		try {
+			await Promise.race([settled, new Promise((resolve) => {
+				timer = setTimeout(resolve, timeoutMs);
+			})]);
+		} finally {
+			if (timer) clearTimeout(timer);
+		}
+	}
 	rawCall(type, payload) {
 		if (!this.transport) throw new Error("SqliteCacheEngine: not connected");
 		const s = getStats();
@@ -2317,7 +2339,17 @@ var SqliteCacheEngine = class {
 		s.maxInFlight = Math.max(s.maxInFlight, s.inFlight);
 		if (this.transport.kind === "port") s.proxiedOps = (s.proxiedOps ?? 0) + 1;
 		const sentAt = performance.now();
+		let settle;
+		const tracked = new Promise((res) => {
+			settle = res;
+		});
+		this.inFlightCalls.add(tracked);
+		const finish = () => {
+			this.inFlightCalls.delete(tracked);
+			settle();
+		};
 		return this.transport.call(type, payload).then((v) => {
+			finish();
 			s.inFlight--;
 			const wt = v?.wt;
 			if (typeof wt === "number") {
@@ -2326,6 +2358,7 @@ var SqliteCacheEngine = class {
 			}
 			return v;
 		}, (e) => {
+			finish();
 			s.inFlight--;
 			throw e;
 		});
@@ -2471,12 +2504,13 @@ var SqliteCacheEngine = class {
 				this.closeRoleGate();
 				return this.storageHealthValue;
 			}
+			if (this.transport) await this.drainInFlight();
 			if (this.hadStore) this.storeEpoch++;
 			if (this.transport) {
 				try {
 					if (this.transport.kind === "worker") await this.rawCall("close");
 				} catch {}
-				this.transport.close("adopting ownership");
+				this.transport.close("adopting ownership", new BrokerPortClosedError("adopting ownership"));
 				this.transport = null;
 			}
 			await this.openInternal(bucketId, {
@@ -2499,8 +2533,9 @@ var SqliteCacheEngine = class {
 	async adoptAttached(dbPort, snapshot, onPortDead) {
 		this.roleLabel = "follower";
 		return this.chainTransition(async () => {
+			if (this.transport) await this.drainInFlight();
 			if (this.hadStore) this.storeEpoch++;
-			this.transport?.close("adopting leader port");
+			this.transport?.close("adopting leader port", new BrokerPortClosedError("adopting leader port"));
 			this.transport = new PortSqliteTransport(dbPort, onPortDead, this.logger);
 			this.bucketId = snapshot.bucketId;
 			this.knownTables.clear();
@@ -2523,10 +2558,11 @@ var SqliteCacheEngine = class {
 	async releaseOwnership() {
 		await this.chainTransition(async () => {
 			if (this.transport) {
+				await this.drainInFlight();
 				try {
 					if (this.transport.kind === "worker") await this.transport.shutdown();
 				} catch {}
-				this.transport.close("ownership released");
+				this.transport.close("ownership released", new BrokerPortClosedError("ownership released"));
 				this.transport = null;
 			}
 			this.storeEpoch++;
@@ -6892,8 +6928,8 @@ function selfAllowlistedVariant(flag, userId) {
 
 //#endregion
 //#region src/modules/devtools/index.ts
-const CORE_VERSION = "0.0.1-canary.176";
-const WASM_VERSION = "0.0.1-canary.176";
+const CORE_VERSION = "0.0.1-canary.178";
+const WASM_VERSION = "0.0.1-canary.178";
 const SURREAL_VERSION = "3.0.3";
 var DevToolsService = class DevToolsService {
 	eventsHistory = [];
@@ -11324,7 +11360,7 @@ var Sp00kyClient = class {
 		return new TabsCoordinator({
 			tabId,
 			fingerprint: computeTabsFingerprint({
-				coreVersion: "0.0.1-canary.176",
+				coreVersion: "0.0.1-canary.178",
 				schemaHash: hash53(this.config.schemaSurql),
 				endpoint: this.config.database.endpoint ?? "",
 				namespace: this.config.database.namespace,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/core",
-  "version": "0.0.1-canary.176",
+  "version": "0.0.1-canary.178",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -60,8 +60,8 @@
     }
   },
   "dependencies": {
-    "@spooky-sync/query-builder": "0.0.1-canary.176",
-    "@spooky-sync/ssp-wasm": "0.0.1-canary.176",
+    "@spooky-sync/query-builder": "0.0.1-canary.178",
+    "@spooky-sync/ssp-wasm": "0.0.1-canary.178",
     "@sqlite.org/sqlite-wasm": "3.53.0-build1",
     "@surrealdb/wasm": "^3.0.3",
     "fast-json-patch": "^3.1.1",

package/src/modules/devtools/versions.ts

@@ -33,6 +33,9 @@ export interface HeartbeatSample {
 export interface HeartbeatInfo {
   enabled: boolean;
   stale: boolean;
+  /** Nothing to measure right now (e.g. no ready SSPs mid-bootstrap). */
+  blocked?: boolean;
+  blocked_reason?: string | null;
   last_e2e_ms: number | null;
   last_ok_epoch_ms: number | null;
   consecutive_failures: number;

package/src/services/database/sqlite-cache-engine.test.ts

@@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest';
 import { RecordId } from 'surrealdb';
 import { pureWriteOpResult, SqliteCacheEngine } from './sqlite-cache-engine';
 import { stubTransport } from './sqlite-transport.fixture';
+import { BrokerPortClosedError } from './sqlite-transport';
 import { translateSurql } from './surql-translate';
 import type { SqlOp } from './surql-translate';
 import { surql } from '../../utils/surql';
@@ -330,6 +331,99 @@ describe('SqliteCacheEngine role modes', () => {
     expect(engine.storageHealth.role).toBe('leader');
   });
 
+  /**
+   * A transport with the REAL pending-map semantics (the shared fixture never
+   * rejects on close, which would make the drain test vacuous): `exec` parks
+   * until `flush()`, and `close` rejects whatever is still parked.
+   */
+  function deferredTransport() {
+    const parked: Array<{ resolve: (v: any) => void; reject: (e: unknown) => void }> = [];
+    let closed = false;
+    const transport: any = {
+      kind: 'worker',
+      get connected() {
+        return !closed;
+      },
+      call(type: string) {
+        if (closed) return Promise.reject(new Error('SQLite worker crashed: transport closed'));
+        if (type === 'open') return Promise.resolve({ persisted: true });
+        if (type !== 'exec') return Promise.resolve({});
+        return new Promise((resolve, reject) => parked.push({ resolve, reject }));
+      },
+      shutdown: () => Promise.resolve(),
+      failAll() {},
+      close(reason = 'closed', err?: Error) {
+        if (closed) return;
+        closed = true;
+        const e = err ?? new Error(`SQLite worker crashed: ${reason}`);
+        for (const p of parked.splice(0)) p.reject(e);
+      },
+      flush() {
+        for (const p of parked.splice(0)) p.resolve({ rows: [] });
+      },
+      get parkedCount() {
+        return parked.length;
+      },
+    };
+    return transport;
+  }
+
+  // Regression (WhitePawn first-login): the bucket switch runs
+  // moveToBucket → teardownLeader → releaseOwnership, which lives on the
+  // transition chain, NOT the opQueue — so a query's fetch can be sitting at
+  // the worker when ownership is released. Tearing the transport down under it
+  // rejected that fetch with "SQLite worker crashed: ownership released", which
+  // nothing retries (and nobody caught: an unhandled rejection in the console).
+  it('releaseOwnership waits for in-flight ops instead of killing them', async () => {
+    const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger(), {
+      shared: true,
+    });
+    const transport = deferredTransport();
+    (engine as any).createTransport = () => transport;
+    await engine.adoptOwner('anon', {
+      workerLockName: 'sp00ky-tabs:fp:anon:worker:1',
+      allowMemoryFallback: false,
+      resumeHeld: false,
+    });
+
+    const read = engine.getById('_00_query', 'h1');
+    await new Promise((r) => setTimeout(r, 0));
+    expect(transport.parkedCount).toBe(1);
+
+    const released = engine.releaseOwnership();
+    await new Promise((r) => setTimeout(r, 0));
+    // Still parked: the teardown is waiting on it rather than failing it.
+    expect(transport.parkedCount).toBe(1);
+    transport.flush();
+
+    await released;
+    await expect(read).resolves.toBeNull();
+  });
+
+  it('fails an undrainable op as a retryable transport loss, not a crash', async () => {
+    const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger(), {
+      shared: true,
+    });
+    const transport = deferredTransport();
+    (engine as any).createTransport = () => transport;
+    await engine.adoptOwner('anon', {
+      workerLockName: 'sp00ky-tabs:fp:anon:worker:1',
+      allowMemoryFallback: false,
+      resumeHeld: false,
+    });
+
+    const read = engine.getById('_00_query', 'h1');
+    const caught = read.catch((e: unknown) => e);
+    await new Promise((r) => setTimeout(r, 0));
+    // The op never answers: the drain times out and the teardown proceeds.
+    (engine as any).drainInFlight = () => Promise.resolve();
+    await engine.releaseOwnership();
+
+    const err = await caught;
+    expect(err).toBeInstanceOf(BrokerPortClosedError);
+    expect((err as Error).message).toContain('ownership released');
+  });
+
   it('bumps the epoch on promotion after having had a store (fences in-flight chains)', async () => {
     const { engine } = makeSharedEngine();
     await engine.adoptOwner('anon', {

package/src/services/database/sqlite-cache-engine.ts

@@ -262,6 +262,33 @@ export class SqliteCacheEngine implements LocalStore {
     return result;
   }
 
+  /**
+   * Ops dispatched to the worker and not yet answered. Role transitions run on
+   * their own chain (see {@link transitionChain}), so they can start while the
+   * opQueue still has an op at the worker; tearing the transport down under it
+   * would reject that op — and its caller may be a query whose only fetch this
+   * was. {@link drainInFlight} lets a deliberate teardown wait them out.
+   */
+  private inFlightCalls = new Set<Promise<unknown>>();
+
+  /** Wait for dispatched ops to answer before a deliberate transport teardown.
+   *  Bounded: a wedged worker must not block the role change forever. */
+  private async drainInFlight(timeoutMs = 2_000): Promise<void> {
+    if (this.inFlightCalls.size === 0) return;
+    const settled = Promise.all([...this.inFlightCalls].map((p) => p.catch(() => undefined)));
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    try {
+      await Promise.race([
+        settled,
+        new Promise<void>((resolve) => {
+          timer = setTimeout(resolve, timeoutMs);
+        }),
+      ]);
+    } finally {
+      if (timer) clearTimeout(timer);
+    }
+  }
+
   private rawCall<T = any>(type: string, payload?: unknown): Promise<T> {
     if (!this.transport) throw new Error('SqliteCacheEngine: not connected');
     // --- instrumentation: live, inspectable via `globalThis.__sqliteStats` ---
@@ -276,8 +303,21 @@ export class SqliteCacheEngine implements LocalStore {
     s.maxInFlight = Math.max(s.maxInFlight, s.inFlight);
     if (this.transport.kind === 'port') s.proxiedOps = (s.proxiedOps ?? 0) + 1;
     const sentAt = performance.now();
-    return this.transport.call<T>(type, payload).then(
+    // Track the dispatch, not the returned promise: attaching a handler to
+    // `result` would mark it handled and swallow the unhandled-rejection
+    // reports that surface a caller which forgot to catch.
+    let settle!: () => void;
+    const tracked = new Promise<void>((res) => {
+      settle = res;
+    });
+    this.inFlightCalls.add(tracked);
+    const finish = () => {
+      this.inFlightCalls.delete(tracked);
+      settle();
+    };
+    const result = this.transport.call<T>(type, payload).then(
       (v: T) => {
+        finish();
         s.inFlight--;
         // Split the round-trip: `wt` is time inside the worker's handler,
         // the remainder is postMessage + scheduling overhead.
@@ -289,10 +329,12 @@ export class SqliteCacheEngine implements LocalStore {
         return v;
       },
       (e: unknown) => {
+        finish();
         s.inFlight--;
         throw e;
       }
     );
+    return result;
   }
 
   /**
@@ -514,6 +556,7 @@ export class SqliteCacheEngine implements LocalStore {
         this.closeRoleGate();
         return this.storageHealthValue;
       }
+      if (this.transport) await this.drainInFlight();
       if (this.hadStore) this.storeEpoch++;
       if (this.transport) {
         try {
@@ -521,7 +564,7 @@ export class SqliteCacheEngine implements LocalStore {
         } catch {
           /* ignore */
         }
-        this.transport.close('adopting ownership');
+        this.transport.close('adopting ownership', new BrokerPortClosedError('adopting ownership'));
         this.transport = null;
       }
       await this.openInternal(bucketId, {
@@ -555,8 +598,12 @@ export class SqliteCacheEngine implements LocalStore {
   ): Promise<void> {
     this.roleLabel = 'follower';
     return this.chainTransition(async () => {
+      if (this.transport) await this.drainInFlight();
       if (this.hadStore) this.storeEpoch++;
-      this.transport?.close('adopting leader port');
+      this.transport?.close(
+        'adopting leader port',
+        new BrokerPortClosedError('adopting leader port')
+      );
       this.transport = new PortSqliteTransport(dbPort, onPortDead, this.logger);
       this.bucketId = snapshot.bucketId;
       // The shared store exists and is seeded; mirror the owner's bookkeeping.
@@ -578,6 +625,11 @@ export class SqliteCacheEngine implements LocalStore {
   async releaseOwnership(): Promise<void> {
     await this.chainTransition(async () => {
       if (this.transport) {
+        // Let ops already at the worker answer first. Without this they die
+        // with the transport — a bucket switch (moveToBucket → teardownLeader)
+        // runs while the opQueue may still have a query's fetch in flight, and
+        // that fetch's caller is not necessarily prepared to retry.
+        await this.drainInFlight();
         try {
           if (this.transport.kind === 'worker') {
             await (this.transport as WorkerSqliteTransport).shutdown();
@@ -585,7 +637,13 @@ export class SqliteCacheEngine implements LocalStore {
         } catch {
           /* worker may already be fenced/dead */
         }
-        this.transport.close('ownership released');
+        // Anything still pending (drain timed out) is a deliberate teardown,
+        // not a crash: fail it the way a follower's lost port does, so the
+        // error text is honest and callers can treat it as retryable.
+        this.transport.close(
+          'ownership released',
+          new BrokerPortClosedError('ownership released')
+        );
         this.transport = null;
       }
       this.storeEpoch++;

package/src/services/database/sqlite-transport.ts

@@ -22,8 +22,10 @@ export interface SqliteTransport {
   call<T = unknown>(type: string, payload?: unknown): Promise<T>;
   /** Reject every pending request with `reason`. Safe to call repeatedly. */
   failAll(reason: string): void;
-  /** failAll + release the underlying channel. Terminal. */
-  close(reason?: string): void;
+  /** failAll + release the underlying channel. Terminal. `err` overrides the
+   *  transport's own error shape — used for a deliberate teardown (role change)
+   *  so callers see a retryable "transport lost", not "the worker crashed". */
+  close(reason?: string, err?: Error): void;
 }
 
 /** Thrown into pending follower calls when the leader (or its port) goes away.
@@ -81,17 +83,17 @@ abstract class BaseTransport implements SqliteTransport {
     });
   }
 
-  failAll(reason: string): void {
+  failAll(reason: string, err?: Error): void {
     if (this.pending.size === 0) return;
-    const err = this.makeError(reason);
-    for (const [, p] of this.pending) p.reject(err);
+    const e = err ?? this.makeError(reason);
+    for (const [, p] of this.pending) p.reject(e);
     this.pending.clear();
   }
 
-  close(reason = 'closed'): void {
+  close(reason = 'closed', err?: Error): void {
     if (this.closed) return;
     this.closed = true;
-    this.failAll(reason);
+    this.failAll(reason, err);
   }
 }
 
@@ -166,9 +168,9 @@ export class WorkerSqliteTransport extends BaseTransport {
     return this.call('shutdown').then(() => undefined);
   }
 
-  close(reason = 'closed'): void {
+  close(reason = 'closed', err?: Error): void {
     if (this.closed) return;
-    super.close(reason);
+    super.close(reason, err);
     this.worker.terminate();
   }
 }
@@ -193,10 +195,10 @@ export class PortSqliteTransport extends BaseTransport {
     this.dead(reason);
   }
 
-  private dead(reason: string): void {
+  private dead(reason: string, err?: Error): void {
     if (this.closed) return;
     this.closed = true;
-    this.failAll(reason);
+    this.failAll(reason, err);
     try {
       this.port.close();
     } catch {
@@ -213,7 +215,7 @@ export class PortSqliteTransport extends BaseTransport {
     return new BrokerPortClosedError(reason);
   }
 
-  close(reason = 'closed'): void {
-    this.dead(reason);
+  close(reason = 'closed', err?: Error): void {
+    this.dead(reason, err);
   }
 }