@spooky-sync/core 0.0.1-canary.177 → 0.0.1-canary.179

package/dist/index.js

@@ -1998,16 +1998,16 @@ var BaseTransport = class {
 			}
 		});
 	}
-	failAll(reason) {
+	failAll(reason, err) {
 		if (this.pending.size === 0) return;
-		const err = this.makeError(reason);
-		for (const [, p] of this.pending) p.reject(err);
+		const e = err ?? this.makeError(reason);
+		for (const [, p] of this.pending) p.reject(e);
 		this.pending.clear();
 	}
-	close(reason = "closed") {
+	close(reason = "closed", err) {
 		if (this.closed) return;
 		this.closed = true;
-		this.failAll(reason);
+		this.failAll(reason, err);
 	}
 };
 var WorkerSqliteTransport = class extends BaseTransport {
@@ -2071,9 +2071,9 @@ var WorkerSqliteTransport = class extends BaseTransport {
 	shutdown() {
 		return this.call("shutdown").then(() => void 0);
 	}
-	close(reason = "closed") {
+	close(reason = "closed", err) {
 		if (this.closed) return;
-		super.close(reason);
+		super.close(reason, err);
 		this.worker.terminate();
 	}
 };
@@ -2092,10 +2092,10 @@ var PortSqliteTransport = class extends BaseTransport {
 	markDead(reason) {
 		this.dead(reason);
 	}
-	dead(reason) {
+	dead(reason, err) {
 		if (this.closed) return;
 		this.closed = true;
-		this.failAll(reason);
+		this.failAll(reason, err);
 		try {
 			this.port.close();
 		} catch {}
@@ -2107,8 +2107,8 @@ var PortSqliteTransport = class extends BaseTransport {
 	makeError(reason) {
 		return new BrokerPortClosedError(reason);
 	}
-	close(reason = "closed") {
-		this.dead(reason);
+	close(reason = "closed", err) {
+		this.dead(reason, err);
 	}
 };
 
@@ -2304,6 +2304,28 @@ var SqliteCacheEngine = class {
 		this.opQueue = result.then(() => void 0, () => void 0);
 		return result;
 	}
+	/**
+	* Ops dispatched to the worker and not yet answered. Role transitions run on
+	* their own chain (see {@link transitionChain}), so they can start while the
+	* opQueue still has an op at the worker; tearing the transport down under it
+	* would reject that op — and its caller may be a query whose only fetch this
+	* was. {@link drainInFlight} lets a deliberate teardown wait them out.
+	*/
+	inFlightCalls = /* @__PURE__ */ new Set();
+	/** Wait for dispatched ops to answer before a deliberate transport teardown.
+	*  Bounded: a wedged worker must not block the role change forever. */
+	async drainInFlight(timeoutMs = 2e3) {
+		if (this.inFlightCalls.size === 0) return;
+		const settled = Promise.all([...this.inFlightCalls].map((p) => p.catch(() => void 0)));
+		let timer;
+		try {
+			await Promise.race([settled, new Promise((resolve) => {
+				timer = setTimeout(resolve, timeoutMs);
+			})]);
+		} finally {
+			if (timer) clearTimeout(timer);
+		}
+	}
 	rawCall(type, payload) {
 		if (!this.transport) throw new Error("SqliteCacheEngine: not connected");
 		const s = getStats();
@@ -2317,7 +2339,17 @@ var SqliteCacheEngine = class {
 		s.maxInFlight = Math.max(s.maxInFlight, s.inFlight);
 		if (this.transport.kind === "port") s.proxiedOps = (s.proxiedOps ?? 0) + 1;
 		const sentAt = performance.now();
+		let settle;
+		const tracked = new Promise((res) => {
+			settle = res;
+		});
+		this.inFlightCalls.add(tracked);
+		const finish = () => {
+			this.inFlightCalls.delete(tracked);
+			settle();
+		};
 		return this.transport.call(type, payload).then((v) => {
+			finish();
 			s.inFlight--;
 			const wt = v?.wt;
 			if (typeof wt === "number") {
@@ -2326,6 +2358,7 @@ var SqliteCacheEngine = class {
 			}
 			return v;
 		}, (e) => {
+			finish();
 			s.inFlight--;
 			throw e;
 		});
@@ -2471,12 +2504,13 @@ var SqliteCacheEngine = class {
 				this.closeRoleGate();
 				return this.storageHealthValue;
 			}
+			if (this.transport) await this.drainInFlight();
 			if (this.hadStore) this.storeEpoch++;
 			if (this.transport) {
 				try {
 					if (this.transport.kind === "worker") await this.rawCall("close");
 				} catch {}
-				this.transport.close("adopting ownership");
+				this.transport.close("adopting ownership", new BrokerPortClosedError("adopting ownership"));
 				this.transport = null;
 			}
 			await this.openInternal(bucketId, {
@@ -2499,8 +2533,9 @@ var SqliteCacheEngine = class {
 	async adoptAttached(dbPort, snapshot, onPortDead) {
 		this.roleLabel = "follower";
 		return this.chainTransition(async () => {
+			if (this.transport) await this.drainInFlight();
 			if (this.hadStore) this.storeEpoch++;
-			this.transport?.close("adopting leader port");
+			this.transport?.close("adopting leader port", new BrokerPortClosedError("adopting leader port"));
 			this.transport = new PortSqliteTransport(dbPort, onPortDead, this.logger);
 			this.bucketId = snapshot.bucketId;
 			this.knownTables.clear();
@@ -2523,10 +2558,11 @@ var SqliteCacheEngine = class {
 	async releaseOwnership() {
 		await this.chainTransition(async () => {
 			if (this.transport) {
+				await this.drainInFlight();
 				try {
 					if (this.transport.kind === "worker") await this.transport.shutdown();
 				} catch {}
-				this.transport.close("ownership released");
+				this.transport.close("ownership released", new BrokerPortClosedError("ownership released"));
 				this.transport = null;
 			}
 			this.storeEpoch++;
@@ -4938,10 +4974,20 @@ function rowToUpEvent(r, logger) {
 
 //#endregion
 //#region src/modules/sync/queue/queue-down.ts
+/**
+* How many times a failing event keeps its place at the head before it is
+* rotated to the back. A transient failure (the SSP still bootstrapping) clears
+* well inside this, so ordering is preserved for the common case; a permanently
+* rejected event (a permission the SSP refuses to lower) stops holding every
+* other query hostage behind it.
+*/
+const MAX_HEAD_RETRIES = 3;
 var DownQueue = class {
 	queue = [];
 	_events;
 	logger;
+	/** Consecutive failures per queued event; cleared when it finally succeeds. */
+	failures = /* @__PURE__ */ new WeakMap();
 	get events() {
 		return this._events;
 	}
@@ -4972,13 +5018,20 @@ var DownQueue = class {
 		const event = this.queue.shift();
 		if (event) try {
 			await fn(event);
+			this.failures.delete(event);
 		} catch (error) {
+			const attempts = (this.failures.get(event) ?? 0) + 1;
+			this.failures.set(event, attempts);
+			const starvingOthers = attempts >= MAX_HEAD_RETRIES && this.queue.length > 0;
+			if (starvingOthers) this.queue.push(event);
+			else this.queue.unshift(event);
 			this.logger.error({
 				error,
 				event,
+				attempts,
+				rotated: starvingOthers,
 				Category: "sp00ky-client::DownQueue::next"
 			}, "Failed to process query");
-			this.queue.unshift(event);
 			throw error;
 		}
 	}
@@ -5342,11 +5395,18 @@ var SyncEngine = class {
 * SyncScheduler manages when to sync: queue management and orchestration.
 * Decides the order and timing of sync operations.
 */
+/** Backoff for re-draining a queue that halted on an error. */
+const RETRY_BASE_MS = 500;
+const RETRY_MAX_MS = 15e3;
 var SyncScheduler = class {
 	isSyncingUp = false;
 	isSyncingDown = false;
 	paused = false;
 	pauseWaiters = [];
+	upRetryTimer;
+	downRetryTimer;
+	upRetryAttempt = 0;
+	downRetryAttempt = 0;
 	constructor(upQueue, downQueue, onProcessUp, onProcessDown, logger, onRollback, onSyncOutcome) {
 		this.upQueue = upQueue;
 		this.downQueue = downQueue;
@@ -5382,12 +5442,16 @@ var SyncScheduler = class {
 	*/
 	pause() {
 		this.paused = true;
+		this.clearRetryTimers();
 		if (!this.isSyncingUp && !this.isSyncingDown) return Promise.resolve();
 		return new Promise((resolve) => this.pauseWaiters.push(resolve));
 	}
 	resume() {
 		this.paused = false;
+		this.upRetryAttempt = 0;
+		this.downRetryAttempt = 0;
 		this.syncUp();
+		this.syncDown();
 	}
 	maybeResolvePause() {
 		if (!this.paused || this.isSyncingUp || this.isSyncingDown) return;
@@ -5395,6 +5459,46 @@ var SyncScheduler = class {
 		this.pauseWaiters = [];
 		for (const resolve of waiters) resolve();
 	}
+	/** Exponential backoff, capped. Attempt 0 is the first retry. */
+	retryDelay(attempt) {
+		return Math.min(RETRY_BASE_MS * 2 ** attempt, RETRY_MAX_MS);
+	}
+	scheduleUpRetry() {
+		if (this.paused || this.upRetryTimer || this.upQueue.size === 0) return;
+		const delay = this.retryDelay(this.upRetryAttempt++);
+		this.upRetryTimer = setTimeout(() => {
+			this.upRetryTimer = void 0;
+			this.syncUp();
+		}, delay);
+	}
+	/**
+	* Re-arm the down pass. With no argument this is failure backoff and the
+	* streak grows; with an explicit delay it is a yield (the up-queue holds the
+	* floor), which is not a failure and must not push the backoff out.
+	*/
+	scheduleDownRetry(delayMs) {
+		if (this.paused || this.downRetryTimer || this.downQueue.size === 0) return;
+		const delay = delayMs ?? this.retryDelay(this.downRetryAttempt++);
+		this.downRetryTimer = setTimeout(() => {
+			this.downRetryTimer = void 0;
+			this.syncDown();
+		}, delay);
+	}
+	clearRetryTimers() {
+		if (this.upRetryTimer) {
+			clearTimeout(this.upRetryTimer);
+			this.upRetryTimer = void 0;
+		}
+		if (this.downRetryTimer) {
+			clearTimeout(this.downRetryTimer);
+			this.downRetryTimer = void 0;
+		}
+	}
+	/** Stop all pending retries. Call when tearing the client down. */
+	dispose() {
+		this.paused = true;
+		this.clearRetryTimers();
+	}
 	/**
 	* Process upload queue
 	*/
@@ -5408,8 +5512,10 @@ var SyncScheduler = class {
 				processedAny = true;
 			}
 			if (processedAny) this.onSyncOutcome?.(true);
+			this.upRetryAttempt = 0;
 		} catch (error) {
 			this.onSyncOutcome?.(false, error);
+			this.scheduleUpRetry();
 			this.logger.debug({
 				error,
 				Category: "sp00ky-client::SyncScheduler::syncUp"
@@ -5425,7 +5531,10 @@ var SyncScheduler = class {
 	*/
 	async syncDown() {
 		if (this.isSyncingDown || this.paused) return;
-		if (this.upQueue.size > 0) return;
+		if (this.upQueue.size > 0) {
+			this.scheduleDownRetry(RETRY_BASE_MS);
+			return;
+		}
 		this.isSyncingDown = true;
 		let processedAny = false;
 		try {
@@ -5435,8 +5544,10 @@ var SyncScheduler = class {
 				processedAny = true;
 			}
 			if (processedAny) this.onSyncOutcome?.(true);
+			this.downRetryAttempt = 0;
 		} catch (error) {
 			this.onSyncOutcome?.(false, error);
+			this.scheduleDownRetry();
 			this.logger.debug({
 				error,
 				Category: "sp00ky-client::SyncScheduler::syncDown"
@@ -6892,8 +7003,8 @@ function selfAllowlistedVariant(flag, userId) {
 
 //#endregion
 //#region src/modules/devtools/index.ts
-const CORE_VERSION = "0.0.1-canary.177";
-const WASM_VERSION = "0.0.1-canary.177";
+const CORE_VERSION = "0.0.1-canary.179";
+const WASM_VERSION = "0.0.1-canary.179";
 const SURREAL_VERSION = "3.0.3";
 var DevToolsService = class DevToolsService {
 	eventsHistory = [];
@@ -11324,7 +11435,7 @@ var Sp00kyClient = class {
 		return new TabsCoordinator({
 			tabId,
 			fingerprint: computeTabsFingerprint({
-				coreVersion: "0.0.1-canary.177",
+				coreVersion: "0.0.1-canary.179",
 				schemaHash: hash53(this.config.schemaSurql),
 				endpoint: this.config.database.endpoint ?? "",
 				namespace: this.config.database.namespace,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/core",
-  "version": "0.0.1-canary.177",
+  "version": "0.0.1-canary.179",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -60,8 +60,8 @@
     }
   },
   "dependencies": {
-    "@spooky-sync/query-builder": "0.0.1-canary.177",
-    "@spooky-sync/ssp-wasm": "0.0.1-canary.177",
+    "@spooky-sync/query-builder": "0.0.1-canary.179",
+    "@spooky-sync/ssp-wasm": "0.0.1-canary.179",
     "@sqlite.org/sqlite-wasm": "3.53.0-build1",
     "@surrealdb/wasm": "^3.0.3",
     "fast-json-patch": "^3.1.1",

package/src/modules/sync/queue/queue-down.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect } from 'vitest';
+import { DownQueue } from './queue-down';
+import type { DownEvent } from './queue-down';
+import type { LocalStore } from '../../../services/database/index';
+
+const silentLogger = {
+  child: () => silentLogger,
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any;
+
+const register = (hash: string) => ({ type: 'register', payload: { hash } }) as DownEvent;
+
+const hashOf = (e: DownEvent) => e.payload.hash;
+
+function makeQueue() {
+  return new DownQueue({} as LocalStore, silentLogger);
+}
+
+describe('DownQueue.next failure handling', () => {
+  it('re-heads a failing event so a transient failure keeps its order', async () => {
+    const q = makeQueue();
+    q.push(register('a'));
+    q.push(register('b'));
+
+    const seen: string[] = [];
+    await expect(
+      q.next(async (e) => {
+        seen.push(hashOf(e));
+        throw new Error('503');
+      })
+    ).rejects.toThrow('503');
+
+    expect(seen).toEqual(['a']);
+    // 'a' is back at the head, ahead of 'b'.
+    await expect(q.next(async (e) => { seen.push(hashOf(e)); throw new Error('503'); })).rejects.toThrow();
+    expect(seen).toEqual(['a', 'a']);
+  });
+
+  it('rotates a persistently failing event to the back so others can proceed', async () => {
+    const q = makeQueue();
+    q.push(register('poison'));
+    q.push(register('healthy'));
+
+    const seen: string[] = [];
+    const failPoison = async (e: DownEvent) => {
+      seen.push(hashOf(e));
+      if (hashOf(e) === 'poison') throw new Error('400 rejected');
+    };
+
+    // Three attempts keep the head; the third rotates it behind 'healthy'.
+    for (let i = 0; i < 3; i++) {
+      await expect(q.next(failPoison)).rejects.toThrow();
+    }
+    expect(seen).toEqual(['poison', 'poison', 'poison']);
+
+    // 'healthy' is no longer starved.
+    await q.next(failPoison);
+    expect(seen).toEqual(['poison', 'poison', 'poison', 'healthy']);
+    expect(q.size).toBe(1); // only the poison event remains
+  });
+
+  it('does not rotate when nothing else is waiting', async () => {
+    const q = makeQueue();
+    q.push(register('only'));
+
+    for (let i = 0; i < 5; i++) {
+      await expect(
+        q.next(async () => {
+          throw new Error('boom');
+        })
+      ).rejects.toThrow();
+    }
+    expect(q.size).toBe(1);
+  });
+
+  it('clears the failure count once an event succeeds', async () => {
+    const q = makeQueue();
+    const event = register('a');
+    q.push(event);
+    q.push(register('b'));
+
+    await expect(
+      q.next(async () => {
+        throw new Error('boom');
+      })
+    ).rejects.toThrow();
+    // Succeeds on the retry, so its streak resets rather than carrying over.
+    await q.next(async () => {});
+    expect(q.size).toBe(1);
+
+    q.push(event);
+    // A fresh streak: it keeps the head again rather than rotating immediately.
+    await expect(
+      q.next(async () => {
+        throw new Error('boom');
+      })
+    ).rejects.toThrow();
+    const drained: string[] = [];
+    await q.next(async (e) => {
+      drained.push(hashOf(e));
+    });
+    expect(drained).toEqual(['b']);
+  });
+});

package/src/modules/sync/queue/queue-down.ts

@@ -37,10 +37,21 @@ export type CleanupEvent = {
 
 export type DownEvent = RegisterEvent | SyncEvent | HeartbeatEvent | CleanupEvent;
 
+/**
+ * How many times a failing event keeps its place at the head before it is
+ * rotated to the back. A transient failure (the SSP still bootstrapping) clears
+ * well inside this, so ordering is preserved for the common case; a permanently
+ * rejected event (a permission the SSP refuses to lower) stops holding every
+ * other query hostage behind it.
+ */
+const MAX_HEAD_RETRIES = 3;
+
 export class DownQueue {
   private queue: DownEvent[] = [];
   private _events: SyncQueueEventSystem;
   private logger: Logger;
+  /** Consecutive failures per queued event; cleared when it finally succeeds. */
+  private failures = new WeakMap<DownEvent, number>();
 
   get events(): SyncQueueEventSystem {
     return this._events;
@@ -83,12 +94,23 @@ export class DownQueue {
     if (event) {
       try {
         await fn(event);
+        this.failures.delete(event);
       } catch (error) {
+        const attempts = (this.failures.get(event) ?? 0) + 1;
+        this.failures.set(event, attempts);
+        // Re-head so a transient failure keeps its ordering, but give up the
+        // head once it looks permanent and there is other work waiting — one
+        // unregisterable query must not stall every other query's registration.
+        const starvingOthers = attempts >= MAX_HEAD_RETRIES && this.queue.length > 0;
+        if (starvingOthers) {
+          this.queue.push(event);
+        } else {
+          this.queue.unshift(event);
+        }
         this.logger.error(
-          { error, event, Category: 'sp00ky-client::DownQueue::next' },
+          { error, event, attempts, rotated: starvingOthers, Category: 'sp00ky-client::DownQueue::next' },
           'Failed to process query'
         );
-        this.queue.unshift(event);
         throw error;
       }
     }

package/src/modules/sync/scheduler.retry.test.ts

@@ -0,0 +1,156 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { SyncScheduler } from './scheduler';
+import type { UpQueue, DownQueue, DownEvent, UpEvent } from './queue/index';
+
+// A queue whose drain throws re-queues the failing item at the HEAD and stops
+// the pass (see DownQueue.next). Nothing used to re-arm it: the queues only
+// moved on a fresh enqueue, so one transient failure — canonically the SSP
+// answering 503 NOT_READY for the whole of its bootstrap window — parked every
+// pending `register` forever, and the `useQuery` waiting on it never left its
+// loading state. These cover the backoff that makes that self-heal.
+
+const silentLogger = {
+  child: () => silentLogger,
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any;
+
+/** A queue that mirrors the real ones: a throwing handler re-heads the item. */
+function makeQueue<E>(items: E[]) {
+  return {
+    queue: [...items],
+    get size() {
+      return this.queue.length;
+    },
+    events: { subscribe: () => {} },
+    loadFromDatabase: async () => {},
+    clear() {
+      this.queue = [];
+    },
+    async next(fn: (event: E) => Promise<void>) {
+      const event = this.queue.shift();
+      if (!event) return;
+      try {
+        await fn(event);
+      } catch (err) {
+        this.queue.unshift(event);
+        throw err;
+      }
+    },
+  };
+}
+
+const downEvent = (hash: string) => ({ type: 'register', payload: { hash } }) as DownEvent;
+const upEvent = (n: number) => ({ type: 'delete', mutation_id: n, record_id: n }) as unknown as UpEvent;
+
+describe('SyncScheduler retry', () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  it('re-drains a failed down event instead of parking it forever', async () => {
+    const downQueue = makeQueue([downEvent('q1')]);
+    const upQueue = makeQueue<UpEvent>([]);
+    let attempts = 0;
+    const scheduler = new SyncScheduler(
+      upQueue as unknown as UpQueue,
+      downQueue as unknown as DownQueue,
+      async () => {},
+      async () => {
+        attempts++;
+        // Fail the way a bootstrapping SSP does, then succeed.
+        if (attempts < 3) throw new Error('503 NOT_READY');
+      },
+      silentLogger
+    );
+
+    await scheduler.syncDown();
+    expect(attempts).toBe(1);
+    expect(downQueue.size).toBe(1); // re-headed, not dropped
+
+    // Backoff: 500ms, then 1000ms.
+    await vi.advanceTimersByTimeAsync(500);
+    expect(attempts).toBe(2);
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(attempts).toBe(3);
+
+    expect(downQueue.size).toBe(0);
+  });
+
+  it('stops retrying once the queue drains', async () => {
+    const downQueue = makeQueue([downEvent('q1')]);
+    const upQueue = makeQueue<UpEvent>([]);
+    let attempts = 0;
+    const scheduler = new SyncScheduler(
+      upQueue as unknown as UpQueue,
+      downQueue as unknown as DownQueue,
+      async () => {},
+      async () => {
+        attempts++;
+        throw new Error('boom');
+      },
+      silentLogger
+    );
+
+    await scheduler.syncDown();
+    await vi.advanceTimersByTimeAsync(500);
+    expect(attempts).toBe(2);
+
+    // Drain it out from under the scheduler; the next retry finds nothing and
+    // schedules no further work.
+    downQueue.clear();
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(attempts).toBe(2);
+    await vi.advanceTimersByTimeAsync(60_000);
+    expect(attempts).toBe(2);
+  });
+
+  it('comes back for the down queue while the up queue holds the floor', async () => {
+    const downQueue = makeQueue([downEvent('q1')]);
+    const upQueue = makeQueue<UpEvent>([upEvent(1)]);
+    let downAttempts = 0;
+    const scheduler = new SyncScheduler(
+      upQueue as unknown as UpQueue,
+      downQueue as unknown as DownQueue,
+      async () => {},
+      async () => {
+        downAttempts++;
+      },
+      silentLogger
+    );
+
+    // Yields to the non-empty up queue.
+    await scheduler.syncDown();
+    expect(downAttempts).toBe(0);
+
+    // Once the up queue empties, the re-armed pass picks the down event up
+    // without needing a fresh enqueue.
+    upQueue.clear();
+    await vi.advanceTimersByTimeAsync(500);
+    expect(downAttempts).toBe(1);
+  });
+
+  it('pause cancels pending retries', async () => {
+    const downQueue = makeQueue([downEvent('q1')]);
+    const upQueue = makeQueue<UpEvent>([]);
+    let attempts = 0;
+    const scheduler = new SyncScheduler(
+      upQueue as unknown as UpQueue,
+      downQueue as unknown as DownQueue,
+      async () => {},
+      async () => {
+        attempts++;
+        throw new Error('boom');
+      },
+      silentLogger
+    );
+
+    await scheduler.syncDown();
+    expect(attempts).toBe(1);
+
+    await scheduler.pause();
+    await vi.advanceTimersByTimeAsync(60_000);
+    expect(attempts).toBe(1);
+  });
+});

package/src/modules/sync/scheduler.ts

@@ -6,11 +6,25 @@ import { SyncQueueEventTypes } from './events/index';
  * SyncScheduler manages when to sync: queue management and orchestration.
  * Decides the order and timing of sync operations.
  */
+/** Backoff for re-draining a queue that halted on an error. */
+const RETRY_BASE_MS = 500;
+const RETRY_MAX_MS = 15_000;
+
 export class SyncScheduler {
   private isSyncingUp: boolean = false;
   private isSyncingDown: boolean = false;
   private paused: boolean = false;
   private pauseWaiters: Array<() => void> = [];
+  // A failed drain re-queues its item at the HEAD (see DownQueue.next) and
+  // stops the pass. Without a timer nothing ever drains it again: the queues
+  // only move on a fresh enqueue, so a transient failure — canonically the
+  // SSP answering 503 NOT_READY for the whole of its bootstrap window — left
+  // every pending `register` parked forever and its `useQuery` loading forever.
+  // Retry on a backoff so that heals itself instead of needing a reload.
+  private upRetryTimer?: ReturnType<typeof setTimeout>;
+  private downRetryTimer?: ReturnType<typeof setTimeout>;
+  private upRetryAttempt = 0;
+  private downRetryAttempt = 0;
 
   constructor(
     private upQueue: UpQueue,
@@ -62,13 +76,18 @@ export class SyncScheduler {
    */
   pause(): Promise<void> {
     this.paused = true;
+    this.clearRetryTimers();
     if (!this.isSyncingUp && !this.isSyncingDown) return Promise.resolve();
     return new Promise<void>((resolve) => this.pauseWaiters.push(resolve));
   }
 
   resume(): void {
     this.paused = false;
+    // A resume is a fresh start, not a continuation of the failing streak.
+    this.upRetryAttempt = 0;
+    this.downRetryAttempt = 0;
     void this.syncUp();
+    void this.syncDown();
   }
 
   private maybeResolvePause() {
@@ -78,6 +97,51 @@ export class SyncScheduler {
     for (const resolve of waiters) resolve();
   }
 
+  /** Exponential backoff, capped. Attempt 0 is the first retry. */
+  private retryDelay(attempt: number): number {
+    return Math.min(RETRY_BASE_MS * 2 ** attempt, RETRY_MAX_MS);
+  }
+
+  private scheduleUpRetry() {
+    if (this.paused || this.upRetryTimer || this.upQueue.size === 0) return;
+    const delay = this.retryDelay(this.upRetryAttempt++);
+    this.upRetryTimer = setTimeout(() => {
+      this.upRetryTimer = undefined;
+      void this.syncUp();
+    }, delay);
+  }
+
+  /**
+   * Re-arm the down pass. With no argument this is failure backoff and the
+   * streak grows; with an explicit delay it is a yield (the up-queue holds the
+   * floor), which is not a failure and must not push the backoff out.
+   */
+  private scheduleDownRetry(delayMs?: number) {
+    if (this.paused || this.downRetryTimer || this.downQueue.size === 0) return;
+    const delay = delayMs ?? this.retryDelay(this.downRetryAttempt++);
+    this.downRetryTimer = setTimeout(() => {
+      this.downRetryTimer = undefined;
+      void this.syncDown();
+    }, delay);
+  }
+
+  private clearRetryTimers() {
+    if (this.upRetryTimer) {
+      clearTimeout(this.upRetryTimer);
+      this.upRetryTimer = undefined;
+    }
+    if (this.downRetryTimer) {
+      clearTimeout(this.downRetryTimer);
+      this.downRetryTimer = undefined;
+    }
+  }
+
+  /** Stop all pending retries. Call when tearing the client down. */
+  dispose(): void {
+    this.paused = true;
+    this.clearRetryTimers();
+  }
+
   /**
    * Process upload queue
    */
@@ -91,8 +155,10 @@ export class SyncScheduler {
         processedAny = true;
       }
       if (processedAny) this.onSyncOutcome?.(true);
+      this.upRetryAttempt = 0;
     } catch (error) {
       this.onSyncOutcome?.(false, error);
+      this.scheduleUpRetry();
       // syncUp runs fire-and-forget — it's wired to the MutationEnqueued event
       // (broadcast synchronously, return value dropped) and is also kicked off
       // via `void this.syncDown()` below. A rejection escaping here therefore
@@ -116,7 +182,15 @@ export class SyncScheduler {
    */
   async syncDown() {
     if (this.isSyncingDown || this.paused) return;
-    if (this.upQueue.size > 0) return;
+    // Down-sync yields to a non-empty up-queue so a register never races ahead
+    // of the mutation it should observe. That yield used to be permanent: if
+    // the up-queue never drained, nothing re-armed the down pass. Come back on
+    // the backoff instead, so a wedged push delays reads rather than killing
+    // them.
+    if (this.upQueue.size > 0) {
+      this.scheduleDownRetry(RETRY_BASE_MS);
+      return;
+    }
 
     this.isSyncingDown = true;
     let processedAny = false;
@@ -127,8 +201,10 @@ export class SyncScheduler {
         processedAny = true;
       }
       if (processedAny) this.onSyncOutcome?.(true);
+      this.downRetryAttempt = 0;
     } catch (error) {
       this.onSyncOutcome?.(false, error);
+      this.scheduleDownRetry();
       // Same fire-and-forget story as syncUp: this is the QueryItemEnqueued
       // subscriber (and is also called via `void this.syncDown()`), so a thrown
       // error here becomes an unhandled rejection. The canonical case is a

package/src/services/database/sqlite-cache-engine.test.ts

@@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest';
 import { RecordId } from 'surrealdb';
 import { pureWriteOpResult, SqliteCacheEngine } from './sqlite-cache-engine';
 import { stubTransport } from './sqlite-transport.fixture';
+import { BrokerPortClosedError } from './sqlite-transport';
 import { translateSurql } from './surql-translate';
 import type { SqlOp } from './surql-translate';
 import { surql } from '../../utils/surql';
@@ -330,6 +331,99 @@ describe('SqliteCacheEngine role modes', () => {
     expect(engine.storageHealth.role).toBe('leader');
   });
 
+  /**
+   * A transport with the REAL pending-map semantics (the shared fixture never
+   * rejects on close, which would make the drain test vacuous): `exec` parks
+   * until `flush()`, and `close` rejects whatever is still parked.
+   */
+  function deferredTransport() {
+    const parked: Array<{ resolve: (v: any) => void; reject: (e: unknown) => void }> = [];
+    let closed = false;
+    const transport: any = {
+      kind: 'worker',
+      get connected() {
+        return !closed;
+      },
+      call(type: string) {
+        if (closed) return Promise.reject(new Error('SQLite worker crashed: transport closed'));
+        if (type === 'open') return Promise.resolve({ persisted: true });
+        if (type !== 'exec') return Promise.resolve({});
+        return new Promise((resolve, reject) => parked.push({ resolve, reject }));
+      },
+      shutdown: () => Promise.resolve(),
+      failAll() {},
+      close(reason = 'closed', err?: Error) {
+        if (closed) return;
+        closed = true;
+        const e = err ?? new Error(`SQLite worker crashed: ${reason}`);
+        for (const p of parked.splice(0)) p.reject(e);
+      },
+      flush() {
+        for (const p of parked.splice(0)) p.resolve({ rows: [] });
+      },
+      get parkedCount() {
+        return parked.length;
+      },
+    };
+    return transport;
+  }
+
+  // Regression (WhitePawn first-login): the bucket switch runs
+  // moveToBucket → teardownLeader → releaseOwnership, which lives on the
+  // transition chain, NOT the opQueue — so a query's fetch can be sitting at
+  // the worker when ownership is released. Tearing the transport down under it
+  // rejected that fetch with "SQLite worker crashed: ownership released", which
+  // nothing retries (and nobody caught: an unhandled rejection in the console).
+  it('releaseOwnership waits for in-flight ops instead of killing them', async () => {
+    const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger(), {
+      shared: true,
+    });
+    const transport = deferredTransport();
+    (engine as any).createTransport = () => transport;
+    await engine.adoptOwner('anon', {
+      workerLockName: 'sp00ky-tabs:fp:anon:worker:1',
+      allowMemoryFallback: false,
+      resumeHeld: false,
+    });
+
+    const read = engine.getById('_00_query', 'h1');
+    await new Promise((r) => setTimeout(r, 0));
+    expect(transport.parkedCount).toBe(1);
+
+    const released = engine.releaseOwnership();
+    await new Promise((r) => setTimeout(r, 0));
+    // Still parked: the teardown is waiting on it rather than failing it.
+    expect(transport.parkedCount).toBe(1);
+    transport.flush();
+
+    await released;
+    await expect(read).resolves.toBeNull();
+  });
+
+  it('fails an undrainable op as a retryable transport loss, not a crash', async () => {
+    const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, makeLogger(), {
+      shared: true,
+    });
+    const transport = deferredTransport();
+    (engine as any).createTransport = () => transport;
+    await engine.adoptOwner('anon', {
+      workerLockName: 'sp00ky-tabs:fp:anon:worker:1',
+      allowMemoryFallback: false,
+      resumeHeld: false,
+    });
+
+    const read = engine.getById('_00_query', 'h1');
+    const caught = read.catch((e: unknown) => e);
+    await new Promise((r) => setTimeout(r, 0));
+    // The op never answers: the drain times out and the teardown proceeds.
+    (engine as any).drainInFlight = () => Promise.resolve();
+    await engine.releaseOwnership();
+
+    const err = await caught;
+    expect(err).toBeInstanceOf(BrokerPortClosedError);
+    expect((err as Error).message).toContain('ownership released');
+  });
+
   it('bumps the epoch on promotion after having had a store (fences in-flight chains)', async () => {
     const { engine } = makeSharedEngine();
     await engine.adoptOwner('anon', {

package/src/services/database/sqlite-cache-engine.ts

@@ -262,6 +262,33 @@ export class SqliteCacheEngine implements LocalStore {
     return result;
   }
 
+  /**
+   * Ops dispatched to the worker and not yet answered. Role transitions run on
+   * their own chain (see {@link transitionChain}), so they can start while the
+   * opQueue still has an op at the worker; tearing the transport down under it
+   * would reject that op — and its caller may be a query whose only fetch this
+   * was. {@link drainInFlight} lets a deliberate teardown wait them out.
+   */
+  private inFlightCalls = new Set<Promise<unknown>>();
+
+  /** Wait for dispatched ops to answer before a deliberate transport teardown.
+   *  Bounded: a wedged worker must not block the role change forever. */
+  private async drainInFlight(timeoutMs = 2_000): Promise<void> {
+    if (this.inFlightCalls.size === 0) return;
+    const settled = Promise.all([...this.inFlightCalls].map((p) => p.catch(() => undefined)));
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    try {
+      await Promise.race([
+        settled,
+        new Promise<void>((resolve) => {
+          timer = setTimeout(resolve, timeoutMs);
+        }),
+      ]);
+    } finally {
+      if (timer) clearTimeout(timer);
+    }
+  }
+
   private rawCall<T = any>(type: string, payload?: unknown): Promise<T> {
     if (!this.transport) throw new Error('SqliteCacheEngine: not connected');
     // --- instrumentation: live, inspectable via `globalThis.__sqliteStats` ---
@@ -276,8 +303,21 @@ export class SqliteCacheEngine implements LocalStore {
     s.maxInFlight = Math.max(s.maxInFlight, s.inFlight);
     if (this.transport.kind === 'port') s.proxiedOps = (s.proxiedOps ?? 0) + 1;
     const sentAt = performance.now();
-    return this.transport.call<T>(type, payload).then(
+    // Track the dispatch, not the returned promise: attaching a handler to
+    // `result` would mark it handled and swallow the unhandled-rejection
+    // reports that surface a caller which forgot to catch.
+    let settle!: () => void;
+    const tracked = new Promise<void>((res) => {
+      settle = res;
+    });
+    this.inFlightCalls.add(tracked);
+    const finish = () => {
+      this.inFlightCalls.delete(tracked);
+      settle();
+    };
+    const result = this.transport.call<T>(type, payload).then(
       (v: T) => {
+        finish();
         s.inFlight--;
         // Split the round-trip: `wt` is time inside the worker's handler,
         // the remainder is postMessage + scheduling overhead.
@@ -289,10 +329,12 @@ export class SqliteCacheEngine implements LocalStore {
         return v;
       },
       (e: unknown) => {
+        finish();
         s.inFlight--;
         throw e;
       }
     );
+    return result;
   }
 
   /**
@@ -514,6 +556,7 @@ export class SqliteCacheEngine implements LocalStore {
         this.closeRoleGate();
         return this.storageHealthValue;
       }
+      if (this.transport) await this.drainInFlight();
       if (this.hadStore) this.storeEpoch++;
       if (this.transport) {
         try {
@@ -521,7 +564,7 @@ export class SqliteCacheEngine implements LocalStore {
         } catch {
           /* ignore */
         }
-        this.transport.close('adopting ownership');
+        this.transport.close('adopting ownership', new BrokerPortClosedError('adopting ownership'));
         this.transport = null;
       }
       await this.openInternal(bucketId, {
@@ -555,8 +598,12 @@ export class SqliteCacheEngine implements LocalStore {
   ): Promise<void> {
     this.roleLabel = 'follower';
     return this.chainTransition(async () => {
+      if (this.transport) await this.drainInFlight();
       if (this.hadStore) this.storeEpoch++;
-      this.transport?.close('adopting leader port');
+      this.transport?.close(
+        'adopting leader port',
+        new BrokerPortClosedError('adopting leader port')
+      );
       this.transport = new PortSqliteTransport(dbPort, onPortDead, this.logger);
       this.bucketId = snapshot.bucketId;
       // The shared store exists and is seeded; mirror the owner's bookkeeping.
@@ -578,6 +625,11 @@ export class SqliteCacheEngine implements LocalStore {
   async releaseOwnership(): Promise<void> {
     await this.chainTransition(async () => {
       if (this.transport) {
+        // Let ops already at the worker answer first. Without this they die
+        // with the transport — a bucket switch (moveToBucket → teardownLeader)
+        // runs while the opQueue may still have a query's fetch in flight, and
+        // that fetch's caller is not necessarily prepared to retry.
+        await this.drainInFlight();
         try {
           if (this.transport.kind === 'worker') {
             await (this.transport as WorkerSqliteTransport).shutdown();
@@ -585,7 +637,13 @@ export class SqliteCacheEngine implements LocalStore {
         } catch {
           /* worker may already be fenced/dead */
         }
-        this.transport.close('ownership released');
+        // Anything still pending (drain timed out) is a deliberate teardown,
+        // not a crash: fail it the way a follower's lost port does, so the
+        // error text is honest and callers can treat it as retryable.
+        this.transport.close(
+          'ownership released',
+          new BrokerPortClosedError('ownership released')
+        );
         this.transport = null;
       }
       this.storeEpoch++;

package/src/services/database/sqlite-transport.ts

@@ -22,8 +22,10 @@ export interface SqliteTransport {
   call<T = unknown>(type: string, payload?: unknown): Promise<T>;
   /** Reject every pending request with `reason`. Safe to call repeatedly. */
   failAll(reason: string): void;
-  /** failAll + release the underlying channel. Terminal. */
-  close(reason?: string): void;
+  /** failAll + release the underlying channel. Terminal. `err` overrides the
+   *  transport's own error shape — used for a deliberate teardown (role change)
+   *  so callers see a retryable "transport lost", not "the worker crashed". */
+  close(reason?: string, err?: Error): void;
 }
 
 /** Thrown into pending follower calls when the leader (or its port) goes away.
@@ -81,17 +83,17 @@ abstract class BaseTransport implements SqliteTransport {
     });
   }
 
-  failAll(reason: string): void {
+  failAll(reason: string, err?: Error): void {
     if (this.pending.size === 0) return;
-    const err = this.makeError(reason);
-    for (const [, p] of this.pending) p.reject(err);
+    const e = err ?? this.makeError(reason);
+    for (const [, p] of this.pending) p.reject(e);
     this.pending.clear();
   }
 
-  close(reason = 'closed'): void {
+  close(reason = 'closed', err?: Error): void {
     if (this.closed) return;
     this.closed = true;
-    this.failAll(reason);
+    this.failAll(reason, err);
   }
 }
 
@@ -166,9 +168,9 @@ export class WorkerSqliteTransport extends BaseTransport {
     return this.call('shutdown').then(() => undefined);
   }
 
-  close(reason = 'closed'): void {
+  close(reason = 'closed', err?: Error): void {
     if (this.closed) return;
-    super.close(reason);
+    super.close(reason, err);
     this.worker.terminate();
   }
 }
@@ -193,10 +195,10 @@ export class PortSqliteTransport extends BaseTransport {
     this.dead(reason);
   }
 
-  private dead(reason: string): void {
+  private dead(reason: string, err?: Error): void {
     if (this.closed) return;
     this.closed = true;
-    this.failAll(reason);
+    this.failAll(reason, err);
     try {
       this.port.close();
     } catch {
@@ -213,7 +215,7 @@ export class PortSqliteTransport extends BaseTransport {
     return new BrokerPortClosedError(reason);
   }
 
-  close(reason = 'closed'): void {
-    this.dead(reason);
+  close(reason = 'closed', err?: Error): void {
+    this.dead(reason, err);
   }
 }