@spooky-sync/core 0.0.1-canary.178 → 0.0.1-canary.179

package/dist/index.js

@@ -4974,10 +4974,20 @@ function rowToUpEvent(r, logger) {
 
 //#endregion
 //#region src/modules/sync/queue/queue-down.ts
+/**
+* How many times a failing event keeps its place at the head before it is
+* rotated to the back. A transient failure (the SSP still bootstrapping) clears
+* well inside this, so ordering is preserved for the common case; a permanently
+* rejected event (a permission the SSP refuses to lower) stops holding every
+* other query hostage behind it.
+*/
+const MAX_HEAD_RETRIES = 3;
 var DownQueue = class {
 	queue = [];
 	_events;
 	logger;
+	/** Consecutive failures per queued event; cleared when it finally succeeds. */
+	failures = /* @__PURE__ */ new WeakMap();
 	get events() {
 		return this._events;
 	}
@@ -5008,13 +5018,20 @@ var DownQueue = class {
 		const event = this.queue.shift();
 		if (event) try {
 			await fn(event);
+			this.failures.delete(event);
 		} catch (error) {
+			const attempts = (this.failures.get(event) ?? 0) + 1;
+			this.failures.set(event, attempts);
+			const starvingOthers = attempts >= MAX_HEAD_RETRIES && this.queue.length > 0;
+			if (starvingOthers) this.queue.push(event);
+			else this.queue.unshift(event);
 			this.logger.error({
 				error,
 				event,
+				attempts,
+				rotated: starvingOthers,
 				Category: "sp00ky-client::DownQueue::next"
 			}, "Failed to process query");
-			this.queue.unshift(event);
 			throw error;
 		}
 	}
@@ -5378,11 +5395,18 @@ var SyncEngine = class {
 * SyncScheduler manages when to sync: queue management and orchestration.
 * Decides the order and timing of sync operations.
 */
+/** Backoff for re-draining a queue that halted on an error. */
+const RETRY_BASE_MS = 500;
+const RETRY_MAX_MS = 15e3;
 var SyncScheduler = class {
 	isSyncingUp = false;
 	isSyncingDown = false;
 	paused = false;
 	pauseWaiters = [];
+	upRetryTimer;
+	downRetryTimer;
+	upRetryAttempt = 0;
+	downRetryAttempt = 0;
 	constructor(upQueue, downQueue, onProcessUp, onProcessDown, logger, onRollback, onSyncOutcome) {
 		this.upQueue = upQueue;
 		this.downQueue = downQueue;
@@ -5418,12 +5442,16 @@ var SyncScheduler = class {
 	*/
 	pause() {
 		this.paused = true;
+		this.clearRetryTimers();
 		if (!this.isSyncingUp && !this.isSyncingDown) return Promise.resolve();
 		return new Promise((resolve) => this.pauseWaiters.push(resolve));
 	}
 	resume() {
 		this.paused = false;
+		this.upRetryAttempt = 0;
+		this.downRetryAttempt = 0;
 		this.syncUp();
+		this.syncDown();
 	}
 	maybeResolvePause() {
 		if (!this.paused || this.isSyncingUp || this.isSyncingDown) return;
@@ -5431,6 +5459,46 @@ var SyncScheduler = class {
 		this.pauseWaiters = [];
 		for (const resolve of waiters) resolve();
 	}
+	/** Exponential backoff, capped. Attempt 0 is the first retry. */
+	retryDelay(attempt) {
+		return Math.min(RETRY_BASE_MS * 2 ** attempt, RETRY_MAX_MS);
+	}
+	scheduleUpRetry() {
+		if (this.paused || this.upRetryTimer || this.upQueue.size === 0) return;
+		const delay = this.retryDelay(this.upRetryAttempt++);
+		this.upRetryTimer = setTimeout(() => {
+			this.upRetryTimer = void 0;
+			this.syncUp();
+		}, delay);
+	}
+	/**
+	* Re-arm the down pass. With no argument this is failure backoff and the
+	* streak grows; with an explicit delay it is a yield (the up-queue holds the
+	* floor), which is not a failure and must not push the backoff out.
+	*/
+	scheduleDownRetry(delayMs) {
+		if (this.paused || this.downRetryTimer || this.downQueue.size === 0) return;
+		const delay = delayMs ?? this.retryDelay(this.downRetryAttempt++);
+		this.downRetryTimer = setTimeout(() => {
+			this.downRetryTimer = void 0;
+			this.syncDown();
+		}, delay);
+	}
+	clearRetryTimers() {
+		if (this.upRetryTimer) {
+			clearTimeout(this.upRetryTimer);
+			this.upRetryTimer = void 0;
+		}
+		if (this.downRetryTimer) {
+			clearTimeout(this.downRetryTimer);
+			this.downRetryTimer = void 0;
+		}
+	}
+	/** Stop all pending retries. Call when tearing the client down. */
+	dispose() {
+		this.paused = true;
+		this.clearRetryTimers();
+	}
 	/**
 	* Process upload queue
 	*/
@@ -5444,8 +5512,10 @@ var SyncScheduler = class {
 				processedAny = true;
 			}
 			if (processedAny) this.onSyncOutcome?.(true);
+			this.upRetryAttempt = 0;
 		} catch (error) {
 			this.onSyncOutcome?.(false, error);
+			this.scheduleUpRetry();
 			this.logger.debug({
 				error,
 				Category: "sp00ky-client::SyncScheduler::syncUp"
@@ -5461,7 +5531,10 @@ var SyncScheduler = class {
 	*/
 	async syncDown() {
 		if (this.isSyncingDown || this.paused) return;
-		if (this.upQueue.size > 0) return;
+		if (this.upQueue.size > 0) {
+			this.scheduleDownRetry(RETRY_BASE_MS);
+			return;
+		}
 		this.isSyncingDown = true;
 		let processedAny = false;
 		try {
@@ -5471,8 +5544,10 @@ var SyncScheduler = class {
 				processedAny = true;
 			}
 			if (processedAny) this.onSyncOutcome?.(true);
+			this.downRetryAttempt = 0;
 		} catch (error) {
 			this.onSyncOutcome?.(false, error);
+			this.scheduleDownRetry();
 			this.logger.debug({
 				error,
 				Category: "sp00ky-client::SyncScheduler::syncDown"
@@ -6928,8 +7003,8 @@ function selfAllowlistedVariant(flag, userId) {
 
 //#endregion
 //#region src/modules/devtools/index.ts
-const CORE_VERSION = "0.0.1-canary.178";
-const WASM_VERSION = "0.0.1-canary.178";
+const CORE_VERSION = "0.0.1-canary.179";
+const WASM_VERSION = "0.0.1-canary.179";
 const SURREAL_VERSION = "3.0.3";
 var DevToolsService = class DevToolsService {
 	eventsHistory = [];
@@ -11360,7 +11435,7 @@ var Sp00kyClient = class {
 		return new TabsCoordinator({
 			tabId,
 			fingerprint: computeTabsFingerprint({
-				coreVersion: "0.0.1-canary.178",
+				coreVersion: "0.0.1-canary.179",
 				schemaHash: hash53(this.config.schemaSurql),
 				endpoint: this.config.database.endpoint ?? "",
 				namespace: this.config.database.namespace,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/core",
-  "version": "0.0.1-canary.178",
+  "version": "0.0.1-canary.179",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -60,8 +60,8 @@
     }
   },
   "dependencies": {
-    "@spooky-sync/query-builder": "0.0.1-canary.178",
-    "@spooky-sync/ssp-wasm": "0.0.1-canary.178",
+    "@spooky-sync/query-builder": "0.0.1-canary.179",
+    "@spooky-sync/ssp-wasm": "0.0.1-canary.179",
     "@sqlite.org/sqlite-wasm": "3.53.0-build1",
     "@surrealdb/wasm": "^3.0.3",
     "fast-json-patch": "^3.1.1",

package/src/modules/sync/queue/queue-down.test.ts

@@ -0,0 +1,107 @@
+import { describe, it, expect } from 'vitest';
+import { DownQueue } from './queue-down';
+import type { DownEvent } from './queue-down';
+import type { LocalStore } from '../../../services/database/index';
+
+const silentLogger = {
+  child: () => silentLogger,
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any;
+
+const register = (hash: string) => ({ type: 'register', payload: { hash } }) as DownEvent;
+
+const hashOf = (e: DownEvent) => e.payload.hash;
+
+function makeQueue() {
+  return new DownQueue({} as LocalStore, silentLogger);
+}
+
+describe('DownQueue.next failure handling', () => {
+  it('re-heads a failing event so a transient failure keeps its order', async () => {
+    const q = makeQueue();
+    q.push(register('a'));
+    q.push(register('b'));
+
+    const seen: string[] = [];
+    await expect(
+      q.next(async (e) => {
+        seen.push(hashOf(e));
+        throw new Error('503');
+      })
+    ).rejects.toThrow('503');
+
+    expect(seen).toEqual(['a']);
+    // 'a' is back at the head, ahead of 'b'.
+    await expect(q.next(async (e) => { seen.push(hashOf(e)); throw new Error('503'); })).rejects.toThrow();
+    expect(seen).toEqual(['a', 'a']);
+  });
+
+  it('rotates a persistently failing event to the back so others can proceed', async () => {
+    const q = makeQueue();
+    q.push(register('poison'));
+    q.push(register('healthy'));
+
+    const seen: string[] = [];
+    const failPoison = async (e: DownEvent) => {
+      seen.push(hashOf(e));
+      if (hashOf(e) === 'poison') throw new Error('400 rejected');
+    };
+
+    // Three attempts keep the head; the third rotates it behind 'healthy'.
+    for (let i = 0; i < 3; i++) {
+      await expect(q.next(failPoison)).rejects.toThrow();
+    }
+    expect(seen).toEqual(['poison', 'poison', 'poison']);
+
+    // 'healthy' is no longer starved.
+    await q.next(failPoison);
+    expect(seen).toEqual(['poison', 'poison', 'poison', 'healthy']);
+    expect(q.size).toBe(1); // only the poison event remains
+  });
+
+  it('does not rotate when nothing else is waiting', async () => {
+    const q = makeQueue();
+    q.push(register('only'));
+
+    for (let i = 0; i < 5; i++) {
+      await expect(
+        q.next(async () => {
+          throw new Error('boom');
+        })
+      ).rejects.toThrow();
+    }
+    expect(q.size).toBe(1);
+  });
+
+  it('clears the failure count once an event succeeds', async () => {
+    const q = makeQueue();
+    const event = register('a');
+    q.push(event);
+    q.push(register('b'));
+
+    await expect(
+      q.next(async () => {
+        throw new Error('boom');
+      })
+    ).rejects.toThrow();
+    // Succeeds on the retry, so its streak resets rather than carrying over.
+    await q.next(async () => {});
+    expect(q.size).toBe(1);
+
+    q.push(event);
+    // A fresh streak: it keeps the head again rather than rotating immediately.
+    await expect(
+      q.next(async () => {
+        throw new Error('boom');
+      })
+    ).rejects.toThrow();
+    const drained: string[] = [];
+    await q.next(async (e) => {
+      drained.push(hashOf(e));
+    });
+    expect(drained).toEqual(['b']);
+  });
+});

package/src/modules/sync/queue/queue-down.ts

@@ -37,10 +37,21 @@ export type CleanupEvent = {
 
 export type DownEvent = RegisterEvent | SyncEvent | HeartbeatEvent | CleanupEvent;
 
+/**
+ * How many times a failing event keeps its place at the head before it is
+ * rotated to the back. A transient failure (the SSP still bootstrapping) clears
+ * well inside this, so ordering is preserved for the common case; a permanently
+ * rejected event (a permission the SSP refuses to lower) stops holding every
+ * other query hostage behind it.
+ */
+const MAX_HEAD_RETRIES = 3;
+
 export class DownQueue {
   private queue: DownEvent[] = [];
   private _events: SyncQueueEventSystem;
   private logger: Logger;
+  /** Consecutive failures per queued event; cleared when it finally succeeds. */
+  private failures = new WeakMap<DownEvent, number>();
 
   get events(): SyncQueueEventSystem {
     return this._events;
@@ -83,12 +94,23 @@ export class DownQueue {
     if (event) {
       try {
         await fn(event);
+        this.failures.delete(event);
       } catch (error) {
+        const attempts = (this.failures.get(event) ?? 0) + 1;
+        this.failures.set(event, attempts);
+        // Re-head so a transient failure keeps its ordering, but give up the
+        // head once it looks permanent and there is other work waiting — one
+        // unregisterable query must not stall every other query's registration.
+        const starvingOthers = attempts >= MAX_HEAD_RETRIES && this.queue.length > 0;
+        if (starvingOthers) {
+          this.queue.push(event);
+        } else {
+          this.queue.unshift(event);
+        }
         this.logger.error(
-          { error, event, Category: 'sp00ky-client::DownQueue::next' },
+          { error, event, attempts, rotated: starvingOthers, Category: 'sp00ky-client::DownQueue::next' },
           'Failed to process query'
         );
-        this.queue.unshift(event);
         throw error;
       }
     }

package/src/modules/sync/scheduler.retry.test.ts

@@ -0,0 +1,156 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { SyncScheduler } from './scheduler';
+import type { UpQueue, DownQueue, DownEvent, UpEvent } from './queue/index';
+
+// A queue whose drain throws re-queues the failing item at the HEAD and stops
+// the pass (see DownQueue.next). Nothing used to re-arm it: the queues only
+// moved on a fresh enqueue, so one transient failure — canonically the SSP
+// answering 503 NOT_READY for the whole of its bootstrap window — parked every
+// pending `register` forever, and the `useQuery` waiting on it never left its
+// loading state. These cover the backoff that makes that self-heal.
+
+const silentLogger = {
+  child: () => silentLogger,
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any;
+
+/** A queue that mirrors the real ones: a throwing handler re-heads the item. */
+function makeQueue<E>(items: E[]) {
+  return {
+    queue: [...items],
+    get size() {
+      return this.queue.length;
+    },
+    events: { subscribe: () => {} },
+    loadFromDatabase: async () => {},
+    clear() {
+      this.queue = [];
+    },
+    async next(fn: (event: E) => Promise<void>) {
+      const event = this.queue.shift();
+      if (!event) return;
+      try {
+        await fn(event);
+      } catch (err) {
+        this.queue.unshift(event);
+        throw err;
+      }
+    },
+  };
+}
+
+const downEvent = (hash: string) => ({ type: 'register', payload: { hash } }) as DownEvent;
+const upEvent = (n: number) => ({ type: 'delete', mutation_id: n, record_id: n }) as unknown as UpEvent;
+
+describe('SyncScheduler retry', () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  it('re-drains a failed down event instead of parking it forever', async () => {
+    const downQueue = makeQueue([downEvent('q1')]);
+    const upQueue = makeQueue<UpEvent>([]);
+    let attempts = 0;
+    const scheduler = new SyncScheduler(
+      upQueue as unknown as UpQueue,
+      downQueue as unknown as DownQueue,
+      async () => {},
+      async () => {
+        attempts++;
+        // Fail the way a bootstrapping SSP does, then succeed.
+        if (attempts < 3) throw new Error('503 NOT_READY');
+      },
+      silentLogger
+    );
+
+    await scheduler.syncDown();
+    expect(attempts).toBe(1);
+    expect(downQueue.size).toBe(1); // re-headed, not dropped
+
+    // Backoff: 500ms, then 1000ms.
+    await vi.advanceTimersByTimeAsync(500);
+    expect(attempts).toBe(2);
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(attempts).toBe(3);
+
+    expect(downQueue.size).toBe(0);
+  });
+
+  it('stops retrying once the queue drains', async () => {
+    const downQueue = makeQueue([downEvent('q1')]);
+    const upQueue = makeQueue<UpEvent>([]);
+    let attempts = 0;
+    const scheduler = new SyncScheduler(
+      upQueue as unknown as UpQueue,
+      downQueue as unknown as DownQueue,
+      async () => {},
+      async () => {
+        attempts++;
+        throw new Error('boom');
+      },
+      silentLogger
+    );
+
+    await scheduler.syncDown();
+    await vi.advanceTimersByTimeAsync(500);
+    expect(attempts).toBe(2);
+
+    // Drain it out from under the scheduler; the next retry finds nothing and
+    // schedules no further work.
+    downQueue.clear();
+    await vi.advanceTimersByTimeAsync(1000);
+    expect(attempts).toBe(2);
+    await vi.advanceTimersByTimeAsync(60_000);
+    expect(attempts).toBe(2);
+  });
+
+  it('comes back for the down queue while the up queue holds the floor', async () => {
+    const downQueue = makeQueue([downEvent('q1')]);
+    const upQueue = makeQueue<UpEvent>([upEvent(1)]);
+    let downAttempts = 0;
+    const scheduler = new SyncScheduler(
+      upQueue as unknown as UpQueue,
+      downQueue as unknown as DownQueue,
+      async () => {},
+      async () => {
+        downAttempts++;
+      },
+      silentLogger
+    );
+
+    // Yields to the non-empty up queue.
+    await scheduler.syncDown();
+    expect(downAttempts).toBe(0);
+
+    // Once the up queue empties, the re-armed pass picks the down event up
+    // without needing a fresh enqueue.
+    upQueue.clear();
+    await vi.advanceTimersByTimeAsync(500);
+    expect(downAttempts).toBe(1);
+  });
+
+  it('pause cancels pending retries', async () => {
+    const downQueue = makeQueue([downEvent('q1')]);
+    const upQueue = makeQueue<UpEvent>([]);
+    let attempts = 0;
+    const scheduler = new SyncScheduler(
+      upQueue as unknown as UpQueue,
+      downQueue as unknown as DownQueue,
+      async () => {},
+      async () => {
+        attempts++;
+        throw new Error('boom');
+      },
+      silentLogger
+    );
+
+    await scheduler.syncDown();
+    expect(attempts).toBe(1);
+
+    await scheduler.pause();
+    await vi.advanceTimersByTimeAsync(60_000);
+    expect(attempts).toBe(1);
+  });
+});

package/src/modules/sync/scheduler.ts

@@ -6,11 +6,25 @@ import { SyncQueueEventTypes } from './events/index';
  * SyncScheduler manages when to sync: queue management and orchestration.
  * Decides the order and timing of sync operations.
  */
+/** Backoff for re-draining a queue that halted on an error. */
+const RETRY_BASE_MS = 500;
+const RETRY_MAX_MS = 15_000;
+
 export class SyncScheduler {
   private isSyncingUp: boolean = false;
   private isSyncingDown: boolean = false;
   private paused: boolean = false;
   private pauseWaiters: Array<() => void> = [];
+  // A failed drain re-queues its item at the HEAD (see DownQueue.next) and
+  // stops the pass. Without a timer nothing ever drains it again: the queues
+  // only move on a fresh enqueue, so a transient failure — canonically the
+  // SSP answering 503 NOT_READY for the whole of its bootstrap window — left
+  // every pending `register` parked forever and its `useQuery` loading forever.
+  // Retry on a backoff so that heals itself instead of needing a reload.
+  private upRetryTimer?: ReturnType<typeof setTimeout>;
+  private downRetryTimer?: ReturnType<typeof setTimeout>;
+  private upRetryAttempt = 0;
+  private downRetryAttempt = 0;
 
   constructor(
     private upQueue: UpQueue,
@@ -62,13 +76,18 @@ export class SyncScheduler {
    */
   pause(): Promise<void> {
     this.paused = true;
+    this.clearRetryTimers();
     if (!this.isSyncingUp && !this.isSyncingDown) return Promise.resolve();
     return new Promise<void>((resolve) => this.pauseWaiters.push(resolve));
   }
 
   resume(): void {
     this.paused = false;
+    // A resume is a fresh start, not a continuation of the failing streak.
+    this.upRetryAttempt = 0;
+    this.downRetryAttempt = 0;
     void this.syncUp();
+    void this.syncDown();
   }
 
   private maybeResolvePause() {
@@ -78,6 +97,51 @@ export class SyncScheduler {
     for (const resolve of waiters) resolve();
   }
 
+  /** Exponential backoff, capped. Attempt 0 is the first retry. */
+  private retryDelay(attempt: number): number {
+    return Math.min(RETRY_BASE_MS * 2 ** attempt, RETRY_MAX_MS);
+  }
+
+  private scheduleUpRetry() {
+    if (this.paused || this.upRetryTimer || this.upQueue.size === 0) return;
+    const delay = this.retryDelay(this.upRetryAttempt++);
+    this.upRetryTimer = setTimeout(() => {
+      this.upRetryTimer = undefined;
+      void this.syncUp();
+    }, delay);
+  }
+
+  /**
+   * Re-arm the down pass. With no argument this is failure backoff and the
+   * streak grows; with an explicit delay it is a yield (the up-queue holds the
+   * floor), which is not a failure and must not push the backoff out.
+   */
+  private scheduleDownRetry(delayMs?: number) {
+    if (this.paused || this.downRetryTimer || this.downQueue.size === 0) return;
+    const delay = delayMs ?? this.retryDelay(this.downRetryAttempt++);
+    this.downRetryTimer = setTimeout(() => {
+      this.downRetryTimer = undefined;
+      void this.syncDown();
+    }, delay);
+  }
+
+  private clearRetryTimers() {
+    if (this.upRetryTimer) {
+      clearTimeout(this.upRetryTimer);
+      this.upRetryTimer = undefined;
+    }
+    if (this.downRetryTimer) {
+      clearTimeout(this.downRetryTimer);
+      this.downRetryTimer = undefined;
+    }
+  }
+
+  /** Stop all pending retries. Call when tearing the client down. */
+  dispose(): void {
+    this.paused = true;
+    this.clearRetryTimers();
+  }
+
   /**
    * Process upload queue
    */
@@ -91,8 +155,10 @@ export class SyncScheduler {
         processedAny = true;
       }
       if (processedAny) this.onSyncOutcome?.(true);
+      this.upRetryAttempt = 0;
     } catch (error) {
       this.onSyncOutcome?.(false, error);
+      this.scheduleUpRetry();
       // syncUp runs fire-and-forget — it's wired to the MutationEnqueued event
       // (broadcast synchronously, return value dropped) and is also kicked off
       // via `void this.syncDown()` below. A rejection escaping here therefore
@@ -116,7 +182,15 @@ export class SyncScheduler {
    */
   async syncDown() {
     if (this.isSyncingDown || this.paused) return;
-    if (this.upQueue.size > 0) return;
+    // Down-sync yields to a non-empty up-queue so a register never races ahead
+    // of the mutation it should observe. That yield used to be permanent: if
+    // the up-queue never drained, nothing re-armed the down pass. Come back on
+    // the backoff instead, so a wedged push delays reads rather than killing
+    // them.
+    if (this.upQueue.size > 0) {
+      this.scheduleDownRetry(RETRY_BASE_MS);
+      return;
+    }
 
     this.isSyncingDown = true;
     let processedAny = false;
@@ -127,8 +201,10 @@ export class SyncScheduler {
         processedAny = true;
       }
       if (processedAny) this.onSyncOutcome?.(true);
+      this.downRetryAttempt = 0;
     } catch (error) {
       this.onSyncOutcome?.(false, error);
+      this.scheduleDownRetry();
       // Same fire-and-forget story as syncUp: this is the QueryItemEnqueued
       // subscriber (and is also called via `void this.syncDown()`), so a thrown
       // error here becomes an unhandled rejection. The canonical case is a