@spooky-sync/core 0.0.1-canary.22 → 0.0.1-canary.221

package/src/modules/sync/queue/queue-down.test.ts

@@ -0,0 +1,228 @@
+import { describe, it, expect } from 'vitest';
+import { DownQueue } from './queue-down';
+import type { DownEvent } from './queue-down';
+import type { LocalStore } from '../../../services/database/index';
+
+const silentLogger = {
+  child: () => silentLogger,
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any;
+
+const register = (hash: string) => ({ type: 'register', payload: { hash } }) as DownEvent;
+const cleanup = (hash: string) => ({ type: 'cleanup', payload: { hash } }) as DownEvent;
+
+const hashOf = (e: DownEvent) => e.payload.hash;
+
+function makeQueue() {
+  return new DownQueue({} as LocalStore, silentLogger);
+}
+
+describe('DownQueue.next failure handling', () => {
+  it('re-heads a failing event so a transient failure keeps its order', async () => {
+    const q = makeQueue();
+    q.push(register('a'));
+    q.push(register('b'));
+
+    const seen: string[] = [];
+    await expect(
+      q.next(async (e) => {
+        seen.push(hashOf(e));
+        throw new Error('503');
+      })
+    ).rejects.toThrow('503');
+
+    expect(seen).toEqual(['a']);
+    // 'a' is back at the head, ahead of 'b'.
+    await expect(q.next(async (e) => { seen.push(hashOf(e)); throw new Error('503'); })).rejects.toThrow();
+    expect(seen).toEqual(['a', 'a']);
+  });
+
+  it('rotates a persistently failing event to the back so others can proceed', async () => {
+    const q = makeQueue();
+    q.push(register('poison'));
+    q.push(register('healthy'));
+
+    const seen: string[] = [];
+    const failPoison = async (e: DownEvent) => {
+      seen.push(hashOf(e));
+      if (hashOf(e) === 'poison') throw new Error('400 rejected');
+    };
+
+    // Three attempts keep the head; the third rotates it behind 'healthy'.
+    for (let i = 0; i < 3; i++) {
+      await expect(q.next(failPoison)).rejects.toThrow();
+    }
+    expect(seen).toEqual(['poison', 'poison', 'poison']);
+
+    // 'healthy' is no longer starved.
+    await q.next(failPoison);
+    expect(seen).toEqual(['poison', 'poison', 'poison', 'healthy']);
+    expect(q.size).toBe(1); // only the poison event remains
+  });
+
+  it('does not rotate when nothing else is waiting', async () => {
+    const q = makeQueue();
+    q.push(register('only'));
+
+    for (let i = 0; i < 5; i++) {
+      await expect(
+        q.next(async () => {
+          throw new Error('boom');
+        })
+      ).rejects.toThrow();
+    }
+    expect(q.size).toBe(1);
+  });
+
+  it('clears the failure count once an event succeeds', async () => {
+    const q = makeQueue();
+    const event = register('a');
+    q.push(event);
+    q.push(register('b'));
+
+    await expect(
+      q.next(async () => {
+        throw new Error('boom');
+      })
+    ).rejects.toThrow();
+    // Succeeds on the retry, so its streak resets rather than carrying over.
+    await q.next(async () => {});
+    expect(q.size).toBe(1);
+
+    q.push(event);
+    // A fresh streak: it keeps the head again rather than rotating immediately.
+    await expect(
+      q.next(async () => {
+        throw new Error('boom');
+      })
+    ).rejects.toThrow();
+    const drained: string[] = [];
+    await q.next(async (e) => {
+      drained.push(hashOf(e));
+    });
+    expect(drained).toEqual(['b']);
+  });
+});
+
+describe('DownQueue.takeNext (per-hash ordering under concurrency)', () => {
+  it('hands out events for distinct hashes so they can run in parallel', () => {
+    const q = makeQueue();
+    q.push(register('a'));
+    q.push(register('b'));
+    q.push(register('c'));
+    const busy = new Set<string>();
+
+    const first = q.takeNext(busy)!;
+    busy.add(hashOf(first));
+    const second = q.takeNext(busy)!;
+    busy.add(hashOf(second));
+
+    expect([hashOf(first), hashOf(second)]).toEqual(['a', 'b']);
+    expect(q.size).toBe(1);
+  });
+
+  it('SKIPS an event whose hash is busy without reordering it', () => {
+    // The whole correctness argument: a `cleanup` must never overtake the
+    // `register` for the same query. A busy hash's event keeps its place in the
+    // queue and is simply passed over.
+    const q = makeQueue();
+    q.push(register('a'));
+    q.push(cleanup('a'));
+    q.push(register('b'));
+
+    const busy = new Set(['a']);
+    const taken = q.takeNext(busy)!;
+    expect(hashOf(taken)).toBe('b');
+
+    // Once 'a' frees up, its two events come back out in their original order.
+    const rest = [q.takeNext(new Set())!, q.takeNext(new Set())!];
+    expect(rest.map((e) => e.type)).toEqual(['register', 'cleanup']);
+    expect(q.size).toBe(0);
+  });
+
+  it('returns undefined when every remaining event is blocked', () => {
+    const q = makeQueue();
+    q.push(register('a'));
+    expect(q.takeNext(new Set(['a']))).toBeUndefined();
+    // Blocked, not consumed.
+    expect(q.size).toBe(1);
+  });
+});
+
+describe('DownQueue.run', () => {
+  it('returns the error instead of throwing, and re-heads the event', async () => {
+    // A concurrent drain has other work in flight when one event fails; a
+    // rejection here would take that work down with it.
+    const q = makeQueue();
+    const event = register('a');
+    q.push(event);
+    const taken = q.takeNext(new Set())!;
+
+    const boom = new Error('nope');
+    const err = await q.run(taken, async () => {
+      throw boom;
+    });
+
+    expect(err).toBe(boom);
+    expect(q.size).toBe(1);
+    expect(hashOf(q.takeNext(new Set())!)).toBe('a');
+  });
+
+  it('returns undefined on success', async () => {
+    const q = makeQueue();
+    q.push(register('a'));
+    const taken = q.takeNext(new Set())!;
+    await expect(q.run(taken, async () => {})).resolves.toBeUndefined();
+    expect(q.size).toBe(0);
+  });
+});
+
+describe('DownQueue.push register coalescing', () => {
+  it('drops a register for a hash that already has one queued', () => {
+    // Four paths re-enqueue `register` for every active hash (reconnect,
+    // self-heal, re-mount, and a re-headed failure) and they stack: one tab was
+    // measured issuing 84 registrations in 93 seconds for 38 hashes. A register
+    // carries nothing but the hash, so the second is identical work.
+    const q = makeQueue();
+    q.push(register('a'));
+    q.push(register('a'));
+    q.push(register('a'));
+
+    expect(q.size).toBe(1);
+  });
+
+  it('keeps registers for DIFFERENT hashes', () => {
+    const q = makeQueue();
+    q.push(register('a'));
+    q.push(register('b'));
+
+    expect(q.size).toBe(2);
+  });
+
+  it('does NOT coalesce a register queued behind a cleanup for the same hash', () => {
+    // Ordering is per hash: dropping this register would let the cleanup tear
+    // the query down with nothing left to re-establish it.
+    const q = makeQueue();
+    q.push(register('a'));
+    q.push(cleanup('a'));
+    q.push(register('a'));
+
+    expect(q.size).toBe(3);
+    expect([q.takeNext(new Set())!, q.takeNext(new Set())!, q.takeNext(new Set())!].map((e) => e.type)).toEqual([
+      'register',
+      'cleanup',
+      'register',
+    ]);
+  });
+
+  it('leaves non-register events alone', () => {
+    const q = makeQueue();
+    q.push(cleanup('a'));
+    q.push(cleanup('a'));
+
+    expect(q.size).toBe(2);
+  });
+});

package/src/modules/sync/queue/queue-down.ts

@@ -1,10 +1,11 @@
-import { LocalDatabaseService } from '../../../services/database/index';
+import type { LocalStore } from '../../../services/database/index';
+import type {
+  SyncQueueEventSystem} from '../events/index';
 import {
   createSyncQueueEventSystem,
-  SyncQueueEventSystem,
   SyncQueueEventTypes,
 } from '../events/index';
-import { Logger } from '../../../services/logger/index';
+import type { Logger } from '../../../services/logger/index';
 
 export type RegisterEvent = {
   type: 'register';
@@ -36,17 +37,28 @@ export type CleanupEvent = {
 
 export type DownEvent = RegisterEvent | SyncEvent | HeartbeatEvent | CleanupEvent;
 
+/**
+ * How many times a failing event keeps its place at the head before it is
+ * rotated to the back. A transient failure (the SSP still bootstrapping) clears
+ * well inside this, so ordering is preserved for the common case; a permanently
+ * rejected event (a permission the SSP refuses to lower) stops holding every
+ * other query hostage behind it.
+ */
+const MAX_HEAD_RETRIES = 3;
+
 export class DownQueue {
   private queue: DownEvent[] = [];
   private _events: SyncQueueEventSystem;
   private logger: Logger;
+  /** Consecutive failures per queued event; cleared when it finally succeeds. */
+  private failures = new WeakMap<DownEvent, number>();
 
   get events(): SyncQueueEventSystem {
     return this._events;
   }
 
   constructor(
-    private local: LocalDatabaseService,
+    private local: LocalStore,
     logger: Logger
   ) {
     this._events = createSyncQueueEventSystem();
@@ -58,10 +70,38 @@ export class DownQueue {
   }
 
   push(event: DownEvent) {
+    // Coalesce repeat registrations. Four separate paths re-enqueue `register`
+    // for every active hash — reconnect, self-heal while degraded, a
+    // re-mounted subscription, and a failed event re-headed by `run` — and
+    // they stack, so one tab was observed issuing 84 `fn::query::register`
+    // round trips in 93 seconds for 38 hashes. A register is idempotent and
+    // carries no payload beyond the hash, so a second one queued behind the
+    // first would re-do identical work.
+    //
+    // Only when the LAST queued event for the hash is itself a `register`.
+    // Ordering never mattered globally but it does per hash (see `takeNext`):
+    // dropping a register queued behind a `cleanup` would let the cleanup tear
+    // the query down with nothing to re-establish it.
+    if (event.type === 'register' && this.lastQueuedTypeFor(event.payload.hash) === 'register') {
+      this.logger.debug(
+        { hash: event.payload.hash, Category: 'sp00ky-client::DownQueue::push' },
+        'Register already queued for this hash; coalescing'
+      );
+      return;
+    }
     this.queue.push(event);
     this.emitPushEvent();
   }
 
+  /** Type of the last queued event for `hash`, or `undefined` if none. */
+  private lastQueuedTypeFor(hash: string): DownEvent['type'] | undefined {
+    for (let i = this.queue.length - 1; i >= 0; i--) {
+      const queued = this.queue[i]!;
+      if (queued.payload.hash === hash) return queued.type;
+    }
+    return undefined;
+  }
+
   private emitPushEvent() {
     this._events.addEvent({
       type: SyncQueueEventTypes.QueryItemEnqueued,
@@ -71,19 +111,74 @@ export class DownQueue {
     });
   }
 
+  /** Drop all queued events (bucket switch: old-bucket register/sync work is
+   *  re-derived after the switch; replaying it would target dead query rows). */
+  clear(): void {
+    this.queue = [];
+  }
+
   async next(fn: (event: DownEvent) => Promise<void>): Promise<void> {
     const event = this.queue.shift();
-    if (event) {
-      try {
-        await fn(event);
-      } catch (error) {
-        this.logger.error(
-          { error, event, Category: 'spooky-client::DownQueue::next' },
-          'Failed to process query'
-        );
+    if (!event) return;
+    const error = await this.run(event, fn);
+    if (error !== undefined) throw error;
+  }
+
+  /**
+   * The next event whose hash is NOT already being processed, or `undefined`
+   * when every remaining event is blocked on a busy hash.
+   *
+   * Ordering only ever mattered PER HASH — a `cleanup` must not overtake the
+   * `register` for the same query — but the queue enforced it globally, so one
+   * registration RPC at a time was the ceiling for the whole client. An event
+   * for a busy hash keeps its place here (it is skipped, not reordered), so
+   * per-hash ordering is preserved exactly while independent hashes proceed
+   * concurrently.
+   */
+  takeNext(busy: ReadonlySet<string>): DownEvent | undefined {
+    for (let i = 0; i < this.queue.length; i++) {
+      const event = this.queue[i]!;
+      if (busy.has(event.payload.hash)) continue;
+      this.queue.splice(i, 1);
+      return event;
+    }
+    return undefined;
+  }
+
+  /**
+   * Process one event, applying the re-head / rotate failure policy.
+   *
+   * NEVER rejects: it RETURNS the error instead (`undefined` on success). A
+   * concurrent drain has other work in flight when one event fails, and a
+   * rejection would either take that work down with it or have to be caught at
+   * every call site. `next` reinstates the throwing contract for the serial
+   * callers that still want it.
+   */
+  async run(
+    event: DownEvent,
+    fn: (event: DownEvent) => Promise<void>
+  ): Promise<unknown | undefined> {
+    try {
+      await fn(event);
+      this.failures.delete(event);
+      return undefined;
+    } catch (error) {
+      const attempts = (this.failures.get(event) ?? 0) + 1;
+      this.failures.set(event, attempts);
+      // Re-head so a transient failure keeps its ordering, but give up the
+      // head once it looks permanent and there is other work waiting — one
+      // unregisterable query must not stall every other query's registration.
+      const starvingOthers = attempts >= MAX_HEAD_RETRIES && this.queue.length > 0;
+      if (starvingOthers) {
+        this.queue.push(event);
+      } else {
         this.queue.unshift(event);
-        throw error;
       }
+      this.logger.error(
+        { error, event, attempts, rotated: starvingOthers, Category: 'sp00ky-client::DownQueue::next' },
+        'Failed to process query'
+      );
+      return error;
     }
   }
 }

package/src/modules/sync/queue/queue-up.forwarded.test.ts

@@ -0,0 +1,164 @@
+import { describe, it, expect, vi } from 'vitest';
+import { UpQueue } from './queue-up';
+import { translateSurql } from '../../../services/database/surql-translate';
+import { surql, classifySyncError } from '../../../utils/index';
+
+function makeLogger(): any {
+  const noop = () => {};
+  const l: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  l.child = () => l;
+  return l;
+}
+
+const ROW = {
+  id: '_00_pending_mutations:0001753875000000_0001_tabB',
+  mutationType: 'create',
+  recordId: 'game:g1',
+  data: { title: 'x' },
+};
+
+// Shared-tabs: followers commit outbox rows into the shared store and notify
+// the leader by id. The notify may be REPLAYED (failover buffering), so the
+// enqueue must be idempotent, and only the leader ever drains.
+describe('UpQueue.enqueueFromDatabase', () => {
+  function makeQueue(rows: Record<string, unknown[]>) {
+    const query = vi.fn(async (_sql: string, vars?: Record<string, unknown>) => {
+      const ids = (vars?.mutation_ids as unknown[]) ?? [];
+      const id = String(ids[0] ?? '');
+      return [rows[id] ?? []];
+    });
+    const local: any = { query };
+    return { queue: new UpQueue(local, makeLogger()), query };
+  }
+
+  it('loads exactly the forwarded row and enqueues it once', async () => {
+    const { queue } = makeQueue({ [ROW.id]: [ROW] });
+    await queue.enqueueFromDatabase(ROW.id);
+    expect(queue.size).toBe(1);
+  });
+
+  it('is idempotent: a replayed notify does not double-enqueue', async () => {
+    const { queue, query } = makeQueue({ [ROW.id]: [ROW] });
+    await queue.enqueueFromDatabase(ROW.id);
+    await queue.enqueueFromDatabase(ROW.id);
+    expect(queue.size).toBe(1);
+    // The second call short-circuits before even touching the store.
+    expect(query).toHaveBeenCalledTimes(1);
+  });
+
+  it('a row that never committed is a silent no-op (the app promise already rejected)', async () => {
+    const { queue } = makeQueue({});
+    await queue.enqueueFromDatabase(ROW.id);
+    expect(queue.size).toBe(0);
+  });
+
+  // The mock above replaces `local.query` wholesale, so on its own it proves
+  // nothing about the SQLite engine actually being able to RUN the statement.
+  // That gap shipped a bug: the emitted `SELECT * FROM $mutation_id` passed a
+  // single RecordId where the engine's `selectByIds` lowering expects an array,
+  // so it threw, the surrounding catch swallowed it at `error` level, and every
+  // follower mutation was silently never pushed. Drive the real translator.
+  it('discards a create with no payload instead of queueing a guaranteed failure', async () => {
+    // `processUpEvent` does `Object.keys(event.data)`, so this row can never be
+    // sent. Queued, it sat at the HEAD and blocked every later mutation for the
+    // whole app: `next()` re-queues on failure, so the backlog never drained.
+    const dropped: any[] = [];
+    const rows: Record<string, unknown[]> = {
+      [ROW.id]: [{ id: ROW.id, mutationType: 'create', recordId: 'comment:c1' }],
+    };
+    const query = vi.fn(async (sql: string, vars?: Record<string, unknown>) => {
+      if (sql.startsWith('DELETE')) return [[]];
+      const ids = (vars?.mutation_ids as unknown[]) ?? [];
+      return [rows[String(ids[0] ?? '')] ?? []];
+    });
+    const queue = new UpQueue({ query } as any, makeLogger(), (d) => dropped.push(d));
+
+    await queue.enqueueFromDatabase(ROW.id);
+
+    expect(queue.size).toBe(0);
+    expect(dropped).toHaveLength(1);
+    // And the row is deleted, so it cannot re-poison the next boot.
+    expect(query.mock.calls.some(([sql]) => String(sql).startsWith('DELETE'))).toBe(true);
+  });
+
+  it('loadFromDatabase drains the replayable rows even when one is unsendable', async () => {
+    const dropped: any[] = [];
+    const query = vi.fn(async (sql: string) => {
+      if (sql.startsWith('DELETE')) return [[]];
+      return [
+        [
+          { id: '_00_pending_mutations:a', mutationType: 'create', recordId: 'comment:c1' },
+          {
+            id: '_00_pending_mutations:b',
+            mutationType: 'update',
+            recordId: 'game:g1',
+            data: { title: 'x' },
+          },
+        ],
+      ];
+    });
+    const queue = new UpQueue({ query } as any, makeLogger(), (d) => dropped.push(d));
+
+    await queue.loadFromDatabase();
+
+    // The good row still loads; the poison one is dropped and reported.
+    expect(queue.size).toBe(1);
+    expect(dropped).toHaveLength(1);
+    expect(dropped[0].mutationId).toBe('_00_pending_mutations:a');
+  });
+
+  it('persists the payload for a create, so it can be replayed at all', () => {
+    // The create branch used to accept `dataVar` and ignore it, so the outbox
+    // row was the ONLY copy of a pending create and it carried no data.
+    const stmt = surql.createMutation('create', 'mid', 'id', 'data');
+    expect(stmt).toContain('data = $data');
+  });
+
+  it('deletes the row using the STORED id, not its escaped display form', async () => {
+    // Outbox ids contain `_` and `-`, so the store returns them escaped:
+    // `_00_pending_mutations:⟨1785…_0005_c960…⟩`. Parsing that verbatim and
+    // re-encoding escapes the brackets AGAIN, so the DELETE targeted
+    // `⟨⟨…\⟩⟩` and matched nothing: a mutation replayed from the store could
+    // never have its row removed, even after a SUCCESSFUL push, so it was
+    // re-sent on every boot.
+    const storedId = '_00_pending_mutations:⟨1785540216679_0005_c960958c-d6ed⟩';
+    const deletes: any[] = [];
+    const query = vi.fn(async (sql: string, vars?: Record<string, unknown>) => {
+      if (sql.startsWith('DELETE')) {
+        deletes.push(vars?.mutation_id);
+        return [[]];
+      }
+      return [[{ id: storedId, mutationType: 'create', recordId: 'comment:c1' }]];
+    });
+    const queue = new UpQueue({ query } as any, makeLogger(), () => {});
+
+    await queue.enqueueFromDatabase(storedId);
+
+    expect(deletes).toHaveLength(1);
+    const rid: any = deletes[0];
+    expect(String(rid.table)).toBe('_00_pending_mutations');
+    expect(rid.id).toBe('1785540216679_0005_c960958c-d6ed');
+    expect(String(rid.id)).not.toContain('⟨');
+  });
+
+  it('a push timeout classifies as network, so it retries instead of rolling back', () => {
+    const err = new Error('Mutation push timed out after 30000ms (create)');
+    expect(classifySyncError(err)).toBe('network');
+  });
+
+  it('emits a statement the SQLite engine can actually translate', async () => {
+    const { queue, query } = makeQueue({ [ROW.id]: [ROW] });
+    await queue.enqueueFromDatabase(ROW.id);
+
+    const [sql, vars] = query.mock.calls[0] as [string, Record<string, unknown>];
+    const translated = translateSurql(sql, vars);
+    const op: any = translated.ops[0];
+
+    expect(op.kind).toBe('selectByIds');
+    // The engine does `ids.length` then `ids.map(...)`: a non-array silently
+    // skips the empty-guard and then throws.
+    expect(Array.isArray(op.ids)).toBe(true);
+    expect(op.ids).toHaveLength(1);
+    expect(() => (op.ids as unknown[]).map((x) => x)).not.toThrow();
+  });
+});