@spooky-sync/core 0.0.1-canary.16 → 0.0.1-canary.160

package/src/modules/sync/queue/queue-down.ts

@@ -1,10 +1,11 @@
-import { LocalDatabaseService } from '../../../services/database/index';
+import type { LocalStore } from '../../../services/database/index';
+import type {
+  SyncQueueEventSystem} from '../events/index';
 import {
   createSyncQueueEventSystem,
-  SyncQueueEventSystem,
   SyncQueueEventTypes,
 } from '../events/index';
-import { Logger } from '../../../services/logger/index';
+import type { Logger } from '../../../services/logger/index';
 
 export type RegisterEvent = {
   type: 'register';
@@ -46,7 +47,7 @@ export class DownQueue {
   }
 
   constructor(
-    private local: LocalDatabaseService,
+    private local: LocalStore,
     logger: Logger
   ) {
     this._events = createSyncQueueEventSystem();
@@ -71,6 +72,12 @@ export class DownQueue {
     });
   }
 
+  /** Drop all queued events (bucket switch: old-bucket register/sync work is
+   *  re-derived after the switch; replaying it would target dead query rows). */
+  clear(): void {
+    this.queue = [];
+  }
+
   async next(fn: (event: DownEvent) => Promise<void>): Promise<void> {
     const event = this.queue.shift();
     if (event) {
@@ -78,7 +85,7 @@ export class DownQueue {
         await fn(event);
       } catch (error) {
         this.logger.error(
-          { error, event, Category: 'spooky-client::DownQueue::next' },
+          { error, event, Category: 'sp00ky-client::DownQueue::next' },
           'Failed to process query'
         );
         this.queue.unshift(event);

package/src/modules/sync/queue/queue-up.forwarded.test.ts

@@ -0,0 +1,137 @@
+import { describe, it, expect, vi } from 'vitest';
+import { UpQueue } from './queue-up';
+import { translateSurql } from '../../../services/database/surql-translate';
+import { surql, classifySyncError } from '../../../utils/index';
+
+function makeLogger(): any {
+  const noop = () => {};
+  const l: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  l.child = () => l;
+  return l;
+}
+
+const ROW = {
+  id: '_00_pending_mutations:0001753875000000_0001_tabB',
+  mutationType: 'create',
+  recordId: 'game:g1',
+  data: { title: 'x' },
+};
+
+// Shared-tabs: followers commit outbox rows into the shared store and notify
+// the leader by id. The notify may be REPLAYED (failover buffering), so the
+// enqueue must be idempotent, and only the leader ever drains.
+describe('UpQueue.enqueueFromDatabase', () => {
+  function makeQueue(rows: Record<string, unknown[]>) {
+    const query = vi.fn(async (_sql: string, vars?: Record<string, unknown>) => {
+      const ids = (vars?.mutation_ids as unknown[]) ?? [];
+      const id = String(ids[0] ?? '');
+      return [rows[id] ?? []];
+    });
+    const local: any = { query };
+    return { queue: new UpQueue(local, makeLogger()), query };
+  }
+
+  it('loads exactly the forwarded row and enqueues it once', async () => {
+    const { queue } = makeQueue({ [ROW.id]: [ROW] });
+    await queue.enqueueFromDatabase(ROW.id);
+    expect(queue.size).toBe(1);
+  });
+
+  it('is idempotent: a replayed notify does not double-enqueue', async () => {
+    const { queue, query } = makeQueue({ [ROW.id]: [ROW] });
+    await queue.enqueueFromDatabase(ROW.id);
+    await queue.enqueueFromDatabase(ROW.id);
+    expect(queue.size).toBe(1);
+    // The second call short-circuits before even touching the store.
+    expect(query).toHaveBeenCalledTimes(1);
+  });
+
+  it('a row that never committed is a silent no-op (the app promise already rejected)', async () => {
+    const { queue } = makeQueue({});
+    await queue.enqueueFromDatabase(ROW.id);
+    expect(queue.size).toBe(0);
+  });
+
+  // The mock above replaces `local.query` wholesale, so on its own it proves
+  // nothing about the SQLite engine actually being able to RUN the statement.
+  // That gap shipped a bug: the emitted `SELECT * FROM $mutation_id` passed a
+  // single RecordId where the engine's `selectByIds` lowering expects an array,
+  // so it threw, the surrounding catch swallowed it at `error` level, and every
+  // follower mutation was silently never pushed. Drive the real translator.
+  it('discards a create with no payload instead of queueing a guaranteed failure', async () => {
+    // `processUpEvent` does `Object.keys(event.data)`, so this row can never be
+    // sent. Queued, it sat at the HEAD and blocked every later mutation for the
+    // whole app: `next()` re-queues on failure, so the backlog never drained.
+    const dropped: any[] = [];
+    const rows: Record<string, unknown[]> = {
+      [ROW.id]: [{ id: ROW.id, mutationType: 'create', recordId: 'comment:c1' }],
+    };
+    const query = vi.fn(async (sql: string, vars?: Record<string, unknown>) => {
+      if (sql.startsWith('DELETE')) return [[]];
+      const ids = (vars?.mutation_ids as unknown[]) ?? [];
+      return [rows[String(ids[0] ?? '')] ?? []];
+    });
+    const queue = new UpQueue({ query } as any, makeLogger(), (d) => dropped.push(d));
+
+    await queue.enqueueFromDatabase(ROW.id);
+
+    expect(queue.size).toBe(0);
+    expect(dropped).toHaveLength(1);
+    // And the row is deleted, so it cannot re-poison the next boot.
+    expect(query.mock.calls.some(([sql]) => String(sql).startsWith('DELETE'))).toBe(true);
+  });
+
+  it('loadFromDatabase drains the replayable rows even when one is unsendable', async () => {
+    const dropped: any[] = [];
+    const query = vi.fn(async (sql: string) => {
+      if (sql.startsWith('DELETE')) return [[]];
+      return [
+        [
+          { id: '_00_pending_mutations:a', mutationType: 'create', recordId: 'comment:c1' },
+          {
+            id: '_00_pending_mutations:b',
+            mutationType: 'update',
+            recordId: 'game:g1',
+            data: { title: 'x' },
+          },
+        ],
+      ];
+    });
+    const queue = new UpQueue({ query } as any, makeLogger(), (d) => dropped.push(d));
+
+    await queue.loadFromDatabase();
+
+    // The good row still loads; the poison one is dropped and reported.
+    expect(queue.size).toBe(1);
+    expect(dropped).toHaveLength(1);
+    expect(dropped[0].mutationId).toBe('_00_pending_mutations:a');
+  });
+
+  it('persists the payload for a create, so it can be replayed at all', () => {
+    // The create branch used to accept `dataVar` and ignore it, so the outbox
+    // row was the ONLY copy of a pending create and it carried no data.
+    const stmt = surql.createMutation('create', 'mid', 'id', 'data');
+    expect(stmt).toContain('data = $data');
+  });
+
+  it('a push timeout classifies as network, so it retries instead of rolling back', () => {
+    const err = new Error('Mutation push timed out after 30000ms (create)');
+    expect(classifySyncError(err)).toBe('network');
+  });
+
+  it('emits a statement the SQLite engine can actually translate', async () => {
+    const { queue, query } = makeQueue({ [ROW.id]: [ROW] });
+    await queue.enqueueFromDatabase(ROW.id);
+
+    const [sql, vars] = query.mock.calls[0] as [string, Record<string, unknown>];
+    const translated = translateSurql(sql, vars);
+    const op: any = translated.ops[0];
+
+    expect(op.kind).toBe('selectByIds');
+    // The engine does `ids.length` then `ids.map(...)`: a non-array silently
+    // skips the empty-guard and then throws.
+    expect(Array.isArray(op.ids)).toBe(true);
+    expect(op.ids).toHaveLength(1);
+    expect(() => (op.ids as unknown[]).map((x) => x)).not.toThrow();
+  });
+});

package/src/modules/sync/queue/queue-up.ts

@@ -1,13 +1,15 @@
-import { RecordId } from 'surrealdb';
-import { LocalDatabaseService } from '../../../services/database/index';
+import type { RecordId } from 'surrealdb';
+import type { LocalStore } from '../../../services/database/index';
+import type { SyncQueueEventSystem } from '../events/index';
+import { createSyncQueueEventSystem, SyncQueueEventTypes } from '../events/index';
 import {
-  createSyncQueueEventSystem,
-  SyncQueueEventSystem,
-  SyncQueueEventTypes,
-} from '../events/index';
-import { parseRecordIdString, extractTablePart, classifySyncError } from '../../../utils/index';
-import { Logger } from '../../../services/logger/index';
-import { PushEventOptions } from '../../../events/index';
+  parseRecordIdString,
+  extractTablePart,
+  classifySyncError,
+  encodeRecordId,
+} from '../../../utils/index';
+import type { Logger } from '../../../services/logger/index';
+import type { PushEventOptions } from '../../../events/index';
 
 export type CreateEvent = {
   type: 'create';
@@ -40,25 +42,80 @@ export type UpEvent = CreateEvent | UpdateEvent | DeleteEvent;
 
 export type RollbackCallback = (event: UpEvent, error: Error) => Promise<void>;
 
+/**
+ * A pending mutation that can never be sent, so it was discarded instead of
+ * being retried forever at the head of the queue.
+ */
+export type DroppedMutation = {
+  mutationId: string;
+  recordId?: string;
+  mutationType?: string;
+  reason: string;
+};
+
+export type DroppedCallback = (dropped: DroppedMutation) => void;
+
 export class UpQueue {
   private queue: UpEvent[] = [];
   private _events: SyncQueueEventSystem;
   private logger: Logger;
-  private debouncedMutations: Map<string, { timer: any; firstBeforeRecord?: Record<string, unknown> }>;
+  private debouncedMutations: Map<
+    string,
+    { timer: any; firstBeforeRecord?: Record<string, unknown> }
+  >;
 
   get events(): SyncQueueEventSystem {
     return this._events;
   }
 
   constructor(
-    private local: LocalDatabaseService,
-    logger: Logger
+    private local: LocalStore,
+    logger: Logger,
+    private onDropped?: DroppedCallback
   ) {
     this._events = createSyncQueueEventSystem();
     this.logger = logger.child({ service: 'UpQueue' });
     this.debouncedMutations = new Map();
   }
 
+  /**
+   * Discard an outbox row that can never be replayed and report it.
+   *
+   * Silently skipping such a row leaves it in the store to be re-read on every
+   * boot; leaving it QUEUED is worse, since `next()` re-queues it on failure and
+   * one unsendable row then blocks every later mutation for the whole app. A
+   * lost write must also be loud: this is the only signal a caller gets.
+   */
+  private async discardUnreplayable(row: any, reason: string): Promise<void> {
+    const mutationId = typeof row?.id === 'string' ? row.id : encodeRecordId(row?.id);
+    this.logger.error(
+      {
+        mutationId,
+        recordId: row?.recordId,
+        mutationType: row?.mutationType,
+        reason,
+        Category: 'sp00ky-client::UpQueue::discardUnreplayable',
+      },
+      'Discarding an unsendable pending mutation'
+    );
+    try {
+      await this.local.query(`DELETE $mutation_id`, {
+        mutation_id: parseRecordIdString(mutationId),
+      });
+    } catch (error) {
+      this.logger.error(
+        { error, mutationId, Category: 'sp00ky-client::UpQueue::discardUnreplayable' },
+        'Failed to delete an unsendable pending mutation'
+      );
+    }
+    this.onDropped?.({
+      mutationId,
+      recordId: row?.recordId,
+      mutationType: row?.mutationType,
+      reason,
+    });
+  }
+
   get size(): number {
     return this.queue.length;
   }
@@ -104,6 +161,20 @@ export class UpQueue {
     this.debouncedMutations.set(key, { timer, firstBeforeRecord });
   }
 
+  /**
+   * Cancel all pending debounce timers WITHOUT enqueueing their events. Used
+   * on local-bucket switches: the mutation's `_00_pending_mutations` row was
+   * already persisted at mutation time, so dropping the in-memory push only
+   * defers it to that bucket's next `loadFromDatabase` — it must NOT be pushed
+   * now, the remote session already belongs to the next user.
+   */
+  clearDebounceTimers(): void {
+    for (const { timer } of this.debouncedMutations.values()) {
+      clearTimeout(timer);
+    }
+    this.debouncedMutations.clear();
+  }
+
   async next(fn: (event: UpEvent) => Promise<void>, onRollback?: RollbackCallback): Promise<void> {
     const event = this.queue.shift();
     if (event) {
@@ -114,7 +185,7 @@ export class UpQueue {
 
         if (errorType === 'network') {
           this.logger.error(
-            { error, event, Category: 'spooky-client::UpQueue::next' },
+            { error, event, Category: 'sp00ky-client::UpQueue::next' },
             'Network error processing mutation, re-queuing'
           );
           this.queue.unshift(event);
@@ -123,14 +194,14 @@ export class UpQueue {
 
         // Application error — rollback instead of re-queuing
         this.logger.error(
-          { error, event, Category: 'spooky-client::UpQueue::next' },
+          { error, event, Category: 'sp00ky-client::UpQueue::next' },
           'Application error processing mutation, rolling back'
         );
         try {
           await this.removeEventFromDatabase(event.mutation_id);
         } catch (removeError) {
           this.logger.error(
-            { error: removeError, event, Category: 'spooky-client::UpQueue::next' },
+            { error: removeError, event, Category: 'sp00ky-client::UpQueue::next' },
             'Failed to remove rolled-back mutation from database'
           );
         }
@@ -139,7 +210,7 @@ export class UpQueue {
             await onRollback(event, error instanceof Error ? error : new Error(String(error)));
           } catch (rollbackError) {
             this.logger.error(
-              { error: rollbackError, event, Category: 'spooky-client::UpQueue::next' },
+              { error: rollbackError, event, Category: 'sp00ky-client::UpQueue::next' },
               'Rollback handler failed'
             );
           }
@@ -154,7 +225,7 @@ export class UpQueue {
         await this.removeEventFromDatabase(event.mutation_id);
       } catch (error) {
         this.logger.error(
-          { error, event, Category: 'spooky-client::UpQueue::next' },
+          { error, event, Category: 'sp00ky-client::UpQueue::next' },
           'Failed to remove mutation from database after successful processing'
         );
       }
@@ -169,55 +240,125 @@ export class UpQueue {
     return this.local.query(`DELETE $mutation_id`, { mutation_id });
   }
 
+  /**
+   * Shared-tabs: a follower committed an outbox row into the shared store and
+   * notified the leader; enqueue exactly that row. Idempotent: a replayed
+   * notify (failover) or a row already queued is a no-op, and a row that never
+   * committed simply is not found (the follower's own write promise already
+   * rejected in that case).
+   */
+  async enqueueFromDatabase(mutationId: string): Promise<void> {
+    if (this.queue.some((e) => encodeUpEventId(e) === mutationId)) return;
+    try {
+      // ARRAY param, matching SyncEngine's `SELECT * FROM $idsToFetch`. A bare
+      // `FROM $singleRecordId` looks fine against SurrealDB but the SQLite
+      // engine lowers any `FROM $param` to `selectByIds` and calls `.map` on
+      // the param (surql-translate.ts, SqliteCacheEngine.selectByIds), so a
+      // single RecordId threw. The throw landed in the catch below, which logs
+      // at `error` — invisible to an app running `logLevel: 'fatal'` — so every
+      // forwarded mutation was silently dropped: the follower's optimistic
+      // write stuck locally, was never pushed, and the next down-sync reverted it.
+      const [records] = await this.local.query<any>(`SELECT * FROM $mutation_ids`, {
+        mutation_ids: [parseRecordIdString(mutationId)],
+      });
+      const row = Array.isArray(records) ? records[0] : undefined;
+      if (!row) return;
+      const event = rowToUpEvent(row, this.logger);
+      if (event) {
+        this.addToQueue(event);
+        return;
+      }
+      await this.discardUnreplayable(row, 'forwarded mutation is not replayable');
+    } catch (error) {
+      this.logger.error(
+        { error, mutationId, Category: 'sp00ky-client::UpQueue::enqueueFromDatabase' },
+        'Failed to load a forwarded mutation'
+      );
+      this.onDropped?.({
+        mutationId,
+        reason: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+
   async loadFromDatabase() {
     try {
+      // ORDER BY id: mutation ids are zero-padded-timestamp-prefixed (see
+      // modules/data/mutation-id.ts), so lexicographic id order IS chronological
+      // order. (The previous created_at was never a defined field on this
+      // table, so the old ordering was undefined.)
       const [records] = await this.local.query<any>(
-        `SELECT * FROM _spooky_pending_mutations ORDER BY created_at ASC`
+        `SELECT * FROM _00_pending_mutations ORDER BY id ASC`
       );
 
-      this.queue = records
-        .map((r: any): UpEvent | null => {
-          switch (r.mutationType) {
-            case 'create':
-              return {
-                type: 'create',
-                mutation_id: parseRecordIdString(r.id),
-                record_id: parseRecordIdString(r.recordId),
-                data: r.data,
-                tableName: extractTablePart(r.recordId),
-              };
-            case 'update':
-              return {
-                type: 'update',
-                mutation_id: parseRecordIdString(r.id),
-                record_id: parseRecordIdString(r.recordId),
-                data: r.data,
-                beforeRecord: r.beforeRecord,
-              };
-            case 'delete':
-              return {
-                type: 'delete',
-                mutation_id: parseRecordIdString(r.id),
-                record_id: parseRecordIdString(r.recordId),
-              };
-            default:
-              this.logger.warn(
-                {
-                  mutationType: r.mutationType,
-                  record: r,
-                  Category: 'spooky-client::UpQueue::loadFromDatabase',
-                },
-                'Unknown mutation type'
-              );
-              return null;
-          }
-        })
-        .filter((e: UpEvent | null): e is UpEvent => e !== null);
+      const loaded: UpEvent[] = [];
+      const unreplayable: any[] = [];
+      for (const row of records as any[]) {
+        const event = rowToUpEvent(row, this.logger);
+        if (event) loaded.push(event);
+        else unreplayable.push(row);
+      }
+      this.queue = loaded;
+      // Drop them AFTER the queue is populated: one unsendable row must not
+      // stop the rest of the backlog from draining, and leaving it in the store
+      // would just re-poison the next boot.
+      for (const row of unreplayable) {
+        await this.discardUnreplayable(row, 'pending mutation is not replayable');
+      }
     } catch (error) {
       this.logger.error(
-        { error, Category: 'spooky-client::UpQueue::loadFromDatabase' },
+        { error, Category: 'sp00ky-client::UpQueue::loadFromDatabase' },
         'Failed to load pending mutations from database'
       );
     }
   }
 }
+
+/** The stable string id of an event's outbox row. */
+function encodeUpEventId(event: UpEvent): string {
+  return encodeRecordId(event.mutation_id);
+}
+
+/** Materialize one `_00_pending_mutations` row into an UpEvent. */
+function rowToUpEvent(r: any, logger: Logger): UpEvent | null {
+  switch (r.mutationType) {
+    case 'create':
+      // `processUpEvent` does `Object.keys(event.data)`, so a create with no
+      // payload throws before it reaches the network and can never succeed.
+      // Rows written before the create branch of `surql.createMutation`
+      // persisted `data` are exactly that, so refuse them here instead of
+      // queueing a guaranteed failure.
+      if (r.data === undefined || r.data === null) return null;
+      return {
+        type: 'create',
+        mutation_id: parseRecordIdString(r.id),
+        record_id: parseRecordIdString(r.recordId),
+        data: r.data,
+        tableName: extractTablePart(r.recordId),
+      };
+    case 'update':
+      return {
+        type: 'update',
+        mutation_id: parseRecordIdString(r.id),
+        record_id: parseRecordIdString(r.recordId),
+        data: r.data,
+        beforeRecord: r.beforeRecord,
+      };
+    case 'delete':
+      return {
+        type: 'delete',
+        mutation_id: parseRecordIdString(r.id),
+        record_id: parseRecordIdString(r.recordId),
+      };
+    default:
+      logger.warn(
+        {
+          mutationType: r.mutationType,
+          record: r,
+          Category: 'sp00ky-client::UpQueue::rowToUpEvent',
+        },
+        'Unknown mutation type'
+      );
+      return null;
+  }
+}

package/src/modules/sync/scheduler.pause.test.ts

@@ -0,0 +1,109 @@
+import { describe, it, expect, vi } from 'vitest';
+import { SyncScheduler } from './scheduler';
+import type { UpQueue, DownQueue, UpEvent } from './queue/index';
+
+// `SyncScheduler.pause()` is the drain step of a local-bucket switch: it must
+// (a) let an IN-FLIGHT queue item finish — including its outbox-row delete,
+// which has to land in the old bucket — and only then resolve, and (b) refuse
+// to start new rounds until `resume()`. The pause point is between items.
+
+const silentLogger = {
+  child: () => silentLogger,
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any;
+
+function makeQueues(items: UpEvent[]) {
+  const upQueue = {
+    queue: [...items],
+    get size() {
+      return this.queue.length;
+    },
+    events: { subscribe: () => {} },
+    loadFromDatabase: async () => {},
+    async next(fn: (event: UpEvent) => Promise<void>) {
+      const event = this.queue.shift();
+      if (event) await fn(event);
+    },
+  } as unknown as UpQueue & { queue: UpEvent[] };
+
+  const downQueue = {
+    queue: [] as unknown[],
+    get size() {
+      return this.queue.length;
+    },
+    events: { subscribe: () => {} },
+    async next() {},
+    clear() {
+      this.queue = [];
+    },
+  } as unknown as DownQueue;
+
+  return { upQueue, downQueue };
+}
+
+const upEvent = (n: number) => ({ type: 'delete', mutation_id: n, record_id: n }) as unknown as UpEvent;
+
+describe('SyncScheduler.pause', () => {
+  it('waits for the in-flight item (push + outbox delete) before resolving', async () => {
+    const { upQueue, downQueue } = makeQueues([upEvent(1), upEvent(2)]);
+    let release!: () => void;
+    const gate = new Promise<void>((r) => (release = r));
+    const processed: unknown[] = [];
+    const scheduler = new SyncScheduler(
+      upQueue,
+      downQueue,
+      async (event) => {
+        await gate; // simulate a slow remote push
+        processed.push(event);
+      },
+      async () => {},
+      silentLogger
+    );
+
+    const round = scheduler.syncUp();
+    const pauseDone = vi.fn();
+    const pause = scheduler.pause().then(pauseDone);
+
+    // The in-flight item hasn't finished — pause must not have resolved.
+    await Promise.resolve();
+    expect(pauseDone).not.toHaveBeenCalled();
+
+    release();
+    await pause;
+    await round;
+    // Exactly the in-flight item completed; the second stayed queued.
+    expect(processed).toHaveLength(1);
+    expect(upQueue.size).toBe(1);
+  });
+
+  it('refuses new rounds while paused and drains them on resume', async () => {
+    const { upQueue, downQueue } = makeQueues([upEvent(1)]);
+    const processed: unknown[] = [];
+    const scheduler = new SyncScheduler(
+      upQueue,
+      downQueue,
+      async (event) => {
+        processed.push(event);
+      },
+      async () => {},
+      silentLogger
+    );
+
+    await scheduler.pause();
+    await scheduler.syncUp();
+    expect(processed).toHaveLength(0);
+
+    scheduler.resume();
+    await vi.waitFor(() => expect(processed).toHaveLength(1));
+    expect(upQueue.size).toBe(0);
+  });
+
+  it('resolves immediately when nothing is in flight', async () => {
+    const { upQueue, downQueue } = makeQueues([]);
+    const scheduler = new SyncScheduler(upQueue, downQueue, async () => {}, async () => {}, silentLogger);
+    await expect(scheduler.pause()).resolves.toBeUndefined();
+  });
+});