@spooky-sync/core 0.0.1-canary.13 → 0.0.1-canary.130

package/src/modules/sync/engine.ts

@@ -1,9 +1,9 @@
-import { RecordId } from 'surrealdb';
-import { SchemaStructure } from '@spooky-sync/query-builder';
-import { RemoteDatabaseService } from '../../services/database/index';
-import { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
-import { RecordVersionDiff } from '../../types';
-import { Logger } from '../../services/logger/index';
+import type { RecordId } from 'surrealdb';
+import type { SchemaStructure } from '@spooky-sync/query-builder';
+import type { RemoteDatabaseService } from '../../services/database/index';
+import type { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
+import type { RecordVersionDiff } from '../../types';
+import type { Logger } from '../../services/logger/index';
 import { SyncEventTypes, createSyncEventSystem } from './events/index';
 import { encodeRecordId } from '../../utils/index';
 import { cleanRecord } from '../../utils/parser';
@@ -12,7 +12,7 @@ import { cleanRecord } from '../../utils/parser';
  * SyncEngine handles the core sync operations: fetching remote records,
  * caching them locally, and ingesting into DBSP.
  *
- * This is extracted from SpookySync to separate "how to sync" from "when to sync".
+ * This is extracted from Sp00kySync to separate "how to sync" from "when to sync".
  */
 export class SyncEngine {
   private logger: Logger;
@@ -24,7 +24,7 @@ export class SyncEngine {
     private schema: SchemaStructure,
     logger: Logger
   ) {
-    this.logger = logger.child({ service: 'SpookySync:SyncEngine' });
+    this.logger = logger.child({ service: 'Sp00kySync:SyncEngine' });
   }
 
   /**
@@ -32,7 +32,9 @@ export class SyncEngine {
    * Main entry point for sync operations.
    * Uses batch processing to minimize events emitted.
    */
-  async syncRecords(diff: RecordVersionDiff): Promise<void> {
+  async syncRecords(
+    diff: RecordVersionDiff
+  ): Promise<{ remoteFetchMs: number; stillRemoteIds: string[] }> {
     const { added, updated, removed } = diff;
 
     this.logger.debug(
@@ -40,55 +42,70 @@ export class SyncEngine {
         added,
         updated,
         removed,
-        Category: 'spooky-client::SyncEngine::syncRecords',
+        Category: 'sp00ky-client::SyncEngine::syncRecords',
       },
       'SyncEngine.syncRecords diff'
     );
 
-    // Handle removed records: verify they don't exist remotely before deleting locally
+    // Handle removed records: verify they don't exist remotely before deleting
+    // locally. Returns ids that LEFT the view's list_ref but still exist upstream
+    // (so they weren't deleted) — the caller converges localArray to drop them.
+    let stillRemoteIds: string[] = [];
     if (removed.length > 0) {
-      await this.handleRemovedRecords(removed);
+      stillRemoteIds = await this.handleRemovedRecords(removed);
     }
 
     // Fetch added/updated records from remote
     const toFetch = [...added, ...updated];
     const idsToFetch = toFetch.map((x) => x.id);
     if (idsToFetch.length === 0) {
-      return;
+      return { remoteFetchMs: 0, stillRemoteIds };
     }
 
+    // Build a version map from the diff (versions come from _00_list_ref)
+    const versionMap = new Map<string, number>();
+    for (const item of toFetch) {
+      versionMap.set(encodeRecordId(item.id), item.version);
+    }
+
+    // Fetch records from remote — avoid SELECT *, <subquery> FROM $param
+    // pattern which drops the * fields in SurrealDB v3 (known bug).
+    // Versions are already known from the diff's list_ref data.
+    const remoteFetchStart = performance.now();
     const [remoteResults] = await this.remote.query<[RecordWithId[]]>(
-      "SELECT (SELECT * FROM ONLY <record>$parent.id) AS record, (SELECT version FROM ONLY _spooky_version WHERE record_id = <record>$parent.id)['version'] as spooky_rv FROM $idsToFetch",
+      'SELECT * FROM $idsToFetch',
       { idsToFetch }
     );
-    console.log('remoteResults>', remoteResults);
+    const remoteFetchMs = performance.now() - remoteFetchStart;
+
     // Prepare batch for cache (which handles both DB and DBSP)
     const cacheBatch: CacheRecord[] = [];
 
-    for (const { spooky_rv, record } of remoteResults) {
+    for (const record of remoteResults) {
       if (!record?.id) {
         this.logger.warn(
           {
             record,
             idsToFetch,
-            Category: 'spooky-client::SyncEngine::syncRecords',
+            Category: 'sp00ky-client::SyncEngine::syncRecords',
           },
-          'Remote record has no id. Skipping record'
+          'Remote record has no id (possibly deleted). Skipping record'
         );
         continue;
       }
       const fullId = encodeRecordId(record.id);
       const table = record.id.table.toString();
       const isAdded = added.some((item) => encodeRecordId(item.id) === fullId);
+      const version = versionMap.get(fullId) ?? 0;
 
       const localVersion = this.cache.lookup(fullId);
-      if (localVersion && spooky_rv <= localVersion) {
+      if (localVersion && version <= localVersion) {
         this.logger.info(
           {
             recordId: fullId,
-            version: spooky_rv,
+            version,
             localVersion,
-            Category: 'spooky-client::SyncEngine::syncRecords',
+            Category: 'sp00ky-client::SyncEngine::syncRecords',
           },
           'Local version is higher than remote version. Skipping record'
         );
@@ -103,7 +120,7 @@ export class SyncEngine {
         table,
         op: isAdded ? 'CREATE' : 'UPDATE',
         record: cleanedRecord as RecordWithId,
-        version: spooky_rv,
+        version,
       });
     }
 
@@ -115,49 +132,96 @@ export class SyncEngine {
     this.events.emit(SyncEventTypes.RemoteDataIngested, {
       records: remoteResults,
     });
+
+    return { remoteFetchMs, stillRemoteIds };
   }
 
   /**
    * Handle records that exist locally but not in remote array.
+   *
+   * "Removed" here is a derived signal: the SSP's `_00_list_ref` array no
+   * longer references a record that exists locally. That can mean the row
+   * was genuinely deleted upstream — but it can also be a benign race
+   * (e.g. a record we just created hasn't propagated into the SSP's
+   * incantation list yet). Before deleting locally we verify against
+   * upstream SurrealDB: if the row still exists there, skip the delete.
+   *
+   * On verification failure we skip deletion too. Losing a stale local
+   * row to a later sync round is recoverable; deleting a fresh row that
+   * upstream still has is not.
    */
-  private async handleRemovedRecords(removed: RecordId[]): Promise<void> {
+  private async handleRemovedRecords(removed: RecordId[]): Promise<string[]> {
     this.logger.debug(
       {
         removed: removed.map((r) => r.toString()),
-        Category: 'spooky-client::SyncEngine::handleRemovedRecords',
+        Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
       },
       'Checking removed records'
     );
 
-    let existingRemoteIds = new Set<string>();
+    // Confirm which of the "removed" ids still exist remotely by selecting the
+    // records directly from the id array (the records ARE the FROM target).
+    //
+    // The exact query form matters on SurrealDB v3.x:
+    //   - `WHERE id IN $ids` is broken: record-id `IN` matches nothing, so every
+    //     removed id looked gone and live local records got deleted (a fresh
+    //     collection vanished mid-session). Do NOT use `IN`.
+    //   - `SELECT id FROM $ids` — a FIELD projection over a record-id ARRAY —
+    //     errors "Specify a database to use" on the deployed engine. The catch
+    //     below then swallowed it and skipped EVERY deletion, so nothing could be
+    //     deleted anywhere (games, comments, …). Do NOT project a field over the
+    //     array. (`SELECT * FROM $ids` works but pulls full records — wasteful.)
+    //   - `SELECT VALUE id FROM $ids` works: a flat array of ids with a NONE entry
+    //     for each id that no longer exists. We filter the NONE entries out; the
+    //     survivors are the ids still present upstream.
+    let existingRemoteIds: Set<string>;
     try {
-      const [existingRemote] = await this.remote.query<[{ id: RecordId }[]]>('SELECT id FROM $ids', {
-        ids: removed,
-      });
-      existingRemoteIds = new Set(existingRemote.map((r) => encodeRecordId(r.id)));
-    } catch {
-      // If remote check fails (e.g., SurrealDB parameter serialization issue),
-      // proceed with deletion — the caller has already determined these should be removed
-      this.logger.debug(
-        { Category: 'spooky-client::SyncEngine::handleRemovedRecords' },
-        'Remote existence check failed, proceeding with deletion'
+      const [existing] = await this.remote.query<[(RecordId | null | undefined)[]]>(
+        'SELECT VALUE id FROM $ids',
+        { ids: removed }
+      );
+      existingRemoteIds = new Set(
+        (existing ?? [])
+          .filter((id): id is RecordId => id != null)
+          .map((id) => encodeRecordId(id))
       );
+    } catch (err) {
+      // Verification failed. Skip deletion entirely — the next sync
+      // round re-derives the diff and we get another shot. The
+      // alternative (delete on uncertainty) destroys freshly-created
+      // rows when the SSP hasn't yet refreshed `_00_list_ref`.
+      this.logger.warn(
+        {
+          err,
+          removed: removed.map((r) => r.toString()),
+          Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
+        },
+        'Remote existence check failed, skipping deletion to avoid clobbering fresh data'
+      );
+      return [];
     }
 
+    // Ids that left the view's list_ref but STILL exist upstream — not deletions,
+    // just a view-membership change (e.g. a record whose field changed so it no
+    // longer matches the query). The caller drops these from `localArray` so the
+    // poll's diff stops re-flagging them every tick (the `job:` churn).
+    const stillRemoteIds: string[] = [];
     for (const recordId of removed) {
       const recordIdStr = encodeRecordId(recordId);
       if (!existingRemoteIds.has(recordIdStr)) {
         this.logger.debug(
           {
             recordId: recordIdStr,
-            Category: 'spooky-client::SyncEngine::handleRemovedRecords',
+            Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
           },
           'Deleting confirmed removed record'
         );
-
         // Use CacheModule to handle both local DB and DBSP deletion
         await this.cache.delete(recordId.table.name, recordIdStr);
+      } else {
+        stillRemoteIds.push(recordIdStr);
       }
     }
+    return stillRemoteIds;
   }
 }

package/src/modules/sync/events/index.ts

@@ -1,5 +1,6 @@
-import { createEventSystem, EventDefinition, EventSystem } from '../../../events/index';
-import { RecordVersionArray } from '../../../types';
+import type { EventDefinition, EventSystem } from '../../../events/index';
+import { createEventSystem } from '../../../events/index';
+import type { RecordVersionArray, SyncHealth } from '../../../types';
 
 export const SyncQueueEventTypes = {
   MutationEnqueued: 'MUTATION_ENQUEUED',
@@ -36,6 +37,7 @@ export const SyncEventTypes = {
   QueryUpdated: 'SYNC_QUERY_UPDATED',
   RemoteDataIngested: 'SYNC_REMOTE_DATA_INGESTED',
   MutationRolledBack: 'SYNC_MUTATION_ROLLED_BACK',
+  SyncHealthChanged: 'SYNC_HEALTH_CHANGED',
 } as const;
 
 export type SyncEventTypeMap = {
@@ -64,6 +66,10 @@ export type SyncEventTypeMap = {
       error: string;
     }
   >;
+  [SyncEventTypes.SyncHealthChanged]: EventDefinition<
+    typeof SyncEventTypes.SyncHealthChanged,
+    SyncHealth
+  >;
 };
 
 export type SyncEventSystem = EventSystem<SyncEventTypeMap>;
@@ -73,5 +79,6 @@ export function createSyncEventSystem(): SyncEventSystem {
     SyncEventTypes.QueryUpdated,
     SyncEventTypes.RemoteDataIngested,
     SyncEventTypes.MutationRolledBack,
+    SyncEventTypes.SyncHealthChanged,
   ]);
 }

package/src/modules/sync/queue/queue-down.ts

@@ -1,10 +1,11 @@
-import { LocalDatabaseService } from '../../../services/database/index';
+import type { LocalStore } from '../../../services/database/index';
+import type {
+  SyncQueueEventSystem} from '../events/index';
 import {
   createSyncQueueEventSystem,
-  SyncQueueEventSystem,
   SyncQueueEventTypes,
 } from '../events/index';
-import { Logger } from '../../../services/logger/index';
+import type { Logger } from '../../../services/logger/index';
 
 export type RegisterEvent = {
   type: 'register';
@@ -46,7 +47,7 @@ export class DownQueue {
   }
 
   constructor(
-    private local: LocalDatabaseService,
+    private local: LocalStore,
     logger: Logger
   ) {
     this._events = createSyncQueueEventSystem();
@@ -71,6 +72,12 @@ export class DownQueue {
     });
   }
 
+  /** Drop all queued events (bucket switch: old-bucket register/sync work is
+   *  re-derived after the switch; replaying it would target dead query rows). */
+  clear(): void {
+    this.queue = [];
+  }
+
   async next(fn: (event: DownEvent) => Promise<void>): Promise<void> {
     const event = this.queue.shift();
     if (event) {
@@ -78,7 +85,7 @@ export class DownQueue {
         await fn(event);
       } catch (error) {
         this.logger.error(
-          { error, event, Category: 'spooky-client::DownQueue::next' },
+          { error, event, Category: 'sp00ky-client::DownQueue::next' },
           'Failed to process query'
         );
         this.queue.unshift(event);

package/src/modules/sync/queue/queue-up.ts

@@ -1,13 +1,14 @@
-import { RecordId } from 'surrealdb';
-import { LocalDatabaseService } from '../../../services/database/index';
+import type { RecordId } from 'surrealdb';
+import type { LocalStore } from '../../../services/database/index';
+import type {
+  SyncQueueEventSystem} from '../events/index';
 import {
   createSyncQueueEventSystem,
-  SyncQueueEventSystem,
   SyncQueueEventTypes,
 } from '../events/index';
 import { parseRecordIdString, extractTablePart, classifySyncError } from '../../../utils/index';
-import { Logger } from '../../../services/logger/index';
-import { PushEventOptions } from '../../../events/index';
+import type { Logger } from '../../../services/logger/index';
+import type { PushEventOptions } from '../../../events/index';
 
 export type CreateEvent = {
   type: 'create';
@@ -51,7 +52,7 @@ export class UpQueue {
   }
 
   constructor(
-    private local: LocalDatabaseService,
+    private local: LocalStore,
     logger: Logger
   ) {
     this._events = createSyncQueueEventSystem();
@@ -104,6 +105,20 @@ export class UpQueue {
     this.debouncedMutations.set(key, { timer, firstBeforeRecord });
   }
 
+  /**
+   * Cancel all pending debounce timers WITHOUT enqueueing their events. Used
+   * on local-bucket switches: the mutation's `_00_pending_mutations` row was
+   * already persisted at mutation time, so dropping the in-memory push only
+   * defers it to that bucket's next `loadFromDatabase` — it must NOT be pushed
+   * now, the remote session already belongs to the next user.
+   */
+  clearDebounceTimers(): void {
+    for (const { timer } of this.debouncedMutations.values()) {
+      clearTimeout(timer);
+    }
+    this.debouncedMutations.clear();
+  }
+
   async next(fn: (event: UpEvent) => Promise<void>, onRollback?: RollbackCallback): Promise<void> {
     const event = this.queue.shift();
     if (event) {
@@ -114,7 +129,7 @@ export class UpQueue {
 
         if (errorType === 'network') {
           this.logger.error(
-            { error, event, Category: 'spooky-client::UpQueue::next' },
+            { error, event, Category: 'sp00ky-client::UpQueue::next' },
             'Network error processing mutation, re-queuing'
           );
           this.queue.unshift(event);
@@ -123,14 +138,14 @@ export class UpQueue {
 
         // Application error — rollback instead of re-queuing
         this.logger.error(
-          { error, event, Category: 'spooky-client::UpQueue::next' },
+          { error, event, Category: 'sp00ky-client::UpQueue::next' },
           'Application error processing mutation, rolling back'
         );
         try {
           await this.removeEventFromDatabase(event.mutation_id);
         } catch (removeError) {
           this.logger.error(
-            { error: removeError, event, Category: 'spooky-client::UpQueue::next' },
+            { error: removeError, event, Category: 'sp00ky-client::UpQueue::next' },
             'Failed to remove rolled-back mutation from database'
           );
         }
@@ -139,7 +154,7 @@ export class UpQueue {
             await onRollback(event, error instanceof Error ? error : new Error(String(error)));
           } catch (rollbackError) {
             this.logger.error(
-              { error: rollbackError, event, Category: 'spooky-client::UpQueue::next' },
+              { error: rollbackError, event, Category: 'sp00ky-client::UpQueue::next' },
               'Rollback handler failed'
             );
           }
@@ -154,7 +169,7 @@ export class UpQueue {
         await this.removeEventFromDatabase(event.mutation_id);
       } catch (error) {
         this.logger.error(
-          { error, event, Category: 'spooky-client::UpQueue::next' },
+          { error, event, Category: 'sp00ky-client::UpQueue::next' },
           'Failed to remove mutation from database after successful processing'
         );
       }
@@ -172,7 +187,7 @@ export class UpQueue {
   async loadFromDatabase() {
     try {
       const [records] = await this.local.query<any>(
-        `SELECT * FROM _spooky_pending_mutations ORDER BY created_at ASC`
+        `SELECT * FROM _00_pending_mutations ORDER BY created_at ASC`
       );
 
       this.queue = records
@@ -205,7 +220,7 @@ export class UpQueue {
                 {
                   mutationType: r.mutationType,
                   record: r,
-                  Category: 'spooky-client::UpQueue::loadFromDatabase',
+                  Category: 'sp00ky-client::UpQueue::loadFromDatabase',
                 },
                 'Unknown mutation type'
               );
@@ -215,7 +230,7 @@ export class UpQueue {
         .filter((e: UpEvent | null): e is UpEvent => e !== null);
     } catch (error) {
       this.logger.error(
-        { error, Category: 'spooky-client::UpQueue::loadFromDatabase' },
+        { error, Category: 'sp00ky-client::UpQueue::loadFromDatabase' },
         'Failed to load pending mutations from database'
       );
     }

package/src/modules/sync/scheduler.pause.test.ts

@@ -0,0 +1,109 @@
+import { describe, it, expect, vi } from 'vitest';
+import { SyncScheduler } from './scheduler';
+import type { UpQueue, DownQueue, UpEvent } from './queue/index';
+
+// `SyncScheduler.pause()` is the drain step of a local-bucket switch: it must
+// (a) let an IN-FLIGHT queue item finish — including its outbox-row delete,
+// which has to land in the old bucket — and only then resolve, and (b) refuse
+// to start new rounds until `resume()`. The pause point is between items.
+
+const silentLogger = {
+  child: () => silentLogger,
+  debug: () => {},
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+} as any;
+
+function makeQueues(items: UpEvent[]) {
+  const upQueue = {
+    queue: [...items],
+    get size() {
+      return this.queue.length;
+    },
+    events: { subscribe: () => {} },
+    loadFromDatabase: async () => {},
+    async next(fn: (event: UpEvent) => Promise<void>) {
+      const event = this.queue.shift();
+      if (event) await fn(event);
+    },
+  } as unknown as UpQueue & { queue: UpEvent[] };
+
+  const downQueue = {
+    queue: [] as unknown[],
+    get size() {
+      return this.queue.length;
+    },
+    events: { subscribe: () => {} },
+    async next() {},
+    clear() {
+      this.queue = [];
+    },
+  } as unknown as DownQueue;
+
+  return { upQueue, downQueue };
+}
+
+const upEvent = (n: number) => ({ type: 'delete', mutation_id: n, record_id: n }) as unknown as UpEvent;
+
+describe('SyncScheduler.pause', () => {
+  it('waits for the in-flight item (push + outbox delete) before resolving', async () => {
+    const { upQueue, downQueue } = makeQueues([upEvent(1), upEvent(2)]);
+    let release!: () => void;
+    const gate = new Promise<void>((r) => (release = r));
+    const processed: unknown[] = [];
+    const scheduler = new SyncScheduler(
+      upQueue,
+      downQueue,
+      async (event) => {
+        await gate; // simulate a slow remote push
+        processed.push(event);
+      },
+      async () => {},
+      silentLogger
+    );
+
+    const round = scheduler.syncUp();
+    const pauseDone = vi.fn();
+    const pause = scheduler.pause().then(pauseDone);
+
+    // The in-flight item hasn't finished — pause must not have resolved.
+    await Promise.resolve();
+    expect(pauseDone).not.toHaveBeenCalled();
+
+    release();
+    await pause;
+    await round;
+    // Exactly the in-flight item completed; the second stayed queued.
+    expect(processed).toHaveLength(1);
+    expect(upQueue.size).toBe(1);
+  });
+
+  it('refuses new rounds while paused and drains them on resume', async () => {
+    const { upQueue, downQueue } = makeQueues([upEvent(1)]);
+    const processed: unknown[] = [];
+    const scheduler = new SyncScheduler(
+      upQueue,
+      downQueue,
+      async (event) => {
+        processed.push(event);
+      },
+      async () => {},
+      silentLogger
+    );
+
+    await scheduler.pause();
+    await scheduler.syncUp();
+    expect(processed).toHaveLength(0);
+
+    scheduler.resume();
+    await vi.waitFor(() => expect(processed).toHaveLength(1));
+    expect(upQueue.size).toBe(0);
+  });
+
+  it('resolves immediately when nothing is in flight', async () => {
+    const { upQueue, downQueue } = makeQueues([]);
+    const scheduler = new SyncScheduler(upQueue, downQueue, async () => {}, async () => {}, silentLogger);
+    await expect(scheduler.pause()).resolves.toBeUndefined();
+  });
+});