@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.210

package/src/modules/ref-tables.test.ts

@@ -0,0 +1,91 @@
+import { describe, it, expect } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { ANON_USER_ID, bucketIdForUser, listRefTableFor, sanitizeUserId } from './ref-tables';
+
+describe('listRefTableFor', () => {
+  it('returns global table in single mode regardless of user', () => {
+    expect(listRefTableFor('single', 'user:abc')).toBe('_00_list_ref');
+    expect(listRefTableFor('single', null)).toBe('_00_list_ref');
+    expect(listRefTableFor('single', undefined)).toBe('_00_list_ref');
+  });
+
+  it('returns per-user table in dedicated mode with valid user id', () => {
+    expect(listRefTableFor('dedicated', 'user:abc')).toBe(
+      '_00_list_ref_user_abc'
+    );
+  });
+
+  it('accepts a RecordId object in dedicated mode', () => {
+    const rid = new RecordId('user', 'def');
+    expect(listRefTableFor('dedicated', rid)).toBe('_00_list_ref_user_def');
+  });
+
+  it('falls back to global in dedicated mode when user id is missing', () => {
+    expect(listRefTableFor('dedicated', null)).toBe('_00_list_ref');
+    expect(listRefTableFor('dedicated', undefined)).toBe('_00_list_ref');
+  });
+
+  it('falls back to global in dedicated mode when user id has invalid chars', () => {
+    // SurrealDB table identifiers only accept alphanumerics + underscore.
+    expect(listRefTableFor('dedicated', 'user:abc-with-dash')).toBe(
+      '_00_list_ref'
+    );
+    expect(listRefTableFor('dedicated', 'user:abc.dot')).toBe('_00_list_ref');
+  });
+
+  it('routes the anon sentinel to the shared anon table in both modes', () => {
+    expect(listRefTableFor('dedicated', ANON_USER_ID)).toBe('_00_list_ref_anon');
+    expect(listRefTableFor('single', ANON_USER_ID)).toBe('_00_list_ref_anon');
+    // A real user whose id sanitizes to "anon" still carries the user: prefix,
+    // so it never collides with the bare sentinel.
+    expect(listRefTableFor('dedicated', 'user:anon')).toBe(
+      '_00_list_ref_user_anon'
+    );
+  });
+});
+
+describe('sanitizeUserId', () => {
+  it('strips the user: prefix', () => {
+    expect(sanitizeUserId('user:abc123')).toBe('abc123');
+  });
+
+  it('accepts plain ids without the user: prefix', () => {
+    expect(sanitizeUserId('abc123')).toBe('abc123');
+  });
+
+  it('accepts RecordId objects', () => {
+    expect(sanitizeUserId(new RecordId('user', 'xyz'))).toBe('xyz');
+  });
+
+  it('returns null for invalid id shapes', () => {
+    expect(sanitizeUserId(null)).toBeNull();
+    expect(sanitizeUserId(undefined)).toBeNull();
+    expect(sanitizeUserId('user:')).toBeNull();
+    expect(sanitizeUserId('user:has-dash')).toBeNull();
+  });
+});
+
+describe('bucketIdForUser', () => {
+  it('routes signed-out sessions to the anon bucket', () => {
+    expect(bucketIdForUser(null)).toBe(ANON_USER_ID);
+    expect(bucketIdForUser(undefined)).toBe(ANON_USER_ID);
+    expect(bucketIdForUser(ANON_USER_ID)).toBe(ANON_USER_ID);
+  });
+
+  it('uses the sanitized id for valid users', () => {
+    expect(bucketIdForUser('user:abc123')).toBe('abc123');
+    expect(bucketIdForUser(new RecordId('user', 'xyz'))).toBe('xyz');
+  });
+
+  it('gives unsanitizable ids a deterministic per-user bucket, never anon', () => {
+    const a = bucketIdForUser('user:has-dash');
+    const b = bucketIdForUser('user:has-dash');
+    const c = bucketIdForUser('user:other-dash');
+    // Falling back to the shared anon bucket here would recreate the
+    // cross-user local-cache leak this helper exists to prevent.
+    expect(a).not.toBe(ANON_USER_ID);
+    expect(a).toBe(b);
+    expect(a).not.toBe(c);
+    expect(a).toMatch(/^u[0-9a-f]+$/);
+  });
+});

package/src/modules/ref-tables.ts

@@ -0,0 +1,88 @@
+// Client-side mirror of `packages/ssp-protocol/src/lib.rs`'s
+// `query_table_for` / `list_ref_table_for`. Same naming convention so
+// the LIVE subscription, the initial-fetch read, and the SSP's writes
+// all land on the same table.
+//
+// The mode is currently hardcoded to `dedicated` because that's the
+// only mode the e2e suite exercises and threading the value through
+// codegen wasn't necessary to land the cross-session fix. If single
+// mode ever needs to be exposed from the TS client too, the SSP server
+// already reads it from `SPKY_SSP_REF_MODE`; add a matching codegen
+// export then.
+
+import { cyrb53 } from '@spooky-sync/query-builder';
+
+export type RefMode = 'single' | 'dedicated';
+
+/**
+ * Sentinel user id for unauthenticated clients when anonymous live queries are
+ * enabled. Mirrors `ssp_protocol::ANON_AUTH_ID`. It carries no `user:` prefix
+ * so it can never collide with a real user id (those arrive as `user:<id>`);
+ * both sides resolve it to the shared `_00_list_ref_anon` table.
+ */
+export const ANON_USER_ID = 'anon';
+
+/**
+ * Default ref-storage mode for this client build. Mirrors the SSP's
+ * default (`RefMode::Dedicated`) so cross-session sync works out of the
+ * box.
+ */
+export const DEFAULT_REF_MODE: RefMode = 'dedicated';
+
+/**
+ * Sanitize a user record id (e.g. `"user:abc"`) into the segment that
+ * goes into a dedicated table name (e.g. `"abc"`). Returns `null` if
+ * the id is missing the `user:` prefix or contains characters that
+ * aren't valid in a SurrealDB table identifier — the server-side
+ * `ssp_protocol::sanitize_user_id` uses the same predicate.
+ *
+ * Accepts both string ids (`"user:abc"`) and SurrealDB `RecordId`
+ * objects (which only stringify cleanly via `.toString()`), since
+ * `AuthService` passes the record-id object as-is to its subscribers.
+ */
+export function sanitizeUserId(userId: unknown): string | null {
+  if (userId === null || userId === undefined) return null;
+  const asString =
+    typeof userId === 'string'
+      ? userId
+      : typeof (userId as { toString?: unknown }).toString === 'function'
+        ? (userId as { toString: () => string }).toString()
+        : null;
+  if (!asString) return null;
+  const raw = asString.startsWith('user:') ? asString.slice('user:'.length) : asString;
+  if (raw.length === 0) return null;
+  if (!/^[A-Za-z0-9_]+$/.test(raw)) return null;
+  return raw;
+}
+
+/**
+ * Resolve the LOCAL storage bucket id for a user. Every user gets their own
+ * IndexedDB-backed local store (`indxdb://sp00ky-<bucketId>`) so cached rows,
+ * query state, and the mutation outbox never leak across accounts on a shared
+ * device. Signed-out sessions share the `anon` bucket.
+ *
+ * An id that fails sanitization still gets a DETERMINISTIC per-user bucket
+ * (cyrb53 hex of the raw id) — falling back to `anon` here would put an
+ * authenticated user in the shared bucket and recreate the cross-user leak.
+ */
+export function bucketIdForUser(userId: unknown): string {
+  if (userId === null || userId === undefined || userId === ANON_USER_ID) return ANON_USER_ID;
+  const uid = sanitizeUserId(userId);
+  if (uid) return uid;
+  return `u${cyrb53(String(userId)).toString(16)}`;
+}
+
+/**
+ * Returns the `_00_list_ref` table name for `(mode, userId)`. Falls
+ * back to the global `_00_list_ref` when sanitization fails or in
+ * single mode.
+ */
+export function listRefTableFor(mode: RefMode, userId: unknown): string {
+  // Anonymous clients (flag-enabled) share one dedicated table in both modes —
+  // checked before the mode split so it never lands on the per-user or the
+  // auth-gated global table. Matches `ssp_protocol::list_ref_table_for`.
+  if (userId === ANON_USER_ID) return '_00_list_ref_anon';
+  if (mode === 'single') return '_00_list_ref';
+  const uid = sanitizeUserId(userId);
+  return uid ? `_00_list_ref_user_${uid}` : '_00_list_ref';
+}

package/src/modules/sync/engine.ts

@@ -1,18 +1,21 @@
-import { RecordId } from 'surrealdb';
-import { SchemaStructure } from '@spooky-sync/query-builder';
-import { RemoteDatabaseService } from '../../services/database/index';
-import { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
-import { RecordVersionDiff } from '../../types';
-import { Logger } from '../../services/logger/index';
+import type { RecordId } from 'surrealdb';
+import type { SchemaStructure } from '@spooky-sync/query-builder';
+import type { RemoteDatabaseService } from '../../services/database/index';
+import type { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
+import type { RecordVersionDiff } from '../../types';
+import type { Logger } from '../../services/logger/index';
 import { SyncEventTypes, createSyncEventSystem } from './events/index';
 import { encodeRecordId } from '../../utils/index';
 import { cleanRecord } from '../../utils/parser';
 
+/** Ids per remote `SELECT * FROM $ids` round trip. */
+const FETCH_CHUNK = 500;
+
 /**
  * SyncEngine handles the core sync operations: fetching remote records,
  * caching them locally, and ingesting into DBSP.
  *
- * This is extracted from SpookySync to separate "how to sync" from "when to sync".
+ * This is extracted from Sp00kySync to separate "how to sync" from "when to sync".
  */
 export class SyncEngine {
   private logger: Logger;
@@ -24,7 +27,7 @@ export class SyncEngine {
     private schema: SchemaStructure,
     logger: Logger
   ) {
-    this.logger = logger.child({ service: 'SpookySync:SyncEngine' });
+    this.logger = logger.child({ service: 'Sp00kySync:SyncEngine' });
   }
 
   /**
@@ -32,7 +35,9 @@ export class SyncEngine {
    * Main entry point for sync operations.
    * Uses batch processing to minimize events emitted.
    */
-  async syncRecords(diff: RecordVersionDiff): Promise<void> {
+  async syncRecords(
+    diff: RecordVersionDiff
+  ): Promise<{ remoteFetchMs: number; stillRemoteIds: string[] }> {
     const { added, updated, removed } = diff;
 
     this.logger.debug(
@@ -40,124 +45,201 @@ export class SyncEngine {
         added,
         updated,
         removed,
-        Category: 'spooky-client::SyncEngine::syncRecords',
+        Category: 'sp00ky-client::SyncEngine::syncRecords',
       },
       'SyncEngine.syncRecords diff'
     );
 
-    // Handle removed records: verify they don't exist remotely before deleting locally
+    // Handle removed records: verify they don't exist remotely before deleting
+    // locally. Returns ids that LEFT the view's list_ref but still exist upstream
+    // (so they weren't deleted) — the caller converges localArray to drop them.
+    let stillRemoteIds: string[] = [];
     if (removed.length > 0) {
-      await this.handleRemovedRecords(removed);
+      stillRemoteIds = await this.handleRemovedRecords(removed);
     }
 
-    // Fetch added/updated records from remote
-    const toFetch = [...added, ...updated];
-    const idsToFetch = toFetch.map((x) => x.id);
-    if (idsToFetch.length === 0) {
-      return;
+    // Fetch added/updated records from remote. Skip ids whose body the local
+    // store already holds at this version (the circuit prime seeded the memo
+    // from `_00_rv`): after a reload every id in the server's list_ref used to
+    // classify as `added` against an empty circuit, and this fetched the
+    // whole working set again.
+    const toFetch = [...added, ...updated].filter((item) => {
+      const localVersion = this.cache.lookup(encodeRecordId(item.id));
+      return !(localVersion && item.version <= localVersion);
+    });
+    if (toFetch.length === 0) {
+      return { remoteFetchMs: 0, stillRemoteIds };
     }
 
-    const [remoteResults] = await this.remote.query<[RecordWithId[]]>(
-      "SELECT (SELECT * FROM ONLY <record>$parent.id) AS record, (SELECT version FROM ONLY _spooky_version WHERE record_id = <record>$parent.id)['version'] as spooky_rv FROM $idsToFetch",
-      { idsToFetch }
-    );
-    console.log('remoteResults>', remoteResults);
-    // Prepare batch for cache (which handles both DB and DBSP)
-    const cacheBatch: CacheRecord[] = [];
-
-    for (const { spooky_rv, record } of remoteResults) {
-      if (!record?.id) {
-        this.logger.warn(
-          {
-            record,
-            idsToFetch,
-            Category: 'spooky-client::SyncEngine::syncRecords',
-          },
-          'Remote record has no id. Skipping record'
-        );
-        continue;
+    // Build a version map from the diff (versions come from _00_list_ref)
+    const versionMap = new Map<string, number>();
+    for (const item of toFetch) {
+      versionMap.set(encodeRecordId(item.id), item.version);
+    }
+    const addedIds = new Set(added.map((item) => encodeRecordId(item.id)));
+
+    // Fetch records from remote — avoid SELECT *, <subquery> FROM $param
+    // pattern which drops the * fields in SurrealDB v3 (known bug).
+    // Versions are already known from the diff's list_ref data.
+    //
+    // Chunked so a genuinely cold load lands progressively: one response
+    // holding thousands of bodies, then one transaction MERGEing all of them,
+    // was a single main-thread stall and a memory spike on top of it.
+    let remoteFetchMs = 0;
+    const remoteResults: RecordWithId[] = [];
+    for (let i = 0; i < toFetch.length; i += FETCH_CHUNK) {
+      const idsToFetch = toFetch.slice(i, i + FETCH_CHUNK).map((x) => x.id);
+      const remoteFetchStart = performance.now();
+      const [chunkResults] = await this.remote.query<[RecordWithId[]]>(
+        'SELECT * FROM $idsToFetch',
+        { idsToFetch }
+      );
+      remoteFetchMs += performance.now() - remoteFetchStart;
+
+      // Prepare batch for cache (which handles both DB and DBSP)
+      const cacheBatch: CacheRecord[] = [];
+      for (const record of chunkResults ?? []) {
+        if (!record?.id) {
+          this.logger.warn(
+            {
+              record,
+              idsToFetch,
+              Category: 'sp00ky-client::SyncEngine::syncRecords',
+            },
+            'Remote record has no id (possibly deleted). Skipping record'
+          );
+          continue;
+        }
+        remoteResults.push(record);
+        const fullId = encodeRecordId(record.id);
+        const table = record.id.table.toString();
+        const version = versionMap.get(fullId) ?? 0;
+
+        const localVersion = this.cache.lookup(fullId);
+        if (localVersion && version <= localVersion) {
+          this.logger.info(
+            {
+              recordId: fullId,
+              version,
+              localVersion,
+              Category: 'sp00ky-client::SyncEngine::syncRecords',
+            },
+            'Local version is higher than remote version. Skipping record'
+          );
+          continue;
+        }
+        const tableSchema = this.schema.tables.find((t) => t.name === table);
+        const cleanedRecord = tableSchema
+          ? cleanRecord(tableSchema.columns, record)
+          : record;
+
+        cacheBatch.push({
+          table,
+          op: addedIds.has(fullId) ? 'CREATE' : 'UPDATE',
+          record: cleanedRecord as RecordWithId,
+          version,
+        });
       }
-      const fullId = encodeRecordId(record.id);
-      const table = record.id.table.toString();
-      const isAdded = added.some((item) => encodeRecordId(item.id) === fullId);
 
-      const localVersion = this.cache.lookup(fullId);
-      if (localVersion && spooky_rv <= localVersion) {
-        this.logger.info(
-          {
-            recordId: fullId,
-            version: spooky_rv,
-            localVersion,
-            Category: 'spooky-client::SyncEngine::syncRecords',
-          },
-          'Local version is higher than remote version. Skipping record'
-        );
-        continue;
+      // Use CacheModule to handle both local DB and DBSP ingestion
+      if (cacheBatch.length > 0) {
+        await this.cache.saveBatch(cacheBatch);
       }
-      const tableSchema = this.schema.tables.find((t) => t.name === table);
-      const cleanedRecord = tableSchema
-        ? cleanRecord(tableSchema.columns, record)
-        : record;
-
-      cacheBatch.push({
-        table,
-        op: isAdded ? 'CREATE' : 'UPDATE',
-        record: cleanedRecord as RecordWithId,
-        version: spooky_rv,
-      });
-    }
-
-    // Use CacheModule to handle both local DB and DBSP ingestion
-    if (cacheBatch.length > 0) {
-      await this.cache.saveBatch(cacheBatch);
     }
 
     this.events.emit(SyncEventTypes.RemoteDataIngested, {
       records: remoteResults,
     });
+
+    return { remoteFetchMs, stillRemoteIds };
   }
 
   /**
    * Handle records that exist locally but not in remote array.
+   *
+   * "Removed" here is a derived signal: the SSP's `_00_list_ref` array no
+   * longer references a record that exists locally. That can mean the row
+   * was genuinely deleted upstream — but it can also be a benign race
+   * (e.g. a record we just created hasn't propagated into the SSP's
+   * incantation list yet). Before deleting locally we verify against
+   * upstream SurrealDB: if the row still exists there, skip the delete.
+   *
+   * On verification failure we skip deletion too. Losing a stale local
+   * row to a later sync round is recoverable; deleting a fresh row that
+   * upstream still has is not.
    */
-  private async handleRemovedRecords(removed: RecordId[]): Promise<void> {
+  private async handleRemovedRecords(removed: RecordId[]): Promise<string[]> {
     this.logger.debug(
       {
         removed: removed.map((r) => r.toString()),
-        Category: 'spooky-client::SyncEngine::handleRemovedRecords',
+        Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
       },
       'Checking removed records'
     );
 
-    let existingRemoteIds = new Set<string>();
+    // Confirm which of the "removed" ids still exist remotely by selecting the
+    // records directly from the id array (the records ARE the FROM target).
+    //
+    // The exact query form matters on SurrealDB v3.x:
+    //   - `WHERE id IN $ids` is broken: record-id `IN` matches nothing, so every
+    //     removed id looked gone and live local records got deleted (a fresh
+    //     collection vanished mid-session). Do NOT use `IN`.
+    //   - `SELECT id FROM $ids` — a FIELD projection over a record-id ARRAY —
+    //     errors "Specify a database to use" on the deployed engine. The catch
+    //     below then swallowed it and skipped EVERY deletion, so nothing could be
+    //     deleted anywhere (games, comments, …). Do NOT project a field over the
+    //     array. (`SELECT * FROM $ids` works but pulls full records — wasteful.)
+    //   - `SELECT VALUE id FROM $ids` works: a flat array of ids with a NONE entry
+    //     for each id that no longer exists. We filter the NONE entries out; the
+    //     survivors are the ids still present upstream.
+    let existingRemoteIds: Set<string>;
     try {
-      const [existingRemote] = await this.remote.query<[{ id: RecordId }[]]>('SELECT id FROM $ids', {
-        ids: removed,
-      });
-      existingRemoteIds = new Set(existingRemote.map((r) => encodeRecordId(r.id)));
-    } catch {
-      // If remote check fails (e.g., SurrealDB parameter serialization issue),
-      // proceed with deletion — the caller has already determined these should be removed
-      this.logger.debug(
-        { Category: 'spooky-client::SyncEngine::handleRemovedRecords' },
-        'Remote existence check failed, proceeding with deletion'
+      const [existing] = await this.remote.query<[(RecordId | null | undefined)[]]>(
+        'SELECT VALUE id FROM $ids',
+        { ids: removed }
+      );
+      existingRemoteIds = new Set(
+        (existing ?? [])
+          .filter((id): id is RecordId => id != null)
+          .map((id) => encodeRecordId(id))
       );
+    } catch (err) {
+      // Verification failed. Skip deletion entirely — the next sync
+      // round re-derives the diff and we get another shot. The
+      // alternative (delete on uncertainty) destroys freshly-created
+      // rows when the SSP hasn't yet refreshed `_00_list_ref`.
+      this.logger.warn(
+        {
+          err,
+          removed: removed.map((r) => r.toString()),
+          Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
+        },
+        'Remote existence check failed, skipping deletion to avoid clobbering fresh data'
+      );
+      return [];
     }
 
+    // Ids that left the view's list_ref but STILL exist upstream — not deletions,
+    // just a view-membership change (e.g. a record whose field changed so it no
+    // longer matches the query). The caller drops these from `localArray` so the
+    // poll's diff stops re-flagging them every tick (the `job:` churn).
+    const stillRemoteIds: string[] = [];
     for (const recordId of removed) {
       const recordIdStr = encodeRecordId(recordId);
       if (!existingRemoteIds.has(recordIdStr)) {
         this.logger.debug(
           {
             recordId: recordIdStr,
-            Category: 'spooky-client::SyncEngine::handleRemovedRecords',
+            Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
           },
           'Deleting confirmed removed record'
         );
-
         // Use CacheModule to handle both local DB and DBSP deletion
         await this.cache.delete(recordId.table.name, recordIdStr);
+      } else {
+        stillRemoteIds.push(recordIdStr);
       }
     }
+    return stillRemoteIds;
   }
 }

package/src/modules/sync/events/index.ts

@@ -1,5 +1,6 @@
-import { createEventSystem, EventDefinition, EventSystem } from '../../../events/index';
-import { RecordVersionArray } from '../../../types';
+import type { EventDefinition, EventSystem } from '../../../events/index';
+import { createEventSystem } from '../../../events/index';
+import type { RecordVersionArray, SyncHealth } from '../../../types';
 
 export const SyncQueueEventTypes = {
   MutationEnqueued: 'MUTATION_ENQUEUED',
@@ -36,6 +37,7 @@ export const SyncEventTypes = {
   QueryUpdated: 'SYNC_QUERY_UPDATED',
   RemoteDataIngested: 'SYNC_REMOTE_DATA_INGESTED',
   MutationRolledBack: 'SYNC_MUTATION_ROLLED_BACK',
+  SyncHealthChanged: 'SYNC_HEALTH_CHANGED',
 } as const;
 
 export type SyncEventTypeMap = {
@@ -64,6 +66,10 @@ export type SyncEventTypeMap = {
       error: string;
     }
   >;
+  [SyncEventTypes.SyncHealthChanged]: EventDefinition<
+    typeof SyncEventTypes.SyncHealthChanged,
+    SyncHealth
+  >;
 };
 
 export type SyncEventSystem = EventSystem<SyncEventTypeMap>;
@@ -73,5 +79,6 @@ export function createSyncEventSystem(): SyncEventSystem {
     SyncEventTypes.QueryUpdated,
     SyncEventTypes.RemoteDataIngested,
     SyncEventTypes.MutationRolledBack,
+    SyncEventTypes.SyncHealthChanged,
   ]);
 }