@spooky-sync/core 0.0.1-canary.6 → 0.0.1-canary.61

package/src/modules/cache/index.ts

@@ -1,13 +1,13 @@
-import { LocalDatabaseService } from '../../services/database/index';
-import {
+import type { LocalDatabaseService } from '../../services/database/index';
+import type {
   StreamProcessorService,
   StreamUpdate,
   StreamUpdateReceiver,
 } from '../../services/stream-processor/index';
-import { Logger } from '../../services/logger/index';
+import type { Logger } from '../../services/logger/index';
 import { parseRecordIdString, encodeRecordId, surql } from '../../utils/index';
-import { CacheRecord, QueryConfig } from './types';
-import { RecordVersionArray } from '../../types';
+import type { CacheRecord, QueryConfig } from './types';
+import type { RecordVersionArray } from '../../types';
 
 export * from './types';
 
@@ -43,7 +43,7 @@ export class CacheModule implements StreamUpdateReceiver {
       {
         queryHash: update.queryHash,
         arrayLength: update.localArray?.length,
-        Category: 'spooky-client::CacheModule::onStreamUpdate',
+        Category: 'sp00ky-client::CacheModule::onStreamUpdate',
       },
       'Stream update received'
     );
@@ -73,7 +73,7 @@ export class CacheModule implements StreamUpdateReceiver {
     this.logger.debug(
       {
         count: records.length,
-        Category: 'spooky-client::CacheModule::saveBatch',
+        Category: 'sp00ky-client::CacheModule::saveBatch',
       },
       'Saving record batch'
     );
@@ -85,7 +85,7 @@ export class CacheModule implements StreamUpdateReceiver {
           ...record,
           record: {
             ...record.record,
-            spooky_rv: record.version,
+            _00_rv: record.version,
           },
         };
       });
@@ -114,7 +114,7 @@ export class CacheModule implements StreamUpdateReceiver {
         await this.local.execute(query, params);
       }
 
-      // 2. Batch ingest into DBSP (use populatedRecords which has spooky_rv set)
+      // 2. Batch ingest into DBSP (use populatedRecords which has _00_rv set)
       for (const record of populatedRecords) {
         const recordId = encodeRecordId(record.record.id);
         this.versionLookups[recordId] = record.version;
@@ -122,12 +122,12 @@ export class CacheModule implements StreamUpdateReceiver {
       }
 
       this.logger.debug(
-        { count: records.length, Category: 'spooky-client::CacheModule::saveBatch' },
+        { count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
         'Batch saved successfully'
       );
     } catch (err) {
       this.logger.error(
-        { err, count: records.length, Category: 'spooky-client::CacheModule::saveBatch' },
+        { err, count: records.length, Category: 'sp00ky-client::CacheModule::saveBatch' },
         'Failed to save batch'
       );
       throw err;
@@ -137,9 +137,9 @@ export class CacheModule implements StreamUpdateReceiver {
   /**
    * Delete a record from local DB and ingest deletion into DBSP
    */
-  async delete(table: string, id: string, skipDbDelete: boolean = false): Promise<void> {
+  async delete(table: string, id: string, skipDbDelete: boolean = false, recordData: Record<string, any> = {}): Promise<void> {
     this.logger.debug(
-      { table, id, Category: 'spooky-client::CacheModule::delete' },
+      { table, id, Category: 'sp00ky-client::CacheModule::delete' },
       'Deleting record'
     );
 
@@ -149,17 +149,17 @@ export class CacheModule implements StreamUpdateReceiver {
         await this.local.query('DELETE $id', { id: parseRecordIdString(id) });
       }
 
-      // 2. Ingest deletion into DBSP
+      // 2. Ingest deletion into DBSP (pass record data so predicates can be matched)
       delete this.versionLookups[id];
-      await this.streamProcessor.ingest(table, 'DELETE', id, {});
+      this.streamProcessor.ingest(table, 'DELETE', id, recordData);
 
       this.logger.debug(
-        { table, id, Category: 'spooky-client::CacheModule::delete' },
+        { table, id, Category: 'sp00ky-client::CacheModule::delete' },
         'Record deleted successfully'
       );
     } catch (err) {
       this.logger.error(
-        { err, table, id, Category: 'spooky-client::CacheModule::delete' },
+        { err, table, id, Category: 'sp00ky-client::CacheModule::delete' },
         'Failed to delete record'
       );
       throw err;
@@ -175,7 +175,7 @@ export class CacheModule implements StreamUpdateReceiver {
       {
         queryHash: config.queryHash,
         surql: config.surql,
-        Category: 'spooky-client::CacheModule::registerQuery',
+        Category: 'sp00ky-client::CacheModule::registerQuery',
       },
       'Registering query'
     );
@@ -202,7 +202,7 @@ export class CacheModule implements StreamUpdateReceiver {
         {
           queryHash: config.queryHash,
           arrayLength: update.localArray?.length,
-          Category: 'spooky-client::CacheModule::registerQuery',
+          Category: 'sp00ky-client::CacheModule::registerQuery',
         },
         'Query registered successfully'
       );
@@ -210,7 +210,7 @@ export class CacheModule implements StreamUpdateReceiver {
       return { localArray: update.localArray };
     } catch (err) {
       this.logger.error(
-        { err, queryHash: config.queryHash, Category: 'spooky-client::CacheModule::registerQuery' },
+        { err, queryHash: config.queryHash, Category: 'sp00ky-client::CacheModule::registerQuery' },
         'Failed to register query'
       );
       throw err;
@@ -222,18 +222,18 @@ export class CacheModule implements StreamUpdateReceiver {
    */
   unregisterQuery(queryHash: string): void {
     this.logger.debug(
-      { queryHash, Category: 'spooky-client::CacheModule::unregisterQuery' },
+      { queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
       'Unregistering query'
     );
     try {
       this.streamProcessor.unregisterQueryPlan(queryHash);
       this.logger.debug(
-        { queryHash, Category: 'spooky-client::CacheModule::unregisterQuery' },
+        { queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
         'Query unregistered successfully'
       );
     } catch (err) {
       this.logger.error(
-        { err, queryHash, Category: 'spooky-client::CacheModule::unregisterQuery' },
+        { err, queryHash, Category: 'sp00ky-client::CacheModule::unregisterQuery' },
         'Failed to unregister query'
       );
     }

package/src/modules/cache/types.ts

@@ -1,5 +1,5 @@
-import { RecordId, Duration } from 'surrealdb';
-import { QueryTimeToLive, RecordVersionArray } from '../../types';
+import type { RecordId, Duration } from 'surrealdb';
+import type { QueryTimeToLive } from '../../types';
 
 export type RecordWithId = Record<string, any> & { id: RecordId<string> };
 

package/src/modules/crdt/crdt-field.ts

@@ -0,0 +1,189 @@
+import { LoroDoc } from 'loro-crdt';
+import type { RemoteDatabaseService } from '../../services/database/index';
+import type { Logger } from '../../services/logger/index';
+import { parseRecordIdString } from '../../utils/index';
+
+// ==================== CURSOR UTILITIES ====================
+
+export const CURSOR_COLORS = [
+  '#3b82f6', '#ef4444', '#22c55e', '#f59e0b',
+  '#8b5cf6', '#ec4899', '#14b8a6', '#f97316',
+];
+
+export function cursorColorFromName(name: string): string {
+  let hash = 0;
+  for (let i = 0; i < name.length; i++) {
+    hash = ((hash << 5) - hash + name.charCodeAt(i)) | 0;
+  }
+  return CURSOR_COLORS[Math.abs(hash) % CURSOR_COLORS.length];
+}
+
+// ==================== CRDT FIELD ====================
+
+export class CrdtField {
+  private doc: LoroDoc;
+  private pushTimer: ReturnType<typeof setTimeout> | null = null;
+  private remote: RemoteDatabaseService | null = null;
+  private recordId: string | null = null;
+  private unsubscribe: (() => void) | null = null;
+  private lastPushTime = 0;
+  private lastCursorPushTime = 0;
+  private loadedFromCrdt = false;
+  private pushRetryCount = 0;
+  private logger: Logger | null;
+
+  private _onCursorUpdate: ((data: Uint8Array) => void) | null = null;
+  private pendingCursorUpdate: Uint8Array | null = null;
+
+  /** Callback set by the editor to receive remote cursor updates.
+   *  Any cursor data that arrived before this callback was set will be replayed. */
+  set onCursorUpdate(cb: ((data: Uint8Array) => void) | null) {
+    this._onCursorUpdate = cb;
+    if (cb && this.pendingCursorUpdate) {
+      try { cb(this.pendingCursorUpdate); } catch (e) {
+        this.logger?.warn(
+          { error: e, Category: 'sp00ky-client::CrdtField::onCursorUpdate' },
+          'Failed to replay pending cursor update'
+        );
+      }
+      this.pendingCursorUpdate = null;
+    }
+  }
+
+  get onCursorUpdate() { return this._onCursorUpdate; }
+
+  constructor(
+    private fieldName: string,
+    initialState?: string,
+    logger?: Logger | null,
+  ) {
+    this.logger = logger ?? null;
+    this.doc = new LoroDoc();
+    if (initialState) {
+      this.doc.import(decodeBase64(initialState));
+      this.loadedFromCrdt = true;
+    }
+  }
+
+  getDoc(): LoroDoc { return this.doc; }
+
+  /** Whether the LoroDoc was loaded from saved CRDT state */
+  hasContent(): boolean {
+    return this.loadedFromCrdt;
+  }
+
+  startSync(remote: RemoteDatabaseService, recordId: string): void {
+    this.remote = remote;
+    this.recordId = recordId;
+    this.unsubscribe = this.doc.subscribeLocalUpdates(() => {
+      this.schedulePush();
+    });
+  }
+
+  stopSync(): void {
+    if (this.unsubscribe) { this.unsubscribe(); this.unsubscribe = null; }
+    if (this.pushTimer) { clearTimeout(this.pushTimer); this.pushTimer = null; }
+    if (this.remote && this.recordId) { void this.pushToRemote(); }
+  }
+
+  importRemote(base64State: string): void {
+    // Suppress echo: skip imports within 500ms of our own push
+    if (Date.now() - this.lastPushTime < 500) return;
+    try {
+      this.doc.import(decodeBase64(base64State));
+    } catch (e) {
+      this.logger?.warn(
+        { error: e, Category: 'sp00ky-client::CrdtField::importRemote' },
+        'Failed to import remote CRDT state'
+      );
+    }
+  }
+
+  exportSnapshot(): string {
+    return encodeBase64(this.doc.export({ mode: 'snapshot' }));
+  }
+
+  /** Push cursor ephemeral state to _00_crdt as a "_cursor_<fieldName>" entry */
+  async pushCursorState(encoded: Uint8Array): Promise<void> {
+    if (!this.remote || !this.recordId) return;
+    this.lastCursorPushTime = Date.now();
+    try {
+      const state = encodeBase64(encoded);
+      await this.remote.query(
+        `INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
+         ON DUPLICATE KEY UPDATE state = $state`,
+        { rid: parseRecordIdString(this.recordId), field: `_cursor_${this.fieldName}`, state }
+      );
+    } catch (e) {
+      this.logger?.warn(
+        { error: e, Category: 'sp00ky-client::CrdtField::pushCursorState' },
+        'Failed to push cursor state'
+      );
+    }
+  }
+
+  /** Import remote cursor state (called by CrdtManager from LIVE SELECT) */
+  importRemoteCursor(base64State: string): void {
+    if (Date.now() - this.lastCursorPushTime < 300) return; // echo suppression
+    try {
+      const data = decodeBase64(base64State);
+      if (this._onCursorUpdate) {
+        this._onCursorUpdate(data);
+      } else {
+        // Only keep the latest cursor state — older positions are useless
+        this.pendingCursorUpdate = data;
+      }
+    } catch (e) {
+      this.logger?.warn(
+        { error: e, Category: 'sp00ky-client::CrdtField::importRemoteCursor' },
+        'Failed to apply remote cursor data'
+      );
+    }
+  }
+
+  private schedulePush(): void {
+    if (this.pushTimer) clearTimeout(this.pushTimer);
+    this.pushTimer = setTimeout(() => void this.pushToRemote(), 300);
+  }
+
+  private async pushToRemote(): Promise<void> {
+    if (!this.remote || !this.recordId) return;
+    this.lastPushTime = Date.now();
+    try {
+      await this.remote.query(
+        `INSERT INTO _00_crdt (record_id, field, state) VALUES ($rid, $field, $state)
+         ON DUPLICATE KEY UPDATE state = $state`,
+        { rid: parseRecordIdString(this.recordId), field: this.fieldName, state: this.exportSnapshot() }
+      );
+      this.pushRetryCount = 0;
+    } catch (e) {
+      this.logger?.warn(
+        { error: e, Category: 'sp00ky-client::CrdtField::pushToRemote' },
+        'Failed to push CRDT state to remote'
+      );
+      if (this.pushRetryCount < 2) {
+        this.pushRetryCount++;
+        this.schedulePush();
+      }
+    }
+  }
+}
+
+export function decodeBase64(b64: string): Uint8Array {
+  if (typeof atob === 'function') {
+    const binary = atob(b64);
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+    return bytes;
+  }
+  return new Uint8Array(Buffer.from(b64, 'base64'));
+}
+
+export function encodeBase64(bytes: Uint8Array): string {
+  if (typeof btoa === 'function') {
+    let binary = '';
+    for (let i = 0; i < bytes.length; i++) binary += String.fromCharCode(bytes[i]);
+    return btoa(binary);
+  }
+  return Buffer.from(bytes).toString('base64');
+}

package/src/modules/crdt/index.ts

@@ -0,0 +1,178 @@
+import type { SchemaStructure } from '@spooky-sync/query-builder';
+import type { RemoteDatabaseService } from '../../services/database/index';
+import type { Logger } from '../../services/logger/index';
+import type { Uuid } from 'surrealdb';
+import { CrdtField } from './crdt-field';
+import { parseRecordIdString } from '../../utils/index';
+
+export { CrdtField, cursorColorFromName, CURSOR_COLORS } from './crdt-field';
+
+/**
+ * CrdtManager manages active CrdtField instances and their sync channels.
+ *
+ * Each open record gets a LIVE SELECT on _00_crdt that delivers remote
+ * changes in real time.
+ */
+export class CrdtManager {
+  private fields = new Map<string, CrdtField>();
+  private liveQueries = new Map<string, { uuid: Uuid; table: string }>();
+  private logger: Logger;
+
+  constructor(
+    private schema: SchemaStructure,
+    private remote: RemoteDatabaseService,
+    logger: Logger
+  ) {
+    this.logger = logger.child({ service: 'CrdtManager' });
+  }
+
+  /**
+   * Open a CRDT field for collaborative editing.
+   *
+   * @param table - Table name
+   * @param recordId - Full record ID (e.g., "thread:abc")
+   * @param field - Field name (e.g., "title", "content")
+   * @param fallbackText - Current plain text from the record, used to seed the
+   *                       LoroDoc if no CRDT state exists yet (migration path)
+   */
+  async open(
+    table: string,
+    recordId: string,
+    field: string,
+    fallbackText?: string,
+  ): Promise<CrdtField> {
+    const key = this.makeKey(table, recordId, field);
+    let crdtField = this.fields.get(key);
+
+    if (crdtField) {
+      return crdtField;
+    }
+
+    // Load saved CRDT state from remote _00_crdt table
+    let initialCrdtState: string | undefined;
+    try {
+      const [result] = await this.remote.query<[string[]]>(
+        'SELECT VALUE state FROM _00_crdt WHERE record_id = $rid AND field = $field LIMIT 1',
+        { rid: parseRecordIdString(recordId), field }
+      );
+      if (result && result.length > 0 && result[0]) {
+        initialCrdtState = result[0];
+      }
+    } catch (e) {
+      this.logger.debug(
+        { error: e, Category: 'sp00ky-client::CrdtManager::open' },
+        'No existing CRDT state found'
+      );
+    }
+
+    crdtField = new CrdtField(field, initialCrdtState, this.logger);
+    crdtField.startSync(this.remote, recordId);
+    this.fields.set(key, crdtField);
+
+    this.logger.info(
+      { key, hasInitialState: !!initialCrdtState, hasFallback: !!fallbackText, Category: 'sp00ky-client::CrdtManager::open' },
+      'CrdtField opened'
+    );
+
+    await this.ensureLiveSelect(table, recordId);
+
+    return crdtField;
+  }
+
+  close(table: string, recordId: string, field: string): void {
+    const key = this.makeKey(table, recordId, field);
+    const crdtField = this.fields.get(key);
+    if (crdtField) {
+      crdtField.stopSync();
+      this.fields.delete(key);
+    }
+
+    const prefix = `${table}:${recordId}:`;
+    const hasOtherFields = Array.from(this.fields.keys()).some(
+      (k) => k !== key && k.startsWith(prefix)
+    );
+    if (!hasOtherFields) {
+      this.killLiveSelect(recordId);
+    }
+
+    this.logger.debug(
+      { key, Category: 'sp00ky-client::CrdtManager::close' },
+      'CrdtField closed'
+    );
+  }
+
+  closeAll(): void {
+    for (const [_, field] of this.fields) {
+      field.stopSync();
+    }
+    this.fields.clear();
+    for (const recordId of this.liveQueries.keys()) {
+      this.killLiveSelect(recordId);
+    }
+  }
+
+  private async ensureLiveSelect(table: string, recordId: string): Promise<void> {
+    if (this.liveQueries.has(recordId)) return;
+
+    try {
+      const [uuid] = await this.remote.query<[Uuid]>(
+        `LIVE SELECT * FROM _00_crdt WHERE record_id = $rid`,
+        { rid: parseRecordIdString(recordId) },
+      );
+
+      this.liveQueries.set(recordId, { uuid, table });
+
+      const subscription = await this.remote.getClient().liveOf(uuid);
+      subscription.subscribe((message) => {
+        if (message.action === 'KILLED') return;
+
+        if (message.action === 'CREATE' || message.action === 'UPDATE') {
+          const fieldName = message.value.field as string;
+          const state = message.value.state as string;
+
+          if (!fieldName || !state) return;
+
+          // Route cursor updates to the corresponding CrdtField
+          if (fieldName.startsWith('_cursor_')) {
+            const actualField = fieldName.slice('_cursor_'.length);
+            const key = this.makeKey(table, recordId, actualField);
+            const crdtField = this.fields.get(key);
+            if (crdtField) {
+              crdtField.importRemoteCursor(state);
+            }
+            return;
+          }
+
+          // Route document updates
+          const key = this.makeKey(table, recordId, fieldName);
+          const crdtField = this.fields.get(key);
+          if (crdtField) {
+            crdtField.importRemote(state);
+          }
+        }
+      });
+
+      this.logger.info(
+        { recordId, Category: 'sp00ky-client::CrdtManager::ensureLiveSelect' },
+        'LIVE SELECT on _00_crdt started'
+      );
+    } catch (e) {
+      this.logger.warn(
+        { error: e, recordId, Category: 'sp00ky-client::CrdtManager::ensureLiveSelect' },
+        'Failed to start LIVE SELECT on _00_crdt'
+      );
+    }
+  }
+
+  private killLiveSelect(recordId: string): void {
+    const entry = this.liveQueries.get(recordId);
+    if (entry) {
+      this.remote.query('KILL $uuid', { uuid: entry.uuid }).catch(() => {});
+      this.liveQueries.delete(recordId);
+    }
+  }
+
+  private makeKey(table: string, recordId: string, field: string): string {
+    return `${table}:${recordId}:${field}`;
+  }
+}