@spooky-sync/core 0.0.0-canary.1

package/src/modules/devtools/index.ts

@@ -0,0 +1,364 @@
+import { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
+import { Logger } from '../../services/logger/index';
+import { SchemaStructure } from '@spooky-sync/query-builder';
+import { RecordId } from 'surrealdb';
+import { StreamUpdate, StreamUpdateReceiver } from '../../services/stream-processor/index';
+import { encodeRecordId } from '../../utils/index';
+
+// DevTools interfaces (matching extension expectations)
+export interface DevToolsEvent {
+  id: number;
+  timestamp: number;
+  eventType: string;
+  payload: any;
+}
+
+import { DataModule } from '../data/index';
+import { AuthService } from '../auth/index';
+import { AuthEventTypes } from '../auth/events/index';
+
+export class DevToolsService implements StreamUpdateReceiver {
+  private eventsHistory: DevToolsEvent[] = [];
+  private eventIdCounter = 0;
+  private version = '1.0.0';
+
+  constructor(
+    private databaseService: LocalDatabaseService,
+    private remoteDatabaseService: RemoteDatabaseService,
+    private logger: Logger,
+    private schema: SchemaStructure,
+    private authService: AuthService<SchemaStructure>,
+    private dataManager?: DataModule<SchemaStructure>
+  ) {
+    this.exposeToWindow();
+
+    // Subscribe to auth events
+    this.authService.eventSystem.subscribe(AuthEventTypes.AuthStateChanged, () => {
+      this.notifyDevTools();
+    });
+
+    this.logger.debug({ Category: 'spooky-client::DevToolsService::init' }, 'Service initialized');
+  }
+
+  // Get active queries directly from DataManager (single source of truth)
+  private getActiveQueries(): Map<number, any> {
+    const result = new Map<number, any>();
+    if (!this.dataManager) return result;
+
+    const queries = this.dataManager.getActiveQueries();
+    queries.forEach((q) => {
+      const queryHash = this.hashString(encodeRecordId(q.config.id));
+      result.set(queryHash, {
+        queryHash,
+        status: 'active',
+        createdAt:
+          q.config.lastActiveAt instanceof Date
+            ? q.config.lastActiveAt.getTime()
+            : new Date(q.config.lastActiveAt || Date.now()).getTime(),
+        lastUpdate: Date.now(),
+        updateCount: q.updateCount,
+        query: q.config.surql,
+        variables: q.config.params || {},
+        dataSize: q.records?.length || 0,
+        data: q.records,
+        localArray: q.config.localArray,
+        remoteArray: q.config.remoteArray,
+      });
+    });
+    return result;
+  }
+
+  public onQueryInitialized(payload: any) {
+    this.logger.debug(
+      { payload, Category: 'spooky-client::DevToolsService::onQueryInitialized' },
+      'QueryInitialized'
+    );
+    const queryHash = this.hashString(payload.queryId.toString());
+
+    this.addEvent('QUERY_REQUEST_INIT', {
+      queryHash,
+      query: payload.sql,
+      variables: {},
+    });
+    this.notifyDevTools();
+  }
+
+  public onQueryUpdated(payload: any) {
+    this.logger.debug(
+      {
+        id: payload.queryId?.toString(),
+        Category: 'spooky-client::DevToolsService::onQueryUpdated',
+      },
+      'QueryUpdated'
+    );
+    const queryHash = this.hashString(payload.queryId.toString());
+
+    this.addEvent('QUERY_UPDATED', {
+      queryHash,
+      data: payload.records,
+    });
+    this.notifyDevTools();
+  }
+
+  public onStreamUpdate(update: StreamUpdate) {
+    this.logger.debug(
+      { update, Category: 'spooky-client::DevToolsService::onStreamUpdate' },
+      'StreamUpdate'
+    );
+    this.addEvent('STREAM_UPDATE', {
+      updates: [update],
+    });
+    this.notifyDevTools();
+  }
+
+  public onMutation(payload: any[]) {
+    const payloads = payload;
+    payloads.forEach((p) => {
+      this.addEvent('MUTATION_REQUEST_EXECUTION', {
+        mutation: {
+          type: 'create', // simplifying
+          data: 'data' in p ? p.data : undefined,
+          selector: encodeRecordId(p.record_id),
+        },
+      });
+    });
+    this.notifyDevTools();
+  }
+
+  private hashString(str: string): number {
+    let hash = 0;
+    if (str.length === 0) return hash;
+    for (let i = 0; i < str.length; i++) {
+      const char = str.charCodeAt(i);
+      hash = (hash << 5) - hash + char;
+      hash = hash & hash; // Convert to 32bit integer
+    }
+    return hash;
+  }
+
+  public logEvent(eventType: string, payload: any) {
+    this.addEvent(eventType, payload);
+    this.notifyDevTools();
+  }
+
+  private addEvent(eventType: string, payload: any) {
+    this.eventsHistory.push({
+      id: this.eventIdCounter++,
+      timestamp: Date.now(),
+      eventType,
+      payload: this.serializeForDevTools(payload),
+    });
+    if (this.eventsHistory.length > 100) this.eventsHistory.shift();
+  }
+
+  private getState() {
+    return this.serializeForDevTools({
+      eventsHistory: [...this.eventsHistory],
+      activeQueries: Object.fromEntries(this.getActiveQueries()),
+      auth: {
+        authenticated: this.authService.isAuthenticated,
+        userId: this.authService.currentUser?.id,
+      },
+      version: this.version,
+      database: {
+        tables: this.schema.tables.map((t) => t.name),
+        tableData: {},
+      },
+    });
+  }
+
+  private notifyDevTools() {
+    if (typeof window !== 'undefined') {
+      window.postMessage(
+        {
+          type: 'SPOOKY_STATE_CHANGED',
+          source: 'spooky-devtools-page',
+          state: this.getState(),
+        },
+        '*'
+      );
+    }
+  }
+
+  private serializeForDevTools(data: any, seen = new WeakSet<object>()): any {
+    if (data === undefined) {
+      return 'undefined';
+    }
+
+    if (data === null) {
+      return null;
+    }
+
+    if (data instanceof RecordId) {
+      return data.toString();
+    }
+
+    if (Array.isArray(data)) {
+      if (seen.has(data)) {
+        return '[Circular Array]';
+      }
+      seen.add(data);
+      return data.map((item) => this.serializeForDevTools(item, seen));
+    }
+
+    if (typeof data === 'bigint') {
+      return data.toString();
+    }
+
+    if (data instanceof Date) {
+      return data.toISOString();
+    }
+
+    if (typeof data === 'object') {
+      if (seen.has(data)) {
+        return '[Circular Object]';
+      }
+      seen.add(data);
+
+      const result: Record<string, any> = {};
+      for (const key in data) {
+        if (Object.prototype.hasOwnProperty.call(data, key)) {
+          result[key] = this.serializeForDevTools(data[key], seen);
+        }
+      }
+      return result;
+    }
+
+    return data;
+  }
+
+  private exposeToWindow() {
+    if (typeof window !== 'undefined') {
+      (window as any).__SPOOKY__ = {
+        version: this.version,
+        getState: () => this.getState(),
+        clearHistory: () => {
+          this.eventsHistory = [];
+          this.notifyDevTools();
+        },
+        getTableData: async (tableName: string) => {
+          try {
+            // Returns the first statement result as T.
+            // SurrealDB query returns [Result1, Result2...].
+            // We want the records from the first result.
+            const result = await this.databaseService.query<any>(`SELECT * FROM ${tableName}`);
+
+            let records: any[] = [];
+
+            if (Array.isArray(result) && result.length > 0) {
+              const first = result[0];
+              if (Array.isArray(first)) {
+                // Legacy or flattened format: [[records]]
+                records = first;
+              } else if (
+                first &&
+                typeof first === 'object' &&
+                'result' in first &&
+                'status' in first
+              ) {
+                // SurrealDB 2.0 format: [{ result: [...records], status: 'OK', ... }]
+                records = Array.isArray(first.result) ? first.result : [];
+              } else {
+                // Fallback: assume result is the array of records itself
+                records = result;
+              }
+            } else if (Array.isArray(result)) {
+              // Empty array
+              records = [];
+            }
+
+            return this.serializeForDevTools(records) || [];
+          } catch (e) {
+            this.logger.error(
+              { err: e, Category: 'spooky-client::DevToolsService::exposeToWindow' },
+              'Failed to get table data'
+            );
+            return [];
+          }
+        },
+        updateTableRow: async (
+          tableName: string,
+          recordId: string,
+          updates: Record<string, unknown>
+        ) => {
+          try {
+            await this.databaseService.query(`UPDATE ${recordId} MERGE $updates`, { updates });
+            return { success: true };
+          } catch (e: any) {
+            return { success: false, error: e.message };
+          }
+        },
+        deleteTableRow: async (tableName: string, recordId: string) => {
+          try {
+            await this.databaseService.query(`DELETE ${recordId}`);
+            return { success: true };
+          } catch (e: any) {
+            return { success: false, error: e.message };
+          }
+        },
+        runQuery: async (query: string, target: 'local' | 'remote' = 'local') => {
+          try {
+            this.logger.debug(
+              { query, target, Category: 'spooky-client::DevToolsService::runQuery' },
+              'Running query (START)'
+            );
+            const service = target === 'remote' ? this.remoteDatabaseService : this.databaseService;
+
+            const startTime = Date.now();
+            const result = await service.query<any>(query);
+            const queryTime = Date.now() - startTime;
+
+            this.logger.debug(
+              {
+                query,
+                time: queryTime,
+                resultType: typeof result,
+                isArray: Array.isArray(result),
+                Category: 'spooky-client::DevToolsService::runQuery',
+              },
+              'Database returned result'
+            );
+
+            // Serialize the result for DevTools
+            const serializeStart = Date.now();
+            const serialized = this.serializeForDevTools(result);
+            const serializeTime = Date.now() - serializeStart;
+
+            this.logger.debug(
+              {
+                serializeTime,
+                serializedLength: JSON.stringify(serialized).length,
+                Category: 'spooky-client::DevToolsService::runQuery',
+              },
+              'Serialization complete'
+            );
+
+            return {
+              success: true,
+              data: serialized,
+              target,
+            };
+          } catch (e: any) {
+            this.logger.error(
+              { err: e, query, target, Category: 'spooky-client::DevToolsService::runQuery' },
+              'Query execution failed'
+            );
+            // Ensure we always return a string for error
+            const errorMessage =
+              e instanceof Error ? e.message : typeof e === 'string' ? e : JSON.stringify(e);
+            return { success: false, error: errorMessage || 'Unknown occurred' };
+          }
+        },
+      };
+
+      window.postMessage(
+        {
+          type: 'SPOOKY_DETECTED',
+          source: 'spooky-devtools-page',
+          data: { version: this.version, detected: true },
+        },
+        '*'
+      );
+    }
+  }
+}

package/src/modules/sync/engine.ts

@@ -0,0 +1,163 @@
+import { RecordId } from 'surrealdb';
+import { SchemaStructure } from '@spooky-sync/query-builder';
+import { RemoteDatabaseService } from '../../services/database/index';
+import { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
+import { RecordVersionDiff } from '../../types';
+import { Logger } from '../../services/logger/index';
+import { SyncEventTypes, createSyncEventSystem } from './events/index';
+import { encodeRecordId } from '../../utils/index';
+import { cleanRecord } from '../../utils/parser';
+
+/**
+ * SyncEngine handles the core sync operations: fetching remote records,
+ * caching them locally, and ingesting into DBSP.
+ *
+ * This is extracted from SpookySync to separate "how to sync" from "when to sync".
+ */
+export class SyncEngine {
+  private logger: Logger;
+  public events = createSyncEventSystem();
+
+  constructor(
+    private remote: RemoteDatabaseService,
+    private cache: CacheModule,
+    private schema: SchemaStructure,
+    logger: Logger
+  ) {
+    this.logger = logger.child({ service: 'SpookySync:SyncEngine' });
+  }
+
+  /**
+   * Sync missing/updated/removed records between local and remote.
+   * Main entry point for sync operations.
+   * Uses batch processing to minimize events emitted.
+   */
+  async syncRecords(diff: RecordVersionDiff): Promise<void> {
+    const { added, updated, removed } = diff;
+
+    this.logger.debug(
+      {
+        added,
+        updated,
+        removed,
+        Category: 'spooky-client::SyncEngine::syncRecords',
+      },
+      'SyncEngine.syncRecords diff'
+    );
+
+    // Handle removed records: verify they don't exist remotely before deleting locally
+    if (removed.length > 0) {
+      await this.handleRemovedRecords(removed);
+    }
+
+    // Fetch added/updated records from remote
+    const toFetch = [...added, ...updated];
+    const idsToFetch = toFetch.map((x) => x.id);
+    if (idsToFetch.length === 0) {
+      return;
+    }
+
+    const [remoteResults] = await this.remote.query<[RecordWithId[]]>(
+      "SELECT (SELECT * FROM ONLY <record>$parent.id) AS record, (SELECT version FROM ONLY _spooky_version WHERE record_id = <record>$parent.id)['version'] as spooky_rv FROM $idsToFetch",
+      { idsToFetch }
+    );
+    console.log('remoteResults>', remoteResults);
+    // Prepare batch for cache (which handles both DB and DBSP)
+    const cacheBatch: CacheRecord[] = [];
+
+    for (const { spooky_rv, record } of remoteResults) {
+      if (!record?.id) {
+        this.logger.warn(
+          {
+            record,
+            idsToFetch,
+            Category: 'spooky-client::SyncEngine::syncRecords',
+          },
+          'Remote record has no id. Skipping record'
+        );
+        continue;
+      }
+      const fullId = encodeRecordId(record.id);
+      const table = record.id.table.toString();
+      const isAdded = added.some((item) => encodeRecordId(item.id) === fullId);
+
+      const localVersion = this.cache.lookup(fullId);
+      if (localVersion && spooky_rv <= localVersion) {
+        this.logger.info(
+          {
+            recordId: fullId,
+            version: spooky_rv,
+            localVersion,
+            Category: 'spooky-client::SyncEngine::syncRecords',
+          },
+          'Local version is higher than remote version. Skipping record'
+        );
+        continue;
+      }
+      const tableSchema = this.schema.tables.find((t) => t.name === table);
+      const cleanedRecord = tableSchema
+        ? cleanRecord(tableSchema.columns, record)
+        : record;
+
+      cacheBatch.push({
+        table,
+        op: isAdded ? 'CREATE' : 'UPDATE',
+        record: cleanedRecord as RecordWithId,
+        version: spooky_rv,
+      });
+    }
+
+    // Use CacheModule to handle both local DB and DBSP ingestion
+    if (cacheBatch.length > 0) {
+      await this.cache.saveBatch(cacheBatch);
+    }
+
+    this.events.emit(SyncEventTypes.RemoteDataIngested, {
+      records: remoteResults,
+    });
+  }
+
+  /**
+   * Handle records that exist locally but not in remote array.
+   */
+  private async handleRemovedRecords(removed: RecordId[]): Promise<void> {
+    this.logger.debug(
+      {
+        removed: removed.map((r) => r.toString()),
+        Category: 'spooky-client::SyncEngine::handleRemovedRecords',
+      },
+      'Checking removed records'
+    );
+
+    let existingRemoteIds = new Set<string>();
+    try {
+      const [existingRemote] = await this.remote.query<[{ id: RecordId }[]]>('SELECT id FROM $ids', {
+        ids: removed,
+      });
+      existingRemoteIds = new Set(existingRemote.map((r) => encodeRecordId(r.id)));
+    } catch {
+      // If remote check fails (e.g., SurrealDB parameter serialization issue),
+      // proceed with deletion — the caller has already determined these should be removed
+      this.logger.debug(
+        { Category: 'spooky-client::SyncEngine::handleRemovedRecords' },
+        'Remote existence check failed, proceeding with deletion'
+      );
+    }
+
+    for (const recordId of removed) {
+      const recordIdStr = encodeRecordId(recordId);
+      if (!existingRemoteIds.has(recordIdStr)) {
+        this.logger.debug(
+          {
+            recordId: recordIdStr,
+            Category: 'spooky-client::SyncEngine::handleRemovedRecords',
+          },
+          'Deleting confirmed removed record'
+        );
+
+        // Use CacheModule to handle both local DB and DBSP deletion
+        await this.cache.delete(recordId.table.name, recordIdStr);
+      }
+    }
+  }
+}

package/src/modules/sync/events/index.ts

@@ -0,0 +1,77 @@
+import { createEventSystem, EventDefinition, EventSystem } from '../../../events/index';
+import { RecordVersionArray } from '../../../types';
+
+export const SyncQueueEventTypes = {
+  MutationEnqueued: 'MUTATION_ENQUEUED',
+  MutationDequeued: 'MUTATION_DEQUEUED',
+  QueryItemEnqueued: 'QUERY_ITEM_ENQUEUED',
+} as const;
+
+export type SyncQueueEventTypeMap = {
+  [SyncQueueEventTypes.MutationEnqueued]: EventDefinition<
+    typeof SyncQueueEventTypes.MutationEnqueued,
+    { queueSize: number }
+  >;
+  [SyncQueueEventTypes.MutationDequeued]: EventDefinition<
+    typeof SyncQueueEventTypes.MutationDequeued,
+    { queueSize: number }
+  >;
+  [SyncQueueEventTypes.QueryItemEnqueued]: EventDefinition<
+    typeof SyncQueueEventTypes.QueryItemEnqueued,
+    { queueSize: number }
+  >;
+};
+
+export type SyncQueueEventSystem = EventSystem<SyncQueueEventTypeMap>;
+
+export function createSyncQueueEventSystem(): SyncQueueEventSystem {
+  return createEventSystem([
+    SyncQueueEventTypes.QueryItemEnqueued,
+    SyncQueueEventTypes.MutationEnqueued,
+    SyncQueueEventTypes.MutationDequeued,
+  ]);
+}
+
+export const SyncEventTypes = {
+  QueryUpdated: 'SYNC_QUERY_UPDATED',
+  RemoteDataIngested: 'SYNC_REMOTE_DATA_INGESTED',
+  MutationRolledBack: 'SYNC_MUTATION_ROLLED_BACK',
+} as const;
+
+export type SyncEventTypeMap = {
+  [SyncEventTypes.QueryUpdated]: EventDefinition<
+    typeof SyncEventTypes.QueryUpdated,
+    {
+      queryId: any; // RecordId<string> but imported
+      localHash?: string;
+      localArray?: RecordVersionArray;
+      remoteHash?: string;
+      remoteArray?: RecordVersionArray;
+      records: Record<string, any>[];
+    }
+  >;
+  [SyncEventTypes.RemoteDataIngested]: EventDefinition<
+    typeof SyncEventTypes.RemoteDataIngested,
+    {
+      records: Record<string, any>[];
+    }
+  >;
+  [SyncEventTypes.MutationRolledBack]: EventDefinition<
+    typeof SyncEventTypes.MutationRolledBack,
+    {
+      eventType: string;
+      recordId: string;
+      error: string;
+    }
+  >;
+};
+
+export type SyncEventSystem = EventSystem<SyncEventTypeMap>;
+
+export function createSyncEventSystem(): SyncEventSystem {
+  return createEventSystem([
+    SyncEventTypes.QueryUpdated,
+    SyncEventTypes.RemoteDataIngested,
+    SyncEventTypes.MutationRolledBack,
+  ]);
+}

package/src/modules/sync/index.ts

@@ -0,0 +1,3 @@
+export * from './events/index';
+export * from './sync';
+export * from './queue/index';

package/src/modules/sync/queue/index.ts

@@ -0,0 +1,2 @@
+export * from './queue-up';
+export * from './queue-down';

package/src/modules/sync/queue/queue-down.ts

@@ -0,0 +1,89 @@
+import { LocalDatabaseService } from '../../../services/database/index';
+import {
+  createSyncQueueEventSystem,
+  SyncQueueEventSystem,
+  SyncQueueEventTypes,
+} from '../events/index';
+import { Logger } from '../../../services/logger/index';
+
+export type RegisterEvent = {
+  type: 'register';
+  payload: {
+    hash: string;
+  };
+};
+
+export type SyncEvent = {
+  type: 'sync';
+  payload: {
+    hash: string;
+  };
+};
+
+export type HeartbeatEvent = {
+  type: 'heartbeat';
+  payload: {
+    hash: string;
+  };
+};
+
+export type CleanupEvent = {
+  type: 'cleanup';
+  payload: {
+    hash: string;
+  };
+};
+
+export type DownEvent = RegisterEvent | SyncEvent | HeartbeatEvent | CleanupEvent;
+
+export class DownQueue {
+  private queue: DownEvent[] = [];
+  private _events: SyncQueueEventSystem;
+  private logger: Logger;
+
+  get events(): SyncQueueEventSystem {
+    return this._events;
+  }
+
+  constructor(
+    private local: LocalDatabaseService,
+    logger: Logger
+  ) {
+    this._events = createSyncQueueEventSystem();
+    this.logger = logger.child({ service: 'DownQueue' });
+  }
+
+  get size(): number {
+    return this.queue.length;
+  }
+
+  push(event: DownEvent) {
+    this.queue.push(event);
+    this.emitPushEvent();
+  }
+
+  private emitPushEvent() {
+    this._events.addEvent({
+      type: SyncQueueEventTypes.QueryItemEnqueued,
+      payload: {
+        queueSize: this.queue.length,
+      },
+    });
+  }
+
+  async next(fn: (event: DownEvent) => Promise<void>): Promise<void> {
+    const event = this.queue.shift();
+    if (event) {
+      try {
+        await fn(event);
+      } catch (error) {
+        this.logger.error(
+          { error, event, Category: 'spooky-client::DownQueue::next' },
+          'Failed to process query'
+        );
+        this.queue.unshift(event);
+        throw error;
+      }
+    }
+  }
+}