@spooky-sync/core 0.0.0-canary.1

package/src/modules/sync/queue/queue-up.ts

@@ -0,0 +1,223 @@
+import { RecordId } from 'surrealdb';
+import { LocalDatabaseService } from '../../../services/database/index';
+import {
+  createSyncQueueEventSystem,
+  SyncQueueEventSystem,
+  SyncQueueEventTypes,
+} from '../events/index';
+import { parseRecordIdString, extractTablePart, classifySyncError } from '../../../utils/index';
+import { Logger } from '../../../services/logger/index';
+import { PushEventOptions } from '../../../events/index';
+
+export type CreateEvent = {
+  type: 'create';
+  mutation_id: RecordId;
+  record_id: RecordId;
+  data: Record<string, unknown>;
+  record?: Record<string, unknown>;
+  tableName?: string;
+  options?: PushEventOptions;
+};
+
+export type UpdateEvent = {
+  type: 'update';
+  mutation_id: RecordId;
+  record_id: RecordId;
+  data: Record<string, unknown>;
+  record?: Record<string, unknown>;
+  beforeRecord?: Record<string, unknown>;
+  options?: PushEventOptions;
+};
+
+export type DeleteEvent = {
+  type: 'delete';
+  mutation_id: RecordId;
+  record_id: RecordId;
+  options?: PushEventOptions;
+};
+
+export type UpEvent = CreateEvent | UpdateEvent | DeleteEvent;
+
+export type RollbackCallback = (event: UpEvent, error: Error) => Promise<void>;
+
+export class UpQueue {
+  private queue: UpEvent[] = [];
+  private _events: SyncQueueEventSystem;
+  private logger: Logger;
+  private debouncedMutations: Map<string, { timer: any; firstBeforeRecord?: Record<string, unknown> }>;
+
+  get events(): SyncQueueEventSystem {
+    return this._events;
+  }
+
+  constructor(
+    private local: LocalDatabaseService,
+    logger: Logger
+  ) {
+    this._events = createSyncQueueEventSystem();
+    this.logger = logger.child({ service: 'UpQueue' });
+    this.debouncedMutations = new Map();
+  }
+
+  get size(): number {
+    return this.queue.length;
+  }
+
+  push(event: UpEvent) {
+    if (event.options?.debounced) {
+      const { key, delay } = event.options.debounced;
+      this.handleDebouncedMutation(event, key, delay);
+      return;
+    }
+    this.addToQueue(event);
+  }
+
+  private addToQueue(event: UpEvent) {
+    this.queue.push(event);
+    this._events.addEvent({
+      type: SyncQueueEventTypes.MutationEnqueued,
+      payload: { queueSize: this.queue.length },
+    });
+  }
+
+  private handleDebouncedMutation(event: UpEvent, key: string, delay: number) {
+    const existing = this.debouncedMutations.get(key);
+    let firstBeforeRecord: Record<string, unknown> | undefined;
+
+    if (existing) {
+      clearTimeout(existing.timer);
+      // Preserve the beforeRecord from the first event in the debounce sequence
+      firstBeforeRecord = existing.firstBeforeRecord;
+    } else if (event.type === 'update') {
+      firstBeforeRecord = event.beforeRecord;
+    }
+
+    const timer = setTimeout(() => {
+      this.debouncedMutations.delete(key);
+      // Attach the first beforeRecord to the final debounced event
+      if (firstBeforeRecord && event.type === 'update') {
+        event.beforeRecord = firstBeforeRecord;
+      }
+      this.addToQueue(event);
+    }, delay);
+
+    this.debouncedMutations.set(key, { timer, firstBeforeRecord });
+  }
+
+  async next(fn: (event: UpEvent) => Promise<void>, onRollback?: RollbackCallback): Promise<void> {
+    const event = this.queue.shift();
+    if (event) {
+      try {
+        await fn(event);
+      } catch (error) {
+        const errorType = classifySyncError(error);
+
+        if (errorType === 'network') {
+          this.logger.error(
+            { error, event, Category: 'spooky-client::UpQueue::next' },
+            'Network error processing mutation, re-queuing'
+          );
+          this.queue.unshift(event);
+          throw error;
+        }
+
+        // Application error — rollback instead of re-queuing
+        this.logger.error(
+          { error, event, Category: 'spooky-client::UpQueue::next' },
+          'Application error processing mutation, rolling back'
+        );
+        try {
+          await this.removeEventFromDatabase(event.mutation_id);
+        } catch (removeError) {
+          this.logger.error(
+            { error: removeError, event, Category: 'spooky-client::UpQueue::next' },
+            'Failed to remove rolled-back mutation from database'
+          );
+        }
+        if (onRollback) {
+          try {
+            await onRollback(event, error instanceof Error ? error : new Error(String(error)));
+          } catch (rollbackError) {
+            this.logger.error(
+              { error: rollbackError, event, Category: 'spooky-client::UpQueue::next' },
+              'Rollback handler failed'
+            );
+          }
+        }
+        this._events.addEvent({
+          type: SyncQueueEventTypes.MutationDequeued,
+          payload: { queueSize: this.queue.length },
+        });
+        return;
+      }
+      try {
+        await this.removeEventFromDatabase(event.mutation_id);
+      } catch (error) {
+        this.logger.error(
+          { error, event, Category: 'spooky-client::UpQueue::next' },
+          'Failed to remove mutation from database after successful processing'
+        );
+      }
+      this._events.addEvent({
+        type: SyncQueueEventTypes.MutationDequeued,
+        payload: { queueSize: this.queue.length },
+      });
+    }
+  }
+
+  async removeEventFromDatabase(mutation_id: RecordId) {
+    return this.local.query(`DELETE $mutation_id`, { mutation_id });
+  }
+
+  async loadFromDatabase() {
+    try {
+      const [records] = await this.local.query<any>(
+        `SELECT * FROM _spooky_pending_mutations ORDER BY created_at ASC`
+      );
+
+      this.queue = records
+        .map((r: any): UpEvent | null => {
+          switch (r.mutationType) {
+            case 'create':
+              return {
+                type: 'create',
+                mutation_id: parseRecordIdString(r.id),
+                record_id: parseRecordIdString(r.recordId),
+                data: r.data,
+                tableName: extractTablePart(r.recordId),
+              };
+            case 'update':
+              return {
+                type: 'update',
+                mutation_id: parseRecordIdString(r.id),
+                record_id: parseRecordIdString(r.recordId),
+                data: r.data,
+                beforeRecord: r.beforeRecord,
+              };
+            case 'delete':
+              return {
+                type: 'delete',
+                mutation_id: parseRecordIdString(r.id),
+                record_id: parseRecordIdString(r.recordId),
+              };
+            default:
+              this.logger.warn(
+                {
+                  mutationType: r.mutationType,
+                  record: r,
+                  Category: 'spooky-client::UpQueue::loadFromDatabase',
+                },
+                'Unknown mutation type'
+              );
+              return null;
+          }
+        })
+        .filter((e: UpEvent | null): e is UpEvent => e !== null);
+    } catch (error) {
+      this.logger.error(
+        { error, Category: 'spooky-client::UpQueue::loadFromDatabase' },
+        'Failed to load pending mutations from database'
+      );
+    }
+  }
+}

package/src/modules/sync/scheduler.ts

@@ -0,0 +1,84 @@
+import { Logger } from '../../services/logger/index';
+import { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
+import { SyncQueueEventTypes } from './events/index';
+
+/**
+ * SyncScheduler manages when to sync: queue management and orchestration.
+ * Decides the order and timing of sync operations.
+ */
+export class SyncScheduler {
+  private isSyncingUp: boolean = false;
+  private isSyncingDown: boolean = false;
+
+  constructor(
+    private upQueue: UpQueue,
+    private downQueue: DownQueue,
+    private onProcessUp: (event: UpEvent) => Promise<void>,
+    private onProcessDown: (event: DownEvent) => Promise<void>,
+    private logger: Logger,
+    private onRollback?: RollbackCallback
+  ) {}
+
+  async init() {
+    await this.upQueue.loadFromDatabase();
+    this.upQueue.events.subscribe(SyncQueueEventTypes.MutationEnqueued, this.syncUp.bind(this));
+    this.downQueue.events.subscribe(
+      SyncQueueEventTypes.QueryItemEnqueued,
+      this.syncDown.bind(this)
+    );
+  }
+
+  /**
+   * Add mutations to the upload queue
+   */
+  enqueueMutation(mutations: UpEvent[]) {
+    for (const mutation of mutations) {
+      this.upQueue.push(mutation);
+    }
+  }
+
+  /**
+   * Add query events to the download queue
+   */
+  enqueueDownEvent(event: DownEvent) {
+    this.downQueue.push(event);
+  }
+
+  /**
+   * Process upload queue
+   */
+  async syncUp() {
+    if (this.isSyncingUp) return;
+    this.isSyncingUp = true;
+    try {
+      while (this.upQueue.size > 0) {
+        await this.upQueue.next(this.onProcessUp, this.onRollback);
+      }
+    } finally {
+      this.isSyncingUp = false;
+      void this.syncDown();
+    }
+  }
+
+  /**
+   * Process download queue
+   */
+  async syncDown() {
+    if (this.isSyncingDown) return;
+    if (this.upQueue.size > 0) return;
+
+    this.isSyncingDown = true;
+    try {
+      while (this.downQueue.size > 0) {
+        if (this.upQueue.size > 0) break;
+        await this.downQueue.next(this.onProcessDown);
+      }
+    } finally {
+      this.isSyncingDown = false;
+    }
+  }
+
+  get isSyncing() {
+    return this.isSyncingUp || this.isSyncingDown;
+  }
+}

package/src/modules/sync/sync.ts

@@ -0,0 +1,407 @@
+import { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
+import { MutationEvent, RecordVersionArray } from '../../types';
+import { createSyncEventSystem, SyncEventTypes, SyncQueueEventTypes } from './events/index';
+import { Logger } from '../../services/logger/index';
+import { DownEvent, DownQueue, UpEvent, UpQueue } from './queue/index';
+import { RecordId, Uuid } from 'surrealdb';
+import { ArraySyncer, createDiffFromDbOp } from './utils';
+import { SyncEngine } from './engine';
+import { SyncScheduler } from './scheduler';
+import { SchemaStructure } from '@spooky-sync/query-builder';
+import { CacheModule } from '../cache/index';
+import { DataModule } from '../data/index';
+import { encodeRecordId, extractTablePart, parseDuration, surql } from '../../utils/index';
+
+/**
+ * The main synchronization engine for Spooky.
+ * Handles the bidirectional synchronization between the local database and the remote backend.
+ * Uses a queue-based architecture with 'up' (local to remote) and 'down' (remote to local) queues.
+ * @template S The schema structure type.
+ */
+export class SpookySync<S extends SchemaStructure> {
+  private clientId: string = '';
+  private upQueue: UpQueue;
+  private downQueue: DownQueue;
+  private isInit: boolean = false;
+  private logger: Logger;
+  private syncEngine: SyncEngine;
+  private scheduler: SyncScheduler;
+  public events = createSyncEventSystem();
+
+  get isSyncing() {
+    return this.scheduler.isSyncing;
+  }
+
+  get pendingMutationCount(): number {
+    return this.upQueue.size;
+  }
+
+  subscribeToPendingMutations(cb: (count: number) => void): () => void {
+    const id1 = this.upQueue.events.subscribe(
+      SyncQueueEventTypes.MutationEnqueued,
+      (event) => cb(event.payload.queueSize)
+    );
+    const id2 = this.upQueue.events.subscribe(
+      SyncQueueEventTypes.MutationDequeued,
+      (event) => cb(event.payload.queueSize)
+    );
+    return () => {
+      this.upQueue.events.unsubscribe(id1);
+      this.upQueue.events.unsubscribe(id2);
+    };
+  }
+
+  constructor(
+    private local: LocalDatabaseService,
+    private remote: RemoteDatabaseService,
+    private cache: CacheModule,
+    private dataModule: DataModule<S>,
+    private schema: S,
+    logger: Logger
+  ) {
+    this.logger = logger.child({ service: 'SpookySync' });
+    this.upQueue = new UpQueue(this.local, this.logger);
+    this.downQueue = new DownQueue(this.local, this.logger);
+    this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
+    this.scheduler = new SyncScheduler(
+      this.upQueue,
+      this.downQueue,
+      this.processUpEvent.bind(this),
+      this.processDownEvent.bind(this),
+      this.logger,
+      this.handleRollback.bind(this)
+    );
+  }
+
+  /**
+   * Initializes the synchronization system.
+   * Starts the scheduler and initiates the initial sync cycles.
+   * @param clientId The unique identifier for this client instance.
+   * @throws Error if already initialized.
+   */
+  public async init(clientId: string) {
+    if (this.isInit) throw new Error('SpookySync is already initialized');
+    this.clientId = clientId;
+    this.isInit = true;
+    await this.scheduler.init();
+    void this.scheduler.syncUp();
+    void this.scheduler.syncUp();
+    void this.scheduler.syncDown();
+    void this.startRefLiveQueries();
+  }
+
+  private async startRefLiveQueries() {
+    this.logger.debug(
+      { clientId: this.clientId, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
+      'Starting ref live queries'
+    );
+
+    const [queryUuid] = await this.remote.query<[Uuid]>(
+      'LIVE SELECT * FROM _spooky_list_ref'
+    );
+
+    (await this.remote.getClient().liveOf(queryUuid)).subscribe((message) => {
+      this.logger.debug(
+        { message, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
+        'Live update received'
+      );
+      if (message.action === 'KILLED') return;
+      this.handleRemoteListRefChange(
+        message.action,
+        message.value.in as RecordId<string>,
+        message.value.out as RecordId<string>,
+        message.value.version as number
+      ).catch((err) => {
+        this.logger.error(
+          { err, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
+          'Error handling remote list ref change'
+        );
+      });
+    });
+  }
+
+  private async handleRemoteListRefChange(
+    action: 'CREATE' | 'UPDATE' | 'DELETE',
+    queryId: RecordId,
+    recordId: RecordId,
+    version: number
+  ) {
+    const existing = this.dataModule.getQueryById(queryId);
+
+    if (!existing) {
+      this.logger.warn(
+        {
+          queryId: queryId.toString(),
+          Category: 'spooky-client::SpookySync::handleRemoteListRefChange',
+        },
+        'Received remote update for unknown local query'
+      );
+      return;
+    }
+
+    const { localArray } = existing.config;
+
+    this.logger.debug(
+      {
+        action,
+        queryId,
+        recordId,
+        version,
+        localArray,
+        Category: 'spooky-client::SpookySync::handleRemoteListRefChange',
+      },
+      'Live update is being processed'
+    );
+    const diff = createDiffFromDbOp(action, recordId, version, localArray);
+    await this.syncEngine.syncRecords(diff);
+  }
+
+  /**
+   * Enqueues a 'down' event (from remote to local) for processing.
+   * @param event The DownEvent to enqueue.
+   */
+  public enqueueDownEvent(event: DownEvent) {
+    this.scheduler.enqueueDownEvent(event);
+  }
+
+  private async processUpEvent(event: UpEvent) {
+    this.logger.debug(
+      { event, Category: 'spooky-client::SpookySync::processUpEvent' },
+      'Processing up event'
+    );
+    console.log('xx1', event);
+    switch (event.type) {
+      case 'create':
+        const dataKeys = Object.keys(event.data).map((key) => ({ key, variable: `data_${key}` }));
+        const prefixedParams = Object.fromEntries(
+          dataKeys.map(({ key, variable }) => [variable, event.data[key]])
+        );
+        const query = surql.seal(surql.createSet('id', dataKeys));
+        await this.remote.query(query, {
+          id: event.record_id,
+          ...prefixedParams,
+        });
+        break;
+      case 'update':
+        await this.remote.query(`UPDATE $id MERGE $data`, {
+          id: event.record_id,
+          data: event.data,
+        });
+        break;
+      case 'delete':
+        await this.remote.query(`DELETE $id`, {
+          id: event.record_id,
+        });
+        break;
+      default:
+        this.logger.error(
+          { event, Category: 'spooky-client::SpookySync::processUpEvent' },
+          'processUpEvent unknown event type'
+        );
+        return;
+    }
+  }
+
+  private async handleRollback(event: UpEvent, error: Error): Promise<void> {
+    const recordId = encodeRecordId(event.record_id);
+    const tableName =
+      event.type === 'create' && event.tableName
+        ? event.tableName
+        : extractTablePart(recordId);
+
+    this.logger.warn(
+      {
+        type: event.type,
+        recordId,
+        tableName,
+        error: error.message,
+        Category: 'spooky-client::SpookySync::handleRollback',
+      },
+      'Rolling back failed mutation'
+    );
+
+    switch (event.type) {
+      case 'create':
+        await this.dataModule.rollbackCreate(event.record_id, tableName);
+        break;
+      case 'update':
+        if (event.beforeRecord) {
+          await this.dataModule.rollbackUpdate(event.record_id, tableName, event.beforeRecord);
+        } else {
+          this.logger.warn(
+            {
+              recordId,
+              Category: 'spooky-client::SpookySync::handleRollback',
+            },
+            'Cannot rollback update: no beforeRecord available. Down-sync will reconcile.'
+          );
+        }
+        break;
+      case 'delete':
+        this.logger.warn(
+          {
+            recordId,
+            Category: 'spooky-client::SpookySync::handleRollback',
+          },
+          'Delete rollback not implemented. Down-sync will reconcile.'
+        );
+        break;
+    }
+
+    this.events.emit(SyncEventTypes.MutationRolledBack, {
+      eventType: event.type,
+      recordId,
+      error: error.message,
+    });
+  }
+
+  private async processDownEvent(event: DownEvent) {
+    this.logger.debug(
+      { event, Category: 'spooky-client::SpookySync::processDownEvent' },
+      'Processing down event'
+    );
+    switch (event.type) {
+      case 'register':
+        return this.registerQuery(event.payload.hash);
+      case 'sync':
+        return this.syncQuery(event.payload.hash);
+      case 'heartbeat':
+        return this.heartbeatQuery(event.payload.hash);
+      case 'cleanup':
+        return this.cleanupQuery(event.payload.hash);
+    }
+  }
+
+  /**
+   * Synchronizes a specific query by hash.
+   * Compares local and remote version arrays and fetches differences.
+   * @param hash The hash of the query to sync.
+   */
+  public async syncQuery(hash: string) {
+    const queryState = this.dataModule.getQueryByHash(hash);
+    if (!queryState) {
+      this.logger.warn(
+        { hash, Category: 'spooky-client::SpookySync::syncQuery' },
+        'Query not found'
+      );
+      return;
+    }
+
+    const diff = new ArraySyncer(
+      queryState.config.localArray,
+      queryState.config.remoteArray
+    ).nextSet();
+
+    if (!diff) {
+      return;
+    }
+    return this.syncEngine.syncRecords(diff);
+  }
+
+  /**
+   * Enqueues a list of mutations (up events) to be sent to the remote.
+   * @param mutations Array of UpEvents (create/update/delete) to enqueue.
+   */
+  public async enqueueMutation(mutations: UpEvent[]) {
+    this.scheduler.enqueueMutation(mutations);
+  }
+
+  private async registerQuery(queryHash: string) {
+    try {
+      this.logger.debug(
+        { queryHash, Category: 'spooky-client::SpookySync::registerQuery' },
+        'Register Query state'
+      );
+      await this.createRemoteQuery(queryHash);
+      await this.syncQuery(queryHash);
+    } catch (e) {
+      this.logger.error(
+        { err: e, Category: 'spooky-client::SpookySync::registerQuery' },
+        'registerQuery error'
+      );
+      throw e;
+    }
+  }
+
+  private async createRemoteQuery(queryHash: string) {
+    const queryState = this.dataModule.getQueryByHash(queryHash);
+
+    if (!queryState) {
+      this.logger.warn(
+        { queryHash, Category: 'spooky-client::SpookySync::createRemoteQuery' },
+        'Query to register not found'
+      );
+      throw new Error('Query to register not found');
+    }
+    // Delegate to remote function which handles DBSP registration & persistence
+    await this.remote.query('fn::query::register($config)', {
+      config: {
+        clientId: this.clientId,
+        id: queryState.config.id,
+        surql: queryState.config.surql,
+        params: queryState.config.params,
+        ttl: queryState.config.ttl,
+      },
+    });
+
+    const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
+      surql.selectByFieldsAnd('_spooky_list_ref', ['in'], ['out', 'version']),
+      {
+        in: queryState.config.id,
+      }
+    );
+
+    this.logger.trace(
+      {
+        queryId: encodeRecordId(queryState.config.id),
+        items,
+        Category: 'spooky-client::SpookySync::createRemoteQuery',
+      },
+      'Got query record version array from remote'
+    );
+
+    const array: RecordVersionArray = items.map((item) => [encodeRecordId(item.out), item.version]);
+
+    this.logger.debug(
+      {
+        queryId: encodeRecordId(queryState.config.id),
+        array,
+        Category: 'spooky-client::SpookySync::createRemoteQuery',
+      },
+      'createdRemoteQuery'
+    );
+
+    if (array) {
+      /// Incantation existed already
+      await this.dataModule.updateQueryRemoteArray(queryHash, array);
+    }
+  }
+
+  private async heartbeatQuery(queryHash: string) {
+    const queryState = this.dataModule.getQueryByHash(queryHash);
+    if (!queryState) {
+      this.logger.warn(
+        { queryHash, Category: 'spooky-client::SpookySync::heartbeatQuery' },
+        'Query to register not found'
+      );
+      throw new Error('Query to register not found');
+    }
+    await this.remote.query('fn::query::heartbeat($id)', {
+      id: queryState.config.id,
+    });
+  }
+
+  private async cleanupQuery(queryHash: string) {
+    const queryState = this.dataModule.getQueryByHash(queryHash);
+    if (!queryState) {
+      this.logger.warn(
+        { queryHash, Category: 'spooky-client::SpookySync::cleanupQuery' },
+        'Query to register not found'
+      );
+      throw new Error('Query to register not found');
+    }
+    await this.remote.query(`DELETE $id`, {
+      id: queryState.config.id,
+    });
+  }
+}