@spooky-sync/core 0.0.1-canary.29 → 0.0.1-canary.290

package/src/modules/sync/engine.ts

@@ -1,173 +0,0 @@
-import { RecordId } from 'surrealdb';
-import { SchemaStructure } from '@spooky-sync/query-builder';
-import { RemoteDatabaseService } from '../../services/database/index';
-import { CacheModule, CacheRecord, RecordWithId } from '../cache/index';
-import { RecordVersionDiff } from '../../types';
-import { Logger } from '../../services/logger/index';
-import { SyncEventTypes, createSyncEventSystem } from './events/index';
-import { encodeRecordId } from '../../utils/index';
-import { cleanRecord } from '../../utils/parser';
-
-/**
- * SyncEngine handles the core sync operations: fetching remote records,
- * caching them locally, and ingesting into DBSP.
- *
- * This is extracted from Sp00kySync to separate "how to sync" from "when to sync".
- */
-export class SyncEngine {
-  private logger: Logger;
-  public events = createSyncEventSystem();
-
-  constructor(
-    private remote: RemoteDatabaseService,
-    private cache: CacheModule,
-    private schema: SchemaStructure,
-    logger: Logger
-  ) {
-    this.logger = logger.child({ service: 'Sp00kySync:SyncEngine' });
-  }
-
-  /**
-   * Sync missing/updated/removed records between local and remote.
-   * Main entry point for sync operations.
-   * Uses batch processing to minimize events emitted.
-   */
-  async syncRecords(diff: RecordVersionDiff): Promise<void> {
-    const { added, updated, removed } = diff;
-
-    this.logger.debug(
-      {
-        added,
-        updated,
-        removed,
-        Category: 'sp00ky-client::SyncEngine::syncRecords',
-      },
-      'SyncEngine.syncRecords diff'
-    );
-
-    // Handle removed records: verify they don't exist remotely before deleting locally
-    if (removed.length > 0) {
-      await this.handleRemovedRecords(removed);
-    }
-
-    // Fetch added/updated records from remote
-    const toFetch = [...added, ...updated];
-    const idsToFetch = toFetch.map((x) => x.id);
-    if (idsToFetch.length === 0) {
-      return;
-    }
-
-    // Build a version map from the diff (versions come from _00_list_ref)
-    const versionMap = new Map<string, number>();
-    for (const item of toFetch) {
-      versionMap.set(encodeRecordId(item.id), item.version);
-    }
-
-    // Fetch records from remote — avoid SELECT *, <subquery> FROM $param
-    // pattern which drops the * fields in SurrealDB v3 (known bug).
-    // Versions are already known from the diff's list_ref data.
-    const [remoteResults] = await this.remote.query<[RecordWithId[]]>(
-      'SELECT * FROM $idsToFetch',
-      { idsToFetch }
-    );
-
-    // Prepare batch for cache (which handles both DB and DBSP)
-    const cacheBatch: CacheRecord[] = [];
-
-    for (const record of remoteResults) {
-      if (!record?.id) {
-        this.logger.warn(
-          {
-            record,
-            idsToFetch,
-            Category: 'sp00ky-client::SyncEngine::syncRecords',
-          },
-          'Remote record has no id (possibly deleted). Skipping record'
-        );
-        continue;
-      }
-      const fullId = encodeRecordId(record.id);
-      const table = record.id.table.toString();
-      const isAdded = added.some((item) => encodeRecordId(item.id) === fullId);
-      const version = versionMap.get(fullId) ?? 0;
-
-      const localVersion = this.cache.lookup(fullId);
-      if (localVersion && version <= localVersion) {
-        this.logger.info(
-          {
-            recordId: fullId,
-            version,
-            localVersion,
-            Category: 'sp00ky-client::SyncEngine::syncRecords',
-          },
-          'Local version is higher than remote version. Skipping record'
-        );
-        continue;
-      }
-      const tableSchema = this.schema.tables.find((t) => t.name === table);
-      const cleanedRecord = tableSchema
-        ? cleanRecord(tableSchema.columns, record)
-        : record;
-
-      cacheBatch.push({
-        table,
-        op: isAdded ? 'CREATE' : 'UPDATE',
-        record: cleanedRecord as RecordWithId,
-        version,
-      });
-    }
-
-    // Use CacheModule to handle both local DB and DBSP ingestion
-    if (cacheBatch.length > 0) {
-      await this.cache.saveBatch(cacheBatch);
-    }
-
-    this.events.emit(SyncEventTypes.RemoteDataIngested, {
-      records: remoteResults,
-    });
-  }
-
-  /**
-   * Handle records that exist locally but not in remote array.
-   */
-  private async handleRemovedRecords(removed: RecordId[]): Promise<void> {
-    this.logger.debug(
-      {
-        removed: removed.map((r) => r.toString()),
-        Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
-      },
-      'Checking removed records'
-    );
-
-    let existingRemoteIds = new Set<string>();
-    try {
-      const [existingRemote] = await this.remote.query<[{ id: RecordId }[]]>('SELECT id FROM $ids', {
-        ids: removed,
-      });
-      existingRemoteIds = new Set(existingRemote.map((r) => encodeRecordId(r.id)));
-    } catch {
-      // If remote check fails (e.g., SurrealDB parameter serialization issue),
-      // proceed with deletion — the caller has already determined these should be removed
-      this.logger.debug(
-        { Category: 'sp00ky-client::SyncEngine::handleRemovedRecords' },
-        'Remote existence check failed, proceeding with deletion'
-      );
-    }
-
-    for (const recordId of removed) {
-      const recordIdStr = encodeRecordId(recordId);
-      if (!existingRemoteIds.has(recordIdStr)) {
-        this.logger.debug(
-          {
-            recordId: recordIdStr,
-            Category: 'sp00ky-client::SyncEngine::handleRemovedRecords',
-          },
-          'Deleting confirmed removed record'
-        );
-
-        // Use CacheModule to handle both local DB and DBSP deletion
-        await this.cache.delete(recordId.table.name, recordIdStr);
-      }
-    }
-  }
-}

package/src/modules/sync/events/index.ts

@@ -1,77 +0,0 @@
-import { createEventSystem, EventDefinition, EventSystem } from '../../../events/index';
-import { RecordVersionArray } from '../../../types';
-
-export const SyncQueueEventTypes = {
-  MutationEnqueued: 'MUTATION_ENQUEUED',
-  MutationDequeued: 'MUTATION_DEQUEUED',
-  QueryItemEnqueued: 'QUERY_ITEM_ENQUEUED',
-} as const;
-
-export type SyncQueueEventTypeMap = {
-  [SyncQueueEventTypes.MutationEnqueued]: EventDefinition<
-    typeof SyncQueueEventTypes.MutationEnqueued,
-    { queueSize: number }
-  >;
-  [SyncQueueEventTypes.MutationDequeued]: EventDefinition<
-    typeof SyncQueueEventTypes.MutationDequeued,
-    { queueSize: number }
-  >;
-  [SyncQueueEventTypes.QueryItemEnqueued]: EventDefinition<
-    typeof SyncQueueEventTypes.QueryItemEnqueued,
-    { queueSize: number }
-  >;
-};
-
-export type SyncQueueEventSystem = EventSystem<SyncQueueEventTypeMap>;
-
-export function createSyncQueueEventSystem(): SyncQueueEventSystem {
-  return createEventSystem([
-    SyncQueueEventTypes.QueryItemEnqueued,
-    SyncQueueEventTypes.MutationEnqueued,
-    SyncQueueEventTypes.MutationDequeued,
-  ]);
-}
-
-export const SyncEventTypes = {
-  QueryUpdated: 'SYNC_QUERY_UPDATED',
-  RemoteDataIngested: 'SYNC_REMOTE_DATA_INGESTED',
-  MutationRolledBack: 'SYNC_MUTATION_ROLLED_BACK',
-} as const;
-
-export type SyncEventTypeMap = {
-  [SyncEventTypes.QueryUpdated]: EventDefinition<
-    typeof SyncEventTypes.QueryUpdated,
-    {
-      queryId: any; // RecordId<string> but imported
-      localHash?: string;
-      localArray?: RecordVersionArray;
-      remoteHash?: string;
-      remoteArray?: RecordVersionArray;
-      records: Record<string, any>[];
-    }
-  >;
-  [SyncEventTypes.RemoteDataIngested]: EventDefinition<
-    typeof SyncEventTypes.RemoteDataIngested,
-    {
-      records: Record<string, any>[];
-    }
-  >;
-  [SyncEventTypes.MutationRolledBack]: EventDefinition<
-    typeof SyncEventTypes.MutationRolledBack,
-    {
-      eventType: string;
-      recordId: string;
-      error: string;
-    }
-  >;
-};
-
-export type SyncEventSystem = EventSystem<SyncEventTypeMap>;
-
-export function createSyncEventSystem(): SyncEventSystem {
-  return createEventSystem([
-    SyncEventTypes.QueryUpdated,
-    SyncEventTypes.RemoteDataIngested,
-    SyncEventTypes.MutationRolledBack,
-  ]);
-}

package/src/modules/sync/index.ts

@@ -1,3 +0,0 @@
-export * from './events/index';
-export * from './sync';
-export * from './queue/index';

package/src/modules/sync/queue/index.ts

@@ -1,2 +0,0 @@
-export * from './queue-up';
-export * from './queue-down';

package/src/modules/sync/queue/queue-down.ts

@@ -1,89 +0,0 @@
-import { LocalDatabaseService } from '../../../services/database/index';
-import {
-  createSyncQueueEventSystem,
-  SyncQueueEventSystem,
-  SyncQueueEventTypes,
-} from '../events/index';
-import { Logger } from '../../../services/logger/index';
-
-export type RegisterEvent = {
-  type: 'register';
-  payload: {
-    hash: string;
-  };
-};
-
-export type SyncEvent = {
-  type: 'sync';
-  payload: {
-    hash: string;
-  };
-};
-
-export type HeartbeatEvent = {
-  type: 'heartbeat';
-  payload: {
-    hash: string;
-  };
-};
-
-export type CleanupEvent = {
-  type: 'cleanup';
-  payload: {
-    hash: string;
-  };
-};
-
-export type DownEvent = RegisterEvent | SyncEvent | HeartbeatEvent | CleanupEvent;
-
-export class DownQueue {
-  private queue: DownEvent[] = [];
-  private _events: SyncQueueEventSystem;
-  private logger: Logger;
-
-  get events(): SyncQueueEventSystem {
-    return this._events;
-  }
-
-  constructor(
-    private local: LocalDatabaseService,
-    logger: Logger
-  ) {
-    this._events = createSyncQueueEventSystem();
-    this.logger = logger.child({ service: 'DownQueue' });
-  }
-
-  get size(): number {
-    return this.queue.length;
-  }
-
-  push(event: DownEvent) {
-    this.queue.push(event);
-    this.emitPushEvent();
-  }
-
-  private emitPushEvent() {
-    this._events.addEvent({
-      type: SyncQueueEventTypes.QueryItemEnqueued,
-      payload: {
-        queueSize: this.queue.length,
-      },
-    });
-  }
-
-  async next(fn: (event: DownEvent) => Promise<void>): Promise<void> {
-    const event = this.queue.shift();
-    if (event) {
-      try {
-        await fn(event);
-      } catch (error) {
-        this.logger.error(
-          { error, event, Category: 'sp00ky-client::DownQueue::next' },
-          'Failed to process query'
-        );
-        this.queue.unshift(event);
-        throw error;
-      }
-    }
-  }
-}

package/src/modules/sync/queue/queue-up.ts

@@ -1,223 +0,0 @@
-import { RecordId } from 'surrealdb';
-import { LocalDatabaseService } from '../../../services/database/index';
-import {
-  createSyncQueueEventSystem,
-  SyncQueueEventSystem,
-  SyncQueueEventTypes,
-} from '../events/index';
-import { parseRecordIdString, extractTablePart, classifySyncError } from '../../../utils/index';
-import { Logger } from '../../../services/logger/index';
-import { PushEventOptions } from '../../../events/index';
-
-export type CreateEvent = {
-  type: 'create';
-  mutation_id: RecordId;
-  record_id: RecordId;
-  data: Record<string, unknown>;
-  record?: Record<string, unknown>;
-  tableName?: string;
-  options?: PushEventOptions;
-};
-
-export type UpdateEvent = {
-  type: 'update';
-  mutation_id: RecordId;
-  record_id: RecordId;
-  data: Record<string, unknown>;
-  record?: Record<string, unknown>;
-  beforeRecord?: Record<string, unknown>;
-  options?: PushEventOptions;
-};
-
-export type DeleteEvent = {
-  type: 'delete';
-  mutation_id: RecordId;
-  record_id: RecordId;
-  options?: PushEventOptions;
-};
-
-export type UpEvent = CreateEvent | UpdateEvent | DeleteEvent;
-
-export type RollbackCallback = (event: UpEvent, error: Error) => Promise<void>;
-
-export class UpQueue {
-  private queue: UpEvent[] = [];
-  private _events: SyncQueueEventSystem;
-  private logger: Logger;
-  private debouncedMutations: Map<string, { timer: any; firstBeforeRecord?: Record<string, unknown> }>;
-
-  get events(): SyncQueueEventSystem {
-    return this._events;
-  }
-
-  constructor(
-    private local: LocalDatabaseService,
-    logger: Logger
-  ) {
-    this._events = createSyncQueueEventSystem();
-    this.logger = logger.child({ service: 'UpQueue' });
-    this.debouncedMutations = new Map();
-  }
-
-  get size(): number {
-    return this.queue.length;
-  }
-
-  push(event: UpEvent) {
-    if (event.options?.debounced) {
-      const { key, delay } = event.options.debounced;
-      this.handleDebouncedMutation(event, key, delay);
-      return;
-    }
-    this.addToQueue(event);
-  }
-
-  private addToQueue(event: UpEvent) {
-    this.queue.push(event);
-    this._events.addEvent({
-      type: SyncQueueEventTypes.MutationEnqueued,
-      payload: { queueSize: this.queue.length },
-    });
-  }
-
-  private handleDebouncedMutation(event: UpEvent, key: string, delay: number) {
-    const existing = this.debouncedMutations.get(key);
-    let firstBeforeRecord: Record<string, unknown> | undefined;
-
-    if (existing) {
-      clearTimeout(existing.timer);
-      // Preserve the beforeRecord from the first event in the debounce sequence
-      firstBeforeRecord = existing.firstBeforeRecord;
-    } else if (event.type === 'update') {
-      firstBeforeRecord = event.beforeRecord;
-    }
-
-    const timer = setTimeout(() => {
-      this.debouncedMutations.delete(key);
-      // Attach the first beforeRecord to the final debounced event
-      if (firstBeforeRecord && event.type === 'update') {
-        event.beforeRecord = firstBeforeRecord;
-      }
-      this.addToQueue(event);
-    }, delay);
-
-    this.debouncedMutations.set(key, { timer, firstBeforeRecord });
-  }
-
-  async next(fn: (event: UpEvent) => Promise<void>, onRollback?: RollbackCallback): Promise<void> {
-    const event = this.queue.shift();
-    if (event) {
-      try {
-        await fn(event);
-      } catch (error) {
-        const errorType = classifySyncError(error);
-
-        if (errorType === 'network') {
-          this.logger.error(
-            { error, event, Category: 'sp00ky-client::UpQueue::next' },
-            'Network error processing mutation, re-queuing'
-          );
-          this.queue.unshift(event);
-          throw error;
-        }
-
-        // Application error — rollback instead of re-queuing
-        this.logger.error(
-          { error, event, Category: 'sp00ky-client::UpQueue::next' },
-          'Application error processing mutation, rolling back'
-        );
-        try {
-          await this.removeEventFromDatabase(event.mutation_id);
-        } catch (removeError) {
-          this.logger.error(
-            { error: removeError, event, Category: 'sp00ky-client::UpQueue::next' },
-            'Failed to remove rolled-back mutation from database'
-          );
-        }
-        if (onRollback) {
-          try {
-            await onRollback(event, error instanceof Error ? error : new Error(String(error)));
-          } catch (rollbackError) {
-            this.logger.error(
-              { error: rollbackError, event, Category: 'sp00ky-client::UpQueue::next' },
-              'Rollback handler failed'
-            );
-          }
-        }
-        this._events.addEvent({
-          type: SyncQueueEventTypes.MutationDequeued,
-          payload: { queueSize: this.queue.length },
-        });
-        return;
-      }
-      try {
-        await this.removeEventFromDatabase(event.mutation_id);
-      } catch (error) {
-        this.logger.error(
-          { error, event, Category: 'sp00ky-client::UpQueue::next' },
-          'Failed to remove mutation from database after successful processing'
-        );
-      }
-      this._events.addEvent({
-        type: SyncQueueEventTypes.MutationDequeued,
-        payload: { queueSize: this.queue.length },
-      });
-    }
-  }
-
-  async removeEventFromDatabase(mutation_id: RecordId) {
-    return this.local.query(`DELETE $mutation_id`, { mutation_id });
-  }
-
-  async loadFromDatabase() {
-    try {
-      const [records] = await this.local.query<any>(
-        `SELECT * FROM _00_pending_mutations ORDER BY created_at ASC`
-      );
-
-      this.queue = records
-        .map((r: any): UpEvent | null => {
-          switch (r.mutationType) {
-            case 'create':
-              return {
-                type: 'create',
-                mutation_id: parseRecordIdString(r.id),
-                record_id: parseRecordIdString(r.recordId),
-                data: r.data,
-                tableName: extractTablePart(r.recordId),
-              };
-            case 'update':
-              return {
-                type: 'update',
-                mutation_id: parseRecordIdString(r.id),
-                record_id: parseRecordIdString(r.recordId),
-                data: r.data,
-                beforeRecord: r.beforeRecord,
-              };
-            case 'delete':
-              return {
-                type: 'delete',
-                mutation_id: parseRecordIdString(r.id),
-                record_id: parseRecordIdString(r.recordId),
-              };
-            default:
-              this.logger.warn(
-                {
-                  mutationType: r.mutationType,
-                  record: r,
-                  Category: 'sp00ky-client::UpQueue::loadFromDatabase',
-                },
-                'Unknown mutation type'
-              );
-              return null;
-          }
-        })
-        .filter((e: UpEvent | null): e is UpEvent => e !== null);
-    } catch (error) {
-      this.logger.error(
-        { error, Category: 'sp00ky-client::UpQueue::loadFromDatabase' },
-        'Failed to load pending mutations from database'
-      );
-    }
-  }
-}

package/src/modules/sync/scheduler.ts

@@ -1,84 +0,0 @@
-import { Logger } from '../../services/logger/index';
-import { UpQueue, DownQueue, DownEvent, UpEvent, RollbackCallback } from './queue/index';
-import { SyncQueueEventTypes } from './events/index';
-
-/**
- * SyncScheduler manages when to sync: queue management and orchestration.
- * Decides the order and timing of sync operations.
- */
-export class SyncScheduler {
-  private isSyncingUp: boolean = false;
-  private isSyncingDown: boolean = false;
-
-  constructor(
-    private upQueue: UpQueue,
-    private downQueue: DownQueue,
-    private onProcessUp: (event: UpEvent) => Promise<void>,
-    private onProcessDown: (event: DownEvent) => Promise<void>,
-    private logger: Logger,
-    private onRollback?: RollbackCallback
-  ) {}
-
-  async init() {
-    await this.upQueue.loadFromDatabase();
-    this.upQueue.events.subscribe(SyncQueueEventTypes.MutationEnqueued, this.syncUp.bind(this));
-    this.downQueue.events.subscribe(
-      SyncQueueEventTypes.QueryItemEnqueued,
-      this.syncDown.bind(this)
-    );
-  }
-
-  /**
-   * Add mutations to the upload queue
-   */
-  enqueueMutation(mutations: UpEvent[]) {
-    for (const mutation of mutations) {
-      this.upQueue.push(mutation);
-    }
-  }
-
-  /**
-   * Add query events to the download queue
-   */
-  enqueueDownEvent(event: DownEvent) {
-    this.downQueue.push(event);
-  }
-
-  /**
-   * Process upload queue
-   */
-  async syncUp() {
-    if (this.isSyncingUp) return;
-    this.isSyncingUp = true;
-    try {
-      while (this.upQueue.size > 0) {
-        await this.upQueue.next(this.onProcessUp, this.onRollback);
-      }
-    } finally {
-      this.isSyncingUp = false;
-      void this.syncDown();
-    }
-  }
-
-  /**
-   * Process download queue
-   */
-  async syncDown() {
-    if (this.isSyncingDown) return;
-    if (this.upQueue.size > 0) return;
-
-    this.isSyncingDown = true;
-    try {
-      while (this.downQueue.size > 0) {
-        if (this.upQueue.size > 0) break;
-        await this.downQueue.next(this.onProcessDown);
-      }
-    } finally {
-      this.isSyncingDown = false;
-    }
-  }
-
-  get isSyncing() {
-    return this.isSyncingUp || this.isSyncingDown;
-  }
-}