@spooky-sync/core 0.0.0-canary.1 → 0.0.1-alpha.2

package/src/modules/sync/sync.ts

@@ -1,407 +0,0 @@
-import { LocalDatabaseService, RemoteDatabaseService } from '../../services/database/index';
-import { MutationEvent, RecordVersionArray } from '../../types';
-import { createSyncEventSystem, SyncEventTypes, SyncQueueEventTypes } from './events/index';
-import { Logger } from '../../services/logger/index';
-import { DownEvent, DownQueue, UpEvent, UpQueue } from './queue/index';
-import { RecordId, Uuid } from 'surrealdb';
-import { ArraySyncer, createDiffFromDbOp } from './utils';
-import { SyncEngine } from './engine';
-import { SyncScheduler } from './scheduler';
-import { SchemaStructure } from '@spooky-sync/query-builder';
-import { CacheModule } from '../cache/index';
-import { DataModule } from '../data/index';
-import { encodeRecordId, extractTablePart, parseDuration, surql } from '../../utils/index';
-
-/**
- * The main synchronization engine for Spooky.
- * Handles the bidirectional synchronization between the local database and the remote backend.
- * Uses a queue-based architecture with 'up' (local to remote) and 'down' (remote to local) queues.
- * @template S The schema structure type.
- */
-export class SpookySync<S extends SchemaStructure> {
-  private clientId: string = '';
-  private upQueue: UpQueue;
-  private downQueue: DownQueue;
-  private isInit: boolean = false;
-  private logger: Logger;
-  private syncEngine: SyncEngine;
-  private scheduler: SyncScheduler;
-  public events = createSyncEventSystem();
-
-  get isSyncing() {
-    return this.scheduler.isSyncing;
-  }
-
-  get pendingMutationCount(): number {
-    return this.upQueue.size;
-  }
-
-  subscribeToPendingMutations(cb: (count: number) => void): () => void {
-    const id1 = this.upQueue.events.subscribe(
-      SyncQueueEventTypes.MutationEnqueued,
-      (event) => cb(event.payload.queueSize)
-    );
-    const id2 = this.upQueue.events.subscribe(
-      SyncQueueEventTypes.MutationDequeued,
-      (event) => cb(event.payload.queueSize)
-    );
-    return () => {
-      this.upQueue.events.unsubscribe(id1);
-      this.upQueue.events.unsubscribe(id2);
-    };
-  }
-
-  constructor(
-    private local: LocalDatabaseService,
-    private remote: RemoteDatabaseService,
-    private cache: CacheModule,
-    private dataModule: DataModule<S>,
-    private schema: S,
-    logger: Logger
-  ) {
-    this.logger = logger.child({ service: 'SpookySync' });
-    this.upQueue = new UpQueue(this.local, this.logger);
-    this.downQueue = new DownQueue(this.local, this.logger);
-    this.syncEngine = new SyncEngine(this.remote, this.cache, this.schema, this.logger);
-    this.scheduler = new SyncScheduler(
-      this.upQueue,
-      this.downQueue,
-      this.processUpEvent.bind(this),
-      this.processDownEvent.bind(this),
-      this.logger,
-      this.handleRollback.bind(this)
-    );
-  }
-
-  /**
-   * Initializes the synchronization system.
-   * Starts the scheduler and initiates the initial sync cycles.
-   * @param clientId The unique identifier for this client instance.
-   * @throws Error if already initialized.
-   */
-  public async init(clientId: string) {
-    if (this.isInit) throw new Error('SpookySync is already initialized');
-    this.clientId = clientId;
-    this.isInit = true;
-    await this.scheduler.init();
-    void this.scheduler.syncUp();
-    void this.scheduler.syncUp();
-    void this.scheduler.syncDown();
-    void this.startRefLiveQueries();
-  }
-
-  private async startRefLiveQueries() {
-    this.logger.debug(
-      { clientId: this.clientId, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
-      'Starting ref live queries'
-    );
-
-    const [queryUuid] = await this.remote.query<[Uuid]>(
-      'LIVE SELECT * FROM _spooky_list_ref'
-    );
-
-    (await this.remote.getClient().liveOf(queryUuid)).subscribe((message) => {
-      this.logger.debug(
-        { message, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
-        'Live update received'
-      );
-      if (message.action === 'KILLED') return;
-      this.handleRemoteListRefChange(
-        message.action,
-        message.value.in as RecordId<string>,
-        message.value.out as RecordId<string>,
-        message.value.version as number
-      ).catch((err) => {
-        this.logger.error(
-          { err, Category: 'spooky-client::SpookySync::startRefLiveQueries' },
-          'Error handling remote list ref change'
-        );
-      });
-    });
-  }
-
-  private async handleRemoteListRefChange(
-    action: 'CREATE' | 'UPDATE' | 'DELETE',
-    queryId: RecordId,
-    recordId: RecordId,
-    version: number
-  ) {
-    const existing = this.dataModule.getQueryById(queryId);
-
-    if (!existing) {
-      this.logger.warn(
-        {
-          queryId: queryId.toString(),
-          Category: 'spooky-client::SpookySync::handleRemoteListRefChange',
-        },
-        'Received remote update for unknown local query'
-      );
-      return;
-    }
-
-    const { localArray } = existing.config;
-
-    this.logger.debug(
-      {
-        action,
-        queryId,
-        recordId,
-        version,
-        localArray,
-        Category: 'spooky-client::SpookySync::handleRemoteListRefChange',
-      },
-      'Live update is being processed'
-    );
-    const diff = createDiffFromDbOp(action, recordId, version, localArray);
-    await this.syncEngine.syncRecords(diff);
-  }
-
-  /**
-   * Enqueues a 'down' event (from remote to local) for processing.
-   * @param event The DownEvent to enqueue.
-   */
-  public enqueueDownEvent(event: DownEvent) {
-    this.scheduler.enqueueDownEvent(event);
-  }
-
-  private async processUpEvent(event: UpEvent) {
-    this.logger.debug(
-      { event, Category: 'spooky-client::SpookySync::processUpEvent' },
-      'Processing up event'
-    );
-    console.log('xx1', event);
-    switch (event.type) {
-      case 'create':
-        const dataKeys = Object.keys(event.data).map((key) => ({ key, variable: `data_${key}` }));
-        const prefixedParams = Object.fromEntries(
-          dataKeys.map(({ key, variable }) => [variable, event.data[key]])
-        );
-        const query = surql.seal(surql.createSet('id', dataKeys));
-        await this.remote.query(query, {
-          id: event.record_id,
-          ...prefixedParams,
-        });
-        break;
-      case 'update':
-        await this.remote.query(`UPDATE $id MERGE $data`, {
-          id: event.record_id,
-          data: event.data,
-        });
-        break;
-      case 'delete':
-        await this.remote.query(`DELETE $id`, {
-          id: event.record_id,
-        });
-        break;
-      default:
-        this.logger.error(
-          { event, Category: 'spooky-client::SpookySync::processUpEvent' },
-          'processUpEvent unknown event type'
-        );
-        return;
-    }
-  }
-
-  private async handleRollback(event: UpEvent, error: Error): Promise<void> {
-    const recordId = encodeRecordId(event.record_id);
-    const tableName =
-      event.type === 'create' && event.tableName
-        ? event.tableName
-        : extractTablePart(recordId);
-
-    this.logger.warn(
-      {
-        type: event.type,
-        recordId,
-        tableName,
-        error: error.message,
-        Category: 'spooky-client::SpookySync::handleRollback',
-      },
-      'Rolling back failed mutation'
-    );
-
-    switch (event.type) {
-      case 'create':
-        await this.dataModule.rollbackCreate(event.record_id, tableName);
-        break;
-      case 'update':
-        if (event.beforeRecord) {
-          await this.dataModule.rollbackUpdate(event.record_id, tableName, event.beforeRecord);
-        } else {
-          this.logger.warn(
-            {
-              recordId,
-              Category: 'spooky-client::SpookySync::handleRollback',
-            },
-            'Cannot rollback update: no beforeRecord available. Down-sync will reconcile.'
-          );
-        }
-        break;
-      case 'delete':
-        this.logger.warn(
-          {
-            recordId,
-            Category: 'spooky-client::SpookySync::handleRollback',
-          },
-          'Delete rollback not implemented. Down-sync will reconcile.'
-        );
-        break;
-    }
-
-    this.events.emit(SyncEventTypes.MutationRolledBack, {
-      eventType: event.type,
-      recordId,
-      error: error.message,
-    });
-  }
-
-  private async processDownEvent(event: DownEvent) {
-    this.logger.debug(
-      { event, Category: 'spooky-client::SpookySync::processDownEvent' },
-      'Processing down event'
-    );
-    switch (event.type) {
-      case 'register':
-        return this.registerQuery(event.payload.hash);
-      case 'sync':
-        return this.syncQuery(event.payload.hash);
-      case 'heartbeat':
-        return this.heartbeatQuery(event.payload.hash);
-      case 'cleanup':
-        return this.cleanupQuery(event.payload.hash);
-    }
-  }
-
-  /**
-   * Synchronizes a specific query by hash.
-   * Compares local and remote version arrays and fetches differences.
-   * @param hash The hash of the query to sync.
-   */
-  public async syncQuery(hash: string) {
-    const queryState = this.dataModule.getQueryByHash(hash);
-    if (!queryState) {
-      this.logger.warn(
-        { hash, Category: 'spooky-client::SpookySync::syncQuery' },
-        'Query not found'
-      );
-      return;
-    }
-
-    const diff = new ArraySyncer(
-      queryState.config.localArray,
-      queryState.config.remoteArray
-    ).nextSet();
-
-    if (!diff) {
-      return;
-    }
-    return this.syncEngine.syncRecords(diff);
-  }
-
-  /**
-   * Enqueues a list of mutations (up events) to be sent to the remote.
-   * @param mutations Array of UpEvents (create/update/delete) to enqueue.
-   */
-  public async enqueueMutation(mutations: UpEvent[]) {
-    this.scheduler.enqueueMutation(mutations);
-  }
-
-  private async registerQuery(queryHash: string) {
-    try {
-      this.logger.debug(
-        { queryHash, Category: 'spooky-client::SpookySync::registerQuery' },
-        'Register Query state'
-      );
-      await this.createRemoteQuery(queryHash);
-      await this.syncQuery(queryHash);
-    } catch (e) {
-      this.logger.error(
-        { err: e, Category: 'spooky-client::SpookySync::registerQuery' },
-        'registerQuery error'
-      );
-      throw e;
-    }
-  }
-
-  private async createRemoteQuery(queryHash: string) {
-    const queryState = this.dataModule.getQueryByHash(queryHash);
-
-    if (!queryState) {
-      this.logger.warn(
-        { queryHash, Category: 'spooky-client::SpookySync::createRemoteQuery' },
-        'Query to register not found'
-      );
-      throw new Error('Query to register not found');
-    }
-    // Delegate to remote function which handles DBSP registration & persistence
-    await this.remote.query('fn::query::register($config)', {
-      config: {
-        clientId: this.clientId,
-        id: queryState.config.id,
-        surql: queryState.config.surql,
-        params: queryState.config.params,
-        ttl: queryState.config.ttl,
-      },
-    });
-
-    const [items] = await this.remote.query<[{ out: RecordId<string>; version: number }[]]>(
-      surql.selectByFieldsAnd('_spooky_list_ref', ['in'], ['out', 'version']),
-      {
-        in: queryState.config.id,
-      }
-    );
-
-    this.logger.trace(
-      {
-        queryId: encodeRecordId(queryState.config.id),
-        items,
-        Category: 'spooky-client::SpookySync::createRemoteQuery',
-      },
-      'Got query record version array from remote'
-    );
-
-    const array: RecordVersionArray = items.map((item) => [encodeRecordId(item.out), item.version]);
-
-    this.logger.debug(
-      {
-        queryId: encodeRecordId(queryState.config.id),
-        array,
-        Category: 'spooky-client::SpookySync::createRemoteQuery',
-      },
-      'createdRemoteQuery'
-    );
-
-    if (array) {
-      /// Incantation existed already
-      await this.dataModule.updateQueryRemoteArray(queryHash, array);
-    }
-  }
-
-  private async heartbeatQuery(queryHash: string) {
-    const queryState = this.dataModule.getQueryByHash(queryHash);
-    if (!queryState) {
-      this.logger.warn(
-        { queryHash, Category: 'spooky-client::SpookySync::heartbeatQuery' },
-        'Query to register not found'
-      );
-      throw new Error('Query to register not found');
-    }
-    await this.remote.query('fn::query::heartbeat($id)', {
-      id: queryState.config.id,
-    });
-  }
-
-  private async cleanupQuery(queryHash: string) {
-    const queryState = this.dataModule.getQueryByHash(queryHash);
-    if (!queryState) {
-      this.logger.warn(
-        { queryHash, Category: 'spooky-client::SpookySync::cleanupQuery' },
-        'Query to register not found'
-      );
-      throw new Error('Query to register not found');
-    }
-    await this.remote.query(`DELETE $id`, {
-      id: queryState.config.id,
-    });
-  }
-}

package/src/modules/sync/utils.test.ts

@@ -1,311 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { RecordId } from 'surrealdb';
-import {
-  diffRecordVersionArray,
-  applyRecordVersionDiff,
-  createDiffFromDbOp,
-  ArraySyncer,
-} from './utils';
-import { RecordVersionArray, RecordVersionDiff } from '../../types';
-import { encodeRecordId } from '../../utils/index';
-
-function rid(table: string, id: string): RecordId<string> {
-  return new RecordId(table, id);
-}
-
-describe('diffRecordVersionArray', () => {
-  it('detects added records (in remote, not local)', () => {
-    const local: RecordVersionArray = [['user:1', 1]];
-    const remote: RecordVersionArray = [
-      ['user:1', 1],
-      ['user:2', 1],
-    ];
-    const diff = diffRecordVersionArray(local, remote);
-
-    expect(diff.added).toHaveLength(1);
-    expect(encodeRecordId(diff.added[0].id)).toBe('user:2');
-    expect(diff.added[0].version).toBe(1);
-    expect(diff.updated).toHaveLength(0);
-    expect(diff.removed).toHaveLength(0);
-  });
-
-  it('detects updated records (remote version > local version)', () => {
-    const local: RecordVersionArray = [['user:1', 1]];
-    const remote: RecordVersionArray = [['user:1', 3]];
-    const diff = diffRecordVersionArray(local, remote);
-
-    expect(diff.updated).toHaveLength(1);
-    expect(encodeRecordId(diff.updated[0].id)).toBe('user:1');
-    expect(diff.updated[0].version).toBe(3);
-    expect(diff.added).toHaveLength(0);
-    expect(diff.removed).toHaveLength(0);
-  });
-
-  it('detects removed records (in local, not remote)', () => {
-    const local: RecordVersionArray = [
-      ['user:1', 1],
-      ['user:2', 1],
-    ];
-    const remote: RecordVersionArray = [['user:1', 1]];
-    const diff = diffRecordVersionArray(local, remote);
-
-    expect(diff.removed).toHaveLength(1);
-    expect(encodeRecordId(diff.removed[0])).toBe('user:2');
-    expect(diff.added).toHaveLength(0);
-    expect(diff.updated).toHaveLength(0);
-  });
-
-  it('handles null arrays', () => {
-    const diff = diffRecordVersionArray(null, null);
-    expect(diff.added).toHaveLength(0);
-    expect(diff.updated).toHaveLength(0);
-    expect(diff.removed).toHaveLength(0);
-  });
-
-  it('handles empty arrays', () => {
-    const diff = diffRecordVersionArray([], []);
-    expect(diff.added).toHaveLength(0);
-    expect(diff.updated).toHaveLength(0);
-    expect(diff.removed).toHaveLength(0);
-  });
-
-  it('no diff when arrays match', () => {
-    const arr: RecordVersionArray = [
-      ['user:1', 1],
-      ['user:2', 2],
-    ];
-    const diff = diffRecordVersionArray(arr, arr);
-    expect(diff.added).toHaveLength(0);
-    expect(diff.updated).toHaveLength(0);
-    expect(diff.removed).toHaveLength(0);
-  });
-
-  it('handles mixed adds/updates/removes', () => {
-    const local: RecordVersionArray = [
-      ['user:1', 1],
-      ['user:2', 1],
-      ['user:3', 1],
-    ];
-    const remote: RecordVersionArray = [
-      ['user:1', 1], // same
-      ['user:2', 3], // updated
-      // user:3 removed
-      ['user:4', 1], // added
-    ];
-    const diff = diffRecordVersionArray(local, remote);
-
-    expect(diff.added).toHaveLength(1);
-    expect(encodeRecordId(diff.added[0].id)).toBe('user:4');
-    expect(diff.updated).toHaveLength(1);
-    expect(encodeRecordId(diff.updated[0].id)).toBe('user:2');
-    expect(diff.removed).toHaveLength(1);
-    expect(encodeRecordId(diff.removed[0])).toBe('user:3');
-  });
-});
-
-describe('applyRecordVersionDiff', () => {
-  it('applies additions', () => {
-    const current: RecordVersionArray = [['user:1', 1]];
-    const diff: RecordVersionDiff = {
-      added: [{ id: rid('user', '2'), version: 1 }],
-      updated: [],
-      removed: [],
-    };
-    const result = applyRecordVersionDiff(current, diff);
-    expect(result).toEqual([
-      ['user:1', 1],
-      ['user:2', 1],
-    ]);
-  });
-
-  it('applies updates', () => {
-    const current: RecordVersionArray = [['user:1', 1]];
-    const diff: RecordVersionDiff = {
-      added: [],
-      updated: [{ id: rid('user', '1'), version: 5 }],
-      removed: [],
-    };
-    const result = applyRecordVersionDiff(current, diff);
-    expect(result).toEqual([['user:1', 5]]);
-  });
-
-  it('applies removals', () => {
-    const current: RecordVersionArray = [
-      ['user:1', 1],
-      ['user:2', 2],
-    ];
-    const diff: RecordVersionDiff = {
-      added: [],
-      updated: [],
-      removed: [rid('user', '1')],
-    };
-    const result = applyRecordVersionDiff(current, diff);
-    expect(result).toEqual([['user:2', 2]]);
-  });
-
-  it('result is sorted by record ID', () => {
-    const current: RecordVersionArray = [['user:c', 1]];
-    const diff: RecordVersionDiff = {
-      added: [
-        { id: rid('user', 'a'), version: 1 },
-        { id: rid('user', 'z'), version: 1 },
-      ],
-      updated: [],
-      removed: [],
-    };
-    const result = applyRecordVersionDiff(current, diff);
-    expect(result).toEqual([
-      ['user:a', 1],
-      ['user:c', 1],
-      ['user:z', 1],
-    ]);
-  });
-
-  it('empty diff returns original (sorted)', () => {
-    const current: RecordVersionArray = [
-      ['user:b', 2],
-      ['user:a', 1],
-    ];
-    const diff: RecordVersionDiff = { added: [], updated: [], removed: [] };
-    const result = applyRecordVersionDiff(current, diff);
-    expect(result).toEqual([
-      ['user:a', 1],
-      ['user:b', 2],
-    ]);
-  });
-});
-
-describe('createDiffFromDbOp', () => {
-  it('CREATE populates added array', () => {
-    const recordId = rid('user', '1');
-    const diff = createDiffFromDbOp('CREATE', recordId, 1);
-    expect(diff.added).toHaveLength(1);
-    expect(diff.added[0].id).toBe(recordId);
-    expect(diff.added[0].version).toBe(1);
-    expect(diff.updated).toHaveLength(0);
-    expect(diff.removed).toHaveLength(0);
-  });
-
-  it('UPDATE populates updated array', () => {
-    const recordId = rid('user', '1');
-    const diff = createDiffFromDbOp('UPDATE', recordId, 2);
-    expect(diff.updated).toHaveLength(1);
-    expect(diff.updated[0].id).toBe(recordId);
-    expect(diff.updated[0].version).toBe(2);
-    expect(diff.added).toHaveLength(0);
-    expect(diff.removed).toHaveLength(0);
-  });
-
-  it('DELETE populates removed array', () => {
-    const recordId = rid('user', '1');
-    const diff = createDiffFromDbOp('DELETE', recordId, 1);
-    expect(diff.removed).toHaveLength(1);
-    expect(diff.removed[0]).toBe(recordId);
-    expect(diff.added).toHaveLength(0);
-    expect(diff.updated).toHaveLength(0);
-  });
-
-  it('skips if existing version >= new version', () => {
-    const recordId = rid('user', '1');
-    const versions: RecordVersionArray = [['user:1', 5]];
-    const diff = createDiffFromDbOp('UPDATE', recordId, 3, versions);
-    expect(diff.added).toHaveLength(0);
-    expect(diff.updated).toHaveLength(0);
-    expect(diff.removed).toHaveLength(0);
-  });
-
-  it('applies if existing version < new version', () => {
-    const recordId = rid('user', '1');
-    const versions: RecordVersionArray = [['user:1', 2]];
-    const diff = createDiffFromDbOp('UPDATE', recordId, 5, versions);
-    expect(diff.updated).toHaveLength(1);
-    expect(diff.updated[0].version).toBe(5);
-  });
-});
-
-describe('ArraySyncer', () => {
-  it('insert adds to local array', () => {
-    const syncer = new ArraySyncer(
-      [['user:1', 1]],
-      [['user:1', 1]]
-    );
-
-    syncer.insert('user:2', 1);
-    const diff = syncer.nextSet();
-    expect(diff).not.toBeNull();
-    // local now has user:2 which remote does not → user:2 is in local removed from remote perspective
-    // Actually: local=[user:1, user:2], remote=[user:1] → user:2 is "removed" (in local, not remote)
-    expect(diff!.removed).toHaveLength(1);
-    expect(encodeRecordId(diff!.removed[0])).toBe('user:2');
-  });
-
-  it('update modifies version in local array', () => {
-    const syncer = new ArraySyncer(
-      [['user:1', 1]],
-      [['user:1', 1]]
-    );
-
-    syncer.update('user:1', 5);
-    const diff = syncer.nextSet();
-    // local version (5) > remote version (1), so no "updated" from diff perspective
-    // diff finds remote additions/updates relative to local; local version higher means no remote update
-    expect(diff).not.toBeNull();
-    expect(diff!.added).toHaveLength(0);
-    expect(diff!.updated).toHaveLength(0);
-    expect(diff!.removed).toHaveLength(0);
-  });
-
-  it('delete removes from local array', () => {
-    const syncer = new ArraySyncer(
-      [
-        ['user:1', 1],
-        ['user:2', 1],
-      ],
-      [
-        ['user:1', 1],
-        ['user:2', 1],
-      ]
-    );
-
-    syncer.delete('user:2');
-    const diff = syncer.nextSet();
-    expect(diff).not.toBeNull();
-    // remote has user:2, local does not → added from remote perspective
-    expect(diff!.added).toHaveLength(1);
-    expect(encodeRecordId(diff!.added[0].id)).toBe('user:2');
-  });
-
-  it('nextSet returns correct diff against remote', () => {
-    const syncer = new ArraySyncer(
-      [['user:1', 1]],
-      [
-        ['user:1', 1],
-        ['user:2', 1],
-      ]
-    );
-
-    const diff = syncer.nextSet();
-    expect(diff).not.toBeNull();
-    expect(diff!.added).toHaveLength(1);
-    expect(encodeRecordId(diff!.added[0].id)).toBe('user:2');
-  });
-
-  it('maintains sorting after mutations', () => {
-    const syncer = new ArraySyncer(
-      [['user:c', 1]],
-      []
-    );
-
-    syncer.insert('user:a', 1);
-    syncer.insert('user:z', 1);
-
-    // nextSet triggers sort
-    const diff = syncer.nextSet();
-    // All 3 are in local but not remote → 3 removed items
-    expect(diff!.removed).toHaveLength(3);
-    // Check they come in sorted order
-    const removedIds = diff!.removed.map((r) => encodeRecordId(r));
-    const sorted = [...removedIds].sort();
-    expect(removedIds).toEqual(sorted);
-  });
-});