@spooky-sync/core 0.0.0-canary.1

package/src/modules/sync/utils.test.ts

@@ -0,0 +1,311 @@
+import { describe, it, expect } from 'vitest';
+import { RecordId } from 'surrealdb';
+import {
+  diffRecordVersionArray,
+  applyRecordVersionDiff,
+  createDiffFromDbOp,
+  ArraySyncer,
+} from './utils';
+import { RecordVersionArray, RecordVersionDiff } from '../../types';
+import { encodeRecordId } from '../../utils/index';
+
+function rid(table: string, id: string): RecordId<string> {
+  return new RecordId(table, id);
+}
+
+describe('diffRecordVersionArray', () => {
+  it('detects added records (in remote, not local)', () => {
+    const local: RecordVersionArray = [['user:1', 1]];
+    const remote: RecordVersionArray = [
+      ['user:1', 1],
+      ['user:2', 1],
+    ];
+    const diff = diffRecordVersionArray(local, remote);
+
+    expect(diff.added).toHaveLength(1);
+    expect(encodeRecordId(diff.added[0].id)).toBe('user:2');
+    expect(diff.added[0].version).toBe(1);
+    expect(diff.updated).toHaveLength(0);
+    expect(diff.removed).toHaveLength(0);
+  });
+
+  it('detects updated records (remote version > local version)', () => {
+    const local: RecordVersionArray = [['user:1', 1]];
+    const remote: RecordVersionArray = [['user:1', 3]];
+    const diff = diffRecordVersionArray(local, remote);
+
+    expect(diff.updated).toHaveLength(1);
+    expect(encodeRecordId(diff.updated[0].id)).toBe('user:1');
+    expect(diff.updated[0].version).toBe(3);
+    expect(diff.added).toHaveLength(0);
+    expect(diff.removed).toHaveLength(0);
+  });
+
+  it('detects removed records (in local, not remote)', () => {
+    const local: RecordVersionArray = [
+      ['user:1', 1],
+      ['user:2', 1],
+    ];
+    const remote: RecordVersionArray = [['user:1', 1]];
+    const diff = diffRecordVersionArray(local, remote);
+
+    expect(diff.removed).toHaveLength(1);
+    expect(encodeRecordId(diff.removed[0])).toBe('user:2');
+    expect(diff.added).toHaveLength(0);
+    expect(diff.updated).toHaveLength(0);
+  });
+
+  it('handles null arrays', () => {
+    const diff = diffRecordVersionArray(null, null);
+    expect(diff.added).toHaveLength(0);
+    expect(diff.updated).toHaveLength(0);
+    expect(diff.removed).toHaveLength(0);
+  });
+
+  it('handles empty arrays', () => {
+    const diff = diffRecordVersionArray([], []);
+    expect(diff.added).toHaveLength(0);
+    expect(diff.updated).toHaveLength(0);
+    expect(diff.removed).toHaveLength(0);
+  });
+
+  it('no diff when arrays match', () => {
+    const arr: RecordVersionArray = [
+      ['user:1', 1],
+      ['user:2', 2],
+    ];
+    const diff = diffRecordVersionArray(arr, arr);
+    expect(diff.added).toHaveLength(0);
+    expect(diff.updated).toHaveLength(0);
+    expect(diff.removed).toHaveLength(0);
+  });
+
+  it('handles mixed adds/updates/removes', () => {
+    const local: RecordVersionArray = [
+      ['user:1', 1],
+      ['user:2', 1],
+      ['user:3', 1],
+    ];
+    const remote: RecordVersionArray = [
+      ['user:1', 1], // same
+      ['user:2', 3], // updated
+      // user:3 removed
+      ['user:4', 1], // added
+    ];
+    const diff = diffRecordVersionArray(local, remote);
+
+    expect(diff.added).toHaveLength(1);
+    expect(encodeRecordId(diff.added[0].id)).toBe('user:4');
+    expect(diff.updated).toHaveLength(1);
+    expect(encodeRecordId(diff.updated[0].id)).toBe('user:2');
+    expect(diff.removed).toHaveLength(1);
+    expect(encodeRecordId(diff.removed[0])).toBe('user:3');
+  });
+});
+
+describe('applyRecordVersionDiff', () => {
+  it('applies additions', () => {
+    const current: RecordVersionArray = [['user:1', 1]];
+    const diff: RecordVersionDiff = {
+      added: [{ id: rid('user', '2'), version: 1 }],
+      updated: [],
+      removed: [],
+    };
+    const result = applyRecordVersionDiff(current, diff);
+    expect(result).toEqual([
+      ['user:1', 1],
+      ['user:2', 1],
+    ]);
+  });
+
+  it('applies updates', () => {
+    const current: RecordVersionArray = [['user:1', 1]];
+    const diff: RecordVersionDiff = {
+      added: [],
+      updated: [{ id: rid('user', '1'), version: 5 }],
+      removed: [],
+    };
+    const result = applyRecordVersionDiff(current, diff);
+    expect(result).toEqual([['user:1', 5]]);
+  });
+
+  it('applies removals', () => {
+    const current: RecordVersionArray = [
+      ['user:1', 1],
+      ['user:2', 2],
+    ];
+    const diff: RecordVersionDiff = {
+      added: [],
+      updated: [],
+      removed: [rid('user', '1')],
+    };
+    const result = applyRecordVersionDiff(current, diff);
+    expect(result).toEqual([['user:2', 2]]);
+  });
+
+  it('result is sorted by record ID', () => {
+    const current: RecordVersionArray = [['user:c', 1]];
+    const diff: RecordVersionDiff = {
+      added: [
+        { id: rid('user', 'a'), version: 1 },
+        { id: rid('user', 'z'), version: 1 },
+      ],
+      updated: [],
+      removed: [],
+    };
+    const result = applyRecordVersionDiff(current, diff);
+    expect(result).toEqual([
+      ['user:a', 1],
+      ['user:c', 1],
+      ['user:z', 1],
+    ]);
+  });
+
+  it('empty diff returns original (sorted)', () => {
+    const current: RecordVersionArray = [
+      ['user:b', 2],
+      ['user:a', 1],
+    ];
+    const diff: RecordVersionDiff = { added: [], updated: [], removed: [] };
+    const result = applyRecordVersionDiff(current, diff);
+    expect(result).toEqual([
+      ['user:a', 1],
+      ['user:b', 2],
+    ]);
+  });
+});
+
+describe('createDiffFromDbOp', () => {
+  it('CREATE populates added array', () => {
+    const recordId = rid('user', '1');
+    const diff = createDiffFromDbOp('CREATE', recordId, 1);
+    expect(diff.added).toHaveLength(1);
+    expect(diff.added[0].id).toBe(recordId);
+    expect(diff.added[0].version).toBe(1);
+    expect(diff.updated).toHaveLength(0);
+    expect(diff.removed).toHaveLength(0);
+  });
+
+  it('UPDATE populates updated array', () => {
+    const recordId = rid('user', '1');
+    const diff = createDiffFromDbOp('UPDATE', recordId, 2);
+    expect(diff.updated).toHaveLength(1);
+    expect(diff.updated[0].id).toBe(recordId);
+    expect(diff.updated[0].version).toBe(2);
+    expect(diff.added).toHaveLength(0);
+    expect(diff.removed).toHaveLength(0);
+  });
+
+  it('DELETE populates removed array', () => {
+    const recordId = rid('user', '1');
+    const diff = createDiffFromDbOp('DELETE', recordId, 1);
+    expect(diff.removed).toHaveLength(1);
+    expect(diff.removed[0]).toBe(recordId);
+    expect(diff.added).toHaveLength(0);
+    expect(diff.updated).toHaveLength(0);
+  });
+
+  it('skips if existing version >= new version', () => {
+    const recordId = rid('user', '1');
+    const versions: RecordVersionArray = [['user:1', 5]];
+    const diff = createDiffFromDbOp('UPDATE', recordId, 3, versions);
+    expect(diff.added).toHaveLength(0);
+    expect(diff.updated).toHaveLength(0);
+    expect(diff.removed).toHaveLength(0);
+  });
+
+  it('applies if existing version < new version', () => {
+    const recordId = rid('user', '1');
+    const versions: RecordVersionArray = [['user:1', 2]];
+    const diff = createDiffFromDbOp('UPDATE', recordId, 5, versions);
+    expect(diff.updated).toHaveLength(1);
+    expect(diff.updated[0].version).toBe(5);
+  });
+});
+
+describe('ArraySyncer', () => {
+  it('insert adds to local array', () => {
+    const syncer = new ArraySyncer(
+      [['user:1', 1]],
+      [['user:1', 1]]
+    );
+
+    syncer.insert('user:2', 1);
+    const diff = syncer.nextSet();
+    expect(diff).not.toBeNull();
+    // local now has user:2 which remote does not → user:2 is in local removed from remote perspective
+    // Actually: local=[user:1, user:2], remote=[user:1] → user:2 is "removed" (in local, not remote)
+    expect(diff!.removed).toHaveLength(1);
+    expect(encodeRecordId(diff!.removed[0])).toBe('user:2');
+  });
+
+  it('update modifies version in local array', () => {
+    const syncer = new ArraySyncer(
+      [['user:1', 1]],
+      [['user:1', 1]]
+    );
+
+    syncer.update('user:1', 5);
+    const diff = syncer.nextSet();
+    // local version (5) > remote version (1), so no "updated" from diff perspective
+    // diff finds remote additions/updates relative to local; local version higher means no remote update
+    expect(diff).not.toBeNull();
+    expect(diff!.added).toHaveLength(0);
+    expect(diff!.updated).toHaveLength(0);
+    expect(diff!.removed).toHaveLength(0);
+  });
+
+  it('delete removes from local array', () => {
+    const syncer = new ArraySyncer(
+      [
+        ['user:1', 1],
+        ['user:2', 1],
+      ],
+      [
+        ['user:1', 1],
+        ['user:2', 1],
+      ]
+    );
+
+    syncer.delete('user:2');
+    const diff = syncer.nextSet();
+    expect(diff).not.toBeNull();
+    // remote has user:2, local does not → added from remote perspective
+    expect(diff!.added).toHaveLength(1);
+    expect(encodeRecordId(diff!.added[0].id)).toBe('user:2');
+  });
+
+  it('nextSet returns correct diff against remote', () => {
+    const syncer = new ArraySyncer(
+      [['user:1', 1]],
+      [
+        ['user:1', 1],
+        ['user:2', 1],
+      ]
+    );
+
+    const diff = syncer.nextSet();
+    expect(diff).not.toBeNull();
+    expect(diff!.added).toHaveLength(1);
+    expect(encodeRecordId(diff!.added[0].id)).toBe('user:2');
+  });
+
+  it('maintains sorting after mutations', () => {
+    const syncer = new ArraySyncer(
+      [['user:c', 1]],
+      []
+    );
+
+    syncer.insert('user:a', 1);
+    syncer.insert('user:z', 1);
+
+    // nextSet triggers sort
+    const diff = syncer.nextSet();
+    // All 3 are in local but not remote → 3 removed items
+    expect(diff!.removed).toHaveLength(3);
+    // Check they come in sorted order
+    const removedIds = diff!.removed.map((r) => encodeRecordId(r));
+    const sorted = [...removedIds].sort();
+    expect(removedIds).toEqual(sorted);
+  });
+});

package/src/modules/sync/utils.ts

@@ -0,0 +1,171 @@
+import { RecordId } from 'surrealdb';
+import { RecordVersionArray, RecordVersionDiff } from '../../types';
+import { parseRecordIdString, encodeRecordId } from '../../utils/index';
+
+export class ArraySyncer {
+  private localArray: RecordVersionArray;
+  private remoteArray: RecordVersionArray;
+  private needsSort = false;
+
+  constructor(localArray: RecordVersionArray, remoteArray: RecordVersionArray) {
+    this.remoteArray = remoteArray.sort((a, b) => a[0].localeCompare(b[0]));
+    this.localArray = localArray.sort((a, b) => a[0].localeCompare(b[0]));
+  }
+
+  /**
+   * Inserts an item into the local array
+   */
+  insert(recordId: string, version: number) {
+    this.localArray.push([recordId, version]);
+    this.needsSort = true;
+  }
+
+  /**
+   * Updates the current local RecordVersionArray state.
+   */
+  update(recordId: string, version: number) {
+    this.localArray = this.localArray.map((record) => {
+      if (record[0] === recordId) {
+        this.needsSort = true;
+        return [recordId, version];
+      }
+      return record;
+    });
+  }
+
+  /**
+   * Deletes an item from the local array
+   */
+  delete(recordId: string) {
+    this.localArray = this.localArray.filter((record) => record[0] !== recordId);
+  }
+
+  /**
+   * Returns the difference between the local and remote arrays.
+   * Includes sets of added, updated, and removed records.
+   */
+  nextSet(): RecordVersionDiff | null {
+    if (this.needsSort) {
+      this.localArray.sort((a, b) => a[0].localeCompare(b[0]));
+      this.needsSort = false;
+    }
+    console.log('xxxx555', this.localArray, this.remoteArray);
+    const diff = diffRecordVersionArray(this.localArray, this.remoteArray);
+    return diff;
+  }
+}
+
+export function diffRecordVersionArray(
+  local: RecordVersionArray | null,
+  remote: RecordVersionArray | null
+): RecordVersionDiff {
+  const localArray = local || [];
+  const remoteArray = remote || [];
+
+  // Convert arrays to Maps for O(1) lookup
+  const localMap = new Map<string, number>(localArray);
+  const remoteMap = new Map<string, number>(remoteArray);
+
+  const added: string[] = [];
+  const updated: string[] = [];
+  const removed: string[] = [];
+
+  // Find added and updated records
+  for (const [recordId, remoteVersion] of remoteMap) {
+    const localVersion = localMap.get(recordId);
+
+    if (localVersion === undefined) {
+      // Record exists in remote but not in local
+      added.push(recordId);
+    } else if (localVersion < remoteVersion) {
+      // Record exists in both but remote has newer version
+      updated.push(recordId);
+    }
+  }
+
+  // Find removed records
+  for (const [recordId] of localMap) {
+    if (!remoteMap.has(recordId)) {
+      removed.push(recordId);
+    }
+  }
+
+  return {
+    added: added.map((id) => ({
+      id: parseRecordIdString(id),
+      version: remoteMap.get(id)!,
+    })),
+    updated: updated.map((id) => ({
+      id: parseRecordIdString(id),
+      version: remoteMap.get(id)!,
+    })),
+    removed: removed.map(parseRecordIdString),
+  };
+}
+
+/**
+ * Applies a RecordVersionDiff to a RecordVersionArray and returns a new sorted array.
+ */
+export function applyRecordVersionDiff(
+  current: RecordVersionArray,
+  diff: RecordVersionDiff
+): RecordVersionArray {
+  const currentMap = new Map(current);
+
+  // Apply removals
+  for (const id of diff.removed) {
+    currentMap.delete(encodeRecordId(id));
+  }
+
+  // Apply additions
+  for (const item of diff.added) {
+    currentMap.set(encodeRecordId(item.id), item.version);
+  }
+
+  // Apply updates
+  for (const item of diff.updated) {
+    currentMap.set(encodeRecordId(item.id), item.version);
+  }
+
+  return Array.from(currentMap).sort((a, b) => a[0].localeCompare(b[0]));
+}
+
+export function createDiffFromDbOp(
+  op: 'CREATE' | 'UPDATE' | 'DELETE',
+  recordId: RecordId,
+  version: number,
+  versions?: RecordVersionArray
+): RecordVersionDiff {
+  // Version guard: skip stale CREATE/UPDATE, but always process DELETE
+  if (op !== 'DELETE') {
+    const old = versions?.find((record) => record[0] === encodeRecordId(recordId));
+
+    if (old && old[1] >= version) {
+      return {
+        added: [],
+        updated: [],
+        removed: [],
+      };
+    }
+  }
+
+  if (op === 'CREATE') {
+    return {
+      added: [{ id: recordId, version }],
+      updated: [],
+      removed: [],
+    };
+  } else if (op === 'UPDATE') {
+    return {
+      added: [],
+      updated: [{ id: recordId, version }],
+      removed: [],
+    };
+  } else {
+    return {
+      added: [],
+      updated: [],
+      removed: [recordId],
+    };
+  }
+}

package/src/services/database/database.ts

@@ -0,0 +1,108 @@
+import { Surreal, SurrealTransaction } from 'surrealdb';
+import { createLogger, Logger } from '../logger/index';
+import {
+  DatabaseEventSystem,
+  DatabaseEventTypes,
+  DatabaseQueryEventPayload,
+} from './events/index';
+import { SealedQuery } from '../../utils/surql';
+
+export abstract class AbstractDatabaseService {
+  protected client: Surreal;
+  protected logger: Logger;
+  protected events: DatabaseEventSystem;
+  protected abstract eventType:
+    | typeof DatabaseEventTypes.LocalQuery
+    | typeof DatabaseEventTypes.RemoteQuery;
+
+  constructor(client: Surreal, logger: Logger, events: DatabaseEventSystem) {
+    this.client = client;
+    this.logger = logger.child({ service: 'Database' });
+    this.events = events;
+  }
+
+  abstract connect(): Promise<void>;
+
+  getClient(): Surreal {
+    return this.client;
+  }
+
+  getEvents(): DatabaseEventSystem {
+    return this.events;
+  }
+
+  tx(): Promise<SurrealTransaction> {
+    return this.client.beginTransaction();
+  }
+
+  private queryQueue: Promise<void> = Promise.resolve();
+
+  /**
+   * Execute a query with serialized execution to prevent WASM transaction issues.
+   */
+  async query<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T> {
+    return new Promise((resolve, reject) => {
+      this.queryQueue = this.queryQueue
+        .then(async () => {
+          const startTime = performance.now();
+          try {
+            this.logger.debug(
+              { query, vars, Category: 'spooky-client::Database::query' },
+              'Executing query'
+            );
+            const pending = this.client.query(query, vars);
+            // In SurrealDB 2.0, .query() collects results by default.
+            // We cast to T directly as proper typing depends on the caller knowing the return structure.
+            const result = (await pending) as unknown as T;
+            const duration = performance.now() - startTime;
+
+            // Emit query event
+            this.events.emit(this.eventType, {
+              query,
+              vars,
+              duration,
+              success: true,
+              timestamp: Date.now(),
+            });
+
+            resolve(result);
+            this.logger.trace(
+              { query, result, Category: 'spooky-client::Database::query' },
+              'Query executed successfully'
+            );
+          } catch (err) {
+            const duration = performance.now() - startTime;
+
+            // Emit query event with error
+            this.events.emit(this.eventType, {
+              query,
+              vars,
+              duration,
+              success: false,
+              error: err instanceof Error ? err.message : String(err),
+              timestamp: Date.now(),
+            });
+
+            this.logger.error(
+              { query, vars, err, Category: 'spooky-client::Database::query' },
+              'Query execution failed'
+            );
+            reject(err);
+          }
+        })
+        .catch(() => {
+          // Ignore queue errors to keep the chain alive; the specific promise was rejected above.
+        });
+    });
+  }
+
+  async execute<T>(query: SealedQuery<T>, vars?: Record<string, unknown>): Promise<T> {
+    const raw = await this.query<unknown[]>(query.sql, vars);
+    return query.extract(raw);
+  }
+
+  async close(): Promise<void> {
+    this.logger.info({ Category: 'spooky-client::Database::close' }, 'Closing database connection');
+    await this.client.close();
+  }
+}

package/src/services/database/events/index.ts

@@ -0,0 +1,32 @@
+import { createEventSystem, EventDefinition, EventSystem } from '../../../events/index';
+
+export const DatabaseEventTypes = {
+  LocalQuery: 'DATABASE_LOCAL_QUERY',
+  RemoteQuery: 'DATABASE_REMOTE_QUERY',
+} as const;
+
+export interface DatabaseQueryEventPayload {
+  query: string;
+  vars?: Record<string, unknown>;
+  duration: number;
+  success: boolean;
+  error?: string;
+  timestamp: number;
+}
+
+export type DatabaseEventTypeMap = {
+  [DatabaseEventTypes.LocalQuery]: EventDefinition<
+    typeof DatabaseEventTypes.LocalQuery,
+    DatabaseQueryEventPayload
+  >;
+  [DatabaseEventTypes.RemoteQuery]: EventDefinition<
+    typeof DatabaseEventTypes.RemoteQuery,
+    DatabaseQueryEventPayload
+  >;
+};
+
+export type DatabaseEventSystem = EventSystem<DatabaseEventTypeMap>;
+
+export function createDatabaseEventSystem(): DatabaseEventSystem {
+  return createEventSystem([DatabaseEventTypes.LocalQuery, DatabaseEventTypes.RemoteQuery]);
+}

package/src/services/database/index.ts

@@ -0,0 +1,5 @@
+export * from './database';
+export * from './local';
+export * from './remote';
+export * from './local-migrator';
+export * from './events/index';