@spooky-sync/core 0.0.0-canary.1

package/src/spooky.ts

@@ -0,0 +1,346 @@
+import { DataModule } from './modules/data/index';
+import {
+  SpookyConfig,
+  QueryTimeToLive,
+  SpookyQueryResultPromise,
+  PersistenceClient,
+  MutationEvent,
+  UpdateOptions,
+  RunOptions,
+} from './types';
+import {
+  LocalDatabaseService,
+  LocalMigrator,
+  RemoteDatabaseService,
+} from './services/database/index';
+import { Surreal } from 'surrealdb';
+import { SpookySync, UpEvent } from './modules/sync/index';
+import {
+  GetTable,
+  InnerQuery,
+  QueryBuilder,
+  QueryOptions,
+  SchemaStructure,
+  TableModel,
+  TableNames,
+  BackendNames,
+  BackendRoutes,
+  RoutePayload,
+} from '@spooky-sync/query-builder';
+
+import { DevToolsService } from './modules/devtools/index';
+import { createLogger } from './services/logger/index';
+import { AuthService } from './modules/auth/index';
+import { StreamProcessorService } from './services/stream-processor/index';
+import { EventSystem } from './events/index';
+import { CacheModule } from './modules/cache/index';
+import { LocalStoragePersistenceClient } from './services/persistence/localstorage';
+import { generateId, parseParams } from './utils/index';
+import { SurrealDBPersistenceClient } from './services/persistence/surrealdb';
+
+export class SpookyClient<S extends SchemaStructure> {
+  private local: LocalDatabaseService;
+  private remote: RemoteDatabaseService;
+  private persistenceClient: PersistenceClient;
+
+  private migrator: LocalMigrator;
+  private cache: CacheModule;
+  private dataModule: DataModule<S>;
+  private sync: SpookySync<S>;
+  private devTools: DevToolsService;
+
+  private logger: ReturnType<typeof createLogger>;
+  public auth: AuthService<S>;
+  public streamProcessor: StreamProcessorService;
+
+  get remoteClient() {
+    return this.remote.getClient();
+  }
+
+  get localClient() {
+    return this.local.getClient();
+  }
+
+  get pendingMutationCount(): number {
+    return this.sync.pendingMutationCount;
+  }
+
+  subscribeToPendingMutations(cb: (count: number) => void): () => void {
+    return this.sync.subscribeToPendingMutations(cb);
+  }
+
+  constructor(private config: SpookyConfig<S>) {
+    const logger = createLogger(config.logLevel ?? 'info', config.otelEndpoint);
+    this.logger = logger.child({ service: 'SpookyClient' });
+
+    this.logger.info(
+      {
+        config: { ...config, schema: '[SchemaStructure]' },
+        Category: 'spooky-client::SpookyClient::constructor',
+      },
+      'SpookyClient initialized'
+    );
+
+    this.local = new LocalDatabaseService(this.config.database, logger);
+    this.remote = new RemoteDatabaseService(this.config.database, logger);
+
+    if (config.persistenceClient === 'surrealdb') {
+      this.persistenceClient = new SurrealDBPersistenceClient(this.local, logger);
+    } else if (config.persistenceClient === 'localstorage' || !config.persistenceClient) {
+      this.persistenceClient = new LocalStoragePersistenceClient(logger);
+    } else {
+      this.persistenceClient = config.persistenceClient;
+    }
+
+    this.streamProcessor = new StreamProcessorService(
+      new EventSystem(['stream_update']),
+      this.local,
+      this.persistenceClient,
+      logger
+    );
+    this.migrator = new LocalMigrator(this.local, logger);
+
+    this.cache = new CacheModule(
+      this.local,
+      this.streamProcessor,
+      (update) => {
+        // Direct callback from cache to data module
+        this.dataModule.onStreamUpdate(update);
+      },
+      logger
+    );
+
+    this.dataModule = new DataModule(
+      this.cache,
+      this.local,
+      this.config.schema,
+      logger,
+      this.config.streamDebounceTime
+    );
+
+    // Initialize Auth
+    this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
+
+    // Initialize Sync
+    this.sync = new SpookySync(this.local, this.remote, this.cache, this.dataModule, this.config.schema, this.logger);
+
+    // Initialize DevTools
+    this.devTools = new DevToolsService(
+      this.local,
+      this.remote,
+      logger,
+      this.config.schema,
+      this.auth,
+      this.dataModule
+    );
+
+    // Register DevTools as a receiver for stream updates
+    this.streamProcessor.addReceiver(this.devTools);
+
+    // Wire up callbacks instead of events
+    this.setupCallbacks();
+  }
+
+  /**
+   * Setup direct callbacks instead of event subscriptions
+   */
+  private setupCallbacks() {
+    // Mutation callback for sync
+    this.dataModule.onMutation((mutations: UpEvent[]) => {
+      // Notify DevTools
+      this.devTools.onMutation(mutations);
+
+      // Enqueue in Sync
+      if (mutations.length > 0) {
+        this.sync.enqueueMutation(mutations);
+      }
+    });
+
+    // Sync events for incoming updates
+    this.sync.events.subscribe('SYNC_QUERY_UPDATED', (event: any) => {
+      this.devTools.logEvent('SYNC_QUERY_UPDATED', event.payload);
+    });
+
+    // Database events for DevTools
+    this.local.getEvents().subscribe('DATABASE_LOCAL_QUERY', (event: any) => {
+      this.devTools.logEvent('LOCAL_QUERY', event.payload);
+    });
+
+    this.remote.getEvents().subscribe('DATABASE_REMOTE_QUERY', (event: any) => {
+      this.devTools.logEvent('REMOTE_QUERY', event.payload);
+    });
+  }
+
+  async init() {
+    this.logger.info(
+      { Category: 'spooky-client::SpookyClient::init' },
+      'SpookyClient initialization started'
+    );
+    try {
+      const clientId = this.config.clientId ?? (await this.loadOrGenerateClientId());
+      this.persistClientId(clientId);
+      this.logger.debug(
+        { clientId, Category: 'spooky-client::SpookyClient::init' },
+        'Client ID loaded'
+      );
+
+      await this.local.connect();
+      this.logger.debug(
+        { Category: 'spooky-client::SpookyClient::init' },
+        'Local database connected'
+      );
+
+      await this.migrator.provision(this.config.schemaSurql);
+      this.logger.debug({ Category: 'spooky-client::SpookyClient::init' }, 'Schema provisioned');
+
+      await this.remote.connect();
+      this.logger.debug(
+        { Category: 'spooky-client::SpookyClient::init' },
+        'Remote database connected'
+      );
+
+      await this.streamProcessor.init();
+      this.logger.debug(
+        { Category: 'spooky-client::SpookyClient::init' },
+        'StreamProcessor initialized'
+      );
+
+      await this.auth.init();
+      this.logger.debug({ Category: 'spooky-client::SpookyClient::init' }, 'Auth initialized');
+
+      await this.dataModule.init();
+      this.logger.debug(
+        { Category: 'spooky-client::SpookyClient::init' },
+        'DataModule initialized'
+      );
+
+      await this.sync.init(clientId);
+      this.logger.debug({ Category: 'spooky-client::SpookyClient::init' }, 'Sync initialized');
+
+      this.logger.info(
+        { Category: 'spooky-client::SpookyClient::init' },
+        'SpookyClient initialization completed successfully'
+      );
+    } catch (e) {
+      this.logger.error(
+        { error: e, Category: 'spooky-client::SpookyClient::init' },
+        'SpookyClient initialization failed'
+      );
+      throw e;
+    }
+  }
+
+  async close() {
+    await this.local.close();
+    await this.remote.close();
+  }
+
+  authenticate(token: string) {
+    return this.remote.getClient().authenticate(token);
+  }
+
+  deauthenticate() {
+    return this.remote.getClient().invalidate();
+  }
+
+  query<Table extends TableNames<S>>(
+    table: Table,
+    options: QueryOptions<TableModel<GetTable<S, Table>>, false>,
+    ttl: QueryTimeToLive = '10m'
+  ): QueryBuilder<S, Table, SpookyQueryResultPromise> {
+    return new QueryBuilder<S, Table, SpookyQueryResultPromise>(
+      this.config.schema,
+      table,
+      async (q) => ({
+        hash: await this.initQuery(table, q, ttl),
+      }),
+      options
+    );
+  }
+
+  private async initQuery<Table extends TableNames<S>>(
+    table: Table,
+    q: InnerQuery<any, any, any>,
+    ttl: QueryTimeToLive
+  ) {
+    const tableSchema = this.config.schema.tables.find((t) => t.name === table);
+    if (!tableSchema) {
+      throw new Error(`Table ${table} not found`);
+    }
+
+    const hash = await this.dataModule.query(
+      table,
+      q.selectQuery.query,
+      parseParams(tableSchema.columns, q.selectQuery.vars ?? {}),
+      ttl
+    );
+
+    await this.sync.enqueueDownEvent({
+      type: 'register',
+      payload: {
+        hash,
+      },
+    });
+
+    return hash;
+  }
+
+  async queryRaw(sql: string, params: Record<string, any>, ttl: QueryTimeToLive) {
+    const tableName = sql.split('FROM ')[1].split(' ')[0];
+    return this.dataModule.query(tableName, sql, params, ttl);
+  }
+
+  async subscribe(
+    queryHash: string,
+    callback: (records: Record<string, any>[]) => void,
+    options?: { immediate?: boolean }
+  ): Promise<() => void> {
+    return this.dataModule.subscribe(queryHash, callback, options);
+  }
+
+  run<
+    B extends BackendNames<S>,
+    R extends BackendRoutes<S, B>,
+  >(backend: B, path: R, payload: RoutePayload<S, B, R>, options?: RunOptions) {
+    return this.dataModule.run(backend, path, payload, options);
+  }
+
+  create(id: string, data: Record<string, unknown>) {
+    return this.dataModule.create(id, data);
+  }
+
+  update(table: string, id: string, data: Record<string, unknown>, options?: UpdateOptions) {
+    return this.dataModule.update(table, id, data, options);
+  }
+
+  delete(table: string, id: string) {
+    return this.dataModule.delete(table, id);
+  }
+
+  async useRemote<T>(fn: (client: Surreal) => Promise<T> | T): Promise<T> {
+    return fn(this.remote.getClient());
+  }
+
+  private persistClientId(id: string) {
+    try {
+      this.persistenceClient.set('spooky_client_id', id);
+    } catch (e) {
+      this.logger.warn(
+        { error: e, Category: 'spooky-client::SpookyClient::persistClientId' },
+        'Failed to persist client ID'
+      );
+    }
+  }
+
+  private async loadOrGenerateClientId(): Promise<string> {
+    const clientId = await this.persistenceClient.get<string>('spooky_client_id');
+
+    if (clientId) {
+      return clientId;
+    }
+
+    const newId = generateId();
+    await this.persistClientId(newId);
+    return newId;
+  }
+}

package/src/types.ts

@@ -0,0 +1,237 @@
+import { RecordId, SchemaStructure } from '@spooky-sync/query-builder';
+import { Level } from 'pino';
+import { PushEventOptions } from './events/index';
+import { UpEvent } from './modules/sync/index';
+
+export type { Level } from 'pino';
+
+/**
+ * The type of storage backend to use for the local database.
+ * - 'memory': In-memory storage (transient).
+ * - 'indexeddb': IndexedDB storage (persistent).
+ */
+export type StoreType = 'memory' | 'indexeddb';
+
+/**
+ * Interface for a custom persistence client.
+ * Allows providing a custom storage mechanism for the local database.
+ */
+export interface PersistenceClient {
+  /**
+   * Sets a value in the storage.
+   * @param key The key to set.
+   * @param value The value to store.
+   */
+  set<T>(key: string, value: T): Promise<void>;
+  /**
+   * Gets a value from the storage.
+   * @param key The key to retrieve.
+   * @returns The stored value or null if not found.
+   */
+  get<T>(key: string): Promise<T | null>;
+  /**
+   * Removes a value from the storage.
+   * @param key The key to remove.
+   */
+  remove(key: string): Promise<void>;
+}
+
+/**
+ * Supported Time-To-Live (TTL) values for cached queries.
+ * Format: number + unit (m=minutes, h=hours, d=days).
+ */
+export type QueryTimeToLive =
+  | '1m'
+  | '5m'
+  | '10m'
+  | '15m'
+  | '20m'
+  | '25m'
+  | '30m'
+  | '1h'
+  | '2h'
+  | '3h'
+  | '4h'
+  | '5h'
+  | '6h'
+  | '7h'
+  | '8h'
+  | '9h'
+  | '10h'
+  | '11h'
+  | '12h'
+  | '1d';
+
+/**
+ * Result object returned when a query is registered or executed.
+ */
+export interface SpookyQueryResult {
+  /** The unique hash identifier for the query. */
+  hash: string;
+}
+
+export type SpookyQueryResultPromise = Promise<SpookyQueryResult>;
+
+export interface EventSubscriptionOptions {
+  priority?: number;
+}
+
+/**
+ * Configuration options for the Spooky client.
+ * @template S The schema structure type.
+ */
+export interface SpookyConfig<S extends SchemaStructure> {
+  /** Database connection configuration. */
+  database: {
+    /** The SurrealDB endpoint URL. */
+    endpoint?: string;
+    /** The namespace to use. */
+    namespace: string;
+    /** The database name. */
+    database: string;
+    /** The local store type implementation. */
+    store?: StoreType;
+    /** Authentication token. */
+    token?: string;
+  };
+  /** Unique client identifier. If not provided, one will be generated. */
+  clientId?: string;
+  /** The schema definition. */
+  schema: S;
+  /** The compiled SURQL schema string. */
+  schemaSurql: string;
+  /** Logging level. */
+  logLevel: Level;
+  /**
+   * Persistence client to use.
+   * Can be a custom implementation, 'surrealdb' (default), or 'localstorage'.
+   */
+  persistenceClient?: PersistenceClient | 'surrealdb' | 'localstorage';
+  /** OpenTelemetry collector endpoint for telemetry data. */
+  otelEndpoint?: string;
+  /**
+   * Debounce time in milliseconds for stream updates.
+   * Defaults to 100ms.
+   */
+  streamDebounceTime?: number;
+}
+
+export type QueryHash = string;
+
+// Flat array format: [[record-id, version], [record-id, version], ...]
+export type RecordVersionArray = Array<[string, number]>;
+
+/**
+ * Represents the difference between two record version sets.
+ * Used for synchronizing local and remote states.
+ */
+export interface RecordVersionDiff {
+  /** List of records added. */
+  added: Array<{ id: RecordId<string>; version: number }>;
+  /** List of records updated. */
+  updated: Array<{ id: RecordId<string>; version: number }>;
+  /** List of record IDs removed. */
+  removed: RecordId<string>[];
+}
+
+/**
+ * Configuration for a specific query instance.
+ * Stores metadata about the query's state, parameters, and versioning.
+ */
+export interface QueryConfig {
+  /** The unique ID of the query config record. */
+  id: RecordId<string>;
+  /** The SURQL query string. */
+  surql: string;
+  /** Parameters used in the query. */
+  params: Record<string, any>;
+  /** The version array representing the local state of results. */
+  localArray: RecordVersionArray;
+  /** The version array representing the remote (server) state of results. */
+  remoteArray: RecordVersionArray;
+  /** Time-To-Live for this query. */
+  ttl: QueryTimeToLive;
+  /** Timestamp when the query was last accessed/active. */
+  lastActiveAt: Date;
+  /** The name of the table this query targets (if applicable). */
+  tableName: string;
+}
+
+export type QueryConfigRecord = QueryConfig & { id: string };
+
+/**
+ * Internal state of a live query.
+ */
+export interface QueryState {
+  /** The configuration for this query. */
+  config: QueryConfig;
+  /** The current cached records for this query. */
+  records: Record<string, any>[];
+  /** Timer for TTL expiration. */
+  ttlTimer: NodeJS.Timeout | null;
+  /** TTL duration in milliseconds. */
+  ttlDurationMs: number;
+  /** Number of times the query has been updated. */
+  updateCount: number;
+}
+
+// Callback types
+export type QueryUpdateCallback = (records: Record<string, any>[]) => void;
+export type MutationCallback = (mutations: UpEvent[]) => void;
+
+export type MutationEventType = 'create' | 'update' | 'delete';
+
+// Mutation event for sync
+/**
+ * Represents a mutation event (create, update, delete) to be synchronized.
+ */
+export interface MutationEvent {
+  /** Example: 'create', 'update', or 'delete'. */
+  type: MutationEventType;
+  /** unique id of the mutation */
+  mutation_id: RecordId<string>;
+  /** The ID of the record being mutated. */
+  record_id: RecordId<string>;
+  /** The data payload for create/update operations. */
+  data?: any;
+  /** The full record data (optional context). */
+  record?: any;
+  /** Options for the mutation event (e.g., debounce settings). */
+  options?: PushEventOptions;
+  /** Timestamp when the event was created. */
+  createdAt: Date;
+}
+
+/**
+ * Options for run operations.
+ */
+export interface RunOptions {
+  assignedTo?: string;
+  max_retries?: number;
+  retry_strategy?: 'linear' | 'exponential';
+}
+
+/**
+ * Options for update operations.
+ */
+export interface UpdateOptions {
+  /**
+   * Debounce configuration for the update.
+   * If boolean, enables default debounce behavior.
+   */
+  debounced?: boolean | DebounceOptions;
+}
+
+/**
+ * Configuration options for debouncing updates.
+ */
+export interface DebounceOptions {
+  /**
+   * The key to use for debouncing.
+   * - 'recordId': Debounce based on the specific record ID. WARNING: IT WILL ONLY ACCEPT THE LATEST CHANGE AND DOES *NOT* MERGE THE PREVIOUS ONCES. IF YOU ARE UNSURE JUST USE 'recordId_x_fields'.
+   * - 'recordId_x_fields': Debounce based on record ID and specific fields.
+   */
+  key?: 'recordId' | 'recordId_x_fields';
+  /** The debounce delay in milliseconds. */
+  delay?: number;
+}

package/src/utils/error-classification.ts

@@ -0,0 +1,28 @@
+const NETWORK_ERROR_PATTERNS = [
+  'connection',
+  'timeout',
+  'timed out',
+  'websocket',
+  'fetch failed',
+  'disconnected',
+  'socket',
+  'network',
+  'econnrefused',
+  'econnreset',
+  'enotfound',
+  'epipe',
+  'abort',
+];
+
+export function classifySyncError(error: unknown): 'network' | 'application' {
+  const message =
+    error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
+
+  for (const pattern of NETWORK_ERROR_PATTERNS) {
+    if (message.includes(pattern)) {
+      return 'network';
+    }
+  }
+
+  return 'application';
+}