@spooky-sync/core 0.0.1-canary.9 → 0.0.1-canary.91

package/src/sp00ky.ts

@@ -0,0 +1,648 @@
+import { DataModule } from './modules/data/index';
+import type {
+  Sp00kyConfig,
+  QueryTimeToLive,
+  QueryStatusCallback,
+  Sp00kyQueryResultPromise,
+  PersistenceClient,
+  UpdateOptions,
+  RunOptions,
+  SyncHealth} from './types';
+import {
+  LocalDatabaseService,
+  LocalMigrator,
+  RemoteDatabaseService,
+} from './services/database/index';
+import type { UpEvent } from './modules/sync/index';
+import { Sp00kySync } from './modules/sync/index';
+import type {
+  GetTable,
+  InnerQuery,
+  QueryOptions,
+  SchemaStructure,
+  TableModel,
+  TableNames,
+  BucketNames,
+  BackendNames,
+  BackendRoutes,
+  RoutePayload} from '@spooky-sync/query-builder';
+import {
+  QueryBuilder
+} from '@spooky-sync/query-builder';
+
+import { DevToolsService } from './modules/devtools/index';
+import { createLogger } from './services/logger/index';
+import { AuthService } from './modules/auth/index';
+import { StreamProcessorService } from './services/stream-processor/index';
+import { extractSelectPermissions } from './services/stream-processor/permissions';
+import { EventSystem } from './events/index';
+import { CacheModule } from './modules/cache/index';
+import type { RecordWithId } from './modules/cache/index';
+import { CrdtManager, CrdtField } from './modules/crdt/index';
+import { FeatureFlagModule, FeatureFlagHandle } from './modules/feature-flag/index';
+import type { FeatureFlagOptions } from './modules/feature-flag/index';
+import { LocalStoragePersistenceClient } from './services/persistence/localstorage';
+import { parseParams, encodeRecordId } from './utils/index';
+import { SurrealDBPersistenceClient } from './services/persistence/surrealdb';
+import { ResilientPersistenceClient } from './services/persistence/resilient';
+
+export class BucketHandle {
+  constructor(private bucketName: string, private remote: RemoteDatabaseService) {}
+
+  async put(path: string, content: string | Uint8Array | Blob): Promise<void> {
+    await this.remote.query(`RETURN f"${this.bucketName}:/${path}".put($content);`, { content });
+  }
+
+  async get(path: string): Promise<unknown> {
+    const [result] = await this.remote.query<[unknown]>(`RETURN f"${this.bucketName}:/${path}".get();`);
+    return result;
+  }
+
+  async delete(path: string): Promise<void> {
+    await this.remote.query(`RETURN f"${this.bucketName}:/${path}".delete();`);
+  }
+
+  async exists(path: string): Promise<boolean> {
+    const [result] = await this.remote.query<[boolean]>(`RETURN f"${this.bucketName}:/${path}".exists();`);
+    return result;
+  }
+
+  async head(path: string): Promise<Record<string, unknown>> {
+    const [result] = await this.remote.query<[Record<string, unknown>]>(`RETURN f"${this.bucketName}:/${path}".head();`);
+    return result;
+  }
+
+  async copy(sourcePath: string, targetPath: string): Promise<void> {
+    await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".copy($target);`, { target: targetPath });
+  }
+
+  async rename(sourcePath: string, targetPath: string): Promise<void> {
+    await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".rename($target);`, { target: targetPath });
+  }
+
+  async list(prefix?: string): Promise<string[]> {
+    const p = prefix ?? '';
+    const [result] = await this.remote.query<[string[]]>(`RETURN f"${this.bucketName}:/${p}".list();`);
+    return result;
+  }
+}
+
+export class Sp00kyClient<S extends SchemaStructure> {
+  private local: LocalDatabaseService;
+  private remote: RemoteDatabaseService;
+  private persistenceClient: PersistenceClient;
+
+  private migrator: LocalMigrator;
+  private cache: CacheModule;
+  private dataModule: DataModule<S>;
+  private sync: Sp00kySync<S>;
+  private devTools: DevToolsService;
+  private crdtManager: CrdtManager;
+  private featureFlags!: FeatureFlagModule<S>;
+
+  private logger: ReturnType<typeof createLogger>;
+  public auth: AuthService<S>;
+  public streamProcessor: StreamProcessorService;
+
+  get remoteClient() {
+    return this.remote.getClient();
+  }
+
+  get localClient() {
+    return this.local.getClient();
+  }
+
+  get pendingMutationCount(): number {
+    return this.sync.pendingMutationCount;
+  }
+
+  /** Number of times the initial list_ref LIVE subscription retried on
+   *  the most recent `setCurrentUserId` call. 0 when the SSP's
+   *  pre-emptive user-table creation got there first; >0 when LIVE
+   *  registration hit a "table not found" race. Exposed so the e2e
+   *  suite can guard the pre-emptive path against regression. */
+  get liveRetryCount(): number {
+    return this.sync.liveRetryCount;
+  }
+
+  subscribeToPendingMutations(cb: (count: number) => void): () => void {
+    return this.sync.subscribeToPendingMutations(cb);
+  }
+
+  /** Current sync-health snapshot. See {@link Sp00kyConfig.syncHealth}. */
+  get syncHealth(): SyncHealth {
+    return this.sync.syncHealth;
+  }
+
+  /**
+   * Observe sync health. Fires immediately with the current status and again
+   * on every healthy↔degraded transition. Returns an unsubscribe.
+   */
+  subscribeToSyncHealth(cb: (health: SyncHealth) => void): () => void {
+    return this.sync.subscribeToSyncHealth(cb);
+  }
+
+  constructor(private config: Sp00kyConfig<S>) {
+    const logger = createLogger(config.logLevel ?? 'info', config.otelTransmit);
+    this.logger = logger.child({ service: 'Sp00kyClient' });
+
+    this.logger.info(
+      {
+        config: { ...config, schema: '[SchemaStructure]' },
+        Category: 'sp00ky-client::Sp00kyClient::constructor',
+      },
+      'Sp00kyClient initialized'
+    );
+
+    this.local = new LocalDatabaseService(this.config.database, logger);
+    this.remote = new RemoteDatabaseService(this.config.database, logger);
+
+    if (config.persistenceClient === 'surrealdb') {
+      this.persistenceClient = new SurrealDBPersistenceClient(this.local, logger);
+    } else if (config.persistenceClient === 'localstorage' || !config.persistenceClient) {
+      this.persistenceClient = new LocalStoragePersistenceClient(logger);
+    } else {
+      this.persistenceClient = config.persistenceClient;
+    }
+
+    this.persistenceClient = new ResilientPersistenceClient(this.persistenceClient, logger);
+
+    this.streamProcessor = new StreamProcessorService(
+      new EventSystem(['stream_update']),
+      this.local,
+      this.persistenceClient,
+      logger
+    );
+    this.migrator = new LocalMigrator(this.local, logger);
+
+    this.cache = new CacheModule(
+      this.local,
+      this.streamProcessor,
+      (update) => {
+        // Direct callback from cache to data module
+        this.dataModule.onStreamUpdate(update);
+      },
+      logger
+    );
+
+    // Initialize CRDT Manager. `local` is used to read the initial
+    // `_00_crdt` snapshot when a field opens AND to mirror every local
+    // edit (so reload/offline see the freshest state); `remote` is used
+    // for the debounced outgoing UPSERTs and the parent-table LIVE feed.
+    // The debounce window is configurable via `crdtDebounceMs`.
+    this.crdtManager = new CrdtManager(
+      this.config.schema,
+      this.local,
+      this.remote,
+      logger,
+      config.crdtDebounceMs ?? 500,
+    );
+
+    this.dataModule = new DataModule(
+      this.cache,
+      this.local,
+      this.config.schema,
+      logger,
+      this.config.streamDebounceTime
+    );
+
+    // Initialize Auth
+    this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
+
+    // Initialize Sync
+    this.sync = new Sp00kySync(
+      this.local,
+      this.remote,
+      this.cache,
+      this.dataModule,
+      this.config.schema,
+      this.logger,
+      {
+        refSyncIntervalMs: this.config.refSyncIntervalMs,
+        anonymousLiveQueries: this.config.enableAnonymousLiveQueries,
+        // `syncHealth: false` (or `{ degradeAfterConsecutiveFailures: 0 }`)
+        // disables degraded reporting; otherwise default to 3.
+        degradeAfterConsecutiveFailures:
+          this.config.syncHealth === false
+            ? 0
+            : this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3,
+      }
+    );
+
+    // Initialize feature flags. Reuses the down-queue to register SSP plans
+    // on `_00_user_feature` and the auth subscription to re-register handles
+    // when the signed-in user changes.
+    this.featureFlags = new FeatureFlagModule({
+      dataModule: this.dataModule,
+      sync: this.sync,
+      auth: this.auth,
+      logger,
+    });
+
+    // Initialize DevTools
+    this.devTools = new DevToolsService(
+      this.local,
+      this.remote,
+      logger,
+      this.config.schema,
+      this.auth,
+      this.dataModule
+    );
+
+    // Register DevTools as a receiver for stream updates
+    this.streamProcessor.addReceiver(this.devTools);
+
+    // Wire up callbacks instead of events
+    this.setupCallbacks();
+  }
+
+  /**
+   * Setup direct callbacks instead of event subscriptions
+   */
+  private setupCallbacks() {
+    // Surface query fetch-status changes (idle/fetching) in DevTools. Logs a
+    // discrete event and triggers a state push so the active-queries panel
+    // reflects the flip immediately.
+    this.dataModule.onQueryStatusChange = (queryHash, status) => {
+      this.devTools.logEvent('QUERY_STATUS_CHANGED', { queryHash, status });
+    };
+
+    // Keep an actively-watched query's remote `_00_query.lastActiveAt` fresh so
+    // the server TTL sweep doesn't expire it out from under live subscribers.
+    // DataModule fires this only while the query still has ≥1 subscriber.
+    this.dataModule.onHeartbeat = (queryHash) => {
+      void this.sync.heartbeatQuery(queryHash).catch((err) => {
+        this.logger.warn(
+          { err, queryHash, Category: 'sp00ky-client::Sp00kyClient::onHeartbeat' },
+          'TTL heartbeat failed'
+        );
+      });
+    };
+
+    // Eager teardown of an opt-in deregistered query: enqueue a `cleanup`
+    // down-event so it's serialized after any in-flight register/sync for the
+    // same query (avoids out-of-order delete-before-create).
+    this.dataModule.onDeregister = (queryHash) => {
+      this.sync.enqueueDownEvent({ type: 'cleanup', payload: { hash: queryHash } });
+    };
+
+    // Mutation callback for sync
+    this.dataModule.onMutation((mutations: UpEvent[]) => {
+      // Notify DevTools
+      this.devTools.onMutation(mutations);
+
+      // Enqueue in Sync
+      if (mutations.length > 0) {
+        this.sync.enqueueMutation(mutations);
+      }
+    });
+
+    // Sync events for incoming updates
+    this.sync.events.subscribe('SYNC_QUERY_UPDATED', (event: any) => {
+      this.devTools.logEvent('SYNC_QUERY_UPDATED', event.payload);
+    });
+
+    // Hand list_ref-driven row ingests to the CrdtManager so CRDT body
+    // / cursor updates reach the receiver even when the cross-session
+    // LIVE on the parent table is filtered out by the SurrealDB
+    // permission-LIVE gap. Same-user clients receive these rows via
+    // CrdtManager's own `LIVE SELECT * FROM <table>`; this hook is the
+    // redundant path that fires when only the list_ref bumped.
+    this.sync.engineEvents.subscribe('SYNC_REMOTE_DATA_INGESTED', (event: any) => {
+      try {
+        const records: Array<Record<string, any>> = event.payload?.records ?? [];
+        for (const row of records) {
+          const id = row?.id;
+          const table =
+            id && typeof id === 'object' && id.table !== undefined
+              ? String(id.table)
+              : undefined;
+          if (!table) continue;
+          this.crdtManager.applyRow(table, row);
+        }
+      } catch (err) {
+        this.logger.debug(
+          { err, Category: 'sp00ky-client::engineEvents::ingested' },
+          'applyRow forwarding from sync ingest failed'
+        );
+      }
+    });
+
+    // Database events for DevTools
+    this.local.getEvents().subscribe('DATABASE_LOCAL_QUERY', (event: any) => {
+      this.devTools.logEvent('LOCAL_QUERY', event.payload);
+    });
+
+    this.remote.getEvents().subscribe('DATABASE_REMOTE_QUERY', (event: any) => {
+      this.devTools.logEvent('REMOTE_QUERY', event.payload);
+    });
+  }
+
+  async init() {
+    this.logger.info(
+      { Category: 'sp00ky-client::Sp00kyClient::init' },
+      'Sp00kyClient initialization started'
+    );
+    try {
+      await this.local.connect();
+      this.logger.debug(
+        { Category: 'sp00ky-client::Sp00kyClient::init' },
+        'Local database connected'
+      );
+
+      await this.migrator.provision(this.config.schemaSurql);
+      this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Schema provisioned');
+
+      await this.remote.connect();
+      this.logger.debug(
+        { Category: 'sp00ky-client::Sp00kyClient::init' },
+        'Remote database connected'
+      );
+
+      await this.streamProcessor.init();
+      // Seed table `select` permissions from the schema before any query is
+      // registered — otherwise the SSP default-denies every non-`_00_` table.
+      this.streamProcessor.setPermissions(
+        extractSelectPermissions(this.config.schemaSurql)
+      );
+      this.logger.debug(
+        { Category: 'sp00ky-client::Sp00kyClient::init' },
+        'StreamProcessor initialized'
+      );
+
+      await this.auth.init();
+      this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Auth initialized');
+
+      // Salt query-id hashing with the SurrealDB session id so two browsers
+      // for the same user don't collide on shared `_00_query` rows. The same
+      // session id is the `session_id` key in `_00_cursor` rows, so the
+      // CrdtManager needs it too.
+      const sessionId = await this.fetchSessionId();
+      await this.dataModule.init(sessionId);
+      this.crdtManager.setSessionId(sessionId);
+      this.logger.debug(
+        { sessionId, Category: 'sp00ky-client::Sp00kyClient::init' },
+        'DataModule initialized'
+      );
+
+      // Refresh the salt whenever auth state flips (sign-in, sign-out).
+      // session::id() changes per WebSocket session, and a sign-in spawns
+      // a new authenticated session, so the salt must follow. Also
+      // forward the user id into `DataModule` and `Sp00kySync` so they
+      // can route to per-user `_00_query_user_<id>` /
+      // `_00_list_ref_user_<id>` tables in `RefMode.Dedicated` — the
+      // LIVE subscription on `_00_list_ref_user_<id>` is restarted
+      // under the new auth context inside `Sp00kySync.setCurrentUserId`
+      // since SurrealDB binds the LIVE permission at registration time.
+      //
+      // Sync prefix BEFORE the first `await`: setting `currentUserId`
+      // synchronously here is critical because the AuthProvider's own
+      // subscribe callback runs right after ours and immediately enables
+      // queries that depend on the user id. Any `await` before
+      // `setCurrentUserId` would let those queries register against the
+      // stale (null) user id and hit the wrong `_00_query[_user_*]`
+      // table.
+      this.auth.subscribe(async (userId) => {
+        this.dataModule.setCurrentUserId(userId);
+        // Mirror the server's `fn::query::register` auth injection for the
+        // in-browser SSP: feed the current user's full record id + access
+        // method so `$auth`-gated table permissions (e.g. `thread`) resolve
+        // locally instead of being rejected. Set synchronously BEFORE the
+        // first `await` (like `setCurrentUserId` above) so queries that
+        // re-register on this auth flip see the fresh context, not a stale one.
+        this.streamProcessor.setSessionAuth(
+          this.auth.currentUser?.id ? encodeRecordId(this.auth.currentUser.id) : null,
+          this.auth.access
+        );
+        const next = await this.fetchSessionId();
+        this.dataModule.setSessionId(next);
+        this.crdtManager.setSessionId(next);
+        try {
+          await this.sync.setCurrentUserId(userId);
+        } catch (e) {
+          this.logger.error(
+            { error: e, Category: 'sp00ky-client::Sp00kyClient::authChange' },
+            'sync.setCurrentUserId failed'
+          );
+        }
+      });
+
+      await this.sync.init();
+      this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Sync initialized');
+
+      this.featureFlags.init();
+      this.logger.debug(
+        { Category: 'sp00ky-client::Sp00kyClient::init' },
+        'FeatureFlagModule initialized'
+      );
+
+      this.logger.info(
+        { Category: 'sp00ky-client::Sp00kyClient::init' },
+        'Sp00kyClient initialization completed successfully'
+      );
+    } catch (e) {
+      this.logger.error(
+        { error: e, Category: 'sp00ky-client::Sp00kyClient::init' },
+        'Sp00kyClient initialization failed'
+      );
+      throw e;
+    }
+  }
+
+  async close() {
+    await this.featureFlags.closeAll();
+    this.crdtManager.closeAll();
+    await this.local.close();
+    await this.remote.close();
+  }
+
+  /**
+   * Subscribe to a feature flag for the current user. Returns a
+   * `FeatureFlagHandle` whose `variant()`, `payload()` and `enabled()`
+   * accessors reflect the latest assignment from `_00_user_feature`,
+   * and whose `subscribe(cb)` fires whenever that assignment changes.
+   *
+   * Permissions are enforced by SurrealDB: a client can only ever see
+   * its own row, and cannot create or modify assignments.
+   */
+  feature(key: string, options?: FeatureFlagOptions): FeatureFlagHandle {
+    return this.featureFlags.feature(key, options);
+  }
+
+  authenticate(token: string) {
+    return this.remote.getClient().authenticate(token);
+  }
+
+  /**
+   * Open a CRDT field for collaborative editing.
+   * Returns a CrdtField with a LoroDoc that can be bound to any editor.
+   * Also starts a LIVE SELECT on the parent table for real-time sync;
+   * incoming events trigger a subquery fetch of `_00_crdt` / `_00_cursor`.
+   */
+  async openCrdtField(
+    table: string,
+    recordId: string,
+    field: string,
+    fallbackText?: string,
+  ): Promise<CrdtField> {
+    return this.crdtManager.open(table, recordId, field, fallbackText);
+  }
+
+  /**
+   * Close a CRDT field when editing is done.
+   */
+  closeCrdtField(table: string, recordId: string, field: string): void {
+    this.crdtManager.close(table, recordId, field);
+  }
+
+  deauthenticate() {
+    return this.remote.getClient().invalidate();
+  }
+
+  query<Table extends TableNames<S>>(
+    table: Table,
+    options: QueryOptions<TableModel<GetTable<S, Table>>, false>,
+    ttl: QueryTimeToLive = '10m'
+  ): QueryBuilder<S, Table, Sp00kyQueryResultPromise> {
+    return new QueryBuilder<S, Table, Sp00kyQueryResultPromise>(
+      this.config.schema,
+      table,
+      async (q) => ({
+        hash: await this.initQuery(table, q, ttl),
+      }),
+      options
+    );
+  }
+
+  private async initQuery<Table extends TableNames<S>>(
+    table: Table,
+    q: InnerQuery<any, any, any>,
+    ttl: QueryTimeToLive
+  ) {
+    const tableSchema = this.config.schema.tables.find((t) => t.name === table);
+    if (!tableSchema) {
+      throw new Error(`Table ${table} not found`);
+    }
+
+    const params = parseParams(tableSchema.columns, q.selectQuery.vars ?? {});
+    const hash = await this.dataModule.query(table, q.selectQuery.query, params, ttl);
+
+    // Instant-hydrate: for a cold query, fetch its rows one-shot from the remote
+    // (its own surql, run directly like useRemote) and display them NOW; the full
+    // realtime registration proceeds below. Hydrated rows carry their `_00_rv`
+    // versions so the registration's syncRecords skips re-pulling unchanged bodies.
+    // Best-effort: any failure (e.g. offline) just falls through to registration.
+    if (this.config.instantHydrate !== false && this.dataModule.isCold(hash)) {
+      try {
+        const [rows] = await this.remote.query<[RecordWithId[]]>(q.selectQuery.query, params);
+        await this.dataModule.applyHydration(hash, rows ?? []);
+      } catch (err) {
+        this.logger.warn(
+          { err, hash, Category: 'sp00ky-client::Sp00kyClient::instantHydrate' },
+          'Instant hydrate failed; proceeding with registration'
+        );
+      }
+    }
+
+    await this.sync.enqueueDownEvent({
+      type: 'register',
+      payload: {
+        hash,
+      },
+    });
+
+    return hash;
+  }
+
+  async queryRaw(sql: string, params: Record<string, any>, ttl: QueryTimeToLive) {
+    const tableName = sql.split('FROM ')[1].split(' ')[0];
+    return this.dataModule.query(tableName, sql, params, ttl);
+  }
+
+  async subscribe(
+    queryHash: string,
+    callback: (records: Record<string, any>[]) => void,
+    options?: { immediate?: boolean }
+  ): Promise<() => void> {
+    return this.dataModule.subscribe(queryHash, callback, options);
+  }
+
+  /**
+   * Opt-in eager teardown for a query whose last subscriber has gone away
+   * (e.g. a viewport-windowed list cancelling an off-screen window). No-op
+   * while any subscriber remains. Tears down the remote `_00_query` view +
+   * local WASM view instead of waiting for the TTL sweep. Default behavior
+   * (no call here) keeps the view resident for cheap re-subscription.
+   */
+  deregisterQuery(queryHash: string): void {
+    this.dataModule.deregisterQuery(queryHash);
+  }
+
+  /**
+   * Subscribe to a query's fetch-status changes (idle/fetching). With
+   * `{ immediate: true }` the callback fires synchronously with the current
+   * status. Powers the `useQuery` hook's `isFetching()` accessor.
+   */
+  subscribeQueryStatus(
+    queryHash: string,
+    callback: QueryStatusCallback,
+    options?: { immediate?: boolean }
+  ): () => void {
+    return this.dataModule.subscribeStatus(queryHash, callback, options);
+  }
+
+  /**
+   * Report the frontend processing time (ms) a client framework spent applying
+   * an update for a query (e.g. `useQuery`'s `reconcile()`), so DevTools/MCP can
+   * surface the "frontend" phase of the per-query timing breakdown.
+   */
+  reportFrontendTiming(queryHash: string, ms: number): void {
+    this.dataModule.recordFrontendTiming(queryHash, ms);
+  }
+
+  run<
+    B extends BackendNames<S>,
+    R extends BackendRoutes<S, B>,
+  >(backend: B, path: R, payload: RoutePayload<S, B, R>, options?: RunOptions) {
+    return this.dataModule.run(backend, path, payload, options);
+  }
+
+  bucket<B extends BucketNames<S>>(name: B): BucketHandle {
+    return new BucketHandle(name, this.remote);
+  }
+
+  create(id: string, data: Record<string, unknown>) {
+    return this.dataModule.create(id, data);
+  }
+
+  update(table: string, id: string, data: Record<string, unknown>, options?: UpdateOptions) {
+    return this.dataModule.update(table, id, data, options);
+  }
+
+  delete(table: string, id: string) {
+    return this.dataModule.delete(table, id);
+  }
+
+  async useRemote<T>(fn: (client: Surreal) => Promise<T> | T): Promise<T> {
+    return fn(this.remote.getClient());
+  }
+
+  /**
+   * Fetch SurrealDB's `session::id()` as a string. Used as a salt for
+   * query-id hashing so two sessions for the same user get distinct
+   * `_00_query` rows. Returns empty string if the query fails (we still
+   * boot, just without session scoping for IDs).
+   */
+  private async fetchSessionId(): Promise<string> {
+    try {
+      const [sid] = await this.remote.query<[string]>('RETURN <string>session::id()');
+      return typeof sid === 'string' ? sid : '';
+    } catch (e) {
+      this.logger.warn(
+        { error: e, Category: 'sp00ky-client::Sp00kyClient::fetchSessionId' },
+        'Failed to fetch session::id() — proceeding with empty salt'
+      );
+      return '';
+    }
+  }
+}