@spooky-sync/core 0.0.1-canary.29 → 0.0.1-canary.291

package/src/sp00ky.ts

@@ -1,395 +1,5 @@
-import { DataModule } from './modules/data/index';
-import {
-  Sp00kyConfig,
-  QueryTimeToLive,
-  Sp00kyQueryResultPromise,
-  PersistenceClient,
-  MutationEvent,
-  UpdateOptions,
-  RunOptions,
-} from './types';
-import {
-  LocalDatabaseService,
-  LocalMigrator,
-  RemoteDatabaseService,
-} from './services/database/index';
-import { Surreal } from 'surrealdb';
-import { Sp00kySync, UpEvent } from './modules/sync/index';
-import {
-  GetTable,
-  InnerQuery,
-  QueryBuilder,
-  QueryOptions,
-  SchemaStructure,
-  TableModel,
-  TableNames,
-  BucketNames,
-  BackendNames,
-  BackendRoutes,
-  RoutePayload,
-} from '@spooky-sync/query-builder';
-
-import { DevToolsService } from './modules/devtools/index';
-import { createLogger } from './services/logger/index';
-import { AuthService } from './modules/auth/index';
-import { StreamProcessorService } from './services/stream-processor/index';
-import { EventSystem } from './events/index';
-import { CacheModule } from './modules/cache/index';
-import { LocalStoragePersistenceClient } from './services/persistence/localstorage';
-import { generateId, parseParams } from './utils/index';
-import { SurrealDBPersistenceClient } from './services/persistence/surrealdb';
-import { ResilientPersistenceClient } from './services/persistence/resilient';
-
-export class BucketHandle {
-  constructor(private bucketName: string, private remote: RemoteDatabaseService) {}
-
-  async put(path: string, content: string | Uint8Array | Blob): Promise<void> {
-    await this.remote.query(`RETURN f"${this.bucketName}:/${path}".put($content);`, { content });
-  }
-
-  async get(path: string): Promise<unknown> {
-    const [result] = await this.remote.query<[unknown]>(`RETURN f"${this.bucketName}:/${path}".get();`);
-    return result;
-  }
-
-  async delete(path: string): Promise<void> {
-    await this.remote.query(`RETURN f"${this.bucketName}:/${path}".delete();`);
-  }
-
-  async exists(path: string): Promise<boolean> {
-    const [result] = await this.remote.query<[boolean]>(`RETURN f"${this.bucketName}:/${path}".exists();`);
-    return result;
-  }
-
-  async head(path: string): Promise<Record<string, unknown>> {
-    const [result] = await this.remote.query<[Record<string, unknown>]>(`RETURN f"${this.bucketName}:/${path}".head();`);
-    return result;
-  }
-
-  async copy(sourcePath: string, targetPath: string): Promise<void> {
-    await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".copy($target);`, { target: targetPath });
-  }
-
-  async rename(sourcePath: string, targetPath: string): Promise<void> {
-    await this.remote.query(`RETURN f"${this.bucketName}:/${sourcePath}".rename($target);`, { target: targetPath });
-  }
-
-  async list(prefix?: string): Promise<string[]> {
-    const p = prefix ?? '';
-    const [result] = await this.remote.query<[string[]]>(`RETURN f"${this.bucketName}:/${p}".list();`);
-    return result;
-  }
-}
-
-export class Sp00kyClient<S extends SchemaStructure> {
-  private local: LocalDatabaseService;
-  private remote: RemoteDatabaseService;
-  private persistenceClient: PersistenceClient;
-
-  private migrator: LocalMigrator;
-  private cache: CacheModule;
-  private dataModule: DataModule<S>;
-  private sync: Sp00kySync<S>;
-  private devTools: DevToolsService;
-
-  private logger: ReturnType<typeof createLogger>;
-  public auth: AuthService<S>;
-  public streamProcessor: StreamProcessorService;
-
-  get remoteClient() {
-    return this.remote.getClient();
-  }
-
-  get localClient() {
-    return this.local.getClient();
-  }
-
-  get pendingMutationCount(): number {
-    return this.sync.pendingMutationCount;
-  }
-
-  subscribeToPendingMutations(cb: (count: number) => void): () => void {
-    return this.sync.subscribeToPendingMutations(cb);
-  }
-
-  constructor(private config: Sp00kyConfig<S>) {
-    const logger = createLogger(config.logLevel ?? 'info', config.otelTransmit);
-    this.logger = logger.child({ service: 'Sp00kyClient' });
-
-    this.logger.info(
-      {
-        config: { ...config, schema: '[SchemaStructure]' },
-        Category: 'sp00ky-client::Sp00kyClient::constructor',
-      },
-      'Sp00kyClient initialized'
-    );
-
-    this.local = new LocalDatabaseService(this.config.database, logger);
-    this.remote = new RemoteDatabaseService(this.config.database, logger);
-
-    if (config.persistenceClient === 'surrealdb') {
-      this.persistenceClient = new SurrealDBPersistenceClient(this.local, logger);
-    } else if (config.persistenceClient === 'localstorage' || !config.persistenceClient) {
-      this.persistenceClient = new LocalStoragePersistenceClient(logger);
-    } else {
-      this.persistenceClient = config.persistenceClient;
-    }
-
-    this.persistenceClient = new ResilientPersistenceClient(this.persistenceClient, logger);
-
-    this.streamProcessor = new StreamProcessorService(
-      new EventSystem(['stream_update']),
-      this.local,
-      this.persistenceClient,
-      logger
-    );
-    this.migrator = new LocalMigrator(this.local, logger);
-
-    this.cache = new CacheModule(
-      this.local,
-      this.streamProcessor,
-      (update) => {
-        // Direct callback from cache to data module
-        this.dataModule.onStreamUpdate(update);
-      },
-      logger
-    );
-
-    this.dataModule = new DataModule(
-      this.cache,
-      this.local,
-      this.config.schema,
-      logger,
-      this.config.streamDebounceTime
-    );
-
-    // Initialize Auth
-    this.auth = new AuthService(this.config.schema, this.remote, this.persistenceClient, logger);
-
-    // Initialize Sync
-    this.sync = new Sp00kySync(this.local, this.remote, this.cache, this.dataModule, this.config.schema, this.logger);
-
-    // Initialize DevTools
-    this.devTools = new DevToolsService(
-      this.local,
-      this.remote,
-      logger,
-      this.config.schema,
-      this.auth,
-      this.dataModule
-    );
-
-    // Register DevTools as a receiver for stream updates
-    this.streamProcessor.addReceiver(this.devTools);
-
-    // Wire up callbacks instead of events
-    this.setupCallbacks();
-  }
-
-  /**
-   * Setup direct callbacks instead of event subscriptions
-   */
-  private setupCallbacks() {
-    // Mutation callback for sync
-    this.dataModule.onMutation((mutations: UpEvent[]) => {
-      // Notify DevTools
-      this.devTools.onMutation(mutations);
-
-      // Enqueue in Sync
-      if (mutations.length > 0) {
-        this.sync.enqueueMutation(mutations);
-      }
-    });
-
-    // Sync events for incoming updates
-    this.sync.events.subscribe('SYNC_QUERY_UPDATED', (event: any) => {
-      this.devTools.logEvent('SYNC_QUERY_UPDATED', event.payload);
-    });
-
-    // Database events for DevTools
-    this.local.getEvents().subscribe('DATABASE_LOCAL_QUERY', (event: any) => {
-      this.devTools.logEvent('LOCAL_QUERY', event.payload);
-    });
-
-    this.remote.getEvents().subscribe('DATABASE_REMOTE_QUERY', (event: any) => {
-      this.devTools.logEvent('REMOTE_QUERY', event.payload);
-    });
-  }
-
-  async init() {
-    this.logger.info(
-      { Category: 'sp00ky-client::Sp00kyClient::init' },
-      'Sp00kyClient initialization started'
-    );
-    try {
-      const clientId = this.config.clientId ?? (await this.loadOrGenerateClientId());
-      this.persistClientId(clientId);
-      this.logger.debug(
-        { clientId, Category: 'sp00ky-client::Sp00kyClient::init' },
-        'Client ID loaded'
-      );
-
-      await this.local.connect();
-      this.logger.debug(
-        { Category: 'sp00ky-client::Sp00kyClient::init' },
-        'Local database connected'
-      );
-
-      await this.migrator.provision(this.config.schemaSurql);
-      this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Schema provisioned');
-
-      await this.remote.connect();
-      this.logger.debug(
-        { Category: 'sp00ky-client::Sp00kyClient::init' },
-        'Remote database connected'
-      );
-
-      await this.streamProcessor.init();
-      this.logger.debug(
-        { Category: 'sp00ky-client::Sp00kyClient::init' },
-        'StreamProcessor initialized'
-      );
-
-      await this.auth.init();
-      this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Auth initialized');
-
-      await this.dataModule.init();
-      this.logger.debug(
-        { Category: 'sp00ky-client::Sp00kyClient::init' },
-        'DataModule initialized'
-      );
-
-      await this.sync.init(clientId);
-      this.logger.debug({ Category: 'sp00ky-client::Sp00kyClient::init' }, 'Sync initialized');
-
-      this.logger.info(
-        { Category: 'sp00ky-client::Sp00kyClient::init' },
-        'Sp00kyClient initialization completed successfully'
-      );
-    } catch (e) {
-      this.logger.error(
-        { error: e, Category: 'sp00ky-client::Sp00kyClient::init' },
-        'Sp00kyClient initialization failed'
-      );
-      throw e;
-    }
-  }
-
-  async close() {
-    await this.local.close();
-    await this.remote.close();
-  }
-
-  authenticate(token: string) {
-    return this.remote.getClient().authenticate(token);
-  }
-
-  deauthenticate() {
-    return this.remote.getClient().invalidate();
-  }
-
-  query<Table extends TableNames<S>>(
-    table: Table,
-    options: QueryOptions<TableModel<GetTable<S, Table>>, false>,
-    ttl: QueryTimeToLive = '10m'
-  ): QueryBuilder<S, Table, Sp00kyQueryResultPromise> {
-    return new QueryBuilder<S, Table, Sp00kyQueryResultPromise>(
-      this.config.schema,
-      table,
-      async (q) => ({
-        hash: await this.initQuery(table, q, ttl),
-      }),
-      options
-    );
-  }
-
-  private async initQuery<Table extends TableNames<S>>(
-    table: Table,
-    q: InnerQuery<any, any, any>,
-    ttl: QueryTimeToLive
-  ) {
-    const tableSchema = this.config.schema.tables.find((t) => t.name === table);
-    if (!tableSchema) {
-      throw new Error(`Table ${table} not found`);
-    }
-
-    const hash = await this.dataModule.query(
-      table,
-      q.selectQuery.query,
-      parseParams(tableSchema.columns, q.selectQuery.vars ?? {}),
-      ttl
-    );
-
-    await this.sync.enqueueDownEvent({
-      type: 'register',
-      payload: {
-        hash,
-      },
-    });
-
-    return hash;
-  }
-
-  async queryRaw(sql: string, params: Record<string, any>, ttl: QueryTimeToLive) {
-    const tableName = sql.split('FROM ')[1].split(' ')[0];
-    return this.dataModule.query(tableName, sql, params, ttl);
-  }
-
-  async subscribe(
-    queryHash: string,
-    callback: (records: Record<string, any>[]) => void,
-    options?: { immediate?: boolean }
-  ): Promise<() => void> {
-    return this.dataModule.subscribe(queryHash, callback, options);
-  }
-
-  run<
-    B extends BackendNames<S>,
-    R extends BackendRoutes<S, B>,
-  >(backend: B, path: R, payload: RoutePayload<S, B, R>, options?: RunOptions) {
-    return this.dataModule.run(backend, path, payload, options);
-  }
-
-  bucket<B extends BucketNames<S>>(name: B): BucketHandle {
-    return new BucketHandle(name, this.remote);
-  }
-
-  create(id: string, data: Record<string, unknown>) {
-    return this.dataModule.create(id, data);
-  }
-
-  update(table: string, id: string, data: Record<string, unknown>, options?: UpdateOptions) {
-    return this.dataModule.update(table, id, data, options);
-  }
-
-  delete(table: string, id: string) {
-    return this.dataModule.delete(table, id);
-  }
-
-  async useRemote<T>(fn: (client: Surreal) => Promise<T> | T): Promise<T> {
-    return fn(this.remote.getClient());
-  }
-
-  private persistClientId(id: string) {
-    try {
-      this.persistenceClient.set('sp00ky_client_id', id);
-    } catch (e) {
-      this.logger.warn(
-        { error: e, Category: 'sp00ky-client::Sp00kyClient::persistClientId' },
-        'Failed to persist client ID'
-      );
-    }
-  }
-
-  private async loadOrGenerateClientId(): Promise<string> {
-    const clientId = await this.persistenceClient.get<string>('sp00ky_client_id');
-
-    if (clientId) {
-      return clientId;
-    }
-
-    const newId = generateId();
-    await this.persistClientId(newId);
-    return newId;
-  }
-}
+// The client lives in client/; this path is kept for `index.ts` and deep imports.
+export { Sp00kyClient } from './client/sp00ky-client';
+export type { Sp00kyClientDeps } from './client/sp00ky-client';
+export { BucketHandle, bucketContentToBlob } from './client/bucket-handle';
+export type { BucketPutOptions, BucketPutResult } from './client/bucket-handle';

package/src/state/client-state.ts

@@ -0,0 +1,200 @@
+import type { QueryPlan, RecordId } from '@spooky-sync/query-builder';
+import type {
+  ConnectionState,
+  MutationEventType,
+  QueryHash,
+  QueryTimeToLive,
+  RecordVersionArray,
+  RegistrationTimings,
+  SyncHealth,
+} from '../types';
+import type { QueryLifecycle } from './lifecycle';
+import type { LaneState } from '../kernel/saga';
+import { emptyLanes } from '../kernel/saga';
+
+export type Row = Record<string, unknown>;
+
+/** Everything a query is, minus anything that changes after registration. */
+export interface QueryDefinition {
+  /** Session-salted `_00_query:<hash>` record id: the remote view id. */
+  readonly id: RecordId<string>;
+  readonly hash: QueryHash;
+  /** Unsalted `sha256({surql, params})`: key of the durable `_00_view` row. */
+  readonly viewKey: string;
+  readonly surql: string;
+  readonly params: Readonly<Record<string, unknown>>;
+  readonly plan?: QueryPlan;
+  readonly ttl: QueryTimeToLive;
+  readonly ttlMs: number;
+  readonly tableName: string;
+  readonly createdAt: number;
+}
+
+export type ServerViewState = 'materializing' | 'ready' | null;
+
+export interface QueryTelemetry {
+  readonly updateCount: number;
+  readonly errorCount: number;
+  readonly lastUpdatedAt: number | null;
+  readonly materializationSamples: ReadonlyArray<number>;
+  readonly lastIngestLatencyMs: number | null;
+  readonly phaseSamples: Readonly<Record<string, ReadonlyArray<number>>>;
+  readonly phaseLast: Readonly<Record<string, number | null>>;
+  readonly registrationTimings: RegistrationTimings;
+}
+
+export interface QueryEntry {
+  readonly def: QueryDefinition;
+  readonly lifecycle: QueryLifecycle;
+  /** Server membership (ids + versions) or the durable seed. */
+  readonly remoteArray: RecordVersionArray;
+  /** The in-browser SSP view's id-set: local truth for "does this row match". */
+  readonly localArray: RecordVersionArray;
+  readonly subqueryRemoteArray: RecordVersionArray;
+  readonly records: ReadonlyArray<Row>;
+  readonly serverState: ServerViewState;
+  readonly subscribers: number;
+  readonly lastSubscriberLeftAt: number | null;
+  readonly lastHeartbeatAt: number | null;
+  readonly lastPolledAt: number | null;
+  readonly registerAttempts: number;
+  /**
+   * Re-registrations issued because the `_00_query` row read as gone, since
+   * the row was last seen. Paces the next one (see `applyMembership`).
+   */
+  readonly viewLostCount: number;
+  /** When the paced re-registration is due; null when none is pending. */
+  readonly viewLostRetryAt: number | null;
+  readonly telemetry: QueryTelemetry;
+}
+
+export type OutboxStatus = 'pending' | 'acked';
+
+/** In-memory mirror of one `_00_pending_mutations` row plus its push progress. */
+export interface OutboxItem {
+  readonly id: string;
+  readonly type: MutationEventType;
+  readonly recordId: string;
+  readonly table: string;
+  readonly status: OutboxStatus;
+  readonly ackedAt: number | null;
+  readonly attempts: number;
+}
+
+export type TabRole = 'solo' | 'leader' | 'follower';
+
+/** A debounced update accumulating locally until its flush writes the outbox row. */
+export interface PendingWrite {
+  readonly key: string;
+  readonly table: string;
+  readonly recordId: string;
+  readonly data: Readonly<Record<string, unknown>>;
+  readonly before: Readonly<Record<string, unknown>> | null;
+  readonly firstAt: number;
+  /** Its durable copy in `_00_pending_writes`; null when `flushOnHide` is off. */
+  readonly mirrorId: string | null;
+}
+
+export interface SyncSlice {
+  readonly health: SyncHealth;
+  readonly consecutiveFailures: number;
+  readonly hasSyncedOnce: boolean;
+  readonly selfHealAttempts: number;
+  readonly pollIdleStreak: number;
+  readonly lastReconnectRefetchAt: number | null;
+  readonly needsResubscribe: boolean;
+  readonly fetchAttempts: number;
+  readonly liveUuid: string | null;
+  readonly liveTable: string | null;
+  readonly lanes: LaneState;
+}
+
+export interface ClientState {
+  readonly sessionId: string | null;
+  readonly userId: string | null;
+  /** The principal the current session salt was minted for. */
+  readonly saltUserId: string | null;
+  /** Latest bucket-switch target; an older switch that wakes up to a newer target skips. */
+  readonly pendingBucket: string | null;
+  readonly tabId: string;
+  readonly tabRole: TabRole;
+  readonly bucketId: string | null;
+  readonly localReady: boolean;
+  readonly primed: boolean;
+  readonly queries: ReadonlyMap<QueryHash, QueryEntry>;
+  /** Hashes whose local registration is in flight (dedupes concurrent `query()` calls). */
+  readonly registering: ReadonlySet<QueryHash>;
+  /** Re-read attempts per hash while the server reports a view as `materializing`. */
+  readonly membershipReread: ReadonlyMap<QueryHash, number>;
+  /** Local body versions (`_00_rv`) by encoded record id. */
+  readonly versions: ReadonlyMap<string, number>;
+  /**
+   * What each durable `_00_view` row vouches for, by view key: its members
+   * and subquery children. A body no row names is an orphan: it never paints
+   * before the server answers and the GC deletes it.
+   */
+  readonly views: ReadonlyMap<string, ReadonlyArray<string>>;
+  readonly outbox: ReadonlyArray<OutboxItem>;
+  readonly pendingWrites: ReadonlyMap<string, PendingWrite>;
+  readonly failedCount: number;
+  /** Queries whose render inputs changed since their last materialization. */
+  readonly dirty: ReadonlySet<QueryHash>;
+  /** Queries whose server membership must be re-read. */
+  readonly membershipDirty: ReadonlySet<QueryHash>;
+  readonly sync: SyncSlice;
+}
+
+export const emptyTelemetry = (): QueryTelemetry => ({
+  updateCount: 0,
+  errorCount: 0,
+  lastUpdatedAt: null,
+  materializationSamples: [],
+  lastIngestLatencyMs: null,
+  phaseSamples: {},
+  phaseLast: {},
+  registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+});
+
+export const initialHealth = (connection: ConnectionState = 'disconnected'): SyncHealth => ({
+  status: 'healthy',
+  consecutiveFailures: 0,
+  everConnected: false,
+  connection,
+});
+
+export function emptyState(init: { tabId: string }): ClientState {
+  return {
+    sessionId: null,
+    userId: null,
+    saltUserId: null,
+    pendingBucket: null,
+    tabId: init.tabId,
+    tabRole: 'solo',
+    bucketId: null,
+    localReady: false,
+    primed: false,
+    queries: new Map(),
+    registering: new Set(),
+    membershipReread: new Map(),
+    versions: new Map(),
+    views: new Map(),
+    outbox: [],
+    pendingWrites: new Map(),
+    failedCount: 0,
+    dirty: new Set(),
+    membershipDirty: new Set(),
+    sync: {
+      health: initialHealth(),
+      consecutiveFailures: 0,
+      hasSyncedOnce: false,
+      selfHealAttempts: 0,
+      pollIdleStreak: 0,
+      lastReconnectRefetchAt: null,
+      needsResubscribe: false,
+      fetchAttempts: 0,
+      liveUuid: null,
+      liveTable: null,
+      lanes: emptyLanes(),
+    },
+  };
+}