@spooky-sync/core 0.0.1-canary.161 → 0.0.1-canary.162

package/src/services/database/connection-supervisor.ts

@@ -0,0 +1,351 @@
+import type { ConnectionState, ReconnectConfig } from '../../types';
+import { withTimeout } from '../../utils/index';
+import type { Logger } from '../logger/index';
+import type { RemoteDatabaseService } from './remote';
+
+/**
+ * Keeps the remote WebSocket alive for the whole life of the page.
+ *
+ * The SurrealDB SDK reconnects on its own after a socket `close`, but that
+ * covers only one of three ways the connection dies:
+ *
+ * 1. **Socket closes, SDK recovers.** Handled entirely by the SDK. This
+ *    supervisor only observes it (to report `reconnecting` upward).
+ * 2. **Socket closes, SDK gives up.** With `attempts: -1` this shouldn't happen
+ *    from exhaustion — but the SDK also terminates the engine permanently when
+ *    its post-reconnect handshake throws (it re-runs `version()`, `use()`,
+ *    `authenticate()` on every reconnect and closes the engine on any error).
+ *    One transient hiccup there would otherwise kill the page's connection for
+ *    good. The revive loop re-opens from scratch.
+ * 3. **Socket never closes at all.** A half-open connection: the peer is gone
+ *    (NAT timeout, wifi switch, laptop sleep) but no FIN ever arrives, so
+ *    `readyState` stays OPEN and the SDK's own 30s ping — fire-and-forget, no
+ *    response deadline — never notices. Nothing ever fires a `close` event, so
+ *    nothing ever triggers a reconnect. The heartbeat detects this and forces
+ *    the teardown that case 2's loop then repairs.
+ *
+ * Plus wake triggers: coming back `online` or un-hiding the tab probes
+ * immediately rather than waiting out a backoff that was scheduled while the
+ * network was known-down.
+ */
+export class ConnectionSupervisor {
+  private readonly logger: Logger;
+  private readonly config: Required<ReconnectConfig>;
+
+  private state: ConnectionState = 'disconnected';
+  private subscribers = new Set<(state: ConnectionState) => void>();
+
+  private started = false;
+  private disposed = false;
+
+  private heartbeatTimer: ReturnType<typeof setTimeout> | null = null;
+  private heartbeatInFlight = false;
+
+  private reviveTimer: ReturnType<typeof setTimeout> | null = null;
+  private reviveAttempts = 0;
+  private reviving = false;
+  /**
+   * Set while the browser reports itself offline. Retrying a socket against a
+   * down interface only burns backoff, so the loop parks until `online` fires.
+   */
+  private suspended = false;
+
+  private teardown: Array<() => void> = [];
+
+  private static readonly REVIVE_BASE_MS = 1_000;
+
+  constructor(
+    private readonly remote: RemoteDatabaseService,
+    logger: Logger,
+    config?: Required<ReconnectConfig>
+  ) {
+    this.logger = logger.child({ service: 'ConnectionSupervisor' });
+    this.config = config ?? remote.getReconnectConfig();
+  }
+
+  /** Latest observed transport state. */
+  get connection(): ConnectionState {
+    return this.state;
+  }
+
+  /**
+   * Observe transport state. Fires immediately with the current value and again
+   * on every change. Returns an unsubscribe.
+   */
+  subscribe(cb: (state: ConnectionState) => void): () => void {
+    cb(this.state);
+    this.subscribers.add(cb);
+    return () => {
+      this.subscribers.delete(cb);
+    };
+  }
+
+  /**
+   * Begin supervising. Call once, after the initial {@link
+   * RemoteDatabaseService.connect}. Idempotent.
+   */
+  start(): void {
+    if (this.started || this.disposed) return;
+    this.started = true;
+
+    this.setState(this.remote.getStatus());
+
+    this.teardown.push(
+      this.remote.subscribeConnection('connecting', () => this.setState('connecting')),
+      this.remote.subscribeConnection('reconnecting', () => {
+        this.setState('reconnecting');
+        // The SDK owns the retry from here; ours would fight it for the socket.
+        this.stopHeartbeat();
+      }),
+      this.remote.subscribeConnection('connected', () => {
+        this.reviveAttempts = 0;
+        this.clearReviveTimer();
+        this.setState('connected');
+        this.startHeartbeat();
+      }),
+      this.remote.subscribeConnection('disconnected', () => {
+        this.setState('disconnected');
+        this.stopHeartbeat();
+        // The SDK has stopped trying (exhausted, terminated, or handshake
+        // failure). From here on, reconnecting is entirely our job.
+        this.scheduleRevive();
+      }),
+      this.remote.subscribeConnection('error', (err) => {
+        this.logger.debug(
+          { err, Category: 'sp00ky-client::ConnectionSupervisor::error' },
+          'Transport error'
+        );
+      })
+    );
+
+    this.installWakeTriggers();
+
+    if (this.state === 'connected') this.startHeartbeat();
+    else this.scheduleRevive();
+  }
+
+  /** Stop all timers and listeners. Safe to call more than once. */
+  dispose(): void {
+    this.disposed = true;
+    this.started = false;
+    this.stopHeartbeat();
+    this.clearReviveTimer();
+    for (const off of this.teardown) {
+      try {
+        off();
+      } catch {
+        /* ignore */
+      }
+    }
+    this.teardown = [];
+    this.subscribers.clear();
+  }
+
+  private setState(next: ConnectionState): void {
+    if (this.state === next) return;
+    this.state = next;
+    this.logger.info(
+      { state: next, Category: 'sp00ky-client::ConnectionSupervisor::state' },
+      'Connection state changed'
+    );
+    for (const cb of this.subscribers) {
+      try {
+        cb(next);
+      } catch (err) {
+        this.logger.debug(
+          { err, Category: 'sp00ky-client::ConnectionSupervisor::state' },
+          'Connection subscriber threw'
+        );
+      }
+    }
+  }
+
+  // ---- Revive loop -------------------------------------------------------
+
+  private clearReviveTimer(): void {
+    if (this.reviveTimer !== null) {
+      clearTimeout(this.reviveTimer);
+      this.reviveTimer = null;
+    }
+  }
+
+  /**
+   * Queue the next `connect()` attempt on exponential backoff, capped at
+   * `superviseRetryDelayMaxMs`. Never gives up — the page is expected to
+   * outlive any outage.
+   */
+  private scheduleRevive(): void {
+    if (this.disposed || this.suspended) return;
+    if (this.reviveTimer !== null || this.reviving) return;
+    const delay = Math.min(
+      this.config.superviseRetryDelayMaxMs,
+      ConnectionSupervisor.REVIVE_BASE_MS * 2 ** this.reviveAttempts
+    );
+    this.reviveTimer = setTimeout(() => {
+      this.reviveTimer = null;
+      void this.revive();
+    }, delay);
+  }
+
+  private async revive(): Promise<void> {
+    if (this.disposed || this.suspended || this.reviving) return;
+    // The SDK may have recovered on its own between scheduling and firing.
+    if (this.remote.getStatus() === 'connected') {
+      this.reviveAttempts = 0;
+      return;
+    }
+    this.reviving = true;
+    this.reviveAttempts++;
+    this.setState('reconnecting');
+    this.logger.info(
+      {
+        attempt: this.reviveAttempts,
+        Category: 'sp00ky-client::ConnectionSupervisor::revive',
+      },
+      'Re-opening the remote connection'
+    );
+    try {
+      await this.remote.connect();
+      // Don't reset `reviveAttempts` or start the heartbeat here — the
+      // `connected` handler does both, and it's the only signal that the
+      // handshake (version/use/authenticate) actually completed.
+    } catch (err) {
+      this.logger.warn(
+        {
+          err,
+          attempt: this.reviveAttempts,
+          Category: 'sp00ky-client::ConnectionSupervisor::revive',
+        },
+        'Reconnect attempt failed; will retry'
+      );
+    } finally {
+      this.reviving = false;
+    }
+    if (this.remote.getStatus() !== 'connected') this.scheduleRevive();
+  }
+
+  // ---- Heartbeat watchdog ------------------------------------------------
+
+  private stopHeartbeat(): void {
+    if (this.heartbeatTimer !== null) {
+      clearTimeout(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
+  }
+
+  private startHeartbeat(): void {
+    this.stopHeartbeat();
+    if (this.disposed || this.suspended) return;
+    if (!(this.config.heartbeatIntervalMs > 0)) return;
+    this.heartbeatTimer = setTimeout(
+      () => void this.beat(),
+      this.config.heartbeatIntervalMs
+    );
+  }
+
+  /**
+   * Probe the server end-to-end. Deliberately goes through
+   * `remote.query` — the same serialized queue every other remote call uses —
+   * so a queue wedged behind a stuck RPC also fails the heartbeat instead of
+   * being invisible to it.
+   */
+  private async beat(): Promise<void> {
+    this.heartbeatTimer = null;
+    if (this.disposed || this.suspended) return;
+    if (this.remote.getStatus() !== 'connected') return;
+    if (this.heartbeatInFlight) {
+      this.startHeartbeat();
+      return;
+    }
+    this.heartbeatInFlight = true;
+    try {
+      await withTimeout(
+        this.remote.query('RETURN true'),
+        this.config.heartbeatTimeoutMs,
+        `Heartbeat timed out after ${this.config.heartbeatTimeoutMs}ms`
+      );
+      this.startHeartbeat();
+    } catch (err) {
+      this.logger.warn(
+        { err, Category: 'sp00ky-client::ConnectionSupervisor::heartbeat' },
+        'Heartbeat failed; tearing the socket down to force a reconnect'
+      );
+      // Force the `close` the transport never delivered. The resulting
+      // `disconnected` event drives the revive loop.
+      await this.remote.forceClose();
+      if (this.remote.getStatus() !== 'connected') this.scheduleRevive();
+    } finally {
+      this.heartbeatInFlight = false;
+    }
+  }
+
+  // ---- Wake triggers -----------------------------------------------------
+
+  /**
+   * A restored network or an un-hidden tab is the strongest available hint that
+   * a reconnect will now succeed, so probe immediately instead of waiting out a
+   * backoff scheduled under worse conditions.
+   */
+  private installWakeTriggers(): void {
+    if (typeof window !== 'undefined' && typeof window.addEventListener === 'function') {
+      const onOnline = () => {
+        this.suspended = false;
+        this.wake('online');
+      };
+      const onOffline = () => {
+        this.logger.info(
+          { Category: 'sp00ky-client::ConnectionSupervisor::offline' },
+          'Browser reports offline; parking reconnects until online'
+        );
+        this.suspended = true;
+        this.stopHeartbeat();
+        this.clearReviveTimer();
+        this.setState('disconnected');
+      };
+      window.addEventListener('online', onOnline);
+      window.addEventListener('offline', onOffline);
+      this.teardown.push(
+        () => window.removeEventListener('online', onOnline),
+        () => window.removeEventListener('offline', onOffline)
+      );
+      // A tab restored from bfcache keeps its dead socket; `pageshow` is the
+      // only event that fires in that path.
+      const onPageShow = () => this.wake('pageshow');
+      window.addEventListener('pageshow', onPageShow);
+      this.teardown.push(() => window.removeEventListener('pageshow', onPageShow));
+    }
+
+    if (typeof document !== 'undefined' && typeof document.addEventListener === 'function') {
+      const onVisibility = () => {
+        if (document.visibilityState !== 'visible') return;
+        this.wake('visibilitychange');
+      };
+      document.addEventListener('visibilitychange', onVisibility);
+      this.teardown.push(() => document.removeEventListener('visibilitychange', onVisibility));
+    }
+  }
+
+  /**
+   * Reset the backoff and act on whichever problem is present: reconnect if the
+   * socket is gone, otherwise probe it (it may be half-open — which is exactly
+   * what a sleep/wake cycle produces).
+   */
+  private wake(reason: string): void {
+    if (this.disposed || this.suspended) return;
+    this.logger.debug(
+      { reason, state: this.state, Category: 'sp00ky-client::ConnectionSupervisor::wake' },
+      'Wake trigger; probing the connection'
+    );
+    this.reviveAttempts = 0;
+    if (this.remote.getStatus() === 'connected') {
+      this.stopHeartbeat();
+      void this.beat();
+      return;
+    }
+    // 'reconnecting' means the SDK's own loop owns the socket; don't race it.
+    if (this.remote.getStatus() === 'disconnected') {
+      this.clearReviveTimer();
+      void this.revive();
+    }
+  }
+}

package/src/services/database/database.query-timeout.test.ts

@@ -0,0 +1,83 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { AbstractDatabaseService } from './database';
+import { DatabaseEventTypes, createDatabaseEventSystem } from './events/index';
+import { classifySyncError } from '../../utils/index';
+
+// Every remote query is serialized through one promise chain (to keep the WASM
+// engine's transactions safe), so a call that never settles doesn't just fail —
+// it blocks every later query behind it, including the sync poll's own health
+// probe. That's how a half-open socket wedges the client while health still
+// reports `healthy`: nothing ever throws, so nothing ever degrades.
+
+const silentLogger: any = (() => {
+  const l: any = {
+    child: () => l,
+    debug: () => {},
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    trace: () => {},
+  };
+  return l;
+})();
+
+class TestService extends AbstractDatabaseService {
+  protected eventType = DatabaseEventTypes.RemoteQuery;
+  constructor(client: any, timeoutMs: number) {
+    super(client, silentLogger, createDatabaseEventSystem());
+    this.queryTimeoutMs = timeoutMs;
+  }
+  async connect(): Promise<void> {}
+}
+
+describe('AbstractDatabaseService query timeout', () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  it('rejects a never-settling query and classifies it as a network failure', async () => {
+    const client = { query: vi.fn().mockImplementation(() => new Promise(() => {})) };
+    const svc = new TestService(client, 1_000);
+
+    const pending = svc.query('SELECT * FROM thread');
+    const assertion = expect(pending).rejects.toThrow(/timed out/);
+    await vi.advanceTimersByTimeAsync(1_000);
+    await assertion;
+
+    // "timed out" in the message is load-bearing: it routes the failure to
+    // `network`, which makes the sync queues retry instead of rolling the
+    // mutation back as a permanent application error.
+    const err = await pending.catch((e) => e);
+    expect(classifySyncError(err)).toBe('network');
+  });
+
+  it('unblocks the serialized queue so later queries still run', async () => {
+    let calls = 0;
+    const client = {
+      query: vi.fn().mockImplementation(() => {
+        calls++;
+        // First call hangs forever; the second would never be reached at all if
+        // the timeout didn't settle the chain link.
+        if (calls === 1) return new Promise(() => {});
+        return Promise.resolve([['ok']]);
+      }),
+    };
+    const svc = new TestService(client, 1_000);
+
+    const stuck = svc.query('SELECT 1');
+    const next = svc.query('SELECT 2');
+    const stuckAssertion = expect(stuck).rejects.toThrow(/timed out/);
+
+    await vi.advanceTimersByTimeAsync(1_000);
+    await stuckAssertion;
+    await expect(next).resolves.toEqual([['ok']]);
+  });
+
+  it('leaves queries unbounded when the timeout is disabled', async () => {
+    const client = { query: vi.fn().mockResolvedValue([['ok']]) };
+    const svc = new TestService(client, 0);
+
+    await expect(svc.query('SELECT 1')).resolves.toEqual([['ok']]);
+    // No timer was armed, so a slow-but-alive query is never cut off.
+    expect(vi.getTimerCount()).toBe(0);
+  });
+});

package/src/services/database/database.ts

@@ -4,11 +4,19 @@ import type {
   DatabaseEventSystem,
   DatabaseEventTypes} from './events/index';
 import type { SealedQuery } from '../../utils/surql';
+import { withTimeout } from '../../utils/index';
 
 export abstract class AbstractDatabaseService {
   protected client: Surreal;
   protected logger: Logger;
   protected events: DatabaseEventSystem;
+  /**
+   * Per-query deadline in ms; `0` disables. Only the remote service sets this
+   * (see `RemoteDatabaseService`) — a local query can be legitimately slow and
+   * has its own retry ladders, and there is no half-open-socket failure mode
+   * for an in-process engine.
+   */
+  protected queryTimeoutMs = 0;
   protected abstract eventType:
     | typeof DatabaseEventTypes.LocalQuery
     | typeof DatabaseEventTypes.RemoteQuery;
@@ -37,6 +45,11 @@ export abstract class AbstractDatabaseService {
 
   /**
    * Execute a query with serialized execution to prevent WASM transaction issues.
+   *
+   * Serialization means every query waits on the previous one, so a call that
+   * never settles blocks the whole chain forever. {@link queryTimeoutMs} bounds
+   * each link: on expiry this promise rejects and the chain moves on, even
+   * though the underlying RPC is still parked in the SDK's pending map.
    */
   async query<T extends unknown[]>(query: string, vars?: Record<string, unknown>): Promise<T> {
     return new Promise((resolve, reject) => {
@@ -52,7 +65,14 @@ export abstract class AbstractDatabaseService {
             const pending = this.client.query(query, vars);
             // In SurrealDB 2.0, .query() collects results by default.
             // We cast to T directly as proper typing depends on the caller knowing the return structure.
-            const result = (await pending) as unknown as T;
+            // "timed out" in the message is load-bearing: `classifySyncError`
+            // keys off it to classify this as `network` so the sync queues
+            // retry rather than rolling the mutation back.
+            const result = (await withTimeout(
+              pending as unknown as Promise<T>,
+              this.queryTimeoutMs,
+              `Remote query timed out after ${this.queryTimeoutMs}ms`
+            )) as T;
             const duration = performance.now() - startTime;
 
             // Emit query event

package/src/services/database/index.ts

@@ -1,6 +1,7 @@
 export * from './database';
 export * from './local';
 export * from './remote';
+export * from './connection-supervisor';
 export * from './local-migrator';
 export * from './events/index';
 export * from './cache-engine';

package/src/services/database/local-migrator.ts

@@ -46,6 +46,7 @@ export class LocalMigrator {
       DEFINE TABLE IF NOT EXISTS _00_stream_processor_state SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_query SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_preload SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
+      DEFINE TABLE IF NOT EXISTS _00_window SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_schema SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
       DEFINE TABLE IF NOT EXISTS _00_pending_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
     `;

package/src/services/database/remote.ts

@@ -1,18 +1,45 @@
 import type {
-  Diagnostic} from 'surrealdb';
+  Diagnostic,
+  SurrealEvents} from 'surrealdb';
 import {
   applyDiagnostics,
   createRemoteEngines,
   Surreal,
 } from 'surrealdb';
-import type { Sp00kyConfig } from '../../types';
+import type { ConnectionState, ReconnectConfig, Sp00kyConfig } from '../../types';
 import type { Logger } from '../logger/index';
 import { AbstractDatabaseService } from './database';
 import { createDatabaseEventSystem, DatabaseEventTypes } from './events/index';
 
+/** Defaults for {@link ReconnectConfig}. See that type for the rationale. */
+export const RECONNECT_DEFAULTS = {
+  attempts: -1,
+  retryDelayMax: 15_000,
+  heartbeatIntervalMs: 20_000,
+  heartbeatTimeoutMs: 10_000,
+  superviseRetryDelayMaxMs: 15_000,
+} as const;
+
+/** Fill in {@link RECONNECT_DEFAULTS} for every field the caller omitted. */
+export function resolveReconnectConfig(
+  input?: ReconnectConfig
+): Required<ReconnectConfig> {
+  return { ...RECONNECT_DEFAULTS, ...input };
+}
+
+/** Transport events the SDK publishes, mapped 1:1 to {@link ConnectionState}. */
+export type RemoteConnectionEvent = ConnectionState | 'error';
+
 export class RemoteDatabaseService extends AbstractDatabaseService {
   private config: Sp00kyConfig<any>['database'];
   protected eventType = DatabaseEventTypes.RemoteQuery;
+  private readonly reconnectConfig: Required<ReconnectConfig>;
+  /**
+   * In-flight `connect()`, so concurrent callers (boot + supervisor revive +
+   * an `online` event landing at the same moment) share one attempt instead of
+   * racing two sockets. Cleared on settle, so a later call always reconnects.
+   */
+  private connecting: Promise<void> | null = null;
 
   constructor(config: Sp00kyConfig<any>['database'], logger: Logger) {
     const events = createDatabaseEventSystem();
@@ -41,13 +68,68 @@ export class RemoteDatabaseService extends AbstractDatabaseService {
       events
     );
     this.config = config;
+    this.reconnectConfig = resolveReconnectConfig(config.reconnect);
+    this.queryTimeoutMs = Math.max(0, config.queryTimeoutMs ?? 60_000);
   }
 
   getConfig(): Sp00kyConfig<any>['database'] {
     return this.config;
   }
 
+  /** Resolved reconnect tunables; the supervisor reads its own knobs here. */
+  getReconnectConfig(): Required<ReconnectConfig> {
+    return this.reconnectConfig;
+  }
+
+  /** Current transport state as reported by the SDK. */
+  getStatus(): ConnectionState {
+    return this.client.status as ConnectionState;
+  }
+
+  /**
+   * Observe transport events. Thin passthrough so callers (the supervisor,
+   * sync, CRDT) don't have to reach through `getClient()`.
+   */
+  subscribeConnection<K extends RemoteConnectionEvent>(
+    event: K,
+    cb: (...payload: SurrealEvents[K]) => void
+  ): () => void {
+    return this.client.subscribe(event, cb);
+  }
+
+  /**
+   * Tear the socket down on purpose. Used by the heartbeat watchdog when a
+   * socket stops answering but never closes: `close()` makes the SDK publish
+   * `disconnected`, which is what drives the supervisor's revive loop.
+   */
+  async forceClose(): Promise<void> {
+    try {
+      await this.client.close();
+    } catch (err) {
+      this.logger.debug(
+        { err, Category: 'sp00ky-client::RemoteDatabaseService::forceClose' },
+        'forceClose failed; treating the socket as gone anyway'
+      );
+    }
+  }
+
+  /**
+   * Open (or re-open) the remote connection.
+   *
+   * Safe to call repeatedly: concurrent calls share the in-flight attempt, and
+   * a call after a `disconnected` builds a fresh socket. `use()` and
+   * `authenticate()` are re-applied here for the cold path; the SDK also
+   * replays them itself on its own internal reconnects.
+   */
   async connect(): Promise<void> {
+    if (this.connecting) return this.connecting;
+    this.connecting = this.doConnect().finally(() => {
+      this.connecting = null;
+    });
+    return this.connecting;
+  }
+
+  private async doConnect(): Promise<void> {
     const { endpoint, token, namespace, database } = this.getConfig();
     if (endpoint) {
       this.logger.info(
@@ -60,7 +142,21 @@ export class RemoteDatabaseService extends AbstractDatabaseService {
         'Connecting to remote database'
       );
       try {
-        await this.client.connect(endpoint);
+        // Without explicit options the SDK caps itself at 5 attempts (~62s of
+        // outage) and then gives up permanently — nothing would re-open the
+        // socket after that. `attempts: -1` keeps it trying; the supervisor
+        // still covers the case where the SDK terminates the engine because its
+        // post-reconnect handshake threw.
+        await this.client.connect(endpoint, {
+          reconnect: {
+            enabled: true,
+            attempts: this.reconnectConfig.attempts,
+            retryDelay: 1_000,
+            retryDelayMax: this.reconnectConfig.retryDelayMax,
+            retryDelayMultiplier: 2,
+            retryDelayJitter: 0.1,
+          },
+        });
         await this.client.use({
           namespace,
           database,

package/src/services/database/sqlite-cache-engine.ts

@@ -53,6 +53,7 @@ const SYSTEM_TABLES = [
   '_00_stream_processor_state',
   '_00_query',
   '_00_preload',
+  '_00_window',
   '_00_schema',
   '_00_pending_mutations',
   // Server-written, synced-down meta tables (see meta_tables_client.surql).