@spooky-sync/core 0.0.1-canary.161 → 0.0.1-canary.162

package/src/sp00ky.ts

@@ -11,7 +11,12 @@ import type {
   SyncHealth,
   StorageHealth,
 } from './types';
-import { LocalMigrator, RemoteDatabaseService, createLocalEngine } from './services/database/index';
+import {
+  ConnectionSupervisor,
+  LocalMigrator,
+  RemoteDatabaseService,
+  createLocalEngine,
+} from './services/database/index';
 import type { LocalStore } from './services/database/index';
 import { StaleEpochError } from './services/database/index';
 import type { UpEvent } from './modules/sync/index';
@@ -148,6 +153,7 @@ function writeBootBucketHint(bucketId: string): void {
 export class Sp00kyClient<S extends SchemaStructure> {
   private local: LocalStore;
   private remote: RemoteDatabaseService;
+  private connectionSupervisor: ConnectionSupervisor;
   private persistenceClient: PersistenceClient;
 
   private migrator: LocalMigrator;
@@ -267,6 +273,10 @@ export class Sp00kyClient<S extends SchemaStructure> {
       shared: tabsSupport.supported,
     });
     this.remote = new RemoteDatabaseService(this.config.database, logger);
+    // Owns socket liveness for the life of the client: re-opens the connection
+    // when the SDK's own reconnect gives up, and heartbeats to catch a socket
+    // that died without ever firing a `close` event.
+    this.connectionSupervisor = new ConnectionSupervisor(this.remote, logger);
 
     if (config.persistenceClient === 'surrealdb') {
       this.persistenceClient = new SurrealDBPersistenceClient(this.local, logger);
@@ -343,6 +353,9 @@ export class Sp00kyClient<S extends SchemaStructure> {
           this.config.syncHealth === false
             ? 0
             : (this.config.syncHealth?.degradeAfterConsecutiveFailures ?? 3),
+        pushTimeoutMs: this.config.pushTimeoutMs,
+        // Read-only: sync mirrors the transport state into `SyncHealth`.
+        connectionSupervisor: this.connectionSupervisor,
       }
     );
 
@@ -616,6 +629,10 @@ export class Sp00kyClient<S extends SchemaStructure> {
       }
 
       await this.remote.connect();
+      // Start supervising only after the first connect succeeds, so a boot-time
+      // failure surfaces as a thrown init() rather than being silently absorbed
+      // into a background retry loop.
+      this.connectionSupervisor.start();
       this.logger.debug(
         { Category: 'sp00ky-client::Sp00kyClient::init' },
         'Remote database connected'
@@ -638,6 +655,16 @@ export class Sp00kyClient<S extends SchemaStructure> {
       // for the same user don't collide on shared `_00_query` rows. The same
       // session id is the `session_id` key in `_00_cursor` rows, so the
       // CrdtManager needs it too.
+      //
+      // Deliberately NOT refreshed on reconnect, even though `session::id()`
+      // does change with every WebSocket session. The salt keys `_00_query`
+      // rows and local cache entries, so rotating it would invalidate every
+      // query hash and force a full re-register plus a cache-key migration on
+      // each blip. Those rows stay valid because the TTL heartbeat keeps them
+      // alive, not because the session that created them is still open — so a
+      // stable salt across reconnects is the correct behavior. Auth flips are
+      // the one case that must rotate it (below): a sign-in is a different
+      // principal, not the same session on a new socket.
       const sessionId = await this.fetchSessionId();
       await this.dataModule.init(sessionId);
       this.crdtManager.setSessionId(sessionId);
@@ -868,9 +895,13 @@ export class Sp00kyClient<S extends SchemaStructure> {
   }
 
   async close() {
+    // Before anything else: a live supervisor would read the intentional
+    // `remote.close()` below as a failure and immediately reconnect.
+    this.connectionSupervisor.dispose();
     await this.featureFlags.closeAll();
     await this.appReleases.closeAll();
     this.crdtManager.closeAll();
+    this.crdtManager.dispose();
     // Leaving the broker first hands leadership to another tab (and releases
     // the OPFS handles via the worker shutdown) before the store closes.
     if (this.tabsCoordinator) await this.tabsCoordinator.stop();

package/src/types.ts

@@ -124,6 +124,22 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
      * multi-hop path (escape hatch while the worker-side path beds in).
      */
     workerSelect?: boolean;
+    /**
+     * WebSocket reconnect + liveness tuning. All fields optional; the defaults
+     * keep the connection alive indefinitely without configuration. See
+     * {@link ReconnectConfig}.
+     */
+    reconnect?: ReconnectConfig;
+    /**
+     * Deadline (ms) for every remote RPC. Remote queries are serialized through
+     * a single promise chain, so one call that never settles (half-open socket:
+     * the WebSocket looks open, the peer is gone, no `close` event fires) would
+     * otherwise wedge ALL later remote traffic behind it — including the sync
+     * poll's own health probe, leaving health pinned at `healthy` with no
+     * banner and no self-heal. The deadline turns that into an ordinary network
+     * failure the queue retries. `0` disables. Defaults to `60_000`.
+     */
+    queryTimeoutMs?: number;
   };
   /** The schema definition. */
   schema: S;
@@ -254,6 +270,15 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
    * (or `degradeAfterConsecutiveFailures: 0`) to never report degraded.
    */
   syncHealth?: SyncHealthConfig | false;
+  /**
+   * Deadline (ms) for a single outgoing mutation push. Tighter than
+   * {@link Sp00kyConfig.database.queryTimeoutMs} because the up-queue drains
+   * one mutation at a time behind an `isSyncingUp` flag: a push that never
+   * settles stops every later mutation for the session, with no retry and no
+   * error. On expiry the push is treated as a network failure and re-queued.
+   * `0` disables. Defaults to `30_000`.
+   */
+  pushTimeoutMs?: number;
 }
 
 /** Tunables for sync-health reporting. See {@link Sp00kyConfig.syncHealth}. */
@@ -267,6 +292,55 @@ export interface SyncHealthConfig {
   degradeAfterConsecutiveFailures?: number;
 }
 
+/**
+ * Tunables for WebSocket reconnect and liveness detection. See
+ * {@link Sp00kyConfig.database.reconnect}.
+ *
+ * Two independent mechanisms cooperate here. The SurrealDB SDK reconnects on
+ * its own after a socket `close` (`attempts` / `retryDelayMax`), and a
+ * supervisor above it re-opens the connection from scratch whenever the SDK
+ * gives up or its post-reconnect handshake fails — the SDK terminates the
+ * engine permanently in that case, so a supervisor is required, not optional.
+ * The heartbeat covers the third case: a socket that never closes at all.
+ */
+export interface ReconnectConfig {
+  /**
+   * SDK reconnect attempts after a socket close. `-1` retries forever.
+   * Defaults to `-1` (the SDK's own default is `5`, which caps recovery at a
+   * ~62s outage and then gives up for the life of the page).
+   */
+  attempts?: number;
+  /** Cap on the SDK's exponential backoff delay. Defaults to `15_000`. */
+  retryDelayMax?: number;
+  /**
+   * Cadence of the application-level liveness probe (`RETURN true`) that
+   * detects a half-open socket the transport never reports as closed.
+   * `0` disables the heartbeat. Defaults to `20_000`.
+   */
+  heartbeatIntervalMs?: number;
+  /**
+   * Deadline for a heartbeat response. Exceeding it means the socket is dead
+   * regardless of what its `readyState` claims, so the connection is torn down
+   * and rebuilt. Defaults to `10_000`.
+   */
+  heartbeatTimeoutMs?: number;
+  /**
+   * Cap on the supervisor's own backoff between `connect()` retries once the
+   * SDK has given up. Defaults to `15_000`.
+   */
+  superviseRetryDelayMaxMs?: number;
+}
+
+/**
+ * Transport-level connection state, independent of {@link SyncHealthStatus}.
+ *
+ * These answer different questions: `connection` is about the socket,
+ * `status` is about whether sync rounds are succeeding. A `connected` socket
+ * can still be `degraded` (server erroring), and a `reconnecting` socket is
+ * usually still `healthy` for the first few seconds.
+ */
+export type ConnectionState = 'connecting' | 'connected' | 'reconnecting' | 'disconnected';
+
 export type SyncHealthStatus = 'healthy' | 'degraded';
 
 /** Snapshot of sync health delivered to `subscribeToSyncHealth` subscribers. */
@@ -286,6 +360,13 @@ export interface SyncHealth {
    * a working session. Never resets back to `false` once set.
    */
   everConnected: boolean;
+  /**
+   * Live transport state of the remote WebSocket. Distinct from `status`: this
+   * one flips the instant the socket drops, whereas `status` only degrades
+   * after a sustained run of failed sync rounds. Use it to show "reconnecting…"
+   * immediately without waiting for the degrade threshold.
+   */
+  connection: ConnectionState;
 }
 
 export type StorageHealthStatus = 'unknown' | 'persistent' | 'memory';
@@ -370,6 +451,24 @@ export interface QueryConfig {
    * `rowCount` / `localArray`.
    */
   subqueryRemoteArray?: RecordVersionArray;
+  /**
+   * Whether authoritative membership (`remoteArray`) has ever been established
+   * for this query — either fetched from `_00_list_ref` this session, or read
+   * back from the durable `_00_window` row on a cold start.
+   *
+   * Tri-state matters: "known and empty" must render an empty list, while
+   * "never established" has to fall back to a predicate scan of the local store
+   * so a query first run on this device still paints offline. A
+   * `remoteArray.length === 0` check cannot tell those apart.
+   */
+  membershipKnown?: boolean;
+  /**
+   * Key of this query's durable `_00_window` membership row: a hash of
+   * `{surql, params}` WITHOUT the `session::id()` salt that `id` carries, so it
+   * survives a reload (which mints a new session id) and a bucket switch.
+   * In-memory only.
+   */
+  membershipKey?: string;
   /** Time-To-Live for this query. */
   ttl: QueryTimeToLive;
   /** Timestamp when the query was last accessed/active. */

package/src/utils/index.ts

@@ -199,3 +199,41 @@ export async function withRetry<T>(
   }
   throw lastError;
 }
+
+/**
+ * Reject after `timeoutMs` if `promise` hasn't settled.
+ *
+ * Exists because a WebSocket RPC has no deadline of its own: the SurrealDB SDK
+ * parks each call in a pending map and only rejects it when the socket reports
+ * a close. On a half-open socket (peer gone, no `close` event, `readyState`
+ * still OPEN) the call never settles at all, which wedges anything serialized
+ * behind it.
+ *
+ * `message` MUST contain "timed out" so `classifySyncError` treats the
+ * rejection as `network` — that's what makes the sync queues retry the
+ * operation instead of rolling the mutation back as an application error.
+ *
+ * Non-positive `timeoutMs` returns `promise` unchanged. The underlying promise
+ * is NOT cancelled (nothing can cancel an in-flight RPC); its eventual
+ * settlement is absorbed, so attach no expectations to it after a timeout.
+ */
+export function withTimeout<T>(
+  promise: Promise<T>,
+  timeoutMs: number,
+  message: string
+): Promise<T> {
+  if (!(timeoutMs > 0)) return promise;
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
+    promise.then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (err) => {
+        clearTimeout(timer);
+        reject(err);
+      }
+    );
+  });
+}