@spooky-sync/core 0.0.1-canary.162 → 0.0.1-canary.164

package/AGENTS.md

@@ -24,7 +24,8 @@ Local mutations are applied optimistically, ingested into a DBSP layer that driv
 ## Key exports (`src/index.ts`)
 
 - `Sp00kyClient<S>` — main class. Methods: `init()`, `create(id, payload)`, `update(table, id, payload, options?)`, `delete(table, idOrSelector)`, `query(table, opts?)`, `run(backend, route, payload)`, `bucket(name)`, `useRemote(fn)`, `authenticate(token)`, `signOut()`. Plus `pendingMutationCount` and `subscribeToPendingMutations(cb)`.
-- `BucketHandle` — file storage handle (`put`, `get`, `delete`, `exists`).
+- `BucketHandle` — file storage handle (`put`, `get`, `delete`, `exists`), plus the cache-aware `read`, `url`, `pin`/`unpin`, `evict`, `prefetch`. `get` is always remote; `read`/`url` go through the blob cache.
+- `services/blobs/` — durable cache for bucket bytes: OPFS holds the files, `_00_blob` holds a manifest that reconcile rebuilds from disk (so a wiped local store costs metadata, not the offline cache). Nothing expires on a timer; eviction is LRU under a byte budget only, skipping pinned and on-screen entries. Config: `blobCache` in `types.ts`.
 - `AuthService` — token management, sign-in/sign-out events.
 - `CrdtManager`, `CrdtField`, `cursorColorFromName`, `CURSOR_COLORS` — Loro-CRDT integration.
 - Types: `Sp00kyConfig`, `SyncedDbConfig` (re-exported by client-solid as the consumer-facing shape), `QueryTimeToLive`, `PersistenceClient`, `StoreType`, `UpdateOptions`, `RunOptions`.

package/dist/index.d.ts

@@ -119,8 +119,12 @@ declare class ConnectionSupervisor {
   private disposed;
   private heartbeatTimer;
   private heartbeatInFlight;
+  /** Consecutive failed probes. See {@link FAILURES_BEFORE_TEARDOWN}. */
+  private heartbeatFailures;
   private reviveTimer;
   private reviveAttempts;
+  /** Timestamp of the last wake-triggered probe, for rate limiting. */
+  private lastWakeProbeAt;
   private reviving;
   /**
    * Set while the browser reports itself offline. Retrying a socket against a
@@ -129,6 +133,23 @@ declare class ConnectionSupervisor {
   private suspended;
   private teardown;
   private static readonly REVIVE_BASE_MS;
+  /**
+   * How many consecutive heartbeat failures it takes to tear the socket down.
+   *
+   * The probe rides the same serialized queue as every other RPC (deliberately
+   * — see {@link beat}), which means it cannot distinguish a WEDGED queue from
+   * a merely BUSY one. A single slow window (a large sync burst, one heavy
+   * app query) used to be enough to force-close a perfectly healthy socket,
+   * and the resulting reconnect re-registered every active query about a
+   * second later. That self-inflicted teardown manufactured the very reconnect
+   * storms this class exists to survive. A genuinely dead socket still fails
+   * every probe, so it is torn down one interval later than before.
+   */
+  private static readonly FAILURES_BEFORE_TEARDOWN;
+  /** Retry delay after an inconclusive (first) heartbeat failure. */
+  private static readonly HEARTBEAT_RETRY_MS;
+  /** Floor between probes triggered by wake events (tab focus, pageshow). */
+  private static readonly WAKE_PROBE_MIN_INTERVAL_MS;
   constructor(remote: RemoteDatabaseService, logger: Logger$1, config?: Required<ReconnectConfig>);
   /** Latest observed transport state. */
   get connection(): ConnectionState;
@@ -1068,6 +1089,14 @@ declare class Sp00kySync<S extends SchemaStructure> {
    * initial connect. See {@link subscribeToReconnect}.
    */
   private needsResubscribe;
+  /** When the last reconnect-driven full refetch ran, for burst coalescing. */
+  private lastReconnectRefetchAt;
+  /**
+   * Minimum gap between reconnect-driven full refetches. Long enough to absorb
+   * a flapping socket (the SDK reconnect ladder starts at 1s), short enough
+   * that a genuine drop minutes later still refetches.
+   */
+  private static readonly RECONNECT_REFETCH_COOLDOWN_MS;
   events: SyncEventSystem;
   private currentUserId;
   private tabRole;
@@ -1712,13 +1741,286 @@ declare class AppReleaseModule<S extends SchemaStructure> {
   private applyRecords;
 }
 //#endregion
+//#region src/services/blobs/blob-store.d.ts
+/**
+ * Byte storage for cached bucket files.
+ *
+ * The default implementation is OPFS. Bucket files never arrive over HTTP in
+ * this client — `BucketHandle.get()` is a SurrealQL RPC on the sync socket — so
+ * neither the browser's HTTP cache nor the Cache API can hold them. We persist
+ * the bytes ourselves, and OPFS is the cheapest place to put them: a read is
+ * `getFile()` → a disk-backed lazy `File` that `URL.createObjectURL` can serve
+ * without ever moving the bytes through the JS heap.
+ *
+ * Layout is real nested directories rather than one hashed filename:
+ *
+ *   sp00ky-blobs/<namespace>/<bucket>/<...path segments>
+ *
+ * That costs a `getDirectoryHandle` per segment on write, and buys the property
+ * the whole orphan story rests on: the full `(bucket, path)` key is recoverable
+ * from a directory walk alone. The `_00_blob` manifest can therefore be wiped
+ * (memory fallback, SQLite pool wipe, IndexedDB corruption recovery) and be
+ * rebuilt from disk instead of taking the cached bytes down with it.
+ */
+/** Identifies one cached file: the bucket it lives in and its path within. */
+interface BlobKey {
+  bucket: string;
+  path: string;
+}
+/** What a directory walk can tell us about a stored file, with no manifest. */
+interface BlobStat {
+  key: BlobKey;
+  size: number;
+  /** File mtime. Seeds `lastAccess` when a manifest row has to be rebuilt. */
+  mtime: number;
+}
+interface BlobStore {
+  /** False for {@link MemoryBlobStore} and for OPFS-less environments: the
+   *  cache still dedupes and serves within a tab, but nothing survives reload. */
+  readonly persistent: boolean;
+  /** Namespace (the local bucketId) all keys are resolved under. */
+  readonly namespace: string;
+  read(key: BlobKey): Promise<Blob | null>;
+  /** Returns the number of bytes written. Throws on quota exhaustion. */
+  write(key: BlobKey, bytes: Blob): Promise<number>;
+  remove(key: BlobKey): Promise<void>;
+  /** Every committed file under the current namespace. Sweeps torn writes. */
+  list(): Promise<BlobStat[]>;
+  /** Drop the whole namespace (sign-out with `clearOnSignOut`, or a reset). */
+  clear(): Promise<void>;
+  /** Point at another namespace. Does not touch the bytes of the old one. */
+  setNamespace(namespace: string): void;
+}
+//#endregion
+//#region src/services/blobs/blob-manifest.d.ts
+interface BlobEntry {
+  /** `${bucket}/${path}` — also the `_00_blob` row id. */
+  id: string;
+  bucket: string;
+  path: string;
+  size: number;
+  contentType: string;
+  createdAt: number;
+  lastAccess: number;
+  hits: number;
+  /** Exempt from pressure eviction. Never expires on its own. */
+  pinned: boolean;
+}
+declare class BlobManifest {
+  private local;
+  private entries;
+  /** Ids whose in-memory state has not been written back yet. */
+  private dirty;
+  private removed;
+  private flushing;
+  constructor(local: LocalStore);
+  /**
+   * Hydrate from the rows matching `keys`. Ids come from the OPFS listing, so
+   * this never needs a full-table scan (and therefore never needs a QueryPlan).
+   * Any read failure yields an empty manifest: reconcile then rebuilds every
+   * row from disk, which is exactly the desired degradation.
+   */
+  load(ids: string[]): Promise<void>;
+  get(key: BlobKey): BlobEntry | undefined;
+  getById(id: string): BlobEntry | undefined;
+  all(): BlobEntry[];
+  totalBytes(): number;
+  pinnedBytes(): number;
+  put(entry: BlobEntry): void;
+  touch(id: string, now: number): void;
+  setPinned(id: string, pinned: boolean): boolean;
+  remove(id: string): void;
+  /** Forget everything without scheduling deletes — for a bucket switch, where
+   *  the rows belong to the store we are leaving and must stay put. */
+  reset(): void;
+  hasPendingWrites(): boolean;
+  /**
+   * Write back pending changes. Serialized: a second concurrent flush awaits
+   * the first rather than racing it into the same rows. Failures are swallowed
+   * on purpose — a lost metadata write costs an LRU timestamp, and the entry is
+   * rebuilt from disk on the next reconcile.
+   */
+  flush(): Promise<void>;
+  private doFlush;
+}
+//#endregion
+//#region src/services/blobs/blob-cache.d.ts
+interface BlobUrlLease {
+  url: string;
+  release(): void;
+}
+interface BlobReadOptions {
+  /** Write through to L1 on a miss. Default true. */
+  persist?: boolean;
+  /** Mark the entry exempt from pressure eviction. */
+  pin?: boolean;
+  /**
+   * Default `'never'`: a bucket path is treated as immutable, which is how the
+   * client writes them (`crypto.randomUUID() + ext`). `'head'` spends a remote
+   * `head()` to compare sizes before trusting L1.
+   */
+  revalidate?: 'never' | 'head';
+  /** Skip L0/L1 entirely and refill from remote. Backs `refetch()`. */
+  reload?: boolean;
+}
+interface BlobCacheStats {
+  entries: number;
+  totalBytes: number;
+  budgetBytes: number;
+  pinnedBytes: number;
+  evictedEntries: number;
+  evictedBytes: number;
+  reconciledEntries: number;
+  hits: number;
+  misses: number;
+  persistent: boolean;
+  /** True when pinned bytes alone exceed the budget: new entries stop being
+   *  written rather than pinned ones being thrown away. */
+  persistPaused: boolean;
+}
+interface BlobCacheOptions {
+  store: BlobStore;
+  manifest: BlobManifest;
+  /** L2 read. Resolves to null when the file does not exist remotely. */
+  fetchRemote(key: BlobKey): Promise<Blob | null>;
+  /** L2 metadata, for `revalidate: 'head'`. */
+  headRemote?(key: BlobKey): Promise<Record<string, unknown> | null>;
+  logger: Logger$1;
+  maxBytes: number;
+  now?: () => number;
+  /** Injected so the URL layer is exercisable off a DOM (node tests). */
+  urls?: {
+    create(blob: Blob): string;
+    revoke(url: string): void;
+  };
+}
+declare class BlobCache {
+  private readonly store;
+  private readonly manifest;
+  private readonly fetchRemote;
+  private readonly headRemote?;
+  private readonly logger;
+  private readonly now;
+  private readonly urlFactory;
+  private maxBytes;
+  private persistPaused;
+  /** Set after a quota failure survives one forced eviction. */
+  private persistDisabled;
+  private readonly urls;
+  /** Ids at zero references, oldest first — the hot-URL window. */
+  private idleUrls;
+  private readonly inflight;
+  private flushTimer;
+  private readonly onPageHide;
+  /**
+   * Resolves once the manifest has been reconciled against disk. Reads await
+   * it, so `start()` does NOT have to be awaited on the boot path — blocking
+   * boot on an OPFS directory walk delayed the WebSocket connect (and with it
+   * the connection supervisor) for no benefit.
+   */
+  private ready;
+  private hits;
+  private misses;
+  private evictedEntries;
+  private evictedBytes;
+  private reconciledEntries;
+  constructor(opts: BlobCacheOptions);
+  /** Coalesce manifest write-back. Metadata only, so losing the tail costs an
+   *  LRU timestamp that reconcile reseeds from the file mtime. */
+  private scheduleFlush;
+  /**
+   * Resolve the bytes for `key`, filling L1 on the way when `persist` is on.
+   * Returns null when the file does not exist remotely and is not cached.
+   */
+  read(key: BlobKey, options?: BlobReadOptions): Promise<Blob | null>;
+  /**
+   * An object URL for `key`, refcounted. Callers MUST `release()`; the URL is
+   * revoked once the last holder lets go and it falls out of the hot window.
+   */
+  acquireUrl(key: BlobKey, options?: BlobReadOptions): Promise<BlobUrlLease | null>;
+  private lease;
+  private releaseUrl;
+  private revokeUrl;
+  /** L1 lookup with the size check that catches torn and cross-tab writes. */
+  private readLocal;
+  /** True when the remote agrees with the cached size, or cannot be reached. */
+  private headMatches;
+  private fetchDeduped;
+  private persist;
+  private writeThrough;
+  /** Forget one path everywhere. Called on `bucket.put()`/`bucket.delete()`. */
+  invalidate(key: BlobKey): Promise<void>;
+  private dropLocal;
+  setPinned(key: BlobKey, pinned: boolean): void;
+  /**
+   * Bring total bytes under budget by dropping the least recently used
+   * entries. Pinned entries and anything with a live object URL are skipped —
+   * evicting bytes that a mounted `<img>` is displaying would blank it.
+   */
+  private enforceBudget;
+  /** Evict LRU-first until at or below `target`. Returns the resulting total. */
+  private evictTo;
+  /**
+   * Rebuild the manifest from what is actually on disk. OPFS wins on existence
+   * in both directions: files with no row get a row (seeded from mtime), rows
+   * are only loaded for files that exist, and torn `.part-` writes are swept by
+   * the walk itself.
+   *
+   * Rows whose file vanished outside our control (a browser origin eviction)
+   * are left in `_00_blob`. They are inert — `load()` only ever asks for ids it
+   * found on disk — and are overwritten if that path is cached again.
+   */
+  reconcile(): Promise<void>;
+  /** Warm the cache for offline use. Skips anything already cached. */
+  prefetch(keys: BlobKey[]): Promise<void>;
+  /** Bind to the boot bucket and hydrate the manifest from disk. Separate from
+   *  {@link setNamespace} because boot must reconcile even when the namespace
+   *  it lands on is the one the store was constructed with. */
+  start(namespace: string): Promise<void>;
+  /** Repoint at another local bucket. The bytes of the old one stay on disk so
+   *  switching back (or signing back in) is still warm. */
+  setNamespace(namespace: string): Promise<void>;
+  setMaxBytes(maxBytes: number): void;
+  /** Delete every cached byte in the current namespace. */
+  clear(): Promise<void>;
+  flush(): Promise<void>;
+  /** Flush metadata and drop every object URL. Must run before the local store
+   *  closes — the flush writes through it. */
+  close(): Promise<void>;
+  stats(): BlobCacheStats;
+}
+//#endregion
 //#region src/sp00ky.d.ts
+/** Coerce whatever the `.get()` RPC hands back into a Blob. */
+declare function bucketContentToBlob(content: unknown): Blob | null;
 declare class BucketHandle {
   private bucketName;
   private remote;
-  constructor(bucketName: string, remote: RemoteDatabaseService);
+  /** Absent on the raw handle the cache itself reads through. */
+  private blobs?;
+  constructor(bucketName: string, remote: RemoteDatabaseService, /** Absent on the raw handle the cache itself reads through. */
+  blobs?: (BlobCache | null) | undefined);
   put(path: string, content: string | Uint8Array | Blob): Promise<void>;
   get(path: string): Promise<unknown>;
+  /**
+   * Read through the local blob cache: OPFS first, the bucket second. Unlike
+   * {@link get} this survives a reload and works offline. Returns null when the
+   * file exists in neither place.
+   */
+  read(path: string, options?: BlobReadOptions): Promise<Blob | null>;
+  /**
+   * A refcounted object URL for `path`, suitable for `<img src>`. The caller
+   * MUST call `release()` when the URL goes off screen. Returns null when the
+   * file does not exist, or when object URLs are unavailable (non-browser).
+   */
+  url(path: string, options?: BlobReadOptions): Promise<BlobUrlLease | null>;
+  /** Exempt `path` from pressure eviction. Pinned bytes never expire. */
+  pin(path: string): void;
+  unpin(path: string): void;
+  /** Drop `path` from the local cache without touching the remote file. */
+  evict(path: string): Promise<void>;
+  /** Warm the cache for offline use. Already-cached paths are skipped. */
+  prefetch(paths: string[]): Promise<void>;
   delete(path: string): Promise<void>;
   exists(path: string): Promise<boolean>;
   head(path: string): Promise<Record<string, unknown>>;
@@ -1730,6 +2032,7 @@ declare class Sp00kyClient<S extends SchemaStructure> {
   private config;
   private local;
   private remote;
+  private blobs;
   private connectionSupervisor;
   private persistenceClient;
   private migrator;
@@ -1909,6 +2212,11 @@ declare class Sp00kyClient<S extends SchemaStructure> {
   reportFrontendTiming(queryHash: string, ms: number): void;
   run<B extends BackendNames<S>, R extends BackendRoutes<S, B>>(backend: B, path: R, payload: RoutePayload<S, B, R>, options?: RunOptions): Promise<void>;
   bucket<B extends BucketNames<S>>(name: B): BucketHandle;
+  /** Cache-free handle. The blob cache reads the remote through this, so a
+   *  cache miss can't loop back into the cache. */
+  private rawBucket;
+  /** Blob cache counters for DevTools. */
+  getBlobCacheStats(): BlobCacheStats;
   create(id: string, data: Record<string, unknown>): Promise<Record<string, unknown>>;
   update(table: string, id: string, data: Record<string, unknown>, options?: UpdateOptions): Promise<{
     [x: string]: /*elided*/any;
@@ -1940,4 +2248,4 @@ declare function textToHtml(text: string): string;
  */
 
 //#endregion
-export { AppReleaseHandle, AppReleaseModule, type AppReleaseOptions, type AppReleaseSnapshot, AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, BucketHandle, CURSOR_COLORS, ConnectionState, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, FeatureFlagHandle, FeatureFlagModule, type FeatureFlagOptions, type FeatureFlagSnapshot, Level, MATERIALIZATION_SAMPLE_WINDOW, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PhaseStat, PinoTransmit, PreloadOptions, PreloadRefresh, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryStatus, QueryStatusCallback, QueryTimeToLive, QueryTimings, QueryUpdateCallback, ReconnectConfig, RecordVersionArray, RecordVersionDiff, RegistrationTimings, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StorageHealth, StorageHealthStatus, StoreType, SyncHealth, SyncHealthConfig, SyncHealthStatus, TimingPhase, UpdateOptions, createAuthEventSystem, cursorColorFromName, fileToUint8Array, semverGt, textToHtml };
+export { AppReleaseHandle, AppReleaseModule, type AppReleaseOptions, type AppReleaseSnapshot, AuthEventSystem, AuthEventTypeMap, AuthEventTypes, AuthService, type BlobCacheStats, type BlobEntry, type BlobKey, type BlobReadOptions, type BlobUrlLease, BucketHandle, CURSOR_COLORS, ConnectionState, CrdtField, CrdtManager, DebounceOptions, EventSubscriptionOptions, FeatureFlagHandle, FeatureFlagModule, type FeatureFlagOptions, type FeatureFlagSnapshot, Level, MATERIALIZATION_SAMPLE_WINDOW, MutationCallback, MutationEvent, MutationEventType, PersistenceClient, PhaseStat, PinoTransmit, PreloadOptions, PreloadRefresh, QueryConfig, QueryConfigRecord, QueryHash, QueryState, QueryStatus, QueryStatusCallback, QueryTimeToLive, QueryTimings, QueryUpdateCallback, ReconnectConfig, RecordVersionArray, RecordVersionDiff, RegistrationTimings, RunOptions, Sp00kyClient, Sp00kyConfig, Sp00kyQueryResult, Sp00kyQueryResultPromise, StorageHealth, StorageHealthStatus, StoreType, SyncHealth, SyncHealthConfig, SyncHealthStatus, TimingPhase, UpdateOptions, bucketContentToBlob, createAuthEventSystem, cursorColorFromName, fileToUint8Array, semverGt, textToHtml };