@spooky-sync/core 0.0.1-canary.152 → 0.0.1-canary.154

package/src/modules/devtools/index.ts

@@ -22,6 +22,7 @@ import {
   parseBackendInfo,
   UNAVAILABLE,
 } from './versions';
+import { walkOpfs, type StorageInfo } from './storage-info';
 
 // Real bundled frontend versions, injected at build time by tsdown's
 // version-define plugin (see tsdown.config.ts). The `typeof` guard keeps these
@@ -96,6 +97,11 @@ export class DevToolsService implements StreamUpdateReceiver {
       }
     });
 
+    // Push state when the local store reports its durability (the open happens
+    // during connect, typically before a panel attaches, so this mostly matters
+    // for a later bucket switch that loses OPFS).
+    this.databaseService.subscribeToStorageHealth?.(() => this.notifyDevTools());
+
     // Fire-and-forget backend version discovery; re-push state when it lands.
     void this.refreshBackendVersions();
 
@@ -318,10 +324,79 @@ export class DevToolsService implements StreamUpdateReceiver {
           ? this.localTables
           : this.schema.tables.map((t) => t.name),
         tableData: {},
+        // Durability of the local store. `fallback: true` means persistence was
+        // requested but the dataset is actually sitting in RAM.
+        storage: this.databaseService.storageHealth ?? { status: 'unknown', fallback: false },
       },
     });
   }
 
+  /**
+   * Full storage diagnostics for the DevTools Storage tab. Every section is
+   * gathered independently and failures land in that section's `error` field,
+   * so one broken source (a mid-switch worker, a browser without OPFS) never
+   * blanks the whole panel.
+   */
+  public async getStorageInfo(opts?: { tableCounts?: boolean }): Promise<StorageInfo> {
+    const nav = typeof navigator !== 'undefined' ? navigator : undefined;
+
+    const info: StorageInfo = {
+      at: Date.now(),
+      engine: {
+        kind: this.databaseService.engineKind ?? 'custom',
+        store: this.databaseService.getConfig()?.store ?? 'memory',
+        bucketId: this.databaseService.currentBucketId,
+      },
+      health: this.databaseService.storageHealth ?? { status: 'unknown', fallback: false },
+      browser: {},
+      opfs: { supported: false, entries: [], totalBytes: 0, truncated: false },
+    };
+
+    try {
+      if (nav?.storage?.estimate) {
+        const est = await nav.storage.estimate();
+        info.browser.usage = est.usage;
+        info.browser.quota = est.quota;
+        // Chrome-only per-storage-system breakdown; absent elsewhere.
+        const details = (est as any).usageDetails;
+        if (details && typeof details === 'object') info.browser.usageDetails = details;
+      }
+      if (nav?.storage?.persisted) {
+        info.browser.persisted = await nav.storage.persisted();
+      }
+    } catch (e) {
+      info.browser.error = e instanceof Error ? e.message : String(e);
+    }
+
+    info.opfs = await walkOpfs();
+
+    const stats = (globalThis as any).__sqliteStats;
+    if (stats && typeof stats === 'object') {
+      info.sqliteStats = { ...stats, byType: { ...(stats.byType ?? {}) } };
+    }
+
+    try {
+      info.engineDiagnostics = await this.databaseService.getStorageDiagnostics?.(opts);
+    } catch (e) {
+      this.logger.warn(
+        { err: e, Category: 'sp00ky-client::DevToolsService::getStorageInfo' },
+        'Engine storage diagnostics failed'
+      );
+    }
+
+    return this.serializeForDevTools(info);
+  }
+
+  /** Ask the browser to exempt this origin's storage from eviction. */
+  public async requestPersistentStorage(): Promise<{ granted: boolean }> {
+    try {
+      const granted = (await navigator.storage?.persist?.()) ?? false;
+      return { granted };
+    } catch {
+      return { granted: false };
+    }
+  }
+
   private notifyDevTools() {
     // No consumer attached → no getState() serialization, no postMessage broadcast.
     if (!this.enabled) return;
@@ -375,6 +450,10 @@ export class DevToolsService implements StreamUpdateReceiver {
       const result: Record<string, any> = {};
       for (const key in data) {
         if (Object.prototype.hasOwnProperty.call(data, key)) {
+          // Skip absent optional fields: recursing them would emit the STRING
+          // 'undefined' (the top-level mapping below), which panels then have
+          // to filter back out (see 3d84fe8a).
+          if (data[key] === undefined) continue;
           result[key] = this.serializeForDevTools(data[key], seen);
         }
       }
@@ -394,6 +473,8 @@ export class DevToolsService implements StreamUpdateReceiver {
           this.notifyDevTools();
         },
         refreshVersions: () => this.refreshBackendVersions(),
+        getStorageInfo: (opts?: { tableCounts?: boolean }) => this.getStorageInfo(opts),
+        requestPersistentStorage: () => this.requestPersistentStorage(),
         getTableData: async (tableName: string) => {
           try {
             // Returns the first statement result as T.

package/src/modules/devtools/storage-info.test.ts

@@ -0,0 +1,79 @@
+import { describe, it, expect, afterEach, vi } from 'vitest';
+import { walkOpfs } from './storage-info';
+
+/** Minimal in-memory OPFS: directories are nested objects, files are numbers
+ *  (their size) or 'locked' (getFile() throws, like a live SAHPool handle). */
+type FakeTree = { [name: string]: FakeTree | number | 'locked' };
+
+function makeDirHandle(tree: FakeTree): any {
+  return {
+    kind: 'directory',
+    entries: async function* () {
+      for (const [name, node] of Object.entries(tree)) {
+        if (typeof node === 'object') {
+          yield [name, makeDirHandle(node)];
+        } else {
+          yield [
+            name,
+            {
+              kind: 'file',
+              getFile: async () => {
+                if (node === 'locked') throw new DOMException('locked', 'NoModificationAllowedError');
+                return { size: node };
+              },
+            },
+          ];
+        }
+      }
+    },
+  };
+}
+
+function stubOpfs(tree: FakeTree | null) {
+  vi.stubGlobal('navigator', tree === null ? {} : {
+    storage: { getDirectory: async () => makeDirHandle(tree) },
+  });
+}
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe('walkOpfs', () => {
+  it('reports unsupported without OPFS APIs', async () => {
+    stubOpfs(null);
+    expect(await walkOpfs()).toEqual({
+      supported: false,
+      entries: [],
+      totalBytes: 0,
+      truncated: false,
+    });
+  });
+
+  it('walks recursively, sums readable sizes, and omits size for locked files', async () => {
+    stubOpfs({
+      '.sp00ky-anon': { '0000000001': 4096, '0000000002': 'locked' },
+      'other.txt': 10,
+    });
+    const res = await walkOpfs();
+    expect(res.supported).toBe(true);
+    expect(res.truncated).toBe(false);
+    // Locked file present but without a size; total counts only readable bytes.
+    expect(res.totalBytes).toBe(4106);
+    expect(res.entries).toEqual([
+      { path: '.sp00ky-anon', kind: 'directory' },
+      { path: '.sp00ky-anon/0000000001', kind: 'file', size: 4096 },
+      { path: '.sp00ky-anon/0000000002', kind: 'file' },
+      { path: 'other.txt', kind: 'file', size: 10 },
+    ]);
+  });
+
+  it('caps the listing and flags truncation', async () => {
+    const big: FakeTree = {};
+    for (let i = 0; i < 10; i++) big[`f${i}`] = 1;
+    stubOpfs(big);
+    const res = await walkOpfs(5);
+    expect(res.truncated).toBe(true);
+    expect(res.entries.length).toBe(5);
+  });
+});

package/src/modules/devtools/storage-info.ts

@@ -0,0 +1,116 @@
+/**
+ * Storage diagnostics for the DevTools Storage tab: what engine backs the
+ * local cache, whether it actually persists, how much of the device's quota
+ * the origin uses, and what is physically sitting in OPFS. Assembled by
+ * `DevToolsService.getStorageInfo()`; everything here is JSON-safe.
+ */
+
+export interface OpfsEntry {
+  /** Path relative to the OPFS root, e.g. `.sp00ky-anon/0000000000000001`. */
+  path: string;
+  kind: 'file' | 'directory';
+  /** Absent when the file's size can't be read (e.g. an exclusive sync access
+   *  handle is held on it — exactly the case during SAHPool contention). */
+  size?: number;
+}
+
+/** Engine-side numbers only the engine can produce (worker round-trips). */
+export interface EngineStorageDiagnostics {
+  engine: 'sqlite';
+  bucketId: string;
+  useOpfs: boolean;
+  workerSelectConfigured: boolean;
+  /** `false` while configured `true` means the runtime downgraded to the
+   *  legacy multi-hop select (stale cached worker bundle). */
+  workerSelectEffective: boolean;
+  /** page_count * page_size. */
+  dbSizeBytes?: number;
+  /** freelist_count * page_size — reclaimable via VACUUM. */
+  freelistBytes?: number;
+  tableCounts?: { table: string; rows: number }[];
+  error?: string;
+}
+
+export interface StorageInfo {
+  at: number;
+  engine: { kind: 'surrealdb' | 'sqlite' | 'custom'; store: string; bucketId: string };
+  health: { status: 'unknown' | 'persistent' | 'memory'; fallback: boolean; error?: string };
+  browser: {
+    /** `navigator.storage.persisted()` — whether the origin's storage is
+     *  exempt from eviction (unrelated to the OPFS pool lock). */
+    persisted?: boolean;
+    usage?: number;
+    quota?: number;
+    /** Chrome-only per-system breakdown from `estimate()`. */
+    usageDetails?: Record<string, number>;
+    error?: string;
+  };
+  opfs: {
+    supported: boolean;
+    entries: OpfsEntry[];
+    totalBytes: number;
+    truncated: boolean;
+    error?: string;
+  };
+  /** Snapshot of `globalThis.__sqliteStats` (SQLite engine only). */
+  sqliteStats?: Record<string, unknown>;
+  engineDiagnostics?: EngineStorageDiagnostics;
+}
+
+/**
+ * Recursively list the origin's OPFS. Sizes come from `handle.getFile()`,
+ * which throws for a file another context holds an exclusive sync access
+ * handle on — SAHPool does exactly that for its whole pool, so a locked file
+ * (size omitted) is a live "who has the pool" signal, not a failure.
+ */
+export async function walkOpfs(maxEntries = 2000, maxDepth = 8): Promise<StorageInfo['opfs']> {
+  const nav = typeof navigator !== 'undefined' ? navigator : undefined;
+  if (!nav?.storage?.getDirectory) {
+    return { supported: false, entries: [], totalBytes: 0, truncated: false };
+  }
+  const entries: OpfsEntry[] = [];
+  let totalBytes = 0;
+  let truncated = false;
+  try {
+    const root = await nav.storage.getDirectory();
+    const walk = async (dir: FileSystemDirectoryHandle, prefix: string, depth: number) => {
+      if (depth > maxDepth) return;
+      // entries() is standard; older lib.dom typings may lack it.
+      for await (const [name, handle] of (dir as any).entries() as AsyncIterable<
+        [string, FileSystemHandle]
+      >) {
+        if (entries.length >= maxEntries) {
+          truncated = true;
+          return;
+        }
+        const path = prefix ? `${prefix}/${name}` : name;
+        if (handle.kind === 'directory') {
+          entries.push({ path, kind: 'directory' });
+          await walk(handle as FileSystemDirectoryHandle, path, depth + 1);
+        } else {
+          let size: number | undefined;
+          try {
+            size = (await (handle as FileSystemFileHandle).getFile()).size;
+            totalBytes += size;
+          } catch {
+            // Locked by an exclusive access handle (e.g. a live SAHPool).
+          }
+          const entry: OpfsEntry = { path, kind: 'file' };
+          if (size !== undefined) entry.size = size;
+          entries.push(entry);
+        }
+      }
+    };
+    await walk(root, '', 0);
+    entries.sort((a, b) => a.path.localeCompare(b.path));
+    return { supported: true, entries, totalBytes, truncated };
+  } catch (e) {
+    return {
+      supported: true,
+      entries,
+      totalBytes,
+      truncated,
+      error: e instanceof Error ? e.message : String(e),
+    };
+  }
+}

package/src/services/database/cache-engine.ts

@@ -1,7 +1,8 @@
 import type { QueryPlan, RelationPlan, WhereNode } from '@spooky-sync/query-builder';
 import type { SealedQuery } from '../../utils/surql';
 import type { DatabaseEventSystem } from './events/index';
-import type { Sp00kyConfig } from '../../types';
+import type { Sp00kyConfig, StorageHealth } from '../../types';
+import type { EngineStorageDiagnostics } from '../../modules/devtools/storage-info';
 
 /**
  * A materialized row. Keys are field names; values are already decoded to the
@@ -121,6 +122,22 @@ export interface LocalStore extends LocalCacheEngine {
   getClient(): unknown;
   getConfig(): Sp00kyConfig<any>['database'];
   readonly currentBucketId: string;
+  /** Which built-in backend this is. OPTIONAL: absent (custom engines) is
+   *  reported as `'custom'` by DevTools. More robust than `instanceof` for
+   *  engines constructed outside this package. */
+  readonly engineKind?: 'surrealdb' | 'sqlite';
+  /** Engine-specific storage numbers for DevTools (DB file size, per-table
+   *  row counts). OPTIONAL: only engines with something to report implement it. */
+  getStorageDiagnostics?(opts?: { tableCounts?: boolean }): Promise<EngineStorageDiagnostics>;
+  /**
+   * Durability of this engine's local store. OPTIONAL: engines that don't
+   * report it (SurrealDB, custom engines) are treated as `'unknown'` by the
+   * client facade, so adding this needs no change on their side.
+   */
+  readonly storageHealth?: StorageHealth;
+  /** Fires immediately with the current snapshot, then on every change.
+   *  Returns an unsubscribe function. */
+  subscribeToStorageHealth?(cb: (health: StorageHealth) => void): () => void;
 }
 
 /** Selected local cache backend. Mirrors the `persistenceClient` config pattern. */

package/src/services/database/sqlite-cache-engine.test.ts

@@ -135,6 +135,165 @@ describe('SqliteCacheEngine system-table seeding', () => {
   });
 });
 
+// The worker's `persisted`/`opfsError` reply used to die in a `logger.info`
+// line, so a host app running pino at `fatal` (whitepawn does) could not tell a
+// disk-backed store from a full-RAM one. It now lands on the engine as
+// observable state the app can render.
+describe('SqliteCacheEngine storage health', () => {
+  /** Engine wired to a worker whose `open` replies with `openReply`. */
+  function makeEngine(openReply: Record<string, unknown>, opts?: { useOpfs?: boolean }) {
+    const logs: { level: string; msg: string; meta: any }[] = [];
+    const logger: any = {};
+    for (const level of ['debug', 'info', 'warn', 'error', 'trace']) {
+      logger[level] = (meta: any, msg: string) => logs.push({ level, msg, meta });
+    }
+    logger.child = () => logger;
+
+    const engine = new SqliteCacheEngine(
+      { namespace: 'n', database: 'd' } as any,
+      logger,
+      opts ?? {}
+    );
+    (engine as any).spawnWorker = () => {
+      const w: any = { onmessage: null, onerror: null, onmessageerror: null, terminate() {} };
+      w.postMessage = (msg: any) => {
+        Promise.resolve().then(() => {
+          const rest = msg.type === 'open' ? openReply : {};
+          const { id, ok, error, ...payload } = { id: msg.id, ok: true, error: undefined, ...rest };
+          const p = (engine as any).pending.get(id);
+          if (!p) return;
+          (engine as any).pending.delete(id);
+          if (ok) p.resolve(payload);
+          else p.reject(new Error(error));
+        });
+      };
+      return w as unknown as Worker;
+    };
+    return { engine, logs };
+  }
+
+  it('publishes a persistent store and logs no error', async () => {
+    const { engine, logs } = makeEngine({ persisted: true });
+    await engine.connect('user:abc');
+
+    expect(engine.storageHealth).toEqual({
+      status: 'persistent',
+      fallback: false,
+      error: undefined,
+    });
+    expect(logs.some((l) => l.level === 'error')).toBe(false);
+  });
+
+  it('publishes the fallback, its reason, and an error log when OPFS is lost', async () => {
+    const { engine, logs } = makeEngine({ persisted: false, opfsError: 'NoModificationAllowedError: locked' });
+    await engine.connect('user:abc');
+
+    expect(engine.storageHealth).toEqual({
+      status: 'memory',
+      fallback: true,
+      error: 'NoModificationAllowedError: locked',
+    });
+    const err = logs.find((l) => l.level === 'error');
+    expect(err?.msg).toContain('IN MEMORY');
+    expect(err?.meta.opfsError).toBe('NoModificationAllowedError: locked');
+    // Inspectable from the console without any logging configured.
+    expect((globalThis as any).__sqliteStats.persisted).toBe(false);
+  });
+
+  // A subscriber almost always attaches AFTER connect() (components mount
+  // later), so an immediate fire is the only way it learns about a fallback.
+  it('fires a late subscriber with the current snapshot', async () => {
+    const { engine } = makeEngine({ persisted: false, opfsError: 'boom' });
+    await engine.connect('user:abc');
+
+    const seen: any[] = [];
+    const unsub = engine.subscribeToStorageHealth((h) => seen.push(h));
+    expect(seen).toEqual([{ status: 'memory', fallback: true, error: 'boom' }]);
+    unsub();
+  });
+
+  // `store: 'memory'` asked for RAM, so it is not a fallback and must not warn.
+  it('does not flag a configured in-memory store as a fallback', async () => {
+    const { engine, logs } = makeEngine({ persisted: false }, { useOpfs: false });
+    await engine.connect('user:abc');
+
+    expect(engine.storageHealth).toEqual({ status: 'memory', fallback: false, error: undefined });
+    expect(logs.some((l) => l.level === 'error')).toBe(false);
+  });
+});
+
+// Storage numbers for the DevTools Storage tab: DB size via the pragmas, row
+// counts on demand, and the configured-vs-effective workerSelect split. Errors
+// must land in `error` (the worker may be mid bucket-switch), never throw.
+describe('SqliteCacheEngine.getStorageDiagnostics', () => {
+  function makeEngine(execRows: (sql: string) => unknown[]) {
+    const noop = () => {};
+    const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+    logger.child = () => logger;
+    const engine = new SqliteCacheEngine({ namespace: 'n', database: 'd' } as any, logger);
+    (engine as any).spawnWorker = () => {
+      const w: any = { onmessage: null, onerror: null, onmessageerror: null, terminate() {} };
+      w.postMessage = (msg: any) => {
+        Promise.resolve().then(() => {
+          const rest =
+            msg.type === 'open'
+              ? { persisted: true }
+              : msg.type === 'exec'
+                ? { rows: execRows(msg.payload.sql) }
+                : {};
+          const p = (engine as any).pending.get(msg.id);
+          if (!p) return;
+          (engine as any).pending.delete(msg.id);
+          p.resolve(rest);
+        });
+      };
+      return w as unknown as Worker;
+    };
+    return engine;
+  }
+
+  it('reports size, freelist, and per-table counts', async () => {
+    const engine = makeEngine((sql) => {
+      if (sql.includes('pragma_page_count')) return [{ bytes: 40960, freelist: 4096 }];
+      if (sql.includes('sqlite_master')) return [{ name: '_00_query' }, { name: 'game' }];
+      if (sql.includes('COUNT(*)'))
+        return [
+          { t: '_00_query', n: 3 },
+          { t: 'game', n: 12 },
+        ];
+      return [];
+    });
+    await engine.connect('user:abc');
+
+    const diag = await engine.getStorageDiagnostics({ tableCounts: true });
+    expect(diag.engine).toBe('sqlite');
+    expect(diag.bucketId).toBe('user:abc');
+    expect(diag.dbSizeBytes).toBe(40960);
+    expect(diag.freelistBytes).toBe(4096);
+    expect(diag.tableCounts).toEqual([
+      { table: '_00_query', rows: 3 },
+      { table: 'game', rows: 12 },
+    ]);
+    // Default config: workerSelect on, never downgraded.
+    expect(diag.workerSelectConfigured).toBe(true);
+    expect(diag.workerSelectEffective).toBe(true);
+  });
+
+  it('skips table counts unless asked and never throws on a dead worker', async () => {
+    const engine = makeEngine(() => [{ bytes: 8192, freelist: 0 }]);
+    await engine.connect('anon');
+
+    const diag = await engine.getStorageDiagnostics();
+    expect(diag.tableCounts).toBeUndefined();
+
+    // No worker at all → the failure lands in `error`, not as a throw.
+    const cold = makeEngine(() => []);
+    const coldDiag = await cold.getStorageDiagnostics();
+    expect(coldDiag.error).toContain('not connected');
+    expect(coldDiag.bucketId).toBe('anon');
+  });
+});
+
 // `pureWriteOpResult` is the single source of truth for what a pure-write op
 // contributes to a query's per-statement results. The batched fast path in
 // `query()` and the per-op `execOp` path BOTH route through it, so a caller that