@spooky-sync/core 0.0.1-canary.20 → 0.0.1-canary.201

package/src/modules/devtools/notify-throttle.test.ts

@@ -0,0 +1,149 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DevToolsService } from './index';
+
+/**
+ * A state push serializes EVERY active query's full record set and postMessage
+ * clones it again, so its cost scales with the whole client dataset. It is
+ * requested per event — including one per local DB query — so without
+ * coalescing a page load's few hundred local queries turn a few MB of rows into
+ * GBs of short-lived large-object garbage and OOM the renderer. These tests pin
+ * the coalescing, not the payload.
+ */
+
+function harness(recordCount = 500) {
+  const posted: any[] = [];
+  const listeners: ((e: any) => void)[] = [];
+  const fakeWindow: any = {
+    postMessage: (msg: any) => posted.push(msg),
+    addEventListener: (_type: string, cb: (e: any) => void) => listeners.push(cb),
+    dispatchEvent: () => true,
+  };
+  fakeWindow.self = fakeWindow;
+  vi.stubGlobal('window', fakeWindow);
+  vi.stubGlobal('CustomEvent', class {
+    type: string;
+    constructor(type: string) {
+      this.type = type;
+    }
+  });
+
+  const noop = () => {};
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+
+  const local: any = {
+    query: async () => [],
+    getConfig: () => ({ store: 'memory' }),
+    currentBucketId: 'anon',
+    storageHealth: { status: 'memory', fallback: false },
+  };
+  const remote: any = { query: async () => [] };
+  const auth: any = {
+    isAuthenticated: false,
+    currentUser: undefined,
+    eventSystem: { subscribe: noop },
+  };
+  // One query holding a lot of rows: the thing whose repeated serialization is
+  // what actually blows the heap.
+  const records = Array.from({ length: recordCount }, (_, i) => ({ id: `game:${i}`, pgn: 'x' }));
+  const dataManager: any = {
+    getActiveQueries: () => [
+      {
+        config: { id: new RecordId('_00_query', 'q1'), params: {} },
+        status: 'idle',
+        records,
+        updateCount: 1,
+      },
+    ],
+    phaseTimings: () => ({}),
+  };
+
+  const service = new DevToolsService(local, remote, logger, { tables: [] } as any, auth, dataManager);
+  // Announce a consumer, exactly like the extension's page-script does.
+  for (const cb of listeners) {
+    cb({ source: fakeWindow, data: { type: 'SP00KY_DEVTOOLS_CONNECT' } });
+  }
+  const statePushes = () => posted.filter((m) => m.type === 'SP00KY_STATE_CHANGED');
+  return { service, statePushes, posted };
+}
+
+afterEach(() => {
+  vi.useRealTimers();
+  vi.unstubAllGlobals();
+});
+
+describe('DevToolsService state-push coalescing', () => {
+  it('pushes once immediately on connect', () => {
+    vi.useFakeTimers();
+    const { statePushes } = harness();
+    expect(statePushes().length).toBe(1);
+  });
+
+  it('collapses a burst of per-query events into ONE trailing push', () => {
+    vi.useFakeTimers();
+    const { service, statePushes } = harness();
+    const before = statePushes().length;
+
+    // What a page load looks like: hundreds of LOCAL_QUERY events, each of which
+    // used to serialize the entire query state.
+    for (let i = 0; i < 400; i++) {
+      (service as any).logEvent('LOCAL_QUERY', { query: 'SELECT * FROM game', vars: {} });
+    }
+    // Nothing extra yet — the burst is queued, not serialized 400 times.
+    expect(statePushes().length).toBe(before);
+
+    vi.advanceTimersByTime(300);
+    expect(statePushes().length).toBe(before + 1);
+  });
+
+  it('still pushes again once the window has passed', () => {
+    vi.useFakeTimers();
+    const { service, statePushes } = harness();
+    const before = statePushes().length;
+
+    (service as any).logEvent('A', {});
+    vi.advanceTimersByTime(300);
+    expect(statePushes().length).toBe(before + 1);
+
+    // An event arriving right after that flush is still inside the window, so it
+    // queues rather than pushing again...
+    (service as any).logEvent('B', {});
+    expect(statePushes().length).toBe(before + 1);
+    vi.advanceTimersByTime(300);
+    expect(statePushes().length).toBe(before + 2);
+
+    // ...but once the tab has been idle past the window, the next event pushes
+    // straight away, so the panel never waits on a quiet app.
+    vi.advanceTimersByTime(1000);
+    (service as any).logEvent('C', {});
+    expect(statePushes().length).toBe(before + 3);
+  });
+
+  it('drops a queued push when the consumer disconnects mid-window', () => {
+    vi.useFakeTimers();
+    const { service, statePushes, posted } = harness();
+    const before = statePushes().length;
+
+    (service as any).logEvent('A', {});
+    (service as any).enabled = false; // panel closed while the push was queued
+    vi.advanceTimersByTime(300);
+    expect(statePushes().length).toBe(before);
+    expect(posted.some((m) => m.type === 'SP00KY_STATE_CHANGED' && m.state === undefined)).toBe(false);
+  });
+
+  it('serializes the LATEST state at flush time, not at request time', () => {
+    vi.useFakeTimers();
+    const { service, statePushes } = harness();
+
+    (service as any).logEvent('FIRST', {});
+    (service as any).logEvent('SECOND', {});
+    vi.advanceTimersByTime(300);
+
+    const last = statePushes().at(-1);
+    const types = last.state.eventsHistory.map((e: any) => e.eventType);
+    // Both events of the coalesced window are present in the single push.
+    expect(types).toContain('FIRST');
+    expect(types).toContain('SECOND');
+  });
+});

package/src/modules/devtools/storage-info.test.ts

@@ -0,0 +1,79 @@
+import { describe, it, expect, afterEach, vi } from 'vitest';
+import { walkOpfs } from './storage-info';
+
+/** Minimal in-memory OPFS: directories are nested objects, files are numbers
+ *  (their size) or 'locked' (getFile() throws, like a live SAHPool handle). */
+type FakeTree = { [name: string]: FakeTree | number | 'locked' };
+
+function makeDirHandle(tree: FakeTree): any {
+  return {
+    kind: 'directory',
+    entries: async function* () {
+      for (const [name, node] of Object.entries(tree)) {
+        if (typeof node === 'object') {
+          yield [name, makeDirHandle(node)];
+        } else {
+          yield [
+            name,
+            {
+              kind: 'file',
+              getFile: async () => {
+                if (node === 'locked') throw new DOMException('locked', 'NoModificationAllowedError');
+                return { size: node };
+              },
+            },
+          ];
+        }
+      }
+    },
+  };
+}
+
+function stubOpfs(tree: FakeTree | null) {
+  vi.stubGlobal('navigator', tree === null ? {} : {
+    storage: { getDirectory: async () => makeDirHandle(tree) },
+  });
+}
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe('walkOpfs', () => {
+  it('reports unsupported without OPFS APIs', async () => {
+    stubOpfs(null);
+    expect(await walkOpfs()).toEqual({
+      supported: false,
+      entries: [],
+      totalBytes: 0,
+      truncated: false,
+    });
+  });
+
+  it('walks recursively, sums readable sizes, and omits size for locked files', async () => {
+    stubOpfs({
+      '.sp00ky-anon': { '0000000001': 4096, '0000000002': 'locked' },
+      'other.txt': 10,
+    });
+    const res = await walkOpfs();
+    expect(res.supported).toBe(true);
+    expect(res.truncated).toBe(false);
+    // Locked file present but without a size; total counts only readable bytes.
+    expect(res.totalBytes).toBe(4106);
+    expect(res.entries).toEqual([
+      { path: '.sp00ky-anon', kind: 'directory' },
+      { path: '.sp00ky-anon/0000000001', kind: 'file', size: 4096 },
+      { path: '.sp00ky-anon/0000000002', kind: 'file' },
+      { path: 'other.txt', kind: 'file', size: 10 },
+    ]);
+  });
+
+  it('caps the listing and flags truncation', async () => {
+    const big: FakeTree = {};
+    for (let i = 0; i < 10; i++) big[`f${i}`] = 1;
+    stubOpfs(big);
+    const res = await walkOpfs(5);
+    expect(res.truncated).toBe(true);
+    expect(res.entries.length).toBe(5);
+  });
+});

package/src/modules/devtools/storage-info.ts

@@ -0,0 +1,168 @@
+/**
+ * Storage diagnostics for the DevTools Storage tab: what engine backs the
+ * local cache, whether it actually persists, how much of the device's quota
+ * the origin uses, and what is physically sitting in OPFS. Assembled by
+ * `DevToolsService.getStorageInfo()`; everything here is JSON-safe.
+ */
+
+export interface OpfsEntry {
+  /** Path relative to the OPFS root, e.g. `.sp00ky-anon/0000000000000001`. */
+  path: string;
+  kind: 'file' | 'directory';
+  /** Absent when the file's size can't be read (e.g. an exclusive sync access
+   *  handle is held on it — exactly the case during SAHPool contention). */
+  size?: number;
+}
+
+/** Engine-side numbers only the engine can produce (worker round-trips). */
+export interface EngineStorageDiagnostics {
+  engine: 'sqlite';
+  bucketId: string;
+  useOpfs: boolean;
+  workerSelectConfigured: boolean;
+  /** `false` while configured `true` means the runtime downgraded to the
+   *  legacy multi-hop select (stale cached worker bundle). */
+  workerSelectEffective: boolean;
+  /** page_count * page_size. */
+  dbSizeBytes?: number;
+  /** freelist_count * page_size — reclaimable via VACUUM. */
+  freelistBytes?: number;
+  tableCounts?: { table: string; rows: number }[];
+  error?: string;
+}
+
+/**
+ * Shared-tabs coordination state. Reported ONLY when `sharedTabs: true` was
+ * configured (apps that never asked for it get `null`, so the panel shows
+ * nothing). `active: false` with a `reason` is itself the useful signal: the
+ * app asked to share one store and this tab is not, so it owns or contends for
+ * the OPFS pool alone.
+ */
+export interface SharedTabsInfo {
+  active: boolean;
+  /** Why inactive: a capability gate reason, or 'fell-back' when the broker
+   *  was reachable but no role landed (election timeout, rejected tab). */
+  reason?: string;
+  role?: 'solo' | 'leader' | 'follower';
+  /** This tab's id (also the suffix of every mutation id it mints). */
+  tabId?: string;
+  /** Monotonic id of the current leadership term; embedded in the worker lock. */
+  leadershipId?: number;
+  /** Who owns the store: this tab when leader, else the leader's tab id. */
+  leaderTabId?: string | null;
+  /** Leader only: attached follower tabs. */
+  followers?: number;
+  /** Leader only: ingest batches relayed to followers so far. */
+  relayedBatches?: number;
+}
+
+export interface StorageInfo {
+  at: number;
+  engine: { kind: 'surrealdb' | 'sqlite' | 'custom'; store: string; bucketId: string };
+  health: { status: 'unknown' | 'persistent' | 'memory'; fallback: boolean; error?: string };
+  /** Shared-tabs state, or null when the feature was never requested. */
+  tabs?: SharedTabsInfo | null;
+  browser: {
+    /** `navigator.storage.persisted()` — whether the origin's storage is
+     *  exempt from eviction (unrelated to the OPFS pool lock). */
+    persisted?: boolean;
+    usage?: number;
+    quota?: number;
+    /** Chrome-only per-system breakdown from `estimate()`. */
+    usageDetails?: Record<string, number>;
+    error?: string;
+  };
+  opfs: {
+    supported: boolean;
+    entries: OpfsEntry[];
+    totalBytes: number;
+    truncated: boolean;
+    error?: string;
+  };
+  /**
+   * Bucket-file cache counters. `evicted*` is cumulative for this tab's
+   * session, so a number that climbs while `totalBytes` sits at the budget is
+   * the signal that `blobCache.maxBytes` is too small for the working set.
+   */
+  blobs?: BlobCacheInfo;
+  /** Snapshot of `globalThis.__sqliteStats` (SQLite engine only). */
+  sqliteStats?: Record<string, unknown>;
+  engineDiagnostics?: EngineStorageDiagnostics;
+}
+
+export interface BlobCacheInfo {
+  entries: number;
+  totalBytes: number;
+  budgetBytes: number;
+  pinnedBytes: number;
+  evictedEntries: number;
+  evictedBytes: number;
+  /** Manifest rows rebuilt from disk at the last reconcile. */
+  reconciledEntries: number;
+  hits: number;
+  misses: number;
+  /** False when OPFS is unwritable or the quota was exhausted — the cache is
+   *  running in per-tab memory and nothing survives a reload. */
+  persistent: boolean;
+  /** Over budget with only pinned or on-screen entries left: new files are no
+   *  longer written rather than pinned ones being discarded. */
+  persistPaused: boolean;
+}
+
+/**
+ * Recursively list the origin's OPFS. Sizes come from `handle.getFile()`,
+ * which throws for a file another context holds an exclusive sync access
+ * handle on — SAHPool does exactly that for its whole pool, so a locked file
+ * (size omitted) is a live "who has the pool" signal, not a failure.
+ */
+export async function walkOpfs(maxEntries = 2000, maxDepth = 8): Promise<StorageInfo['opfs']> {
+  const nav = typeof navigator !== 'undefined' ? navigator : undefined;
+  if (!nav?.storage?.getDirectory) {
+    return { supported: false, entries: [], totalBytes: 0, truncated: false };
+  }
+  const entries: OpfsEntry[] = [];
+  let totalBytes = 0;
+  let truncated = false;
+  try {
+    const root = await nav.storage.getDirectory();
+    const walk = async (dir: FileSystemDirectoryHandle, prefix: string, depth: number) => {
+      if (depth > maxDepth) return;
+      // entries() is standard; older lib.dom typings may lack it.
+      for await (const [name, handle] of (dir as any).entries() as AsyncIterable<
+        [string, FileSystemHandle]
+      >) {
+        if (entries.length >= maxEntries) {
+          truncated = true;
+          return;
+        }
+        const path = prefix ? `${prefix}/${name}` : name;
+        if (handle.kind === 'directory') {
+          entries.push({ path, kind: 'directory' });
+          await walk(handle as FileSystemDirectoryHandle, path, depth + 1);
+        } else {
+          let size: number | undefined;
+          try {
+            size = (await (handle as FileSystemFileHandle).getFile()).size;
+            totalBytes += size;
+          } catch {
+            // Locked by an exclusive access handle (e.g. a live SAHPool).
+          }
+          const entry: OpfsEntry = { path, kind: 'file' };
+          if (size !== undefined) entry.size = size;
+          entries.push(entry);
+        }
+      }
+    };
+    await walk(root, '', 0);
+    entries.sort((a, b) => a.path.localeCompare(b.path));
+    return { supported: true, entries, totalBytes, truncated };
+  } catch (e) {
+    return {
+      supported: true,
+      entries,
+      totalBytes,
+      truncated,
+      error: e instanceof Error ? e.message : String(e),
+    };
+  }
+}

package/src/modules/devtools/versions.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, it } from 'vitest';
+import {
+  emptyBackendInfo,
+  emptyBackendVersions,
+  parseBackendInfo,
+  toEntityArray,
+  UNAVAILABLE,
+} from './versions';
+
+describe('toEntityArray', () => {
+  it('passes through an array, dropping non-objects', () => {
+    expect(toEntityArray([{ entity: 'ssp' }, null, 'x'])).toEqual([{ entity: 'ssp' }]);
+  });
+
+  it('wraps a single object', () => {
+    expect(toEntityArray({ entity: 'ssp' })).toEqual([{ entity: 'ssp' }]);
+  });
+
+  it('returns [] for null/undefined/primitives', () => {
+    expect(toEntityArray(null)).toEqual([]);
+    expect(toEntityArray(undefined)).toEqual([]);
+    expect(toEntityArray(42)).toEqual([]);
+  });
+});
+
+describe('parseBackendInfo', () => {
+  it('parses the singlenode shape (ssp only) incl. surrealdb_version', () => {
+    const { versions, entities } = parseBackendInfo([
+      { entity: 'ssp', version: '0.0.1-canary.69', surrealdb_version: '2.0.3', status: 'ready' },
+    ]);
+    expect(versions.ssp).toBe('0.0.1-canary.69');
+    expect(versions.surrealdb).toBe('2.0.3');
+    expect(versions.scheduler).toBe(UNAVAILABLE);
+    expect(entities).toHaveLength(1);
+  });
+
+  it('parses the cluster shape (scheduler + ssp + backend)', () => {
+    const { versions, entities } = parseBackendInfo([
+      { entity: 'scheduler', version: '0.9.0', surrealdb_version: '2.0.3', status: 'ready' },
+      { entity: 'ssp', version: '0.9.0', status: 'ready' },
+      { entity: 'backend', id: 'surrealdb', status: 'healthy' },
+    ]);
+    expect(versions.scheduler).toBe('0.9.0');
+    expect(versions.ssp).toBe('0.9.0');
+    expect(versions.surrealdb).toBe('2.0.3');
+    expect(entities).toHaveLength(3);
+  });
+
+  it('strips a leading surrealdb- prefix', () => {
+    const { versions } = parseBackendInfo([
+      { entity: 'ssp', version: '1.0.0', surrealdb_version: 'surrealdb-2.1.0' },
+    ]);
+    expect(versions.surrealdb).toBe('2.1.0');
+  });
+
+  it('takes surrealdb_version from whichever entity reports it', () => {
+    const { versions } = parseBackendInfo([
+      { entity: 'scheduler', version: '0.9.0' },
+      { entity: 'ssp', version: '0.9.0', surrealdb_version: '3.1.0' },
+    ]);
+    expect(versions.surrealdb).toBe('3.1.0');
+  });
+
+  it('tolerates a single object instead of an array', () => {
+    const { versions } = parseBackendInfo({ entity: 'ssp', version: '1.2.3' });
+    expect(versions.ssp).toBe('1.2.3');
+  });
+
+  it('returns all-unavailable / empty for null or garbage', () => {
+    expect(parseBackendInfo(null)).toEqual(emptyBackendInfo());
+    expect(parseBackendInfo('nope').versions).toEqual(emptyBackendVersions());
+    expect(parseBackendInfo([{ foo: 'bar' }]).versions).toEqual(emptyBackendVersions());
+  });
+});

package/src/modules/devtools/versions.ts

@@ -0,0 +1,110 @@
+/**
+ * Backend versions of the stack components, derived from the entity list the
+ * backend `/info` endpoint exposes (read via the `fn::spooky::info()` SurrealQL
+ * function). Any component that isn't reported degrades to `'unavailable'`.
+ */
+export interface BackendVersions {
+  ssp: string;
+  scheduler: string;
+  surrealdb: string;
+}
+
+export const UNAVAILABLE = 'unavailable';
+
+/**
+ * A single stack entity as reported by `/info` (one per ssp / scheduler /
+ * backend). Carries far more than versions — status, uptime, ip, views — so the
+ * DevTools can render the whole stack. Extra fields are preserved verbatim.
+ */
+/** One end-to-end sync-pipeline probe cycle recorded by the scheduler. */
+export interface HeartbeatSample {
+  ts: number;
+  /** `null` for a failed cycle. */
+  ms: number | null;
+  ok: boolean;
+}
+
+/**
+ * E2E heartbeat state, reported on the scheduler entity only. The scheduler
+ * times a probe row's full round trip (DB event → ingest → broadcast → SSP
+ * circuit step); `samples` is its rolling window of recent cycles, which the
+ * DevTools sparkline renders.
+ */
+export interface HeartbeatInfo {
+  enabled: boolean;
+  stale: boolean;
+  /** Nothing to measure right now (e.g. no ready SSPs mid-bootstrap). */
+  blocked?: boolean;
+  blocked_reason?: string | null;
+  last_e2e_ms: number | null;
+  last_ok_epoch_ms: number | null;
+  consecutive_failures: number;
+  interval_secs: number;
+  samples?: HeartbeatSample[];
+}
+
+export interface BackendEntity {
+  entity: string;
+  id?: string;
+  ip?: string | null;
+  status?: string;
+  version?: string;
+  surrealdb_version?: string;
+  uptime_seconds?: number;
+  views?: number;
+  /** Scheduler entity only. */
+  heartbeat?: HeartbeatInfo;
+  [key: string]: unknown;
+}
+
+export interface BackendInfo {
+  versions: BackendVersions;
+  entities: BackendEntity[];
+}
+
+export function emptyBackendVersions(): BackendVersions {
+  return { ssp: UNAVAILABLE, scheduler: UNAVAILABLE, surrealdb: UNAVAILABLE };
+}
+
+export function emptyBackendInfo(): BackendInfo {
+  return { versions: emptyBackendVersions(), entities: [] };
+}
+
+/** Strip a leading `surrealdb-` so versions read as bare semver (e.g. `2.0.3`). */
+function normalizeServerVersion(v: string): string {
+  return String(v).replace(/^surrealdb-/i, '').trim();
+}
+
+/**
+ * Normalize whatever `RETURN fn::spooky::info()` resolves to into the entity
+ * array. The SurrealQL function returns the parsed `/info` array; depending on
+ * how the result is unwrapped it may arrive as the array itself, a single
+ * object, or `null`. Tolerant of all three.
+ */
+export function toEntityArray(raw: unknown): BackendEntity[] {
+  if (Array.isArray(raw)) return raw.filter((e): e is BackendEntity => !!e && typeof e === 'object');
+  if (raw && typeof raw === 'object') return [raw as BackendEntity];
+  return [];
+}
+
+/**
+ * Derive component versions + the full entity list from a `/info` entity array.
+ * `surrealdb` is taken from whichever entity reports `surrealdb_version` (ssp or
+ * scheduler). Never throws; missing pieces stay `'unavailable'`.
+ */
+export function parseBackendInfo(raw: unknown): BackendInfo {
+  const entities = toEntityArray(raw);
+  const versions = emptyBackendVersions();
+
+  for (const entity of entities) {
+    const version = entity.version ? String(entity.version) : undefined;
+    if (entity.entity === 'ssp' && version) versions.ssp = version;
+    else if (entity.entity === 'scheduler' && version) versions.scheduler = version;
+
+    if (versions.surrealdb === UNAVAILABLE && entity.surrealdb_version) {
+      versions.surrealdb = normalizeServerVersion(String(entity.surrealdb_version));
+    }
+  }
+
+  return { versions, entities };
+}