@volter/world-core 2.0.0

package/src/world-store.ts

@@ -0,0 +1,572 @@
+// WORLD STORE — the transport seam for the control-plane's persistence.
+//
+// The kernel's on-disk surface is a CLOSED SET of synchronous primitives: JSONL
+// append-logs (read-all-rows / append-row), whole-file JSON sidecars
+// (read/write/exists/remove), directory listing, and atomic-write + cross-process
+// file-lock. `WorldStore` names exactly that set, and nothing more. Route the whole
+// kernel through it and the twin persists through ANY backend — the local
+// filesystem today (`FsWorldStore`, the default), an in-memory Map for tests and
+// serverless proofs (`MemoryWorldStore`), a Durable Object / KV / redis tomorrow —
+// without touching a line of twin business logic.
+//
+// The interface is SYNCHRONOUS on purpose. The kernel's internals (append-then-
+// project, read-then-append critical sections) are synchronous, and `clerk-twin.ts`'s
+// request handler is a pure sync function keyed by `root`. Making the store async
+// would ripple `await` through every one of those call sites and change the handler's
+// shape — precisely the blast radius this seam exists to avoid. An async backend is
+// handled at a SEPARATE persistence boundary: `hydrate()` the durable source into a
+// sync store, run the sync kernel, `flush()` back out. See `hydrateInto`/`flushFrom`.
+//
+// NOT part of this store, deliberately: `rateBudget.ts`'s ledger. That is a
+// client-side guard on REAL vendor calls, keyed by CREDENTIAL and living in the user's
+// home dir (NOT under the project state root) — it is not twin persistence and never
+// runs on the serve path. Its bespoke inode-identity lock and reservation semantics
+// cannot be expressed through this generic seam without weakening them, so it stays on
+// `node:fs`. See its module header.
+import {
+  appendFileSync,
+  chmodSync,
+  closeSync,
+  existsSync,
+  fsyncSync,
+  fstatSync,
+  lstatSync,
+  mkdirSync,
+  openSync,
+  readdirSync,
+  readSync,
+  realpathSync,
+  readFileSync,
+  renameSync,
+  rmSync,
+  statSync,
+  unlinkSync,
+  writeFileSync,
+} from 'node:fs';
+import { hostname } from 'node:os';
+import { basename, dirname, join, resolve } from 'node:path';
+
+/** Metadata a caller needs about a stored path. `isDirectory` distinguishes a JSON
+ *  sidecar from a per-service subdir (validate/scrub walk on it); `size`+`mtimeMs`
+ *  drive the append-dedupe index's cache-validity check in storage.ts. */
+export type WorldStat = { size: number; mtimeMs: number; isDirectory: boolean; isSymbolicLink?: boolean };
+
+/**
+ * The synchronous persistence seam. Every path is an ABSOLUTE key the caller already
+ * computed (via `worldPaths`/`worldStateRoot`); the store treats it as an opaque key,
+ * so a single active store partitions cleanly by `root` (the root is embedded in the
+ * key) and no per-root wiring is needed.
+ */
+export interface WorldStore {
+  /** Whole-file read; `null` when the path does not exist. */
+  read(path: string): string | null;
+  /** File content split on `\n` (blank lines and the trailing empty included, so a
+   *  caller can report 1-based row numbers); `[]` when the path does not exist. */
+  readLines(path: string): string[];
+  /** Raw append of exactly `data` (the caller owns any trailing newline). On the fs
+   *  backend this is the VOLTER_DURABLE-aware durable append. Does NOT create parents —
+   *  call `mkdir` first, matching the historical `appendDurable` contract. */
+  append(path: string, data: string): void;
+  /** Non-atomic whole-file write, creating parent dirs. Matches the plain
+   *  `writeFileSync` sidecar writers (cursors, refs, leases, plans, fork-meta). `secret` makes the
+   *  file owner-only (0600 in a 0700 directory) where the backend has permissions. */
+  write(path: string, data: string, options?: { secret?: boolean }): void;
+  /** Atomic whole-file write (write-temp-then-rename on fs), creating parent dirs.
+   *  Used where a reader must never observe a half-written document (state.json). `secret` makes
+   *  the file owner-only where the backend has permissions. */
+  writeAtomic(path: string, data: string, options?: { secret?: boolean }): void;
+  /** Does the path exist? */
+  exists(path: string): boolean;
+  /** Recursively remove the path (force; a missing path is not an error). */
+  remove(path: string): void;
+  /** Ensure a directory (and parents) exists. A no-op on backends with implicit dirs. */
+  mkdir(dirPath: string): void;
+  /** Immediate child names of a directory; `[]` when it does not exist. */
+  list(dirPath: string): string[];
+  /** Metadata for a path, or `null` when it does not exist. */
+  stat(path: string): WorldStat | null;
+  /** The file from byte `start` to its end, decoded as UTF-8 (`start` must fall on a character boundary: a line's
+   *  start in a log); `null` when the path does not exist. Optional: a reader of an append-only log uses it to read
+   *  only what was appended, and reads the whole file on a store without it. */
+  readRange?(path: string, start: number): string | null;
+  /** Resolve filesystem aliases for durable ownership; absent on opaque-key stores. */
+  canonicalPath?(path: string): string;
+  /** Run `fn` holding an exclusive lock on `lockPath`, releasing it afterwards. The fs
+   *  backend uses the historical cross-process reclaimable file lock; an in-memory,
+   *  single-instance backend is a synchronous pass-through. */
+  withLock<T>(lockPath: string, fn: () => T, options?: { retainLiveOwner?: boolean }): T;
+}
+
+// ── FsWorldStore — the DEFAULT, byte-identical to the historical node:fs behavior ────
+
+let fsAtomicSequence = 0;
+
+/** How long a lock may live before an acquirer treats it as abandoned. Above the 10s
+ *  wait timeout so a live-but-slow holder is never reclaimed out from under itself. */
+const LOCK_STALE_MS = 60_000;
+
+type LockHolder = { pid: number; hostname: string; at: string };
+
+function readLockHolder(lockPath: string): LockHolder | null {
+  try {
+    return JSON.parse(readFileSync(lockPath, 'utf8')) as LockHolder;
+  } catch {
+    return null; // missing, empty (mid-write), or malformed
+  }
+}
+
+function pidAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // ESRCH → no such process (dead); EPERM → exists but not ours to signal (alive)
+    return (error as NodeJS.ErrnoException).code === 'EPERM';
+  }
+}
+
+/** A lock is stale when its holder is provably gone (same host + dead pid) or it has
+ *  outlived LOCK_STALE_MS. The age falls back to the lockfile's own mtime when the
+ *  `at` record is unreadable — so a writer that crashed between creating the lock and
+ *  recording itself is still eventually reclaimed, while a freshly-created (recent
+ *  mtime) empty lock is left alone, avoiding a race with the live writer. */
+function lockIsStale(lockPath: string, retainLiveOwner = false): boolean {
+  const holder = readLockHolder(lockPath);
+  if (holder && holder.hostname === hostname() && Number.isInteger(holder.pid) && !pidAlive(holder.pid)) {
+    return true;
+  }
+  if (retainLiveOwner) return false;
+  const recordedAt = holder && typeof holder.at === 'string' ? Date.parse(holder.at) : NaN;
+  let stamp = recordedAt;
+  if (!Number.isFinite(stamp)) {
+    try {
+      stamp = statSync(lockPath).mtimeMs;
+    } catch {
+      return false; // lock vanished — let the next openSync settle it
+    }
+  }
+  return Date.now() - stamp > LOCK_STALE_MS;
+}
+
+function sleepSync(ms: number): void {
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/**
+ * The default store: a thin, faithful pass-through to `node:fs`. Every method preserves
+ * the exact behavior the kernel had before the seam existed — the durable append
+ * (VOLTER_DURABLE), the atomic write (temp + rename), and the reclaimable cross-process
+ * file lock — so the existing twins and the conformance suite are unaffected.
+ */
+export class FsWorldStore implements WorldStore {
+  read(path: string): string | null {
+    if (!existsSync(path)) return null;
+    return readFileSync(path, 'utf8');
+  }
+
+  readLines(path: string): string[] {
+    if (!existsSync(path)) return [];
+    return readFileSync(path, 'utf8').split('\n');
+  }
+
+  readRange(path: string, start: number): string | null {
+    let fd: number;
+    try { fd = openSync(path, 'r'); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; throw error; }
+    try {
+      const size = fstatSync(fd).size;
+      if (start >= size) return '';
+      const buffer = Buffer.allocUnsafe(size - start);
+      let read = 0;
+      while (read < buffer.length) { const n = readSync(fd, buffer, read, buffer.length - read, start + read); if (n === 0) break; read += n; }
+      return buffer.toString('utf8', 0, read);
+    } finally { closeSync(fd); }
+  }
+
+  append(path: string, data: string): void {
+    // Historical appendDurable: a completed append syscall survives a process crash;
+    // fsync additionally survives kernel-panic/power-loss when VOLTER_DURABLE=1.
+    const fd = openSync(path, 'a');
+    try {
+      appendFileSync(fd, data);
+      if (process.env.VOLTER_DURABLE === '1') fsyncSync(fd);
+    } finally {
+      closeSync(fd);
+    }
+  }
+
+  write(path: string, data: string, options: { secret?: boolean } = {}): void {
+    if (!options.secret) {
+      mkdirSync(dirname(path), { recursive: true });
+      writeFileSync(path, data);
+      return;
+    }
+    mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
+    writeFileSync(path, data, { mode: 0o600 });
+    chmodSync(path, 0o600);
+  }
+
+  writeAtomic(path: string, data: string, options: { secret?: boolean } = {}): void {
+    mkdirSync(dirname(path), { recursive: true });
+    const tmp = `${path}.${process.pid}.${Date.now()}.${fsAtomicSequence++}.tmp`;
+    try {
+      writeFileSync(tmp, data, options.secret ? { mode: 0o600 } : undefined);
+      renameSync(tmp, path);
+      if (options.secret) chmodSync(path, 0o600);
+    } finally {
+      rmSync(tmp, { force: true });
+    }
+  }
+
+  exists(path: string): boolean {
+    return existsSync(path);
+  }
+
+  remove(path: string): void {
+    rmSync(path, { recursive: true, force: true });
+  }
+
+  mkdir(dirPath: string): void {
+    mkdirSync(dirPath, { recursive: true });
+  }
+
+  list(dirPath: string): string[] {
+    if (!existsSync(dirPath)) return [];
+    return readdirSync(dirPath);
+  }
+
+  canonicalPath(path: string): string {
+    const absolute = resolve(path);
+    try { return realpathSync(absolute); }
+    catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== 'ENOENT' || dirname(absolute) === absolute) throw error;
+      return join(this.canonicalPath(dirname(absolute)), basename(absolute));
+    }
+  }
+
+  stat(path: string): WorldStat | null {
+    if (!existsSync(path)) return null;
+    const s = statSync(path);
+    return { size: s.size, mtimeMs: s.mtimeMs, isDirectory: s.isDirectory(), ...(lstatSync(path).isSymbolicLink() ? { isSymbolicLink: true } : {}) };
+  }
+
+  /** Run `fn` holding an exclusive cross-process file lock. The lockfile records
+   *  `{pid, hostname, at}`; a contender that finds a stale lock (dead pid or age >
+   *  LOCK_STALE_MS) reclaims it by atomically renaming it aside — so a crashed holder
+   *  can't wedge the world forever. Reclaim is race-safe: only the process that wins
+   *  the rename clears the stale inode, and the exclusive `wx` create still decides the
+   *  winner. (Moved verbatim from storage.ts `withFileLock`.) */
+  withLock<T>(lockPath: string, fn: () => T, options: { retainLiveOwner?: boolean } = {}): T {
+    mkdirSync(dirname(lockPath), { recursive: true });
+    const started = Date.now();
+    let fd: number | null = null;
+    while (fd === null) {
+      try {
+        fd = openSync(lockPath, 'wx');
+      } catch (error) {
+        const code = (error as NodeJS.ErrnoException).code;
+        if (code !== 'EEXIST') throw error;
+        if (lockIsStale(lockPath, options.retainLiveOwner)) {
+          // Serialize reclaimers, then re-read: a contender's earlier stale observation
+          // must never rename another contender's newly acquired, live lock.
+          const recoveryPath = `${lockPath}.recovery`;
+          let recovery: number;
+          try { recovery = openSync(recoveryPath, 'wx'); }
+          catch (error) {
+            if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error;
+            if (Date.now() - started > 10_000) throw new Error(`Lock recovery is active or interrupted: ${recoveryPath}. Inspect its owner before removing the recovery record.`);
+            sleepSync(25); continue;
+          }
+          try {
+            writeFileSync(recovery, JSON.stringify({ pid: process.pid, hostname: hostname() }));
+            if (lockIsStale(lockPath, options.retainLiveOwner)) {
+              const holder = readLockHolder(lockPath);
+              const salvage = `${lockPath}.stale.${process.pid}.${Date.now()}`;
+              try {
+                renameSync(lockPath, salvage);
+                console.warn(`[world] reclaiming stale storage lock ${lockPath} (held by pid ${holder?.pid ?? '?'} on ${holder?.hostname ?? '?'})`);
+                unlinkSync(salvage);
+              } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
+            }
+          } finally { closeSync(recovery); unlinkSync(recoveryPath); }
+          continue;
+        }
+        if (Date.now() - started > 10_000) {
+          const holder = readLockHolder(lockPath);
+          const since = holder && typeof holder.at === 'string' ? Date.parse(holder.at) : NaN;
+          throw new Error(`Timed out waiting for world storage lock: ${lockPath}${holder ? ` (held by pid ${holder.pid} on ${holder.hostname}${Number.isFinite(since) ? ` for ${Date.now() - since}ms` : ''})` : ''}`);
+        }
+        sleepSync(25);
+      }
+    }
+
+    // Record the holder so a later contender can detect staleness. Best-effort: a write
+    // failure here doesn't weaken exclusivity, only staleness diagnostics.
+    try {
+      writeFileSync(fd, `${JSON.stringify({ pid: process.pid, hostname: hostname(), at: new Date().toISOString() } satisfies LockHolder)}\n`);
+    } catch { /* ignore */ }
+
+    const acquired = Date.now();
+    try {
+      return fn();
+    } finally {
+      // Every World on the machine waits on some of these locks; a long hold stalls them all.
+      const heldMs = Date.now() - acquired;
+      if (heldMs > 2_000) console.warn(`[world] held storage lock ${lockPath} for ${heldMs}ms (pid ${process.pid})\n${new Error().stack?.split('\n').slice(2, 8).join('\n') ?? ''}`);
+      const held = fstatSync(fd);
+      closeSync(fd);
+      try {
+        const current = lstatSync(lockPath);
+        if (held.dev === current.dev && held.ino === current.ino) unlinkSync(lockPath);
+      } catch { /* already reclaimed by a stale-lock sweep */ }
+    }
+  }
+}
+
+// ── MemoryWorldStore — a Map-backed store; the proof a non-fs transport works ─────────
+
+// A process-global, strictly-increasing stamp source shared by every MemoryWorldStore, so
+// no two writes (across instances or requests) ever collide on `stat().mtimeMs`. See `bump`.
+let memoryStoreClock = 0;
+
+const byteLength = (text: string): number => new TextEncoder().encode(text).length;
+
+/**
+ * An in-memory store. Files are entries in a `Map<path, string>`; directories are
+ * synthesized from the key set (a path is a directory when it is a strict prefix of a
+ * file key). `withLock` is a synchronous pass-through — a single in-process instance has
+ * no concurrent writers to serialize, so the read-then-append critical sections the
+ * kernel guards are already atomic under the single-threaded event loop.
+ *
+ * This is the portability proof: point the active store here and the entire kernel — and
+ * the clerk twin on top of it — runs with the filesystem untouched.
+ */
+export class MemoryWorldStore implements WorldStore {
+  /** path → content, and path → version stamp driving `stat().mtimeMs`. */
+  private readonly files = new Map<string, string>();
+  private readonly versions = new Map<string, number>();
+  /** path → UTF-8 byte size, kept as content changes so `stat` never re-encodes a growing log. */
+  private readonly sizes = new Map<string, number>();
+  /** The stamp of this store's most recent mutation (0 = never mutated). A serverless
+   *  entry snapshots this before running the sync kernel and flushes ONLY when it moved —
+   *  read-only requests (the common case) then skip the durable write entirely. */
+  private lastMutation = 0;
+
+  mutationStamp(): number {
+    return this.lastMutation;
+  }
+
+  /** Paths whose content changed strictly after `stamp` — the DELTA-FLUSH primitive
+   *  (runtime contract R11b): a serverless entry records `mutationStamp()` at hydration,
+   *  runs the sync kernel, then persists ONLY these paths (plus removals, which the caller
+   *  detects by diffing its hydrated path set against `entries()` — a removed path has no
+   *  version to report). O(paths) scan, O(delta) write. */
+  changedSince(stamp: number): string[] {
+    const changed: string[] = [];
+    for (const [path, version] of this.versions) {
+      if (version > stamp && this.files.has(path)) changed.push(path);
+    }
+    return changed.sort();
+  }
+
+  private bump(path: string): void {
+    // Draw the stamp from a PROCESS-GLOBAL monotonic counter, not a per-instance one.
+    // storage.ts's append-dedupe cache is keyed by path and validated by (size, mtimeMs),
+    // and it persists across requests in a Worker isolate while a fresh MemoryWorldStore is
+    // created per request (the hydrate→sync-kernel→flush shape). A per-instance counter
+    // resets to 0 each request, so two instances could present the SAME (size, mtimeMs) for
+    // one path with DIFFERENT content — serving a stale index (dedupe bypass) or a false
+    // conflict. A global counter makes every write across every instance a distinct,
+    // strictly-increasing stamp, so a cross-instance collision is impossible.
+    this.lastMutation = memoryStoreClock += 1;
+    this.versions.set(path, this.lastMutation);
+  }
+
+  read(path: string): string | null {
+    return this.files.has(path) ? this.files.get(path)! : null;
+  }
+
+  readLines(path: string): string[] {
+    const content = this.files.get(path);
+    return content === undefined ? [] : content.split('\n');
+  }
+
+  append(path: string, data: string): void {
+    this.files.set(path, (this.files.get(path) ?? '') + data);
+    this.sizes.set(path, (this.sizes.get(path) ?? 0) + byteLength(data));
+    this.bump(path);
+  }
+
+  write(path: string, data: string): void {
+    this.files.set(path, data);
+    this.sizes.set(path, byteLength(data));
+    this.bump(path);
+  }
+
+  writeAtomic(path: string, data: string): void {
+    // Atomic by nature in a single process: the assignment is indivisible, so no reader
+    // ever observes a partial document.
+    this.files.set(path, data);
+    this.sizes.set(path, byteLength(data));
+    this.bump(path);
+  }
+
+  exists(path: string): boolean {
+    if (this.files.has(path)) return true;
+    // A directory "exists" when some file lives under it.
+    const prefix = path.endsWith('/') ? path : `${path}/`;
+    for (const key of this.files.keys()) if (key.startsWith(prefix)) return true;
+    return false;
+  }
+
+  remove(path: string): void {
+    let removedAny = false;
+    if (this.files.delete(path)) {
+      this.versions.delete(path);
+      this.sizes.delete(path);
+      removedAny = true;
+    }
+    const prefix = path.endsWith('/') ? path : `${path}/`;
+    for (const key of [...this.files.keys()]) {
+      if (key.startsWith(prefix)) {
+        this.files.delete(key);
+        this.versions.delete(key);
+        this.sizes.delete(key);
+        removedAny = true;
+      }
+    }
+    // A pure removal is a mutation too: without this, a request that only deletes state
+    // would read as clean and the delta-flush would never persist the removal (R11b).
+    if (removedAny) this.lastMutation = memoryStoreClock += 1;
+  }
+
+  mkdir(_dirPath: string): void {
+    // Directories are implicit in the key set; nothing to do.
+  }
+
+  list(dirPath: string): string[] {
+    const prefix = dirPath.endsWith('/') ? dirPath : `${dirPath}/`;
+    const children = new Set<string>();
+    for (const key of this.files.keys()) {
+      if (!key.startsWith(prefix)) continue;
+      const rest = key.slice(prefix.length);
+      const slash = rest.indexOf('/');
+      children.add(slash === -1 ? rest : rest.slice(0, slash));
+    }
+    return [...children];
+  }
+
+  stat(path: string): WorldStat | null {
+    const content = this.files.get(path);
+    if (content !== undefined) {
+      return { size: this.sizes.get(path) ?? byteLength(content), mtimeMs: this.versions.get(path) ?? 0, isDirectory: false };
+    }
+    if (this.exists(path)) {
+      return { size: 0, mtimeMs: 0, isDirectory: true };
+    }
+    return null;
+  }
+
+  withLock<T>(_lockPath: string, fn: () => T): T {
+    return fn();
+  }
+
+  /** Test/inspection helper: a plain snapshot of every stored file. Not part of the
+   *  `WorldStore` contract — used by `flushFrom` and by tests asserting no fs was touched. */
+  entries(): Record<string, string> {
+    return Object.fromEntries(this.files);
+  }
+}
+
+// ── the active store (the injection point) ───────────────────────────────────────────
+
+let activeStore: WorldStore = new FsWorldStore();
+
+/** The request-scoped store: `withWorldStore` runs its callback inside an AsyncLocalStorage
+ *  context where the runtime has one (Node, Bun, workerd with nodejs_compat), so concurrent
+ *  requests for different worlds in one isolate each see their own store across awaits. Reached
+ *  through `process.getBuiltinModule`, never a static import, so a browser bundle stays clean and
+ *  falls back to the module-global swap. */
+type StoreScope = { getStore(): WorldStore | undefined; run<R>(store: WorldStore, fn: () => R): R };
+const scope: StoreScope | null = (() => {
+  try {
+    const hooks = (globalThis as { process?: { getBuiltinModule?: (id: string) => unknown } }).process?.getBuiltinModule?.('node:async_hooks') as { AsyncLocalStorage?: new () => StoreScope } | undefined;
+    return hooks?.AsyncLocalStorage ? new hooks.AsyncLocalStorage() : null;
+  } catch {
+    return null;
+  }
+})();
+
+/** The store the kernel currently persists through: the request's scoped store, else the
+ *  module-global one (`FsWorldStore` by default). */
+export function getActiveWorldStore(): WorldStore {
+  return scope?.getStore() ?? activeStore;
+}
+
+/** Swap the active store. A serverless/DO/test entry sets a `MemoryWorldStore` (or a
+ *  backend-backed store) here before running the sync kernel, and restores the previous
+ *  one afterwards. Returns the store that was active, so callers can restore it. */
+export function setActiveWorldStore(store: WorldStore): WorldStore {
+  const previous = activeStore;
+  activeStore = store;
+  return previous;
+}
+
+/** Run `fn` with `store` active, restoring the previous store afterwards (even on
+ *  throw). ASYNC-AWARE: when `fn` returns a promise, the restore happens after it
+ *  SETTLES — the first version restored at `fn`'s return, i.e. at an async callback's
+ *  FIRST await, un-scoping the rest of the request (both serverless pack lanes hit this
+ *  independently and hand-rolled the same try/finally, 2026-09-01).
+ *
+ *  CONCURRENCY: where the runtime has AsyncLocalStorage the scope is per async context, so
+ *  unserialized requests for different worlds interleave without bleeding stores. Only a runtime
+ *  without it (a browser bundle) falls back to the module-global swap, which is safe only when
+ *  store-scoped work is serialized. */
+export function withWorldStore<T>(store: WorldStore, fn: () => Promise<T>): Promise<T>;
+export function withWorldStore<T>(store: WorldStore, fn: () => T): T;
+export function withWorldStore<T>(store: WorldStore, fn: () => T | Promise<T>): T | Promise<T> {
+  if (scope) return scope.run(store, fn);
+  const previous = setActiveWorldStore(store);
+  let result: T | Promise<T>;
+  try {
+    result = fn();
+  } catch (error) {
+    setActiveWorldStore(previous);
+    throw error;
+  }
+  if (result instanceof Promise) {
+    return result.finally(() => setActiveWorldStore(previous));
+  }
+  setActiveWorldStore(previous);
+  return result;
+}
+
+// ── async persistence boundary ───────────────────────────────────────────────────────
+//
+// The kernel is synchronous; a durable backend (Durable Object storage, KV, redis, S3)
+// is asynchronous. These two helpers are the ONLY async in the persistence story: a
+// serverless entry `await hydrate()`s the durable snapshot into a sync store, runs the
+// sync kernel (which never awaits), then `await flush()`es the mutated snapshot back.
+// The shapes are proven against `MemoryWorldStore` (see world-store.test.ts); a real DO
+// adapter implements `HydrationSource`/`HydrationSink` over its own async storage.
+
+/** A durable, async source of a world snapshot (path → content). */
+export interface HydrationSource {
+  load(): Promise<Record<string, string>>;
+}
+
+/** A durable, async sink for a world snapshot (path → content). */
+export interface HydrationSink {
+  save(snapshot: Record<string, string>): Promise<void>;
+}
+
+/** Fill `store` from an async durable `source` before the sync kernel runs. */
+export async function hydrateInto(store: MemoryWorldStore, source: HydrationSource): Promise<void> {
+  const snapshot = await source.load();
+  for (const [path, content] of Object.entries(snapshot)) store.write(path, content);
+}
+
+/** Persist `store`'s current contents back to an async durable `sink` after the sync
+ *  kernel has run. */
+export async function flushFrom(store: MemoryWorldStore, sink: HydrationSink): Promise<void> {
+  await sink.save(store.entries());
+}

package/src/worldConfig.ts

@@ -0,0 +1,27 @@
+// World service config — generic per-service settings for the twin runtime
+// (`.volter/world/config.json`). Kept deliberately small + open: the index
+// signature lets a service carry arbitrary extra config that downstream tools
+// define. (The tracker's annotation adapter defines + reads `annotationPolicy` and
+// `browseUrlTemplate` on top of this — those are verification concerns, not the
+// twin's, so they are NOT typed here.)
+import { join } from 'node:path';
+import { worldStateRoot } from './storage.ts';
+import { getActiveWorldStore } from './world-store.ts';
+
+export type WorldServiceConfig = {
+  provider?: string;
+  tool?: string;
+  pollLimit?: number;
+} & Record<string, unknown>;
+
+export type WorldConfig = {
+  services: Record<string, WorldServiceConfig>;
+} & Record<string, unknown>;
+
+export function loadWorldConfig(root?: string): WorldConfig {
+  const path = join(worldStateRoot(root), 'config.json');
+  const raw = getActiveWorldStore().read(path);
+  if (raw === null) return { services: {} };
+  const parsed = JSON.parse(raw) as Partial<WorldConfig>;
+  return { ...parsed, services: parsed.services ?? {} };
+}

package/stream-bridge.cjs

@@ -0,0 +1,80 @@
+// THE STREAM BRIDGE: a World's byte-stream doors (a TCP protocol a twin serves — smtp's own, planetscale's
+// mysql — carried over a WebSocket, because a hosted World takes no inbound TCP) as loopback listeners,
+// so an UNMODIFIED client (nodemailer, mysql2, Python's smtplib) connects to 127.0.0.1:<port> as it would
+// to the twin's own listener. One WebSocket per TCP connection; its first message is the World token,
+// then bytes both ways. Used by the attach entry (in the app's process) and by `volter-world attach`
+// (beside a command in any language).
+'use strict';
+
+const net = require('node:net');
+
+/** The bridged connections whose WebSocket has not closed yet, across every bridge. */
+const live = new Set();
+
+/** Serve one stream door on 127.0.0.1:`port` (0: any). Resolves to the listening server. */
+function bridgeStream(url, token, port = 0) {
+  const server = net.createServer((socket) => {
+    const ws = new WebSocket(url);
+    const done = new Promise((resolve) => ws.addEventListener('close', resolve));
+    live.add(done);
+    done.then(() => live.delete(done));
+    ws.binaryType = 'arraybuffer';
+    const early = [];
+    let open = false;
+    // the client may write and hang up before the WebSocket opens: its bytes still go, then the close
+    let hangUpWhenOpen = false;
+    ws.addEventListener('open', () => {
+      ws.send(token);
+      open = true;
+      for (const chunk of early) ws.send(chunk);
+      early.length = 0;
+      if (hangUpWhenOpen) ws.close(1000);
+    });
+    ws.addEventListener('message', (event) => { if (typeof event.data !== 'string') socket.write(Buffer.from(event.data)); });
+    ws.addEventListener('close', (event) => {
+      if (event.code >= 4000) process.stderr.write(`[twin-stream] ${url}: ${event.reason || event.code}\n`);
+      socket.end();
+    });
+    ws.addEventListener('error', () => socket.destroy());
+    socket.on('data', (chunk) => { if (open) ws.send(chunk); else early.push(chunk); });
+    const hangUp = () => {
+      if (!open) { hangUpWhenOpen = true; return; }
+      try { ws.close(1000); } catch { /* already closed */ }
+    };
+    socket.on('close', hangUp);
+    socket.on('error', hangUp);
+  });
+  return new Promise((resolve, reject) => {
+    server.once('error', reject);
+    server.listen(port, '127.0.0.1', () => { server.off('error', reject); resolve(server); });
+  });
+}
+
+/** The env a client reads for a stream, with the bridge's listener in place of `${host}`/`${port}`. */
+function streamEnv(templates, port) {
+  const out = {};
+  for (const [name, template] of Object.entries(templates || {})) {
+    out[name] = String(template).replace(/\$\{host\}/g, '127.0.0.1').replace(/\$\{port\}/g, String(port));
+  }
+  return out;
+}
+
+/** Resolves once every bridged connection has finished carrying its bytes (or after `ms`): what a
+ *  launcher waits for after its command exits, since a client may write and exit at once. */
+function drainStreams(ms = 10_000) {
+  return Promise.race([Promise.all([...live]), new Promise((resolve) => setTimeout(resolve, ms).unref())]);
+}
+
+/** Bridge every stream of a manifest; resolves to the env naming the listeners and the servers. */
+async function bridgeStreams(streams, token) {
+  const env = {};
+  const servers = [];
+  for (const stream of Object.values(streams || {})) {
+    const server = await bridgeStream(stream.url, token);
+    servers.push(server);
+    Object.assign(env, streamEnv(stream.env, server.address().port));
+  }
+  return { env, servers };
+}
+
+module.exports = { bridgeStream, bridgeStreams, drainStreams, streamEnv };