@nimbus-sh/fabric 0.1.0

package/src/launch-journal.ts

@@ -0,0 +1,229 @@
+/**
+ * launch-journal.ts — durable record of the resident launches a Durable Object
+ * owes, and their recovery after an instance reset.
+ *
+ * The platform resets a session Durable Object over what one turn has
+ * outstanding in storage ("Internal error in Durable Object storage caused
+ * object to be reset"), and a resident launch is the largest writer a session
+ * has. Everything a launch holds is in memory, so the process it is building
+ * and the terminal watching it both go with the instance — the journal is what
+ * a LATER instance reads to know that happened, and this module is the whole
+ * of that mechanism: the put→sync durability barrier on the way in, the
+ * delete→sync release on the way out, and the once-per-instance recovery pump
+ * that re-drives what a previous generation left behind.
+ *
+ * What a launch IS stays the embedder's: the journal stores the record it is
+ * given and hands it back on recovery. The mechanism reads only the fields in
+ * {@link ResidentLaunchRecord}; everything else in the record rides through
+ * opaquely.
+ */
+
+/**
+ * Prefix for the resident-process journal: one row per resident this session
+ * owes the user, keyed by the pid it was built for.
+ *
+ * A resident holds its state in memory — the process table entry, the facet
+ * handle, the terminal — so an instance reset destroys it silently. The row
+ * is what a LATER instance reads to know a resident ended that way rather
+ * than on purpose: a pid at or below the reader's own pid base was allocated
+ * by a previous generation (PID_GEN_STRIDE, core's process-table). Written
+ * (and synced) before the launch's first byte of work, rewritten as `running`
+ * when the launch settles, and released only when the PROCESS ends — because
+ * the resets this row survives strike after the launch as often as during it
+ * (measured live, staging 2026-08-13: every observed reset landed seconds
+ * AFTER settle).
+ *
+ * The VALUE is live production DO storage and must never change — renaming a
+ * storage key is a migration, and orphaned rows are the least of what it
+ * breaks.
+ */
+export const RESIDENT_LAUNCH_KEY_PREFIX = 'resident-launch:';
+
+/** A launch is re-driven once. A reset that recurs is not the transient one. */
+export const RESIDENT_LAUNCH_MAX_ATTEMPT = 1;
+
+/**
+ * A resident process this session owes the user, as a later instance would
+ * have to re-drive it.
+ *
+ * The launch's own inputs and nothing derived from them: everything a launch
+ * builds is a pure function of these, and the images it writes are content-
+ * addressed, so re-driving is the same work again rather than a repair. The
+ * inputs themselves are the embedder's — a record type extends this base with
+ * whatever its `redrive` needs, and the journal never reads those fields.
+ *
+ * The row lives for the PROCESS's lifetime, not the launch's. Measured live
+ * (staging, 2026-08-13): every observed reset struck seconds AFTER the launch
+ * settled — the platform kills the object while the resident runs, which is
+ * when a launch-scoped row had already been deleted and recovery had nothing
+ * to find. A resident's facet cannot outlive its session instance (the
+ * process host's held-open leg dies with it), so a row from a previous
+ * generation always names a process that is genuinely gone.
+ */
+export interface ResidentLaunchRecord {
+  pid: number;
+  command: string;
+  /** 0 for a launch the user asked for; 1 for the one re-drive it may get. */
+  attempt: number;
+  /** Where the resident was when its instance died: still being built, or
+   *  booted and running. Running residents re-drive with a fresh attempt
+   *  budget — their launch already proved itself once. */
+  phase: 'starting' | 'running';
+}
+
+/**
+ * The slice of Durable Object storage the journal writes through. Exactly a
+ * `DurableObjectStorage`, narrowed to what the mechanism performs — `sync()`
+ * is load-bearing, see {@link ResidentLaunchJournal.journal}.
+ */
+export interface LaunchJournalStorage {
+  put(key: string, value: unknown): Promise<void>;
+  delete(key: string): Promise<boolean>;
+  list<T = unknown>(options: { prefix: string }): Promise<Map<string, T>>;
+  sync(): Promise<void>;
+}
+
+/** What the journal's recovery needs from its embedder. */
+export interface LaunchJournalHost<R extends ResidentLaunchRecord> {
+  /**
+   * The current instance generation's pid floor. A pid at or below it was
+   * allocated by a PREVIOUS instance (core's process-table, PID_GEN_STRIDE),
+   * so its launch never finished; above it is this instance's own, still
+   * running. The journal takes the base rather than the predicate so the one
+   * definition of what a prior-generation pid is stays in the process table.
+   */
+  generationBase(): number;
+  /**
+   * Root a recovery re-drive on the instance (`ctx.waitUntil`) so it is not
+   * an abandoned promise between turns.
+   */
+  waitUntil(promise: Promise<unknown>): void;
+  /**
+   * Re-drive an interrupted launch from its journalled inputs. `attempt` is
+   * the budget the re-drive spends — the mechanism computes it, the embedder
+   * carries it into the launch it starts. The result is discarded: a re-drive
+   * owns its own process, and nobody is waiting on the pid it allocates.
+   */
+  redrive(record: R, attempt: number): Promise<unknown>;
+  /** A re-drive is being started for this record. */
+  onRedrive?(record: R): void;
+  /** The record's re-drive budget is spent; the resident stays stopped. */
+  onAbandoned?(record: R): void;
+  /** The re-drive itself failed. */
+  onRedriveFailed?(record: R, error: unknown): void;
+}
+
+/**
+ * The resident-launch journal of one Durable Object instance.
+ *
+ * In-memory state here is per-instance on purpose: `journalledPids` tracks the
+ * rows THIS instance wrote, and `recovered` whether this instance has already
+ * read the journal a reset leaves behind. Rows from a previous instance are
+ * recovery's to consume, never the release path's.
+ */
+export class ResidentLaunchJournal<R extends ResidentLaunchRecord> {
+  /**
+   * Pids THIS instance holds journal rows for. What keeps the terminal hook —
+   * which fires for every process, shells and one-shots included — from
+   * paying a storage delete for pids that never had a row.
+   */
+  private journalledPids = new Set<number>();
+  /** Whether this instance has already read the journal a reset leaves behind. */
+  private recovered = false;
+
+  constructor(
+    private readonly storage: LaunchJournalStorage,
+    private readonly host: LaunchJournalHost<R>,
+  ) {}
+
+  /**
+   * Record a launch as in flight, so an instance that replaces this one knows
+   * it never finished. Best-effort: a launch that cannot be journalled still
+   * runs, and a reset then costs exactly what it cost before the journal.
+   *
+   * Synced, not merely put: `await put()` resolves before durability, and the
+   * reset this journal exists for destroys every write its turn still had
+   * outstanding — measured live, a launch killed in its first chunks left NO
+   * row for the replacement instance to find, which is how the recovery this
+   * feeds sat inert while its own test stayed green. `sync()` is the storage
+   * layer's durability barrier: the row is on disk before the launch performs
+   * its first byte of real work. What remains is a reset between the put and
+   * the sync's completion — and a launch that dies there has not started, so
+   * losing its row costs a retype, not a recovery.
+   */
+  async journal(record: R): Promise<void> {
+    try {
+      this.journalledPids.add(record.pid);
+      await this.storage.put(`${RESIDENT_LAUNCH_KEY_PREFIX}${record.pid}`, record);
+      await this.storage.sync();
+    } catch (e: unknown) {
+      console.warn('[nimbus] resident launch journal write failed:', errorMessage(e));
+    }
+  }
+
+  /** True while this instance holds a journal row for `pid`. */
+  has(pid: number): boolean {
+    return this.journalledPids.has(pid);
+  }
+
+  /**
+   * The journal row's one release: the process is over, nothing is owed.
+   * Synced so an instance reset moments later cannot roll the delete back and
+   * resurrect a process the user watched end.
+   */
+  async release(pid: number): Promise<void> {
+    if (!this.journalledPids.delete(pid)) return;
+    try {
+      await this.storage.delete(`${RESIDENT_LAUNCH_KEY_PREFIX}${pid}`);
+      await this.storage.sync();
+    } catch (e: unknown) {
+      console.warn('[nimbus] resident launch journal delete failed:', errorMessage(e));
+    }
+  }
+
+  /**
+   * Re-drive the launches a previous instance was building when it was reset.
+   *
+   * Sited on the launch-turn pump because the pump is what an alarm calls, and
+   * a launch that was suspended has an alarm armed for it — a reset during a
+   * chunk fails that alarm, and the platform re-delivers it to the instance
+   * that replaces this one. So the first turn after a reset is already this
+   * one.
+   *
+   * Runs once per instance: the journal only changes when a launch of THIS
+   * instance starts or settles, and those are rows this instance wrote.
+   */
+  async recoverInterrupted(): Promise<void> {
+    if (this.recovered) return;
+    this.recovered = true;
+    const journal = await this.storage.list<R>({ prefix: RESIDENT_LAUNCH_KEY_PREFIX });
+    const base = this.host.generationBase();
+    for (const [key, record] of journal) {
+      // A pid at or below this instance's base was allocated by a PREVIOUS one
+      // (process-table.ts, PID_GEN_STRIDE), so its launch never finished; above
+      // the base is this instance's own, still running. Same predicate as
+      // `session/rpc.ts` uses to attribute a prior generation's pid.
+      if (!(record.pid > 0 && record.pid <= base)) continue;
+      await this.storage.delete(key);
+      if (record.attempt >= RESIDENT_LAUNCH_MAX_ATTEMPT) {
+        this.host.onAbandoned?.(record);
+        continue;
+      }
+      this.host.onRedrive?.(record);
+      // Not awaited: this call is running inside the alarm that granted the
+      // turn, and the launch it starts asks for turns of its own through that
+      // same alarm — awaiting it here would be waiting on an alarm that cannot
+      // be scheduled until this one returns.
+      this.host.waitUntil(
+        this.host.redrive(record, record.attempt + 1)
+          .catch((e: unknown) => {
+            this.host.onRedriveFailed?.(record, e);
+          }),
+      );
+    }
+  }
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/launch-pacer.ts

@@ -0,0 +1,231 @@
+/**
+ * launch-pacer.ts — spreading a resident launch across Durable Object turns.
+ *
+ * Building a resident process is the largest single span of computation this
+ * session performs: for pi it walks a 17 MB source tree through eight
+ * enrichment passes, serializes a 22.9 MB module map, and writes that map into
+ * the image store. Done in one turn it occupied the session DO's only thread
+ * for 15-35 s, and a session that cannot reach its thread cannot service the
+ * terminal WebSocket — the launch turn finished `outcome=ok` and the terminal
+ * died anyway, painting "[process terminal closed]" over a process that was
+ * still running.
+ *
+ * A faster launch does not fix that. A launch half the length still blocks the
+ * thread for as long as it runs, and the socket is dropped inside that window
+ * whether or not the work succeeds. What fixes it is never holding the thread
+ * for long in the first place, which means suspending the launch at bounded
+ * intervals and resuming it on a fresh turn. Responsiveness stops depending on
+ * how long the total work takes.
+ *
+ * A fresh turn is also a fresh CPU budget. The same launches that dropped the
+ * socket were also being killed with `exceededCpu` at 31.8 s and 32.5 s
+ * against a 30 s ceiling, and no amount of yielding *within* one invocation
+ * moves that: CPU accrues to the invocation, not to the pause. Only genuinely
+ * re-entering the object resets it.
+ *
+ * Progress is measured in bytes rather than milliseconds because workerd's
+ * clock does not advance without I/O — a wall-clock guard inside a span of
+ * pure computation reads zero however many seconds it burns, which is why the
+ * phase costs behind this module had to be recovered from per-turn `cpuTime`
+ * rather than measured in place. Bytes are what the work is actually
+ * proportional to, and they are exact. The same reasoning is why
+ * `git/network-facet.ts` bounds its checkout chunks by entries and decoded
+ * bytes and treats its wall guard as coarse.
+ */
+
+/** How a paced launch gets back onto a fresh Durable Object turn. */
+export interface LaunchTurnScheduler {
+  /**
+   * Suspend until a fresh turn is running this launch again.
+   *
+   * `chunkEnded` settles when the resumed launch reaches its next suspension
+   * point or finishes, so whoever grants the turn can await the work it just
+   * released rather than letting it run detached in a handler's microtask
+   * drain.
+   */
+  nextTurn(chunkEnded: Promise<void>): Promise<void>;
+}
+
+/**
+ * Bytes of launch work one turn may perform before it must yield.
+ *
+ * Sized so a chunk stays far below both the CPU ceiling and the span in which
+ * a terminal socket is at risk, while keeping the number of turn handoffs —
+ * each an alarm round trip — small enough not to dominate a launch. pi's
+ * 22.9 MB map crosses this about a dozen times per phase that handles it.
+ */
+export const LAUNCH_CHUNK_MAX_BYTES = 2_000_000;
+
+/**
+ * Accounts launch progress and ends the turn when a chunk's worth has been
+ * spent.
+ *
+ * Callers report the work they are about to do or have just done and await
+ * the result; a pacer that is not yielding returns without suspending, so the
+ * one-shot exec path — which passes no pacer at all — keeps its exact
+ * behaviour and cost. Nothing here decides WHAT the launch does, only where it
+ * is allowed to stop.
+ */
+export class LaunchPacer {
+  /** Turn handoffs this launch has taken. Reported with the launch. */
+  chunks = 0;
+  /** Total work accounted, for the same report. */
+  bytes = 0;
+
+  private spent = 0;
+  private chunkEnded: { promise: Promise<void>; resolve: () => void } | undefined;
+
+  /**
+   * @param stillWanted Checked every time the launch resumes. A launch spans
+   *   many turns, so anything may have happened to what it is building for
+   *   while it was suspended; throwing from here is how a launch stops instead
+   *   of spending turn after turn on work nothing will use. Checked at the one
+   *   place a launch can be interrupted, rather than at whichever phases
+   *   remembered to ask.
+   */
+  constructor(
+    private readonly scheduler: LaunchTurnScheduler,
+    private readonly maxChunkBytes: number = LAUNCH_CHUNK_MAX_BYTES,
+    private readonly stillWanted?: () => void,
+  ) {}
+
+  /**
+   * Account `bytes` of completed work, ending the turn if a chunk is full.
+   *
+   * Safe to call anywhere the launch holds no state that a concurrent turn
+   * could invalidate — which is why the image store registers its whole root
+   * set before the first call rather than one entry at a time.
+   */
+  async spend(bytes: number): Promise<void> {
+    this.bytes += bytes;
+    this.spent += bytes;
+    if (this.spent < this.maxChunkBytes) return;
+    this.spent = 0;
+    this.chunks++;
+    // Release the turn that resumed us before asking for the next one.
+    this.chunkEnded?.resolve();
+    const ended = withResolvers();
+    this.chunkEnded = ended;
+    await this.scheduler.nextTurn(ended.promise);
+    this.stillWanted?.();
+  }
+
+  /**
+   * The launch has finished (or failed). Releases the turn still waiting on
+   * the chunk it resumed, so a launch that ends mid-chunk does not strand the
+   * handler that granted it.
+   */
+  settle(): void {
+    this.chunkEnded?.resolve();
+    this.chunkEnded = undefined;
+  }
+}
+
+function withResolvers(): { promise: Promise<void>; resolve: () => void } {
+  let resolve!: () => void;
+  const promise = new Promise<void>((r) => { resolve = r; });
+  return { promise, resolve };
+}
+
+/** What {@link LaunchTurnPump} needs from the Durable Object hosting it. */
+export interface LaunchTurnPumpHost {
+  /**
+   * Arrange for {@link LaunchTurnPump.pump} to run on a fresh Durable Object
+   * turn.
+   *
+   * The embedder satisfies this with an alarm, which is the only primitive
+   * that genuinely re-enters the object: a fresh turn is both a released
+   * thread and a fresh CPU budget, and a launch needs each for a different
+   * reason. Without it the pump degrades to a same-context timer — see
+   * {@link LaunchTurnPump.nextTurn}.
+   */
+  requestTurn?: () => void;
+  /**
+   * Awaited first on every pump, before any waiter resumes. Where the
+   * resident-launch journal's recovery sits: the pump is what an alarm calls,
+   * and the first turn after a reset is the re-delivered alarm of a launch
+   * the reset interrupted.
+   */
+  recover?: () => Promise<void>;
+}
+
+/**
+ * The granting side of {@link LaunchTurnScheduler}: parks suspended launches
+ * and resumes every one of them when the host grants a fresh turn.
+ */
+export class LaunchTurnPump implements LaunchTurnScheduler {
+  /**
+   * Launches suspended between chunks, waiting for a turn of their own.
+   *
+   * In-memory on purpose: a launch is only meaningful while the process table
+   * entry it is building for exists, and both are lost together if the isolate
+   * resets. What survives a reset is the journal, which names the launch's
+   * INPUTS rather than its position — a resumed queue would be resurrecting
+   * half-built work for pids that no longer exist, where re-driving a launch
+   * from its inputs is the same idempotent work again.
+   */
+  private waiters: Array<{ resume: () => void; chunkEnded: Promise<void> }> = [];
+
+  constructor(private readonly host: LaunchTurnPumpHost) {}
+
+  /**
+   * How a paced launch asks for a fresh turn.
+   *
+   * The host grants one by calling {@link pump} from a context that is
+   * genuinely a new invocation — the session's alarm. Without such a host
+   * there is no fresh turn to be had, and the launch continues on this one
+   * rather than hanging: that is exactly the single-turn launch this path has
+   * always performed, so a harness or a runtime without alarms loses the
+   * responsiveness but keeps the behaviour.
+   */
+  nextTurn(chunkEnded: Promise<void>): Promise<void> {
+    return new Promise<void>((resume) => {
+      this.waiters.push({ resume, chunkEnded });
+      if (this.host.requestTurn) {
+        this.host.requestTurn();
+        return;
+      }
+      setTimeout(() => { void this.pump(); }, 0);
+    });
+  }
+
+  /**
+   * Run one chunk of every launch waiting for a turn.
+   *
+   * Awaits the chunk each resumed launch then performs, so the invocation that
+   * granted the turn is the invocation that pays for the work — rather than
+   * releasing it into a handler's microtask drain, where nothing owns it and
+   * the runtime may tear the context down mid-chunk.
+   */
+  async pump(): Promise<void> {
+    await this.host.recover?.();
+    const waiting = this.waiters;
+    if (waiting.length === 0) return;
+    this.waiters = [];
+    for (const waiter of waiting) waiter.resume();
+    await Promise.all(waiting.map((waiter) => waiter.chunkEnded));
+  }
+
+  /** Whether any launch is suspended waiting for a turn. */
+  get hasPending(): boolean {
+    return this.waiters.length > 0;
+  }
+}
+
+/**
+ * Chunk bound for this session, honouring the verification knob.
+ *
+ * `NIMBUS_LAUNCH_CHUNK_BYTES` forces a small bound so an ordinary launch —
+ * not just a pathological one — crosses several turns and exercises every
+ * suspension point. Without it the multi-turn path would only ever be
+ * reached by the largest programs, which is the same reason
+ * `git/commands.ts` carries `NIMBUS_GIT_CHECKOUT_CHUNK_ENTRIES`. Unset in
+ * production, where the default applies.
+ */
+export function launchChunkMaxBytes(env: unknown): number {
+  const raw = (env as { NIMBUS_LAUNCH_CHUNK_BYTES?: string } | null | undefined)
+    ?.NIMBUS_LAUNCH_CHUNK_BYTES;
+  if (!raw) return LAUNCH_CHUNK_MAX_BYTES;
+  const parsed = Number(raw);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : LAUNCH_CHUNK_MAX_BYTES;
+}

package/src/loader-ledger.ts

@@ -0,0 +1,112 @@
+/**
+ * loader-ledger.ts — per-DO accounting for the Worker Loader's two caps.
+ *
+ * Measured on production workerd: a Durable Object admits ~5–6 concurrent
+ * dynamic workers before the platform refuses with "Too many concurrent
+ * dynamic workers", one DO method can drive at most 4 concurrent Loader
+ * fetches, and loader-cache entries are never released — every DISTINCT
+ * `loader.get(id)` permanently consumes one of the dynamic-worker slots for
+ * the object's lifetime. Nimbus stays under the caps by construction
+ * (`IN_DO_THRESHOLD` = 5 in the fanout pool), which until now meant the slots
+ * were counted in prose. This ledger counts them at the fabric's loader call
+ * sites instead — the loader pool's slots, a resident process's keyed worker,
+ * a one-shot's load — so proximity is measurable and a cap failure can name
+ * the ids actually holding slots.
+ *
+ * Measurement only: no admission control. The caps are the platform's, they
+ * are approximate ("~5–6"), and a gate on an approximate number would refuse
+ * work the platform would have run.
+ *
+ * Keyed weakly off the hosting actor's `ctx`, like the facet slot books: the
+ * caps are per Durable Object, and dynamic workers die with the isolate that
+ * loaded them, so a ledger that goes away with its host describes nothing
+ * that still exists.
+ */
+
+import { classifyError } from '@nimbus-sh/core/observability/oom-classify.js';
+
+interface LoaderLedger {
+  /** Distinct loader ids ever gotten — each one a permanently consumed slot. */
+  ids: Set<string>;
+  /** In-flight calls into dynamic workers, right now. */
+  liveFetches: number;
+  /** The most that were ever in flight at once. */
+  peakLiveFetches: number;
+}
+
+const ledgers = new WeakMap<object, LoaderLedger>();
+
+function ledger(ctx: object): LoaderLedger {
+  let entry = ledgers.get(ctx);
+  if (!entry) {
+    entry = { ids: new Set(), liveFetches: 0, peakLiveFetches: 0 };
+    ledgers.set(ctx, entry);
+  }
+  return entry;
+}
+
+/** Record a keyed `loader.get(id)` — a permanent slot if the id is new. */
+export function recordLoaderId(ctx: object, id: string): void {
+  ledger(ctx).ids.add(id);
+}
+
+/**
+ * Count one call into a dynamic worker as a live Loader fetch; the returned
+ * function ends it (idempotently), from the caller's own `finally`.
+ *
+ * A begin/end pair rather than a wrapper on purpose, and the shape is
+ * load-bearing: wrapping the stub call in a ledger-owned async frame
+ * (`trackLoaderFetch(ctx, () => entrypoint.execute(...))`) left the hosting
+ * Durable Object poisoned after every pooled dispatch — the next fabric
+ * activity hung the object or reset the instance outright (pid base jumped,
+ * every attached WebSocket dropped with no close frame), measured 7/7 on
+ * staging and gone 3/3 with the direct call restored. Same seam-quirk class
+ * as pipelined `fetch.call`, which workerd refuses for dynamically-loaded
+ * workers: an RPC stub call must stay a direct property call awaited by the
+ * frame that made it, so the ledger only brackets it.
+ */
+export function beginLoaderFetch(ctx: object): () => void {
+  const entry = ledger(ctx);
+  entry.liveFetches++;
+  entry.peakLiveFetches = Math.max(entry.peakLiveFetches, entry.liveFetches);
+  let ended = false;
+  return () => {
+    if (ended) return;
+    ended = true;
+    entry.liveFetches--;
+  };
+}
+
+/** Snapshot for the diag surface. Pure read; no I/O. */
+export function loaderLedgerStats(ctx: object): {
+  idsEverGotten: string[];
+  liveFetches: number;
+  peakLiveFetches: number;
+} {
+  const entry = ledger(ctx);
+  return {
+    idsEverGotten: [...entry.ids],
+    liveFetches: entry.liveFetches,
+    peakLiveFetches: entry.peakLiveFetches,
+  };
+}
+
+/**
+ * Name the per-DO accounting on a "Too many concurrent dynamic workers"
+ * failure; hand every other error back untouched. The platform's message
+ * says only that the cap was hit — which ids hold the slots, and that a
+ * keyed id can never give one back, is what the operator needs to know to
+ * shrink anything.
+ */
+export function withDynamicWorkerCapNamed<E>(ctx: object, error: E): E | Error {
+  if (classifyError(error) !== 'dynamic_worker_cap') return error;
+  const entry = ledger(ctx);
+  const platform = error instanceof Error ? error.message : String(error);
+  return new Error(
+    `${platform} — this Durable Object has ${entry.ids.size} loader id(s) permanently `
+      + `holding dynamic-worker slots (a loader.get id is never released): `
+      + `${[...entry.ids].join(', ') || '(none recorded)'}; live Loader fetches ${entry.liveFetches}, `
+      + `peak ${entry.peakLiveFetches}`,
+    { cause: error },
+  );
+}