@nimbus-sh/fabric 0.1.0 → 0.3.0

package/src/do-calls.ts

@@ -0,0 +1,156 @@
+/**
+ * do-calls.ts — the two verbs for calling another Durable Object, split by
+ * the one property that decides whether a retry is safe.
+ *
+ * Both consumers asked for this. Proteus hand-wrote the retry
+ * (`cf-backend/src/lib/do-rpc.ts`) with the rule its header states:
+ * "An operation that appends, sends, charges or mints is never wrapped: a
+ * dropped call there may already have run, so a retry is a correctness bug
+ * wearing resilience as a costume." agent-core has no retry machinery at all
+ * and its backlog calls the gap "the most production-proven gap in the
+ * corpus". Here the rule is a type: `idempotent` retries, `mutating` cannot.
+ *
+ * What the platform contract requires, and this keeps:
+ *   - a FRESH stub per attempt. Cloudflare documents that many exceptions
+ *     leave a stub permanently broken, so both verbs take a stub RESOLVER,
+ *     not a stub — which is also what lets placement pins and auth wrappers
+ *     compose (agent-core's PlacementResolver pins an Actor to one
+ *     jurisdiction for life; the resolver seam is where that lives).
+ *   - `overloaded` is never retried, by either verb: retrying an overloaded
+ *     object is what overloaded it.
+ *   - attempts and backoff are the consumer-proven bounds: 3 attempts total,
+ *     full-jitter delays in [0, 2**attempt * 60ms).
+ *
+ * The resolver MINTS a stub per call and the verb disposes each one it
+ * minted — that ownership is what makes the fresh-stub retry real.
+ *
+ * The stub method call itself happens inside the CALLER's closure
+ * (`(stub) => stub.method(args)`): nothing here proxies property resolution
+ * or dispatches by method name. Do NOT use these verbs around a dynamically
+ * loaded worker's entrypoint stub — those calls must stay direct property
+ * calls bracketed by `beginLoaderFetch` (budgets.ts records the 7/7 staging
+ * poisoning that rule comes from). These verbs are for Durable Object
+ * namespace stubs, where the thunk shape is production-proven in Proteus.
+ */
+
+import { classifyDoCall, isRetryableDoCall, type DoCallClass } from '@nimbus-sh/platform/oom-classify.js';
+import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
+
+/** Total attempts. Two retries is what a dropped connection or a deploy
+ *  bounce needs; beyond that the object is not coming back inside this
+ *  request (the consumer's measured bound). */
+const MAX_ATTEMPTS = 3;
+/** Full-jitter base, in the shape the Agents SDK itself uses. */
+const BASE_DELAY_MS = 60;
+
+export interface DoCallRetryPolicy {
+  maxAttempts?: number;
+  baseDelayMs?: number;
+  /**
+   * Called once per retry, before its backoff delay, with the failure the
+   * retry is answering. The consumer's logging seam: Proteus's hand-rolled
+   * predecessor logged every retry so a flaky object is visible in Workers
+   * Logs rather than silently absorbed, and `operation` names it there.
+   */
+  onRetry?(info: DoCallRetryInfo): void;
+}
+
+/** What one retry is answering: which call, which platform class, which
+ *  attempt just failed out of how many. */
+export interface DoCallRetryInfo {
+  operation: string;
+  classification: DoCallClass;
+  /** The 1-based attempt that failed; the retry about to run is attempt+1. */
+  attempt: number;
+  maxAttempts: number;
+  error: unknown;
+}
+
+/**
+ * Mints one stub per call. `idempotent` calls it once per attempt.
+ *
+ * MINT means mint. Both verbs dispose the stub they were handed when the
+ * call settles — on success as much as on failure (`disposeRpcResource`).
+ * A resolver that returns a shared, long-lived stub hands its other users
+ * a disposed stub, and only in production: a test double is a plain object
+ * with nothing to dispose, so the test stays green while the deployed
+ * Worker breaks on the second call.
+ */
+export type DoStubResolver<S> = () => S | Promise<S>;
+
+/**
+ * A failed `mutating` call, typed so the caller can act on WHAT failed:
+ * `classification` names the platform condition, and a transient class on a
+ * mutating call means the call may already have run — the indeterminacy the
+ * consumer's rule exists to surface rather than paper over.
+ */
+export class DoCallError extends Error {
+  constructor(
+    readonly operation: string,
+    readonly verb: 'idempotent' | 'mutating',
+    readonly classification: DoCallClass,
+    cause: unknown,
+  ) {
+    const text = cause instanceof Error ? cause.message : String(cause);
+    const indeterminate = isRetryableDoCall(classification)
+      ? ' — a dropped mutating call may already have run, so it is not retried'
+      : '';
+    super(`${verb} call '${operation}' failed [${classification}]: ${text}${indeterminate}`, { cause });
+    this.name = 'DoCallError';
+  }
+}
+
+/**
+ * Call another Durable Object with an operation that is safe to repeat: a
+ * read, or a converge-to-a-value write. Transient failures retry on a fresh
+ * stub with full-jitter backoff; overloaded and permanent failures surface
+ * unchanged, as does the last error at exhaustion.
+ */
+export async function idempotent<S, T>(
+  operation: string,
+  stub: DoStubResolver<S>,
+  call: (stub: S) => Promise<T>,
+  policy: DoCallRetryPolicy = {},
+): Promise<T> {
+  const maxAttempts = policy.maxAttempts ?? MAX_ATTEMPTS;
+  const baseDelayMs = policy.baseDelayMs ?? BASE_DELAY_MS;
+  for (let attempt = 1; ; attempt++) {
+    const minted = await stub();
+    try {
+      const result = await call(minted);
+      disposeRpcResource(minted);
+      return result;
+    } catch (error) {
+      // A stub that threw may be permanently broken; it is never reused.
+      disposeRpcResource(minted);
+      const classification = classifyDoCall(error);
+      if (!isRetryableDoCall(classification) || attempt >= maxAttempts) throw error;
+      policy.onRetry?.({ operation, classification, attempt, maxAttempts, error });
+      await new Promise<void>((resolve) => {
+        setTimeout(resolve, Math.floor(Math.random() * 2 ** attempt * baseDelayMs));
+      });
+    }
+  }
+}
+
+/**
+ * Call another Durable Object with an operation that appends, sends, charges
+ * or mints. NEVER retried — a dropped call may already have run. Failure
+ * surfaces as a {@link DoCallError} carrying the classification, so the
+ * caller can tell a refusal from an indeterminate drop.
+ */
+export async function mutating<S, T>(
+  operation: string,
+  stub: DoStubResolver<S>,
+  call: (stub: S) => Promise<T>,
+): Promise<T> {
+  const minted = await stub();
+  try {
+    const result = await call(minted);
+    disposeRpcResource(minted);
+    return result;
+  } catch (error) {
+    disposeRpcResource(minted);
+    throw new DoCallError(operation, 'mutating', classifyDoCall(error), error);
+  }
+}

package/src/facet-pool.ts

@@ -0,0 +1,157 @@
+/**
+ * facet-pool.ts — leased facets, so reclaiming storage is the default and
+ * leaking it takes intent.
+ *
+ * Proteus's facet-spawn.ts (313 lines) exists because the platform's two
+ * teardown verbs are indistinguishable to a caller and only one gives
+ * storage back: `abort` is mid-flight eviction with storage KEPT, `delete`
+ * is terminal with storage WIPED. Its docstring records the cost of
+ * confusing them: "the leak this module previously had, in which every head
+ * and every MCTS branch abandoned a permanent database inside the
+ * orchestrator DO". The lease makes that leak unreachable: disposal retires
+ * the facet (evict, then wipe), and keeping storage is the explicit opt-in
+ * (`detach()`, today's abort).
+ *
+ * The constraints a caller must not be surprised by, from Proteus's platform
+ * catalog (all proven by probe or by source):
+ *   - a facet cannot set alarms (`do.facet.no_alarms`) — a head cannot
+ *     schedule its own resumption; everything time-driven routes through the
+ *     root's single alarm (`timers`).
+ *   - a facet stub is coordinator-local (`do.facet.stub_local`) — it cannot
+ *     be transferred, stored, or re-invoked indirectly.
+ *   - facet storage is charged to the ROOT's shared budget, and a clone that
+ *     crosses it is an uncatchable reset, not an error (`do.storage.bytes`).
+ *   - a parent and its facets are evicted JOINTLY after minutes idle, so
+ *     in-memory facet state is never safe to assume between two RPCs.
+ *   - 65,536 facet ids per DO lifetime (`do.facet.count`), append-only and
+ *     never reclaimed — the binding constraint for the leak, reached an
+ *     order of magnitude before the byte quota. The pool counts first-use
+ *     names in the durable ledger (budgets.ts) and refuses a NEW name at the
+ *     wall by name, instead of letting the platform fail opaquely. Refusal
+ *     at the wall is exact, not a threshold: the ledger never overcounts.
+ *
+ * A failed reclaim stays loud (facet-spawn's `runOnceAndReclaim`): storage
+ * that was not given back is a permanent charge against the root's quota,
+ * and swallowing that is how the original leak stayed invisible.
+ *
+ * The pool drives the RAW `ctx.facets` container and assumes it is the only
+ * thing naming facets on this actor. The Agents SDK's sub-agent layer makes
+ * the same assumption from the other side — it owns facet naming and runs
+ * its own cleanup — so the two are mutually exclusive on one actor:
+ * whichever acts second aborts or retires facets the other still tracks,
+ * and the facet-id ledger here counts only the names this pool minted.
+ */
+
+import {
+  FACET_ID_LIFETIME_BUDGET,
+  facetNameCount,
+  recordFacetNameMinted,
+  withFacetBudgetNamed,
+} from './budgets.js';
+
+/** `ctx.facets`, as the pool drives it — same surface the facet host uses. */
+export interface FacetPoolContainer {
+  get(name: string, start: () => Promise<{ class: unknown }>): unknown;
+  abort(name: string, reason?: unknown): void;
+  delete(name: string): void;
+}
+
+/** The hosting actor's context: its facet container, and the storage the
+ *  facet-id ledger persists through. */
+export interface FacetPoolContext {
+  facets?: FacetPoolContainer;
+  storage: {
+    get(key: string): Promise<unknown> | unknown;
+    put(key: string, value: unknown): Promise<void>;
+  };
+}
+
+/**
+ * One leased facet. Dispose (or `retire()`) evicts the instance and WIPES
+ * its storage; `detach()` first to keep the storage — after it, disposal
+ * only evicts. The stub is coordinator-local: do not store it past the turn
+ * or hand it to anything else.
+ */
+export interface FacetLease<S> {
+  readonly name: string;
+  readonly stub: S;
+  /** Keep the facet's storage: disposal becomes eviction only. */
+  detach(): void;
+  /** Idempotent. Throws, loudly, when the platform refuses the wipe. */
+  retire(): Promise<void>;
+  [Symbol.asyncDispose](): Promise<void>;
+}
+
+/** Names this ctx's pool has already charged to the lifetime ledger. */
+const chargedNames = new WeakMap<object, Set<string>>();
+
+/** The facet pool of one hosting actor. Cheap accessor, like `timers()`. */
+export function facetPool(ctx: FacetPoolContext): FacetPool {
+  return new FacetPool(ctx);
+}
+
+export class FacetPool {
+  constructor(private readonly ctx: FacetPoolContext) {}
+
+  /**
+   * Open (or re-enter) the named facet under a lease. A first-use name
+   * consumes one of the object's 65,536 lifetime facet ids and is refused at
+   * the wall; a reused name costs nothing, in this incarnation or any other.
+   */
+  async acquire<S = unknown>(
+    name: string,
+    start: () => Promise<{ class: unknown }>,
+  ): Promise<FacetLease<S>> {
+    const facets = this.ctx.facets;
+    if (!facets || typeof facets.get !== 'function') {
+      throw new Error('fabric: ctx.facets is unavailable in this Durable Object; facets cannot be leased');
+    }
+    let charged = chargedNames.get(this.ctx);
+    if (!charged) {
+      charged = new Set();
+      chargedNames.set(this.ctx, charged);
+    }
+    if (!charged.has(name)) {
+      const consumed = facetNameCount(this.ctx);
+      if (consumed >= FACET_ID_LIFETIME_BUDGET) {
+        throw withFacetBudgetNamed(
+          consumed,
+          new Error(`facet '${name}' refused before creation: no lifetime ids remain`),
+        );
+      }
+      recordFacetNameMinted(this.ctx, consumed + 1);
+      charged.add(name);
+    }
+    const stub = facets.get(name, start) as S;
+    let settled = false;
+    let keepStorage = false;
+    const retire = async (): Promise<void> => {
+      if (settled) return;
+      settled = true;
+      // Evict first so the wipe never lands under a live writer; a facet
+      // already gone makes the abort a no-op.
+      try { facets.abort(name, new Error('fabric: facet lease retired')); } catch { /* already gone */ }
+      if (keepStorage) return;
+      try {
+        facets.delete(name);
+      } catch (e) {
+        throw new Error(
+          `fabric: facet '${name}' was evicted but its storage was not reclaimed — `
+            + `it is leaked into the root Durable Object's shared quota: ${errorText(e)}`,
+          { cause: e },
+        );
+      }
+    };
+    return {
+      name,
+      stub,
+      detach(): void { keepStorage = true; },
+      retire,
+      [Symbol.asyncDispose]: retire,
+    };
+  }
+}
+
+function errorText(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/{fanout-pool.ts → fanout.ts}

@@ -4,7 +4,7 @@
  *
  * A single Durable Object method can drive at most four concurrent
  * Worker Loader fetches before extra dispatches serialize or fail. Small
- * batches therefore run in the coordinator DO through LoaderPool.
+ * batches therefore run in the coordinator DO through IsolatePool.
  * Wider batches are sharded across sibling NimbusSession DOs, each of
  * which owns its own four-loader budget.
  *
@@ -16,9 +16,9 @@
 
 import { serializeFunction } from './vendor/serialize.js';
 import { BindingError } from './vendor/errors.js';
-import { LoaderPool, type FacetTaskFn } from './loader-pool.js';
-import { disposeRpcResource } from '@nimbus-sh/core/_shared/rpc-dispose.js';
-import { describeError, isDoOverloaded, isTransientDoReset } from '@nimbus-sh/core/observability/oom-classify.js';
+import { IsolatePool, type FacetTaskFn } from './isolate-pool.js';
+import { disposeRpcResource } from '@nimbus-sh/platform/rpc-dispose.js';
+import { describeError, isDoOverloaded, isTransientDoReset } from '@nimbus-sh/platform/oom-classify.js';
 import type { WorkerLoader } from './vendor/types.js';
 
 /**
@@ -31,7 +31,7 @@ interface PeerSessionNamespace {
 }
 
 /** The bindings a fan-out needs off the coordinator DO's env. */
-export interface FanoutPoolEnv {
+export interface FanoutEnv {
   LOADER?: WorkerLoader;
   NIMBUS_SESSION?: PeerSessionNamespace;
 }
@@ -104,45 +104,45 @@ export interface FanoutTask<A> {
   args: A;
 }
 
-/** Options handed to FanoutPool's constructor. */
-export interface FanoutPoolOptions {
+/** Options handed to Fanout's constructor. */
+export interface FanoutOptions {
   /**
    * Tag prepended to peer-DO ids and in-DO loader ids for debugging
    * (e.g. "npm-install-batch"). Affects neither isolate identity (in-DO
-   * path uses the existing LoaderPool's tag-fold) nor peer-DO
+   * path uses the existing IsolatePool's tag-fold) nor peer-DO
    * deterministic placement (peer ids fold tag + key).
    */
   tag: string;
   /**
    * Per-task timeout in ms. Default 60_000. Forwarded to the in-DO
-   * LoaderPool's submit calls and to the peer-DO RPC's own
-   * LoaderPool.
+   * IsolatePool's submit calls and to the peer-DO RPC's own
+   * IsolatePool.
    */
   timeoutMs?: number;
   /**
    * Preamble bundled into every facet (in-DO and inside each peer
-   * DO). Same semantics as LoaderPool's preamble option.
+   * DO). Same semantics as IsolatePool's preamble option.
    */
   preamble?: string;
   /**
    * Wasm modules forwarded to every facet. Same semantics as
-   * LoaderPool's wasmModules option.
+   * IsolatePool's wasmModules option.
    */
   wasmModules?: Record<string, ArrayBuffer>;
   /**
    * Extra bindings forwarded to every facet. Same semantics as
-   * LoaderPool's extraBindings option.
+   * IsolatePool's extraBindings option.
    */
   extraBindings?: Record<string, unknown>;
   /**
    * If set, skip the supervisor-RPC binding injection (mirrors
-   * LoaderPool's omitSupervisor flag).
+   * IsolatePool's omitSupervisor flag).
    */
   omitSupervisor?: boolean;
   /**
    * Invoking process pid, baked into each facet's SUPERVISOR binding so
    * filesystem RPCs (writeBatchStream) are authorized under the caller's
-   * credential (mirrors LoaderPool's supervisorPid). Threaded to both
+   * credential (mirrors IsolatePool's supervisorPid). Threaded to both
    * the in-DO loader pool and, via `_rpcFanoutExecute`, the peer-DO pools.
    * npm install passes the shell command's `ctx.pid`; resolve leaves it 0.
    */
@@ -186,10 +186,10 @@ function isFanoutPeerStub(value: unknown): value is FanoutPeerStub {
 
 function fanoutPeerStub(value: unknown): FanoutPeerStub {
   if ((typeof value !== 'object' && typeof value !== 'function') || value === null) {
-    throw new BindingError('FanoutPool: NIMBUS_SESSION.get() did not return a peer stub.');
+    throw new BindingError('Fanout: NIMBUS_SESSION.get() did not return a peer stub.');
   }
   if (!isFanoutPeerStub(value)) {
-    throw new BindingError('FanoutPool: peer stub does not expose _rpcFanoutExecute().');
+    throw new BindingError('Fanout: peer stub does not expose _rpcFanoutExecute().');
   }
   return value;
 }
@@ -203,23 +203,23 @@ function fanoutPeerStub(value: unknown): FanoutPeerStub {
  * exists primarily as a clean API surface; per-call dispatch state
  * lives only inside submitMany's promise.
  */
-export class FanoutPool {
-  private readonly env: FanoutPoolEnv;
+export class Fanout {
+  private readonly env: FanoutEnv;
   private readonly ctx: DurableObjectState;
-  private readonly opts: FanoutPoolOptions;
+  private readonly opts: FanoutOptions;
   private readonly coordDoId: string;
   private readonly coordDoIdShort: string;
 
-  constructor(rawEnv: unknown, ctx: DurableObjectState, opts: FanoutPoolOptions) {
+  constructor(rawEnv: unknown, ctx: DurableObjectState, opts: FanoutOptions) {
     // A host hands its whole env over; the bindings are claimed here and the
     // LOADER claim is checked immediately. Hard-fail on a missing LOADER —
-    // LoaderPool also enforces this, but checking up front points the
-    // diagnostic at the fanout-pool construction site rather than the
-    // deferred loader-pool one.
-    const env = (rawEnv as FanoutPoolEnv | null | undefined) ?? {};
+    // IsolatePool also enforces this, but checking up front points the
+    // diagnostic at the fanout construction site rather than the
+    // deferred isolate-pool one.
+    const env = (rawEnv as FanoutEnv | null | undefined) ?? {};
     if (!env.LOADER || typeof env.LOADER.get !== 'function') {
       throw new BindingError(
-        'FanoutPool: env.LOADER binding missing or invalid. ' +
+        'Fanout: env.LOADER binding missing or invalid. ' +
           'Add a [[worker_loaders]] entry to wrangler.jsonc.',
       );
     }
@@ -235,21 +235,21 @@ export class FanoutPool {
    * results in input order.
    *
    * Routing:
-   *   tasks.length < 5   -> coordinator-local LoaderPool
+   *   tasks.length < 5   -> coordinator-local IsolatePool
    *   tasks.length >= 5  -> sibling NimbusSession DOs
    *
    * Backpressure: if `tasks.length > MAX_PEER_FANOUT (32)`, tasks
    * are sharded modulo `MAX_PEER_FANOUT` and each shard's bucket
    * runs serially inside its assigned peer DO via the in-peer
-   * LoaderPool's concurrency (capped at 4 there too). A
+   * IsolatePool's concurrency (capped at 4 there too). A
    * single submitMany call returns when ALL tasks complete (or any
    * throws).
    *
    * `fn` is the user function executed per task. It runs INSIDE a
    * Worker Loader isolate (in the in-DO path) or inside a peer DO's
    * Worker Loader isolate (in the peer-DO path); same trust posture
-   * as LoaderPool.submit. The function is serialized via
-   * the vendored serializeFunction (same as LoaderPool#prepare).
+   * as IsolatePool.submit. The function is serialized via
+   * the vendored serializeFunction (same as IsolatePool#prepare).
    */
   async submitMany<A, R>(
     tasks: FanoutTask<A>[],
@@ -287,12 +287,12 @@ export class FanoutPool {
     tasks: FanoutTask<A>[],
     fn: FacetTaskFn<A, R>,
   ): Promise<R[]> {
-    // Use the existing LoaderPool. Concurrency = task count
+    // Use the existing IsolatePool. Concurrency = task count
     // (capped at 4 by constructor — tasks.length is already < 5
     // here, so the cap won't bite). Each task = one pool.submit;
     // pool.map runs them with stable-slot reuse.
     const concurrency = Math.min(tasks.length, IN_DO_THRESHOLD - 1);
-    const pool = new LoaderPool(this.env, this.ctx, {
+    const pool = new IsolatePool(this.env, this.ctx, {
       concurrency,
       timeoutMs: this.opts.timeoutMs,
       tag: this.opts.tag,
@@ -326,7 +326,7 @@ export class FanoutPool {
     const ns = this.env?.NIMBUS_SESSION;
     if (!ns || typeof ns.idFromName !== 'function' || typeof ns.get !== 'function') {
       throw new BindingError(
-        'FanoutPool: env.NIMBUS_SESSION binding missing or invalid. ' +
+        'Fanout: env.NIMBUS_SESSION binding missing or invalid. ' +
           'The peer-DO topology requires it. ' +
           'Add the binding via durable_objects.bindings in wrangler.jsonc.',
       );
@@ -340,7 +340,7 @@ export class FanoutPool {
 
     // Cap peer count at MAX_PEER_FANOUT. Tasks beyond N=32 are
     // bucketed into existing shards — each shard's peer DO then
-    // runs its bucket through its in-DO LoaderPool.map
+    // runs its bucket through its in-DO IsolatePool.map
     // (concurrency capped at 4 there).
     const peerCount = Math.min(tasks.length, this.opts.maxPeers ?? MAX_PEER_FANOUT);
     // Group tasks by deterministic shard. Same key → same shard, so
@@ -397,7 +397,7 @@ export class FanoutPool {
                 extraBindings: this.opts.extraBindings,
                 omitSupervisor: this.opts.omitSupervisor,
                 // INSTALL-HONESTY: forward the COORDINATOR's full doId so
-                // the peer's LoaderPool can mint a SUPERVISOR
+                // the peer's IsolatePool can mint a SUPERVISOR
                 // binding that routes back HERE (the user's session DO),
                 // not to the peer DO itself. Without this, peer DOs'
                 // env.SUPERVISOR.writeBatch / writeBatchStream / stdout /