@jr2/orchestrator 0.3.0 → 0.4.1

package/src/workspace.ts

@@ -1,6 +1,6 @@
-// `workspace(body, { input, image, user, repos, spec })` (ADR-0012, ADR-0049, ADR-0051): the
-// jr2-owned wrapper Machine that owns ONLY Sandbox lifecycle — provision the Sandbox (out of the
-// STATIC `image`/`user`/`repos` options it carries) + attach one worktree per Repo Slot, run the
+// `workspace(body, { input, image, user, repos, resources, spec })` (ADR-0012, ADR-0049, ADR-0051,
+// ADR-0060): the jr2-owned wrapper Machine that owns ONLY Sandbox lifecycle — provision the Sandbox
+// (out of the STATIC `image`/`user`/`repos`/`resources` options it carries) + attach one worktree per Repo Slot, run the
 // author's body Machine inside it as the named slot `body`, with `{ workspace: { repos, branch } }`
 // appended to its input (the mechanism-facing endpoint/sandbox are published ambiently
 // — ADR-0016, ambient.ts), and destroy the Sandbox when the body reaches a final state. Teardown lives INSIDE the
@@ -14,14 +14,19 @@
 // table): one cluster per orchestrator instance, so the port rides `RunHostOptions.sandbox`,
 // never workflow code — `workspace` stays a plain static import (ADR-0011 doctrine) and unit
 // tests bind a fake port. Every port operation is invoked from a state that RE-RUNS on restore
-// (invoked actors re-execute from persisted input), so all four operations must be idempotent.
+// (invoked actors re-execute from persisted input), so every operation must be idempotent.
 //
-// Restore-reconcile (ADR-0012): the `running` state co-invokes a reconcile probe beside the
-// body. Invoked callback actors restart on every (re)entry — including snapshot restore — so
-// after an orchestrator restart the probe re-checks the Sandbox CR mechanically: present →
-// nothing (agent admissions re-attach — ADR-0016); absent → the pod-local clone and any unpushed
-// commits are gone, so it delivers `workspace.lost` INTO the restored body (same channel as
-// `agent.fault`) and the body's policy decides. Never silently re-provision.
+// The lifecycle (ADR-0012, ADR-0064): `placing` writes the Sandbox and waits, with no deadline,
+// for its pod to be scheduled; `provisioning` waits for Ready under a budget that starts there;
+// `attaching` runs the Harness's attach; `running` runs the body; `teardown` destroys.
+//
+// Restore-reconcile (ADR-0012, ADR-0021): every state from `placing` to `running` co-invokes the
+// lease, and in `running` the lease judges Continuity from the Orchestrator's watch for its
+// Sandbox (ADR-0063). Invoked callback actors restart on every (re)entry — including snapshot
+// restore — so after an orchestrator restart the watch's first list is the reconcile: present and
+// not Lost → nothing (agent admissions re-attach — ADR-0016); gone or Lost → the pod-local clone
+// and any unpushed commits are gone, so it delivers `workspace.lost` INTO the restored body (same
+// channel as `agent.fault`) and the body's policy decides. Never silently re-provision.
 
 import {
   assign,
@@ -37,6 +42,7 @@ import {
 import type { z } from "zod";
 import { registerAmbientHandles, type AmbientHandles } from "./ambient.ts";
 import {
+  actorRoute,
   actorSlotPath,
   assertRepoSlot,
   attachSandboxParts,
@@ -49,16 +55,33 @@ import {
   type JR2Repos,
   type JR2Wrapper,
   type RepoSlot,
+  type PlacementRoute,
   type SandboxParts,
   type WrapperActors,
 } from "./parts.ts";
-import { runBindingOf, type AnyActorSystem } from "./registration.ts";
+import { actorPath, runBindingOf, type AnyActorSystem } from "./registration.ts";
+import { assertSize, assertUserSeat, userSeatOf, type Size, type UserContainer } from "./size.ts";
 import { attachInputSchema, inputSchemaOf, invokingMachine, type HostInjectedInput } from "./vocabulary.ts";
 
 /** Lease cadence when the backend names none. Well inside the 30m default idle timeout, so a
- * few missed renewals in a row are survivable; also the worst-case detection latency for a
- * workspace that went away (ADR-0021). */
-const DEFAULT_LEASE_INTERVAL_MS = 5 * 60_000;
+ * few missed renewals in a row are survivable. It bounds only how long an orphan waits to be
+ * reaped: loss is the watch's news, within seconds (ADR-0021, ADR-0063). */
+export const DEFAULT_LEASE_INTERVAL_MS = 5 * 60_000;
+
+/** Each renewal lands at the interval ±20% (ADR-0021), so Leases restored together after an
+ * Orchestrator restart never renew in lockstep. */
+const LEASE_JITTER = 0.2;
+
+/**
+ * When the lease renews next (ADR-0021), for a draw `random` in [0, 1]: every renewal at the
+ * interval ±20%, and the FIRST within a fifth of it — spread, like the rest, but soon, because a
+ * restore may follow an Orchestrator that was away for a while and the idle timeout has been
+ * running since its last stamp. Exported for the tests; the lease draws `Math.random()`.
+ */
+export function leaseDelay(interval: number, which: "first" | "next", random: number): number {
+  if (which === "first") return interval * LEASE_JITTER * random;
+  return interval * (1 - LEASE_JITTER + 2 * LEASE_JITTER * random);
+}
 
 /** What to attach, in workspace vocabulary only (ADR-0012 boundary): the one branch the body
  * works on, the pod's work group, and the review sha. Derived PER RUN from the wrapper's input,
@@ -131,16 +154,52 @@ export type WorkspaceHandles<TSlots extends string> = {
 export type Workspaced<TInput, TSlots extends string> = TInput & { workspace: WorkspaceHandles<TSlots> };
 
 /**
- * What a renewal learned about the workspace it just stamped (ADR-0021).
- *
- * `identity` is the backing pod's identity, NOT its address. Addresses are deterministic — the
- * CR name, the Service DNS, the worktree paths are all derived from the run — so every name a
- * live run holds still resolves after an eviction or a node loss, while the pod behind them is
- * a replacement with an empty `work` volume: clones, worktrees, and unpushed commits gone.
- * Presence cannot see that; identity can. A backend with no such notion may omit it, and its
- * workspaces are then reconciled on presence alone.
+ * What the watch says about a workspace (ADR-0021, ADR-0063): gone, or present — and, once the
+ * operator has judged its one pod gone or terminal, `lost` with the operator's reason and the pod's
+ * own words. A Sandbox has one pod for its life, so a Lost Sandbox never comes back; the operator
+ * judges identity because it knows which pod it created, and nothing here compares pod UIDs.
+ */
+export type Continuity = { present: false } | { present: true; lost?: SandboxLoss };
+
+/** Why a Sandbox is Lost (ADR-0021): the pod's own reason (`Evicted`, `NodeShutdown`, ...) or the
+ * operator's (`PodDeleted`, `NodeLost`, `PodFailed`, `PodSucceeded`), and its message. */
+export type SandboxLoss = { reason: string; message: string };
+
+/**
+ * Why a Sandbox with no node waits (ADR-0064): the scheduler found no node for it, or a
+ * ResourceQuota refused its pod. `message` is the scheduler's or the API server's own words, which
+ * can name nodes and taints — so it rides the authenticated status and the per-run feed only.
  */
-export type Continuity = { present: false } | { present: true; identity?: string };
+export type PlacingWait = { on: "node" | "quota"; message: string };
+
+/** The Harness container's restarts on the Sandbox's one pod, and how it last ended — as the
+ * operator publishes them (ADR-0021, ADR-0063). The signal that ends the Turns a restart took. */
+export type HarnessRestarts = {
+  restartCount: number;
+  lastTerminated?: { reason?: string; exitCode?: number; finishedAt?: string };
+};
+
+/** What the port writes when a Workspace is placed (ADR-0012, ADR-0064). */
+export type PlaceRequest = {
+  name: string;
+  runId: string;
+  workflow: string;
+  /** The placement route from the registered Machine to this `workspace()` (`actorRoute`). With
+   * `workflow`, it names the pod whose Held secrets and Sandbox Effector `jr2 up` placed (ADR-0059). */
+  route: PlacementRoute;
+  image?: string;
+  user?: string;
+  workGroup?: number;
+  repos: ProvisionedRepo[];
+  resources?: Size;
+  userResources?: Size;
+  /** The Sandbox Effector's image as the `workspace()` names it (ADR-0068): the port resolves it,
+   * and adds the container only when `jr2 up` placed an Effect there. Absent → the Sandbox Image. */
+  effector?: string;
+  /** The hash of the Assembly the run holds (ADR-0069). When it is not the current one, an image
+   * the map lacks fails naming it. Absent for a run no Assembly started. */
+  assembly?: string;
+};
 
 /** One Repo Slot as the port receives it at provision (ADR-0051): resolved to a Binding, and
  * flagged when the run — not the Machine — chose the url, because that is what the credentials
@@ -149,27 +208,34 @@ export type ProvisionedRepo = { slot: string; url: string; ref?: string; perRun:
 
 /**
  * The Sandbox backend a host supplies (`RunHostOptions.sandbox`) — the seam between the
- * workspace Machine and the cluster. All four operations MUST be idempotent: the invoking
+ * workspace Machine and the cluster. Every operation MUST be idempotent: the invoking
  * states re-run on snapshot restore (create-if-absent, attach-if-absent, delete-if-present).
  */
 export interface SandboxPort {
-  /** Ensure the Sandbox CR exists (labeled with its run for `jr2 ls`) and await `phase: Ready`;
-   * resolve with the Harness endpoint the orchestrator can reach, and the identity the lease
-   * will hold this workspace to. `image`/`user` are the wrapper's static image options
-   * (ADR-0037/0005/0049) — a `file:` context or a registry ref, the port resolves both, and a
-   * context the last converge did not build fails here rather than converge-time. `repos` are the
-   * wrapper's slots, resolved, in declaration order (ADR-0051): the port names each Repo on the
-   * CR so the cluster mounts its cache, and refuses a per-run url no `git.credentials` entry
-   * admits. `workGroup` is the pod's `fsGroup`; the port owns the default. */
-  provision(req: {
-    name: string;
-    runId: string;
-    workflow: string;
-    image?: string;
-    user?: string;
-    workGroup?: number;
-    repos: ProvisionedRepo[];
-  }): Promise<{ endpoint: string; identity?: string }>;
+  /**
+   * Write the Sandbox (its token Secret, then the CR, labeled with its run for `jr2 ls`) and wait,
+   * with NO deadline, until the operator says its pod is scheduled (ADR-0064). `image`/`user` are
+   * the wrapper's static image options (ADR-0037/0005/0049) — a `file:` context or a registry ref,
+   * the port resolves both, and a context the last converge did not build fails here rather than
+   * converge-time. `repos` are the wrapper's slots, resolved, in declaration order (ADR-0051): the
+   * port names each Repo on the CR so the cluster mounts its cache, and refuses a per-run url no
+   * `git.credentials` entry admits. `workGroup` is the pod's `fsGroup`; the port owns the default.
+   * `resources` is the Size the Machine states and `userResources` the User Container's split of it
+   * (ADR-0060), both as stated — the port resolves the rest of the chain and splits the Size inside
+   * the pod.
+   *
+   * While the pod has no node, `onWait` hears why, each time the reason changes. A Sandbox that is
+   * Lost, or deleted, before it is placed rejects. `signal` ends the wait: the promise rejects once
+   * any write in flight has settled, so a caller that deletes after it never races the apply.
+   */
+  place(req: PlaceRequest, opts: { onWait: (wait: PlacingWait) => void; signal?: AbortSignal }): Promise<void>;
+  /**
+   * Wait for the placed Sandbox's `phase: Ready` under the port's budget, which starts NOW — the
+   * pod is scheduled, so it measures only what jr2 controls: the image pull, the Harness start, the
+   * Repos (ADR-0064). Resolves with the Harness endpoint the orchestrator can reach. Lost, or gone,
+   * rejects naming why.
+   */
+  provision(name: string): Promise<{ endpoint: string }>;
   /** Post-Ready attach (ADR-0004): per slot, `git clone --shared` off the node's read-only
    * cache (the `default/` checkout), then a branch worktree sibling — and, with `spec.reviewSha`, the detached
    * review worktree (ADR-0028). Resolves with the worktree paths by slot; `stale` names the slots
@@ -185,22 +251,53 @@ export interface SandboxPort {
     stale?: Record<string, string>;
   }>;
   /**
-   * Renew this workspace's keepalive lease AND report what the renewal found — one exchange,
-   * because it is one question: is the thing I am keeping alive still the thing I attached to?
-   * Nothing in the cluster represents a run (ADR-0001), so the lease is how the Orchestrator
-   * asserts liveness; the answer is how it learns the truth. Idempotent, called on a timer.
-   *
-   * A renewal that FAILS must reject, not resolve `{present: false}` — an unreachable API server
-   * is "unknown", and fabricating loss would settle a live run holding real work.
+   * Renew this workspace's keepalive lease — a WRITE, and only that (ADR-0021, ADR-0063). Nothing
+   * in the cluster represents a run (ADR-0001), so the lease is the Orchestrator's assertion
+   * that lets the operator reap an orphan and no one else. It learns nothing: Continuity is the
+   * watch's. Idempotent, called on a jittered timer; a failure rejects and the caller shrugs —
+   * the next renewal is minutes away and the idle timeout is thirty.
+   */
+  renew(name: string): Promise<void>;
+  /**
+   * Hear this workspace's Continuity from the watch (ADR-0021, ADR-0063): once the watch has
+   * listed, what it holds now, then again on every change the cluster reports — a Sandbox gone, or
+   * one the operator has made Lost. A dropped watch reports NOTHING: unknown is never loss, and the
+   * listener never hears a fabricated `{present: false}`. Returns the unsubscribe.
+   */
+  continuity(name: string, listener: (seen: Continuity) => void): () => void;
+  /**
+   * Hear the Harness container's restarts on this Sandbox's pod (ADR-0021): once the watch has
+   * listed, then on every change. A Turn records the count at its admission, and a count above it
+   * means the conversation it waits on is gone. Silent for a name the watch does not hold — the
+   * Instance Harness is a StatefulSet, not a Sandbox. Returns the unsubscribe.
+   */
+  harnessRestarts(name: string, listener: (seen: HarnessRestarts) => void): () => void;
+  /**
+   * Name a memory kill (ADR-0061): when the Harness container of this Sandbox last ended
+   * `OOMKilled` at or after `since`, the kill — its fault reason, starting with the fixed prefix
+   * `memory limit`, and the data the next Turn's notice is made of (ADR-0062) — else undefined.
+   * Asked when a Turn faults on a lost conversation; it may wait a moment for the operator's word,
+   * which can trail the Harness's restart. A backend that cannot see the container answers
+   * undefined, and every lost conversation stays lost — never an optional method (ADR-0021).
    */
-  renew(name: string): Promise<Continuity>;
+  memoryFault(name: string, since: Date): Promise<MemoryKill | undefined>;
   /** Delete the Sandbox CR. Absent is success. */
   destroy(name: string): Promise<void>;
   /** How often to renew. Must be well inside the backend's idle-timeout, since a lapsed lease is
-   * what lets the operator reap. Also the detection latency for a lost workspace. */
+   * what lets the operator reap. */
   readonly leaseIntervalMs?: number;
 }
 
+/** A kernel memory kill of a Sandbox's Harness container (ADR-0061, layer 5). */
+export type MemoryKill = {
+  /** The fault reason, starting with the fixed prefix `memory limit`. */
+  reason: string;
+  /** The limit the kernel enforced, as the Sandbox states it (`1920Mi`). */
+  limit?: string;
+  /** When the container ended — which kill this was, so every Turn it ended names the same one. */
+  at?: string;
+};
+
 /** Resolve the host's Sandbox backend, failing with a pointed message on a host without one. */
 export function sandboxOf(system: AnyActorSystem): SandboxPort {
   const port = runBindingOf(system).sandbox;
@@ -215,7 +312,7 @@ export function sandboxOf(system: AnyActorSystem): SandboxPort {
 
 /**
  * The Sandbox CR name for one workspace invocation: DNS-1123, deterministic from the run and
- * the wrapper's actor id (both stable across restore — that is what lets the reconcile probe
+ * the wrapper's actor id (both stable across restore — that is what lets the lease
  * and a re-run provision find the SAME CR), collision-proofed by a content suffix.
  */
 export function workspaceName(runId: string, wsId: string): string {
@@ -239,6 +336,17 @@ type MechanismHandles = AmbientHandles;
  * mapper is not re-evaluated against an input that may since have been re-parsed. */
 export type ResolvedBinding = { url: string; ref?: string; perRun: boolean };
 
+/**
+ * A Workspace's wait for capacity as its context holds it (ADR-0064): the reason as the port last
+ * heard it, and when the wait began. `since` survives a changed reason and a restart — it is how
+ * long the Workspace has waited, which is what the feed's closing line reports.
+ */
+export type WorkspaceWait = PlacingWait & { since: string };
+
+/** The context key the wait rides under — what `RunStatus.waiting` reads off a Workspace's
+ * snapshot, live or persisted. */
+export const WORKSPACE_WAIT_KEY = "wait";
+
 type WsContext = {
   /** The wrapper's own input, passed through to the body untouched (plus `workspace`). */
   runInput: Record<string, unknown>;
@@ -249,10 +357,9 @@ type WsContext = {
   /** Every slot's resolved Binding, by slot, in declaration order — assigned at provision. */
   bindings?: Record<string, ResolvedBinding>;
   endpoint?: string;
-  /** The pod identity this workspace attached to, captured at provision and persisted so the
-   * lease can hold the workspace to it across a restart (ADR-0021). Plain serializable data,
-   * exactly like `endpoint` — ADR-0007's rule about what may ride context. */
-  identity?: string;
+  /** Why the Sandbox waits for a node, while `placing` (ADR-0064) — persisted, so a restart keeps
+   * when the wait began and `RunStatus.waiting` reads it off the snapshot. Cleared once placed. */
+  wait?: WorkspaceWait;
   /** Persisted in context so the registrar can re-publish them on restore. */
   handles?: MechanismHandles;
   output?: unknown;
@@ -281,7 +388,7 @@ export type WorkspaceMachine<TInput, TOutput, TBody extends AnyStateMachine, TSl
   any,
   any,
   any,
-  WrapperActors<"body", TBody, "provision" | "attach" | "registrar" | "lease" | "destroy">,
+  WrapperActors<"body", TBody, "place" | "provision" | "attach" | "registrar" | "lease" | "destroy">,
   any,
   any,
   any,
@@ -389,11 +496,21 @@ type BodyAcceptsSlots<TBody extends AnyStateMachine, TSlots extends string> =
 export type SandboxOptions<TSlots extends string, TInput = unknown> = {
   /** The Sandbox Image. Absent → the Instance's `images/default`, then the stock Harness. */
   image?: string;
-  /** The User Container's image (ADR-0005). Absent → the pod has no third container: there is no
-   * default, because the seat's whole identity is "what jr2 does not own" and jr2 has nothing to put
-   * there. One string is the entire authoring surface — env, ports, and resources are deliberately
-   * not forwarded. */
-  user?: string;
+  /** The User Container (ADR-0005): its image, or `{ image, resources }` when the Machine states
+   * the seat's split of the Size (ADR-0060) — the widening ADR-0005 foresaw. Absent → the pod has
+   * no third container: there is no default, because the seat's whole identity is "what jr2 does
+   * not own" and jr2 has nothing to put there. Env and ports are deliberately not forwarded. With
+   * no split stated, the seat has no limit of its own and shares the pod's budget. */
+  user?: string | UserContainer;
+  /**
+   * The Size (ADR-0060): the ceiling for the whole Sandbox, `limits.memory` and `limits.cpu` and
+   * nothing else — Agent Substrate's template size, so the Workspace moves there unchanged. A fact
+   * of the Machine, like its image: the author knows the toolchain, the runner, the worker count.
+   * jr2 reserves all of it (every request equals its limit) and enforces the CPU limit. Absent, or
+   * a field left out → the Instance's `sandbox.resources`, then the kit default (2Gi, 1 cpu). A
+   * composer retunes it with `customize(machine, { resources })`.
+   */
+  resources?: Size;
   /**
    * The Repo Slots (ADR-0051), keyed by the Machine's own word for each — the key of the body's
    * `workspace.repos` handles and the directory under `/work`. The handles keep this map's order,
@@ -407,6 +524,22 @@ export type SandboxOptions<TSlots extends string, TInput = unknown> = {
    * `task`, whose first slot is the one the coder edits). Under it `TSlots` is `string`, and the body's handles must say so.
    */
   repos: Record<TSlots, RepoSlot<TInput>> | typeof open;
+  /**
+   * The Held secrets this Workspace's pod gets (ADR-0059): names of `jr2.config.ts`'s `secrets`
+   * list. Its Custodian holds each one, and the Harness container sees its Stand-in. Naming is
+   * placement: a pod gets the Held secrets something in it names — this list, and the model keys
+   * of the Agents whose Turns run here, which their providers claim without being listed.
+   */
+  secrets?: readonly string[];
+  /**
+   * The Sandbox Effector's image (ADR-0068, ADR-0005): the container that runs this Workspace's
+   * `push`, file-body `http` and credentialed `command` Effects, with the secrets they name. Absent → the
+   * Sandbox Image, whose floor already holds `git` (ADR-0037). Name one for the tools and the
+   * credentials kept out of the Agent's image (`terraform`, `aws`). A `file:` context or a
+   * registry ref, resolved like the User Container's image; it runs as its own `USER`. The
+   * container exists only when an Effect below this `workspace()` is placed there.
+   */
+  effector?: { image: string };
 };
 
 /**
@@ -477,16 +610,18 @@ export function workspace(
   }
   // Static, so checkable NOW rather than at the first provision — an empty or non-string image is
   // the same derives-from-a-typo bug `assertSpec` catches for the spec, one build earlier.
-  for (const seat of ["image", "user"] as const) {
-    const value = options[seat];
-    if (value !== undefined && (typeof value !== "string" || !value)) {
-      throw new Error(
-        `workspace(): \`${seat}\` must be a non-empty string (got ${JSON.stringify(value)}) — either a \`file:\` ` +
-          'URL to a docker context this module ships (`import.meta.resolve("./image")`) or a registry ref ' +
-          "(ADR-0037).",
-      );
-    }
+  const image = options.image;
+  if (image !== undefined && (typeof image !== "string" || !image)) {
+    throw new Error(
+      `workspace(): \`image\` must be a non-empty string (got ${JSON.stringify(image)}) — either a \`file:\` ` +
+        'URL to a docker context this module ships (`import.meta.resolve("./image")`) or a registry ref ' +
+        "(ADR-0037).",
+    );
   }
+  if (options.user !== undefined) assertUserSeat("workspace()", options.user);
+  // The Size, refused by field where it is written (ADR-0060): only the two limits. A request, or
+  // any key Substrate would refuse, is a dependency the move to it would drop.
+  if (options.resources !== undefined) assertSize("workspace(): resources", options.resources);
   // The slots, checked NOW for the same reason (ADR-0051): every value is one of the three forms,
   // every key is a directory name, and there is at least one — a Workspace exists to work on a
   // repository, and a wrapper with no slot would attach nothing and hand the body no checkout.
@@ -502,6 +637,33 @@ export function workspace(
     }
     for (const [slot, value] of Object.entries(repos)) assertRepoSlot("workspace()", slot, value);
   }
+  // The Held secrets it names (ADR-0059), checked NOW: each a name, each once. Whether the list
+  // holds the name is `jr2 up`'s to say — the config is not here.
+  const secrets = options.secrets;
+  if (secrets !== undefined) {
+    const names = Array.isArray(secrets) ? (secrets as unknown[]) : undefined;
+    const bad = names?.find((n) => typeof n !== "string" || !n);
+    if (names === undefined || bad !== undefined || new Set(names).size !== names.length) {
+      throw new Error(
+        `workspace(): \`secrets\` must be a list of distinct names from jr2.config.ts's \`secrets\` list (got ` +
+          `${JSON.stringify(secrets)}) — secrets: ["JIRA_TOKEN"] (ADR-0059).`,
+      );
+    }
+  }
+  const effector = options.effector;
+  if (
+    effector !== undefined &&
+    (typeof effector !== "object" ||
+      effector === null ||
+      typeof effector.image !== "string" ||
+      !effector.image ||
+      Object.keys(effector).some((k) => k !== "image"))
+  ) {
+    throw new Error(
+      `workspace(): \`effector\` is { image } — a \`file:\` URL to a docker context or a registry ref (got ` +
+        `${JSON.stringify(effector)}) — effector: { image: import.meta.resolve("./effector") } (ADR-0068).`,
+    );
+  }
   const wrapper = buildWorkspaceMachine(body, options.spec);
   // The wrapper is TRANSPARENT to its body (ADR-0049): `customize(machine, { agents })` on a
   // Workspace means the Machine inside, so the composer never spells `body` and never has to know
@@ -516,6 +678,9 @@ export function workspace(
     ...(options.image !== undefined ? { image: options.image } : {}),
     ...(options.user !== undefined ? { user: options.user } : {}),
     repos: repos === open ? open : { ...repos },
+    ...(options.resources !== undefined ? { resources: options.resources } : {}),
+    ...(secrets !== undefined ? { secrets: [...secrets] } : {}),
+    ...(effector !== undefined ? { effector: { image: effector.image } } : {}),
   });
   // The body's vocabulary stays the BODY's (ADR-0011, ADR-0049): the wrapper declares no events
   // of its own and merges none, because the actors that use the body's names resolve against the
@@ -624,10 +789,30 @@ function attachedRepos(bindings: Record<string, ResolvedBinding>): Array<{ slot:
 }
 
 function buildWorkspaceMachine(body: AnyStateMachine, spec: (args: { input: any }) => WorkspaceSpec): AnyStateMachine {
-  const provision = fromPromise<
-    { endpoint: string; identity?: string; bindings: Record<string, ResolvedBinding> },
+  /**
+   * `placing` (ADR-0064): write the Sandbox and wait, with no deadline, for its pod to be
+   * scheduled. jr2 cannot tell "full now" from "never fits", so a bound is the Machine's own
+   * `after` on the state that holds the Workspace.
+   *
+   * The wait is said, never sent to the body: each reason the port hears is assigned into this
+   * wrapper's context (`workspace.waiting`, where `RunStatus.waiting` reads it), and the feed gets
+   * two lines — the wait starting, with its reason, and the wait ending ("placed after …"). A
+   * changed reason updates the context and adds no line. A restart re-enters here: the apply is
+   * idempotent, the watch's first list answers at once, and `since` is read back off the
+   * wrapper's context, so neither line is said twice.
+   *
+   * Delete what never ran: stopped before its pod has a node — a parent `after`, a cancelled run, a
+   * run faulted elsewhere — this Sandbox is deleted at once, because nothing on it can be inspected
+   * and left alone it could take a node later for nobody. xstate's stop is synchronous, so the
+   * delete rides the abort, after any write in flight has settled. A wait that fails on its own —
+   * the Sandbox Lost before its node, a write that half landed — is the same end: the delete comes
+   * before the fault, and the fault carries the reason. The host stopping the run for a restart is
+   * not an end: the restored run re-enters `placing` and keeps its place.
+   */
+  const place = fromPromise<
+    { bindings: Record<string, ResolvedBinding> },
     { wsId: string; spec: WorkspaceSpec; runInput: unknown }
-  >(async ({ input, self, system }) => {
+  >(async ({ input, self, system, signal }) => {
     assertSpec(input.spec); // before the port: a bad spec must never cost a pod
     const binding = runBindingOf(system);
     // The images and the slots come off the WRAPPER, at invoke time, not out of context and not
@@ -637,31 +822,118 @@ function buildWorkspaceMachine(body: AnyStateMachine, spec: (args: { input: any
     // content-addressed tag, which would outlive the image it names. The RESOLVED bindings are
     // persisted, because a per-run mapper's answer is this run's fact.
     const parts = sandboxPartsOf(invokingMachine(self));
-    const bindings = resolveBindings(
-      { workflow: binding.workflow, path: actorSlotPath(self._parent) },
-      parts.repos,
-      input.runInput,
+    const user = userSeatOf(parts.user);
+    const path = actorSlotPath(self._parent);
+    // Read before the port: a Workspace no walk can see placed nothing, and is refused with the slot form.
+    const route = actorRoute(self._parent);
+    const bindings = resolveBindings({ workflow: binding.workflow, path }, parts.repos, input.runInput);
+    const port = sandboxOf(system);
+    const name = workspaceName(binding.runId, input.wsId);
+    const wrapper = self._parent;
+    const child = wrapper ? actorPath(wrapper).join("/") : "";
+    // A wait opened before a restart is the same wait: its start is in the wrapper's persisted
+    // context, never in this actor's input, which a restore replays as it was first invoked.
+    const opened = (): string | undefined =>
+      ((wrapper?.getSnapshot() as { context?: WsContext } | undefined)?.context?.wait as WorkspaceWait | undefined)
+        ?.since;
+    let since: string | undefined;
+    /** When THIS actor opened the wait, on the monotonic clock: `after` is a duration, and two
+     * wall-clock reads go negative when the host's clock steps back between them. */
+    let openedAt: number | undefined;
+    let heard: PlacingWait | undefined;
+    const onWait = (wait: PlacingWait) => {
+      if (signal.aborted || (heard?.on === wait.on && heard.message === wait.message)) return;
+      heard = wait;
+      since ??= opened();
+      if (since === undefined) {
+        since = new Date().toISOString();
+        openedAt = performance.now();
+        binding.marker?.({ kind: "placing", child, on: wait.on, message: wait.message, since });
+      }
+      wrapper?.send({ type: "workspace.waiting", wait: { ...wait, since } });
+    };
+    const placed = port.place(
+      {
+        name,
+        runId: binding.runId,
+        workflow: binding.workflow,
+        route,
+        // The image strings straight through (ADR-0037/0005) — the port owns resolution, and the
+        // work group's default (ADR-0005 puts it in pod composition, where the pod is built).
+        ...(parts.image !== undefined ? { image: parts.image } : {}),
+        ...(user.image !== undefined ? { user: user.image } : {}),
+        ...(input.spec.workGroup !== undefined ? { workGroup: input.spec.workGroup } : {}),
+        repos: Object.entries(bindings).map(([slot, b]) => ({ slot, ...b })),
+        // The Size as the Machine states it, re-read here like the images (ADR-0060): a redeploy
+        // that retuned it reaches the next provision, and no snapshot holds it.
+        ...(parts.resources !== undefined ? { resources: parts.resources } : {}),
+        ...(user.resources !== undefined ? { userResources: user.resources } : {}),
+        ...(parts.effector !== undefined ? { effector: parts.effector.image } : {}),
+        ...(binding.assembly !== undefined ? { assembly: binding.assembly } : {}),
+      },
+      { onWait, signal },
     );
-    const provisioned = await sandboxOf(system).provision({
-      name: workspaceName(binding.runId, input.wsId),
-      runId: binding.runId,
-      workflow: binding.workflow,
-      // The image strings straight through (ADR-0037/0005) — the port owns resolution, and the
-      // work group's default (ADR-0005 puts it in pod composition, where the pod is built).
-      ...(parts.image !== undefined ? { image: parts.image } : {}),
-      ...(parts.user !== undefined ? { user: parts.user } : {}),
-      ...(input.spec.workGroup !== undefined ? { workGroup: input.spec.workGroup } : {}),
-      repos: Object.entries(bindings).map(([slot, b]) => ({ slot, ...b })),
-    });
-    return { ...provisioned, bindings };
+    signal.addEventListener(
+      "abort",
+      () => {
+        if (binding.hostStopping) return;
+        void placed
+          .catch(() => {})
+          .then(() => port.destroy(name))
+          .catch(() => {}); // a delete that fails leaves the CR to the idle GC, as ever
+      },
+      { once: true },
+    );
+    try {
+      await placed;
+    } catch (err) {
+      // An abort is the listener's to delete; a failure of the wait itself is deleted here.
+      if (!signal.aborted && !binding.hostStopping) await port.destroy(name).catch(() => {});
+      throw err;
+    }
+    since ??= opened();
+    if (since !== undefined) {
+      // A wait opened before a restart has only its stored wall-clock start, so its duration is two
+      // wall-clock reads; it is clamped at zero, as a clock that stepped back since the start would
+      // make it negative.
+      const after =
+        openedAt !== undefined ? Math.round(performance.now() - openedAt) : Math.max(0, Date.now() - Date.parse(since));
+      binding.marker?.({ kind: "placed", child, since, after });
+    }
+    return { bindings };
   });
 
+  /** `provisioning`: the placed pod's Ready, under the port's budget, which starts here (ADR-0064). */
+  const provision = fromPromise<{ endpoint: string }, { wsId: string }>(async ({ input, system }) =>
+    sandboxOf(system).provision(workspaceName(runBindingOf(system).runId, input.wsId)),
+  );
+
   const attach = fromPromise<
     { repos: Record<string, string>; review?: Record<string, string>; stale?: Record<string, string> },
     { wsId: string; spec: WorkspaceSpec; bindings: Record<string, ResolvedBinding> }
-  >(async ({ input, system }) => {
+  >(async ({ input, system, signal }) => {
+    const port = sandboxOf(system);
     const name = workspaceName(runBindingOf(system).runId, input.wsId);
-    const out = await sandboxOf(system).attach({ name, spec: input.spec, repos: attachedRepos(input.bindings) });
+    // The attach is loss-aware (ADR-0021): a Sandbox gone or Lost under it fails the attach at
+    // once and by name, as it fails the placing and the provisioning — never a transport error
+    // one attach window later. Only the cluster's word: a dropped watch says nothing.
+    let unsubscribe = (): void => {};
+    const loss = new Promise<never>((_, reject) => {
+      unsubscribe = port.continuity(name, (seen) => {
+        const lost = !seen.present ? GONE : seen.lost;
+        if (lost === undefined) return;
+        const what = seen.present ? "is Lost" : "is gone";
+        reject(new Error(`Sandbox "${name}" ${what} while it attached: ${lost.reason}: ${lost.message} (ADR-0021)`));
+      });
+    });
+    loss.catch(() => {});
+    signal.addEventListener("abort", () => unsubscribe(), { once: true });
+    let out: Awaited<ReturnType<typeof port.attach>>;
+    try {
+      out = await Promise.race([port.attach({ name, spec: input.spec, repos: attachedRepos(input.bindings) }), loss]);
+    } finally {
+      unsubscribe();
+    }
     // Announced, never persisted (ADR-0051): a stale cache is a degraded attach the run proceeds
     // through on the objects the node holds. The pod cannot heal it — the worktree's `origin`
     // fetches from the cache, not the remote (ADR-0005: the pod holds no credential) — so stale
@@ -696,65 +968,92 @@ function buildWorkspaceMachine(body: AnyStateMachine, spec: (args: { input: any
   });
 
   /**
-   * The lease (ADR-0021). One actor owns the whole exchange with the cluster for one workspace:
-   * it asserts liveness (nothing in the cluster represents a run, so the Orchestrator must keep
-   * saying "still mine" or the operator's idle GC reaps — ADR-0001) and, in the same call, reads
-   * back whether what it just stamped is still what the body attached to.
+   * The lease (ADR-0021). One actor owns both halves of the exchange with the cluster for one
+   * workspace: it ASSERTS liveness with a write (nothing in the cluster represents a run, so the
+   * Orchestrator must keep saying "still mine" or the operator's idle GC reaps — ADR-0001), and it
+   * LISTENS to the watch for whether what the body attached to is still there (ADR-0063).
    *
-   * Being an INVOKED actor is the whole design. Its lifetime IS `running`'s lifetime, which
-   * xstate already manages: it re-invokes on snapshot restore (so a restart reconciles for free,
-   * with no restore-specific code path), and it stops on every exit — body final, run stopped,
-   * run faulted. That last one is why there is no `release()`: a faulted run stops its actors,
-   * the lease stops with them, and the abandoned pod ages out of the idle timeout on its own.
+   * Being an INVOKED actor is the whole design. It is invoked beside the work of every state from
+   * `placing` to `running`, so the renewal covers the Sandbox from its write to its teardown — a
+   * Sandbox that waits for capacity is never reaped as abandoned (ADR-0064) — and there is one at
+   * a time, because a state's invocations stop when it exits. It re-invokes on snapshot restore (so
+   * a restart reconciles for free: the watch's first list is the first thing it hears), and it
+   * stops on every exit — body final, run stopped, run faulted. That last one is why there is no
+   * `release()`: a faulted run stops its actors, the lease stops with them, and the abandoned pod
+   * ages out of the idle timeout on its own.
    *
-   * Level-triggered on purpose. The one-shot probe this replaces could only fire on entry, so a
-   * run parked on a gate for hours — the state most likely to outlive its Sandbox — never
-   * rechecked anything until the next restart.
+   * Continuity is JUDGED only in `running`, once there is an attached pod: before that, a Sandbox
+   * that is Lost or gone fails the placing, the provisioning or the attach, by name, on its own. The two halves
+   * share no call: loss arrives within seconds of the cluster saying so; the renewal is a merge
+   * patch every interval ±20%, whose only job is to keep the Sandbox from being reaped.
    */
-  const lease = fromCallback<{ type: string }, { wsId: string; identity?: string }>(({ input, system, sendBack }) => {
-    const port = sandboxOf(system);
-    const name = workspaceName(runBindingOf(system).runId, input.wsId);
-    let stopped = false;
-
-    const renew = async (): Promise<void> => {
-      let seen: Continuity;
-      try {
-        seen = await port.renew(name);
-      } catch {
-        return; // unknown, never lost: an API blip must not settle a run holding real work
-      }
-      if (stopped) return;
-      // Two ways to lose a workspace, one event. Gone: reaped, deleted, namespace cleared.
-      // Replaced: the CR survived an eviction or node loss but the pod behind it did not, so
-      // every name still resolves over an empty `work` volume. Re-provisioning either silently
-      // would resume into an inconsistent world — the body decides (ADR-0012).
-      const replaced =
-        seen.present && input.identity !== undefined && seen.identity !== undefined
-          ? seen.identity !== input.identity
-          : false;
-      if (!seen.present || replaced) sendBack({ type: "workspace.lost" });
-    };
+  const lease = fromCallback<{ type: string }, { wsId: string; judging: boolean }>(
+    ({ input, self, system, sendBack }) => {
+      const port = sandboxOf(system);
+      const name = workspaceName(runBindingOf(system).runId, input.wsId);
+      let stopped = false;
+      let lost = false;
 
-    void renew(); // immediately on entry: this is the restore-reconcile, no longer a special case
-    const timer = setInterval(() => void renew(), port.leaseIntervalMs ?? DEFAULT_LEASE_INTERVAL_MS);
-    timer.unref?.(); // a lease never holds the process open; it matters only while the run runs
-    return () => {
-      stopped = true;
-      clearInterval(timer);
-    };
-  });
+      const unsubscribe = input.judging
+        ? port.continuity(name, (seen) => {
+            if (stopped || lost) return;
+            // Two ways to lose a workspace, one event. Gone: reaped, deleted, namespace cleared. Lost:
+            // the CR survived but its one pod did not — evicted, its node lost or shut down — so
+            // every name still resolves over an empty `work` volume, and the operator never gives it
+            // a second pod. Re-provisioning either silently would resume into an inconsistent world
+            // — the body decides (ADR-0012).
+            const loss: SandboxLoss | undefined = !seen.present ? GONE : seen.lost;
+            if (loss !== undefined) {
+              lost = true; // once: the body has been told, and the workspace does not come back
+              sendBack({ type: "workspace.lost", reason: loss.reason, message: loss.message });
+            }
+          })
+        : () => {};
+
+      // The assertion, jittered from the first renewal on (see `leaseDelay`).
+      const interval = port.leaseIntervalMs ?? DEFAULT_LEASE_INTERVAL_MS;
+      let timer: ReturnType<typeof setTimeout>;
+      const schedule = (ms: number) => {
+        timer = setTimeout(() => {
+          // A Workspace that ended in error stopped processing without stopping its children (xstate
+          // does not), and a parent that handled the error only stops what still processes: the
+          // lease asks, and asserts nothing for a Workspace that is over.
+          if (stopped || self._parent?.getSnapshot().status !== "active") return;
+          // A failed renewal is unknown, never lost: the next one is an interval away, well inside
+          // the idle timeout, and loss is the watch's to report. Before the CR's write lands it is a
+          // 404, and the same shrug.
+          port.renew(name).catch(() => {});
+          schedule(leaseDelay(interval, "next", Math.random()));
+        }, ms);
+        timer.unref?.(); // a lease never holds the process open; it matters only while the run runs
+      };
+      schedule(leaseDelay(interval, "first", Math.random()));
+      return () => {
+        stopped = true;
+        clearTimeout(timer);
+        unsubscribe();
+      };
+    },
+  );
 
   const destroy = fromPromise<void, { wsId: string }>(async ({ input, system }) =>
     sandboxOf(system).destroy(workspaceName(runBindingOf(system).runId, input.wsId)),
   );
 
+  /** The lease beside a state's own work — renewing everywhere, judging in `running` alone. */
+  const leaseInvoke = (judging: boolean) => ({
+    id: "lease",
+    src: "lease" as const,
+    input: ({ context }: { context: unknown }) => ({ wsId: (context as WsContext).wsId, judging }),
+  });
+
   // Every actor this wrapper runs is a NAMED SLOT (ADR-0049), the body first among them: a Machine
   // composes by invoking a declared `src`, and `body` is what `provide()`, `customize()`, Stately,
-  // the Console's join key and the `jr2 up` parts walk all reach it by. The mechanism's own four —
-  // provision, attach, registrar, lease, destroy — are named for the same price, and the Console
-  // now shows what each state is doing instead of "inline".
+  // the Console's join key and the `jr2 up` parts walk all reach it by. The mechanism's own six —
+  // place, provision, attach, registrar, lease, destroy — are named for the same price, and the
+  // Console shows what each state is doing instead of "inline".
   return setup({
-    actors: { body, provision, attach, registrar, lease, destroy },
+    actors: { body, place, provision, attach, registrar, lease, destroy },
   }).createMachine({
     id: "workspace",
     context: ({ input, self }: { input: unknown; self: { id: string } }): WsContext => ({
@@ -762,60 +1061,95 @@ function buildWorkspaceMachine(body: AnyStateMachine, spec: (args: { input: any
       wsId: self.id,
       spec: spec({ input }),
     }),
-    initial: "provisioning",
+    initial: "placing",
     states: {
-      provisioning: {
-        invoke: {
-          src: "provision",
-          input: ({ context }) => ({
-            wsId: (context as unknown as WsContext).wsId,
-            spec: (context as unknown as WsContext).spec,
-            runInput: (context as unknown as WsContext).runInput,
-          }),
-          onDone: {
-            target: "attaching",
-            actions: assign({
-              endpoint: ({ event }) => (event as unknown as { output: { endpoint: string } }).output.endpoint,
-              identity: ({ event }) => (event as unknown as { output: { identity?: string } }).output.identity,
-              bindings: ({ event }) =>
-                (event as unknown as { output: { bindings: Record<string, ResolvedBinding> } }).output.bindings,
+      placing: {
+        invoke: [
+          {
+            id: "place",
+            src: "place",
+            input: ({ context }) => ({
+              wsId: (context as unknown as WsContext).wsId,
+              spec: (context as unknown as WsContext).spec,
+              runInput: (context as unknown as WsContext).runInput,
             }),
+            onDone: {
+              target: "provisioning",
+              actions: assign({
+                bindings: ({ event }) =>
+                  (event as unknown as { output: { bindings: Record<string, ResolvedBinding> } }).output.bindings,
+                wait: undefined,
+              }),
+            },
+          },
+          leaseInvoke(false),
+        ],
+        // The wait's reason, into context and nowhere else: scheduling detail is not the body's
+        // concern (ADR-0064), and a bound is the parent's `after`.
+        on: {
+          "workspace.waiting": {
+            actions: assign({ wait: ({ event }) => (event as unknown as { wait: WorkspaceWait }).wait }),
           },
         },
       },
+      provisioning: {
+        invoke: [
+          {
+            id: "provision",
+            src: "provision",
+            input: ({ context }) => ({ wsId: (context as unknown as WsContext).wsId }),
+            onDone: {
+              target: "attaching",
+              actions: assign({
+                endpoint: ({ event }) => (event as unknown as { output: { endpoint: string } }).output.endpoint,
+              }),
+            },
+          },
+          leaseInvoke(false),
+        ],
+      },
       attaching: {
-        invoke: {
-          src: "attach",
-          input: ({ context }) => ({
-            wsId: (context as unknown as WsContext).wsId,
-            spec: (context as unknown as WsContext).spec,
-            bindings: (context as unknown as WsContext).bindings!,
-          }),
-          onDone: {
-            target: "running",
-            actions: assign({
-              handles: ({ context, event, system }): MechanismHandles => {
-                const ctx = context as WsContext;
-                const out = (
-                  event as unknown as {
-                    output: { repos: Record<string, string>; review?: Record<string, string> };
-                  }
-                ).output;
-                return {
-                  endpoint: ctx.endpoint!,
-                  // Derived, not remembered: the same function every port operation names the CR
-                  // with, so the Sandbox the registrar publishes — and the Agent actor records on its
-                  // registration — is the one the Custodian's token is scoped to, by construction
-                  // (ADR-0013).
-                  sandbox: workspaceName(runBindingOf(system as AnyActorSystem).runId, ctx.wsId),
-                  repos: out.repos,
-                  branch: ctx.spec.branch,
-                  ...(out.review ? { review: out.review } : {}),
-                };
-              },
+        invoke: [
+          {
+            id: "attach",
+            src: "attach",
+            input: ({ context }) => ({
+              wsId: (context as unknown as WsContext).wsId,
+              spec: (context as unknown as WsContext).spec,
+              bindings: (context as unknown as WsContext).bindings!,
             }),
+            onDone: {
+              target: "running",
+              actions: assign({
+                handles: ({ context, event, system }): MechanismHandles => {
+                  const ctx = context as WsContext;
+                  const out = (
+                    event as unknown as {
+                      output: { repos: Record<string, string>; review?: Record<string, string> };
+                    }
+                  ).output;
+                  return {
+                    endpoint: ctx.endpoint!,
+                    // Derived, not remembered: the same function every port operation names the CR
+                    // with, so the Sandbox the registrar publishes — and the Agent actor records on its
+                    // registration — is the one the Custodian's token is scoped to, by construction
+                    // (ADR-0013).
+                    sandbox: workspaceName(runBindingOf(system as AnyActorSystem).runId, ctx.wsId),
+                    repos: out.repos,
+                    // Each slot's Binding url: where a push goes, and the Repo cache its import
+                    // borrows from (ADR-0051, ADR-0067).
+                    ...(ctx.bindings
+                      ? { bindings: Object.fromEntries(Object.entries(ctx.bindings).map(([s, b]) => [s, b.url])) }
+                      : {}),
+                    branch: ctx.spec.branch,
+                    ...(out.review ? { review: out.review } : {}),
+                  };
+                },
+              }),
+            },
           },
-        },
+          leaseInvoke(false),
+        ],
       },
       running: {
         invoke: [
@@ -844,17 +1178,11 @@ function buildWorkspaceMachine(body: AnyStateMachine, spec: (args: { input: any
               actions: assign({ output: ({ event }) => (event as unknown as { output: unknown }).output }),
             },
           },
-          {
-            id: "lease",
-            src: "lease",
-            input: ({ context }) => ({
-              wsId: (context as unknown as WsContext).wsId,
-              identity: (context as unknown as WsContext).identity,
-            }),
-          },
+          leaseInvoke(true),
         ],
-        // The wrapper emits, the body decides (ADR-0012): forward loss into the body's policy.
-        on: { "workspace.lost": { actions: sendTo("body", { type: "workspace.lost" }) } },
+        // The wrapper emits, the body decides (ADR-0012): forward loss, with its reason, into the
+        // body's policy.
+        on: { "workspace.lost": { actions: sendTo("body", ({ event }) => event) } },
       },
       teardown: {
         invoke: {
@@ -872,3 +1200,10 @@ function buildWorkspaceMachine(body: AnyStateMachine, spec: (args: { input: any
     output: ({ context }) => (context as unknown as WsContext).output,
   });
 }
+
+/** A Sandbox the watch says is gone — reaped by the idle GC, deleted by hand, its namespace
+ * cleared — as the loss `workspace.lost` carries. */
+const GONE: SandboxLoss = {
+  reason: "Deleted",
+  message: "the Sandbox is gone: deleted, or reaped by the operator's idle GC (ADR-0001)",
+};