@jr2/orchestrator 0.2.1 → 0.4.1

package/src/harness-client.ts

@@ -10,20 +10,23 @@
 // socket-free through an injected `fetch`.
 //
 // Channel split (ADR-0002, refined by ADR-0016): the Harness wire carries **lifecycle only** —
-// domain events go up the MCP channel via the Adapter. The client's `send`/`wait` pair is exactly
+// domain events go up the Menu, through the pod's Custodian. The client's `send`/`wait` pair is exactly
 // that lifecycle surface: `send` answers with a serializable Admission
 // (`{ streamUrl, offset, submissionId }` — jr2's durable re-attach handle, stored in the host
 // ledger), and `wait(admission)` follows the durable stream from the admission offset to the
 // Submission's Settlement — a long-poll loop that advances by the `stream-next-offset` header and
-// reconnects from the SAME offset on network failure (capped backoff, indefinitely: pod death is
-// `workspace.lost`'s job to report, not this loop's to guess). Re-attach after a restart is `wait`
+// reconnects from the SAME offset on network failure (capped backoff, indefinitely: a lost pod is
+// `workspace.lost`'s to report and a restarted Harness the Agent actor's, both off the watch —
+// ADR-0021 — never this loop's to guess). Re-attach after a restart is `wait`
 // with the SAME persisted admission; replay cost is bounded by one Submission's chunks. A 404 is a
-// LOST conversation (ADR-0027: a conversation lives as long as its Harness process) — a
-// `SettlementFault`, never an endless poll.
+// LOST conversation (ADR-0027: the Harness no longer holds it — a persisted conversation survives a
+// Harness restart, ADR-0031, and one it cannot rebuild or has freed does not) — a
+// `SettlementFault`, never an endless poll. A Submission a restart cut settles `failed` on the
+// rebuilt stream (ADR-0031), which is read like any other Settlement.
 //
 // Both verbs re-send on a network failure, for one reason stated twice: an unanswered request is
 // not an answer. They differ in bound, and the difference is where the Submission is — `wait`'s is
-// already admitted, so the lease may report a dead pod and this loop need never give up, while
+// already admitted, so the watch may report a dead pod or Harness and this loop need never give up, while
 // `send`'s does not exist yet, so its window closes and faults. `send` also re-sends ONLY what
 // provably never left this host (`postAdmission` — ADR-0042: it is the first thing ever to dial a
 // Sandbox's Service, and a CR at `phase: Ready` is not yet routable).
@@ -34,13 +37,42 @@
 // (the actor is stopped by then; see actor.ts).
 
 import { agentActorWith } from "./actor.ts";
-import type { AgentAdmission, AgentAdmitOptions, AgentLogic, AgentRunInput, AgentRunPort } from "./actor.ts";
+import type {
+  AgentAdmission,
+  AgentAdmitOptions,
+  AgentLogic,
+  AgentRunInput,
+  AgentRunPort,
+  AgentSettleOptions,
+} from "./actor.ts";
 import type { AgentDeclaration, AgentDefinition, ThinkingLevel } from "./agent.ts";
+import {
+  commandWith,
+  ExecNotSent,
+  ExecRefused,
+  type Arg,
+  type CommandInput,
+  type CommandOptions,
+  type CommandOutput,
+  type OpenCommandInput,
+  type EnvOpenParts,
+  type EffectLogic,
+  type ExecPort,
+} from "./effect.ts";
+import { open } from "./open.ts";
+import type { OpenEffect, OpenOf } from "./open-effect.ts";
 import {
   LIVE_LONG_POLL,
   STREAM_NEXT_OFFSET_HEADER,
+  VIEW_HISTORY,
   VIEW_UPDATES,
   type EchoEvent,
+  type ExecRead,
+  type ExecRequest,
+  type LiveConversation,
+  type LiveSetRequest,
+  type LiveSetResponse,
+  type Notice,
   type Settlement,
   type StreamEvent,
   type SubmissionSettledEvent,
@@ -67,12 +99,19 @@ export type HarnessClient = {
       definition: AgentDefinition;
       model?: string;
       thinkingLevel?: ThinkingLevel;
+      /** The pending notices in this admission's scope (ADR-0062). Omitted when there are none. */
+      notices?: Notice[];
       signal?: AbortSignal;
     },
   ): Promise<AgentAdmission>;
   /** Follow the stream to this Submission's Settlement: resolve on `completed`, reject with
-   * `SettlementFault` on `failed`/`aborted`/404. */
-  wait(admission: AgentAdmission, opts?: { signal?: AbortSignal }): Promise<void>;
+   * `SettlementFault` on `failed`/`aborted`/404. A guard kill read on the way is handed to
+   * `onMemoryLimit` (ADR-0061). */
+  wait(admission: AgentAdmission, opts?: AgentSettleOptions): Promise<void>;
+  /** `GET /agents/:name/:id?view=history`: does the Harness still hold this conversation? Only a
+   * 404 says no (ADR-0027: a GET on an unknown conversation is 404); anything else — another
+   * status, a transport failure — says yes, because unknown is never loss (ADR-0062). */
+  holds(agentName: string, instanceId: string, opts?: { signal?: AbortSignal }): Promise<boolean>;
   /** `POST /agents/:name/:id/abort`. The `{ aborted }` answer is dropped (ADR-0024). */
   abort(agentName: string, instanceId: string, opts?: { signal?: AbortSignal }): Promise<void>;
 };
@@ -114,10 +153,14 @@ export type HarnessClientOptions = {
 export class SettlementFault extends Error {
   /** The Settlement as the stream carried it; absent when the conversation itself was lost. */
   readonly settlement?: Settlement;
-  constructor(message: string, settlement?: Settlement) {
+  /** The conversation itself is gone (404): its Harness no longer holds it (ADR-0027). One of the
+   * faults the Agent actor asks the Sandbox about, to name a memory kill (ADR-0061). */
+  readonly lost: boolean;
+  constructor(message: string, settlement?: Settlement, opts: { lost?: boolean } = {}) {
     super(message);
     this.name = "SettlementFault";
     this.settlement = settlement;
+    this.lost = opts.lost === true;
   }
 }
 
@@ -138,11 +181,11 @@ export function createHarnessClient(options: HarnessClientOptions): HarnessClien
    * POST the admission, re-sending it while the request DEMONSTRABLY never left this host.
    *
    * This is `wait`'s reconnect rule, applied one step earlier and for the same reason: a request
-   * that could not connect is not a Harness that refused the prompt. It matters here because the
-   * admission is the FIRST thing jr2 ever sends over the Sandbox's Service — provisioning waits on
-   * the CR's `phase: Ready`, which the operator computes from the POD, and the attach reaches the
-   * pod through the API server, so nothing before this has proven the Service dialable. Ready is
-   * not routable: the EndpointSlice behind the ClusterIP is programmed after the pod passes its
+   * that could not connect is not a Harness that refused the prompt. It matters here because
+   * provisioning waits on the CR's `phase: Ready`, which the operator computes from the POD, and
+   * a Workspace-less Turn (the Instance Harness) has no attach before it — so this can be the first
+   * request ever sent over a Service. (A Sandbox's attach, `POST /attach` since ADR-0063, dials it
+   * first and retries the same way.) Ready is not routable: the EndpointSlice behind the ClusterIP is programmed after the pod passes its
    * probe, and until it is, kube-proxy REJECTs — which arrives here as a refused connection.
    * Un-retried, that single blip lost the whole turn (the actor calls an admission failure a
    * terminal `agent.fault`), which is exactly the flake that kept the `@kind` tier serial.
@@ -153,9 +196,15 @@ export function createHarnessClient(options: HarnessClientOptions): HarnessClien
    * question "did this reach the wire at all", never "does this look transient".
    *
    * And the window is BOUNDED where `wait`'s is not. `wait` may reconnect forever because its
-   * Submission is already admitted, so the lease owns the reporting (`workspace.lost` — ADR-0021).
+   * Submission is already admitted, so the watch owns the reporting (`workspace.lost`, or the
+   * Harness restart that ends the Turn — ADR-0021).
    * Nothing is admitted yet here, so there is no turn for a lease to be about: an address that
    * never answers is a fault this call has to name itself.
+   *
+   * A draining Harness's 503 is re-sent on the same ladder (ADR-0031): the Harness answers it
+   * before it reads the body, so it queued nothing, and it closes the connection with it, so the
+   * next attempt dials the Service again — which reaches the replacement once it serves. It is the
+   * one answer re-sent: every other status is the Harness's word about this admission.
    */
   const postAdmission = async (url: string, init: RequestInit, signal?: AbortSignal): Promise<Response> => {
     const startedAt = Date.now();
@@ -165,12 +214,9 @@ export function createHarnessClient(options: HarnessClientOptions): HarnessClien
     let lastCode = "?";
     for (;;) {
       attempts += 1;
+      let res: Response;
       try {
-        const res = await fetchImpl(url, init);
-        // Only when it actually cost something. A window that is never approached should be silent,
-        // so that a line appearing at all is already the signal (see `routabilityLine`).
-        if (attempts > 1) log(routabilityLine("admission", url, attempts, Date.now() - startedAt, lastCode));
-        return res;
+        res = await fetchImpl(url, init);
       } catch (err) {
         lastCode = transportCode(err);
         // A local abandon propagates untranslated, exactly as in `wait` — the stopped actor
@@ -186,7 +232,19 @@ export function createHarnessClient(options: HarnessClientOptions): HarnessClien
         }
         await sleep(jittered(backoffMs), signal);
         backoffMs = Math.min(backoffMs * 2, backoffMaxMs);
+        continue;
+      }
+      if (res.status === DRAINING && Date.now() < deadline) {
+        lastCode = `HTTP${DRAINING}`;
+        await res.body?.cancel();
+        await sleep(jittered(backoffMs), signal);
+        backoffMs = Math.min(backoffMs * 2, backoffMaxMs);
+        continue;
       }
+      // Only when it actually cost something. A window that is never approached should be silent,
+      // so that a line appearing at all is already the signal (see `routabilityLine`).
+      if (attempts > 1) log(routabilityLine("admission", url, attempts, Date.now() - startedAt, lastCode));
+      return res;
     }
   };
 
@@ -215,6 +273,9 @@ export function createHarnessClient(options: HarnessClientOptions): HarnessClien
             // Omitted when unset, so an admission with no dials is byte-identical to before.
             ...(sendOptions.model ? { model: sendOptions.model } : {}),
             ...(sendOptions.thinkingLevel ? { thinkingLevel: sendOptions.thinkingLevel } : {}),
+            // The Briefing's news (ADR-0062): typed data, the Harness writes the words. Omitted
+            // when there is none, so an admission without notices is byte-identical to before.
+            ...(sendOptions.notices?.length ? { notices: sendOptions.notices } : {}),
           }),
           signal: sendOptions.signal,
         },
@@ -247,7 +308,9 @@ export function createHarnessClient(options: HarnessClientOptions): HarnessClien
           if (res.status === 404) {
             throw new SettlementFault(
               `conversation lost: the harness answered 404 for submission "${admission.submissionId}" — ` +
-                `a conversation lives as long as its Harness process (ADR-0027)`,
+                `the Harness no longer holds this conversation (ADR-0027, ADR-0031)`,
+              undefined,
+              { lost: true },
             );
           }
           // Refused, not unreachable: the bearer is derived, so a Harness that rejects it now
@@ -276,6 +339,11 @@ export function createHarnessClient(options: HarnessClientOptions): HarnessClien
         backoffMs = backoffInitialMs;
 
         for (const event of events) {
+          if (event.type === "memory-limit") {
+            const { peak, limit, conversationId, position } = event;
+            opts?.onMemoryLimit?.({ peak, limit, source: `${conversationId}@${position.batch}.${position.index}` });
+            continue;
+          }
           if (event.type !== "submission-settled" || event.submissionId !== admission.submissionId) continue;
           if (event.outcome === "completed") return;
           throw new SettlementFault(faultMessage(event), toSettlement(event));
@@ -284,6 +352,20 @@ export function createHarnessClient(options: HarnessClientOptions): HarnessClien
       }
     },
 
+    async holds(agentName, instanceId, opts) {
+      const url = new URL(conversationUrl(agentName, instanceId));
+      url.searchParams.set("view", VIEW_HISTORY);
+      try {
+        const res = await fetchImpl(url.toString(), { headers: authorization, signal: opts?.signal });
+        // Drain the body so the socket is released; the snapshot itself is not the question.
+        await res.body?.cancel().catch(() => undefined);
+        return res.status !== 404;
+      } catch (err) {
+        if (opts?.signal?.aborted) throw err;
+        return true;
+      }
+    },
+
     async abort(agentName, instanceId, opts) {
       const res = await fetchImpl(`${conversationUrl(agentName, instanceId)}/abort`, {
         method: "POST",
@@ -324,12 +406,17 @@ export function harnessAgentRunPort(client: HarnessClient): AgentRunPort {
         definition: opts.definition,
         model: input.model,
         thinkingLevel: input.thinkingLevel,
+        ...(opts.notices?.length ? { notices: opts.notices } : {}),
         signal: opts.signal,
       });
     },
 
-    async settle(admission: AgentAdmission, opts?: { signal?: AbortSignal }): Promise<void> {
-      await client.wait(admission, { signal: opts?.signal });
+    async settle(admission: AgentAdmission, opts?: AgentSettleOptions): Promise<void> {
+      await client.wait(admission, opts);
+    },
+
+    async holds(agentName: string, instanceId: string, opts?: { signal?: AbortSignal }): Promise<boolean> {
+      return await client.holds(agentName, instanceId, opts);
     },
 
     async abort(agentName: string, instanceId: string, opts?: { signal?: AbortSignal }): Promise<void> {
@@ -386,6 +473,40 @@ export function createEchoPush(options: {
   };
 }
 
+/**
+ * Build the live-set statement for the Instance Harness (ADR-0031): `PUT /agents
+ * { live: [{ agent, instanceId }] }` → `{ freed }`, and the Harness frees memory and files of every
+ * conversation not named. The whole set every time — level-triggered, so a missed statement costs
+ * nothing and the next one says it all again. Un-retried, like the echo: the caller is a timer.
+ */
+export function createLiveSetPush(options: {
+  baseUrl: string;
+  /** The Instance Harness's bearer (ADR-0058). Absent only against the stub Harness. */
+  token?: string;
+  /** Injectable for socket-free tests. Default: global `fetch`. */
+  fetch?: typeof fetch;
+}): (live: LiveConversation[]) => Promise<LiveSetResponse> {
+  const fetchImpl = options.fetch ?? fetch;
+  return async (live) => {
+    const url = new URL("/agents", options.baseUrl).toString();
+    const res = await fetchImpl(url, {
+      method: "PUT",
+      headers: {
+        "content-type": "application/json",
+        ...(options.token ? { authorization: `Bearer ${options.token}` } : {}),
+      },
+      body: JSON.stringify({ live } satisfies LiveSetRequest),
+    }).catch((err: unknown) => {
+      throw new Error(`live-set statement to ${url} failed: ${transportDetail(err)}`, { cause: err });
+    });
+    if (!res.ok) {
+      throw new Error(`live-set statement failed (${res.status}): ${await errorDetail(res)}`);
+    }
+    const { freed } = (await res.json().catch(() => ({}))) as Partial<Record<keyof LiveSetResponse, unknown>>;
+    return { freed: typeof freed === "number" ? freed : 0 };
+  };
+}
+
 /**
  * THE authoring surface for an Agent (ADR-0049): one definition in, one actor slot out —
  * `jr2Setup({ actors: { coder: agent({ model, instructions }) } })`, invoked as `src: "coder"`.
@@ -409,6 +530,107 @@ export function agent(declaration: AgentDeclaration): AgentLogic {
   );
 }
 
+/**
+ * The Harness's exec (ADR-0067), as a `command` Effect drives it: `POST /exec/:id` to start,
+ * `GET /exec/:id` long-polled by offset, `POST /exec/:id/kill` when the Effect's state exits (sent
+ * again until the Harness keeps it), `DELETE /exec/:id` once the output is ledgered. A start that
+ * never reached the wire rejects `ExecNotSent`, the one start failure sent again: a Harness runs one
+ * id once only within its own process, so a start that reached one that then restarted would run
+ * twice. A read's transport failure or 5xx rejects, and the Effect asks again; a 404 on a read is
+ * an exec the Harness does not know (it restarted); any other 4xx is a jr2 defect, never retried.
+ */
+export function createExecPort(options: {
+  baseUrl: string;
+  /** The Workspace's Harness bearer (ADR-0058). Absent only against a Harness that checks nothing. */
+  token?: string;
+  /** Injectable for socket-free tests. Default: global `fetch`. */
+  fetch?: typeof fetch;
+}): ExecPort {
+  const fetchImpl = options.fetch ?? fetch;
+  const headers = {
+    "content-type": "application/json",
+    ...(options.token ? { authorization: `Bearer ${options.token}` } : {}),
+  };
+  const url = (id: string, suffix = "") => new URL(`/exec/${encodeURIComponent(id)}${suffix}`, options.baseUrl);
+  const call = async (target: URL, init: RequestInit): Promise<Response> =>
+    fetchImpl(target.toString(), { ...init, headers }).catch((err: unknown) => {
+      throw new Error(`harness exec ${init.method ?? "GET"} ${target.pathname} failed: ${transportDetail(err)}`, {
+        cause: err,
+      });
+    });
+  const refusedOrRetry = async (res: Response, what: string): Promise<never> => {
+    const detail = await errorDetail(res);
+    if (res.status >= 400 && res.status < 500)
+      throw new ExecRefused(`harness refused the ${what} (${res.status}): ${detail}`);
+    throw new Error(`harness ${what} failed (${res.status}): ${detail}`);
+  };
+  return {
+    start: async (id, req: ExecRequest, signal) => {
+      const res = await call(url(id), { method: "POST", body: JSON.stringify(req), signal }).catch((err: unknown) => {
+        // Only a request that never reached the wire is safe to send again (see `startOnce`).
+        if (!signal.aborted && neverDelivered(err)) throw new ExecNotSent((err as Error).message, { cause: err });
+        throw err;
+      });
+      if (!res.ok) return refusedOrRetry(res, "exec");
+      await res.json().catch(() => undefined);
+    },
+    read: async (id, offset, signal) => {
+      const target = url(id);
+      if (offset !== undefined) target.searchParams.set("offset", String(offset));
+      target.searchParams.set("live", LIVE_LONG_POLL);
+      const res = await call(target, { method: "GET", signal });
+      if (res.status === 404) {
+        await res.body?.cancel();
+        return undefined;
+      }
+      if (!res.ok) return refusedOrRetry(res, "exec read");
+      return (await res.json()) as ExecRead;
+    },
+    kill: async (id) => {
+      // Not kept until the Harness answers 2xx: the Effect sends it again until then.
+      const res = await call(url(id, "/kill"), { method: "POST" });
+      if (!res.ok) return refusedOrRetry(res, "exec kill");
+      await res.body?.cancel();
+    },
+    forget: async (id) => {
+      const res = await call(url(id), { method: "DELETE" });
+      await res.body?.cancel();
+    },
+  };
+}
+
+/**
+ * THE authoring surface for a `command` Effect (ADR-0067): `actors: { tests: command(["pnpm",
+ * "test"]) }`, invoked as `src: "tests"`. `argv` is never shell text — an `arg("x")` entry is ONE
+ * argument, filled from the invoke's input — and `sh -c` is the author's explicit choice. It runs
+ * in the Harness container of the enclosing Workspace's Sandbox, in a Repo Slot's Worktree
+ * (`slot`, needed only where the Workspace carries several); outside a Workspace it is refused.
+ * Every outcome is output: `exited | timeout | killed | unknown`.
+ */
+export function command<const Opt extends CommandOptions = CommandOptions>(
+  argv: typeof open,
+  options?: Opt,
+): EffectLogic<OpenCommandInput<Opt["pin"]>, CommandOutput> & OpenEffect<"argv" | EnvOpenParts<Opt>>;
+export function command<const A extends readonly (string | Arg)[], const Opt extends CommandOptions = CommandOptions>(
+  argv: A,
+  options?: Opt,
+): EffectLogic<CommandInput<A, Opt["pin"]>, CommandOutput> & OpenOf<EnvOpenParts<Opt>>;
+export function command(
+  argv: readonly (string | Arg)[] | typeof open,
+  options: CommandOptions = {},
+  // The implementation of the overloads above: each one states the input type.
+): EffectLogic<any, CommandOutput> {
+  // An Open argv (ADR-0054) is the composer's to bind with `customize()`; see open-effect.ts.
+  return commandWith(
+    (endpoint, bearer) => createExecPort({ baseUrl: endpoint, ...(bearer ? { token: bearer } : {}) }),
+    argv as readonly (string | Arg)[],
+    options,
+  );
+}
+
+/** What a draining Harness answers an admission (ADR-0031): refused before anything is queued. */
+const DRAINING = 503;
+
 /** A stream read worth retrying (server hiccup) — internal to the reconnect loop, never thrown out. */
 class ReconnectableError extends Error {}
 
@@ -477,7 +699,7 @@ async function errorDetail(res: Response): Promise<string> {
 
 /**
  * The marker the `@kind` tier greps for, and therefore a CONTRACT — duplicated verbatim in
- * `@jr2/adapter` (the two packages share no runtime dependency) and matched in
+ * `@jr2/harness`'s Menu (the two packages share no runtime dependency) and matched in
  * `features/steps/kind.steps.ts`. Renaming it on one side does not break a build; it silently turns
  * the tier's routability budget into a check that passes because it matches nothing.
  */