@agentswf/engine 0.1.0-beta.4 → 0.1.0

package/src/workflow-testing/host.ts

@@ -1,4 +1,4 @@
-import type { SandboxRecord } from "@agentswf/contract/records";
+import type { SandboxRecord, TurnRecord } from "@agentswf/contract/records";
 import {
   type MessageResponse,
   REPLY_CODES,
@@ -16,8 +16,10 @@ import {
   type Effort,
   type JsonObject,
   type KeepPane,
-  type PaneLayout,
+  type PaneSpec,
+  type PaneWorkspace,
   placementOf,
+  positiveDurationMilliseconds,
 } from "@agentswf/contract/workflow";
 import {
   createPlacementHostFactory,
@@ -37,28 +39,28 @@ import {
   type FakeFork,
   type FakeSet,
 } from "@agentswf/harness/testing";
-import { submitControl, submitResult, submitWaiting } from "../agent-cli/client";
+import { submitControl } from "../agent-cli/client";
 import {
   type Message,
   type MessageOutcome,
   type MessageScripts,
+  outcomeOf,
   type ReceivedMessage,
-  reply,
   type Scripts,
   type SendOptions,
   type Step,
+  type TestTurnOutcome,
   type ToolOutcome,
   type Turn,
-  type TurnOutcome,
 } from "./script";
 
 /**
  * A turn as the workflow wrote it, and how it ended. `messages` are those it was shown, absent when
  * none were.
  */
-export type TurnRecord = Omit<Turn, "signal" | "messages" | "send" | "reply" | "tool"> & {
+export type TestTurn = Omit<Turn, "signal" | "messages" | "send" | "reply" | "tool"> & {
   messages?: readonly Message[];
-  outcome: TurnOutcome;
+  outcome: TestTurnOutcome;
 };
 
 /** A compaction as the workflow asked for it, and how it ended. */
@@ -69,7 +71,7 @@ export type CompactionRecord = {
   id: string;
   /** The spec's prompt: what the harness is to keep and drop. */
   focus: string;
-  outcome: TurnOutcome;
+  outcome: TestTurnOutcome;
 };
 
 /** A switch of an agent's model or effort, as the host was asked for it. */
@@ -87,7 +89,6 @@ export type OpenedAgent = {
   key: string;
   execution: AgentExecution;
   instructions?: string;
-  labels?: JsonObject;
   /** The skills it was given by name; absent when the workflow named none. */
   skills?: readonly string[];
   /** The sandbox it ran in; absent on the host. */
@@ -98,28 +99,37 @@ export type OpenedAgent = {
    */
   forkedFrom?: { key: string; turns: number } | { caller: true; turns: number };
   /**
-   * Its pane's layout as the workflow wrote it; this host has no screen, so nothing says where a
-   * pane landed or whether a `beside` fell back.
+   * Its `pane` as the host was asked to place it: its `tab`, or `beside` another, and `front`. This
+   * host has no screen, so nothing says where a pane landed or whether a `beside` fell back.
    */
-  layout?: PaneLayout;
-  /** Why the engine already knew `layout` could not be used: a `beside` an agent with no pane. */
-  layoutFallback?: string;
-  keepPane?: KeepPane;
-  /** Its pane was kept when it was done: `keepPane` asked, and how its last operation ended let it. */
+  pane?: PaneSpec;
+  /**
+   * The run's `panes` as the host was asked to apply them to this pane: the `workspace` its tab
+   * opens in, which a `beside` takes from the pane it splits, and whether it is `keep`.
+   */
+  panes?: { workspace?: PaneWorkspace; keep?: KeepPane };
+  /** Why the engine already knew `pane` could not be used: a `beside` an agent with no pane. */
+  paneFallback?: string;
+  /** Its pane was kept when it was done: `panes.keep` asked, and its last operation's end let it. */
   keptPane?: true;
 };
 
 /** The sandbox an agent ran in, as the run's record keeps it. */
 export type AgentSandbox = Pick<SandboxRecord, "key" | "provider" | "spec" | "domains">;
 
-export type ScriptedHost = {
+type ScriptedHost = {
   factory: AgentRunHostFactory;
-  turns: TurnRecord[];
+  turns: TestTurn[];
   compactions: CompactionRecord[];
   sets: SetRecord[];
   agents: OpenedAgent[];
   /** Turns started and not yet ended, for a stalled run's message. */
-  inFlight(): TurnRecord[];
+  inFlight(): TestTurn[];
+  /**
+   * Once the run has ended, gives each operation's last turn, and each compaction, the outcome the
+   * engine recorded for it in `turns`: a turn before that, a nudge's or a wait's, keeps its own.
+   */
+  settle(recorded: readonly TurnRecord[]): void;
 };
 
 /**
@@ -138,7 +148,7 @@ export function createScriptedHost(
     /** The script could not meet a turn: the test fails. */
     onScriptError(message: string): void;
     /** A turn began, before its script answers: what a test sends during it goes now. */
-    onTurnStarted?(turn: TurnRecord): Promise<void>;
+    onTurnStarted?(turn: TestTurn): Promise<void>;
   },
   /**
    * The session the run is started from, as `awf run --here` finds one, and the model its files
@@ -146,12 +156,14 @@ export function createScriptedHost(
    */
   caller?: { harness: string; cwd: string; model?: string; here?: boolean },
 ): ScriptedHost {
-  const turns: TurnRecord[] = [];
+  const turns: TestTurn[] = [];
   const compactions: CompactionRecord[] = [];
   const compactionCounts = new Map<string, number>();
   const sets: SetRecord[] = [];
   const agents: OpenedAgent[] = [];
-  const open = new Set<TurnRecord>();
+  const open = new Set<TestTurn>();
+  /** Each turn's operation, which the engine's record of it names. */
+  const operationOf = new Map<TestTurn, string>();
   const counts = new Map<string, number>();
   /** Each fork's session, by the agent it copied and that agent's turns so far. */
   const forks = new Map<string, NonNullable<OpenedAgent["forkedFrom"]>>();
@@ -172,7 +184,7 @@ export function createScriptedHost(
               turns: callerKey === undefined ? 0 : (counts.get(callerKey) ?? 0),
             }),
         }));
-  const end = (record: TurnRecord, outcome: TurnOutcome) => {
+  const end = (record: TestTurn, outcome: TestTurnOutcome) => {
     record.outcome = outcome;
     open.delete(record);
     events.onActivity();
@@ -206,13 +218,17 @@ export function createScriptedHost(
       failTest(why);
       throw new Error(why);
     }
+    const timeoutMs =
+      options.timeout == null
+        ? undefined
+        : positiveDurationMilliseconds(options.timeout, `a ${command}'s timeout`);
     const call = submitControl(operation.binding.endpoint, {
       version: WIRE_VERSION,
       command,
       peer,
       text,
       ...(options.awaitReply ? { awaitReply: true as const } : {}),
-      ...(options.timeoutMs === undefined ? {} : { timeoutMs: options.timeoutMs }),
+      ...(timeoutMs === undefined ? {} : { timeoutMs }),
     });
     invocation.calls.add(call);
     let response: MessageResponse;
@@ -304,7 +320,7 @@ export function createScriptedHost(
   const invoke = (
     context: FakeAdapterTurnContext,
     operation: Operation,
-    record: TurnRecord,
+    record: TestTurn,
     shown: readonly Message[],
   ): Invocation => {
     const { activation, authored } = context;
@@ -338,8 +354,6 @@ export function createScriptedHost(
           reply: (text, options) => turn.reply(woke!.from, text, options),
           send: turn.send,
           tool: turn.tool,
-          silent: reply.silent,
-          fail: (reason) => reply.failed(reason),
         } satisfies ReceivedMessage,
         operation.message!,
       );
@@ -353,7 +367,9 @@ export function createScriptedHost(
       if (was === "running") invocation.stopped.resolve({ step: taken });
       else if (was === "suspended") {
         failTest(
-          `agent "${turn.agent}" turn ${turn.n}: its script went on after its turn ended awaiting a reply; await only the host's calls`,
+          taken.kind === "script-error"
+            ? taken.message
+            : `agent "${turn.agent}" turn ${turn.n}: its script went on after its turn ended awaiting a reply; await only the host's calls`,
         );
       }
     });
@@ -364,7 +380,7 @@ export function createScriptedHost(
   const until = async (
     invocation: Invocation,
     context: FakeAdapterTurnContext,
-    record: TurnRecord,
+    record: TestTurn,
   ): Promise<FakeAdapterTurn> => {
     const { operation } = invocation;
     const stopped = await Promise.race([invocation.stopped.promise, whenAborted(context.signal)]);
@@ -400,7 +416,7 @@ export function createScriptedHost(
   const perform = (
     step: Step,
     context: FakeAdapterTurnContext,
-    record: TurnRecord,
+    record: TestTurn,
     invocation?: Invocation,
   ): FakeAdapterTurn => {
     if (step.kind === "script-error") {
@@ -411,12 +427,12 @@ export function createScriptedHost(
       return {
         act: async () => {
           // Submitted is answered, even though the engine then releases the turn.
-          let outcome: TurnOutcome = "cancelled";
+          let outcome: TestTurnOutcome = "cancelled";
           try {
             // As an agent's `wfa send` returns before it can run `wfa result`.
             if (invocation) await Promise.allSettled([...invocation.calls]);
             const { endpoint, operationId } = context.binding!;
-            const response = await submitResult(endpoint, {
+            const response = await submitControl(endpoint, {
               version: WIRE_VERSION,
               command: "result",
               operationId,
@@ -447,7 +463,7 @@ export function createScriptedHost(
       return {
         act: async () => {
           const { endpoint, operationId } = context.binding!;
-          const response = await submitWaiting(endpoint, {
+          const response = await submitControl(endpoint, {
             version: WIRE_VERSION,
             command: "waiting",
             operationId,
@@ -465,7 +481,7 @@ export function createScriptedHost(
       return {
         act: async (turn) => {
           await whenAborted(turn.signal);
-          end(record, "hang");
+          end(record, "cancelled");
         },
       };
     }
@@ -476,10 +492,10 @@ export function createScriptedHost(
           `agent "${record.agent}" turn ${record.n}: only the calling session can be interrupted by the operator`,
         );
       }
-      end(record, "interrupted");
+      end(record, "cancelled");
       return { state: "cancelled", detail: "interrupted by the operator" };
     }
-    end(record, ending.kind);
+    end(record, outcomeOf(ending.kind));
     return ended(ending, context.activation.execution.harness);
   };
 
@@ -492,10 +508,11 @@ export function createScriptedHost(
       n,
       id: context.id,
       focus: context.prompt,
-      outcome: "hang",
+      // Until it ends: one the run leaves running is cancelled with it.
+      outcome: "cancelled",
     };
     compactions.push(record);
-    const done = (outcome: TurnOutcome) => {
+    const done = (outcome: TestTurnOutcome) => {
       record.outcome = outcome;
       events.onActivity();
     };
@@ -546,7 +563,7 @@ export function createScriptedHost(
       return {
         act: async (turn) => {
           await whenAborted(turn.signal);
-          done("hang");
+          done("cancelled");
         },
       };
     }
@@ -555,7 +572,7 @@ export function createScriptedHost(
         `agent "${activation.key}" compaction ${n}: a compaction is not interrupted by the operator`,
       );
     }
-    done(ending.kind);
+    done(outcomeOf(ending.kind));
     return ended(ending, activation.execution.harness);
   };
 
@@ -591,7 +608,7 @@ export function createScriptedHost(
           ]
         : [],
     );
-    const record: TurnRecord = {
+    const record: TestTurn = {
       agent: activation.key,
       n,
       // A continuation that delivers nothing is a reminder of a reply owed: a nudge.
@@ -605,9 +622,11 @@ export function createScriptedHost(
       ...(context.settings.effort === undefined ? {} : { effort: context.settings.effort }),
       ...(authored.cause === undefined ? {} : { cause: authored.cause }),
       ...(shown.length === 0 ? {} : { messages: shown }),
-      outcome: "hang",
+      // Until it ends: one the run leaves running is cancelled with it.
+      outcome: "cancelled",
     };
     turns.push(record);
+    operationOf.set(record, binding.operationId);
     open.add(record);
     if (!record.nudge && context.kind === "turn") await events.onTurnStarted?.(record);
 
@@ -657,7 +676,7 @@ export function createScriptedHost(
       createFakeAdapter({
         harnesses: PLACEMENT_HARNESSES[placement],
         supportsWaiting: (activation) =>
-          placement === "pane" &&
+          placement === "tui" &&
           activation.execution.harness === "claude" &&
           !activation.execution.caller,
         placement,
@@ -690,7 +709,7 @@ export function createScriptedHost(
       }),
     );
   const placed = createPlacementHostFactory({
-    pane: side("pane"),
+    tui: side("tui"),
     headless: side("headless"),
     ...(caller && caller.here !== false
       ? {
@@ -701,7 +720,7 @@ export function createScriptedHost(
               createSingleSessionHostFactory(
                 createFakeAdapter({
                   harnesses: [caller.harness],
-                  placement: "pane",
+                  placement: "tui",
                   script: (context) => {
                     callerKey = context.activation.key;
                     return script(context);
@@ -731,14 +750,13 @@ export function createScriptedHost(
               key: request.key,
               execution: request.execution,
               ...(request.instructions ? { instructions: request.instructions } : {}),
-              ...(request.labels ? { labels: request.labels } : {}),
               ...(request.skills ? { skills: request.skills.names } : {}),
               ...(request.continues && forks.has(request.continues.sessionRef)
                 ? { forkedFrom: forks.get(request.continues.sessionRef)! }
                 : {}),
-              ...(request.layout ? { layout: request.layout } : {}),
-              ...(request.layoutFallback ? { layoutFallback: request.layoutFallback } : {}),
-              ...(request.keepPane ? { keepPane: request.keepPane } : {}),
+              ...paneOf(request),
+              ...panesOf(request),
+              ...(request.layoutFallback ? { paneFallback: request.layoutFallback } : {}),
             };
             agents.push(opened);
             return {
@@ -758,6 +776,26 @@ export function createScriptedHost(
     sets,
     agents,
     inFlight: () => [...open],
+    settle(recorded) {
+      const engine = new Map(
+        recorded.flatMap((line) =>
+          line.kind === "turn" ? [[line.operationId, line.outcome]] : [],
+        ),
+      );
+      const last = new Map<string, TestTurn>();
+      for (const turn of turns) last.set(operationOf.get(turn)!, turn);
+      for (const [operation, turn] of last) turn.outcome = engine.get(operation) ?? turn.outcome;
+      // Matched by place: the harness's compact() carries no operation id, and an agent's compactions run in order,
+      // one `compact` line per compaction the host saw.
+      const compacted = recorded.filter((line) => line.kind === "compact");
+      const seen = new Map<string, number>();
+      for (const compaction of compactions) {
+        const n = seen.get(compaction.agent) ?? 0;
+        seen.set(compaction.agent, n + 1);
+        const line = compacted.filter((each) => each.agent === compaction.agent)[n];
+        if (line) compaction.outcome = line.outcome;
+      }
+    },
   };
 }
 
@@ -816,12 +854,42 @@ async function noTools(): Promise<ToolOutcome> {
   throw new Error("a compaction calls no tools");
 }
 
+/** The workflow's part of what the host was asked for a pane, as `pane` writes it. */
+function paneOf({ layout, front }: HarnessActivation): { pane?: PaneSpec } {
+  if (layout?.beside !== undefined) {
+    const { beside, side, share } = layout;
+    return {
+      pane: {
+        beside,
+        side,
+        ...(share === undefined ? {} : { share }),
+        ...(front ? { front } : {}),
+      },
+    };
+  }
+  const tab = layout?.tab;
+  if (tab === undefined && !front) return {};
+  return { pane: { ...(tab === undefined ? {} : { tab }), ...(front ? { front } : {}) } };
+}
+
+/** The run's `panes` as the host was asked to apply them to this pane. */
+function panesOf({ layout, keepPane }: HarnessActivation): Pick<OpenedAgent, "panes"> {
+  const workspace = layout?.beside === undefined ? layout?.workspace : undefined;
+  if (workspace === undefined && keepPane === undefined) return {};
+  return {
+    panes: {
+      ...(workspace === undefined ? {} : { workspace }),
+      ...(keepPane === undefined ? {} : { keep: keepPane }),
+    },
+  };
+}
+
 /** What the pane host refuses before it opens a pane. */
 function paneRefusal(activation: HarnessActivation): string | undefined {
   const { harness } = activation.execution;
   const spec = findHarness(harness);
   if (activation.continues && spec && !spec.interactiveResume) {
-    return `${harness} panes cannot continue a forked session: ${spec.absent.interactiveResume}`;
+    return `${harness} tui agents cannot continue a forked session: ${spec.absent.interactiveResume}`;
   }
   return undefined;
 }
@@ -839,7 +907,7 @@ function refusedCompaction(activation: HarnessActivation, hasRun: boolean): stri
   }
   const spec = findHarness(activation.execution.harness);
   const capability =
-    placementOf(activation.execution) === "pane" ? "compactPane" : "compactHeadless";
+    placementOf(activation.execution) === "tui" ? "compactPane" : "compactHeadless";
   if (spec && !spec[capability]) {
     return `${activation.execution.harness} has no compaction of its own: ${spec.absent[capability]}`;
   }
@@ -848,7 +916,7 @@ function refusedCompaction(activation: HarnessActivation, hasRun: boolean): stri
 }
 
 type Ended = Exclude<
-  Extract<Step, { kind: "reply" }>["ending"],
+  Extract<Step, { kind: "no-answer" }>["ending"],
   { kind: "waiting" | "hang" | "interrupted" }
 >;
 
@@ -861,7 +929,7 @@ function ended(ending: Ended, harness: string): FakeAdapterTurn {
   return check
     ? {
         state: "failed",
-        ...loginFailure(harness, check, { ...provider, said: "reply.needsLogin()" }),
+        ...loginFailure(harness, check, { ...provider, said: "noAnswer.needsLogin()" }),
       }
     : {
         state: "failed",