@agentswf/engine 0.1.0-beta.2 → 0.1.0-beta.4

package/src/workflow-testing/coordinator.typecheck.ts

@@ -0,0 +1,54 @@
+// Checked by `tsc --noEmit`, never run: story 032's workflow and its test, as an author writes them.
+import type {
+  ParticipantRef,
+  WorkflowContext,
+  WorkflowDefinition,
+} from "@agentswf/contract/workflow";
+import { answer, type ParticipantScript, testWorkflow } from ".";
+
+declare const runtime: string;
+
+const investigate: WorkflowDefinition<null, string> = {
+  meta: { name: "investigate", description: "watches checkout-api" },
+  async run(wf: WorkflowContext) {
+    const investigator = await wf.agents.open({ key: "investigator", runtime });
+    const coordinator: ParticipantRef = await wf.participants.connect({
+      key: "coordinator",
+      labels: { audience: "operator" },
+    });
+    await wf.messages.allow({
+      between: [investigator, coordinator],
+      purpose: "report findings that hold up; ask when the logs can't settle it",
+      timeoutMs: 2 * 60 * 60_000,
+    });
+    const { outcome } = await investigator.run({ prompt: "Check the checkout-api logs." });
+    await coordinator.release("investigation done");
+    // A run has no deadline unless `awf run --timeout` gives it one.
+    const left = wf.deadline ? wf.deadline.unixMilliseconds - Date.now() : undefined;
+    return outcome.answered ? `${outcome.value} (${left ?? "no"} ms left)` : outcome.kind;
+  },
+};
+
+export async function story(): Promise<string> {
+  const run = await testWorkflow(investigate, null, {
+    agents: { investigator: [answer("…")] },
+    participants: {
+      coordinator: (event) =>
+        event.kind === "message" && event.asks
+          ? event.reply("Keep two, and link them.")
+          : undefined,
+    },
+  });
+  return run.value;
+}
+
+export const sendsFirst: ParticipantScript = async (event) => {
+  if (event.kind !== "route") return;
+  const sent = await event.send("Also check the 09:12 deploy.");
+  if (sent.code !== "delivered") throw new Error(sent.code);
+};
+
+export async function refused(wf: WorkflowContext): Promise<void> {
+  // @ts-expect-error bindingPath is gone: the socket's place is fixed.
+  await wf.participants.connect({ key: "coordinator", bindingPath: "/tmp/x" });
+}

package/src/workflow-testing/coordinators.ts

@@ -0,0 +1,137 @@
+import type { CoordinatorHolder, CoordinatorPort } from "../coordinator-port";
+import type { HolderResult, MessagingEventBody } from "../messaging/messaging";
+import type {
+  ParticipantMessage,
+  ParticipantOutcome,
+  ParticipantRoute,
+  ParticipantScript,
+} from "./script";
+
+/**
+ * Answers for a test's coordinators from their scripts, as a holder would: each route to one as it
+ * opens, and each message an agent sends one once recorded, is handed to its script.
+ */
+export class ParticipantScripts implements CoordinatorPort {
+  readonly #scripts: Readonly<Record<string, ParticipantScript>>;
+  readonly #attempt: number;
+  readonly #events: { onActivity(): void; onScriptError(message: string): void };
+  #holder: CoordinatorHolder | undefined;
+  readonly #connected = new Set<string>();
+  /** Script calls not yet settled. */
+  #working = 0;
+
+  constructor(
+    scripts: Readonly<Record<string, ParticipantScript>> | undefined,
+    attempt: number,
+    events: { onActivity(): void; onScriptError(message: string): void },
+  ) {
+    this.#scripts = scripts ?? {};
+    this.#attempt = attempt;
+    this.#events = events;
+  }
+
+  attach(holder: CoordinatorHolder): void {
+    this.#holder = holder;
+  }
+
+  /** Nothing a script was handed is still running. */
+  get idle(): boolean {
+    return this.#working === 0;
+  }
+
+  /** Questions open to coordinators now; `closesAt` where one closes unanswered. */
+  open(): { closesAt?: number }[] {
+    return this.#holder?.open() ?? [];
+  }
+
+  recorded(event: MessagingEventBody): void {
+    if (event.kind === "participant-connected") {
+      this.#connected.add(event.participant);
+      return;
+    }
+    if (event.kind === "route-opened") {
+      const [a, b] = event.between ?? [event.from, event.to];
+      for (const [coordinator, agent] of [
+        [a, b],
+        [b, a],
+      ] as const) {
+        if (!this.#connected.has(coordinator)) continue;
+        const direction = event.between
+          ? "both"
+          : event.from === coordinator
+            ? "from-participant"
+            : "to-participant";
+        const route: ParticipantRoute = {
+          kind: "route",
+          agent,
+          direction,
+          purpose: event.purpose,
+          send: (text) => this.#send(coordinator, agent, text),
+        };
+        this.#call(coordinator, route);
+      }
+      return;
+    }
+    if (event.kind === "message-sent" && this.#connected.has(event.to)) {
+      const coordinator = event.to;
+      const message: ParticipantMessage = {
+        kind: "message",
+        id: this.#id(event.message),
+        from: event.from,
+        text: event.text,
+        ...(event.awaitsReply ? { asks: true as const } : {}),
+        reply: (text) =>
+          this.#outcome(
+            this.#holder!.reply({
+              coordinator,
+              agent: event.from,
+              to: event.message,
+              text,
+              via: "test",
+            }),
+          ),
+        send: (agent, text) => this.#send(coordinator, agent, text),
+      };
+      this.#call(coordinator, message);
+    }
+  }
+
+  #call(coordinator: string, event: ParticipantRoute | ParticipantMessage): void {
+    const script = this.#scripts[coordinator];
+    if (!script) {
+      this.#events.onScriptError(
+        `${event.kind === "route" ? "a route" : "a message"} reached coordinator "${coordinator}", which has no script under coordinators`,
+      );
+      return;
+    }
+    this.#working += 1;
+    this.#events.onActivity();
+    // Never inside the record's write: a script's reply is a holder's, arriving after it.
+    void Promise.resolve()
+      .then(() => script(event))
+      .catch((error: unknown) =>
+        this.#events.onScriptError(
+          `coordinator "${coordinator}"'s script threw: ${error instanceof Error ? error.message : String(error)}`,
+        ),
+      )
+      .finally(() => {
+        this.#working -= 1;
+        this.#events.onActivity();
+      });
+  }
+
+  #send(coordinator: string, agent: string, text: string): Promise<ParticipantOutcome> {
+    return this.#outcome(this.#holder!.send({ coordinator, agent, text, via: "test" }));
+  }
+
+  async #outcome(result: Promise<HolderResult>): Promise<ParticipantOutcome> {
+    const settled = await result;
+    return settled.code === "delivered"
+      ? { code: "delivered", id: this.#id(settled.message) }
+      : { code: settled.code };
+  }
+
+  #id(message: number): string {
+    return `${this.#attempt}.${message}`;
+  }
+}

package/src/workflow-testing/host.ts

@@ -5,6 +5,9 @@ import {
   type ReplyCode,
   SEND_CODES,
   type SendCode,
+  TOOL_CODES,
+  type ToolCode,
+  type ToolResponse,
   WIRE_VERSION,
 } from "@agentswf/contract/wire";
 import {
@@ -34,7 +37,7 @@ import {
   type FakeFork,
   type FakeSet,
 } from "@agentswf/harness/testing";
-import { submitControl, submitResult, submitWaiting } from "@agentswf/wf/client";
+import { submitControl, submitResult, submitWaiting } from "../agent-cli/client";
 import {
   type Message,
   type MessageOutcome,
@@ -44,6 +47,7 @@ import {
   type Scripts,
   type SendOptions,
   type Step,
+  type ToolOutcome,
   type Turn,
   type TurnOutcome,
 } from "./script";
@@ -52,7 +56,7 @@ import {
  * A turn as the workflow wrote it, and how it ended. `messages` are those it was shown, absent when
  * none were.
  */
-export type TurnRecord = Omit<Turn, "signal" | "messages" | "send" | "reply"> & {
+export type TurnRecord = Omit<Turn, "signal" | "messages" | "send" | "reply" | "tool"> & {
   messages?: readonly Message[];
   outcome: TurnOutcome;
 };
@@ -133,6 +137,8 @@ export function createScriptedHost(
     onActivity(): void;
     /** The script could not meet a turn: the test fails. */
     onScriptError(message: string): void;
+    /** A turn began, before its script answers: what a test sends during it goes now. */
+    onTurnStarted?(turn: TurnRecord): Promise<void>;
   },
   /**
    * The session the run is started from, as `awf run --here` finds one, and the model its files
@@ -186,7 +192,7 @@ export function createScriptedHost(
     return n;
   };
 
-  /** `wf send` or `wf reply` from a running script; with `awaitReply`, resolves with the reply. */
+  /** `wfa send` or `wfa reply` from a running script; with `awaitReply`, resolves with the reply. */
   const message = async (
     invocation: Invocation,
     command: "send" | "reply",
@@ -220,7 +226,7 @@ export function createScriptedHost(
         suspendIfAwaiting(invocation);
         return { kind: "refused", code: response.code as SendCode | ReplyCode };
       }
-      throw new Error(`wf ${command} was refused (${response.code}): ${response.error}`);
+      throw new Error(`wfa ${command} was refused (${response.code}): ${response.error}`);
     }
     if (!options.awaitReply) {
       suspendIfAwaiting(invocation);
@@ -232,9 +238,54 @@ export function createScriptedHost(
     });
   };
 
+  /** `wfa tool` from a running script, as JSON or no input; waits as long as the engine takes. */
+  const callTool = async (
+    invocation: Invocation,
+    name: string,
+    input: JsonObject | undefined,
+  ): Promise<ToolOutcome> => {
+    if (invocation.state !== "running") {
+      const why = `agent "${invocation.turn.agent}" turn ${invocation.turn.n}: its script called a tool after its turn ended`;
+      failTest(why);
+      throw new Error(why);
+    }
+    const call = submitControl(
+      invocation.operation.binding.endpoint,
+      {
+        version: WIRE_VERSION,
+        command: "tool",
+        tool: name,
+        ...(input === undefined ? {} : { raw: JSON.stringify(input) }),
+      },
+      null,
+    );
+    invocation.calls.add(call);
+    let response: ToolResponse;
+    try {
+      response = await call;
+    } finally {
+      invocation.calls.delete(call);
+      // A reply it awaits ends the turn once nothing of the host's is in flight, this call included.
+      suspendIfAwaiting(invocation);
+    }
+    switch (response.kind) {
+      case "tool-result":
+        return {
+          kind: "result",
+          ...(response.output === undefined ? {} : { output: response.output }),
+        };
+      case "tool-failed":
+        return { kind: "failed", outcome: response.outcome, error: response.error };
+      default:
+        if (response.code === "internal-error" || !TOOL_REFUSALS.has(response.code))
+          throw new Error(`wfa tool was refused (${response.code}): ${response.error}`);
+        return { kind: "refused", code: response.code as ToolCode, error: response.error };
+    }
+  };
+
   /**
    * Ends the native turn when its script has gone still awaiting a reply, as an agent ends its turn
-   * after `wf send --await-reply`: nothing of the host's in flight, and no step taken since.
+   * after `wfa send --await-reply`: nothing of the host's in flight, and no step taken since.
    */
   const suspendIfAwaiting = (invocation: Invocation) => {
     setTimeout(() => {
@@ -271,6 +322,7 @@ export function createScriptedHost(
       messages,
       send: (peer, text, options) => message(invocation, "send", peer, text, options),
       reply: (peer, text, options) => message(invocation, "reply", peer, text, options),
+      tool: (name, input) => callTool(invocation, name, input),
       signal: context.signal,
     };
     invocation.turn = turn;
@@ -285,6 +337,7 @@ export function createScriptedHost(
           turn,
           reply: (text, options) => turn.reply(woke!.from, text, options),
           send: turn.send,
+          tool: turn.tool,
           silent: reply.silent,
           fail: (reason) => reply.failed(reason),
         } satisfies ReceivedMessage,
@@ -360,7 +413,7 @@ export function createScriptedHost(
           // Submitted is answered, even though the engine then releases the turn.
           let outcome: TurnOutcome = "cancelled";
           try {
-            // As an agent's `wf send` returns before it can run `wf result`.
+            // As an agent's `wfa send` returns before it can run `wfa result`.
             if (invocation) await Promise.allSettled([...invocation.calls]);
             const { endpoint, operationId } = context.binding!;
             const response = await submitResult(endpoint, {
@@ -465,6 +518,7 @@ export function createScriptedHost(
         messages: [],
         send: noMessages,
         reply: noMessages,
+        tool: noTools,
         cwd: activation.cwd,
         model: context.settings.model,
         ...(context.settings.effort === undefined ? {} : { effort: context.settings.effort }),
@@ -555,6 +609,7 @@ export function createScriptedHost(
     };
     turns.push(record);
     open.add(record);
+    if (!record.nudge && context.kind === "turn") await events.onTurnStarted?.(record);
 
     // A reply or a failure settles the send awaiting it, and resumes the script that awaits it.
     const settled: [Awaiting, MessageOutcome][] = [];
@@ -727,7 +782,7 @@ type Awaiting = { peer: string; by: Invocation; resolve(outcome: MessageOutcome)
 type Invocation = {
   operation: Operation;
   turn: Turn;
-  /** Its `wf` calls in flight. */
+  /** Its `wfa` calls in flight. */
   calls: Set<Promise<unknown>>;
   state: "running" | "suspended" | "settled" | "cancelled";
   /** Where it stopped in its current native turn. */
@@ -735,6 +790,7 @@ type Invocation = {
 };
 
 const MESSAGE_CODES = new Set<string>([...SEND_CODES, ...REPLY_CODES]);
+const TOOL_REFUSALS = new Set<string>(TOOL_CODES);
 /** The call ended before the script's answer reached it: the turn was let go, not refused. */
 const CLOSED_CODES = new Set<string>([
   "closed-operation",
@@ -756,6 +812,10 @@ async function noMessages(): Promise<MessageOutcome> {
   throw new Error("a compaction sends no messages");
 }
 
+async function noTools(): Promise<ToolOutcome> {
+  throw new Error("a compaction calls no tools");
+}
+
 /** What the pane host refuses before it opens a pane. */
 function paneRefusal(activation: HarnessActivation): string | undefined {
   const { harness } = activation.execution;