@forwardimpact/libeval 0.1.21 → 0.1.23

package/src/orchestration-toolkit.js

@@ -2,6 +2,12 @@
  * OrchestrationToolkit — tool schemas, per-role tool sets, and handler
  * factories for orchestration between supervisors, facilitators, and agents.
  *
+ * The tool surface is Ask / Answer / Announce + Redirect / Conclude / RollCall,
+ * shared across facilitation and supervision. Ask registers a pending-ask in
+ * the context; Answer clears it and routes the reply. The orchestrator's
+ * turn-complete guard (see checkPendingAsk) holds the request-response
+ * contract at the runtime instead of the prompt layer.
+ *
  * Handlers communicate via a shared context object. The orchestrator reads
  * context at natural checkpoints (after resume(), after onBatch).
  */
@@ -20,6 +26,12 @@ export function createOrchestrationContext() {
     redirect: null,
     participants: [],
     messageBus: null,
+    // Map<addresseeName, {askId, askerName, question, reminded}>
+    // Always keyed by an addressee name. Broadcast asks write one entry
+    // per named participant, so every pending entry has a concrete
+    // addressee and the match rule is uniform.
+    pendingAsks: new Map(),
+    askIdCounter: 0,
   };
 }
 
@@ -40,46 +52,150 @@ export function createRedirectHandler(ctx) {
   };
 }
 
-export function createAskHandler(ctx, { onAsk }) {
-  return async ({ question }) => {
-    try {
-      const answer = await onAsk(question);
-      return { content: [{ type: "text", text: answer }] };
-    } catch (err) {
+export function createRollCallHandler(ctx) {
+  return async () => {
+    return {
+      content: [{ type: "text", text: JSON.stringify(ctx.participants) }],
+    };
+  };
+}
+
+/**
+ * Create an Ask handler for a given caller. Ask registers a pending-ask
+ * in ctx and routes the question to the addressee via the message bus.
+ *
+ * @param {object} ctx
+ * @param {object} opts
+ * @param {string} opts.from - Canonical name of the asker.
+ * @param {string|undefined} opts.defaultTo - Default addressee when the
+ *   caller omits `to`. Use `undefined` to signal "broadcast across all
+ *   non-asker participants" (facilitator-only).
+ */
+export function createAskHandler(ctx, { from, defaultTo }) {
+  return async ({ question, to }) => {
+    const explicitTo = typeof to === "string" && to.length > 0 ? to : null;
+    const effectiveTo = explicitTo ?? defaultTo ?? null;
+
+    const addressees = effectiveTo
+      ? [effectiveTo]
+      : ctx.participants.map((p) => p.name).filter((name) => name !== from);
+
+    if (addressees.length === 0) {
       return {
-        content: [{ type: "text", text: `Error: ${err.message}` }],
+        content: [{ type: "text", text: "No addressee for Ask." }],
         isError: true,
       };
     }
+
+    for (const addressee of addressees) {
+      const askId = ++ctx.askIdCounter;
+      ctx.pendingAsks.set(addressee, {
+        askId,
+        askerName: from,
+        question,
+        reminded: false,
+      });
+      ctx.messageBus.ask(from, addressee, question, askId);
+    }
+
+    return { content: [{ type: "text", text: "Ask delivered." }] };
   };
 }
 
-export function createRollCallHandler(ctx) {
-  return async () => {
-    return {
-      content: [{ type: "text", text: JSON.stringify(ctx.participants) }],
-    };
+/**
+ * Create an Answer handler for a given caller. Answer clears the caller's
+ * pending-ask entry (keyed by the caller's canonical name) and routes the
+ * reply to the original asker via the message bus.
+ *
+ * @param {object} ctx
+ * @param {object} opts
+ * @param {string} opts.from - Canonical name of the answerer.
+ */
+export function createAnswerHandler(ctx, { from }) {
+  return async ({ message }) => {
+    const entry = ctx.pendingAsks.get(from);
+    if (!entry) {
+      return {
+        content: [{ type: "text", text: "No pending ask to answer." }],
+        isError: true,
+      };
+    }
+    ctx.pendingAsks.delete(from);
+    ctx.messageBus.answer(from, entry.askerName, message, entry.askId);
+    return { content: [{ type: "text", text: "Answer delivered." }] };
   };
 }
 
-export function createShareHandler(ctx, { from }) {
+/**
+ * Create an Announce handler. Announce broadcasts a message to every
+ * participant except the sender; it never touches pendingAsks.
+ *
+ * @param {object} ctx
+ * @param {object} opts
+ * @param {string} opts.from
+ */
+export function createAnnounceHandler(ctx, { from }) {
   return async ({ message }) => {
-    ctx.messageBus.share(from, message);
-    return { content: [{ type: "text", text: "Message shared." }] };
+    ctx.messageBus.announce(from, message);
+    return { content: [{ type: "text", text: "Announcement delivered." }] };
   };
 }
 
-export function createTellHandler(ctx, { from }) {
-  return async ({ message, to }) => {
-    ctx.messageBus.tell(from, to, message);
-    return { content: [{ type: "text", text: "Message sent." }] };
-  };
+/**
+ * Shared turn-complete guard. Consulted by Facilitator#runAgent and
+ * Supervisor#runAgentTurn / #endOfTurnReview before finalising an agent's
+ * turn. Returns "advance" when no pending-ask is owed by `addresseeName`;
+ * "recheck" after queueing a single synthetic reminder; "advance" after
+ * emitting a protocol_violation event and injecting a synthetic null
+ * answer so the original asker unblocks.
+ *
+ * @param {object} args
+ * @param {object} args.ctx
+ * @param {object} args.messageBus
+ * @param {string} args.addresseeName
+ * @param {"facilitated"|"supervised"} args.mode
+ * @param {(event: object) => void} args.emitViolation
+ * @returns {"advance"|"recheck"}
+ */
+export function checkPendingAsk({
+  ctx,
+  messageBus,
+  addresseeName,
+  mode,
+  emitViolation,
+}) {
+  const entry = ctx.pendingAsks.get(addresseeName);
+  if (!entry) return "advance";
+
+  if (!entry.reminded) {
+    entry.reminded = true;
+    messageBus.synthetic(
+      addresseeName,
+      `You have an unanswered ask from ${entry.askerName}. Reply via Answer.`,
+    );
+    return "recheck";
+  }
+
+  emitViolation({
+    type: "protocol_violation",
+    agent: addresseeName,
+    askId: entry.askId,
+    mode,
+  });
+  messageBus.answer(
+    "@orchestrator",
+    entry.askerName,
+    `[no answer: ${addresseeName} did not reply to ask ${entry.askId}]`,
+    entry.askId,
+  );
+  ctx.pendingAsks.delete(addresseeName);
+  return "advance";
 }
 
 // --- Per-role MCP server factories ---
 
 /**
- * Supervisor tools: Conclude + Redirect.
+ * Supervisor tools: Ask + Announce + Conclude + Redirect + RollCall.
  * @param {object} ctx - Orchestration context
  * @returns {object} MCP server config (type: "sdk")
  */
@@ -87,46 +203,86 @@ export function createSupervisorToolServer(ctx) {
   return createSdkMcpServer({
     name: "orchestration",
     tools: [
+      tool(
+        "Ask",
+        "Send a question to the agent. The reply arrives via Answer.",
+        { question: z.string() },
+        createAskHandler(ctx, { from: "supervisor", defaultTo: "agent" }),
+      ),
+      tool(
+        "Announce",
+        "Broadcast a message with no reply expected.",
+        { message: z.string() },
+        createAnnounceHandler(ctx, { from: "supervisor" }),
+      ),
       tool(
         "Conclude",
-        "Signal that the evaluation is done. Provide a summary.",
+        "End the session with a summary.",
         { summary: z.string() },
         createConcludeHandler(ctx),
       ),
       tool(
         "Redirect",
-        "Interrupt the agent with a corrective message.",
+        "Interrupt the agent with replacement instructions.",
         { message: z.string(), to: z.string().optional() },
         createRedirectHandler(ctx),
       ),
+      tool(
+        "RollCall",
+        "List all participants in the session.",
+        {},
+        createRollCallHandler(ctx),
+      ),
     ],
   });
 }
 
 /**
- * Supervised agent tools: Ask.
+ * Supervised agent tools: Ask + Answer + Announce + RollCall.
  * @param {object} ctx - Orchestration context
- * @param {object} opts
- * @param {function} opts.onAsk - Async callback: (question) → answer string
  * @returns {object} MCP server config (type: "sdk")
  */
-export function createSupervisedAgentToolServer(ctx, { onAsk }) {
+export function createSupervisedAgentToolServer(ctx) {
   return createSdkMcpServer({
     name: "orchestration",
     tools: [
       tool(
         "Ask",
-        "Ask the supervisor a clarifying question. Blocks until answered.",
+        "Send a question to the supervisor. The reply arrives via Answer.",
         { question: z.string() },
-        createAskHandler(ctx, { onAsk }),
+        createAskHandler(ctx, { from: "agent", defaultTo: "supervisor" }),
+      ),
+      tool(
+        "Answer",
+        "Reply to an ask addressed to you.",
+        { message: z.string() },
+        createAnswerHandler(ctx, { from: "agent" }),
+      ),
+      tool(
+        "Announce",
+        "Broadcast a message with no reply expected.",
+        { message: z.string() },
+        createAnnounceHandler(ctx, { from: "agent" }),
+      ),
+      tool(
+        "RollCall",
+        "List all participants in the session.",
+        {},
+        createRollCallHandler(ctx),
       ),
     ],
   });
 }
 
 /**
- * Facilitator tools: Conclude + Redirect + RollCall + Share + Tell.
- * No Ask — the facilitator answers Ask calls, not issues them.
+ * Facilitator tools: Ask + Announce + Conclude + RollCall.
+ *
+ * Redirect is intentionally omitted. In facilitated mode the facilitator
+ * can re-Ask a participant to course-correct — Ask overwrites the pending
+ * slot, giving the agent a proper round-trip path. Redirect (abort +
+ * direct message) belongs in supervised mode where a single agent is
+ * steered by a supervisor.
+ *
  * @param {object} ctx - Orchestration context
  * @returns {object} MCP server config (type: "sdk")
  */
@@ -134,75 +290,68 @@ export function createFacilitatorToolServer(ctx) {
   return createSdkMcpServer({
     name: "orchestration",
     tools: [
+      tool(
+        "Ask",
+        "Send a question to a participant. Omit 'to' to broadcast. The reply arrives via Answer.",
+        { question: z.string(), to: z.string().optional() },
+        createAskHandler(ctx, { from: "facilitator", defaultTo: undefined }),
+      ),
+      tool(
+        "Announce",
+        "Broadcast a message with no reply expected.",
+        { message: z.string() },
+        createAnnounceHandler(ctx, { from: "facilitator" }),
+      ),
       tool(
         "Conclude",
-        "Signal that the task is done. Provide a summary.",
+        "End the session with a summary.",
         { summary: z.string() },
         createConcludeHandler(ctx),
       ),
-      tool(
-        "Redirect",
-        "Interrupt agents with a corrective message. Use to='all' for all agents or a specific agent name.",
-        { message: z.string(), to: z.string().optional() },
-        createRedirectHandler(ctx),
-      ),
       tool(
         "RollCall",
         "List all participants in the session.",
         {},
         createRollCallHandler(ctx),
       ),
-      tool(
-        "Share",
-        "Broadcast a message to all participants. After sending, stop making tool calls to receive responses.",
-        { message: z.string() },
-        createShareHandler(ctx, { from: "facilitator" }),
-      ),
-      tool(
-        "Tell",
-        "Send a direct message to one participant. After sending, stop making tool calls to receive their response.",
-        { message: z.string(), to: z.string() },
-        createTellHandler(ctx, { from: "facilitator" }),
-      ),
     ],
   });
 }
 
 /**
- * Facilitated agent tools: Ask + RollCall + Share + Tell.
+ * Facilitated agent tools: Ask + Answer + Announce + RollCall.
  * @param {object} ctx - Orchestration context
  * @param {object} opts
- * @param {string} opts.from - Agent name (for Share/Tell)
- * @param {function} opts.onAsk - Async callback: (question) → answer string
+ * @param {string} opts.from - Agent name (canonical, used for handler wiring)
  * @returns {object} MCP server config (type: "sdk")
  */
-export function createFacilitatedAgentToolServer(ctx, { from, onAsk }) {
+export function createFacilitatedAgentToolServer(ctx, { from }) {
   return createSdkMcpServer({
     name: "orchestration",
     tools: [
       tool(
         "Ask",
-        "Ask the facilitator a clarifying question. Blocks until answered.",
-        { question: z.string() },
-        createAskHandler(ctx, { onAsk }),
+        "Send a question to another participant. Omit 'to' to ask the facilitator.",
+        { question: z.string(), to: z.string().optional() },
+        createAskHandler(ctx, { from, defaultTo: "facilitator" }),
       ),
       tool(
-        "RollCall",
-        "List all participants in the session.",
-        {},
-        createRollCallHandler(ctx),
+        "Answer",
+        "Reply to an ask addressed to you.",
+        { message: z.string() },
+        createAnswerHandler(ctx, { from }),
       ),
       tool(
-        "Share",
-        "Broadcast a message to all participants.",
+        "Announce",
+        "Broadcast a message with no reply expected.",
         { message: z.string() },
-        createShareHandler(ctx, { from }),
+        createAnnounceHandler(ctx, { from }),
       ),
       tool(
-        "Tell",
-        "Send a direct message to one participant.",
-        { message: z.string(), to: z.string() },
-        createTellHandler(ctx, { from }),
+        "RollCall",
+        "List all participants in the session.",
+        {},
+        createRollCallHandler(ctx),
       ),
     ],
   });

package/src/orchestrator-helpers.js

@@ -0,0 +1,58 @@
+/**
+ * Shared helpers for Facilitator and Supervisor orchestrators:
+ * - `createAsyncQueue`  — simple promise-based queue used by the facilitator
+ *                         event loop.
+ * - `formatMessages`    — render a drained message batch as tagged lines.
+ */
+
+export function createAsyncQueue() {
+  const items = [];
+  let waiter = null;
+  let closed = false;
+  return {
+    enqueue(item) {
+      items.push(item);
+      if (waiter) {
+        waiter();
+        waiter = null;
+      }
+    },
+    async dequeue() {
+      if (items.length > 0) return items.shift();
+      if (closed) return null;
+      await new Promise((resolve) => {
+        waiter = resolve;
+      });
+      return items.length > 0 ? items.shift() : null;
+    },
+    close() {
+      closed = true;
+      if (waiter) {
+        waiter();
+        waiter = null;
+      }
+    },
+  };
+}
+
+/**
+ * Render a drained batch of bus messages as tagged text lines.
+ * @param {Array<{from: string, text: string, kind?: string, direct?: boolean}>} messages
+ * @returns {string}
+ */
+export function formatMessages(messages) {
+  return messages.map(formatMessage).join("\n");
+}
+
+function formatMessage(m) {
+  return `${tagFor(m)} ${m.from}: ${m.text}`;
+}
+
+function tagFor(m) {
+  if (m.kind === "ask") return "[ask]";
+  if (m.kind === "answer") return "[answer]";
+  if (m.kind === "announce") return "[shared]";
+  if (m.kind === "synthetic") return "[system]";
+  if (m.kind === "direct") return "[direct]";
+  return m.direct ? "[direct]" : "[shared]";
+}

package/src/render/tool-hints.js

@@ -78,7 +78,7 @@ const HINT_HANDLERS = {
 
 /**
  * Strip the `mcp__<server>__` prefix from MCP-namespaced tool names so logs
- * show the bare method (e.g. `mcp__orchestration__Tell` → `Tell`). Non-MCP
+ * show the bare method (e.g. `mcp__orchestration__Ask` → `Ask`). Non-MCP
  * names and malformed inputs pass through unchanged.
  * @param {string} name
  * @returns {string}
@@ -92,7 +92,7 @@ export function simplifyToolName(name) {
 }
 
 /**
- * MCP-prefixed tool names (e.g. `mcp__orchestration__Tell`) take a different
+ * MCP-prefixed tool names (e.g. `mcp__orchestration__Ask`) take a different
  * handler path. The method name itself is surfaced via `simplifyToolName`,
  * so this only adds the `to/from` decorators for orchestration calls.
  * Returns null if the name does not match any MCP prefix.
@@ -121,7 +121,7 @@ function hintForMcp(name, input) {
  * ends with `sanitize`, so the output is guaranteed free of `{`, `}`, `"`
  * from the input object (success criterion #2).
  *
- * @param {string} name - Tool name (e.g. "Bash", "Read", "mcp__orchestration__Tell")
+ * @param {string} name - Tool name (e.g. "Bash", "Read", "mcp__orchestration__Ask")
  * @param {object|null|undefined} input - Raw tool input object from the trace
  * @returns {string} One-line hint, or "" when no rule matches
  */