@forwardimpact/libeval 0.1.48 → 0.1.49

package/README.md

@@ -176,11 +176,9 @@ downloadable through retention.
 ## fit-selfedit
 
 A narrow, audited bypass for sessions where `Edit`/`Write` (and bash
-writes) are blocked against paths the project's own allowlist permits —
-see [#1162](https://github.com/forwardimpact/monorepo/issues/1162) and
-[#441](https://github.com/forwardimpact/monorepo/issues/441) for the
-original episodes. Reads stdin, writes the target, exits 0 / 2
-(safeguard violation) / 1 (I/O error).
+writes) are blocked against paths the project's own allowlist permits.
+Reads stdin, writes the target, exits 0 / 2 (safeguard violation) / 1
+(I/O error).
 
 ```sh
 echo "<content>" | bunx fit-selfedit <path>

package/bin/fit-benchmark.js

@@ -134,7 +134,7 @@ export const definition = {
     "fit-benchmark run --family=./families/coding --runs=10 --agent-model=claude-sonnet-4-6",
     "fit-benchmark score --family=./families/coding --task=todo-api --workdir=./benchmark-runs/runs/todo-api/0",
     "fit-benchmark report --format=text",
-    "fit-benchmark report --input=./runs/2026-05-11 --k=1,3,5 --format=text",
+    "fit-benchmark report --input=./runs/today --k=1,3,5 --format=text",
   ],
   documentation: [
     {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.48",
+  "version": "0.1.49",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/discuss-tools.js

@@ -20,28 +20,44 @@ import { tool } from "@anthropic-ai/claude-agent-sdk";
 import { z } from "zod";
 
 import {
+  ADJOURN_DESC,
   baseTools,
   concludeSession,
   orchestrationServer,
+  RECESS_DESC,
   requestForCommentTool,
+  requireNoPendingAsks,
 } from "./orchestration-toolkit.js";
 
 /** System prompt for discuss-mode agent participants. L0 mechanics only per COALIGNED. */
 export const DISCUSS_AGENT_SYSTEM_PROMPT =
   "You are a participant in a discussion.\n" +
-  "Each question arrives as `[ask#N] <name>: <text>`.\n" +
+  "Each question arrives as `[ask#N] <name>: <text>` in your inbox.\n" +
   "Quote N as askId on your `Answer` to route the reply correctly.\n" +
   "Your `Answer` is posted to the discussion thread as a separate reply.\n" +
   "If the task already contains a completed response with no new human input after it, `Answer` that no further action is needed.\n" +
   "Do not redo completed work.";
 
-const RESUME_TRIGGER_SCHEMA = z
-  .object({
-    kind: z.enum(["responses", "elapsed", "either"]),
-    responses: z.number().optional(),
-    elapsed: z.string().optional(),
-  })
-  .strict();
+const RESUME_TRIGGER_SCHEMA = z.discriminatedUnion("kind", [
+  z
+    .object({
+      kind: z.literal("missing_input"),
+      replies: z.number().int().positive(),
+    })
+    .strict(),
+  z
+    .object({
+      kind: z.literal("escalation_needed"),
+      signal: z.string().min(1),
+    })
+    .strict(),
+  z
+    .object({
+      kind: z.literal("elapsed"),
+      elapsed: z.string().min(1),
+    })
+    .strict(),
+]);
 
 /** Discuss-mode lead tool server. */
 export function createDiscussLeadToolServer(ctx) {
@@ -49,13 +65,13 @@ export function createDiscussLeadToolServer(ctx) {
     ...baseTools(ctx, { from: "lead", defaultTo: undefined, broadcast: true }),
     tool(
       "Recess",
-      "Suspend the run. The bridge re-dispatches the workflow when the trigger fires.",
+      RECESS_DESC,
       { reason: z.string(), trigger: RESUME_TRIGGER_SCHEMA },
       createRecessHandler(ctx),
     ),
     tool(
       "Adjourn",
-      "End the discussion with a verdict ('adjourned' / 'failed') and a summary.",
+      ADJOURN_DESC,
       {
         verdict: z.enum(["adjourned", "failed"]),
         summary: z.string(),
@@ -83,6 +99,8 @@ export function createDiscussAgentToolServer(ctx, { from }) {
  */
 export function createRecessHandler(ctx) {
   return async ({ reason, trigger }) => {
+    const guard = requireNoPendingAsks(ctx);
+    if (guard) return guard;
     ctx.recessTrigger = trigger;
     concludeSession(ctx, {
       verdict: "recessed",
@@ -96,6 +114,8 @@ export function createRecessHandler(ctx) {
 /** Adjourn handler — ends the discussion with a verdict. */
 export function createAdjournHandler(ctx) {
   return async ({ verdict, summary, outcome }) => {
+    const guard = requireNoPendingAsks(ctx);
+    if (guard) return guard;
     if (outcome !== undefined) ctx.outcome = outcome;
     concludeSession(ctx, {
       verdict,

package/src/discusser.js

@@ -36,10 +36,11 @@ export const DISCUSS_SYSTEM_PROMPT =
   "Use `Ask` to delegate work to the best-suited participant.\n" +
   "Participants are domain experts; state the task, not how to do it.\n" +
   "Each participant's `Answer` is posted to the discussion thread as a separate reply.\n" +
-  "`Ask` returns {askIds:[N,…]} immediately.\n" +
-  "Answers arrive on your next turn as `[answer#N] <participant>: <text>`.\n" +
-  "Multiple `Ask` calls in one turn run participants concurrently.\n" +
-  "Wait for all participants to `Answer` before calling `Adjourn` or `Recess`.";
+  "`Ask` is async and returns {askIds:[N,…]} immediately.\n" +
+  "Answers arrive on your next turn as `[answer#N] <participant>: <text>` in your inbox.\n" +
+  "End your turn while Asks are pending. The system resumes you when answers arrive.\n" +
+  "Multiple `Ask` calls in one turn run participants in parallel.\n" +
+  "End the discussion by calling `Adjourn` with a verdict and summary, or `Recess` only to wait on an external reply or duration.";
 
 /**
  * Augment a base orchestration context with discuss-mode fields.

package/src/facilitator.js

@@ -25,15 +25,16 @@ export const FACILITATOR_SYSTEM_PROMPT =
   "Use `RollCall` to list participants.\n" +
   "Use `Ask` to delegate work to the best-suited participant.\n" +
   "Participants are domain experts; state the task, not how to do it.\n" +
-  "`Ask` returns {askIds:[N,…]} immediately.\n" +
-  "Answers arrive on your next turn as `[answer#N] <participant>: <text>`.\n" +
-  "Multiple `Ask` calls in one turn run participants concurrently.\n" +
-  "Wait for all participants to `Answer` before calling `Conclude`.";
+  "`Ask` is async and returns {askIds:[N,…]} immediately.\n" +
+  "Answers arrive on your next turn as `[answer#N] <participant>: <text>` in your inbox.\n" +
+  "End your turn while Asks are pending. The system resumes you when answers arrive.\n" +
+  "Multiple `Ask` calls in one turn run participants in parallel.\n" +
+  "End every session by calling `Conclude` with a verdict and summary.";
 
 /** System prompt for facilitated agent participants. L0 mechanics only per COALIGNED. */
 export const FACILITATED_AGENT_SYSTEM_PROMPT =
   "You are a participant in a facilitated session.\n" +
-  "Each question arrives as `[ask#N] <name>: <text>`.\n" +
+  "Each question arrives as `[ask#N] <name>: <text>` in your inbox.\n" +
   "Quote N as askId on your `Answer` to route the reply correctly.\n" +
   "If the task already contains a completed response with no new human input after it, `Answer` that no further action is needed.\n" +
   "Do not redo completed work.";

package/src/orchestration-toolkit.js

@@ -46,9 +46,24 @@ export function createOrchestrationContext() {
 
 // --- Handler factories ---
 
+/**
+ * Guard for terminal tools (`Conclude`, `Adjourn`, `Recess`). Returns an
+ * error result when the caller still has Asks in flight, telling them to
+ * end the turn and wait for the auto-resume. Returns `null` when no Asks
+ * are pending and the terminal tool is free to run.
+ */
+export function requireNoPendingAsks(ctx) {
+  if (ctx.pendingAsks.size === 0) return null;
+  return errorResult(
+    "Asks are still pending. End your turn. You will be resumed when answers arrive.",
+  );
+}
+
 /** Mark the session as concluded; cancel any open Asks so askers see the synthetic null on their next turn. */
 export function createConcludeHandler(ctx) {
   return async ({ verdict, summary }) => {
+    const guard = requireNoPendingAsks(ctx);
+    if (guard) return guard;
     concludeSession(ctx, { verdict, summary, reason: "session concluded" });
     return { content: [{ type: "text", text: "Session concluded." }] };
   };
@@ -235,8 +250,18 @@ const ANNOUNCE_DESC = "Broadcast a message with no reply expected.";
 
 const ROLLCALL_DESC = "List all participants in the session.";
 
+// Terminal-tool descriptions. Each one ends the run. Group them so the
+// contrast is visible: Conclude (success/failure), Adjourn (settled in
+// thread), Recess (paused for out-of-session input). Each description
+// leads with the cost.
 const CONCLUDE_DESC =
-  "End the session with a verdict ('success' or 'failure') and a summary.";
+  "End the session. Provide a verdict ('success' or 'failure') and a summary.";
+
+const ADJOURN_DESC =
+  "End the discussion. Provide a verdict ('adjourned' or 'failed') and a summary. Cancels any unanswered Asks.";
+
+const RECESS_DESC =
+  "End the run and schedule an out-of-session re-dispatch. Cancels any unanswered Asks. Use only when waiting on an external reply or duration. Do not use to wait on in-flight Asks.";
 
 // --- Tool builders ---
 
@@ -244,6 +269,7 @@ const CONCLUDE_DESC =
 function textResult(text) {
   return { content: [{ type: "text", text }] };
 }
+/** Build an MCP tool error result wrapping a single text message. */
 function errorResult(text) {
   return { content: [{ type: "text", text }], isError: true };
 }
@@ -391,4 +417,11 @@ function requestForCommentTool(ctx) {
 
 // Re-export the building blocks discuss-tools.js needs to assemble its
 // own lead tool surface (it has two extra terminal tools).
-export { baseTools, orchestrationServer, requestForCommentTool };
+export {
+  ADJOURN_DESC,
+  baseTools,
+  errorResult,
+  orchestrationServer,
+  RECESS_DESC,
+  requestForCommentTool,
+};

package/src/render/tool-hints.js

@@ -101,7 +101,8 @@ export function simplifyToolName(name) {
  *
  * Three branches, in priority order:
  *  - A built-in tool with an entry in `HINT_HANDLERS` → sanitized hint, no
- *    `{` / `"` from the input (spec 540 criterion #2 for non-MCP tools).
+ *    `{` / `"` from the input (built-in tool hints stay free of JSON
+ *    punctuation so readers see clean one-liners).
  *  - An MCP-prefixed tool (`mcp__*`) → full input rendered as compact
  *    single-line JSON; `{` and `"` intentionally appear so readers see
  *    the actual MCP payload.

package/src/render/turn-renderer.js

@@ -2,8 +2,7 @@
  * Turn renderer — maps a structured turn into formatted text lines.
  *
  * Shared by `TeeWriter.flushTurns()` (live stream) and
- * `TraceCollector.toText()` (offline replay) so both emit identical output
- * (spec 540).
+ * `TraceCollector.toText()` (offline replay) so both emit identical output.
  */
 
 import {

package/src/supervisor.js

@@ -32,15 +32,16 @@ export const SUPERVISOR_SYSTEM_PROMPT =
   "You supervise one agent.\n" +
   "You have no tools to perform work yourself.\n" +
   "Use `Ask` to delegate work to the agent.\n" +
-  "`Ask` returns {askIds:[N]} immediately.\n" +
-  "The reply arrives on your next turn as `[answer#N] agent: <text>`.\n" +
+  "`Ask` is async and returns {askIds:[N]} immediately.\n" +
+  "The reply arrives on your next turn as `[answer#N] agent: <text>` in your inbox.\n" +
+  "End your turn while Asks are pending. The system resumes you when an answer arrives.\n" +
   "If the agent goes off-track, send a corrective `Ask`.\n" +
-  "End every session by calling `Conclude`.";
+  "End every session by calling `Conclude` with a verdict and summary.";
 
 /** System prompt for the supervised agent. L0 mechanics only per COALIGNED. */
 export const AGENT_SYSTEM_PROMPT =
   "A supervisor directs your work.\n" +
-  "Each question arrives as `[ask#N] supervisor: <text>`.\n" +
+  "Each question arrives as `[ask#N] supervisor: <text>` in your inbox.\n" +
   "Quote N as askId on your `Answer` to route the reply correctly.\n" +
   "If the task already contains a completed response with no new human input after it, `Answer` that no further action is needed.\n" +
   "Do not redo completed work.";

package/src/tee-writer.js

@@ -9,7 +9,7 @@
  *
  * Human text rendering is delegated to the pure modules under `./render/`
  * so the live stream and the offline `TraceCollector.toText()` replay share
- * one formatting path (spec 540). The NDJSON going to `fileStream` is
+ * one formatting path. The NDJSON going to `fileStream` is
  * untouched — only what reaches `textStream` changes.
  *
  * Follows OO+DI: constructor injection, factory function, tests bypass factory.
@@ -67,10 +67,9 @@ export class TeeWriter extends Writable {
     }
 
     // Emit the trailing `--- Result: ... ---` footer — the one summary line
-    // humans want (spec 540). This is the same tail TraceCollector.toText()
-    // appends, so the live stream and the offline replay stay in sync
-    // (spec 540 criterion #6). The superseded `--- Evaluation ... ---`
-    // footer is gone in every mode.
+    // humans want. This is the same tail TraceCollector.toText()
+    // appends, so the live stream and the offline replay stay in sync.
+    // The superseded `--- Evaluation ... ---` footer is gone in every mode.
     if (this.collector.result) {
       const text = this.collector.toText();
       const idx = text.lastIndexOf("\n---");
@@ -78,7 +77,7 @@ export class TeeWriter extends Writable {
         // Slice past the leading `\n` — the previously-streamed body
         // already ended with its own newline, so re-emitting `\n---` here
         // would produce a blank line before the footer and desync from
-        // the offline replay (spec 540 #6).
+        // the offline replay.
         this.textStream.write(text.slice(idx + 1) + "\n");
       }
     }
@@ -107,7 +106,8 @@ export class TeeWriter extends Writable {
       this.collector.addLine(line);
 
       // Orchestrator lifecycle events are suppressed from the text stream
-      // entirely (spec 540). They still reached fileStream above.
+      // entirely — humans only want agent-visible content. They still
+      // reached fileStream above.
       if (
         parsed.source === "orchestrator" &&
         isSuppressedOrchestratorEvent(parsed.event)
@@ -118,7 +118,7 @@ export class TeeWriter extends Writable {
       return;
     }
 
-    // Bare event (run mode pre-migration or direct feed)
+    // Bare event (unwrapped run mode line or direct feed)
     this.collector.addLine(line);
     this.flushTurns();
   }

package/src/trace-collector.js

@@ -6,7 +6,7 @@
  *
  * Human text rendering is delegated to the pure modules under `./render/`
  * so the live `TeeWriter` stream and the offline `toText()` replay share
- * one formatting path (spec 540).
+ * one formatting path.
  */
 
 import { renderTurnLines } from "./render/turn-renderer.js";
@@ -293,7 +293,7 @@ export class TraceCollector {
   }
 
   /**
-   * Format the trailing result summary line (spec 540). When an orchestrator
+   * Format the trailing result summary line. When an orchestrator
    * summary is present (supervised / facilitated mode), the headline word is
    * the supervisor's verdict ("success" / "failure") rather than the SDK's
    * per-runner subtype, so the footer aligns with the CI exit code.