@forwardimpact/libharness 0.1.22 → 1.0.0

package/src/judge.js

@@ -0,0 +1,210 @@
+/**
+ * Judge — one agent session that inspects a completed agent's work and emits
+ * a verdict via the orchestration `Conclude` tool. Parallel concept to
+ * `Supervisor` and `Facilitator`, but post-hoc and solo: no peer agents,
+ * no message bus, no orchestration loop. The judge reads the task, optionally
+ * inspects the working directory and trace via read-only tools, and calls
+ * Conclude exactly once.
+ *
+ * Trace lines are tagged `source: "judge"` so consumers can distinguish
+ * judge sessions from supervisor or facilitator sessions in a unified
+ * NDJSON envelope.
+ *
+ * Follows OO+DI: constructor injection, factory function, tests bypass factory.
+ */
+
+import { resolve } from "node:path";
+import { Writable } from "node:stream";
+
+import { createAgentRunner } from "./agent-runner.js";
+import { composeSystemPrompt } from "./profile-prompt.js";
+import { SequenceCounter } from "./sequence-counter.js";
+import {
+  createJudgeToolServer,
+  createOrchestrationContext,
+} from "./orchestration-toolkit.js";
+
+/**
+ * System-prompt trailer appended to the judge's main thread. Always applied,
+ * even when a `judgeProfile` is supplied — the profile layers on top of the
+ * trailer, the same way `SUPERVISOR_SYSTEM_PROMPT` and
+ * `FACILITATOR_SYSTEM_PROMPT` work for their respective roles.
+ */
+export const JUDGE_SYSTEM_PROMPT =
+  "You are a post-hoc judge for an agent task benchmark. " +
+  "The agent has already completed its work and an objective invariants step has already run; your role is to confirm or override the verdict by inspecting the agent's working directory and trace. " +
+  "You have read-only inspection tools — Read, Glob, Grep, Bash — to investigate; do not modify the working directory. " +
+  "Conclude ends the session with a verdict ('success' or 'failure') and a one-paragraph summary; verdict='success' iff the agent's work meets the criteria stated in the task. " +
+  "Call Conclude as your final action — do not deliberate across multiple turns.";
+
+const DEFAULT_JUDGE_ALLOWED_TOOLS = ["Read", "Glob", "Grep", "Bash"];
+
+const devNull = new Writable({
+  write(_chunk, _enc, cb) {
+    cb();
+  },
+});
+
+/** Run a single post-hoc judge session and emit a verdict via Conclude. */
+export class Judge {
+  /**
+   * @param {object} deps
+   * @param {import("./agent-runner.js").AgentRunner} deps.runner - The judge's AgentRunner.
+   * @param {import("stream").Writable} deps.output - Stream to emit tagged NDJSON to.
+   * @param {object} deps.ctx - Orchestration context (the Conclude handler writes to it).
+   * @param {import("./redaction.js").Redactor} deps.redactor
+   * @param {string} [deps.taskAmend] - Opaque addendum appended to the task before delivery.
+   */
+  constructor({ runner, output, ctx, redactor, taskAmend }) {
+    if (!runner) throw new Error("runner is required");
+    if (!output) throw new Error("output is required");
+    if (!ctx) throw new Error("ctx is required");
+    if (!redactor) throw new Error("redactor is required");
+    this.runner = runner;
+    this.output = output;
+    this.ctx = ctx;
+    this.redactor = redactor;
+    this.taskAmend = taskAmend ?? null;
+    this.counter = new SequenceCounter();
+  }
+
+  /**
+   * Run the judge session.
+   * @param {string} task - The judge prompt (with placeholders already substituted).
+   * @returns {Promise<{success: boolean, verdict: string|null, summary: string|null, turns: number}>}
+   */
+  async run(task) {
+    const fullTask = this.taskAmend ? `${task}\n\n${this.taskAmend}` : task;
+    const result = await this.runner.run(fullTask);
+
+    if (this.ctx.concluded) {
+      const success = this.ctx.verdict === "success";
+      const outcome = {
+        success,
+        verdict: this.ctx.verdict,
+        summary: this.ctx.summary ?? null,
+        turns: 1,
+      };
+      this.emitSummary(outcome);
+      return outcome;
+    }
+
+    // The judge ended without calling Conclude. Surface that explicitly so
+    // callers can distinguish "judge said fail" from "judge never voted."
+    const outcome = {
+      success: false,
+      verdict: null,
+      summary: null,
+      turns: result.success ? 1 : 0,
+    };
+    this.emitSummary(outcome);
+    return outcome;
+  }
+
+  /**
+   * Tag a single NDJSON line with `source: "judge"` and emit it to the
+   * judge's output stream. Wired into the underlying AgentRunner via the
+   * `onLine` callback so the judge's stream is the single source of truth
+   * for the session's trace.
+   * @param {string} line
+   */
+  emitLine(line) {
+    const event = JSON.parse(line);
+    const tagged = { source: "judge", seq: this.counter.next(), event };
+    this.output.write(JSON.stringify(this.redactor.redactValue(tagged)) + "\n");
+  }
+
+  /**
+   * Emit a final orchestrator summary line, wrapped in the universal envelope.
+   * @param {{success: boolean, verdict?: string|null, summary?: string|null, turns: number}} result
+   */
+  emitSummary(result) {
+    this.output.write(
+      JSON.stringify(
+        this.redactor.redactValue({
+          source: "orchestrator",
+          seq: this.counter.next(),
+          event: {
+            type: "summary",
+            success: result.success,
+            ...(result.verdict && { verdict: result.verdict }),
+            turns: result.turns,
+            ...(result.summary && { summary: result.summary }),
+          },
+        }),
+      ) + "\n",
+    );
+  }
+}
+
+/**
+ * Factory function — wires the AgentRunner with the judge orchestration server
+ * and the JUDGE_SYSTEM_PROMPT trailer. A `judgeProfile` (when supplied) layers
+ * on top of the trailer via `composeSystemPrompt`, matching the
+ * supervisor/facilitator pattern.
+ *
+ * @param {object} deps
+ * @param {string} deps.cwd - Judge working directory. Defaults to the directory whose `.claude/agents` holds `judgeProfile`.
+ * @param {function} deps.query - SDK query function (injected for testing).
+ * @param {import("stream").Writable} deps.output - Trace output stream.
+ * @param {import("./redaction.js").Redactor} deps.redactor
+ * @param {string} [deps.model]
+ * @param {number} [deps.maxTurns] - Default 5 (the judge is expected to act in turn 1; 5 leaves headroom for tool inspection).
+ * @param {string[]} [deps.allowedTools] - Default `["Read","Glob","Grep","Bash"]` — read-only inspection.
+ * @param {string} [deps.judgeProfile] - Profile name; resolved into the system prompt via `composeSystemPrompt`.
+ * @param {string} [deps.profilesDir] - Defaults to `<cwd>/.claude/agents`.
+ * @param {string} [deps.taskAmend]
+ * @returns {Judge}
+ */
+export function createJudge({
+  cwd,
+  query,
+  output,
+  redactor,
+  model,
+  maxTurns,
+  allowedTools,
+  judgeProfile,
+  profilesDir,
+  taskAmend,
+  runtime,
+}) {
+  if (!cwd) throw new Error("cwd is required");
+  if (!query) throw new Error("query is required");
+  if (!output) throw new Error("output is required");
+  if (!redactor) throw new Error("redactor is required");
+  if (!runtime) throw new Error("runtime is required");
+
+  const resolvedProfilesDir = profilesDir ?? resolve(cwd, ".claude/agents");
+  const systemPrompt = composeSystemPrompt({
+    role: "agent",
+    profile: judgeProfile,
+    profilesDir: resolvedProfilesDir,
+    trailer: JUDGE_SYSTEM_PROMPT,
+    runtime,
+  });
+
+  const ctx = createOrchestrationContext();
+  ctx.participants = [{ name: "judge", role: "judge" }];
+  const judgeServer = createJudgeToolServer(ctx);
+
+  let judge;
+  const onLine = (line) => judge.emitLine(line);
+
+  const runner = createAgentRunner({
+    cwd,
+    query,
+    output: devNull,
+    model,
+    maxTurns: maxTurns ?? 5,
+    allowedTools: allowedTools ?? DEFAULT_JUDGE_ALLOWED_TOOLS,
+    onLine,
+    settingSources: ["project"],
+    systemPrompt,
+    mcpServers: { orchestration: judgeServer },
+    redactor,
+  });
+
+  judge = new Judge({ runner, output, ctx, redactor, taskAmend });
+  return judge;
+}

package/src/message-bus.js

@@ -0,0 +1,118 @@
+/**
+ * MessageBus — in-memory per-participant message queues.
+ *
+ * Four message kinds, each pushed onto the addressee's queue:
+ *
+ * - `ask(from, to, text, askId)` — direct question; the toolkit owns the
+ *   pending-ask state separately. Fan-out (broadcast Ask) happens at the
+ *   handler level by calling `ask()` once per addressee.
+ * - `answer(from, to, text, askId)` — direct reply to the original asker.
+ *   The orchestrator may inject synthetic answers (`from === "@orchestrator"`)
+ *   when an Ask times out.
+ * - `announce(from, text)` — broadcast, no reply expected; lands on every
+ *   participant's queue except the sender's.
+ * - `synthetic(to, text)` — orchestrator-only reminder injection.
+ *
+ * Follows OO+DI: constructor injection, factory function, tests bypass factory.
+ */
+
+/** In-memory per-participant message queues. */
+export class MessageBus {
+  /**
+   * @param {object} deps
+   * @param {string[]} deps.participants - Canonical participant names.
+   */
+  constructor({ participants }) {
+    this.queues = new Map();
+    this.waiters = new Map();
+    for (const name of participants) {
+      this.queues.set(name, []);
+      this.waiters.set(name, null);
+    }
+  }
+
+  /** Send a question to one participant. */
+  ask(from, to, text, askId) {
+    this.#assertParticipant(from);
+    this.#assertParticipant(to);
+    this.queues.get(to).push({ from, text, kind: "ask", askId });
+    this.#resolveWaiter(to);
+  }
+
+  /**
+   * Reply to a pending ask. `from === "@orchestrator"` is allowed for
+   * synthetic null answers — the orchestrator is not a real participant
+   * but it routes through the bus.
+   */
+  answer(from, to, text, askId) {
+    this.#assertParticipant(to);
+    if (from !== "@orchestrator") this.#assertParticipant(from);
+    this.queues.get(to).push({ from, text, kind: "answer", askId });
+    this.#resolveWaiter(to);
+  }
+
+  /** Broadcast a message to every participant except the sender. */
+  announce(from, text) {
+    this.#assertParticipant(from);
+    const msg = { from, text, kind: "announce" };
+    for (const [name, queue] of this.queues) {
+      if (name === from) continue;
+      queue.push(msg);
+      this.#resolveWaiter(name);
+    }
+  }
+
+  /** Inject an orchestrator-originated reminder onto one participant's queue. */
+  synthetic(to, text) {
+    this.#assertParticipant(to);
+    this.queues
+      .get(to)
+      .push({ from: "@orchestrator", text, kind: "synthetic" });
+    this.#resolveWaiter(to);
+  }
+
+  /** Check whether a participant has pending messages without draining them. */
+  hasPending(participant) {
+    this.#assertParticipant(participant);
+    return this.queues.get(participant).length > 0;
+  }
+
+  /** Return and clear pending messages for a participant. */
+  drain(participant) {
+    this.#assertParticipant(participant);
+    return this.queues.get(participant).splice(0);
+  }
+
+  /**
+   * Return a Promise that resolves when at least one message is pending.
+   * Resolves immediately if messages are already queued.
+   */
+  waitForMessages(participant) {
+    this.#assertParticipant(participant);
+    if (this.queues.get(participant).length > 0) {
+      return Promise.resolve();
+    }
+    return new Promise((resolve) => {
+      this.waiters.set(participant, resolve);
+    });
+  }
+
+  #assertParticipant(name) {
+    if (!this.queues.has(name)) {
+      throw new Error(`Unknown participant: ${name}`);
+    }
+  }
+
+  #resolveWaiter(name) {
+    const waiter = this.waiters.get(name);
+    if (waiter) {
+      this.waiters.set(name, null);
+      waiter();
+    }
+  }
+}
+
+/** Factory function. */
+export function createMessageBus(deps) {
+  return new MessageBus(deps);
+}

package/src/orchestration-loop.js

@@ -0,0 +1,330 @@
+/**
+ * OrchestrationLoop — N agent sessions coordinated by one lead LLM session.
+ *
+ * Ask is **async**: the tool returns immediately, the actual reply arrives
+ * on a later turn as `[answer#N] participant: …` on the asker's bus queue.
+ * Pending state keys by `askId` (visible in the `[ask#N]` tag), so duplicate
+ * Asks to the same addressee coexist without overwriting each other, and
+ * the asker can map each reply unambiguously back to its question.
+ *
+ * Both lead and participants follow the same outer pattern: drain the bus
+ * queue, run / resume the LLM with the drained messages, then settle any
+ * unanswered Asks the participant owes. They differ only in how the first
+ * turn starts (the lead receives the task; participants wait for traffic).
+ *
+ * Termination signals:
+ * - `ctx.concluded` — explicit Conclude / Adjourn / Recess.
+ * - `stopped` — broader: also true on lead error, agent crash, or any
+ *   other abort path. Loops watch `stopped`; `ctx.concluded` is only used
+ *   for the summary's success/verdict.
+ */
+import { SequenceCounter } from "./sequence-counter.js";
+import {
+  cancelPendingAsks,
+  pendingAsksOwedBy,
+  remindOwedAsks,
+} from "./orchestration-toolkit.js";
+import { formatMessages } from "./orchestrator-helpers.js";
+
+/** Default per-session lead-turn budget — accommodates multi-round injected conversations. */
+const DEFAULT_MAX_LEAD_TURNS = 200;
+
+/** Orchestrate N agent sessions coordinated by a single lead LLM session. */
+export class OrchestrationLoop {
+  /**
+   * @param {object} deps
+   * @param {import("./agent-runner.js").AgentRunner} deps.leadRunner
+   * @param {Array<{name: string, role: string, runner: import("./agent-runner.js").AgentRunner}>} deps.agents
+   * @param {import("./message-bus.js").MessageBus} deps.messageBus
+   * @param {import("stream").Writable} deps.output
+   * @param {string} deps.leadName - Canonical name of the lead participant on the bus.
+   * @param {"facilitated"|"discussion"|"supervised"} deps.mode - Carries through to `protocol_violation` events.
+   * @param {object} deps.ctx - Orchestration context (from `createOrchestrationContext()`).
+   * @param {object} deps.redactor
+   * @param {number} [deps.maxLeadTurns] - Cap on lead resumes per session (default 200).
+   * @param {string} [deps.taskAmend] - Appended to the task before delivery.
+   * @param {import("./inbox-poller.js").InboxPoller} [deps.inboxPoller]
+   * @param {AbortController} [deps.abortController]
+   */
+  constructor({
+    leadRunner,
+    agents,
+    messageBus,
+    output,
+    leadName,
+    mode,
+    maxLeadTurns,
+    ctx,
+    taskAmend,
+    redactor,
+    inboxPoller,
+    abortController,
+  }) {
+    if (!leadRunner) throw new Error("leadRunner is required");
+    if (!agents) throw new Error("agents is required");
+    if (!messageBus) throw new Error("messageBus is required");
+    if (!output) throw new Error("output is required");
+    if (!leadName) throw new Error("leadName is required");
+    if (!mode) throw new Error("mode is required");
+    if (!ctx) throw new Error("ctx is required");
+    if (!redactor) throw new Error("redactor is required");
+    this.leadRunner = leadRunner;
+    this.agents = agents;
+    this.messageBus = messageBus;
+    this.output = output;
+    this.leadName = leadName;
+    this.mode = mode;
+    this.ctx = ctx;
+    this.redactor = redactor;
+    this.taskAmend = taskAmend ?? null;
+    this.maxLeadTurns = maxLeadTurns ?? DEFAULT_MAX_LEAD_TURNS;
+    this.inboxPoller = inboxPoller ?? null;
+    this.abortController = abortController ?? null;
+    this.counter = new SequenceCounter();
+    this.leadTurns = 0;
+    this.stopped = false;
+    let resolveDone;
+    this.donePromise = new Promise((r) => {
+      resolveDone = r;
+    });
+    this.#signalDone = resolveDone;
+  }
+
+  /** Internal — resolved when `stopped` flips true so waiters unblock. */
+  #signalDone;
+
+  /**
+   * Run the full orchestrated session.
+   * @param {string} task
+   * @returns {Promise<{success: boolean, turns: number}>}
+   */
+  async run(task) {
+    this.emitOrchestratorEvent({ type: "session_start" });
+    const initialTask = this.taskAmend
+      ? task
+        ? `${task}\n\n${this.taskAmend}`
+        : this.taskAmend
+      : task;
+
+    let firstError = null;
+    const abort = (err) => {
+      if (err && !firstError) firstError = err;
+      this.#stop();
+    };
+
+    // Start agent loops in parallel. Wrapped so a crash flips `stopped`
+    // but the wrapper itself resolves — Promise.allSettled below never
+    // sees an unhandled rejection.
+    const agentPromises = this.agents.map((a) =>
+      this.#runAgent(a).catch(abort),
+    );
+    const pollerPromise = this.inboxPoller?.run().catch(() => {});
+
+    try {
+      await this.#runLead(initialTask);
+    } catch (err) {
+      abort(err);
+    } finally {
+      this.#stop();
+    }
+
+    await Promise.allSettled([...agentPromises, pollerPromise].filter(Boolean));
+    if (firstError) throw firstError;
+
+    const success = this.ctx.concluded && this.ctx.verdict === "success";
+    this.emitSummary({
+      success,
+      verdict: this.ctx.verdict,
+      turns: this.leadTurns,
+      summary: this.ctx.summary,
+    });
+    return { success, turns: this.leadTurns };
+  }
+
+  #stop() {
+    if (this.stopped) return;
+    this.stopped = true;
+    this.#signalDone();
+    this.abortController?.abort();
+    for (const agent of this.agents) {
+      agent.runner.currentAbortController?.abort();
+    }
+    this.leadRunner.currentAbortController?.abort();
+  }
+
+  /**
+   * Lead loop. The lead's first turn carries the task; every subsequent
+   * turn is a resume triggered by something landing on its inbox.
+   *
+   * `messages.length === 0` from `#drainOrWait` means the session ended
+   * before any message arrived — that's the natural exit. If
+   * `drainOrWait` returned messages, deliver them even if the session
+   * concluded in the microtask window between wake-up and this check;
+   * the inbox already has them and they deserve to be seen.
+   */
+  async #runLead(initialTask) {
+    this.leadTurns = 1;
+    this.emitOrchestratorEvent({ type: "agent_start", agent: this.leadName });
+    await this.leadRunner.run(initialTask);
+    if (this.#exiting()) return;
+    await this.#settleOwedAsks(this.leadName, this.leadRunner);
+
+    while (!this.#exiting()) {
+      if (this.leadTurns >= this.maxLeadTurns) {
+        this.emitOrchestratorEvent({
+          type: "lead_turn_limit",
+          limit: this.maxLeadTurns,
+        });
+        return;
+      }
+      const messages = await this.#drainOrWait(this.leadName);
+      if (messages.length === 0) return;
+
+      this.leadTurns++;
+      const hasSynthetic = messages.some((m) => m.kind === "synthetic");
+      await this.leadRunner.resume(formatMessages(messages));
+      if (hasSynthetic) this.inboxPoller?.markActed();
+      if (this.#exiting()) return;
+      await this.#settleOwedAsks(this.leadName, this.leadRunner);
+    }
+  }
+
+  /**
+   * Agent loop. The first message off the inbox triggers `run()`; every
+   * subsequent batch triggers `resume()`. No turn budget — the agent
+   * runner's own `maxTurns` caps each SDK call.
+   */
+  async #runAgent({ name, runner }) {
+    let started = false;
+    while (!this.#exiting()) {
+      const messages = await this.#drainOrWait(name);
+      if (messages.length === 0) return;
+
+      if (!started) {
+        started = true;
+        this.emitOrchestratorEvent({ type: "agent_start", agent: name });
+        await runner.run(formatMessages(messages));
+      } else {
+        await runner.resume(formatMessages(messages));
+      }
+      if (this.#exiting()) return;
+      await this.#settleOwedAsks(name, runner);
+    }
+  }
+
+  /** Either an explicit Conclude or any abort path. */
+  #exiting() {
+    return this.stopped || this.ctx.concluded;
+  }
+
+  /**
+   * Drain the queue, or wait for the first message to arrive. Returns an
+   * empty array when the session ended before any message landed.
+   */
+  async #drainOrWait(name) {
+    let messages = this.messageBus.drain(name);
+    if (messages.length > 0) return messages;
+    await Promise.race([
+      this.messageBus.waitForMessages(name),
+      this.donePromise,
+    ]);
+    if (this.stopped) return [];
+    messages = this.messageBus.drain(name);
+    return messages;
+  }
+
+  /**
+   * If `name` left a pending Ask unanswered, inject one synthetic reminder
+   * and resume once more. If still unanswered after the reminder, emit a
+   * `protocol_violation` event per outstanding ask and cancel them — the
+   * asker's queue gets a synthetic `[no answer: …]` so it doesn't deadlock
+   * on a participant that's silently ignoring its inbox.
+   */
+  async #settleOwedAsks(name, runner) {
+    if (pendingAsksOwedBy(this.ctx, name).length === 0) return;
+    if (this.stopped) return;
+
+    const reminded = remindOwedAsks(this.ctx, name);
+    if (!reminded) return;
+    const reminders = this.messageBus.drain(name);
+    if (reminders.length === 0) return;
+
+    await runner.resume(formatMessages(reminders));
+    if (this.stopped) return;
+
+    const stillOwed = pendingAsksOwedBy(this.ctx, name);
+    if (stillOwed.length === 0) return;
+
+    for (const entry of stillOwed) {
+      this.emitOrchestratorEvent({
+        type: "protocol_violation",
+        agent: name,
+        askId: entry.askId,
+        mode: this.mode,
+      });
+    }
+    cancelPendingAsks(this.ctx, `${name} did not answer after reminder`, name);
+  }
+
+  /**
+   * Emit one NDJSON line tagged with its source (participant name) and a
+   * monotonic seq, wrapped in the universal `{source, seq, event}` envelope.
+   * Called from each runner's `onLine` callback.
+   * @param {string} source
+   * @param {string} line - Raw NDJSON line from the SDK iterator.
+   */
+  emitLine(source, line) {
+    const event = JSON.parse(line);
+    this.output.write(
+      JSON.stringify(
+        this.redactor.redactValue({
+          source,
+          seq: this.counter.next(),
+          event,
+        }),
+      ) + "\n",
+    );
+  }
+
+  /**
+   * Emit one orchestrator-source event (`session_start`, `agent_start`,
+   * `protocol_violation`, `lead_turn_limit`) wrapped in the universal
+   * envelope.
+   * @param {object} event
+   */
+  emitOrchestratorEvent(event) {
+    this.output.write(
+      JSON.stringify(
+        this.redactor.redactValue({
+          source: "orchestrator",
+          seq: this.counter.next(),
+          event,
+        }),
+      ) + "\n",
+    );
+  }
+
+  /**
+   * Emit the terminal summary line. `Discusser` emits its own discuss-
+   * augmented summary after this one; trace consumers keep the last
+   * summary they see.
+   * @param {{success: boolean, verdict?: string|null, turns: number, summary?: string|null}} result
+   */
+  emitSummary(result) {
+    this.output.write(
+      JSON.stringify(
+        this.redactor.redactValue({
+          source: "orchestrator",
+          seq: this.counter.next(),
+          event: {
+            type: "summary",
+            success: result.success,
+            ...(result.verdict && { verdict: result.verdict }),
+            turns: result.turns,
+            ...(result.summary && { summary: result.summary }),
+          },
+        }),
+      ) + "\n",
+    );
+  }
+}