@forwardimpact/libharness 1.3.0 → 1.4.0

package/bin/fit-harness.js

@@ -27,6 +27,20 @@ const LEAD_OPTIONS = {
   },
 };
 
+// Advisor consult flags, shared by all four session modes.
+const ADVISOR_OPTIONS = {
+  "advisor-model": {
+    type: "string",
+    description:
+      "Claude model for advisor consults; omitting the flag disables the Advisor tool (default: off)",
+  },
+  "advisor-max-uses": {
+    type: "string",
+    description:
+      "Session-wide consult budget shared by all participants (default: 3; requires --advisor-model)",
+  },
+};
+
 // Shared task-input flags: --task-file (path), --task-text (inline), and
 // --task-event (path to native GitHub event JSON composed into a task via
 // libharness/src/events/github.js). Exactly one of the three is required.
@@ -94,6 +108,7 @@ const definition = {
           description:
             "Connect to the MCP service (e.g. --mcp-server=guide); adds mcp__<name>__* to allowed tools",
         },
+        ...ADVISOR_OPTIONS,
       },
     },
     {
@@ -143,6 +158,7 @@ const definition = {
           description:
             "Connect to the MCP service (e.g. --mcp-server=guide); adds mcp__<name>__* to allowed tools",
         },
+        ...ADVISOR_OPTIONS,
       },
     },
     {
@@ -185,6 +201,7 @@ const definition = {
           description:
             "Active work-item tracker (github|filesystem, default: github)",
         },
+        ...ADVISOR_OPTIONS,
       },
     },
     {
@@ -231,6 +248,7 @@ const definition = {
           description:
             "Active work-item tracker (github|filesystem, default: github)",
         },
+        ...ADVISOR_OPTIONS,
       },
     },
     {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libharness",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Autonomous agent team harness — coordinate a lead and participant agents in one async session, with eval, benchmark, and trace tooling to prove the changes worked.",
   "keywords": [
     "orchestration",

package/src/advisor.js

@@ -0,0 +1,218 @@
+/**
+ * Advisor — the judge's mid-loop sibling: a solo, tool-restricted, one-shot
+ * `AgentRunner` session on a stronger model whose final text is the advice.
+ * Each consult forwards the caller's whole recorded context (system prompt,
+ * delivered prompts, transcript so far) plus a focused question; the advisor
+ * can inspect files read-only but holds no write, execute, subagent, or
+ * orchestration tools and never appears on the message bus.
+ *
+ * Consults are stateless — one fresh session per call, re-reading the
+ * caller's context as it stands — and fail-open: timeout, error, and abort
+ * all resolve to an in-band `{unavailable}` result so the caller's session
+ * never stalls or crashes on a consult.
+ *
+ * Follows OO+DI: factory function, tests inject a fake `query`.
+ */
+
+import { Writable } from "node:stream";
+
+import { createAgentRunner } from "./agent-runner.js";
+import { composeSystemPrompt } from "./profile-prompt.js";
+
+/**
+ * System-prompt trailer for the advisor session. Fixes the response
+ * contract (spec criterion "Advice is bounded"): assessment,
+ * recommendation, unsolicited findings, with a stated length ceiling.
+ */
+export const ADVISOR_SYSTEM_PROMPT =
+  "You are a consulted specialist, not a worker. " +
+  "Another agent paused its work to ask you one question; its full session context and the question are in the task. " +
+  "You may Read, Glob, and Grep the files the transcript names to ground your advice; never modify anything. " +
+  "Respond in one turn of prose — your final text is delivered to the caller verbatim. " +
+  "Structure the response as: assessment (what you see), recommendation (what to do and why), and unsolicited findings (anything important the caller did not ask about). " +
+  "Keep the whole response to at most three short paragraphs. " +
+  "Do not ask follow-up questions — the caller cannot reply.";
+
+/**
+ * Consult-guidance fragment for caller system prompts, present only when
+ * the session runs with an advisor model. Steers the caller's judgment; it
+ * mandates nothing.
+ * @param {number} maxUses - The session-wide consult budget.
+ * @returns {string}
+ */
+export function advisorGuidance(maxUses) {
+  return (
+    "An `Advisor` tool is available: one focused question per call, answered by a stronger model that sees your full session context. " +
+    "A consult pays off at hard decision points — architectural forks, unclear root causes, trade-offs you cannot rank — and early, before work builds on an unvalidated assumption. " +
+    "It does not pay off for routine reads, writes, or searches. " +
+    `The session-wide budget is ${maxUses} consult${maxUses === 1 ? "" : "s"}, shared across all participants. ` +
+    "Consulting is your judgment, never mandatory."
+  );
+}
+
+/**
+ * Create the session-wide consult budget, shared by every caller's tool
+ * handler. Enforced in code by the tool handler, not in the prompt.
+ * @param {number} maxUses
+ * @returns {{maxUses: number, used: number}}
+ */
+export function createAdvisorBudget(maxUses) {
+  return { maxUses, used: 0 };
+}
+
+/**
+ * Fold the consult guidance into an existing run-specific amendment when
+ * the advisor is enabled (budget present); return the amendment unchanged
+ * otherwise, so advisor-off prompts stay byte-identical.
+ * @param {string|undefined} amend - The existing amendment, if any.
+ * @param {{maxUses: number}|null} budget
+ * @returns {string|undefined}
+ */
+export function withAdvisorGuidance(amend, budget) {
+  if (!budget) return amend;
+  return [amend, advisorGuidance(budget.maxUses)].filter(Boolean).join("\n\n");
+}
+
+/** Consult timeout — generous for a read-a-few-files-and-answer session, and the universal guard in modes with no stop path. */
+export const DEFAULT_CONSULT_TIMEOUT_MS = 300_000;
+
+const ADVISOR_ALLOWED_TOOLS = ["Read", "Glob", "Grep"];
+
+// Under the harness's always-on bypassPermissions, `allowedTools` alone is
+// not structural — `disallowedTools` is what removes tools from the model's
+// context, the same treatment the lead runners get. The advisor's contract
+// is stricter than the leads' (read-only inspection, nothing else), so the
+// list also removes the write-capable and non-inspection built-ins the
+// lead convention leaves in.
+const ADVISOR_DISALLOWED_TOOLS = [
+  "Bash",
+  "Write",
+  "Edit",
+  "NotebookEdit",
+  "Agent",
+  "Task",
+  "TaskOutput",
+  "TaskStop",
+  "TodoWrite",
+  "WebFetch",
+  "WebSearch",
+];
+
+const devNull = new Writable({
+  write(_chunk, _enc, cb) {
+    cb();
+  },
+});
+
+/**
+ * Create a per-caller advisor closed over that caller's transcript
+ * recorder.
+ *
+ * @param {object} deps
+ * @param {string} deps.model - Advisor model id.
+ * @param {string} deps.cwd - The caller's working directory, so read-only inspection sees the caller's files.
+ * @param {function} deps.query - SDK query function (injected for testing).
+ * @param {{render: () => string}} deps.recorder - The caller's transcript recorder.
+ * @param {import("./redaction.js").Redactor} deps.redactor
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} deps.runtime - Clock surface for timeout and duration.
+ * @param {function} deps.onLine - Re-emitter for the advisor session's NDJSON lines (tagged `source: "advisor"` by the caller).
+ * @param {number} [deps.maxTurns] - Default 5 — single-digit per the spec criterion.
+ * @param {number} [deps.timeoutMs] - Default `DEFAULT_CONSULT_TIMEOUT_MS`.
+ * @returns {{consult: (question: string) => Promise<{advice?: string, unavailable?: boolean, reason?: string, durationMs: number}>, abort: () => void}}
+ */
+export function createAdvisor({
+  model,
+  cwd,
+  query,
+  recorder,
+  redactor,
+  runtime,
+  onLine,
+  maxTurns,
+  timeoutMs,
+}) {
+  if (!model) throw new Error("model is required");
+  if (!cwd) throw new Error("cwd is required");
+  if (!query) throw new Error("query is required");
+  if (!recorder) throw new Error("recorder is required");
+  if (!redactor) throw new Error("redactor is required");
+  if (!runtime) throw new Error("runtime is required");
+  if (!onLine) throw new Error("onLine is required");
+  const resolvedMaxTurns = maxTurns ?? 5;
+  const resolvedTimeoutMs = timeoutMs ?? DEFAULT_CONSULT_TIMEOUT_MS;
+
+  /** @type {import("./agent-runner.js").AgentRunner|null} */
+  let currentRunner = null;
+
+  return {
+    /**
+     * Run one fresh advisor session over the caller's context as it
+     * stands plus the question. Never rejects — every failure shape
+     * resolves to `{unavailable, reason}` (fail-open).
+     * @param {string} question
+     */
+    async consult(question) {
+      const started = runtime.clock.now();
+      const runner = createAgentRunner({
+        cwd,
+        query,
+        output: devNull,
+        model,
+        maxTurns: resolvedMaxTurns,
+        allowedTools: ADVISOR_ALLOWED_TOOLS,
+        disallowedTools: ADVISOR_DISALLOWED_TOOLS,
+        onLine,
+        settingSources: ["project"],
+        systemPrompt: composeSystemPrompt({
+          role: "agent",
+          trailer: ADVISOR_SYSTEM_PROMPT,
+          runtime,
+        }),
+        redactor,
+      });
+      currentRunner = runner;
+      const task = `${recorder.render()}\n\n<consult_question>\n${question}\n</consult_question>`;
+      const timer = runtime.clock.setTimeout(
+        () => runner.currentAbortController?.abort(),
+        resolvedTimeoutMs,
+      );
+      try {
+        const result = await runner.run(task);
+        const durationMs = runtime.clock.now() - started;
+        if (result.aborted) {
+          return {
+            unavailable: true,
+            reason: "timed out or aborted",
+            durationMs,
+          };
+        }
+        if (!result.success) {
+          return {
+            unavailable: true,
+            reason: result.error?.message ?? "advisor session failed",
+            durationMs,
+          };
+        }
+        return { advice: result.text, durationMs };
+      } catch (err) {
+        return {
+          unavailable: true,
+          reason: err?.message ?? "advisor session failed",
+          durationMs: runtime.clock.now() - started,
+        };
+      } finally {
+        runtime.clock.clearTimeout(timer);
+        currentRunner = null;
+      }
+    },
+
+    /**
+     * Abort the in-flight consult, if any. A consult is a blocking tool
+     * call, so one caller cannot overlap its own consults; advisors are
+     * per-caller, so at most one runner is ever tracked.
+     */
+    abort() {
+      currentRunner?.currentAbortController?.abort();
+    },
+  };
+}

package/src/agent-runner.js

@@ -53,6 +53,7 @@ export class AgentRunner {
    * @param {number} [deps.maxTurns] - Maximum agentic turns; 0 means unlimited
    * @param {string[]} [deps.allowedTools] - Tools the agent may use
    * @param {function} [deps.onLine] - Callback invoked with each NDJSON line as it's produced
+   * @param {function} [deps.onPrompt] - Callback invoked with the effective (amend-applied) prompt of each run/resume
    * @param {string[]} [deps.settingSources] - SDK setting sources (e.g. ['project'] to load CLAUDE.md)
    * @param {string|object} [deps.systemPrompt] - SDK system prompt (string replaces default; {type:'preset', preset:'claude_code', append} appends)
    * @param {string[]} [deps.disallowedTools] - Tools to explicitly remove from the model's context
@@ -80,6 +81,9 @@ export class AgentRunner {
     this.maxTurns = deps.maxTurns ?? 50;
     this.allowedTools = deps.allowedTools ?? DEFAULT_ALLOWED_TOOLS;
     this.onLine = deps.onLine ?? null;
+    // Optional; read only through a truthy guard in run()/resume(), so an
+    // absent value stays undefined rather than needing a `?? null` default.
+    this.onPrompt = deps.onPrompt;
     this.settingSources = deps.settingSources ?? [];
     this.systemPrompt = deps.systemPrompt ?? null;
     this.disallowedTools = deps.disallowedTools ?? [];
@@ -106,6 +110,7 @@ export class AgentRunner {
         ? `${task}\n\n${this.taskAmend}`
         : this.taskAmend
       : task;
+    if (this.onPrompt) this.onPrompt(effectiveTask);
     try {
       const iterator = this.query({
         prompt: effectiveTask,
@@ -125,6 +130,7 @@ export class AgentRunner {
   async resume(prompt) {
     const abortController = new AbortController();
     this.currentAbortController = abortController;
+    if (this.onPrompt) this.onPrompt(prompt);
     try {
       const iterator = this.query({
         prompt,

package/src/benchmark/report.js

@@ -143,11 +143,19 @@ export function renderTextReport(report, kValues) {
 }
 
 // ---------------------------------------------------------------------------
-// Compact report (legacy path)
+// Compact report — status line + pass@k table, no per-task detail. Selected by
+// `report --detail=compact` (aggregate without `includeRuns`); the per-shard
+// summary uses it so a sharded run stays short while the merge job renders the
+// full report over the combined ledger.
 // ---------------------------------------------------------------------------
 
 function renderCompactReport(report, kValues) {
+  const { totals } = report;
+  const passing = report.tasks.filter((t) => t.c > 0 && t.c === t.n).length;
+  const icon = statusIcon(passing === totals.tasks);
   const lines = [
+    `${icon} **${passing}/${totals.tasks} tasks passing** | ${totals.runs} runs${totals.skipped ? ` | ${totals.skipped} skipped` : ""}`,
+    "",
     renderPassAtKTable(report, kValues),
     "",
     renderTotalsLine(report),

package/src/commands/advisor-flags.js

@@ -0,0 +1,28 @@
+/**
+ * Shared advisor-flag parsing for the four session-mode commands. The two
+ * flags are identical everywhere: `--advisor-model` (no default — absent
+ * means the Advisor tool is not offered) and `--advisor-max-uses`
+ * (default 3), which is a usage error without the model flag.
+ */
+
+/**
+ * Parse `--advisor-model` / `--advisor-max-uses` from parsed option values.
+ * A malformed max-uses is a usage error, not a silent fallback: NaN would
+ * make the budget check (`used >= maxUses`) permanently false and disable
+ * the code-enforced cap the flag exists to guarantee.
+ * @param {object} values - Parsed option values from cli.parse()
+ * @returns {{advisorModel: string|undefined, advisorMaxUses: number}}
+ */
+export function parseAdvisorOptions(values) {
+  if (values["advisor-max-uses"] && !values["advisor-model"]) {
+    throw new Error("--advisor-max-uses requires --advisor-model");
+  }
+  const advisorMaxUses = parseInt(values["advisor-max-uses"] || "3", 10);
+  if (Number.isNaN(advisorMaxUses) || advisorMaxUses < 1) {
+    throw new Error("--advisor-max-uses must be a positive integer");
+  }
+  return {
+    advisorModel: values["advisor-model"] || undefined,
+    advisorMaxUses,
+  };
+}

package/src/commands/benchmark-definition.js

@@ -140,6 +140,11 @@ export const definition = {
           type: "string",
           description: "Output format (json|text, default: json)",
         },
+        detail: {
+          type: "string",
+          description:
+            "Text report verbosity (full|compact, default: full). compact omits per-task detail — useful for sharded run summaries.",
+        },
       },
     },
   ],

package/src/commands/benchmark-report.js

@@ -1,7 +1,9 @@
 /**
  * `fit-benchmark report` — aggregate `results.jsonl` into pass@k via the
  * OpenAI HumanEval estimator. Output is JSON by default; pass --format=text
- * to render a markdown table.
+ * to render a markdown table. --detail=compact drops the per-task detail
+ * sections so a sharded run's per-shard summary stays short (the merge job
+ * renders the full report over the combined ledger).
  */
 
 import { resolve } from "node:path";
@@ -35,11 +37,19 @@ export async function runBenchmarkReportCommand(ctx) {
   if (format !== "json" && format !== "text") {
     return { ok: false, code: 1, error: "--format must be 'json' or 'text'" };
   }
+  const detail = values.detail ?? "full";
+  if (detail !== "full" && detail !== "compact") {
+    return {
+      ok: false,
+      code: 1,
+      error: "--detail must be 'full' or 'compact'",
+    };
+  }
 
   const report = await aggregate({
     inputDir: resolve(inputDir),
     kValues,
-    includeRuns: format === "text",
+    includeRuns: format === "text" && detail === "full",
     runtime,
   });
   if (format === "text") {

package/src/commands/discuss.js

@@ -3,6 +3,7 @@ import { isoTimestamp } from "@forwardimpact/libutil";
 import { createDiscusser } from "../discusser.js";
 import { createRedactor } from "../redaction.js";
 import { createTeeWriter } from "../tee-writer.js";
+import { parseAdvisorOptions } from "./advisor-flags.js";
 import { resolveTaskContent } from "./task-input.js";
 import { resolveWorkTracker } from "./work-tracker.js";
 import { AGENT_MODEL, LEAD_MODEL } from "@forwardimpact/libutil/models";
@@ -67,6 +68,7 @@ export function parseDiscussOptions(values, runtime) {
     callbackUrl: runtime.proc.env.CALLBACK_URL ?? null,
     inboxUrl: runtime.proc.env.INBOX_URL ?? null,
     correlationId: runtime.proc.env.CORRELATION_ID ?? null,
+    ...parseAdvisorOptions(values),
   };
 }
 
@@ -121,6 +123,8 @@ export async function runDiscussCommand(ctx) {
     inboxUrl: opts.inboxUrl,
     correlationId: opts.correlationId,
     runtime,
+    advisorModel: opts.advisorModel,
+    advisorMaxUses: opts.advisorMaxUses,
   });
 
   const result = await discusser.run(opts.taskContent);

package/src/commands/facilitate.js

@@ -3,6 +3,7 @@ import { isoTimestamp } from "@forwardimpact/libutil";
 import { createFacilitator } from "../facilitator.js";
 import { createRedactor } from "../redaction.js";
 import { createTeeWriter } from "../tee-writer.js";
+import { parseAdvisorOptions } from "./advisor-flags.js";
 import { resolveTaskContent } from "./task-input.js";
 import { resolveWorkTracker } from "./work-tracker.js";
 import { AGENT_MODEL, LEAD_MODEL } from "@forwardimpact/libutil/models";
@@ -60,6 +61,7 @@ export function parseFacilitateOptions(values, runtime) {
     outputPath: values.output,
     facilitatorProfile: values["lead-profile"] || undefined,
     workTracker: resolveWorkTracker(values, runtime?.proc?.env),
+    ...parseAdvisorOptions(values),
   };
 }
 
@@ -112,6 +114,8 @@ export async function runFacilitateCommand(ctx) {
     taskAmend: opts.taskAmend,
     redactor,
     runtime,
+    advisorModel: opts.advisorModel,
+    advisorMaxUses: opts.advisorMaxUses,
   });
 
   const result = await facilitator.run(opts.taskContent);