@forwardimpact/libharness 1.3.0 → 2.0.0

package/bin/fit-harness.js

@@ -27,6 +27,20 @@ const LEAD_OPTIONS = {
   },
 };
 
+// Advisor consult flags, shared by all four session modes.
+const ADVISOR_OPTIONS = {
+  "advisor-model": {
+    type: "string",
+    description:
+      "Claude model for advisor consults; omitting the flag disables the Advisor tool (default: off)",
+  },
+  "advisor-max-uses": {
+    type: "string",
+    description:
+      "Session-wide consult budget shared by all participants (default: 3; requires --advisor-model)",
+  },
+};
+
 // Shared task-input flags: --task-file (path), --task-text (inline), and
 // --task-event (path to native GitHub event JSON composed into a task via
 // libharness/src/events/github.js). Exactly one of the three is required.
@@ -94,6 +108,7 @@ const definition = {
           description:
             "Connect to the MCP service (e.g. --mcp-server=guide); adds mcp__<name>__* to allowed tools",
         },
+        ...ADVISOR_OPTIONS,
       },
     },
     {
@@ -143,6 +158,7 @@ const definition = {
           description:
             "Connect to the MCP service (e.g. --mcp-server=guide); adds mcp__<name>__* to allowed tools",
         },
+        ...ADVISOR_OPTIONS,
       },
     },
     {
@@ -185,6 +201,7 @@ const definition = {
           description:
             "Active work-item tracker (github|filesystem, default: github)",
         },
+        ...ADVISOR_OPTIONS,
       },
     },
     {
@@ -231,6 +248,7 @@ const definition = {
           description:
             "Active work-item tracker (github|filesystem, default: github)",
         },
+        ...ADVISOR_OPTIONS,
       },
     },
     {

package/bin/fit-trace.js

@@ -398,6 +398,16 @@ const definition = {
           type: "string",
           description: "Custom failure message",
         },
+        gate: {
+          type: "boolean",
+          description:
+            "Mark the row a gate check (any failing gate fails the run)",
+        },
+        weight: {
+          type: "string",
+          description:
+            "Attach a numeric weight to the scored row; 0 marks the row diagnostic",
+        },
       },
     },
   ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libharness",
-  "version": "1.3.0",
+  "version": "2.0.0",
   "description": "Autonomous agent team harness — coordinate a lead and participant agents in one async session, with eval, benchmark, and trace tooling to prove the changes worked.",
   "keywords": [
     "orchestration",

package/src/advisor.js

@@ -0,0 +1,218 @@
+/**
+ * Advisor — the judge's mid-loop sibling: a solo, tool-restricted, one-shot
+ * `AgentRunner` session on a stronger model whose final text is the advice.
+ * Each consult forwards the caller's whole recorded context (system prompt,
+ * delivered prompts, transcript so far) plus a focused question; the advisor
+ * can inspect files read-only but holds no write, execute, subagent, or
+ * orchestration tools and never appears on the message bus.
+ *
+ * Consults are stateless — one fresh session per call, re-reading the
+ * caller's context as it stands — and fail-open: timeout, error, and abort
+ * all resolve to an in-band `{unavailable}` result so the caller's session
+ * never stalls or crashes on a consult.
+ *
+ * Follows OO+DI: factory function, tests inject a fake `query`.
+ */
+
+import { Writable } from "node:stream";
+
+import { createAgentRunner } from "./agent-runner.js";
+import { composeSystemPrompt } from "./profile-prompt.js";
+
+/**
+ * System-prompt trailer for the advisor session. Fixes the response
+ * contract (spec criterion "Advice is bounded"): assessment,
+ * recommendation, unsolicited findings, with a stated length ceiling.
+ */
+export const ADVISOR_SYSTEM_PROMPT =
+  "You are a consulted specialist, not a worker. " +
+  "Another agent paused its work to ask you one question; its full session context and the question are in the task. " +
+  "You may Read, Glob, and Grep the files the transcript names to ground your advice; never modify anything. " +
+  "Respond in one turn of prose — your final text is delivered to the caller verbatim. " +
+  "Structure the response as: assessment (what you see), recommendation (what to do and why), and unsolicited findings (anything important the caller did not ask about). " +
+  "Keep the whole response to at most three short paragraphs. " +
+  "Do not ask follow-up questions — the caller cannot reply.";
+
+/**
+ * Consult-guidance fragment for caller system prompts, present only when
+ * the session runs with an advisor model. Steers the caller's judgment; it
+ * mandates nothing.
+ * @param {number} maxUses - The session-wide consult budget.
+ * @returns {string}
+ */
+export function advisorGuidance(maxUses) {
+  return (
+    "An `Advisor` tool is available: one focused question per call, answered by a stronger model that sees your full session context. " +
+    "A consult pays off at hard decision points — architectural forks, unclear root causes, trade-offs you cannot rank — and early, before work builds on an unvalidated assumption. " +
+    "It does not pay off for routine reads, writes, or searches. " +
+    `The session-wide budget is ${maxUses} consult${maxUses === 1 ? "" : "s"}, shared across all participants. ` +
+    "Consulting is your judgment, never mandatory."
+  );
+}
+
+/**
+ * Create the session-wide consult budget, shared by every caller's tool
+ * handler. Enforced in code by the tool handler, not in the prompt.
+ * @param {number} maxUses
+ * @returns {{maxUses: number, used: number}}
+ */
+export function createAdvisorBudget(maxUses) {
+  return { maxUses, used: 0 };
+}
+
+/**
+ * Fold the consult guidance into an existing run-specific amendment when
+ * the advisor is enabled (budget present); return the amendment unchanged
+ * otherwise, so advisor-off prompts stay byte-identical.
+ * @param {string|undefined} amend - The existing amendment, if any.
+ * @param {{maxUses: number}|null} budget
+ * @returns {string|undefined}
+ */
+export function withAdvisorGuidance(amend, budget) {
+  if (!budget) return amend;
+  return [amend, advisorGuidance(budget.maxUses)].filter(Boolean).join("\n\n");
+}
+
+/** Consult timeout — generous for a read-a-few-files-and-answer session, and the universal guard in modes with no stop path. */
+export const DEFAULT_CONSULT_TIMEOUT_MS = 300_000;
+
+const ADVISOR_ALLOWED_TOOLS = ["Read", "Glob", "Grep"];
+
+// Under the harness's always-on bypassPermissions, `allowedTools` alone is
+// not structural — `disallowedTools` is what removes tools from the model's
+// context, the same treatment the lead runners get. The advisor's contract
+// is stricter than the leads' (read-only inspection, nothing else), so the
+// list also removes the write-capable and non-inspection built-ins the
+// lead convention leaves in.
+const ADVISOR_DISALLOWED_TOOLS = [
+  "Bash",
+  "Write",
+  "Edit",
+  "NotebookEdit",
+  "Agent",
+  "Task",
+  "TaskOutput",
+  "TaskStop",
+  "TodoWrite",
+  "WebFetch",
+  "WebSearch",
+];
+
+const devNull = new Writable({
+  write(_chunk, _enc, cb) {
+    cb();
+  },
+});
+
+/**
+ * Create a per-caller advisor closed over that caller's transcript
+ * recorder.
+ *
+ * @param {object} deps
+ * @param {string} deps.model - Advisor model id.
+ * @param {string} deps.cwd - The caller's working directory, so read-only inspection sees the caller's files.
+ * @param {function} deps.query - SDK query function (injected for testing).
+ * @param {{render: () => string}} deps.recorder - The caller's transcript recorder.
+ * @param {import("./redaction.js").Redactor} deps.redactor
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} deps.runtime - Clock surface for timeout and duration.
+ * @param {function} deps.onLine - Re-emitter for the advisor session's NDJSON lines (tagged `source: "advisor"` by the caller).
+ * @param {number} [deps.maxTurns] - Default 5 — single-digit per the spec criterion.
+ * @param {number} [deps.timeoutMs] - Default `DEFAULT_CONSULT_TIMEOUT_MS`.
+ * @returns {{consult: (question: string) => Promise<{advice?: string, unavailable?: boolean, reason?: string, durationMs: number}>, abort: () => void}}
+ */
+export function createAdvisor({
+  model,
+  cwd,
+  query,
+  recorder,
+  redactor,
+  runtime,
+  onLine,
+  maxTurns,
+  timeoutMs,
+}) {
+  if (!model) throw new Error("model is required");
+  if (!cwd) throw new Error("cwd is required");
+  if (!query) throw new Error("query is required");
+  if (!recorder) throw new Error("recorder is required");
+  if (!redactor) throw new Error("redactor is required");
+  if (!runtime) throw new Error("runtime is required");
+  if (!onLine) throw new Error("onLine is required");
+  const resolvedMaxTurns = maxTurns ?? 5;
+  const resolvedTimeoutMs = timeoutMs ?? DEFAULT_CONSULT_TIMEOUT_MS;
+
+  /** @type {import("./agent-runner.js").AgentRunner|null} */
+  let currentRunner = null;
+
+  return {
+    /**
+     * Run one fresh advisor session over the caller's context as it
+     * stands plus the question. Never rejects — every failure shape
+     * resolves to `{unavailable, reason}` (fail-open).
+     * @param {string} question
+     */
+    async consult(question) {
+      const started = runtime.clock.now();
+      const runner = createAgentRunner({
+        cwd,
+        query,
+        output: devNull,
+        model,
+        maxTurns: resolvedMaxTurns,
+        allowedTools: ADVISOR_ALLOWED_TOOLS,
+        disallowedTools: ADVISOR_DISALLOWED_TOOLS,
+        onLine,
+        settingSources: ["project"],
+        systemPrompt: composeSystemPrompt({
+          role: "agent",
+          trailer: ADVISOR_SYSTEM_PROMPT,
+          runtime,
+        }),
+        redactor,
+      });
+      currentRunner = runner;
+      const task = `${recorder.render()}\n\n<consult_question>\n${question}\n</consult_question>`;
+      const timer = runtime.clock.setTimeout(
+        () => runner.currentAbortController?.abort(),
+        resolvedTimeoutMs,
+      );
+      try {
+        const result = await runner.run(task);
+        const durationMs = runtime.clock.now() - started;
+        if (result.aborted) {
+          return {
+            unavailable: true,
+            reason: "timed out or aborted",
+            durationMs,
+          };
+        }
+        if (!result.success) {
+          return {
+            unavailable: true,
+            reason: result.error?.message ?? "advisor session failed",
+            durationMs,
+          };
+        }
+        return { advice: result.text, durationMs };
+      } catch (err) {
+        return {
+          unavailable: true,
+          reason: err?.message ?? "advisor session failed",
+          durationMs: runtime.clock.now() - started,
+        };
+      } finally {
+        runtime.clock.clearTimeout(timer);
+        currentRunner = null;
+      }
+    },
+
+    /**
+     * Abort the in-flight consult, if any. A consult is a blocking tool
+     * call, so one caller cannot overlap its own consults; advisors are
+     * per-caller, so at most one runner is ever tracked.
+     */
+    abort() {
+      currentRunner?.currentAbortController?.abort();
+    },
+  };
+}

package/src/agent-runner.js

@@ -53,6 +53,7 @@ export class AgentRunner {
    * @param {number} [deps.maxTurns] - Maximum agentic turns; 0 means unlimited
    * @param {string[]} [deps.allowedTools] - Tools the agent may use
    * @param {function} [deps.onLine] - Callback invoked with each NDJSON line as it's produced
+   * @param {function} [deps.onPrompt] - Callback invoked with the effective (amend-applied) prompt of each run/resume
    * @param {string[]} [deps.settingSources] - SDK setting sources (e.g. ['project'] to load CLAUDE.md)
    * @param {string|object} [deps.systemPrompt] - SDK system prompt (string replaces default; {type:'preset', preset:'claude_code', append} appends)
    * @param {string[]} [deps.disallowedTools] - Tools to explicitly remove from the model's context
@@ -80,6 +81,9 @@ export class AgentRunner {
     this.maxTurns = deps.maxTurns ?? 50;
     this.allowedTools = deps.allowedTools ?? DEFAULT_ALLOWED_TOOLS;
     this.onLine = deps.onLine ?? null;
+    // Optional; read only through a truthy guard in run()/resume(), so an
+    // absent value stays undefined rather than needing a `?? null` default.
+    this.onPrompt = deps.onPrompt;
     this.settingSources = deps.settingSources ?? [];
     this.systemPrompt = deps.systemPrompt ?? null;
     this.disallowedTools = deps.disallowedTools ?? [];
@@ -106,6 +110,7 @@ export class AgentRunner {
         ? `${task}\n\n${this.taskAmend}`
         : this.taskAmend
       : task;
+    if (this.onPrompt) this.onPrompt(effectiveTask);
     try {
       const iterator = this.query({
         prompt: effectiveTask,
@@ -125,6 +130,7 @@ export class AgentRunner {
   async resume(prompt) {
     const abortController = new AbortController();
     this.currentAbortController = abortController;
+    if (this.onPrompt) this.onPrompt(prompt);
     try {
       const iterator = this.query({
         prompt,

package/src/benchmark/grade.js

@@ -0,0 +1,222 @@
+/**
+ * Grading derivation — the sole home of the check-row arithmetic.
+ *
+ * Check rows are the single authoritative grading channel. Every row is a
+ * check by default; a row declares its role with its own fields, checked in
+ * order:
+ *
+ *   1. Gate       — `gate` is exactly `true`, `pass` is boolean, and no
+ *                   `weight` key is present. Any failing gate → `gatesPass`
+ *                   false.
+ *   2. Diagnostic — no `gate` key and `weight` is exactly `0`. Free-form;
+ *                   never graded.
+ *   3. Scored     — no `gate` key, boolean `pass`, `weight` absent (defaults
+ *                   to 1) or finite > 0.
+ *   4. Malformed  — everything else: any `gate`+`weight` co-occurrence (a
+ *                   stray weight must never silently disarm a gate), a
+ *                   non-boolean `gate`, a missing or non-boolean `pass` on a
+ *                   graded row, an invalid `weight`, an fd-3 line that failed
+ *                   to parse, a non-object row. Counts as a **failing scored
+ *                   check** — dropping a defect could mint full marks;
+ *                   failing the whole run would zero completed work.
+ *
+ * The producers' `source` stamp is display metadata, never a grading input.
+ */
+
+/**
+ * @typedef {object} GradeResult
+ * @property {"pass" | "fail"} verdict - `healthy ∧ gatesPass ∧ fullMarks`.
+ * @property {boolean} gatesPass - Every gate row passes (vacuously true).
+ * @property {number | null} score - Weighted fraction of passing scored
+ *   checks; `null` when the cell has zero scored checks (binary task).
+ * @property {boolean} fullMarks - Integer count predicate: no malformed rows
+ *   and every scored check passes. Never a float comparison, so fractional
+ *   weights carry no equality hazard. Vacuously true with zero scored checks.
+ * @property {number} malformed - Malformed row count.
+ */
+
+/**
+ * Grade the merged check rows against grader health.
+ *
+ * `healthy` is the completion signal a crashed grader cannot fake: when it is
+ * false the verdict is `fail` whatever the rows say, so a hook that dies
+ * after emitting passing rows can never mint marks.
+ * @param {unknown[]} details - Merged check rows from both producers.
+ * @param {boolean} healthy - Invariants exited 0 AND the hidden-test engine
+ *   did not throw.
+ * @returns {GradeResult}
+ */
+export function gradeChecks(details, healthy) {
+  const tally = {
+    gatesPass: true,
+    malformed: 0,
+    scored: 0,
+    passing: 0,
+    weightAll: 0,
+    weightPassing: 0,
+  };
+  for (const row of details) tallyRow(tally, row);
+
+  const score =
+    tally.scored + tally.malformed === 0
+      ? null
+      : tally.weightPassing / tally.weightAll;
+  const fullMarks = tally.malformed === 0 && tally.passing === tally.scored;
+  const verdict = healthy && tally.gatesPass && fullMarks ? "pass" : "fail";
+  return {
+    verdict,
+    gatesPass: tally.gatesPass,
+    score,
+    fullMarks,
+    malformed: tally.malformed,
+  };
+}
+
+/**
+ * Fold one row into the running tally per its classified role.
+ * @param {{gatesPass: boolean, malformed: number, scored: number, passing: number, weightAll: number, weightPassing: number}} tally
+ * @param {unknown} row
+ */
+function tallyRow(tally, row) {
+  const role = classifyRow(row);
+  if (role === "gate") {
+    if (!row.pass) tally.gatesPass = false;
+  } else if (role === "scored") {
+    const weight = row.weight ?? 1;
+    tally.scored++;
+    tally.weightAll += weight;
+    if (row.pass) {
+      tally.passing++;
+      tally.weightPassing += weight;
+    }
+  } else if (role === "malformed") {
+    tally.malformed++;
+    tally.weightAll += malformedWeight(row);
+  }
+}
+
+/**
+ * Run both check-row producers and grade the merged rows — the one
+ * composition shared by the runner and the `grade` subcommand. An engine
+ * throw is grader fault: its message lands on the returned `engineError`
+ * and health fails, so a crashed grader can never mint marks from rows it
+ * happened to emit first.
+ * @param {import("./task-family.js").Task} task
+ * @param {{cwd: string, port: number, runDir: string, familyDir?: string|null}} ctx
+ * @param {import("@forwardimpact/libutil/runtime").Runtime} runtime
+ * @param {{runInvariants: Function, runHiddenTests: Function}} producers -
+ *   The two producer functions (real implementations or test seams).
+ * @returns {Promise<{invariants: object, hiddenRows: object[], engineError: Error|null, rows: unknown[], healthy: boolean, grade: object}>}
+ */
+export async function runProducersAndGrade(task, ctx, runtime, producers) {
+  const invariants = await producers.runInvariants(task, ctx, runtime);
+  let hiddenRows = [];
+  let engineError = null;
+  try {
+    const hidden = await producers.runHiddenTests(task, ctx, runtime);
+    hiddenRows = hidden.details;
+  } catch (e) {
+    engineError = e;
+  }
+  const rows = mergeRows(invariants.details, hiddenRows);
+  const healthy = invariants.exitCode === 0 && !engineError;
+  const grade = normalizeGrade(gradeChecks(rows, healthy));
+  return { invariants, hiddenRows, engineError, rows, healthy, grade };
+}
+
+/**
+ * Merge the two producers' rows (invariants first) and stamp each row's
+ * provenance. The stamp is display metadata, never a grading input, and
+ * non-object rows (malformed by contract) pass through verbatim.
+ * @param {unknown[]} invariantsDetails
+ * @param {unknown[]} hiddenDetails
+ * @returns {unknown[]}
+ */
+export function mergeRows(invariantsDetails, hiddenDetails) {
+  return [
+    ...invariantsDetails.map((row) => stampSource(row, "invariants")),
+    ...hiddenDetails.map((row) => stampSource(row, "tests")),
+  ];
+}
+
+function stampSource(row, source) {
+  if (row === null || typeof row !== "object" || Array.isArray(row)) {
+    return row;
+  }
+  return { ...row, source };
+}
+
+/**
+ * Project the raw `gradeChecks` return onto the record schema: `fullMarks`
+ * is derivable and dropped, `score` is omitted on binary tasks (`null`),
+ * `malformed` is omitted when clean.
+ * @param {GradeResult} raw
+ * @returns {{verdict: "pass"|"fail", gatesPass: boolean, score?: number, malformed?: number}}
+ */
+export function normalizeGrade({ verdict, gatesPass, score, malformed }) {
+  return {
+    verdict,
+    gatesPass,
+    ...(score !== null && { score }),
+    ...(malformed > 0 && { malformed }),
+  };
+}
+
+/**
+ * Classify one row per the role order in the module contract.
+ * @param {unknown} row
+ * @returns {"gate" | "diagnostic" | "scored" | "malformed"}
+ */
+function classifyRow(row) {
+  if (row === null || typeof row !== "object" || Array.isArray(row)) {
+    return "malformed";
+  }
+  if ("gate" in row) return classifyGateRow(row);
+  if ("weight" in row) return classifyWeightedRow(row);
+  return typeof row.pass === "boolean" ? "scored" : "malformed";
+}
+
+/**
+ * A row carrying a `gate` key: valid only as `gate: true` with a boolean
+ * `pass` and no `weight` key — any co-occurring weight is malformed so a
+ * stray weight can never silently disarm a gate.
+ * @param {object} row
+ * @returns {"gate" | "malformed"}
+ */
+function classifyGateRow(row) {
+  if ("weight" in row) return "malformed";
+  return row.gate === true && typeof row.pass === "boolean"
+    ? "gate"
+    : "malformed";
+}
+
+/**
+ * A gate-less row carrying a `weight` key: exactly 0 is a diagnostic, a
+ * finite positive weight with a boolean `pass` is scored, anything else is
+ * malformed.
+ * @param {object} row
+ * @returns {"diagnostic" | "scored" | "malformed"}
+ */
+function classifyWeightedRow(row) {
+  if (row.weight === 0) return "diagnostic";
+  return isValidWeight(row.weight) && typeof row.pass === "boolean"
+    ? "scored"
+    : "malformed";
+}
+
+/**
+ * A malformed row fails at its own weight when it carries a valid positive
+ * one, else at unit weight 1.
+ * @param {unknown} row
+ * @returns {number}
+ */
+function malformedWeight(row) {
+  if (row !== null && typeof row === "object" && isValidWeight(row.weight)) {
+    return row.weight;
+  }
+  return 1;
+}
+
+function isValidWeight(w) {
+  return typeof w === "number" && Number.isFinite(w) && w > 0;
+}