@forwardimpact/libeval 0.1.54 → 0.1.56

package/bin/fit-trace.js

@@ -41,7 +41,7 @@ const definition = {
       argsUsage: "[pattern]",
       handler: runRunsCommand,
       description:
-        "List recent GitHub Actions workflow runs (default pattern: agent)",
+        "List recent GitHub Actions workflow runs (default pattern: kata|agent)",
       options: {
         lookback: {
           type: "string",
@@ -59,7 +59,8 @@ const definition = {
       args: ["run-id"],
       argsUsage: "<run-id>",
       handler: runDownloadCommand,
-      description: "Download trace artifact and convert to structured JSON",
+      description:
+        "Download trace artifact and convert to structured JSON; pass --artifact to pick one when a matrix workflow emits multiple `trace--*` artifacts",
       options: {
         dir: { type: "string", description: "Output directory" },
         artifact: { type: "string", description: "Artifact name override" },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.54",
+  "version": "0.1.56",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/commands/trace.js

@@ -25,9 +25,8 @@ export async function runRunsCommand(ctx) {
     repo: ctx.options.repo,
     runtime,
   });
-  const pattern = ctx.args.pattern ?? "agent";
   const lookback = ctx.options.lookback ?? "7d";
-  const runs = await gh.listRuns({ pattern, lookback });
+  const runs = await gh.listRuns({ pattern: ctx.args.pattern, lookback });
   writeJSON(runtime, runs, ctx.options);
   return { ok: true };
 }

package/src/index.js

@@ -6,6 +6,7 @@ export {
   createTraceGitHub,
   detectRepoSlug,
   parseGitRemote,
+  pickTraceArtifact,
 } from "./trace-github.js";
 export { AgentRunner, createAgentRunner } from "./agent-runner.js";
 export {

package/src/supervisor.js

@@ -30,8 +30,7 @@ import { OrchestrationLoop } from "./orchestration-loop.js";
 /** System prompt for the supervisor lead. L0 mechanics only per COALIGNED. */
 export const SUPERVISOR_SYSTEM_PROMPT =
   "You supervise one agent.\n" +
-  "You have no tools to perform work yourself.\n" +
-  "Use `Ask` to delegate work to the agent.\n" +
+  "Use `Ask` to delegate the agent's task to the agent.\n" +
   "`Ask` is async and returns {askIds:[N]} immediately.\n" +
   "The reply arrives on your next turn as `[answer#N] agent: <text>` in your inbox.\n" +
   "End your turn while Asks are pending. The system resumes you when an answer arrives.\n" +
@@ -196,7 +195,6 @@ export function createSupervisor({
     "Task",
     "TaskOutput",
     "TaskStop",
-    "Bash",
     "Write",
     "Edit",
   ];
@@ -210,7 +208,7 @@ export function createSupervisor({
     output: devNull,
     model: supervisorModel ?? model,
     maxTurns: perRunBudget,
-    allowedTools: supervisorAllowedTools ?? ["Read", "Glob", "Grep"],
+    allowedTools: supervisorAllowedTools ?? ["Read", "Glob", "Grep", "Bash"],
     disallowedTools,
     onLine: (line) => supervisor.emitLine("supervisor", line),
     settingSources: ["project"],

package/src/trace-github.js

@@ -31,13 +31,13 @@ export class TraceGitHub {
    * List recent workflow runs, optionally filtered by name pattern.
    *
    * @param {object} [opts]
-   * @param {string} [opts.pattern] - Case-insensitive substring to match workflow name (default: "agent")
+   * @param {string} [opts.pattern] - Case-insensitive regex to match workflow name (default: "kata|agent" — covers `Kata: Shift`, `Kata: Dispatch`, and any `agent`-named workflow)
    * @param {number} [opts.limit=50] - Max runs to return from GitHub API
    * @param {string} [opts.lookback="7d"] - How far back to search (e.g. "7d", "24h", "2w")
    * @returns {Promise<object[]>} Array of {workflow, runId, status, conclusion, createdAt, branch, url}
    */
   async listRuns(opts = {}) {
-    const { pattern = "agent", limit = 50, lookback = "7d" } = opts;
+    const { pattern = "kata|agent", limit = 50, lookback = "7d" } = opts;
     const cutoff = parseLookback(lookback, this.runtime.clock.now());
 
     const params = new URLSearchParams({
@@ -68,10 +68,10 @@ export class TraceGitHub {
   /**
    * Download a trace artifact from a workflow run and extract it.
    *
-   * When `opts.name` is set, looks up that exact artifact. Otherwise picks the
-   * best match from the unified `trace--<case>--<participant>.<role>` naming
-   * convention: prefer a `*.raw` artifact (combined log), then any `*.agent`,
-   * then the first `trace--*` artifact found.
+   * When `opts.name` is set, looks up that exact artifact. Otherwise picks
+   * the single `trace--*` artifact if exactly one exists, or throws with a
+   * disambiguation list when matrix workflows emit multiple per-participant
+   * artifacts (see {@link pickTraceArtifact}).
    *
    * @param {number|string} runId
    * @param {object} [opts]
@@ -88,28 +88,7 @@ export class TraceGitHub {
     const url = `${API}/repos/${this.owner}/${this.repo}/actions/runs/${runId}/artifacts`;
     const data = await this.#get(url);
     const artifacts = data.artifacts ?? [];
-
-    // Find the trace artifact.
-    let artifact = null;
-    if (opts.name) {
-      artifact = artifacts.find((a) => a.name === opts.name);
-    } else {
-      const traceArtifacts = artifacts.filter((a) =>
-        a.name.startsWith("trace--"),
-      );
-      artifact =
-        traceArtifacts.find((a) => a.name.endsWith(".raw")) ??
-        traceArtifacts.find((a) => a.name.endsWith(".agent")) ??
-        traceArtifacts[0] ??
-        null;
-    }
-
-    if (!artifact) {
-      const available = artifacts.map((a) => a.name).join(", ");
-      throw new Error(
-        `No trace artifact found for run ${runId}. Available: ${available || "none"}`,
-      );
-    }
+    const artifact = pickTraceArtifact(artifacts, opts.name, runId);
 
     // Download the zip.
     const zipPath = path.join(dir, `${artifact.name}.zip`);
@@ -172,6 +151,45 @@ export class TraceGitHub {
   }
 }
 
+/**
+ * Pick the trace artifact to download from a workflow run's artifact list.
+ *
+ * When `name` is given, returns the exact match or throws with the available
+ * names. When `name` is omitted, returns the only `trace--*` artifact if
+ * there is exactly one; if there are multiple (matrix workflows like
+ * `kata-shift.yml` emit one `trace--<participant>` per cell), throws and
+ * lists them so the caller can pass `--name` to disambiguate.
+ *
+ * @param {Array<{name: string}>} artifacts - Artifact list from the GitHub API.
+ * @param {string} [name] - Exact artifact name to match.
+ * @param {number|string} [runId] - Run id for error messages.
+ * @returns {{name: string}} The selected artifact.
+ */
+export function pickTraceArtifact(artifacts, name, runId) {
+  const runRef = runId == null ? "" : ` for run ${runId}`;
+  if (name) {
+    const found = artifacts.find((a) => a.name === name);
+    if (found) return found;
+    const available = artifacts.map((a) => a.name).join(", ");
+    throw new Error(
+      `No artifact named "${name}"${runRef}. Available: ${available || "none"}`,
+    );
+  }
+
+  const traceArtifacts = artifacts.filter((a) => a.name.startsWith("trace--"));
+  if (traceArtifacts.length === 1) return traceArtifacts[0];
+  if (traceArtifacts.length === 0) {
+    const available = artifacts.map((a) => a.name).join(", ");
+    throw new Error(
+      `No trace artifact found${runRef}. Available: ${available || "none"}`,
+    );
+  }
+  const names = traceArtifacts.map((a) => a.name).join(", ");
+  throw new Error(
+    `Multiple trace artifacts found${runRef}: ${names}. Pass --name to choose one.`,
+  );
+}
+
 /**
  * Parse a lookback duration string into an ISO date string.
  * Supports: Nd (days), Nh (hours), Nw (weeks).