@forwardimpact/libeval 0.1.63 → 0.1.65

package/LICENSE

@@ -1,201 +1,21 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to the Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright 2026 Dick Olsson
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+MIT License
+
+Copyright (c) 2019 Zachary Rice
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/bin/fit-trace.js

@@ -9,6 +9,7 @@ import { createLogger } from "@forwardimpact/libtelemetry";
 
 import {
   runRunsCommand,
+  runFindCommand,
   runDownloadCommand,
   runOverviewCommand,
   runCountCommand,
@@ -22,14 +23,30 @@ import {
   runReasoningCommand,
   runTimelineCommand,
   runStatsCommand,
+  runCostCommand,
   runInitCommand,
   runTurnCommand,
   runFilterCommand,
   runSplitCommand,
+  runToolCallsCommand,
+  runCommandsCommand,
+  runPathsCommand,
+  runCompareCommand,
 } from "../src/commands/trace.js";
 import { runAssertCommand } from "../src/commands/assert.js";
 import { runByDiscussionCommand } from "../src/commands/by-discussion.js";
 
+// Cross-trace verbs take one or more trace files via repeated `--file`
+// (libcli's named-slot `dispatch()` has no variadic positional). A value with
+// glob metacharacters is expanded by the handler via `runtime.fsSync.globSync`.
+const fileOption = () => ({
+  file: {
+    type: "string",
+    multiple: true,
+    description: "Trace file (repeat or pass a quoted glob for several)",
+  },
+});
+
 const definition = {
   name: "fit-trace",
   description:
@@ -47,6 +64,30 @@ const definition = {
           type: "string",
           description: "How far back to search (default: 7d)",
         },
+        participant: {
+          type: "string",
+          description:
+            "Filter to runs carrying this participant's trace lane; in-progress candidates are labeled, not dropped",
+        },
+        repo: {
+          type: "string",
+          description:
+            "GitHub repo override (default: $GITHUB_REPOSITORY or 'origin' git remote)",
+        },
+      },
+    },
+    {
+      name: "find",
+      args: ["run-id", "participant"],
+      argsUsage: "<run-id> <participant>",
+      handler: runFindCommand,
+      description:
+        "Resolve a participant's lane trace for a known run id in one keyed lookup (no run enumeration, no content inspection)",
+      options: {
+        dir: {
+          type: "string",
+          description: "Output directory for a downloaded dispatch artifact",
+        },
         repo: {
           type: "string",
           description:
@@ -73,17 +114,17 @@ const definition = {
     },
     {
       name: "overview",
-      args: ["file"],
-      argsUsage: "<file>",
+      args: [],
       handler: runOverviewCommand,
       description: "Metadata, summary, turn count, tool frequency",
+      options: fileOption(),
     },
     {
       name: "count",
-      args: ["file"],
-      argsUsage: "<file>",
+      args: [],
       handler: runCountCommand,
       description: "Number of turns",
+      options: fileOption(),
     },
     {
       name: "batch",
@@ -94,17 +135,23 @@ const definition = {
     },
     {
       name: "head",
-      args: ["file", "n"],
-      argsUsage: "<file> [N]",
+      args: [],
       handler: runHeadCommand,
-      description: "First N turns (default 10)",
+      description: "First N turns (default 10; set with --lines)",
+      options: {
+        ...fileOption(),
+        lines: { type: "string", description: "Number of turns (default: 10)" },
+      },
     },
     {
       name: "tail",
-      args: ["file", "n"],
-      argsUsage: "<file> [N]",
+      args: [],
       handler: runTailCommand,
-      description: "Last N turns (default 10)",
+      description: "Last N turns (default 10; set with --lines)",
+      options: {
+        ...fileOption(),
+        lines: { type: "string", description: "Number of turns (default: 10)" },
+      },
     },
     {
       name: "search",
@@ -129,56 +176,127 @@ const definition = {
     },
     {
       name: "tools",
-      args: ["file"],
-      argsUsage: "<file>",
+      args: [],
       handler: runToolsCommand,
-      description: "Tool usage frequency (descending)",
+      description:
+        "Tool usage frequency (descending). See also `tool` (turns for one tool) and `tool-calls` (paired use+result records)",
+      options: fileOption(),
     },
     {
       name: "tool",
       args: ["file", "name"],
       argsUsage: "<file> <name>",
       handler: runToolCommand,
-      description: "All turns involving a specific tool",
+      description:
+        "All turns involving a specific tool. See also `tools` (frequency) and `tool-calls` (paired use+result records)",
+    },
+    {
+      name: "tool-calls",
+      args: [],
+      handler: runToolCallsCommand,
+      description:
+        "One record per tool_use block, each paired with its tool_result by toolUseId (orphans emit result:null). See also `tool` and `tools`",
+      options: fileOption(),
+    },
+    {
+      name: "commands",
+      args: [],
+      handler: runCommandsCommand,
+      description:
+        "One record per Bash tool_use block, carrying the command text",
+      options: {
+        ...fileOption(),
+        match: {
+          type: "string",
+          description: "Filter to commands whose text matches this regex",
+        },
+      },
+    },
+    {
+      name: "paths",
+      args: [],
+      handler: runPathsCommand,
+      description:
+        "Distinct Read/Edit/Write file_path arguments, frequency-sorted",
+      options: {
+        ...fileOption(),
+        prefix: {
+          type: "string",
+          description: "Filter to paths beginning with this prefix",
+        },
+      },
+    },
+    {
+      name: "compare",
+      args: ["file-a", "file-b"],
+      argsUsage: "<file-a> <file-b>",
+      handler: runCompareCommand,
+      description:
+        "Side-by-side comparison of two traces: turns, tools, paths, cost, and per-tool delta",
     },
     {
       name: "errors",
-      args: ["file"],
-      argsUsage: "<file>",
+      args: [],
       handler: runErrorsCommand,
       description: "Tool results with isError=true",
+      options: fileOption(),
     },
     {
       name: "reasoning",
-      args: ["file"],
-      argsUsage: "<file>",
+      args: [],
       handler: runReasoningCommand,
       description: "Agent reasoning text only",
       options: {
+        ...fileOption(),
         from: { type: "string", description: "Start at turn index" },
         to: { type: "string", description: "Stop before turn index" },
       },
     },
     {
       name: "timeline",
-      args: ["file"],
-      argsUsage: "<file>",
+      args: [],
       handler: runTimelineCommand,
       description: "Compact one-line-per-turn overview",
+      options: fileOption(),
     },
     {
       name: "stats",
-      args: ["file"],
-      argsUsage: "<file>",
+      args: [],
       handler: runStatsCommand,
       description: "Token usage and cost breakdown",
+      options: {
+        ...fileOption(),
+        "by-tool": {
+          type: "boolean",
+          description: "Per-tool token attribution and cost share",
+        },
+        summary: {
+          type: "boolean",
+          description: "Totals only (suppress the per-turn array)",
+        },
+      },
     },
     {
-      name: "init",
+      name: "cost",
       args: ["file"],
       argsUsage: "<file>",
+      handler: runCostCommand,
+      description:
+        "Total run cost across every participant (agent, supervisor, judge, named profiles), with a per-source breakdown",
+      options: {
+        markdown: {
+          type: "boolean",
+          description:
+            "Emit a GitHub-flavored markdown block (redirect into $GITHUB_STEP_SUMMARY)",
+        },
+      },
+    },
+    {
+      name: "init",
+      args: [],
       handler: runInitCommand,
       description: "Full system/init event",
+      options: fileOption(),
     },
     {
       name: "turn",
@@ -203,11 +321,11 @@ const definition = {
     },
     {
       name: "filter",
-      args: ["file"],
-      argsUsage: "<file>",
+      args: [],
       handler: runFilterCommand,
       description: "Filter turns by role, tool, or error status",
       options: {
+        ...fileOption(),
         role: {
           type: "string",
           description: "Turn role (system, user, assistant, tool_result)",
@@ -286,7 +404,15 @@ const definition = {
   globalOptions: {
     help: { type: "boolean", short: "h", description: "Show this help" },
     version: { type: "boolean", description: "Show version" },
-    json: { type: "boolean", description: "Output help as JSON" },
+    json: {
+      type: "boolean",
+      description: "Output help as JSON (use --format json for command output)",
+    },
+    format: {
+      type: "string",
+      default: "text",
+      description: "Command output format: text (default) or json",
+    },
     signatures: {
       type: "boolean",
       description: "Include thinking.signature blobs in output",
@@ -294,14 +420,23 @@ const definition = {
   },
   examples: [
     "fit-trace runs --lookback 7d",
+    "fit-trace runs --participant release-engineer",
+    "fit-trace find 27401632821 release-engineer",
     "fit-trace download 24497273755",
     "fit-trace split structured.json --mode=facilitate",
-    "fit-trace overview structured.json",
-    "fit-trace timeline structured.json",
-    "fit-trace stats structured.json",
+    "fit-trace overview --file structured.json",
+    "fit-trace overview --file structured.json --format json",
+    "fit-trace timeline --file structured.json",
+    "fit-trace stats --file structured.json --by-tool",
+    "fit-trace cost trace.ndjson",
+    "fit-trace cost trace.ndjson --markdown",
     "fit-trace tool structured.json Conclude",
+    "fit-trace tool-calls --file structured.json",
+    "fit-trace commands --file structured.json --match '^git'",
+    "fit-trace paths --file 'traces/*.ndjson' --prefix /app",
+    "fit-trace compare trace-a.ndjson trace-b.ndjson",
     "fit-trace search structured.json 'error|fail' --context 1",
-    "fit-trace filter structured.json --tool Bash --error",
+    "fit-trace filter --file structured.json --tool Bash --error",
     "fit-trace turn structured.json 3",
     "fit-trace assert has-heading --grep '^## Problem' spec.md",
     "fit-trace assert no-leak --not --grep 'password' output.log",
@@ -336,7 +471,7 @@ const logger = createLogger("trace", runtime);
 
 // Commands that talk to the GitHub API need a config-backed token resolver;
 // the rest only read local trace files through the runtime.
-const NEEDS_CONFIG = new Set(["runs", "download"]);
+const NEEDS_CONFIG = new Set(["runs", "find", "download"]);
 
 async function main() {
   const cli = createCli(definition, {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.63",
+  "version": "0.1.65",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/benchmark/judge.js

@@ -23,11 +23,13 @@
 
 import { createJudge } from "../judge.js";
 import { createRedactor } from "../redaction.js";
+import { sumTraceCost } from "../cost.js";
 
 /**
  * @typedef {object} JudgeVerdict
  * @property {"pass" | "fail"} verdict
  * @property {string} summary
+ * @property {number} costUsd - Cost of the judge's own SDK session.
  */
 
 /**
@@ -81,12 +83,25 @@ export async function runJudge(task, workdir, invariants, deps, context) {
     await new Promise((r) => output.end(r));
   }
 
+  // The judge is its own SDK session; its spend lands in the judge trace we
+  // just wrote, not in the supervisor's combined trace. Read it back so the
+  // benchmark record's cost includes the judge.
+  const judgeTrace = await fs
+    .readFile(workdir.judgeTracePath, "utf8")
+    .catch(() => "");
+  const { totalCostUsd } = sumTraceCost(judgeTrace.split("\n"));
+
   if (outcome.verdict === null) {
-    return { verdict: "fail", summary: "judge did not conclude" };
+    return {
+      verdict: "fail",
+      summary: "judge did not conclude",
+      costUsd: totalCostUsd,
+    };
   }
   return {
     verdict: outcome.verdict === "success" ? "pass" : "fail",
     summary: outcome.summary ?? "",
+    costUsd: totalCostUsd,
   };
 }
 

package/src/benchmark/result.js

@@ -27,6 +27,17 @@ const JUDGE_VERDICT_SHAPE = z.object({
   summary: z.string(),
 });
 
+/**
+ * Per-participant cost attribution. `costUsd` is the sum of these; the
+ * breakdown lets reports show where the spend went. The judge runs as its
+ * own SDK session, so its cost is tracked separately from agent/supervisor.
+ */
+const COST_BREAKDOWN_SHAPE = z.object({
+  agent: z.number(),
+  supervisor: z.number(),
+  judge: z.number(),
+});
+
 const PROFILES_SHAPE = z.object({
   agent: z.union([z.string(), z.null()]),
   supervisor: z.union([z.string(), z.null()]),
@@ -44,6 +55,7 @@ const COMMON_FIELDS = {
   runIndex: z.number().int().min(0),
   verdict: VERDICT_ENUM,
   costUsd: z.number(),
+  costBreakdown: COST_BREAKDOWN_SHAPE.optional(),
   turns: z.number().int().min(0),
   profiles: PROFILES_SHAPE,
   model: z.object({