@forwardimpact/libeval 0.1.20 → 0.1.21

package/bin/fit-trace.js

@@ -20,6 +20,9 @@ import {
   runReasoningCommand,
   runTimelineCommand,
   runStatsCommand,
+  runInitCommand,
+  runTurnCommand,
+  runFilterCommand,
 } from "../src/commands/trace.js";
 
 const { version: VERSION } = JSON.parse(
@@ -99,6 +102,10 @@ const definition = {
           type: "string",
           description: "Surrounding turns per hit (default: 0)",
         },
+        full: {
+          type: "boolean",
+          description: "Full content block in match descriptions",
+        },
       },
     },
     {
@@ -135,11 +142,45 @@ const definition = {
       args: "<file>",
       description: "Token usage and cost breakdown",
     },
+    {
+      name: "init",
+      args: "<file>",
+      description: "Full system/init event",
+    },
+    {
+      name: "turn",
+      args: "<file> <index>",
+      description: "Single turn by index",
+    },
+    {
+      name: "filter",
+      args: "<file>",
+      description: "Filter turns by structural properties",
+      options: {
+        role: {
+          type: "string",
+          description: "Turn role (system, user, assistant, tool_result)",
+        },
+        tool: {
+          type: "string",
+          description: "Tool name (matches assistant turns)",
+        },
+        error: {
+          type: "boolean",
+          description:
+            "Error tool_result turns only (flag-only; for non-errors use the API)",
+        },
+      },
+    },
   ],
   globalOptions: {
     help: { type: "boolean", short: "h", description: "Show this help" },
     version: { type: "boolean", description: "Show version" },
     json: { type: "boolean", description: "Output help as JSON" },
+    signatures: {
+      type: "boolean",
+      description: "Include thinking.signature blobs in output",
+    },
   },
   examples: [
     "fit-trace runs --lookback 7d",
@@ -149,6 +190,11 @@ const definition = {
     "fit-trace search structured.json 'error|fail' --context 1",
     "fit-trace tool structured.json Bash",
     "fit-trace batch structured.json 0 20",
+    "fit-trace init structured.json",
+    "fit-trace turn structured.json 3",
+    "fit-trace filter structured.json --role system",
+    "fit-trace filter structured.json --tool Bash --role assistant",
+    "fit-trace search structured.json 'error' --full",
   ],
 };
 
@@ -170,6 +216,9 @@ const COMMANDS = {
   reasoning: runReasoningCommand,
   timeline: runTimelineCommand,
   stats: runStatsCommand,
+  init: runInitCommand,
+  turn: runTurnCommand,
+  filter: runFilterCommand,
 };
 
 async function main() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "description": "Process Claude Code stream-json output into structured traces",
   "license": "Apache-2.0",
   "author": "D. Olsson <hi@senzilla.io>",

package/src/commands/trace.js

@@ -3,6 +3,7 @@ import { join } from "node:path";
 import { createTraceCollector } from "@forwardimpact/libeval";
 import { createTraceQuery } from "../trace-query.js";
 import { createTraceGitHub } from "../trace-github.js";
+import { stripSignatures } from "../signature-filter.js";
 
 // --- GitHub commands ---
 
@@ -20,7 +21,7 @@ export async function runRunsCommand(values, args, ctx) {
   const pattern = args[0] ?? "agent";
   const lookback = values.lookback ?? "7d";
   const runs = await gh.listRuns({ pattern, lookback });
-  writeJSON(runs);
+  writeJSON(runs, values);
 }
 
 /**
@@ -51,14 +52,14 @@ export async function runDownloadCommand(values, args, ctx) {
     result.files.push("structured.json");
   }
 
-  writeJSON(result);
+  writeJSON(result, values);
 }
 
 // --- Query commands ---
 
 /** @param {object} values @param {string[]} args - [file] */
 export async function runOverviewCommand(values, args) {
-  writeJSON(loadTrace(args[0]).overview());
+  writeJSON(loadTrace(args[0]).overview(), values);
 }
 
 /** @param {object} values @param {string[]} args - [file] */
@@ -70,48 +71,53 @@ export async function runCountCommand(values, args) {
 export async function runBatchCommand(values, args) {
   writeJSON(
     loadTrace(args[0]).batch(parseInt(args[1], 10), parseInt(args[2], 10)),
+    values,
   );
 }
 
 /** @param {object} values @param {string[]} args - [file, N?] */
 export async function runHeadCommand(values, args) {
   const n = args[1] ? parseInt(args[1], 10) : 10;
-  writeJSON(loadTrace(args[0]).head(n));
+  writeJSON(loadTrace(args[0]).head(n), values);
 }
 
 /** @param {object} values @param {string[]} args - [file, N?] */
 export async function runTailCommand(values, args) {
   const n = args[1] ? parseInt(args[1], 10) : 10;
-  writeJSON(loadTrace(args[0]).tail(n));
+  writeJSON(loadTrace(args[0]).tail(n), values);
 }
 
 /** @param {object} values @param {string[]} args - [file, pattern] */
 export async function runSearchCommand(values, args) {
   const limit = values.limit ? parseInt(values.limit, 10) : 50;
   const context = values.context ? parseInt(values.context, 10) : 0;
-  writeJSON(loadTrace(args[0]).search(args[1], { limit, context }));
+  const full = values.full ?? false;
+  writeJSON(
+    loadTrace(args[0]).search(args[1], { limit, context, full }),
+    values,
+  );
 }
 
 /** @param {object} values @param {string[]} args - [file] */
 export async function runToolsCommand(values, args) {
-  writeJSON(loadTrace(args[0]).toolFrequency());
+  writeJSON(loadTrace(args[0]).toolFrequency(), values);
 }
 
 /** @param {object} values @param {string[]} args - [file, name] */
 export async function runToolCommand(values, args) {
-  writeJSON(loadTrace(args[0]).tool(args[1]));
+  writeJSON(loadTrace(args[0]).tool(args[1]), values);
 }
 
 /** @param {object} values @param {string[]} args - [file] */
 export async function runErrorsCommand(values, args) {
-  writeJSON(loadTrace(args[0]).errors());
+  writeJSON(loadTrace(args[0]).errors(), values);
 }
 
 /** @param {object} values @param {string[]} args - [file] */
 export async function runReasoningCommand(values, args) {
   const from = values.from ? parseInt(values.from, 10) : undefined;
   const to = values.to ? parseInt(values.to, 10) : undefined;
-  writeJSON(loadTrace(args[0]).reasoning({ from, to }));
+  writeJSON(loadTrace(args[0]).reasoning({ from, to }), values);
 }
 
 /** @param {object} values @param {string[]} args - [file] */
@@ -122,7 +128,26 @@ export async function runTimelineCommand(values, args) {
 
 /** @param {object} values @param {string[]} args - [file] */
 export async function runStatsCommand(values, args) {
-  writeJSON(loadTrace(args[0]).stats());
+  writeJSON(loadTrace(args[0]).stats(), values);
+}
+
+/** @param {object} values @param {string[]} args - [file] */
+export async function runInitCommand(values, args) {
+  writeJSON(loadTrace(args[0]).init(), values);
+}
+
+/** @param {object} values @param {string[]} args - [file, index] */
+export async function runTurnCommand(values, args) {
+  writeJSON(loadTrace(args[0]).turn(parseInt(args[1], 10)), values);
+}
+
+/** @param {object} values @param {string[]} args - [file] */
+export async function runFilterCommand(values, args) {
+  const opts = {};
+  if (values.role) opts.role = values.role;
+  if (values.tool) opts.toolName = values.tool;
+  if (values.error) opts.isError = true;
+  writeJSON(loadTrace(args[0]).filter(opts), values);
 }
 
 // --- Shared helpers ---
@@ -151,7 +176,14 @@ function loadTrace(file) {
   return createTraceQuery(collector.toJSON());
 }
 
-/** @param {object} data */
-function writeJSON(data) {
-  process.stdout.write(JSON.stringify(data, null, 2) + "\n");
+/**
+ * Write JSON output to stdout. By default strips `thinking.signature`
+ * base64 blobs from the payload so they don't dominate terminal output;
+ * pass `--signatures` (surfaced as `values.signatures`) to keep them.
+ * @param {*} data
+ * @param {object} [values]
+ */
+function writeJSON(data, values = {}) {
+  const output = values.signatures ? data : stripSignatures(data);
+  process.stdout.write(JSON.stringify(output, null, 2) + "\n");
 }

package/src/index.js

@@ -1,5 +1,6 @@
 export { TraceCollector, createTraceCollector } from "./trace-collector.js";
 export { TraceQuery, createTraceQuery } from "./trace-query.js";
+export { stripSignatures } from "./signature-filter.js";
 export {
   TraceGitHub,
   createTraceGitHub,

package/src/signature-filter.js

@@ -0,0 +1,27 @@
+/**
+ * Strip `thinking.signature` base64 blobs from a JSON-serializable value.
+ *
+ * Applied at the CLI output boundary — the stored structured trace keeps
+ * signatures intact (lossless storage), and the display filter drops them
+ * by default because they dominate output without helping analysis.
+ *
+ * Recursively walks the input. For any object whose `type === "thinking"`,
+ * the `signature` field is removed after copying. Signatures on objects of
+ * any other type are preserved.
+ *
+ * @param {*} value - Any JSON-serializable value
+ * @returns {*} A deep-copy with thinking signatures removed
+ */
+export function stripSignatures(value) {
+  if (value === null || typeof value !== "object") return value;
+  if (Array.isArray(value)) return value.map(stripSignatures);
+
+  const result = {};
+  for (const [key, val] of Object.entries(value)) {
+    result[key] = stripSignatures(val);
+  }
+  if (result.type === "thinking") {
+    delete result.signature;
+  }
+  return result;
+}

package/src/tee-writer.js

@@ -163,6 +163,27 @@ export class TeeWriter extends Writable {
             withPrefix,
           }),
         );
+      } else if (turn.role === "system") {
+        const label = turn.subtype ?? "system";
+        this.textStream.write(
+          renderTextLine({
+            source: turn.source,
+            text: `[${label}]`,
+            withPrefix,
+          }),
+        );
+      } else if (turn.role === "user") {
+        for (const block of turn.content) {
+          if (block.type === "text") {
+            this.textStream.write(
+              renderTextLine({
+                source: turn.source,
+                text: `[user] ${block.text}`,
+                withPrefix,
+              }),
+            );
+          }
+        }
       }
     }
   }

package/src/trace-collector.js

@@ -37,6 +37,8 @@ export class TraceCollector {
     this.result = null;
     /** @type {number} */
     this.turnIndex = 0;
+    /** @type {object|null} */
+    this.initEvent = null;
   }
 
   /**
@@ -73,7 +75,7 @@ export class TraceCollector {
 
     switch (event.type) {
       case "system":
-        this.handleSystem(event);
+        this.handleSystem(event, source);
         break;
       case "assistant":
         this.handleAssistant(event, source);
@@ -91,8 +93,11 @@ export class TraceCollector {
 
   /**
    * @param {object} event
+   * @param {string|null} source
    */
-  handleSystem(event) {
+  handleSystem(event, source) {
+    const { type: _type, ...payload } = event;
+
     if (event.subtype === "init") {
       this.metadata = {
         timestamp: event.timestamp ?? this.now(),
@@ -102,7 +107,16 @@ export class TraceCollector {
         tools: event.tools ?? [],
         permissionMode: event.permissionMode ?? null,
       };
+      this.initEvent = payload;
     }
+
+    this.turns.push({
+      index: this.turnIndex++,
+      role: "system",
+      source,
+      subtype: event.subtype ?? null,
+      data: payload,
+    });
   }
 
   /**
@@ -158,6 +172,19 @@ export class TraceCollector {
     const contentItems = message.content;
     if (!Array.isArray(contentItems)) return;
 
+    const textBlocks = contentItems
+      .filter((item) => item.type === "text")
+      .map((item) => ({ type: "text", text: item.text }));
+
+    if (textBlocks.length > 0) {
+      this.turns.push({
+        index: this.turnIndex++,
+        role: "user",
+        source,
+        content: textBlocks,
+      });
+    }
+
     for (const item of contentItems) {
       if (item.type === "tool_result") {
         this.turns.push({
@@ -204,7 +231,7 @@ export class TraceCollector {
    */
   toJSON() {
     return {
-      version: "1.0.0",
+      version: "1.1.0",
       metadata: this.metadata ?? {
         timestamp: this.now(),
         sessionId: null,
@@ -213,6 +240,7 @@ export class TraceCollector {
         tools: [],
         permissionMode: null,
       },
+      initEvent: this.initEvent ?? null,
       turns: this.turns,
       summary: this.result ?? {
         result: "unknown",
@@ -271,6 +299,27 @@ export class TraceCollector {
             withPrefix,
           }),
         );
+      } else if (turn.role === "system") {
+        const label = turn.subtype ?? "system";
+        out.push(
+          renderTextLine({
+            source: turn.source,
+            text: `[${label}]`,
+            withPrefix,
+          }),
+        );
+      } else if (turn.role === "user") {
+        for (const block of turn.content) {
+          if (block.type === "text") {
+            out.push(
+              renderTextLine({
+                source: turn.source,
+                text: `[user] ${block.text}`,
+                withPrefix,
+              }),
+            );
+          }
+        }
       }
     }
 

package/src/trace-query.js

@@ -17,18 +17,90 @@ export class TraceQuery {
   }
 
   /**
-   * High-level overview: metadata, summary, turn count, and tool frequency.
+   * High-level overview: metadata, summary, turn count, tool frequency,
+   * and the first user message text (taskPrompt) when present.
    * @returns {object}
    */
   overview() {
+    const firstUser = this.turns.find((t) => t.role === "user");
+    const taskPrompt = firstUser
+      ? firstUser.content
+          .filter((b) => b.type === "text")
+          .map((b) => b.text)
+          .join("\n")
+      : null;
     return {
       metadata: this.metadata,
       summary: this.summary,
       turnCount: this.turns.length,
       tools: this.toolFrequency(),
+      taskPrompt,
     };
   }
 
+  /**
+   * Full system/init event — the single most diagnostic message for
+   * root-cause analysis. Returns null for traces collected before this
+   * field existed.
+   * @returns {object|null}
+   */
+  init() {
+    return this.trace.initEvent ?? null;
+  }
+
+  /**
+   * Retrieve a single turn by its index.
+   * @param {number} index
+   * @returns {object|null}
+   */
+  turn(index) {
+    return this.turns.find((t) => t.index === index) ?? null;
+  }
+
+  /**
+   * Filter turns by composable structural criteria. All criteria are
+   * combined as AND. `tool()` and `errors()` remain as convenience
+   * shortcuts for pre-existing workflows.
+   *
+   * `toolName` matches assistant turns only. Applying `toolName` without
+   * `role: "assistant"` still drops every non-assistant turn, because
+   * resolving tool_use → tool_result pairs requires the `tool()` method.
+   * `isError` matches tool_result turns only. Combining `toolName` with
+   * `isError` therefore always returns `[]` (no turn is both assistant
+   * and tool_result) — use `tool(name)` for "errors from Bash"–shaped
+   * queries.
+   *
+   * @param {object} [opts]
+   * @param {string} [opts.role] - Exact role match (system | user |
+   *   assistant | tool_result).
+   * @param {string} [opts.toolName] - Matches assistant turns with a
+   *   tool_use block of this name. Drops all non-assistant turns.
+   * @param {boolean} [opts.isError] - Matches tool_result turns by
+   *   `isError` value. Drops all non-tool_result turns.
+   * @returns {object[]}
+   */
+  filter(opts = {}) {
+    const { role, toolName, isError } = opts;
+    return this.turns.filter((turn) => {
+      if (role !== undefined && turn.role !== role) return false;
+      if (isError !== undefined) {
+        if (turn.role !== "tool_result") return false;
+        if (turn.isError !== isError) return false;
+      }
+      if (toolName !== undefined) {
+        if (turn.role === "assistant") {
+          const has = turn.content.some(
+            (b) => b.type === "tool_use" && b.name === toolName,
+          );
+          if (!has) return false;
+        } else {
+          return false;
+        }
+      }
+      return true;
+    });
+  }
+
   /** @returns {number} */
   count() {
     return this.turns.length;
@@ -73,16 +145,18 @@ export class TraceQuery {
    * @param {object} [opts]
    * @param {number} [opts.context=0] - Number of surrounding turns to include
    * @param {number} [opts.limit=50] - Max results
+   * @param {boolean} [opts.full=false] - Emit full content block text in
+   *   match descriptions instead of the default narrow excerpt window.
    * @returns {object[]} Array of {turn, matches, context?}
    */
   search(pattern, opts = {}) {
-    const { context = 0, limit = 50 } = opts;
+    const { context = 0, limit = 50, full = false } = opts;
     // eslint-disable-next-line security/detect-non-literal-regexp -- pattern is caller-controlled, not untrusted input
     const re = new RegExp(pattern, "gi");
     const hits = [];
 
     for (const turn of this.turns) {
-      const matches = matchTurn(turn, re);
+      const matches = matchTurn(turn, re, full);
       if (matches.length > 0) {
         const entry = { turn, matches };
         if (context > 0) {
@@ -273,40 +347,79 @@ export class TraceQuery {
  * Search a single turn for regex matches. Returns array of match descriptions.
  * @param {object} turn
  * @param {RegExp} re
+ * @param {boolean} [full=false] - Emit full block text instead of an excerpt.
  * @returns {string[]}
  */
-function matchTurn(turn, re) {
+function matchTurn(turn, re, full = false) {
+  if (turn.role === "assistant") return matchAssistantTurn(turn, re, full);
+  if (turn.role === "tool_result") return matchToolResultTurn(turn, re, full);
+  if (turn.role === "user") return matchUserTurn(turn, re, full);
+  return [];
+}
+
+function matchAssistantTurn(turn, re, full) {
   const matches = [];
-  if (turn.role === "assistant") {
-    for (const block of turn.content) {
-      if (block.type === "text" && re.test(block.text)) {
-        re.lastIndex = 0;
-        matches.push(`text: ${excerptAround(block.text, re)}`);
-      }
-      if (block.type === "tool_use") {
-        if (re.test(block.name)) {
-          re.lastIndex = 0;
-          matches.push(`tool_name: ${block.name}`);
-        }
-        const inputStr = JSON.stringify(block.input);
-        if (re.test(inputStr)) {
-          re.lastIndex = 0;
-          matches.push(
-            `tool_input(${block.name}): ${excerptAround(inputStr, re)}`,
-          );
-        }
-      }
+  for (const block of turn.content) {
+    if (block.type === "text") {
+      const desc = describeText(block.text, re, "text", full);
+      if (desc) matches.push(desc);
+    } else if (block.type === "tool_use") {
+      matches.push(...matchToolUseBlock(block, re, full));
     }
-  } else if (turn.role === "tool_result") {
-    const content = turn.content ?? "";
-    if (re.test(content)) {
-      re.lastIndex = 0;
-      matches.push(`result: ${excerptAround(content, re)}`);
+  }
+  return matches;
+}
+
+function matchToolUseBlock(block, re, full) {
+  const matches = [];
+  if (re.test(block.name)) {
+    re.lastIndex = 0;
+    matches.push(`tool_name: ${block.name}`);
+  }
+  const inputStr = JSON.stringify(block.input);
+  const inputDesc = describeText(
+    inputStr,
+    re,
+    `tool_input(${block.name})`,
+    full,
+  );
+  if (inputDesc) matches.push(inputDesc);
+  return matches;
+}
+
+function matchToolResultTurn(turn, re, full) {
+  const content = turn.content ?? "";
+  const desc = describeText(content, re, "result", full);
+  return desc ? [desc] : [];
+}
+
+function matchUserTurn(turn, re, full) {
+  const matches = [];
+  for (const block of turn.content ?? []) {
+    if (block.type === "text") {
+      const desc = describeText(block.text, re, "user_text", full);
+      if (desc) matches.push(desc);
     }
   }
   return matches;
 }
 
+/**
+ * Return a `<prefix>: <text-or-excerpt>` description when `text` matches
+ * the regex, or null when it does not. Centralises the full-vs-excerpt
+ * choice so each call site just supplies its prefix.
+ * @param {string} text
+ * @param {RegExp} re
+ * @param {string} prefix
+ * @param {boolean} full
+ * @returns {string|null}
+ */
+function describeText(text, re, prefix, full) {
+  if (!re.test(text)) return null;
+  re.lastIndex = 0;
+  return `${prefix}: ${full ? text : excerptAround(text, re)}`;
+}
+
 /**
  * Extract a short excerpt around the first regex match in text.
  * @param {string} text