@forwardimpact/libeval 0.1.59 → 0.1.61

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@forwardimpact/libeval",
-  "version": "0.1.59",
+  "version": "0.1.61",
   "description": "Agent evaluation framework — prove whether agent changes improved outcomes with reproducible evidence.",
   "keywords": [
     "eval",

package/src/commands/benchmark-run.js

@@ -58,9 +58,9 @@ function parseRunOptions(values) {
     family,
     runs,
     output: resolve(output),
-    agentModel: values["agent-model"] ?? BENCHMARK_AGENT_MODEL,
-    supervisorModel: values["lead-model"] ?? LEAD_MODEL,
-    judgeModel: values["judge-model"] ?? LEAD_MODEL,
+    agentModel: values["agent-model"] || BENCHMARK_AGENT_MODEL,
+    supervisorModel: values["lead-model"] || LEAD_MODEL,
+    judgeModel: values["judge-model"] || LEAD_MODEL,
     profiles: {
       agent: values["agent-profile"] ?? null,
       judge: values["judge-profile"] ?? null,

package/src/commands/discuss.js

@@ -53,8 +53,8 @@ export function parseDiscussOptions(values, runtime) {
     taskAmend,
     agentConfigs,
     leadProfile: values["lead-profile"] ?? undefined,
-    leadModel: values["lead-model"] ?? LEAD_MODEL,
-    agentModel: values["agent-model"] ?? AGENT_MODEL,
+    leadModel: values["lead-model"] || LEAD_MODEL,
+    agentModel: values["agent-model"] || AGENT_MODEL,
     maxTurns,
     maxLeadTurns,
     outputPath: values.output,

package/src/commands/facilitate.js

@@ -51,8 +51,8 @@ export function parseFacilitateOptions(values, runtime) {
     taskAmend,
     agentConfigs,
     facilitatorCwd: resolve(values["facilitator-cwd"] ?? "."),
-    agentModel: values["agent-model"] ?? AGENT_MODEL,
-    facilitatorModel: values["lead-model"] ?? LEAD_MODEL,
+    agentModel: values["agent-model"] || AGENT_MODEL,
+    facilitatorModel: values["lead-model"] || LEAD_MODEL,
     maxTurns,
     outputPath: values.output,
     facilitatorProfile: values["lead-profile"] ?? undefined,

package/src/commands/run.js

@@ -27,7 +27,7 @@ function parseRunOptions(values, runtime) {
     taskContent,
     taskAmend,
     cwd: resolve(values.cwd ?? "."),
-    agentModel: values["agent-model"] ?? AGENT_MODEL,
+    agentModel: values["agent-model"] || AGENT_MODEL,
     maxTurns: maxTurnsRaw === "0" ? 0 : parseInt(maxTurnsRaw, 10),
     outputPath: values.output,
     agentProfile: values["agent-profile"] ?? undefined,

package/src/commands/supervise.js

@@ -31,8 +31,8 @@ export async function parseSuperviseOptions(values, runtime) {
     taskAmend,
     supervisorCwd: resolve(values["supervisor-cwd"] ?? "."),
     agentCwd,
-    agentModel: values["agent-model"] ?? AGENT_MODEL,
-    supervisorModel: values["lead-model"] ?? LEAD_MODEL,
+    agentModel: values["agent-model"] || AGENT_MODEL,
+    supervisorModel: values["lead-model"] || LEAD_MODEL,
     maxTurns: (() => {
       const raw = values["max-turns"] ?? "200";
       return raw === "0" ? 0 : parseInt(raw, 10);

package/src/trace-collector.js

@@ -218,25 +218,24 @@ export class TraceCollector {
   }
 
   /**
+   * Accumulate a result event into the running summary. Facilitated and
+   * supervised sessions emit one result event per runner invocation, so a
+   * single trace can carry several — cost, duration, turn, and token
+   * figures sum across all of them. `result` reflects the latest event;
+   * `isError` is true once any event errored.
    * @param {object} event
    */
   handleResult(event) {
+    const prev = this.result ?? EMPTY_RESULT;
+
     this.result = {
       result: event.subtype ?? "unknown",
-      isError: event.is_error ?? false,
-      totalCostUsd: event.total_cost_usd ?? 0,
-      durationMs: event.duration_ms ?? 0,
-      numTurns: event.num_turns ?? 0,
-      tokenUsage: event.usage
-        ? {
-            inputTokens: event.usage.input_tokens ?? 0,
-            outputTokens: event.usage.output_tokens ?? 0,
-            cacheReadInputTokens: event.usage.cache_read_input_tokens ?? 0,
-            cacheCreationInputTokens:
-              event.usage.cache_creation_input_tokens ?? 0,
-          }
-        : null,
-      modelUsage: event.modelUsage ?? null,
+      isError: prev.isError || (event.is_error ?? false),
+      totalCostUsd: prev.totalCostUsd + (event.total_cost_usd ?? 0),
+      durationMs: prev.durationMs + (event.duration_ms ?? 0),
+      numTurns: prev.numTurns + (event.num_turns ?? 0),
+      tokenUsage: sumTokenUsage(prev.tokenUsage, normalizeUsage(event.usage)),
+      modelUsage: event.modelUsage ?? prev.modelUsage,
     };
   }
 
@@ -303,7 +302,9 @@ export class TraceCollector {
    * Format the trailing result summary line. When an orchestrator
    * summary is present (supervised / facilitated mode), the headline word is
    * the supervisor's verdict ("success" / "failure") rather than the SDK's
-   * per-runner subtype, so the footer aligns with the CI exit code.
+   * per-runner subtype, so the footer aligns with the CI exit code. Turn,
+   * cost, and duration figures are the accumulated totals across every
+   * result event in the trace, not the last event's.
    * @returns {string}
    */
   #formatResultTail() {
@@ -318,6 +319,50 @@ export class TraceCollector {
   }
 }
 
+/** Identity element for result-event accumulation in handleResult. */
+const EMPTY_RESULT = {
+  isError: false,
+  totalCostUsd: 0,
+  durationMs: 0,
+  numTurns: 0,
+  tokenUsage: null,
+  modelUsage: null,
+};
+
+/**
+ * Normalize an SDK snake_case usage block to camelCase token fields.
+ * @param {object|null|undefined} usage
+ * @returns {object|null}
+ */
+function normalizeUsage(usage) {
+  if (!usage) return null;
+  return {
+    inputTokens: usage.input_tokens ?? 0,
+    outputTokens: usage.output_tokens ?? 0,
+    cacheReadInputTokens: usage.cache_read_input_tokens ?? 0,
+    cacheCreationInputTokens: usage.cache_creation_input_tokens ?? 0,
+  };
+}
+
+/**
+ * Sum two token-usage records field-by-field. Either side may be null
+ * (a result event without usage); the sum is null only when both are.
+ * @param {object|null} a
+ * @param {object|null} b
+ * @returns {object|null}
+ */
+function sumTokenUsage(a, b) {
+  if (!a) return b;
+  if (!b) return a;
+  return {
+    inputTokens: a.inputTokens + b.inputTokens,
+    outputTokens: a.outputTokens + b.outputTokens,
+    cacheReadInputTokens: a.cacheReadInputTokens + b.cacheReadInputTokens,
+    cacheCreationInputTokens:
+      a.cacheCreationInputTokens + b.cacheCreationInputTokens,
+  };
+}
+
 /**
  * Format milliseconds into a human-readable duration.
  * @param {number} ms - Duration in milliseconds

package/src/trace-query.js

@@ -278,38 +278,20 @@ export class TraceQuery {
 
   /**
    * Token usage and cost breakdown per assistant turn, plus totals.
+   *
+   * Token totals prefer the summary's result-event usage — the SDK's
+   * authoritative ledger, accumulated across every result event in the
+   * trace — over per-turn sums, whose stream-time snapshots double-count
+   * re-emitted messages. Traces without a result event (truncated or
+   * in-flight) fall back to the per-turn sums.
    * @returns {object}
    */
   stats() {
-    let totalInput = 0;
-    let totalOutput = 0;
-    let totalCacheRead = 0;
-    let totalCacheCreate = 0;
-    const perTurn = [];
-
-    for (const turn of this.turns) {
-      if (turn.role !== "assistant" || !turn.usage) continue;
-      const u = turn.usage;
-      totalInput += u.inputTokens ?? 0;
-      totalOutput += u.outputTokens ?? 0;
-      totalCacheRead += u.cacheReadInputTokens ?? 0;
-      totalCacheCreate += u.cacheCreationInputTokens ?? 0;
-
-      perTurn.push({
-        index: turn.index,
-        inputTokens: u.inputTokens ?? 0,
-        outputTokens: u.outputTokens ?? 0,
-        cacheReadInputTokens: u.cacheReadInputTokens ?? 0,
-        cacheCreationInputTokens: u.cacheCreationInputTokens ?? 0,
-      });
-    }
-
+    const { perTurn, totals: turnTotals } = perTurnUsage(this.turns);
+    const tokenTotals = this.summary.tokenUsage ?? turnTotals;
     return {
       totals: {
-        inputTokens: totalInput,
-        outputTokens: totalOutput,
-        cacheReadInputTokens: totalCacheRead,
-        cacheCreationInputTokens: totalCacheCreate,
+        ...tokenTotals,
         totalCostUsd: this.summary.totalCostUsd ?? 0,
         durationMs: this.summary.durationMs ?? 0,
       },
@@ -318,6 +300,38 @@ export class TraceQuery {
   }
 }
 
+/**
+ * Sum per-turn assistant usage and build the per-turn breakdown rows.
+ * @param {object[]} turns
+ * @returns {{perTurn: object[], totals: object}}
+ */
+function perTurnUsage(turns) {
+  const totals = {
+    inputTokens: 0,
+    outputTokens: 0,
+    cacheReadInputTokens: 0,
+    cacheCreationInputTokens: 0,
+  };
+  const perTurn = [];
+
+  for (const turn of turns) {
+    if (turn.role !== "assistant" || !turn.usage) continue;
+    const row = {
+      index: turn.index,
+      inputTokens: turn.usage.inputTokens ?? 0,
+      outputTokens: turn.usage.outputTokens ?? 0,
+      cacheReadInputTokens: turn.usage.cacheReadInputTokens ?? 0,
+      cacheCreationInputTokens: turn.usage.cacheCreationInputTokens ?? 0,
+    };
+    totals.inputTokens += row.inputTokens;
+    totals.outputTokens += row.outputTokens;
+    totals.cacheReadInputTokens += row.cacheReadInputTokens;
+    totals.cacheCreationInputTokens += row.cacheCreationInputTokens;
+    perTurn.push(row);
+  }
+  return { perTurn, totals };
+}
+
 /**
  * @param {object} turn
  * @param {string|undefined} role