@skill-harness/core 0.18.0 → 0.19.0

package/dist/adapters/types.d.ts

@@ -81,6 +81,8 @@ export interface StructuredRun {
 }
 export interface HarnessAdapter {
     name: string;
+    /** Prefer structured execution for ordinary runs so subject usage is retained. */
+    preferStructured?: boolean;
     available(): Promise<boolean>;
     run(req: RunReq): Promise<string>;
     /**

package/dist/capture-trace-types.d.ts

@@ -83,14 +83,16 @@ export interface TraceResultMeta {
  * conversation carries in the session dir, not in the event stream). This
  * mirrors the loop the adapter already runs.
  */
-export type CostSource = "provider-reported" | "subscription" | "unreported";
+export type CostSource = "provider-reported" | "subscription" | "price-table" | "unreported";
 export interface TraceMetrics {
-    input_tokens: number;
-    output_tokens: number;
-    cache_read_tokens: number;
-    cache_write_tokens: number;
-    cost_usd: number;
+    input_tokens: number | null;
+    output_tokens: number | null;
+    cache_read_tokens: number | null;
+    cache_write_tokens: number | null;
+    cost_usd: number | null;
     cost_source: CostSource;
+    /** Date of the repository price table used for cost_usd; null when unpriced. */
+    price_as_of: string | null;
     tool_calls: number;
     delegated_children: number;
     /** Maximum simultaneously outstanding tool calls observed in the pi stream. */

package/dist/execution-trace.d.ts

@@ -40,22 +40,6 @@ export declare function traceSha256(trace: ExecutionTraceV1): string;
 export declare function serializeTrace(trace: ExecutionTraceV1): string;
 /** Read a saved trace artifact back, for `regate`. Returns null when unusable. */
 export declare function deserializeTrace(text: string): ExecutionTraceV1 | null;
-/**
- * Collapse a scenario's per-turn traces into one view for gate evaluation.
- *
- * Assertions are written about the scenario ("it delegated to `plan` at least
- * once"), not about turn 3 — a multi-turn scenario would otherwise need the
- * author to know which turn a tool call landed in, which is a property of the
- * model's choices, not of the test.
- *
- * Indices are renumbered across the whole scenario so `issueIndex` stays a total
- * order. `completionIndex` is renumbered within the concatenation too: turns are
- * strictly sequential (each is a separate `pi` invocation), so no completion in
- * turn 2 can precede one in turn 1.
- *
- * Returns null for an empty list — "no turns produced evidence" must not look
- * like "a run in which nothing happened".
- */
 export declare function mergeTraces(traces: ExecutionTraceV1[]): ExecutionTraceV1 | null;
 /** Split a raw stdout blob into lines. Prefer streaming; this is for saved blobs. */
 export declare function lines(text: string): Generator<string>;

package/dist/execution-trace.js

@@ -43,12 +43,10 @@ export function parseTrace(lines, meta) {
     let sawTerminal = false;
     let finalText = "";
     let lastAssistantText = "";
-    let cost = null;
-    let inputTokens = 0;
-    let outputTokens = 0;
-    let cacheReadTokens = 0;
-    let cacheWriteTokens = 0;
-    let sawUsage = false;
+    let inputTokens = null;
+    let outputTokens = null;
+    let cacheReadTokens = null;
+    let cacheWriteTokens = null;
     let activeCalls = 0;
     let maxConcurrency = 0;
     for (const line of lines) {
@@ -130,21 +128,10 @@ export function parseTrace(lines, meta) {
                 if (msg.stopReason === "stop")
                     finalText = text;
             }
-            if (msg.usage && (typeof msg.usage.input === "number" || typeof msg.usage.output === "number" ||
-                typeof msg.usage.cacheRead === "number" || typeof msg.usage.cacheWrite === "number" ||
-                typeof msg.usage.cost?.total === "number"))
-                sawUsage = true;
-            const total = msg.usage?.cost?.total;
-            if (typeof total === "number")
-                cost = (cost ?? 0) + total;
-            if (typeof msg.usage?.input === "number")
-                inputTokens += msg.usage.input;
-            if (typeof msg.usage?.output === "number")
-                outputTokens += msg.usage.output;
-            if (typeof msg.usage?.cacheRead === "number")
-                cacheReadTokens += msg.usage.cacheRead;
-            if (typeof msg.usage?.cacheWrite === "number")
-                cacheWriteTokens += msg.usage.cacheWrite;
+            inputTokens = addReported(inputTokens, msg.usage?.input);
+            outputTokens = addReported(outputTokens, msg.usage?.output);
+            cacheReadTokens = addReported(cacheReadTokens, msg.usage?.cacheRead);
+            cacheWriteTokens = addReported(cacheWriteTokens, msg.usage?.cacheWrite);
             continue;
         }
         if (type === "turn_end" || type === "agent_end" || type === "agent_settled") {
@@ -157,19 +144,14 @@ export function parseTrace(lines, meta) {
         }
     }
     const toolCalls = [...calls.values()].sort((a, b) => a.issueIndex - b.issueIndex);
-    const subscription = meta.subject.provider === "openai-codex" || meta.subject.provider === "claude-code";
-    const costSource = subscription
-        ? "subscription"
-        : cost !== null
-            ? "provider-reported"
-            : "unreported";
     const metrics = {
         input_tokens: inputTokens,
         output_tokens: outputTokens,
         cache_read_tokens: cacheReadTokens,
         cache_write_tokens: cacheWriteTokens,
-        cost_usd: cost ?? 0,
-        cost_source: costSource,
+        cost_usd: null,
+        cost_source: "unreported",
+        price_as_of: null,
         tool_calls: toolCalls.length,
         delegated_children: toolCalls
             .filter((call) => call.name === "Agent")
@@ -197,15 +179,17 @@ export function parseTrace(lines, meta) {
         // overwrites this after observing the workspace. Defaulting to `[]` claimed
         // "observed, nothing changed" for every trace ever parsed.
         changed_paths: meta.changedPaths ? [...meta.changedPaths].sort() : null,
-        cost_usd: cost,
-        // Tool calls remain in the trace for objective gates. Aggregate usage/cost/
-        // tool metrics are published only when pi actually reported usage; otherwise
-        // zero would mean "free" instead of "unavailable".
-        ...(sawUsage ? { metrics } : {}),
+        cost_usd: null,
+        // Tool metrics are observed directly. Token and cost fields stay null when
+        // Pi does not report them; zero would falsely mean "free".
+        metrics,
     };
     trace.trace_sha256 = traceSha256(trace);
     return { trace, isComplete: sawTerminal, malformedLines };
 }
+function addReported(current, value) {
+    return typeof value === "number" && value > 0 ? (current ?? 0) + value : current;
+}
 /** Visible assistant text. Thinking is dropped here, and at every other reader. */
 function assistantText(msg) {
     return (msg.content ?? [])
@@ -308,6 +292,9 @@ export function deserializeTrace(text) {
  * Returns null for an empty list — "no turns produced evidence" must not look
  * like "a run in which nothing happened".
  */
+function sumNullable(left, right) {
+    return left === null ? right : right === null ? left : left + right;
+}
 export function mergeTraces(traces) {
     if (traces.length === 0)
         return null;
@@ -350,18 +337,20 @@ export function mergeTraces(traces) {
     const completeMetrics = traces.every((trace) => trace.metrics !== undefined);
     const metrics = completeMetrics
         ? traces.reduce((sum, trace) => ({
-            input_tokens: sum.input_tokens + trace.metrics.input_tokens,
-            output_tokens: sum.output_tokens + trace.metrics.output_tokens,
-            cache_read_tokens: sum.cache_read_tokens + trace.metrics.cache_read_tokens,
-            cache_write_tokens: sum.cache_write_tokens + trace.metrics.cache_write_tokens,
-            cost_usd: sum.cost_usd + trace.metrics.cost_usd,
+            input_tokens: sumNullable(sum.input_tokens, trace.metrics.input_tokens),
+            output_tokens: sumNullable(sum.output_tokens, trace.metrics.output_tokens),
+            cache_read_tokens: sumNullable(sum.cache_read_tokens, trace.metrics.cache_read_tokens),
+            cache_write_tokens: sumNullable(sum.cache_write_tokens, trace.metrics.cache_write_tokens),
+            cost_usd: sumNullable(sum.cost_usd, trace.metrics.cost_usd),
             cost_source: sum.cost_source === trace.metrics.cost_source ? sum.cost_source : "unreported",
+            price_as_of: sum.price_as_of === trace.metrics.price_as_of ? sum.price_as_of : null,
             tool_calls: sum.tool_calls + trace.metrics.tool_calls,
             delegated_children: sum.delegated_children + trace.metrics.delegated_children,
             max_concurrency: Math.max(sum.max_concurrency, trace.metrics.max_concurrency),
         }), {
-            input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0,
-            cost_usd: 0, cost_source: traces[0].metrics.cost_source, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
+            input_tokens: null, output_tokens: null, cache_read_tokens: null, cache_write_tokens: null,
+            cost_usd: null, cost_source: traces[0].metrics.cost_source, price_as_of: traces[0].metrics.price_as_of,
+            tool_calls: 0, delegated_children: 0, max_concurrency: 0,
         })
         : undefined;
     const last = traces[traces.length - 1];

package/dist/metrics.d.ts

@@ -1,4 +1,5 @@
 import type { ScenarioResult } from "./results.js";
+import type { CostSource } from "./capture-trace-types.js";
 export interface AggregateMetrics {
     wall_time_ms: number;
     judge_calls: number;
@@ -10,7 +11,7 @@ export interface AggregateMetrics {
     cache_read_tokens: number | null;
     cache_write_tokens: number | null;
     subject_cost_usd: number | null;
-    cost_source: "provider-reported" | "subscription" | "unreported" | null;
+    cost_source: CostSource | null;
     tool_calls: number | null;
     delegated_children: number | null;
     max_concurrency: number | null;

package/dist/reps.js

@@ -67,17 +67,32 @@ export function outcomesToResult(id, outcomes, repCount, threshold) {
     const objectiveField = objective ? { objective } : {};
     const metrics = aggregateMetrics(outcomes);
     const metricsField = metrics ? { metrics } : {};
+    const usage = outcomes.flatMap((outcome, repetition) => {
+        const subject = outcome.metrics?.subject;
+        if (!subject)
+            return [];
+        const reported = (value) => value !== null && value > 0 ? value : null;
+        return [{
+                repetition,
+                inputTokens: reported(subject.input_tokens),
+                outputTokens: reported(subject.output_tokens),
+                cacheReadTokens: reported(subject.cache_read_tokens),
+                costUsd: reported(subject.cost_usd),
+                priceAsOf: subject.price_as_of,
+            }];
+    });
+    const usageField = usage.length ? { usage } : {};
     const repJudgments = outcomes.map((outcome, repetition) => ({ repetition, judgments: outcome.judgment ? [outcome.judgment] : [], recorded_verdict: outcome.verdict, ...(outcome.objective ? { objective: outcome.objective } : {}) }));
     const repJudgmentField = outcomes.some(outcome => outcome.judgment) ? { rep_judgments: repJudgments } : {};
     if (repCount === 1) {
         const o = outcomes[0];
-        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...metricsField, override: null, note: "", ...objectiveField, ...repJudgmentField };
+        return { id, judge_verdict: o.verdict, judge_reason: o.reason, suspect: o.suspect, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField };
     }
     const agg = aggregateReps(outcomes, threshold);
     return {
         id, judge_verdict: agg.verdict, judge_reason: agg.reason, suspect: agg.suspect,
         reps: agg.reps, passes: agg.passes, clean: agg.clean, flakiness: agg.flakiness,
-        pass_threshold: threshold, ...metricsField, override: null, note: "", ...objectiveField, ...repJudgmentField,
+        pass_threshold: threshold, ...metricsField, ...usageField, override: null, note: "", ...objectiveField, ...repJudgmentField,
     };
 }
 function aggregateMetrics(outcomes) {
@@ -85,22 +100,32 @@ function aggregateMetrics(outcomes) {
     if (present.length === 0)
         return undefined;
     const subjects = present.map((metrics) => metrics.subject).filter((metrics) => metrics !== undefined);
+    const reportedSubjects = subjects.filter(metrics => metrics.input_tokens !== null || metrics.output_tokens !== null || metrics.cache_read_tokens !== null);
     const base = {
         wall_time_ms: present.reduce((sum, metrics) => sum + metrics.wall_time_ms, 0),
         judge_calls: present.reduce((sum, metrics) => sum + metrics.judge_calls, 0),
         judge_rejudge_calls: present.reduce((sum, metrics) => sum + metrics.judge_rejudge_calls, 0),
-        subject_metrics_reps: subjects.length,
+        subject_metrics_reps: reportedSubjects.length,
         total_reps: outcomes.length,
     };
     if (subjects.length === 0)
         return base;
+    const sumReported = (field) => {
+        const values = subjects.map(metrics => metrics[field]).filter((value) => value !== null && value > 0);
+        return values.length ? values.reduce((sum, value) => sum + value, 0) : undefined;
+    };
+    const inputTokens = sumReported("input_tokens");
+    const outputTokens = sumReported("output_tokens");
+    const cacheReadTokens = sumReported("cache_read_tokens");
+    const cacheWriteTokens = sumReported("cache_write_tokens");
+    const subjectCost = sumReported("cost_usd");
     return {
         ...base,
-        input_tokens: subjects.reduce((sum, metrics) => sum + metrics.input_tokens, 0),
-        output_tokens: subjects.reduce((sum, metrics) => sum + metrics.output_tokens, 0),
-        cache_read_tokens: subjects.reduce((sum, metrics) => sum + metrics.cache_read_tokens, 0),
-        cache_write_tokens: subjects.reduce((sum, metrics) => sum + metrics.cache_write_tokens, 0),
-        subject_cost_usd: subjects.reduce((sum, metrics) => sum + metrics.cost_usd, 0),
+        ...(inputTokens === undefined ? {} : { input_tokens: inputTokens }),
+        ...(outputTokens === undefined ? {} : { output_tokens: outputTokens }),
+        ...(cacheReadTokens === undefined ? {} : { cache_read_tokens: cacheReadTokens }),
+        ...(cacheWriteTokens === undefined ? {} : { cache_write_tokens: cacheWriteTokens }),
+        ...(subjectCost === undefined ? {} : { subject_cost_usd: subjectCost }),
         cost_source: subjects.every((metrics) => metrics.cost_source === subjects[0].cost_source)
             ? subjects[0].cost_source
             : "unreported",

package/dist/results.d.ts

@@ -3,6 +3,14 @@ import { type ScenarioVerdict } from "./score.js";
 import type { Verdict } from "./score.js";
 import type { ShipBar, Scenario } from "./spec.js";
 import type { CostSource } from "./capture-trace-types.js";
+export interface SubjectUsage {
+    repetition: number;
+    inputTokens: number | null;
+    outputTokens: number | null;
+    cacheReadTokens: number | null;
+    costUsd: number | null;
+    priceAsOf: string | null;
+}
 export interface ScenarioMetrics {
     wall_time_ms: number;
     judge_calls: number;
@@ -56,6 +64,8 @@ export interface ScenarioResult {
     pass_threshold?: number;
     /** Cost/latency counters. Separate from verdicts: cheaper failure is still failure. */
     metrics?: ScenarioMetrics;
+    /** Per-repetition subject usage; null values mean Pi did not report that counter. */
+    usage?: SubjectUsage[];
     /**
      * Objective trace-gate evidence. ADDITIVE and optional.
      *

package/dist/results.js

@@ -599,7 +599,7 @@ export function diffPath(runDir, scenarioId, mode, rep) {
 export function rebuildScenarioResult(fresh, prior, policy) {
     // Exhaustive destructure. Do not replace with a spread: the spread is what
     // allowed a new field to pass through unconsidered in the first place.
-    const { id, criterion_count: freshCriterionCount, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, objective: freshObjective, adjudication: freshAdjudication, rep_judgments: freshRepJudgments, ...rest } = fresh;
+    const { id, criterion_count: freshCriterionCount, judge_verdict, judge_reason, suspect, override: _freshOverride, note: _freshNote, reps, passes, clean, flakiness, pass_threshold, metrics: freshMetrics, usage: freshUsage, objective: freshObjective, adjudication: freshAdjudication, rep_judgments: freshRepJudgments, ...rest } = fresh;
     const _exhaustive = rest;
     void _exhaustive;
     void _freshOverride;
@@ -652,6 +652,7 @@ export function rebuildScenarioResult(fresh, prior, policy) {
         ...(flakiness === undefined ? {} : { flakiness }),
         ...(pass_threshold === undefined ? {} : { pass_threshold }),
         ...((freshMetrics ?? prior?.metrics) ? { metrics: freshMetrics ?? prior.metrics } : {}),
+        ...((freshUsage ?? prior?.usage) ? { usage: freshUsage ?? prior.usage } : {}),
         // The author owns the verdict; a re-measurement never discards their call.
         override: prior?.override ?? null,
         note: prior?.note ?? "",

package/dist/run.js

@@ -276,7 +276,7 @@ async function runRep(scenario, rep, repCount, ctx) {
                 throw new Error(`scenario \`${scenario.id}\` declares structured objective assertions, but the \`${ctx.adapter.name}\` adapter` +
                     ` cannot produce execution traces — the gate would have no evidence to read.`);
             }
-            const useStructured = (Boolean(ctx.structured) || needsStructuredEvidence) && Boolean(ctx.adapter.runStructured);
+            const useStructured = (Boolean(ctx.structured) || needsStructuredEvidence || Boolean(ctx.adapter.preferStructured)) && Boolean(ctx.adapter.runStructured);
             // A blank assistant turn is a harness timeout, not model behavior: retry ONCE in a
             // fresh workspace (the first attempt may have half-mutated a seeded repo), and if
             // it happens again the verdict is ERROR — never a judged FAIL on an empty reply.
@@ -581,7 +581,9 @@ export function formatScorecard(summary, lift, stability) {
                     ? " · WARNING: subject tokens were used but the provider reported $0 cost"
                     : metrics.subject_cost_usd === null
                         ? " · subject cost unavailable"
-                        : ` · $${metrics.subject_cost_usd.toFixed(6)} provider-reported`;
+                        : metrics.cost_source === "price-table"
+                            ? ` · $${metrics.subject_cost_usd.toFixed(6)} price table`
+                            : ` · $${metrics.subject_cost_usd.toFixed(6)} provider-reported`;
         lines.push(`  COST:   ${tokens} (${metrics.subject_metrics_reps}/${metrics.total_reps} reps reported)${cost} · ${metrics.judge_calls} judge + ${metrics.judge_rejudge_calls} re-judge call(s) · ${metrics.wall_time_ms}ms · ${tools}`);
     }
     // Lift is a statement about a skill-delivered run (green or force). On a red run

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.18.0",
+    "version": "0.19.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -130,6 +130,19 @@
         }
       }
     },
+    "usage": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["repetition", "inputTokens", "outputTokens", "cacheReadTokens", "costUsd", "priceAsOf"],
+      "properties": {
+        "repetition": { "type": "integer", "minimum": 0 },
+        "inputTokens": { "type": ["integer", "null"], "minimum": 1 },
+        "outputTokens": { "type": ["integer", "null"], "minimum": 1 },
+        "cacheReadTokens": { "type": ["integer", "null"], "minimum": 1 },
+        "costUsd": { "type": ["number", "null"], "exclusiveMinimum": 0 },
+        "priceAsOf": { "type": ["string", "null"] }
+      }
+    },
     "metrics": {
       "type": "object",
       "required": [
@@ -184,6 +197,7 @@
           "enum": [
             "provider-reported",
             "subscription",
+            "price-table",
             "unreported"
           ]
         },
@@ -308,6 +322,10 @@
         "metrics": {
           "$ref": "#/$defs/metrics"
         },
+        "usage": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/usage" }
+        },
         "objective": {
           "type": "object",
           "additionalProperties": false,

package/schemas/results-v3.schema.json

@@ -141,6 +141,19 @@
         }
       }
     },
+    "usage": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["repetition", "inputTokens", "outputTokens", "cacheReadTokens", "costUsd", "priceAsOf"],
+      "properties": {
+        "repetition": { "type": "integer", "minimum": 0 },
+        "inputTokens": { "type": ["integer", "null"], "minimum": 1 },
+        "outputTokens": { "type": ["integer", "null"], "minimum": 1 },
+        "cacheReadTokens": { "type": ["integer", "null"], "minimum": 1 },
+        "costUsd": { "type": ["number", "null"], "exclusiveMinimum": 0 },
+        "priceAsOf": { "type": ["string", "null"] }
+      }
+    },
     "metrics": {
       "type": "object",
       "required": [
@@ -195,6 +208,7 @@
           "enum": [
             "provider-reported",
             "subscription",
+            "price-table",
             "unreported"
           ]
         },
@@ -334,6 +348,10 @@
         "metrics": {
           "$ref": "#/$defs/metrics"
         },
+        "usage": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/usage" }
+        },
         "objective": {
           "type": "object",
           "additionalProperties": false,