@skill-harness/core 0.8.0 → 0.10.0

package/dist/adapters/types.d.ts

@@ -1,4 +1,6 @@
 import type { ExecutionTraceV1 } from "../capture-trace-types.js";
+import type { TrajectoryEventSource } from "../spec.js";
+import type { TrajectoryEventV1 } from "../trajectory-gates.js";
 export type RunMode = "red" | "green" | "force";
 /** A provider+model pair, e.g. { provider: "fireworks", model: "accounts/.../deepseek-v4-pro" }. */
 export interface ModelRef {
@@ -40,6 +42,8 @@ export interface RunReq {
      * extension the developer happens to have installed cannot join the test.
      */
     extensions?: string[];
+    /** Workspace-local native ledgers to normalize after the subject finishes. */
+    eventSources?: TrajectoryEventSource[];
 }
 /** A judge request: single prompt, no skills, no session. */
 export interface JudgeReq {
@@ -52,6 +56,10 @@ export interface StructuredRun {
     transcript: string;
     /** One trace per turn — each `pi` invocation emits an independent event stream. */
     traces: ExecutionTraceV1[];
+    /** Adapter-neutral workflow/tool events. Additive; absent on older adapters. */
+    events?: TrajectoryEventV1[];
+    /** Native sources that were required but missing/malformed. Never treated as an empty success. */
+    eventErrors?: string[];
 }
 export interface HarnessAdapter {
     name: string;

package/dist/adjudication.d.ts

@@ -103,6 +103,7 @@ export declare function collapseJudgments(judgments: Judgment[], trigger: Trigge
  * resolving this needs to see what each judge actually said, not just the
  * collapsed answer.
  */
+export declare function boundAdjudicationToRepetitions(result: ScenarioResult, scenario: Scenario, adj: AdjudicationResult): AdjudicationResult;
 export declare function projectAdjudication(result: ScenarioResult, adj: AdjudicationResult): ScenarioResult;
 /** Ask one judge about one cell's saved transcript. Supplied by the caller. */
 export type RejudgeFn = (id: string, judge: ModelRef) => Promise<{

package/dist/adjudication.js

@@ -1,6 +1,6 @@
 import { existsSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
-import { judgeRawPath, writeResults, scoreContextFor, findTranscriptFiles, repIndexOf, rebuildScenarioResult, } from "./results.js";
+import { judgeRawPath, writeResults, scoreContextFor, findTranscriptFiles, repIndexOf, rebuildScenarioResult, mergeScenarioMetrics, effectiveThreshold, } from "./results.js";
 import { score } from "./score.js";
 import { buildJudgePrompt, judgeInWorkspace, parseVerdict } from "./grade.js";
 import { appendJournal } from "./journal.js";
@@ -119,6 +119,14 @@ export function collapseJudgments(judgments, trigger) {
  * resolving this needs to see what each judge actually said, not just the
  * collapsed answer.
  */
+export function boundAdjudicationToRepetitions(result, scenario, adj) {
+    const criticalAggregate = (result.reps ?? 1) > 1 && effectiveThreshold(result, scenario) === 1;
+    // A cell-level re-judge reads one transcript. It can confirm the aggregate,
+    // but cannot erase another clean repetition's failure.
+    return criticalAggregate && result.judge_verdict !== "PASS" && adj.verdict === "PASS"
+        ? { ...adj, state: "unresolved", verdict: undefined }
+        : adj;
+}
 export function projectAdjudication(result, adj) {
     if (adj.state === "unresolved") {
         return {
@@ -231,9 +239,23 @@ export async function adjudicateRun(opts) {
         // Adjudication asks judges again and re-measures nothing else, so the run's
         // objective evidence carries. Overrides survive too — a judge panel does not
         // outvote the author.
-        return adj
-            ? rebuildScenarioResult(projectAdjudication(s, adj), s, { objective: "carry", adjudication: "fresh" })
-            : s;
+        if (!adj)
+            return s;
+        const scenario = byIdScenario.get(s.id);
+        const boundedAdj = scenario ? boundAdjudicationToRepetitions(s, scenario, adj) : adj;
+        const projected = projectAdjudication(s, boundedAdj);
+        if (boundedAdj !== adj) {
+            projected.judge_reason = `${adj.judgments.length} judgments on one transcript cannot replace a critical all-repetitions aggregate`;
+        }
+        const extraCalls = adj.judgments.filter((judgment) => judgment.ordinal > 1).length;
+        projected.metrics = mergeScenarioMetrics(s.metrics, {
+            wall_time_ms: 0,
+            judge_calls: extraCalls,
+            judge_rejudge_calls: extraCalls,
+            subject_metrics_reps: 0,
+            total_reps: s.metrics?.total_reps ?? s.reps ?? 1,
+        });
+        return rebuildScenarioResult(projected, s, { objective: "carry", adjudication: "fresh" });
     });
     appendJournal(opts.runDir, {
         event: "adjudication",

package/dist/capture-trace-types.d.ts

@@ -41,8 +41,12 @@ export interface TraceToolCall {
     args: SanitizedArgs;
     /** 0-based position in ISSUE order (the order pi emitted `tool_execution_start`). */
     issueIndex: number;
+    /** Wall-clock issue time correlated from pi's assistant tool-call message, when available. */
+    started_at?: string;
     /** 0-based position in COMPLETION order. Differs from `issueIndex` under parallelism. */
     completionIndex: number;
+    /** Wall-clock completion time correlated from pi's tool-result message, when available. */
+    completed_at?: string;
     /** pi's `isError` on the `tool_execution_end` event. */
     isError: boolean;
     /**
@@ -79,6 +83,17 @@ export interface TraceResultMeta {
  * conversation carries in the session dir, not in the event stream). This
  * mirrors the loop the adapter already runs.
  */
+export interface TraceMetrics {
+    input_tokens: number;
+    output_tokens: number;
+    cache_read_tokens: number;
+    cache_write_tokens: number;
+    cost_usd: number;
+    tool_calls: number;
+    delegated_children: number;
+    /** Maximum simultaneously outstanding tool calls observed in the pi stream. */
+    max_concurrency: number;
+}
 export interface ExecutionTraceV1 {
     trace_version: typeof EXECUTION_TRACE_VERSION;
     /** `pi --version` at capture time. Null when it could not be determined. */
@@ -127,6 +142,10 @@ export interface ExecutionTraceV1 {
      * disclosure, never for grading.
      */
     cost_usd: number | null;
+    /** Capture defects that make absence-based assertions unsafe (for example malformed native JSONL). */
+    capture_errors?: string[];
+    /** Token/tool/concurrency counters reported or directly observed by the adapter. */
+    metrics?: TraceMetrics;
     /** SHA-256 over the deterministic serialization of this trace, minus this field. */
     trace_sha256?: string;
 }

package/dist/comparison.d.ts

@@ -0,0 +1,96 @@
+import type { ResultsFile, ScenarioResult } from "./results.js";
+import type { Verdict } from "./score.js";
+export interface ComparisonSideDigests {
+    skill: string;
+    spec: string;
+    fixtures: Record<string, string>;
+    /** Every non-candidate test input: scenario facets, persona, fixtures, extensions, post-tests, and system prompts. */
+    inputs: Record<string, string>;
+}
+export interface ComparisonDigests {
+    reference: ComparisonSideDigests;
+    candidate: ComparisonSideDigests;
+    harness: string;
+    model: string;
+    judge: string;
+    environment: {
+        node: string;
+        platform: string;
+        arch: string;
+        [key: string]: string;
+    };
+}
+export interface ComparisonThresholds {
+    max_subject_token_increase?: number;
+    max_wall_time_increase?: number;
+    max_tool_call_increase?: number;
+}
+export interface ComparisonCell {
+    id: string;
+    reference: Verdict;
+    candidate: Verdict;
+    change: "regression" | "lift" | "held-pass" | "held-fail" | "inconclusive";
+    critical: boolean;
+    reference_flakiness: number | null;
+    candidate_flakiness: number | null;
+}
+export interface AggregateMetrics {
+    wall_time_ms: number;
+    judge_calls: number;
+    judge_rejudge_calls: number;
+    subject_metrics_reps: number;
+    total_reps: number;
+    input_tokens: number | null;
+    output_tokens: number | null;
+    cache_read_tokens: number | null;
+    cache_write_tokens: number | null;
+    subject_cost_usd: number | null;
+    tool_calls: number | null;
+    delegated_children: number | null;
+    max_concurrency: number | null;
+}
+export interface ComparisonReport {
+    schema: 1;
+    skill: string;
+    model: string;
+    mode: string;
+    reps: number;
+    judge: string;
+    paired_setup: true;
+    deterministic_sampling: false;
+    sampling_note: string;
+    partial: boolean;
+    release_eligible: boolean;
+    ship: boolean;
+    note: string;
+    cells: ComparisonCell[];
+    behavioral_regressions: string[];
+    critical_regressions: string[];
+    infrastructure_errors: string[];
+    metrics: {
+        reference: AggregateMetrics;
+        candidate: AggregateMetrics;
+    };
+    cost_regressions: string[];
+    thresholds?: ComparisonThresholds;
+    digests: ComparisonDigests;
+    reference: ResultsFile;
+    candidate: ResultsFile;
+}
+export declare function buildComparison(input: {
+    skill: string;
+    model: string;
+    mode: string;
+    reps: number;
+    judge: string;
+    reference: ResultsFile;
+    candidate: ResultsFile;
+    critical: string[];
+    digests: ComparisonDigests;
+    partial?: boolean;
+    thresholds?: ComparisonThresholds;
+}): ComparisonReport;
+/** 2 = critical behavioral regression, 1 = ordinary/ship/infra regression, 0 = clean or partial feedback. */
+export declare function comparisonExitCode(report: ComparisonReport): 0 | 1 | 2;
+export declare function formatComparison(report: ComparisonReport): string;
+export declare function aggregateMetrics(scenarios: ScenarioResult[]): AggregateMetrics;

package/dist/comparison.js

@@ -0,0 +1,183 @@
+export function buildComparison(input) {
+    const { reference, candidate } = input;
+    if (reference.model !== candidate.model || reference.model !== input.model)
+        throw new Error("comparison subject models differ");
+    if (reference.mode !== candidate.mode || reference.mode !== input.mode)
+        throw new Error("comparison delivery modes differ");
+    const referenceJudge = `${reference.judge.provider}:${reference.judge.model}`;
+    const candidateJudge = `${candidate.judge.provider}:${candidate.judge.model}`;
+    if (referenceJudge !== candidateJudge || referenceJudge !== input.judge)
+        throw new Error("comparison judge policies differ");
+    const referenceIds = reference.scenarios.map((scenario) => scenario.id);
+    const candidateIds = candidate.scenarios.map((scenario) => scenario.id);
+    if (JSON.stringify([...referenceIds].sort()) !== JSON.stringify([...candidateIds].sort())) {
+        throw new Error("comparison scenario IDs differ — paired comparisons require the same scenario plan");
+    }
+    if (input.digests.reference.spec !== input.digests.candidate.spec ||
+        JSON.stringify(sorted(input.digests.reference.fixtures)) !== JSON.stringify(sorted(input.digests.candidate.fixtures)) ||
+        JSON.stringify(sorted(input.digests.reference.inputs)) !== JSON.stringify(sorted(input.digests.candidate.inputs))) {
+        throw new Error("comparison spec/test-input digests differ — only the skill candidate may change in a paired run");
+    }
+    const referencePlan = Object.fromEntries(reference.scenarios.map((scenario) => [scenario.id, [scenario.reps ?? 1, scenario.pass_threshold ?? null]]));
+    const candidatePlan = Object.fromEntries(candidate.scenarios.map((scenario) => [scenario.id, [scenario.reps ?? 1, scenario.pass_threshold ?? null]]));
+    if (JSON.stringify(sorted(referencePlan)) !== JSON.stringify(sorted(candidatePlan))) {
+        throw new Error("comparison repetition/threshold plans differ");
+    }
+    const critical = new Set(input.critical);
+    const byCandidate = new Map(candidate.scenarios.map((scenario) => [scenario.id, scenario]));
+    const cells = reference.scenarios.map((left) => {
+        const right = byCandidate.get(left.id);
+        const a = effective(left), b = effective(right);
+        const conclusive = !untrustworthy(left, a) && !untrustworthy(right, b);
+        const change = !conclusive
+            ? "inconclusive"
+            : a === "PASS" && b !== "PASS"
+                ? "regression"
+                : a !== "PASS" && b === "PASS"
+                    ? "lift"
+                    : a === "PASS"
+                        ? "held-pass"
+                        : "held-fail";
+        return {
+            id: left.id,
+            reference: a,
+            candidate: b,
+            change,
+            critical: critical.has(left.id),
+            reference_flakiness: left.flakiness ?? null,
+            candidate_flakiness: right.flakiness ?? null,
+        };
+    });
+    const behavioral = cells.filter((cell) => cell.change === "regression").map((cell) => cell.id);
+    const criticalRegressions = cells.filter((cell) => cell.change === "regression" && cell.critical).map((cell) => cell.id);
+    const byReference = new Map(reference.scenarios.map((scenario) => [scenario.id, scenario]));
+    const infrastructureErrors = cells.filter((cell) => cell.reference === "ERROR" || cell.reference === "JUDGE-AMBIGUOUS" || byReference.get(cell.id)?.suspect ||
+        cell.candidate === "ERROR" || cell.candidate === "JUDGE-AMBIGUOUS" || byCandidate.get(cell.id)?.suspect).map((cell) => cell.id);
+    const metrics = { reference: aggregateMetrics(reference.scenarios), candidate: aggregateMetrics(candidate.scenarios) };
+    const costRegressions = thresholdFindings(metrics.reference, metrics.candidate, input.thresholds);
+    const partial = input.partial === true || reference.partial === true || candidate.partial === true;
+    const releaseEligible = !partial;
+    const ship = releaseEligible && candidate.effective_grade.ship && behavioral.length === 0 && infrastructureErrors.length === 0 && costRegressions.length === 0;
+    const note = partial
+        ? "branch feedback only — partial/affected comparisons never SHIP; run the full paired plan for a release decision"
+        : ship
+            ? "candidate clears its ship bar with no paired behavioral regression"
+            : "candidate is NOT READY under the paired behavioral/cost release policy";
+    return {
+        schema: 1,
+        skill: input.skill,
+        model: input.model,
+        mode: input.mode,
+        reps: input.reps,
+        judge: input.judge,
+        paired_setup: true,
+        deterministic_sampling: false,
+        sampling_note: "paired setup only: prompts/fixtures/models/policy/repetition plan match; provider-deterministic seeding is not claimed",
+        partial,
+        release_eligible: releaseEligible,
+        ship,
+        note,
+        cells,
+        behavioral_regressions: behavioral,
+        critical_regressions: criticalRegressions,
+        infrastructure_errors: [...new Set(infrastructureErrors)],
+        metrics,
+        cost_regressions: costRegressions,
+        ...(input.thresholds ? { thresholds: input.thresholds } : {}),
+        digests: input.digests,
+        reference,
+        candidate,
+    };
+}
+/** 2 = critical behavioral regression, 1 = ordinary/ship/infra regression, 0 = clean or partial feedback. */
+export function comparisonExitCode(report) {
+    if (report.critical_regressions.length)
+        return 2;
+    if (report.behavioral_regressions.length || report.infrastructure_errors.length || report.cost_regressions.length || (report.release_eligible && !report.ship))
+        return 1;
+    return 0;
+}
+export function formatComparison(report) {
+    const lines = [
+        `── compare ${report.skill} · ${report.model} · mode=${report.mode} ×${report.reps} ──`,
+        "  sampling: paired setup; provider-deterministic seeding NOT claimed",
+    ];
+    for (const cell of report.cells) {
+        const mark = cell.change === "regression" ? "↓" : cell.change === "lift" ? "↑" : cell.change === "inconclusive" ? "?" : "=";
+        const variance = cell.reference_flakiness === null && cell.candidate_flakiness === null
+            ? ""
+            : ` · flakiness ${cell.reference_flakiness ?? "n/a"} → ${cell.candidate_flakiness ?? "n/a"}`;
+        lines.push(`  ${mark} ${cell.id}${cell.critical ? " CRITICAL" : ""}: ${cell.reference} → ${cell.candidate} (${cell.change})${variance}`);
+    }
+    lines.push(`  behavior: ${report.behavioral_regressions.length} regression(s), ${report.cells.filter((cell) => cell.change === "lift").length} lift(s), ${report.infrastructure_errors.length} infrastructure/inconclusive error(s)`);
+    lines.push(`  subject tokens: input ${metricDelta(report.metrics.reference.input_tokens, report.metrics.candidate.input_tokens)} · output ${metricDelta(report.metrics.reference.output_tokens, report.metrics.candidate.output_tokens)} · cache-read ${metricDelta(report.metrics.reference.cache_read_tokens, report.metrics.candidate.cache_read_tokens)} (${report.metrics.candidate.subject_metrics_reps}/${report.metrics.candidate.total_reps} candidate reps reported)`);
+    lines.push(`  judge calls: ${report.metrics.reference.judge_calls} → ${report.metrics.candidate.judge_calls} (re-judge ${report.metrics.reference.judge_rejudge_calls} → ${report.metrics.candidate.judge_rejudge_calls})`);
+    lines.push(`  wall time: ${report.metrics.reference.wall_time_ms}ms → ${report.metrics.candidate.wall_time_ms}ms`);
+    lines.push(`  tool calls: ${metricDelta(report.metrics.reference.tool_calls, report.metrics.candidate.tool_calls)}`);
+    if (report.cost_regressions.length)
+        lines.push(`  cost/latency thresholds (separate from behavior): ${report.cost_regressions.join("; ")}`);
+    lines.push(`  ${report.ship ? "SHIP" : "NOT READY"}: ${report.note}`);
+    return lines.join("\n");
+}
+export function aggregateMetrics(scenarios) {
+    const metrics = scenarios.map((scenario) => scenario.metrics).filter((value) => value !== undefined);
+    const sumOptional = (field) => {
+        const values = metrics.map((value) => value[field]).filter((value) => typeof value === "number");
+        return values.length ? values.reduce((sum, value) => sum + value, 0) : null;
+    };
+    const maxValues = metrics.map((value) => value.max_concurrency).filter((value) => typeof value === "number");
+    return {
+        wall_time_ms: metrics.reduce((sum, value) => sum + value.wall_time_ms, 0),
+        judge_calls: metrics.reduce((sum, value) => sum + value.judge_calls, 0),
+        judge_rejudge_calls: metrics.reduce((sum, value) => sum + value.judge_rejudge_calls, 0),
+        subject_metrics_reps: metrics.reduce((sum, value) => sum + value.subject_metrics_reps, 0),
+        total_reps: metrics.reduce((sum, value) => sum + value.total_reps, 0),
+        input_tokens: sumOptional("input_tokens"),
+        output_tokens: sumOptional("output_tokens"),
+        cache_read_tokens: sumOptional("cache_read_tokens"),
+        cache_write_tokens: sumOptional("cache_write_tokens"),
+        subject_cost_usd: sumOptional("subject_cost_usd"),
+        tool_calls: sumOptional("tool_calls"),
+        delegated_children: sumOptional("delegated_children"),
+        max_concurrency: maxValues.length ? Math.max(...maxValues) : null,
+    };
+}
+function effective(result) {
+    if (result.override)
+        return result.override;
+    if (result.objective?.status === "ERROR")
+        return "ERROR";
+    if (result.objective?.status === "FAIL")
+        return "FAIL";
+    return result.judge_verdict;
+}
+function untrustworthy(result, verdict) { return result.suspect || verdict === "ERROR" || verdict === "JUDGE-AMBIGUOUS"; }
+function sorted(value) { return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))); }
+function increase(reference, candidate) { return reference === null || candidate === null || reference <= 0 ? null : (candidate - reference) / reference; }
+function thresholdFindings(reference, candidate, thresholds) {
+    if (!thresholds)
+        return [];
+    const out = [];
+    const check = (label, left, right, max) => {
+        if (max === undefined)
+            return;
+        if (left === null || right === null) {
+            out.push(`${label} unavailable/incomplete — cannot evaluate configured ${(max * 100).toFixed(1)}% maximum`);
+            return;
+        }
+        if (left === 0) {
+            if (right > 0)
+                out.push(`${label} increased from 0 to ${right}, exceeding the configured maximum`);
+            return;
+        }
+        const delta = increase(left, right);
+        if (delta > max)
+            out.push(`${label} +${(delta * 100).toFixed(1)}% exceeds ${(max * 100).toFixed(1)}%`);
+    };
+    check("subject tokens", reference.input_tokens, candidate.input_tokens, thresholds.max_subject_token_increase);
+    check("wall time", reference.wall_time_ms, candidate.wall_time_ms, thresholds.max_wall_time_increase);
+    check("tool calls", reference.tool_calls, candidate.tool_calls, thresholds.max_tool_call_increase);
+    return out;
+}
+function metricDelta(reference, candidate) { return reference === null || candidate === null ? "unavailable" : `${reference} → ${candidate}`; }
+//# sourceMappingURL=comparison.js.map

package/dist/execution-trace.js

@@ -1,4 +1,5 @@
 import { createHash } from "node:crypto";
+import { normalizeSubagentCall } from "./trace-gates.js";
 import { EXECUTION_TRACE_VERSION } from "./capture-trace-types.js";
 import { redactArgs, redactText } from "./capture.js";
 /**
@@ -34,6 +35,8 @@ const MAX_DETAILS_CHARS = 2000;
  */
 export function parseTrace(lines, meta) {
     const calls = new Map();
+    const issuedAt = new Map();
+    const completedAt = new Map();
     let issueCounter = 0;
     let completionCounter = 0;
     let malformedLines = 0;
@@ -41,6 +44,13 @@ export function parseTrace(lines, meta) {
     let finalText = "";
     let lastAssistantText = "";
     let cost = null;
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let cacheReadTokens = 0;
+    let cacheWriteTokens = 0;
+    let sawUsage = false;
+    let activeCalls = 0;
+    let maxConcurrency = 0;
     for (const line of lines) {
         const trimmed = line.trim();
         if (!trimmed)
@@ -65,10 +75,13 @@ export function parseTrace(lines, meta) {
                 name: str(ev.toolName) ?? "(unknown)",
                 args: redactArgs(ev.args, meta.homeDir),
                 issueIndex: issueCounter++,
+                ...(issuedAt.get(id) ? { started_at: issuedAt.get(id) } : {}),
                 completionIndex: -1, // filled in on `end`; -1 means it never completed
                 isError: false,
                 result: { bytes: 0, sha256: sha256("") },
             });
+            activeCalls++;
+            maxConcurrency = Math.max(maxConcurrency, activeCalls);
             continue;
         }
         if (type === "tool_execution_end") {
@@ -78,7 +91,11 @@ export function parseTrace(lines, meta) {
             const call = calls.get(id);
             if (!call)
                 continue; // an end with no start is not evidence of a call
+            if (call.completionIndex < 0)
+                activeCalls = Math.max(0, activeCalls - 1);
             call.completionIndex = completionCounter++;
+            if (completedAt.get(id))
+                call.completed_at = completedAt.get(id);
             call.isError = ev.isError === true;
             call.result = resultMeta(ev.result, meta.homeDir);
             continue;
@@ -86,6 +103,23 @@ export function parseTrace(lines, meta) {
         if (type === "message_end") {
             sawTerminal = true;
             const msg = ev.message;
+            const at = isoTime(msg?.timestamp);
+            if (msg?.role === "assistant" && at) {
+                for (const block of msg.content ?? []) {
+                    if (block.type !== "toolCall" || typeof block.id !== "string")
+                        continue;
+                    issuedAt.set(block.id, at);
+                    const call = calls.get(block.id);
+                    if (call)
+                        call.started_at = at;
+                }
+            }
+            if (msg?.role === "toolResult" && typeof msg.toolCallId === "string" && at) {
+                completedAt.set(msg.toolCallId, at);
+                const call = calls.get(msg.toolCallId);
+                if (call)
+                    call.completed_at = at;
+            }
             if (msg?.role !== "assistant")
                 continue;
             const text = assistantText(msg);
@@ -96,9 +130,21 @@ export function parseTrace(lines, meta) {
                 if (msg.stopReason === "stop")
                     finalText = text;
             }
+            if (msg.usage && (typeof msg.usage.input === "number" || typeof msg.usage.output === "number" ||
+                typeof msg.usage.cacheRead === "number" || typeof msg.usage.cacheWrite === "number" ||
+                typeof msg.usage.cost?.total === "number"))
+                sawUsage = true;
             const total = msg.usage?.cost?.total;
             if (typeof total === "number")
                 cost = (cost ?? 0) + total;
+            if (typeof msg.usage?.input === "number")
+                inputTokens += msg.usage.input;
+            if (typeof msg.usage?.output === "number")
+                outputTokens += msg.usage.output;
+            if (typeof msg.usage?.cacheRead === "number")
+                cacheReadTokens += msg.usage.cacheRead;
+            if (typeof msg.usage?.cacheWrite === "number")
+                cacheWriteTokens += msg.usage.cacheWrite;
             continue;
         }
         if (type === "turn_end" || type === "agent_end" || type === "agent_settled") {
@@ -110,6 +156,19 @@ export function parseTrace(lines, meta) {
             continue;
         }
     }
+    const toolCalls = [...calls.values()].sort((a, b) => a.issueIndex - b.issueIndex);
+    const metrics = {
+        input_tokens: inputTokens,
+        output_tokens: outputTokens,
+        cache_read_tokens: cacheReadTokens,
+        cache_write_tokens: cacheWriteTokens,
+        cost_usd: cost ?? 0,
+        tool_calls: toolCalls.length,
+        delegated_children: toolCalls
+            .filter((call) => call.name === "Agent")
+            .reduce((count, call) => count + normalizeSubagentCall(call.args).length, 0),
+        max_concurrency: maxConcurrency,
+    };
     const trace = {
         trace_version: EXECUTION_TRACE_VERSION,
         pi_version: meta.piVersion,
@@ -126,12 +185,16 @@ export function parseTrace(lines, meta) {
         // — an assertion that used to pass only because the smoke model happened not
         // to echo one.
         final_text: redactText(finalText || lastAssistantText, meta.homeDir),
-        tool_calls: [...calls.values()].sort((a, b) => a.issueIndex - b.issueIndex),
+        tool_calls: toolCalls,
         // `null`, not `[]`: the stream says nothing about the filesystem. The runner
         // overwrites this after observing the workspace. Defaulting to `[]` claimed
         // "observed, nothing changed" for every trace ever parsed.
         changed_paths: meta.changedPaths ? [...meta.changedPaths].sort() : null,
         cost_usd: cost,
+        // Tool calls remain in the trace for objective gates. Aggregate usage/cost/
+        // tool metrics are published only when pi actually reported usage; otherwise
+        // zero would mean "free" instead of "unavailable".
+        ...(sawUsage ? { metrics } : {}),
     };
     trace.trace_sha256 = traceSha256(trace);
     return { trace, isComplete: sawTerminal, malformedLines };
@@ -171,6 +234,12 @@ function resultMeta(result, homeDir) {
 function str(v) {
     return typeof v === "string" && v.length > 0 ? v : undefined;
 }
+function isoTime(value) {
+    if (typeof value !== "number" && typeof value !== "string")
+        return undefined;
+    const date = new Date(value);
+    return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
+}
 function sha256(text) {
     return createHash("sha256").update(text, "utf8").digest("hex");
 }
@@ -241,7 +310,7 @@ export function mergeTraces(traces) {
         // without one would be un-regatable — and an adapter is exactly the layer
         // most likely to forget.
         const only = traces[0];
-        return only.trace_sha256 ? only : { ...only, trace_sha256: traceSha256(only) };
+        return { ...only, trace_sha256: traceSha256(only) };
     }
     const calls = [];
     const changed = new Set();
@@ -271,15 +340,36 @@ export function mergeTraces(traces) {
         if (t.cost_usd !== null)
             cost = (cost ?? 0) + t.cost_usd;
     }
+    const completeMetrics = traces.every((trace) => trace.metrics !== undefined);
+    const metrics = completeMetrics
+        ? traces.reduce((sum, trace) => ({
+            input_tokens: sum.input_tokens + trace.metrics.input_tokens,
+            output_tokens: sum.output_tokens + trace.metrics.output_tokens,
+            cache_read_tokens: sum.cache_read_tokens + trace.metrics.cache_read_tokens,
+            cache_write_tokens: sum.cache_write_tokens + trace.metrics.cache_write_tokens,
+            cost_usd: sum.cost_usd + trace.metrics.cost_usd,
+            tool_calls: sum.tool_calls + trace.metrics.tool_calls,
+            delegated_children: sum.delegated_children + trace.metrics.delegated_children,
+            max_concurrency: Math.max(sum.max_concurrency, trace.metrics.max_concurrency),
+        }), {
+            input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0,
+            cost_usd: 0, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
+        })
+        : undefined;
     const last = traces[traces.length - 1];
+    const captureErrors = traces.flatMap((trace) => trace.capture_errors ?? []);
+    const { capture_errors: _lastCaptureErrors, ...lastWithoutCaptureErrors } = last;
+    void _lastCaptureErrors;
     const merged = {
-        ...last,
+        ...lastWithoutCaptureErrors,
         // The scenario's answer is its LAST turn's answer, matching how the
         // transcript reads and how the judge is asked to grade it.
         final_text: last.final_text,
         tool_calls: calls,
         changed_paths: anyUnobserved ? null : [...changed].sort(),
         cost_usd: cost,
+        ...(captureErrors.length ? { capture_errors: captureErrors } : {}),
+        ...(metrics ? { metrics } : {}),
     };
     merged.trace_sha256 = traceSha256(merged);
     return merged;

package/dist/index.d.ts

@@ -33,6 +33,9 @@ export * from "./spec-write.js";
 export * from "./capture.js";
 export * from "./execution-trace.js";
 export * from "./trace-gates.js";
+export * from "./trajectory-gates.js";
 export * from "./instruction-coverage.js";
 export * from "./affected.js";
 export * from "./adjudication.js";
+export * from "./comparison.js";
+export * from "./sandbox.js";

package/dist/index.js

@@ -33,7 +33,10 @@ export * from "./spec-write.js";
 export * from "./capture.js";
 export * from "./execution-trace.js";
 export * from "./trace-gates.js";
+export * from "./trajectory-gates.js";
 export * from "./instruction-coverage.js";
 export * from "./affected.js";
 export * from "./adjudication.js";
+export * from "./comparison.js";
+export * from "./sandbox.js";
 //# sourceMappingURL=index.js.map

package/dist/lint.js

@@ -60,6 +60,12 @@ export function lintSkill(skillDir) {
         findings.push({ skill, code: "consistency", message: stale.replace(/^warning: /, "") });
     }
     // ship_bar sanity
+    if (!Number.isInteger(spec.ship_bar.total)) {
+        findings.push({ skill, code: "ship_bar", message: "ship_bar.total must be an integer" });
+    }
+    if (!Number.isInteger(spec.ship_bar.min_pass)) {
+        findings.push({ skill, code: "ship_bar", message: "ship_bar.min_pass must be an integer" });
+    }
     if (spec.ship_bar.total < 1) {
         findings.push({ skill, code: "ship_bar", message: "ship_bar.total must be >= 1" });
     }

package/dist/regate.d.ts

@@ -15,7 +15,7 @@ export interface RegateChange {
     from: Verdict;
     to: Verdict;
     /** What the corrected needles say about this scenario now. */
-    gate: "pass" | "fail";
+    gate: "pass" | "fail" | "error";
     /** Whether re-deciding it required a judge call (only a gate-FAIL → gate-pass flip does). */
     judged: boolean;
 }