@skill-harness/core 0.17.0 → 0.19.0

package/README.md

@@ -1,8 +1,7 @@
 # @skill-harness/core
 
 The engine behind [skill-harness](https://github.com/mojomanyana/skill-harness):
-spec parsing, discovery, run orchestration, LLM-judge grading, scoring, results,
-and the separately versioned durable qualification-runner-v1 primitives.
+spec parsing, discovery, run orchestration, LLM-judge grading, scoring, and results.
 
 Internal API — use the `skill-harness` CLI. See the repo for docs.
 

package/dist/adapters/types.d.ts

@@ -1,6 +1,4 @@
 import type { ExecutionTraceV1 } from "../capture-trace-types.js";
-import type { TrajectoryEventSource } from "../spec.js";
-import type { TrajectoryEventV1 } from "../trajectory-gates.js";
 export type RunMode = "red" | "green" | "force";
 /** A provider+model pair, e.g. { provider: "fireworks", model: "accounts/.../deepseek-v4-pro" }. */
 export interface ModelRef {
@@ -57,16 +55,12 @@ export interface RunReq {
      * extension the developer happens to have installed cannot join the test.
      */
     extensions?: string[];
-    /** Workspace-local native ledgers to normalize after the subject finishes. */
-    eventSources?: TrajectoryEventSource[];
     /**
      * Extra env for the subject process, from the arm. Merged over `process.env` by
      * the adapter — an arm must be able to add `PI_GRANTS_*` without the harness
      * inheriting a caller's copy of them.
      */
     armEnv?: Record<string, string>;
-    /** Adapter-produced observation callback. Callers receive evidence; they never supply delivery status. */
-    onPromptObservation?: (observation: PromptProvenance) => void;
 }
 /** A judge request: single prompt, no skills, no session. */
 export interface JudgeReq {
@@ -79,10 +73,6 @@ export interface StructuredRun {
     transcript: string;
     /** One trace per turn — each `pi` invocation emits an independent event stream. */
     traces: ExecutionTraceV1[];
-    /** Adapter-neutral workflow/tool events. Additive; absent on older adapters. */
-    events?: TrajectoryEventV1[];
-    /** Native sources that were required but missing/malformed. Never treated as an empty success. */
-    eventErrors?: string[];
     /**
      * Set when pi failed provider-side (auth, transport) rather than the model
      * answering badly. `run.ts` turns this into ERROR — never a model verdict.
@@ -91,8 +81,8 @@ export interface StructuredRun {
 }
 export interface HarnessAdapter {
     name: string;
-    /** True only when the adapter computes final provider-payload observations. */
-    observesPrompts?: boolean;
+    /** Prefer structured execution for ordinary runs so subject usage is retained. */
+    preferStructured?: boolean;
     available(): Promise<boolean>;
     run(req: RunReq): Promise<string>;
     /**

package/dist/capture-trace-types.d.ts

@@ -83,14 +83,16 @@ export interface TraceResultMeta {
  * conversation carries in the session dir, not in the event stream). This
  * mirrors the loop the adapter already runs.
  */
-export type CostSource = "provider-reported" | "subscription" | "unreported";
+export type CostSource = "provider-reported" | "subscription" | "price-table" | "unreported";
 export interface TraceMetrics {
-    input_tokens: number;
-    output_tokens: number;
-    cache_read_tokens: number;
-    cache_write_tokens: number;
-    cost_usd: number;
+    input_tokens: number | null;
+    output_tokens: number | null;
+    cache_read_tokens: number | null;
+    cache_write_tokens: number | null;
+    cost_usd: number | null;
     cost_source: CostSource;
+    /** Date of the repository price table used for cost_usd; null when unpriced. */
+    price_as_of: string | null;
     tool_calls: number;
     delegated_children: number;
     /** Maximum simultaneously outstanding tool calls observed in the pi stream. */
@@ -151,99 +153,3 @@ export interface ExecutionTraceV1 {
     /** SHA-256 over the deterministic serialization of this trace, minus this field. */
     trace_sha256?: string;
 }
-export declare const CAPTURE_SCHEMA_VERSION = 1;
-/** Why a conversation was captured. */
-export type CaptureClassification = "failure" | "good_example";
-/** Where a capture is in the human review pipeline. */
-export type CapturePromotionStatus = "pending" | "promoted" | "rejected";
-/** What the human confirmed as responsible for the behavior. */
-export interface CaptureTarget {
-    kind: "skill" | "subagent";
-    /** Path relative to the skills root or repo root — never absolute. */
-    path: string;
-    /** SHA-256 of the target's content at capture time, so drift is detectable. */
-    content_sha256: string;
-}
-/**
- * Provenance for a capture.
- *
- * Hashed, not absolute: an absolute session path identifies a machine and a
- * user, and the capture file is meant to be committed. The hash is enough to
- * recognize the same session again locally, which is all provenance needs to do.
- */
-export interface CaptureProvenance {
-    session_sha256: string;
-    /** Indices of the selected turn range within the active branch, inclusive. */
-    turn_range: {
-        start: number;
-        end: number;
-    };
-    subject?: ModelRef;
-    git_commit?: string;
-    /** True when the working tree was dirty at capture time. */
-    git_dirty?: boolean;
-}
-/**
- * A reviewed conversation awaiting promotion into a scenario.
- *
- * Lives under `<skill>/tests/captures/`, NOT in `specification.yaml`. A pending
- * capture is not a test: putting it in the spec would drag it into ship-bar
- * totals, staleness, lift, and stability, and the alternative — a `draft: true`
- * flag every runner, scorer and linter has to honor — is a state that only has
- * to be forgotten once to either corrupt a grade or silently drop a real
- * scenario from a release run.
- */
-export interface CaptureCaseV1 {
-    capture_schema: typeof CAPTURE_SCHEMA_VERSION;
-    id: string;
-    created: string;
-    classification: CaptureClassification;
-    /** Sanitized user turns only. Assistant prose is evidence, never an oracle. */
-    turns: string[];
-    /** Human-written, in their own words. Required — a capture with no expectation is not reviewable. */
-    expected_behavior: string;
-    /** Editable draft checklist. Offline-derived by default; LLM drafting is opt-in and costs tokens. */
-    checklist: string[];
-    target: CaptureTarget;
-    provenance: CaptureProvenance;
-    status: CapturePromotionStatus;
-    /** Set once promoted: the scenario id appended to `specification.yaml`. */
-    scenario_id?: string;
-    /**
-     * `covers` refs for this pending case, so `coverage` can park it against the
-     * instructions it is about before anyone promotes it.
-     *
-     * Derived from `target` — the author has already been made to choose which
-     * instructions are responsible, and asking again in different words would be
-     * asking the same question twice. File granularity, not `#section`: the target
-     * choice attributes a file, and inventing a section would be a guess the
-     * session cannot support.
-     *
-     * Written relative to the SPEC dir (`<skill>/tests`), matching how a scenario's
-     * own `covers` resolve. `coverage` read this field from the start; nothing ever
-     * wrote it, so every pending case parked against nothing and the feature was
-     * inert.
-     */
-    covers?: string[];
-}
-/**
- * Local-only evidence sidecar for a capture.
- *
- * Written to `<skill>/tests/captures/.local/` and git-ignored by default. It may
- * hold a sanitized assistant excerpt and tool-call summaries to help a human
- * review the capture, and must NEVER hold hidden thinking, complete tool-result
- * bodies, or the effective system prompt.
- *
- * pi's stream carries thinking in `message_end`, `turn_end` AND `agent_end`, so
- * dropping it is an explicit filter at every one of those, not a side effect of
- * reading the convenient field.
- */
-export interface CaptureEvidenceV1 {
-    capture_id: string;
-    /** Sanitized, truncated excerpt of the assistant text of every selected turn, joined. */
-    assistant_excerpt: string;
-    /** Tool name, error state and redacted arguments — no bodies. */
-    tool_calls: Array<Pick<TraceToolCall, "name" | "isError"> & {
-        args: SanitizedArgs;
-    }>;
-}

package/dist/capture-trace-types.js

@@ -16,8 +16,4 @@
 // Execution trace
 // ---------------------------------------------------------------------------
 export const EXECUTION_TRACE_VERSION = 2;
-// ---------------------------------------------------------------------------
-// Capture case
-// ---------------------------------------------------------------------------
-export const CAPTURE_SCHEMA_VERSION = 1;
 //# sourceMappingURL=capture-trace-types.js.map

package/dist/command-cost.d.ts

@@ -4,6 +4,6 @@
  * Keep this list narrow: commands that can conditionally spend (notably `regate`)
  * do not belong here even when their common path only replays saved artifacts.
  */
-export declare const FREE_OFFLINE_COMMANDS: readonly ["affected", "archive", "coverage", "init", "judge-agreement", "learning", "lint", "list", "rescore", "restamp", "screen", "stability"];
+export declare const FREE_OFFLINE_COMMANDS: readonly ["coverage", "init", "lint", "list", "rescore", "restamp", "stability"];
 export type FreeOfflineCommand = typeof FREE_OFFLINE_COMMANDS[number];
 export declare function isFreeOfflineCommand(command: string): command is FreeOfflineCommand;

package/dist/command-cost.js

@@ -5,17 +5,12 @@
  * do not belong here even when their common path only replays saved artifacts.
  */
 export const FREE_OFFLINE_COMMANDS = [
-    "affected",
-    "archive",
     "coverage",
     "init",
-    "judge-agreement",
-    "learning",
     "lint",
     "list",
     "rescore",
     "restamp",
-    "screen",
     "stability",
 ];
 export function isFreeOfflineCommand(command) {

package/dist/execution-trace.d.ts

@@ -40,22 +40,6 @@ export declare function traceSha256(trace: ExecutionTraceV1): string;
 export declare function serializeTrace(trace: ExecutionTraceV1): string;
 /** Read a saved trace artifact back, for `regate`. Returns null when unusable. */
 export declare function deserializeTrace(text: string): ExecutionTraceV1 | null;
-/**
- * Collapse a scenario's per-turn traces into one view for gate evaluation.
- *
- * Assertions are written about the scenario ("it delegated to `plan` at least
- * once"), not about turn 3 — a multi-turn scenario would otherwise need the
- * author to know which turn a tool call landed in, which is a property of the
- * model's choices, not of the test.
- *
- * Indices are renumbered across the whole scenario so `issueIndex` stays a total
- * order. `completionIndex` is renumbered within the concatenation too: turns are
- * strictly sequential (each is a separate `pi` invocation), so no completion in
- * turn 2 can precede one in turn 1.
- *
- * Returns null for an empty list — "no turns produced evidence" must not look
- * like "a run in which nothing happened".
- */
 export declare function mergeTraces(traces: ExecutionTraceV1[]): ExecutionTraceV1 | null;
 /** Split a raw stdout blob into lines. Prefer streaming; this is for saved blobs. */
 export declare function lines(text: string): Generator<string>;

package/dist/execution-trace.js

@@ -1,7 +1,7 @@
 import { createHash } from "node:crypto";
 import { normalizeSubagentCall } from "./trace-gates.js";
 import { EXECUTION_TRACE_VERSION } from "./capture-trace-types.js";
-import { redactArgs, redactText } from "./capture.js";
+import { redactArgs, redactText } from "./redaction.js";
 /**
  * Parse pi's `--mode json` event stream into an `ExecutionTraceV1`.
  *
@@ -43,12 +43,10 @@ export function parseTrace(lines, meta) {
     let sawTerminal = false;
     let finalText = "";
     let lastAssistantText = "";
-    let cost = null;
-    let inputTokens = 0;
-    let outputTokens = 0;
-    let cacheReadTokens = 0;
-    let cacheWriteTokens = 0;
-    let sawUsage = false;
+    let inputTokens = null;
+    let outputTokens = null;
+    let cacheReadTokens = null;
+    let cacheWriteTokens = null;
     let activeCalls = 0;
     let maxConcurrency = 0;
     for (const line of lines) {
@@ -130,21 +128,10 @@ export function parseTrace(lines, meta) {
                 if (msg.stopReason === "stop")
                     finalText = text;
             }
-            if (msg.usage && (typeof msg.usage.input === "number" || typeof msg.usage.output === "number" ||
-                typeof msg.usage.cacheRead === "number" || typeof msg.usage.cacheWrite === "number" ||
-                typeof msg.usage.cost?.total === "number"))
-                sawUsage = true;
-            const total = msg.usage?.cost?.total;
-            if (typeof total === "number")
-                cost = (cost ?? 0) + total;
-            if (typeof msg.usage?.input === "number")
-                inputTokens += msg.usage.input;
-            if (typeof msg.usage?.output === "number")
-                outputTokens += msg.usage.output;
-            if (typeof msg.usage?.cacheRead === "number")
-                cacheReadTokens += msg.usage.cacheRead;
-            if (typeof msg.usage?.cacheWrite === "number")
-                cacheWriteTokens += msg.usage.cacheWrite;
+            inputTokens = addReported(inputTokens, msg.usage?.input);
+            outputTokens = addReported(outputTokens, msg.usage?.output);
+            cacheReadTokens = addReported(cacheReadTokens, msg.usage?.cacheRead);
+            cacheWriteTokens = addReported(cacheWriteTokens, msg.usage?.cacheWrite);
             continue;
         }
         if (type === "turn_end" || type === "agent_end" || type === "agent_settled") {
@@ -157,19 +144,14 @@ export function parseTrace(lines, meta) {
         }
     }
     const toolCalls = [...calls.values()].sort((a, b) => a.issueIndex - b.issueIndex);
-    const subscription = meta.subject.provider === "openai-codex" || meta.subject.provider === "claude-code";
-    const costSource = subscription
-        ? "subscription"
-        : cost !== null
-            ? "provider-reported"
-            : "unreported";
     const metrics = {
         input_tokens: inputTokens,
         output_tokens: outputTokens,
         cache_read_tokens: cacheReadTokens,
         cache_write_tokens: cacheWriteTokens,
-        cost_usd: cost ?? 0,
-        cost_source: costSource,
+        cost_usd: null,
+        cost_source: "unreported",
+        price_as_of: null,
         tool_calls: toolCalls.length,
         delegated_children: toolCalls
             .filter((call) => call.name === "Agent")
@@ -197,15 +179,17 @@ export function parseTrace(lines, meta) {
         // overwrites this after observing the workspace. Defaulting to `[]` claimed
         // "observed, nothing changed" for every trace ever parsed.
         changed_paths: meta.changedPaths ? [...meta.changedPaths].sort() : null,
-        cost_usd: cost,
-        // Tool calls remain in the trace for objective gates. Aggregate usage/cost/
-        // tool metrics are published only when pi actually reported usage; otherwise
-        // zero would mean "free" instead of "unavailable".
-        ...(sawUsage ? { metrics } : {}),
+        cost_usd: null,
+        // Tool metrics are observed directly. Token and cost fields stay null when
+        // Pi does not report them; zero would falsely mean "free".
+        metrics,
     };
     trace.trace_sha256 = traceSha256(trace);
     return { trace, isComplete: sawTerminal, malformedLines };
 }
+function addReported(current, value) {
+    return typeof value === "number" && value > 0 ? (current ?? 0) + value : current;
+}
 /** Visible assistant text. Thinking is dropped here, and at every other reader. */
 function assistantText(msg) {
     return (msg.content ?? [])
@@ -308,6 +292,9 @@ export function deserializeTrace(text) {
  * Returns null for an empty list — "no turns produced evidence" must not look
  * like "a run in which nothing happened".
  */
+function sumNullable(left, right) {
+    return left === null ? right : right === null ? left : left + right;
+}
 export function mergeTraces(traces) {
     if (traces.length === 0)
         return null;
@@ -350,18 +337,20 @@ export function mergeTraces(traces) {
     const completeMetrics = traces.every((trace) => trace.metrics !== undefined);
     const metrics = completeMetrics
         ? traces.reduce((sum, trace) => ({
-            input_tokens: sum.input_tokens + trace.metrics.input_tokens,
-            output_tokens: sum.output_tokens + trace.metrics.output_tokens,
-            cache_read_tokens: sum.cache_read_tokens + trace.metrics.cache_read_tokens,
-            cache_write_tokens: sum.cache_write_tokens + trace.metrics.cache_write_tokens,
-            cost_usd: sum.cost_usd + trace.metrics.cost_usd,
+            input_tokens: sumNullable(sum.input_tokens, trace.metrics.input_tokens),
+            output_tokens: sumNullable(sum.output_tokens, trace.metrics.output_tokens),
+            cache_read_tokens: sumNullable(sum.cache_read_tokens, trace.metrics.cache_read_tokens),
+            cache_write_tokens: sumNullable(sum.cache_write_tokens, trace.metrics.cache_write_tokens),
+            cost_usd: sumNullable(sum.cost_usd, trace.metrics.cost_usd),
             cost_source: sum.cost_source === trace.metrics.cost_source ? sum.cost_source : "unreported",
+            price_as_of: sum.price_as_of === trace.metrics.price_as_of ? sum.price_as_of : null,
             tool_calls: sum.tool_calls + trace.metrics.tool_calls,
             delegated_children: sum.delegated_children + trace.metrics.delegated_children,
             max_concurrency: Math.max(sum.max_concurrency, trace.metrics.max_concurrency),
         }), {
-            input_tokens: 0, output_tokens: 0, cache_read_tokens: 0, cache_write_tokens: 0,
-            cost_usd: 0, cost_source: traces[0].metrics.cost_source, tool_calls: 0, delegated_children: 0, max_concurrency: 0,
+            input_tokens: null, output_tokens: null, cache_read_tokens: null, cache_write_tokens: null,
+            cost_usd: null, cost_source: traces[0].metrics.cost_source, price_as_of: traces[0].metrics.price_as_of,
+            tool_calls: 0, delegated_children: 0, max_concurrency: 0,
         })
         : undefined;
     const last = traces[traces.length - 1];

package/dist/index.d.ts

@@ -1,4 +1,3 @@
-export * from "./trusted-host-supervision.js";
 export * from "./spec.js";
 export * from "./discover.js";
 export * from "./run.js";
@@ -13,6 +12,7 @@ export * from "./rescore.js";
 export * from "./workspace.js";
 export * from "./seeded.js";
 export * from "./report.js";
+export * from "./metrics.js";
 export * from "./trends.js";
 export * from "./lint.js";
 export * from "./restamp.js";
@@ -26,39 +26,19 @@ export * from "./version.js";
 export * from "./defaults.js";
 export * from "./command-cost.js";
 export * from "./judge-policy.js";
-export * from "./judge-agreement.js";
 export * from "./regate.js";
-export * from "./screen.js";
 export * from "./prompt-normalization.js";
 export * from "./downgrade.js";
 export * from "./canary.js";
 export * from "./stability.js";
 export * from "./capture-trace-types.js";
-export * from "./work-capture.js";
-export * from "./work-signals.js";
-export * from "./work-capture-schema.js";
-export * from "./factory-calibration.js";
-export * from "./adoption.js";
-export * from "./investigation.js";
-export * from "./intervention.js";
-export * from "./intervention-results.js";
 export * from "./spec-write.js";
-export * from "./capture.js";
+export * from "./redaction.js";
 export * from "./execution-trace.js";
 export * from "./trace-gates.js";
-export * from "./trajectory-gates.js";
+export * from "./trajectory-events.js";
 export * from "./instruction-coverage.js";
-export * from "./affected.js";
-export * from "./adjudication.js";
 export * from "./vote-panel.js";
-export * from "./qualification-panels.js";
-export * from "./qualification-panel-store.js";
-export * from "./qualification-validation.js";
-export * from "./comparison.js";
 export * from "./sandbox.js";
 export * from "./provider-failure.js";
 export * from "./arms.js";
-export * from "./qualification-config.js";
-export * from "./qualification-oauth-directory.js";
-export * from "./qualification-store.js";
-export * from "./qualification-runner.js";

package/dist/index.js

@@ -1,4 +1,3 @@
-export * from "./trusted-host-supervision.js";
 export * from "./spec.js";
 export * from "./discover.js";
 export * from "./run.js";
@@ -13,6 +12,7 @@ export * from "./rescore.js";
 export * from "./workspace.js";
 export * from "./seeded.js";
 export * from "./report.js";
+export * from "./metrics.js";
 export * from "./trends.js";
 export * from "./lint.js";
 export * from "./restamp.js";
@@ -26,40 +26,20 @@ export * from "./version.js";
 export * from "./defaults.js";
 export * from "./command-cost.js";
 export * from "./judge-policy.js";
-export * from "./judge-agreement.js";
 export * from "./regate.js";
-export * from "./screen.js";
 export * from "./prompt-normalization.js";
 export * from "./downgrade.js";
 export * from "./canary.js";
 export * from "./stability.js";
 export * from "./capture-trace-types.js";
-export * from "./work-capture.js";
-export * from "./work-signals.js";
-export * from "./work-capture-schema.js";
-export * from "./factory-calibration.js";
-export * from "./adoption.js";
-export * from "./investigation.js";
-export * from "./intervention.js";
-export * from "./intervention-results.js";
 export * from "./spec-write.js";
-export * from "./capture.js";
+export * from "./redaction.js";
 export * from "./execution-trace.js";
 export * from "./trace-gates.js";
-export * from "./trajectory-gates.js";
+export * from "./trajectory-events.js";
 export * from "./instruction-coverage.js";
-export * from "./affected.js";
-export * from "./adjudication.js";
 export * from "./vote-panel.js";
-export * from "./qualification-panels.js";
-export * from "./qualification-panel-store.js";
-export * from "./qualification-validation.js";
-export * from "./comparison.js";
 export * from "./sandbox.js";
 export * from "./provider-failure.js";
 export * from "./arms.js";
-export * from "./qualification-config.js";
-export * from "./qualification-oauth-directory.js";
-export * from "./qualification-store.js";
-export * from "./qualification-runner.js";
 //# sourceMappingURL=index.js.map

package/dist/instruction-coverage.d.ts

@@ -61,8 +61,6 @@ export interface SectionCoverage {
     section: Section;
     /** Scenario ids that declare a reference to this section. */
     scenarios: string[];
-    /** Capture ids parked against this section but not yet promoted. */
-    pendingCaptures: string[];
 }
 export interface BrokenRef {
     scenarioId: string;
@@ -87,11 +85,6 @@ export interface CoverageOptions {
     scenarios: Scenario[];
     /** Instruction files to report on even if nothing references them. */
     baseFiles?: string[];
-    /** capture id → covers refs, for parking a pending case against a section. */
-    pendingCaptures?: {
-        id: string;
-        covers: string[];
-    }[];
 }
 /**
  * Build the coverage report.

package/dist/instruction-coverage.js

@@ -130,7 +130,7 @@ export function computeCoverage(opts) {
         const k = key(file, section.slug);
         let entry = bySection.get(k);
         if (!entry) {
-            entry = { file, section, scenarios: [], pendingCaptures: [] };
+            entry = { file, section, scenarios: [] };
             bySection.set(k, entry);
         }
         return entry;
@@ -142,7 +142,7 @@ export function computeCoverage(opts) {
             ensure(file, s);
     const broken = [];
     const unmapped = [];
-    const attach = (id, refs, into) => {
+    const attach = (id, refs) => {
         for (const raw of refs) {
             const ref = parseCoversRef(raw);
             const sections = readSections(ref.file);
@@ -155,7 +155,7 @@ export function computeCoverage(opts) {
             if (ref.slug === undefined) {
                 // A whole-file reference covers every section in it.
                 for (const s of sections)
-                    ensure(ref.file, s)[into].push(id);
+                    ensure(ref.file, s).scenarios.push(id);
                 continue;
             }
             const match = sections.find((s) => s.slug === ref.slug);
@@ -168,7 +168,7 @@ export function computeCoverage(opts) {
                 });
                 continue;
             }
-            ensure(ref.file, match)[into].push(id);
+            ensure(ref.file, match).scenarios.push(id);
         }
     };
     for (const s of opts.scenarios) {
@@ -176,10 +176,8 @@ export function computeCoverage(opts) {
             unmapped.push(s.id);
             continue;
         }
-        attach(s.id, s.covers, "scenarios");
+        attach(s.id, s.covers);
     }
-    for (const c of opts.pendingCaptures ?? [])
-        attach(c.id, c.covers, "pendingCaptures");
     const sections = [...bySection.values()].sort((a, b) => a.file.localeCompare(b.file) || a.section.startLine - b.section.startLine);
     const covered = sections.filter((s) => s.scenarios.length > 0);
     const uncovered = sections.filter((s) => s.scenarios.length === 0);

package/dist/journal.d.ts

@@ -63,14 +63,6 @@ export type JournalEvent = {
     ok: boolean;
     detail: string;
     rep?: number;
-}
-/** One adjudication pass: which cells were re-judged, what it cost in CALLS, what stayed unresolved. */
- | {
-    event: "adjudication";
-    ts: string;
-    triggered: string[];
-    judge_calls: number;
-    unresolved: string[];
 } | {
     event: "judge-verdict";
     ts: string;

package/dist/metrics.d.ts

@@ -0,0 +1,19 @@
+import type { ScenarioResult } from "./results.js";
+import type { CostSource } from "./capture-trace-types.js";
+export interface AggregateMetrics {
+    wall_time_ms: number;
+    judge_calls: number;
+    judge_rejudge_calls: number;
+    subject_metrics_reps: number;
+    total_reps: number;
+    input_tokens: number | null;
+    output_tokens: number | null;
+    cache_read_tokens: number | null;
+    cache_write_tokens: number | null;
+    subject_cost_usd: number | null;
+    cost_source: CostSource | null;
+    tool_calls: number | null;
+    delegated_children: number | null;
+    max_concurrency: number | null;
+}
+export declare function aggregateMetrics(scenarios: ScenarioResult[]): AggregateMetrics;

package/dist/metrics.js

@@ -0,0 +1,28 @@
+export function aggregateMetrics(scenarios) {
+    const metrics = scenarios.map((scenario) => scenario.metrics).filter((value) => value !== undefined);
+    const sumOptional = (field) => {
+        const values = metrics.map((value) => value[field]).filter((value) => typeof value === "number");
+        return values.length ? values.reduce((sum, value) => sum + value, 0) : null;
+    };
+    const maxValues = metrics.map((value) => value.max_concurrency).filter((value) => typeof value === "number");
+    return {
+        wall_time_ms: metrics.reduce((sum, value) => sum + value.wall_time_ms, 0),
+        judge_calls: metrics.reduce((sum, value) => sum + value.judge_calls, 0),
+        judge_rejudge_calls: metrics.reduce((sum, value) => sum + value.judge_rejudge_calls, 0),
+        subject_metrics_reps: metrics.reduce((sum, value) => sum + value.subject_metrics_reps, 0),
+        total_reps: metrics.reduce((sum, value) => sum + value.total_reps, 0),
+        input_tokens: sumOptional("input_tokens"),
+        output_tokens: sumOptional("output_tokens"),
+        cache_read_tokens: sumOptional("cache_read_tokens"),
+        cache_write_tokens: sumOptional("cache_write_tokens"),
+        subject_cost_usd: sumOptional("subject_cost_usd"),
+        cost_source: (() => {
+            const sources = scenarios.map((scenario) => scenario.metrics?.cost_source).filter((source) => source !== undefined);
+            return sources.length === 0 ? null : sources.every((source) => source === sources[0]) ? sources[0] : "unreported";
+        })(),
+        tool_calls: sumOptional("tool_calls"),
+        delegated_children: sumOptional("delegated_children"),
+        max_concurrency: maxValues.length ? Math.max(...maxValues) : null,
+    };
+}
+//# sourceMappingURL=metrics.js.map

package/dist/redaction.d.ts

@@ -0,0 +1,6 @@
+import type { SanitizedArgs } from "./capture-trace-types.js";
+/** Values longer than this are truncated rather than persisted whole. */
+export declare const MAX_VALUE_CHARS = 2000;
+export declare function redactText(input: string, homeDir?: string): string;
+export declare function truncate(input: string, max?: number): string;
+export declare function redactArgs(args: unknown, homeDir?: string, depth?: number): SanitizedArgs;