@tangle-network/agent-eval 0.123.7 → 0.124.0

package/dist/rollout/index.d.ts

@@ -0,0 +1,1053 @@
+import { DatabaseSync } from 'node:sqlite';
+
+/**
+ * `tangle.rollout.v1` — THE canonical rollout serialization, owned by
+ * agent-eval. One JSONL line per agent invocation (a solo eval run, a
+ * supervisor episode, a worker session, a proposer shot, a judge call, an
+ * analyst pass), labeled with its task/split coordinates and a single
+ * scalar reward, carrying the FULL message transcript inline.
+ *
+ * This schema is the reconciliation of two prior producers:
+ *   - agent-eval's RunRecord-joined rollout rows (PR #410): identity,
+ *     provenance hashes, the realness gate travelling into the reward,
+ *     trace-derived steps.
+ *   - the bench rollout-ledger (agent-runtime PR #591): the wire shape —
+ *     role, task.split/rep, parent_rollout_id, policy provenance, capture
+ *     provenance, inline canonical chat-with-tools messages.
+ * Where the two conflicted, RunRecord-derived semantics won; the wire
+ * field names follow the ledger (snake_case). See `docs/rollout.md` for
+ * the field-by-field decision table.
+ *
+ * Messages are inlined — never referenced — because every harness store a
+ * rollout can be recovered from is mutable or garbage-collected. A line
+ * must stay a complete training/eval example on its own.
+ *
+ * `outcome.reward` is THE single scalar (null = no verdict exists — a
+ * labeled gap, never 0). `outcome.realness_gated` is the anti-Goodhart
+ * flag: a gated line must never export as a positive training example.
+ */
+declare const ROLLOUT_SCHEMA = "tangle.rollout.v1";
+/** @deprecated alias kept for consumers of the pre-unification constant name. */
+declare const ROLLOUT_FORMAT = "tangle.rollout.v1";
+/** `agent` = a solo evaluation run (no multi-agent topology). */
+type RolloutRole = 'agent' | 'supervisor' | 'worker' | 'proposer' | 'judge' | 'analyst';
+declare const ROLLOUT_ROLES: readonly RolloutRole[];
+/**
+ * Split vocabulary follows `RunRecord.splitTag` ('search' is the pool the
+ * optimizer may read — the trainable split), extended with the ledger's
+ * 'canary'. 'train' is a legacy alias for 'search' emitted by
+ * pre-unification ledgers; it validates and counts as trainable, but new
+ * producers must emit 'search'.
+ */
+type RolloutSplit = 'search' | 'dev' | 'holdout' | 'canary' | 'train';
+declare const ROLLOUT_SPLITS: readonly RolloutSplit[];
+/** Splits that may ship in training exports. Everything else is fail-closed excluded. */
+declare const TRAINABLE_SPLITS: readonly RolloutSplit[];
+declare function isTrainableSplit(split: RolloutSplit): boolean;
+/** 'mint' = joined live from RunRecord + trace by `mintRolloutRows`. */
+type RolloutCapture = 'mint' | 'settle-time' | 'backfill';
+declare const ROLLOUT_CAPTURES: readonly RolloutCapture[];
+type ChatRole = 'system' | 'user' | 'assistant' | 'tool';
+declare const CHAT_ROLES: readonly ChatRole[];
+interface ChatToolCall {
+    id: string;
+    type: 'function';
+    function: {
+        name: string;
+        /** JSON-encoded argument object, exactly as the model emitted it. */
+        arguments: string;
+    };
+}
+interface ChatMessage {
+    role: ChatRole;
+    content: string | null;
+    /** Reasoning/thinking channel where the harness captured it (full fidelity). */
+    reasoning_content?: string;
+    tool_calls?: ChatToolCall[];
+    /** Required on role:"tool" — the ChatToolCall this result answers. */
+    tool_call_id?: string;
+    name?: string;
+}
+interface ToolDef {
+    type: 'function';
+    function: {
+        name: string;
+        description?: string;
+        parameters?: Record<string, unknown>;
+    };
+}
+/**
+ * Compact trace-span projection (llm/tool step) carried alongside the
+ * conversation when the line was minted from a trace. Optional: lines
+ * recovered from harness stores have no span structure.
+ */
+interface RolloutStep {
+    kind: string;
+    name: string;
+    /** llm: last-message summary · tool: stringified args. Scrubbed. */
+    input?: string;
+    /** llm: output text · tool: stringified result. Scrubbed. */
+    output?: string;
+    status?: 'ok' | 'error';
+    durationMs?: number;
+}
+interface RolloutTask {
+    /** Benchmark/suite id (e.g. "swe-bench-verified") or the experiment id. */
+    suite: string;
+    instance_id: string;
+    split: RolloutSplit;
+    /** Sampling seed the campaign pinned; null = not recorded. */
+    seed: number | null;
+    /** Replicate index (0-based). */
+    rep: number;
+}
+interface RolloutPolicy {
+    /** Harness that drove the invocation (e.g. "opencode", "claude", "pi-loops"). */
+    harness: string | null;
+    harness_version: string | null;
+    model: string | null;
+    provider: string | null;
+    /** Commit of the agent profile / candidate under evaluation. */
+    profile_commit: string | null;
+    /** sha256 of the effective prompt (post-steering), when recorded. */
+    prompt_hash?: string | null;
+    /** sha256 of the effective run config, when recorded. */
+    config_hash?: string | null;
+    /** Canonical agent-profile cell identity, when the run carries one. */
+    agent_profile_cell_id?: string | null;
+    /** Sampling params (temperature, top_p, max_tokens…); null = not recorded. */
+    sampling: Record<string, unknown> | null;
+}
+interface RolloutOutcome {
+    /**
+     * THE single scalar training signal — the official verdict.
+     * null = no verdict exists for this invocation (a labeled gap, never 0).
+     */
+    reward: number | null;
+    /** Where the reward came from (judge id; "/inherited" = parent episode's). */
+    reward_source: string | null;
+    /** Raw judge verdict record, verbatim. */
+    verdict: unknown;
+    /** Everything that is NOT the scalar reward. */
+    metrics: Record<string, unknown>;
+    is_completed: boolean;
+    is_truncated: boolean;
+    error: string | null;
+    /**
+     * Anti-Goodhart flag from `RunRecord.outcome.realness.gated`: the run
+     * faked its success signal. Reward is forced to 0 at mint time and the
+     * line never qualifies for SFT. Optional on the wire (absent = false)
+     * so pre-unification ledgers stay readable.
+     */
+    realness_gated?: boolean;
+}
+interface RolloutCostBlock {
+    usd: number | null;
+    tokens_in: number | null;
+    tokens_out: number | null;
+    tokens_reasoning: number | null;
+    cache_read: number | null;
+    cache_write: number | null;
+    wall_s: number | null;
+}
+interface RolloutArtifacts {
+    patch_path: string | null;
+    run_dir: string | null;
+    /** Source-of-truth transcript pointer (session id / jsonl path) for audit. */
+    transcript_ref: string | null;
+}
+interface RolloutProvenance {
+    captured_at: string;
+    capture: RolloutCapture;
+    /** Present on gap lines: why `messages` could not be recovered. */
+    gap?: string;
+}
+interface RolloutLine {
+    schema: typeof ROLLOUT_SCHEMA;
+    rollout_id: string;
+    /** Spawning invocation within the same episode (worker → supervisor). */
+    parent_rollout_id: string | null;
+    run_id: string;
+    /** Logical experiment grouping from `RunRecord.experimentId`. Optional on
+     *  the wire (pre-unification ledgers lack it); null = not recorded. */
+    experiment_id?: string | null;
+    /** Stable candidate identity from `RunRecord.candidateId`; null = not recorded. */
+    candidate_id?: string | null;
+    /** Improvement-loop generation (-1 = baseline); null = not an improvement loop. */
+    generation: number | null;
+    /** Improvement-loop candidate index (-1 = baseline); null = not an improvement loop. */
+    candidate_index: number | null;
+    role: RolloutRole;
+    task: RolloutTask;
+    policy: RolloutPolicy;
+    /** Full transcript, inline. [] = gap line (see provenance.gap). */
+    messages: ChatMessage[];
+    tool_defs: ToolDef[];
+    /** Trace-span projections, when minted from a trace. */
+    steps?: RolloutStep[];
+    outcome: RolloutOutcome;
+    cost: RolloutCostBlock;
+    artifacts: RolloutArtifacts;
+    provenance: RolloutProvenance;
+}
+declare function validateRolloutLine(value: unknown): string[];
+declare function assertRolloutLine(value: unknown, context?: string): asserts value is RolloutLine;
+declare function isRolloutLine(value: unknown): value is RolloutLine;
+
+/**
+ * Pure exporters over `tangle.rollout.v1` lines → the training-data shapes
+ * the improvement loops feed:
+ *   - SFT chat JSONL           (clean trainable successes, {messages, metadata})
+ *   - reward rows              (every scored line, success or failure, with steps)
+ *   - Prime Intellect verifiers RolloutOutput (prompt/completion split + reward)
+ *   - OpenAI RFT items         (prompt turns + verdict reference fields)
+ *
+ * All exporters are pure functions of the lines — filtering (never train on
+ * holdout, reward thresholds, the realness gate) happens HERE, on inline
+ * labels, no joins.
+ */
+
+interface SftExportOptions {
+    /** Export only lines with reward ≥ this (default 1 = clean successes only). */
+    minReward?: number;
+}
+interface SftRow {
+    messages: ChatMessage[];
+    metadata: {
+        rollout_id: string;
+        run_id: string;
+        candidate_id: string | null;
+        instance_id: string;
+        reward: number;
+    };
+}
+/**
+ * Supervised fine-tune rows: the completed conversation of each qualifying
+ * line. Fail-closed filters: trainable split only (never holdout/canary),
+ * reward ≥ minReward, realness-gated lines never qualify, gap lines carry
+ * no trainable content.
+ */
+declare function toSftRows(lines: RolloutLine[], options?: SftExportOptions): SftRow[];
+interface RewardRow {
+    /** First user turn — the task prompt. */
+    prompt: string;
+    steps: RolloutStep[];
+    reward: number;
+    metadata: {
+        rollout_id: string;
+        run_id: string;
+        candidate_id: string | null;
+        instance_id: string;
+        split: RolloutSplit;
+    };
+}
+/**
+ * Reward-labeled rows: every line with a scalar reward, success or
+ * failure. Failures are signal here — only the realness-gate zeroing
+ * (applied at mint time) touches the reward, never filtering. Lines with
+ * no verdict (reward null) are excluded: an unlabeled example is a gap,
+ * not a zero.
+ */
+declare function toRewardRows(lines: RolloutLine[]): RewardRow[];
+interface VerifiersTokenUsage {
+    input_tokens: number | null;
+    output_tokens: number | null;
+    reasoning_tokens: number | null;
+    cache_read_tokens: number | null;
+    cache_write_tokens: number | null;
+}
+interface VerifiersRolloutOutput {
+    /** Messages through the last turn BEFORE the first assistant turn. */
+    prompt: ChatMessage[];
+    /** The first assistant turn onward — what the policy produced. */
+    completion: ChatMessage[];
+    reward: number | null;
+    metrics: Record<string, unknown>;
+    tool_defs: ToolDef[];
+    token_usage: VerifiersTokenUsage;
+    info: {
+        task: RolloutLine['task'];
+        policy: RolloutLine['policy'];
+        rollout_id: string;
+        run_id: string;
+        experiment_id: string | null;
+        candidate_id: string | null;
+        generation: number | null;
+        candidate_index: number | null;
+        role: RolloutLine['role'];
+    };
+}
+declare function toVerifiersRolloutOutput(line: RolloutLine): VerifiersRolloutOutput;
+declare function toVerifiersRolloutOutputs(lines: RolloutLine[]): VerifiersRolloutOutput[];
+interface RftItem {
+    /** Prompt turns only — the graded completion is re-sampled during RFT. */
+    messages: ChatMessage[];
+    /** Verdict/label fields the grader references as item.reference.* */
+    reference: {
+        reward: number | null;
+        reward_source: string | null;
+        verdict: unknown;
+        instance_id: string;
+        suite: string;
+        split: RolloutSplit;
+        rollout_id: string;
+    };
+}
+declare function toRftItem(line: RolloutLine): RftItem;
+/** RFT needs a real prompt: lines whose transcript starts with prompt turns. */
+declare function toRftItems(lines: RolloutLine[]): RftItem[];
+declare function toJsonl(rows: ReadonlyArray<unknown>): string;
+
+/**
+ * Rollout-ledger file API — append-only JSONL of validated `tangle.rollout.v1`
+ * lines. Writes validate BEFORE touching disk (a bad line never lands);
+ * reads validate line-by-line and fail loud with the line number, because a
+ * silently-skipped rollout is a corrupted dataset.
+ */
+
+/** Replace the ledger file with exactly `lines`. */
+declare function writeRolloutLedger(path: string, lines: RolloutLine[]): Promise<void>;
+/** Append `lines` to the ledger file (created if absent). */
+declare function appendRolloutLines(path: string, lines: RolloutLine[]): Promise<void>;
+/**
+ * Read and validate every line. Throws on the first malformed/invalid line
+ * (with its 1-based line number) — fail-closed, never a silent drop.
+ */
+declare function readRolloutLedger(path: string): Promise<RolloutLine[]>;
+
+type AgentProfileCellSchemaVersion = 'agent-profile-cell/v1';
+type AgentProfileDimensionValue = string | number | boolean | null;
+interface AgentProfileSource {
+    /** Runtime/profile contract being fingerprinted, e.g. `agent-interface-profile`. */
+    kind: string;
+    /** sha256 over the canonical source profile object. */
+    hash: string;
+}
+interface AgentProfileHarness {
+    id: string;
+    version?: string;
+    hash?: string;
+}
+interface AgentProfileCell {
+    schemaVersion: AgentProfileCellSchemaVersion;
+    cellId: string;
+    profileId: string;
+    sourceProfile: AgentProfileSource;
+    harness?: AgentProfileHarness;
+    model?: string;
+    promptHash?: string;
+    dimensions?: Record<string, AgentProfileDimensionValue>;
+}
+
+type RunStatus = 'running' | 'completed' | 'failed' | 'aborted';
+interface BudgetSpec {
+    tokens?: number;
+    wallMs?: number;
+    calls?: number;
+    usd?: number;
+}
+interface RunOutcome$1 {
+    score?: number;
+    pass?: boolean;
+    failureClass?: FailureClass;
+    notes?: string;
+}
+/**
+ * Layer — optional classification in a nested build workflow.
+ * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).
+ * `app-build`: sandbox harness that compiled + tested the generated scaffold.
+ * `app-runtime`: a run of the generated agent against a domain scenario.
+ * `meta`: any meta-eval (judge replay, correlation analysis).
+ */
+type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom';
+interface Run {
+    runId: string;
+    /**
+     * Stable identifier of the scenario being executed.
+     *
+     * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts
+     * input WITHOUT this field, substituting a sensible default
+     * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no
+     * curated scenario to anchor to (runtime / operator / meta-eval runs). This
+     * keeps the persisted shape unambiguous for downstream filters + aggregations
+     * while removing the boilerplate of inventing placeholder ids at the call site.
+     */
+    scenarioId: string;
+    variantId?: string;
+    datasetVersion?: string;
+    /** Git SHA of agent code at run time. */
+    codeSha?: string;
+    /** Hash of the prompt template + any system prompt. */
+    promptSha?: string;
+    /** Model id + date + system-prompt hash, concatenated. */
+    modelFingerprint?: string;
+    seed?: number;
+    /** Arbitrary environment markers (shell, docker version, tz). */
+    envFingerprint?: Record<string, string>;
+    /** Version of the redaction rules applied to this run. */
+    redactionVersion?: string;
+    /** Parent run in a nested build workflow. A builder run's children are
+     *  app-build runs; those children are app-runtime runs. */
+    parentRunId?: string;
+    /** Stable project identifier — groups runs across chats + sessions. */
+    projectId?: string;
+    /** Chat/conversation identifier within a project. */
+    chatId?: string;
+    /** Layer classification — hint for aggregation; not enforced. */
+    layer?: RunLayer;
+    startedAt: number;
+    endedAt?: number;
+    status: RunStatus;
+    outcome?: RunOutcome$1;
+    budget?: BudgetSpec;
+    /** Free-form labels for downstream grouping. */
+    tags?: Record<string, string>;
+}
+type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom';
+type SpanStatus = 'ok' | 'error';
+interface SpanBase {
+    spanId: string;
+    parentSpanId?: string;
+    runId: string;
+    kind: SpanKind;
+    name: string;
+    startedAt: number;
+    endedAt?: number;
+    status?: SpanStatus;
+    error?: string;
+    /** Anything not covered by typed fields. Kept deliberately free-form. */
+    attributes?: Record<string, unknown>;
+}
+interface Message {
+    role: 'system' | 'user' | 'assistant' | 'tool';
+    content: string;
+    tokens?: number;
+    /** Multi-modal content descriptors; blobs themselves live in Artifacts. */
+    images?: Array<{
+        artifactId?: string;
+        url?: string;
+        mime?: string;
+    }>;
+}
+interface LlmSpan extends SpanBase {
+    kind: 'llm';
+    model: string;
+    messages: Message[];
+    output?: string;
+    inputTokens?: number;
+    /** All generated tokens, including the reasoning subset when present. */
+    outputTokens?: number;
+    cachedTokens?: number;
+    cacheWriteTokens?: number;
+    /** Reasoning-token subset of `outputTokens`. */
+    reasoningTokens?: number;
+    costUsd?: number;
+    finishReason?: string;
+}
+interface ToolSpan extends SpanBase {
+    kind: 'tool';
+    toolName: string;
+    args: unknown;
+    /** False when the source observed the call but did not capture its arguments. */
+    argsCaptured?: boolean;
+    result?: unknown;
+    latencyMs?: number;
+}
+interface RetrievalSpan extends SpanBase {
+    kind: 'retrieval';
+    query: string;
+    hits: Array<{
+        docId: string;
+        score: number;
+        content?: string;
+    }>;
+}
+interface JudgeSpan extends SpanBase {
+    kind: 'judge';
+    judgeId: string;
+    /** Span this judgment applies to. */
+    targetSpanId: string;
+    dimension: string;
+    /** Numeric score (free-range; interpretation up to the judge). */
+    score: number;
+    rationale?: string;
+    evidence?: string;
+}
+interface SandboxSpan extends SpanBase {
+    kind: 'sandbox';
+    image?: string;
+    command?: string;
+    exitCode?: number;
+    testsTotal?: number;
+    testsPassed?: number;
+    stdoutHash?: string;
+    stderrHash?: string;
+    /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */
+    wallMs?: number;
+}
+interface GenericSpan extends SpanBase {
+    kind: 'agent' | 'custom';
+}
+type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan;
+type EventKind = 'log' | 'error' | 'budget_decrement' | 'budget_breach' | 'state_mutation' | 'policy_violation' | 'redaction_applied' | 'custom';
+interface TraceEvent {
+    eventId: string;
+    runId: string;
+    spanId?: string;
+    kind: EventKind;
+    timestamp: number;
+    payload: Record<string, unknown>;
+}
+interface BudgetLedgerEntry {
+    runId: string;
+    dimension: keyof BudgetSpec;
+    limit: number;
+    consumed: number;
+    remaining: number;
+    timestamp: number;
+    breached: boolean;
+    /** Span that triggered this entry, if any. */
+    spanId?: string;
+}
+interface Artifact {
+    artifactId: string;
+    runId: string;
+    spanId?: string;
+    contentType: string;
+    sizeBytes: number;
+    /** sha256 in hex. */
+    hash: string;
+    /** External storage URL (R2, S3, filesystem path). */
+    storageUrl?: string;
+    /** Inline content for small blobs — keep under ~64KB. */
+    inlineContent?: string;
+}
+type FailureClass = 'success' | 'reasoning_error' | 'tool_selection_error' | 'tool_argument_error' | 'tool_recovery_failure' | 'hallucination' | 'instruction_following' | 'safety_refusal_miss' | 'policy_violation' | 'budget_exceeded' | 'format_drift' | 'permission_escalation' | 'pii_leak' | 'cost_overrun' | 'timeout' | 'sandbox_failure' | 'missing_user_data' | 'missing_domain_data' | 'missing_codebase_context' | 'missing_runtime_context' | 'missing_credentials' | 'missing_integration_connection' | 'missing_integration_scope' | 'integration_approval_required' | 'integration_auth_expired' | 'integration_provider_failure' | 'bad_integration_manifest' | 'unsafe_integration_write_denied' | 'stale_external_data' | 'bad_retrieval' | 'insufficient_evidence' | 'contradictory_evidence' | 'ambiguous_user_intent' | 'knowledge_readiness_blocked' | 'unknown';
+
+/**
+ * Paper-grade RunRecord schema + runtime validator.
+ *
+ * Every run that participates in a promotion gate, paper table, or
+ * researcher loop SHOULD be recorded as a `RunRecord`. The mandatory
+ * fields are exactly those the paper "Two Loops, Three Roles" requires
+ * for reproducibility: who/what/when/cost/seed/hash, plus the search vs
+ * holdout split tag and either a `searchScore` or a `holdoutScore`.
+ *
+ * This is intentionally NOT a replacement for the rich `Run` /
+ * `ProposeReviewReport` / `ScenarioResult` types already in the
+ * package. Those are runtime structures with full provenance. A
+ * `RunRecord` is the analysis-time projection — the JSON-friendly
+ * row you'd put in a parquet file or paste into a notebook.
+ *
+ * Validate at the boundary:
+ *
+ *   const rec = validateRunRecord(rawJson)         // throws on missing
+ *   const ok  = isRunRecord(rawJson)               // boolean check
+ *   const rec = parseRunRecordSafe(rawJson)        // { ok, value | error }
+ *
+ * The validator runs in pure TS — zod is intentionally NOT a
+ * dependency. Round-trip tested in `tests/run-record.test.ts`.
+ */
+
+/** Search/dev/holdout split tag. 'search' is the paper-grade alias for the
+ *  combined train+test pool that the optimizer is allowed to read. */
+type RunSplitTag = 'search' | 'dev' | 'holdout';
+interface RunTokenUsage {
+    input: number;
+    /** All generated tokens charged as output, including reasoning tokens. */
+    output: number;
+    /** Reasoning-token subset of `output`, when the provider reports it. */
+    reasoning?: number;
+    /** Prompt tokens served from a provider cache. */
+    cached?: number;
+    /** Prompt tokens written into a provider cache. */
+    cacheWrite?: number;
+}
+/**
+ * How a run's USD amount was obtained.
+ *
+ * `costUsd` remains mandatory for wire compatibility. New producers should
+ * always populate this discriminated union so a missing bill is never
+ * mistaken for an observed zero-dollar run. For `uncaptured`, `costUsd` uses
+ * the legacy `0` sentinel while this field carries the truthful null.
+ */
+type RunCostProvenance = {
+    kind: 'observed';
+    usd: number;
+} | {
+    kind: 'estimated';
+    usd: number;
+} | {
+    kind: 'uncaptured';
+    usd: null;
+};
+interface RunJudgeMetadata {
+    model: string;
+    promptVersion: string;
+    /** [0,1] confidence the judge declared. Constant judge confidence
+     *  across many runs is a fallback signal (see `canary.ts`). */
+    confidence: number;
+    /** True if the judge degraded to a fallback path (rules-only,
+     *  prior-call cache, etc.). The canary uses this to alert. */
+    fallback: boolean;
+}
+/**
+ * Per-judge / per-dimension breakdown for runs scored by an ensemble of
+ * judges over a multi-dimensional rubric.
+ *
+ * The collapsed `outcome.searchScore` / `holdoutScore` carries the
+ * composite the gate uses. The full breakdown belongs here so consumers
+ * can answer "which judge disagreed?", "which dimension dragged the
+ * composite down?", and "did half the panel fail?" without re-running.
+ *
+ * `perJudge[judgeId][dim]` is the canonical source; `perDimMean` and
+ * `composite` are convenience projections — derivable but precomputed so
+ * downstream IRR primitives (`interRaterReliability`,
+ * `corpusInterRaterAgreement`) and reporters don't pay the same
+ * aggregation twice.
+ *
+ * Fail-loud discipline: judges that errored out land in `failedJudges`
+ * by id. A missing key in `perJudge` is ambiguous (silent zero vs not
+ * run); the explicit list makes a partial-failure recorded as such.
+ */
+interface JudgeScoresRecord {
+    /** Per-judge per-dimension scores. `{ "kimi-k2.6": { helpfulness: 0.8, clarity: 0.7 }, ... }`. */
+    perJudge: Record<string, Record<string, number>>;
+    /** Per-dim mean across judges. Convenience — derivable from `perJudge`. */
+    perDimMean: Record<string, number>;
+    /** Composite mean across all dims and judges. Mirrors the score
+     *  the gate sees on `outcome.searchScore` / `holdoutScore`. */
+    composite: number;
+    /** Judges that errored or returned an unparseable verdict. Recorded
+     *  by id (e.g. `['glm-5.1']`) so a partial-failure case is explicit,
+     *  not inferred from missing keys in `perJudge`. */
+    failedJudges?: string[];
+    /** Free-form notes the judges emitted (joined across judges or
+     *  first-judge only — consumer's choice). */
+    notes?: string;
+}
+interface RunOutcome {
+    /** Score on the search/optimization split. Optional because a
+     *  holdout-only evaluation only fills `holdoutScore`. */
+    searchScore?: number;
+    /** Score on the held-out split. Optional because a search-only run
+     *  only fills `searchScore`. At least one must be present. */
+    holdoutScore?: number;
+    /** Bag of any other metric the run produced — judge dimensions,
+     *  pass/fail counters, latency stats, etc. Numeric only — keeps
+     *  reporters honest. */
+    raw: Record<string, number>;
+    /** Per-judge / per-dim breakdown. Consumers writing ensemble
+     *  judgements populate this; substrate primitives like
+     *  `interRaterReliability` and `corpusInterRaterAgreement` accept
+     *  these records as input. Optional — single-judge or scalar-only
+     *  runs leave it unset. */
+    judgeScores?: JudgeScoresRecord;
+    /** Authenticity / realness verdict — did the run build the REAL thing on the
+     *  intended infra, or fake it (see `./authenticity`)? Optional: only domains
+     *  with an authenticity config populate it. Carried in the corpus so the
+     *  flywheel / off-policy learning can optimize for real completion, not gamed
+     *  pass-rate. `score` is 0-1; `gated` is the anti-Goodhart flag — a gated run
+     *  must not count as a real success regardless of `score`. */
+    realness?: {
+        score: number;
+        gated: boolean;
+        reason?: string;
+    };
+}
+/**
+ * Mandatory paper-grade fields for a single evaluation run. Optional
+ * fields are extension points; mandatory fields throw if missing.
+ *
+ * Hash discipline:
+ *   - `promptHash` is the sha256 of the EFFECTIVE prompt sent to the
+ *     model (after any steering bundle merge).
+ *   - `configHash` is the sha256 of the effective run config (model,
+ *     temperature, tools, judges, splits). The pair (promptHash,
+ *     configHash) uniquely identifies an experiment cell.
+ *
+ * Model snapshot discipline:
+ *   - `model` MUST encode a snapshot version. Bare aliases like
+ *     `claude-sonnet-4` or `gpt-4o` are banned — they remap silently.
+ *     Use `claude-sonnet-4-6@2025-04-15` or `gpt-4o-2024-11-20`.
+ */
+interface RunRecord {
+    /** UUID for the run. */
+    runId: string;
+    /** Logical experiment grouping (a treatment vs a baseline within
+     *  the same sweep should share `experimentId`). */
+    experimentId: string;
+    /** Stable identifier for the candidate (variant) being run. The
+     *  promotion gate compares two `candidateId`s on matched items. */
+    candidateId: string;
+    /** RNG seed for the run. Always recorded — silent re-seeding is
+     *  the most common cause of non-reproducible numbers. */
+    seed: number;
+    /** Model identifier WITH snapshot version. */
+    model: string;
+    /** sha256 of the effective prompt (post-steering). */
+    promptHash: string;
+    /** sha256 of the effective config. */
+    configHash: string;
+    /** Git SHA the harness was run from. */
+    commitSha: string;
+    /** End-to-end wall-clock duration in milliseconds. */
+    wallMs: number;
+    /** Time spent queued before execution started, if known. */
+    queueMs?: number;
+    /** Total USD cost. Mandatory — runs without a cost number are
+     *  unbounded by definition and must not be admitted into the gate.
+     *  `0` is retained as the compatibility sentinel for an uncaptured amount;
+     *  inspect `costProvenance` before treating it as observed. */
+    costUsd: number;
+    /** Observed, model-priced estimate, or genuinely uncaptured USD amount.
+     *  Optional only so existing serialized RunRecords remain valid. */
+    costProvenance?: RunCostProvenance;
+    /** Token usage breakdown. */
+    tokenUsage: RunTokenUsage;
+    /** Judge-side metadata, if a judge was used. */
+    judgeMetadata?: RunJudgeMetadata;
+    /** Per-split scores + raw bag. */
+    outcome: RunOutcome;
+    /** Canonical, cross-agent failure class drawn from the shared
+     *  `FAILURE_CLASSES` taxonomy. This is the aggregation key that makes
+     *  "which failure dominates across the whole fleet" answerable in ONE
+     *  vocabulary — every agent classifies against the same enum. Producers
+     *  set it via the substrate classifier; leave unset only when the failure
+     *  genuinely can't be classified. */
+    failureClass?: FailureClass;
+    /** Free-form domain-specific failure detail, scoped UNDER `failureClass`
+     *  (e.g. failureClass='tool_recovery_failure', failureMode='forge_build_unsatisfied').
+     *  The within-agent drill-down; `failureClass` is the cross-agent key. */
+    failureMode?: string;
+    /** Which split this run was drawn from. */
+    splitTag: RunSplitTag;
+    /**
+     * Stable scenario identifier the run was scored against. Optional for
+     * backwards compatibility, but **strongly recommended**: every primitive
+     * that pairs runs by scenario (preferences, paired stats, BT tournament)
+     * keys on this. The campaign artifact populates it canonically; legacy
+     * runs without it fall back to inference from `outcome.raw.scenario_id`
+     * or `experimentId`.
+     */
+    scenarioId?: string;
+    /**
+     * Canonical identity for the agent profile cell that produced this row:
+     * profile artifact hash plus optional harness/model/prompt/reporting
+     * dimensions. Use `agentProfile.cellId` to group persona sweeps and
+     * longitudinal reports by the complete source profile, not by a loose
+     * candidate label or opaque config hash.
+     */
+    agentProfile?: AgentProfileCell;
+}
+
+interface RunFilter {
+    scenarioId?: string;
+    variantId?: string;
+    status?: RunStatus;
+    since?: number;
+    until?: number;
+    tag?: {
+        key: string;
+        value: string;
+    };
+    parentRunId?: string;
+    projectId?: string;
+    chatId?: string;
+    layer?: RunLayer;
+}
+interface SpanFilter {
+    runId?: string;
+    parentSpanId?: string;
+    kind?: SpanKind;
+    name?: string;
+    toolName?: string;
+    judgeId?: string;
+    since?: number;
+    until?: number;
+}
+interface EventFilter {
+    runId?: string;
+    spanId?: string;
+    kind?: EventKind;
+    since?: number;
+    until?: number;
+}
+interface TraceStore {
+    appendRun(run: Run): Promise<void>;
+    updateRun(runId: string, patch: Partial<Run>): Promise<void>;
+    appendSpan(span: Span): Promise<void>;
+    updateSpan(spanId: string, patch: Partial<Span>): Promise<void>;
+    appendEvent(event: TraceEvent): Promise<void>;
+    appendArtifact(artifact: Artifact): Promise<void>;
+    appendBudgetEntry(entry: BudgetLedgerEntry): Promise<void>;
+    getRun(runId: string): Promise<Run | undefined>;
+    listRuns(filter?: RunFilter): Promise<Run[]>;
+    spans(filter?: SpanFilter): Promise<Span[]>;
+    events(filter?: EventFilter): Promise<TraceEvent[]>;
+    budget(runId: string): Promise<BudgetLedgerEntry[]>;
+    artifacts(runId: string): Promise<Artifact[]>;
+}
+
+/**
+ * Rollout minting — `tangle.rollout.v1` lines joined from the records the
+ * substrate ALREADY keeps. There is no separate rollout store: a rollout
+ * is the JOIN of a RunRecord (identity, provenance, cost, outcome) with
+ * its trace (spans share `runId`), projected into the canonical line.
+ *
+ * Composition, not duplication:
+ *   - identity/provenance     → `RunRecord` (candidateId, splitTag, agentProfile, hashes)
+ *   - step structure          → `buildTrajectory` over the shared TraceStore
+ *   - preference-pair export  → `feedbackTrajectoryToOptimizerRow` (feedback-trajectory.ts)
+ *   - PRM / reward-model      → `reward-model-export.ts`
+ *
+ * Anti-Goodhart invariant: a run whose `outcome.realness.gated` is true
+ * is never exported with a positive reward — the gate travels into the
+ * training data (`reward` forced to 0, `realness_gated: true`), so a
+ * fine-tune cannot learn from gamed successes.
+ *
+ * Records without spans become labeled GAP LINES (messages: [],
+ * provenance.gap) — present in the output AND surfaced in
+ * `missingTraces`; a capture gap is a finding, never a silent omission.
+ */
+
+/** Redactor applied to every exported string (secrets, PII). Identity by default. */
+type RolloutScrubber = (text: string) => string;
+interface MintRolloutOptions {
+    scrub?: RolloutScrubber;
+    /** Cap steps per line (longest runs first drop middle steps). Default: no cap. */
+    maxSteps?: number;
+    /** Role recorded on every minted line. Default 'agent' (a solo eval run). */
+    role?: RolloutRole;
+    /** Task suite label. Default: the record's `experimentId`. */
+    suite?: string;
+    /** Injected clock for deterministic output. */
+    now?: () => Date;
+}
+interface MintRolloutResult {
+    rows: RolloutLine[];
+    /** runIds that had a RunRecord but no spans — emitted as gap lines AND listed here. */
+    missingTraces: string[];
+}
+declare function rolloutReward(record: RunRecord): {
+    reward: number;
+    gated: boolean;
+};
+/**
+ * Join RunRecords with their traces into canonical rollout lines. Records
+ * without spans are emitted as labeled gap lines and reported in
+ * `missingTraces` — a capture gap is a finding, not a silent omission.
+ */
+declare function mintRolloutRows(records: RunRecord[], store: TraceStore, options?: MintRolloutOptions): Promise<MintRolloutResult>;
+
+/**
+ * Backfill reader over Claude Code project transcripts
+ * (~/.claude/projects/<cwd-slug>/<sessionId>.jsonl) → canonical
+ * chat-with-tools messages plus per-session token usage.
+ *
+ * Transcript lines consumed: type:"user" (string content or content blocks —
+ * text + tool_result) and type:"assistant" (content blocks — thinking, text,
+ * tool_use; message.usage carries tokens). Sidechain lines (isSidechain=true,
+ * subagent threads) are separate invocations and are excluded from the main
+ * transcript. Everything else (queue-operation, attachment, last-prompt…) is
+ * transport metadata, not conversation.
+ */
+
+declare const DEFAULT_CLAUDE_PROJECTS_DIR: string;
+/** Claude Code's project-directory slug for a working directory. */
+declare function claudeProjectSlug(cwd: string): string;
+interface ClaudeTranscriptRef {
+    sessionId: string;
+    path: string;
+}
+/** Transcript files recorded for sessions launched from `cwd`. */
+declare function findClaudeTranscripts(cwd: string, projectsDir?: string): Promise<ClaudeTranscriptRef[]>;
+interface ClaudeUsageTotals {
+    tokensIn: number;
+    tokensOut: number;
+    cacheRead: number;
+    cacheWrite: number;
+}
+interface ClaudeTranscript {
+    messages: ChatMessage[];
+    usage: ClaudeUsageTotals;
+    /** Timestamp of the first conversation line; null = empty transcript. */
+    startedAt: string | null;
+    endedAt: string | null;
+    model: string | null;
+}
+/** Parse one transcript jsonl into canonical messages + usage totals. */
+declare function readClaudeTranscript(path: string): Promise<ClaudeTranscript>;
+
+/**
+ * Read-only backfill reader over the opencode sqlite store
+ * (~/.local/share/opencode/opencode.db) → canonical chat-with-tools messages.
+ *
+ * Schema consumed (observed, 2026-07): `session` rows carry directory /
+ * parent_id / agent / model / cost / tokens_*; `message` rows carry a JSON
+ * `data` blob ({role, modelID, providerID, tokens, cost, finish}); `part`
+ * rows carry the actual content ({type: text|reasoning|tool|step-start|
+ * step-finish|snapshot…}). Tool parts hold {callID, state:{input, output,
+ * status}} — both the call and its result, which we split into an assistant
+ * tool_call plus a role:"tool" result message.
+ *
+ * The store is mutable and can be corrupt (a `.corrupt-bak` sibling ships
+ * next to it in the wild), so `openOpencodeDb` returns null instead of
+ * throwing — callers record a gap line, never crash the backfill.
+ */
+
+declare const DEFAULT_OPENCODE_DB: string;
+interface OpencodeSessionRow {
+    id: string;
+    parentId: string | null;
+    directory: string;
+    agent: string | null;
+    /** Raw session.model JSON: {id, providerID, variant} where present. */
+    model: {
+        id?: string;
+        providerID?: string;
+    } | null;
+    costUsd: number;
+    tokensInput: number;
+    tokensOutput: number;
+    tokensReasoning: number;
+    tokensCacheRead: number;
+    tokensCacheWrite: number;
+    timeCreated: number;
+    timeUpdated: number;
+}
+/** Open the store read-only; null = unavailable/corrupt (caller records a gap). */
+declare function openOpencodeDb(path?: string): Promise<DatabaseSync | null>;
+/** Sessions whose cwd is `directory` (the worker-clone join key). */
+declare function findOpencodeSessionsByDirectory(db: DatabaseSync, directory: string): OpencodeSessionRow[];
+declare function findOpencodeSessionById(db: DatabaseSync, sessionId: string): OpencodeSessionRow | null;
+/**
+ * Convert one session's message+part rows into canonical messages.
+ * An opencode assistant message row spans several model steps; each step's
+ * parts (reasoning → text → tool …) become one assistant message followed by
+ * the role:"tool" results of its calls, preserving order.
+ */
+declare function readOpencodeSessionMessages(db: DatabaseSync, sessionId: string): ChatMessage[];
+
+/**
+ * Deterministic scrubbing pass over rollout-ledger lines before public release.
+ *
+ * Every rule is a pure regex rewrite applied to every string value in a line
+ * (messages, artifacts, run ids, tool arguments — everywhere), so the scrubbed
+ * line is still a valid `tangle.rollout.v1` line. Rules are idempotent:
+ * scrub(scrub(x)) === scrub(x), and a second pass counts zero hits — that is
+ * the property the release pipeline relies on to prove nothing half-scrubbed
+ * ships. Rule order matters: whole `KEY=value` env pairs are redacted before
+ * the bare-key rule so one secret is never counted twice.
+ */
+
+interface ScrubRule {
+    name: string;
+    pattern: RegExp;
+    /** Rewrite for one match; `g1` is the first capture group when present. */
+    rewrite: (match: string, g1?: string) => string;
+}
+declare const SCRUB_RULES: readonly ScrubRule[];
+/** Rule name → number of matches rewritten. Always carries every rule (0 is data). */
+type ScrubCounts = Record<string, number>;
+declare function emptyScrubCounts(): ScrubCounts;
+declare function addScrubCounts(into: ScrubCounts, from: ScrubCounts): ScrubCounts;
+declare function scrubText(text: string, counts: ScrubCounts): string;
+/** Scrub every string value in a line; structure and key order are preserved. */
+declare function scrubRolloutLine(line: RolloutLine, counts: ScrubCounts): RolloutLine;
+declare function scrubLines(lines: RolloutLine[]): {
+    lines: RolloutLine[];
+    counts: ScrubCounts;
+};
+/**
+ * A `RolloutScrubber` (text → text) applying the full rule set — the
+ * default hook to pass to `mintRolloutRows({ scrub })` so lines are
+ * scrubbed at mint time, before they ever reach a ledger file. Release
+ * builds re-run `scrubLines` regardless (idempotent), so double-scrubbing
+ * is safe and counted as zero.
+ */
+declare function defaultRolloutScrubber(text: string): string;
+
+/**
+ * HuggingFace dataset-card (README.md) generation for a rollout-ledger release.
+ *
+ * The card is a pure function of the SCRUBBED lines plus the release options —
+ * no timestamps, no environment reads — so rebuilding from the same ledger
+ * yields byte-identical output. It documents the schema, provenance (run ids,
+ * generations, the official judge), per-role reward semantics including the
+ * inherited/contribution caveat, and a role × reward counts table.
+ */
+
+declare const RELEASE_FORMATS: readonly ["sft", "verifiers", "rft", "raw"];
+type ReleaseFormat = (typeof RELEASE_FORMATS)[number];
+/** Format → data file path inside the dataset dir (train split only). */
+declare const FORMAT_FILES: Record<ReleaseFormat, string>;
+interface DatasetCardInputs {
+    /** Scrubbed, release-filtered lines (what actually ships). */
+    lines: RolloutLine[];
+    formats: ReleaseFormat[];
+    includeProposers: boolean;
+    /** Source ledger basenames, for provenance. */
+    sourceFiles: string[];
+    scrubTotals: ScrubCounts;
+    excluded: {
+        proposers: number;
+        nonTrain: number;
+    };
+    formatCounts: Partial<Record<ReleaseFormat, number>>;
+}
+declare function buildDatasetCard(inputs: DatasetCardInputs): string;
+
+/**
+ * One-command HuggingFace dataset release from rollout ledgers:
+ *
+ *   agent-eval rollout-release <ledger.jsonl...> --out <dir> \
+ *     [--formats sft,verifiers,rft,raw] [--include-proposers] [--push <org/name>]
+ *
+ * Pipeline per input ledger: read + validate → fail-closed filters
+ * (trainable split only; proposer sessions dropped unless
+ * --include-proposers, they contain improvement-loop harness source) →
+ * deterministic scrub → export the requested formats + scrub-report.json +
+ * auto-generated README.md card. Deterministic: same inputs and flags →
+ * byte-identical output dir.
+ *
+ * --push uploads the built dir with `huggingface-cli upload` only when the
+ * CLI exists on PATH and HF_TOKEN is present in the env; the token is
+ * never printed. Everything else runs fully offline.
+ */
+
+interface BuildOptions {
+    out: string;
+    formats: ReleaseFormat[];
+    includeProposers: boolean;
+}
+interface ScrubReport {
+    /** Input ledger path → rule → rewrite count (only shipped lines are scrubbed). */
+    files: Record<string, ScrubCounts>;
+    totals: ScrubCounts;
+    excluded: {
+        proposers: number;
+        nonTrain: number;
+    };
+}
+interface BuildSummary {
+    inputs: string[];
+    read: number;
+    kept: number;
+    scrub: ScrubReport;
+    formatCounts: Partial<Record<ReleaseFormat, number>>;
+    files: string[];
+}
+declare function buildHfDataset(inputs: string[], options: BuildOptions): Promise<BuildSummary>;
+declare function planPushCommand(repo: string, outDir: string): string[];
+declare function pushDataset(repo: string, outDir: string): void;
+interface RolloutReleaseCliArgs extends BuildOptions {
+    inputs: string[];
+    push: string | null;
+}
+declare const ROLLOUT_RELEASE_USAGE = "usage: agent-eval rollout-release <ledger.jsonl...> --out <dir> [--formats sft,verifiers,rft,raw] [--include-proposers] [--push <org/name>]";
+declare function parseRolloutReleaseArgs(argv: string[]): RolloutReleaseCliArgs;
+/** CLI driver for `agent-eval rollout-release`. Returns the process exit code. */
+declare function runRolloutReleaseCli(argv: string[]): Promise<number>;
+
+export { type BuildOptions, type BuildSummary, CHAT_ROLES, type ChatMessage, type ChatRole, type ChatToolCall, type ClaudeTranscript, type ClaudeTranscriptRef, type ClaudeUsageTotals, DEFAULT_CLAUDE_PROJECTS_DIR, DEFAULT_OPENCODE_DB, type DatasetCardInputs, FORMAT_FILES, type MintRolloutOptions, type MintRolloutResult, type OpencodeSessionRow, RELEASE_FORMATS, ROLLOUT_CAPTURES, ROLLOUT_FORMAT, ROLLOUT_RELEASE_USAGE, ROLLOUT_ROLES, ROLLOUT_SCHEMA, ROLLOUT_SPLITS, type ReleaseFormat, type RewardRow, type RftItem, type RolloutArtifacts, type RolloutCapture, type RolloutCostBlock, type RolloutLine, type RolloutOutcome, type RolloutPolicy, type RolloutProvenance, type RolloutReleaseCliArgs, type RolloutRole, type RolloutScrubber, type RolloutSplit, type RolloutStep, type RolloutTask, SCRUB_RULES, type ScrubCounts, type ScrubReport, type ScrubRule, type SftExportOptions, type SftRow, TRAINABLE_SPLITS, type ToolDef, type VerifiersRolloutOutput, type VerifiersTokenUsage, addScrubCounts, appendRolloutLines, assertRolloutLine, buildDatasetCard, buildHfDataset, claudeProjectSlug, defaultRolloutScrubber, emptyScrubCounts, findClaudeTranscripts, findOpencodeSessionById, findOpencodeSessionsByDirectory, isRolloutLine, isTrainableSplit, mintRolloutRows, openOpencodeDb, parseRolloutReleaseArgs, planPushCommand, pushDataset, readClaudeTranscript, readOpencodeSessionMessages, readRolloutLedger, rolloutReward, runRolloutReleaseCli, scrubLines, scrubRolloutLine, scrubText, toJsonl, toRewardRows, toRftItem, toRftItems, toSftRows, toVerifiersRolloutOutput, toVerifiersRolloutOutputs, validateRolloutLine, writeRolloutLedger };