@markstanden/opencode-test-container 0.0.0-stage → 0.1.1

package/dist/evaluators/docs/analysis/pairing.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Pairing for the analysis framework.
+ *
+ * Pairs come from the declared pair identities in the manifest schedule and
+ * are matched to observations by declared slot identity — never by array
+ * adjacency. A complete pair holds an assessed observation for every declared
+ * arm; an incomplete pair yields its assessed sides as unmatched observations,
+ * and observations outside any declared pair stay visible as unpaired.
+ */
+import type { TrialScored } from "./accounting.ts";
+import type { Interval } from "./bounds.ts";
+import type { SlotSpec } from "./model.ts";
+/** One declared pair with its observations by arm. */
+export interface PairView {
+    /** The declared pair id. */
+    pairId: string;
+    /** The pair's scenario. */
+    scenario: string;
+    /** The pair's repetition. */
+    repetition: number;
+    /** Observations keyed by declared arm; undefined when absent. */
+    arms: Record<string, TrialScored | undefined>;
+    /** Whether every declared arm has a fully assessed observation. */
+    complete: boolean;
+    /** Declared arms without a fully assessed observation. */
+    missingArms: string[];
+}
+/** The pairing result for one measured population. */
+export interface PairingResult {
+    /** Declared pairs in declared order. */
+    pairs: PairView[];
+    /** Assessed observations in incomplete pairs. */
+    unmatched: TrialScored[];
+    /** Observations outside any declared pair. */
+    unpaired: TrialScored[];
+    /** Evidence problems found while pairing (e.g. duplicate observations). */
+    problems: string[];
+}
+/** A candidate-minus-control comparison over complete pairs. */
+export interface PairedComparison {
+    /** Complete pairs carrying both compared arms. */
+    completePairs: number;
+    /** Pairs where the control scored higher. */
+    controlBetter: number;
+    /** Pairs where the candidate scored higher. */
+    candidateBetter: number;
+    /** Pairs with equal scores. */
+    ties: number;
+    /** Summed candidate-minus-control delta over complete pairs. */
+    delta: number;
+    /** Closed delta interval from the per-arm bounds. */
+    interval: Interval;
+}
+/**
+ * Pair a measured population by declared identity.
+ * @param scored The scored trials.
+ * @param slots The declared schedule slots.
+ * @returns Complete pairs, unmatched observations, unpaired evidence and problems.
+ */
+export declare function pairObservations(scored: readonly TrialScored[], slots: readonly SlotSpec[]): PairingResult;
+/**
+ * Compare a candidate arm against a control arm over complete pairs.
+ * @param pairs The paired population.
+ * @param controlArm The control arm id.
+ * @param candidateArm The candidate arm id.
+ * @returns Pair counts, the summed delta and its interval.
+ */
+export declare function pairedComparison(pairs: readonly PairView[], controlArm: string, candidateArm: string): PairedComparison;

package/dist/evaluators/docs/analysis/pairing.js

@@ -0,0 +1,95 @@
+/**
+ * Build one pair view from its declared slots.
+ * @param pairId The declared pair id.
+ * @param group The pair's slots, in declared order.
+ * @param bySlot First-attempt observations keyed by slot number.
+ * @returns The pair view.
+ */
+function pairViewOf(pairId, group, bySlot) {
+    const arms = {};
+    const missingArms = [];
+    for (const slot of group) {
+        const row = bySlot.get(slot.slot);
+        arms[slot.arm] = row;
+        if (row === undefined || !row.utility.assessed) {
+            missingArms.push(slot.arm);
+        }
+    }
+    return {
+        pairId,
+        scenario: group[0]?.scenario ?? "",
+        repetition: group[0]?.repetition ?? 0,
+        arms,
+        complete: missingArms.length === 0,
+        missingArms,
+    };
+}
+/**
+ * Pair a measured population by declared identity.
+ * @param scored The scored trials.
+ * @param slots The declared schedule slots.
+ * @returns Complete pairs, unmatched observations, unpaired evidence and problems.
+ */
+export function pairObservations(scored, slots) {
+    const observed = scored.filter((row) => row.disposition === "first-attempt");
+    const problems = [];
+    const bySlot = new Map();
+    for (const row of observed) {
+        if (row.identity.slot === null)
+            continue;
+        if (bySlot.has(row.identity.slot)) {
+            problems.push(`duplicate first-attempt observation for slot ${String(row.identity.slot)}`);
+        }
+        else {
+            bySlot.set(row.identity.slot, row);
+        }
+    }
+    const pairIds = [
+        ...new Set(slots.flatMap((slot) => slot.pairId === undefined ? [] : [slot.pairId])),
+    ];
+    const pairs = pairIds.map((pairId) => pairViewOf(pairId, slots.filter((slot) => slot.pairId === pairId), bySlot));
+    const unmatched = pairs
+        .filter((pair) => !pair.complete)
+        .flatMap((pair) => Object.values(pair.arms).filter((row) => row !== undefined && row.utility.assessed));
+    const pairedSlots = new Set(slots.flatMap((slot) => (slot.pairId === undefined ? [] : [slot.slot])));
+    const unpaired = observed.filter((row) => row.identity.slot === null || !pairedSlots.has(row.identity.slot));
+    return { pairs, unmatched, unpaired, problems };
+}
+/**
+ * Compare a candidate arm against a control arm over complete pairs.
+ * @param pairs The paired population.
+ * @param controlArm The control arm id.
+ * @param candidateArm The candidate arm id.
+ * @returns Pair counts, the summed delta and its interval.
+ */
+export function pairedComparison(pairs, controlArm, candidateArm) {
+    const comparison = {
+        completePairs: 0,
+        controlBetter: 0,
+        candidateBetter: 0,
+        ties: 0,
+        delta: 0,
+        interval: { low: 0, high: 0 },
+    };
+    for (const pair of pairs) {
+        if (!pair.complete)
+            continue;
+        const control = pair.arms[controlArm];
+        const candidate = pair.arms[candidateArm];
+        if (control === undefined || candidate === undefined)
+            continue;
+        comparison.completePairs += 1;
+        const delta = candidate.utility.score - control.utility.score;
+        comparison.delta += delta;
+        comparison.interval.low += candidate.utility.low - control.utility.high;
+        comparison.interval.high +=
+            candidate.utility.high - control.utility.low;
+        if (delta > 0)
+            comparison.candidateBetter += 1;
+        else if (delta < 0)
+            comparison.controlBetter += 1;
+        else
+            comparison.ties += 1;
+    }
+    return comparison;
+}

package/dist/evaluators/docs/analysis/records.d.ts

@@ -0,0 +1,229 @@
+/**
+ * Evidence loading for the trial analysis framework.
+ *
+ * Every record is read through an injected {@link EvidenceReader} and lands
+ * in one of four explicit states: available, missing, malformed or
+ * unsupported-loaded. Absence is never conflated with emptiness — a missing
+ * snapshot is not an empty lake, and an absent judgement is not "none". The
+ * ac-v1 adapter reads the layout the 2026-11-10 campaigns recorded (per-arm
+ * result roots holding run metadata, gate and primary-check records, judge
+ * attempt directories and trial artefacts). A legacy record without the
+ * fields it needs stays limited or unknown; it is never silently upgraded.
+ *
+ * Nothing here touches the filesystem or calls a model: the CLI supplies the
+ * reader, and tests supply an in-memory one.
+ */
+import { type DocumentationOutcome } from "../scoring.ts";
+import type { LakeDocument } from "../campaign.ts";
+import type { SelectionPolicy } from "./model.ts";
+import { type RecoveryPolicy } from "./recovery.ts";
+import { type JournalVerification } from "./journal.ts";
+/** The explicit state of one loaded record. */
+export type Loaded<T> = {
+    state: "available";
+    value: T;
+    source: string;
+} | {
+    state: "missing";
+    source: string;
+} | {
+    state: "not-a-directory";
+    source: string;
+} | {
+    state: "malformed";
+    source: string;
+    problems: string[];
+} | {
+    state: "unsupported-loaded";
+    source: string;
+    reason: string;
+};
+/** The filesystem boundary the loader reads through. */
+export interface EvidenceReader {
+    /** Read one UTF-8 text file; a missing file is a state, not an error. */
+    readTextFile(path: string): Loaded<string>;
+    /** List one directory's entry names, sorted; missing is a state. */
+    readDirectoryEntries(path: string): Loaded<string[]>;
+}
+/** Whether one trial directory proves a recorded run. */
+export type AttemptState = "recorded" | "reservation-only" | "unknown" | "absent";
+/** The gate record as the harness wrote it. */
+export interface GateRecord {
+    /** The recorded gate status string (e.g. `pass`, `blocked`). */
+    status: string;
+    /** The recorded reason, or null. */
+    reason: string | null;
+}
+/** One preservation observation as a judgement recorded it. */
+export interface PreservationObservation {
+    /** The recorded assessment state (e.g. `retained`, `lost`). */
+    state: string;
+    /** The decisive excerpt the judgement quoted. */
+    quote: string;
+}
+/** Per-fact preservation observations keyed by inventory fact id. */
+export type PreservationByFact = Record<string, Loaded<PreservationObservation>>;
+/** One token field's subtotal and independent step coverage. */
+export interface TokenFieldUsage {
+    /** Sum over steps that reported this field; meaningful only with coverage. */
+    total: number;
+    /** Observed steps reporting a numeric value, including explicit zero. */
+    measuredSteps: number;
+    /** Observed steps not reporting this field. */
+    unknownSteps: number;
+}
+/** Token accounting summed from a recorded session trail, per field. */
+export interface TokenUsage {
+    /** Observed step_finish events, with or without token telemetry. */
+    steps: number;
+    /** Input-token subtotal and coverage. */
+    input: TokenFieldUsage;
+    /** Output-token subtotal and coverage. */
+    output: TokenFieldUsage;
+    /** Reasoning-token subtotal and coverage. */
+    reasoning: TokenFieldUsage;
+    /** Cache-read-token subtotal and coverage. */
+    cacheRead: TokenFieldUsage;
+}
+/** One recorded campaign-attempt ledger event. */
+export interface LaunchRecord {
+    /** Event timestamp, or null when the event carried none. */
+    at: string | null;
+    /** The recorded event name (e.g. `launch`, `slot-complete`). */
+    event: string;
+    /** The trial stamp, or null when the event carried none. */
+    stamp: string | null;
+}
+/** The assessment status of one trial's judgement layer. */
+export type AssessmentStatus = "judged" | "invalid-verdict" | "judge-error" | "unassessed" | "not-run" | "unknown";
+/** One trial's retrieval-application observation as the judge recorded it. */
+export interface RetrievalObservation {
+    /** The recorded state (e.g. applied, read-not-applied). */
+    state: string;
+    /** The decisive excerpt connecting the retrieved detail to the work. */
+    quote: string;
+}
+/** One trial's resolved assessment. */
+export interface TrialAssessment {
+    /** The status of the selected judgement layer. */
+    status: AssessmentStatus;
+    /** Which layer the recorded selection policy chose. */
+    layer: "judge" | "deterministic-resolution" | "none";
+    /** The documentation outcome, undefined when no usable one was recorded. */
+    outcome: DocumentationOutcome | undefined;
+    /** The judgement's stated reason, when present. */
+    reason: string | undefined;
+    /** The judgement's evidence quotes, when present. */
+    evidence: string[];
+    /** Per-fact preservation observations for the scenario's inventory. */
+    preservation: Loaded<PreservationByFact>;
+    /** The judge's retrieval-application finding, when one was recorded. */
+    retrieval: Loaded<RetrievalObservation>;
+    /** The published selection's attempt, when a selection record exists. */
+    selectedAttemptId?: string;
+    /** Every problem found while loading or validating the assessment. */
+    problems: string[];
+}
+/** The artefacts a static screen reads for one trial. */
+export interface TrialArtefacts {
+    /** The trial's final answer text. */
+    finalAnswer: Loaded<string>;
+    /** The recorded writes report text. */
+    writesText: Loaded<string>;
+    /** The recorded docs-operation journal lines. */
+    docsOperations: Loaded<string[]>;
+    /** The trial's prompt text. */
+    promptText: Loaded<string>;
+}
+/** One trial's loaded evidence. */
+export interface TrialRecord {
+    /** The declared identity this record was loaded for. */
+    identity: {
+        resultRoot: string;
+        stamp: string;
+        scenario: string;
+    };
+    /** What the trial directory proves about the run. */
+    attempt: {
+        /** The attempt state derived from directory contents. */
+        state: AttemptState;
+        /** Usability: true, false, or unknown without a run record. */
+        usable: boolean | "unknown";
+        /** Why usability holds or fails, for visible limitations. */
+        problems: string[];
+    };
+    /** The run metadata record, for policy and cost fields. */
+    run: Loaded<Record<string, unknown>>;
+    /** The recorded deterministic gate verdict. */
+    gate: Loaded<GateRecord>;
+    /** The independent primary-task result. */
+    primary: {
+        status: "pass" | "fail" | "unknown";
+        problems: string[];
+    };
+    /** The resolved judgement layer. */
+    assessment: TrialAssessment;
+    /** The judge model identity, when recorded. */
+    judgeModel: Loaded<string>;
+    /** Token accounting from the session trail. */
+    tokens: Loaded<TokenUsage>;
+    /** The trial artefacts a static screen reads. */
+    artefacts: TrialArtefacts;
+    /**
+     * The verified operation-journal evidence. Absent only on a legacy record
+     * built without one; the loader always sets it, and an unverified journal
+     * earns no credit.
+     */
+    journal?: JournalVerification;
+    /** Why the captured answer is not bound to the run, or null. */
+    answerProblem?: string | null;
+}
+/** Options for loading one trial under the ac-v1 layout. */
+export interface TrialRecordOptions {
+    /** The arm's result root, relative to the evidence root. */
+    resultRoot: string;
+    /** The selected judge attempt directory; defaults to `attempt-001`. */
+    attemptId?: string;
+    /** The trial stamp to load. */
+    stamp: string;
+    /** The scenario the slot declared. */
+    scenario: string;
+    /** The scheduled slot number, when the caller knows it. */
+    slot?: number;
+    /** The pair id, when the caller knows it. */
+    pairId?: string;
+    /** The arm id, when the caller knows it. */
+    arm?: string;
+    /** The scenario's inventory fact ids, for per-fact preservation loading. */
+    inventoryFactIds: string[];
+    /** The manifest's recorded judgement-layer selection policy. */
+    selectionPolicy: SelectionPolicy;
+    /** The registered recovery policy, when the manifest declares one. */
+    recoveryPolicy?: RecoveryPolicy | undefined;
+    /** Whether legacy evidence without an answer hash is accepted. */
+    allowUnboundAnswer?: boolean;
+}
+/**
+ * Read the Markdown documents under one directory, recursively and sorted.
+ * A missing directory stays missing; an existing directory with no Markdown
+ * documents is available and empty — absence is never conflated with
+ * emptiness.
+ * @param reader The evidence reader.
+ * @param dir The directory, relative to the evidence root.
+ * @returns The documents sorted by path, or the explicit state.
+ */
+export declare function readMarkdownTree(reader: EvidenceReader, dir: string): Loaded<LakeDocument[]>;
+/**
+ * Load one trial's evidence under the ac-v1 layout.
+ * @param reader The evidence reader rooted at the evidence root.
+ * @param options Which trial to load and how to select judgement layers.
+ * @returns The trial record.
+ */
+export declare function loadTrialRecord(reader: EvidenceReader, options: TrialRecordOptions): TrialRecord;
+/**
+ * Load the campaign attempt ledger (one JSON event per line).
+ * @param reader The evidence reader.
+ * @param path The ledger path, relative to the evidence root.
+ * @returns The events, or the explicit state.
+ */
+export declare function loadLaunchLedger(reader: EvidenceReader, path: string): Loaded<LaunchRecord[]>;