@markstanden/opencode-test-container 0.0.0-stage → 0.1.0

package/dist/evaluators/docs/analysis/closure.js

@@ -0,0 +1,251 @@
+/**
+ * Campaign accounting and closure, derived from durable evidence.
+ *
+ * The report never trusts a hard-coded checkpoint. Counts come from the
+ * frozen schedule, the launch claims, the recorded trial directories and each
+ * attempt's own token/duration record. A stopped partial campaign closes with
+ * the real scope it ran, and a closed campaign cannot be reopened by either
+ * the runner or the recovery path.
+ */
+/** An empty token accumulator. */
+function emptyTokens() {
+    return { input: null, output: null, reasoning: null, cacheRead: null };
+}
+/**
+ * Add one recorded field. Once an observation is unknown the total stays
+ * unknown: a later known value never restarts the subtotal at zero.
+ */
+function addField(account, field, value) {
+    if (value === null) {
+        account.unknown.set(field, true);
+    }
+    else {
+        const total = account[field];
+        account[field] =
+            total === null || account.unknown.get(field) === true
+                ? value
+                : total + value;
+    }
+    if (account.unknown.get(field) === true)
+        account[field] = null;
+}
+/** Add one attempt's tokens into an accumulator. */
+function addTokens(account, tokens) {
+    const fields = ["input", "output", "reasoning", "cacheRead"];
+    if (tokens === null) {
+        for (const field of fields)
+            account.unknownTokens.add(field);
+    }
+    else {
+        for (const field of fields) {
+            const value = tokens[field];
+            if (value === null) {
+                account.unknownTokens.add(field);
+            }
+            else {
+                account.tokens[field] = (account.tokens[field] ?? 0) + value;
+            }
+        }
+    }
+    for (const field of account.unknownTokens)
+        account.tokens[field] = null;
+}
+/** Token coverage counters. */
+function emptyCoverage() {
+    return { input: 0, output: 0, reasoning: 0, cacheRead: 0 };
+}
+/**
+ * Charge a claimed-but-unspawned attempt explicitly. A reservation consumes
+ * budget and appears as an unresolved attempt, never as a launch.
+ * @param claims The durable claims.
+ * @returns The count of claims that never spawned.
+ */
+export function unresolvedReservations(claims) {
+    return (claims ?? []).filter((claim) => claim.state === "reserved");
+}
+/** The numeric ordinal of an `attempt-NNN` id, or 0 when malformed. */
+function attemptOrdinal(attemptId) {
+    const match = /^attempt-(\d{3})$/.exec(attemptId);
+    return match === null ? 0 : Number(match[1]);
+}
+/**
+ * Derive the campaign accounts from durable observations.
+ * @param input The schedule, slot observations, claims and ceilings.
+ * @returns The closure report.
+ */
+export function deriveClosure(input) {
+    const trial = emptyAccount();
+    const judge = {
+        ...emptyAccount(),
+        usableJudgements: 0,
+        unusableJudgements: 0,
+    };
+    const observed = input.observations.filter(isObserved);
+    const judgeAttempts = observed.flatMap((slot) => slot.judgeAttempts);
+    for (const slot of observed) {
+        addTrial(trial, slot);
+        for (const attempt of slot.judgeAttempts)
+            addJudge(judge, attempt);
+    }
+    const problems = [
+        ...budgetProblems(input, trial.attempts, judge.attempts),
+        ...observationProblems(observed),
+    ];
+    const recordedTrials = observed.filter((slot) => slot.recorded).length;
+    problems.push(...closureProblems(input.closed, observed.length, recordedTrials));
+    return {
+        state: input.closed ? "closed" : "open",
+        scheduledSlots: input.scheduledSlots,
+        observedSlots: observed.length,
+        unrunSlots: unrunOf(input.scheduledSlots, observed),
+        trialLaunches: trial.attempts,
+        recordedTrials,
+        trial,
+        judge: {
+            ...judge,
+            initialAttempts: judgeAttempts.filter((attempt) => attemptOrdinal(attempt.attemptId) === 1).length,
+            recoveryAttempts: judgeAttempts.filter((attempt) => attemptOrdinal(attempt.attemptId) > 1).length,
+        },
+        stoppedAt: [...observed].at(-1)?.stamp ?? null,
+        problems,
+    };
+}
+/** An empty model accumulator. */
+function emptyAccount() {
+    return {
+        attempts: 0,
+        inferred: 0,
+        steps: null,
+        tokens: emptyTokens(),
+        tokenCoverage: emptyCoverage(),
+        durationMs: null,
+        unknown: new Map(),
+        unknownTokens: new Set(),
+    };
+}
+/** Whether one slot has any evidence at all. */
+function isObserved(slot) {
+    return slot.launched || slot.recorded || slot.judgeAttempts.length > 0;
+}
+/** Add one slot's trial evidence. */
+function addTrial(account, slot) {
+    if (!slot.launched)
+        return;
+    account.attempts += 1;
+    if (slot.trialSteps !== null && slot.trialSteps > 0)
+        account.inferred += 1;
+    addField(account, "steps", slot.trialSteps);
+    addTokens(account, slot.trialTokens);
+    addField(account, "durationMs", slot.trialDurationMs);
+    addCoverage(account, slot.trialTokens);
+}
+/** Add one judge attempt's evidence. */
+function addJudge(account, attempt) {
+    if (attempt.spawned)
+        account.attempts += 1;
+    if (attempt.reachedInference)
+        account.inferred += 1;
+    addField(account, "steps", attempt.steps);
+    addTokens(account, attempt.tokens);
+    addField(account, "durationMs", attempt.durationMs);
+    addCoverage(account, attempt.tokens);
+    if (attempt.status === "judged")
+        account.usableJudgements += 1;
+    else
+        account.unusableJudgements += 1;
+}
+/** Count which token fields an observation reported. */
+function addCoverage(account, tokens) {
+    if (tokens === null)
+        return;
+    for (const field of [
+        "input",
+        "output",
+        "reasoning",
+        "cacheRead",
+    ]) {
+        if (tokens[field] !== null)
+            account.tokenCoverage[field] += 1;
+    }
+}
+/** The slot numbers with no evidence. */
+function unrunOf(scheduledSlots, observed) {
+    const observedNumbers = new Set(observed.map((slot) => slot.slot));
+    return Array.from({ length: scheduledSlots }, (_, index) => index + 1).filter((number) => !observedNumbers.has(number));
+}
+/** Ceiling and reservation disagreements. */
+function budgetProblems(input, trialLaunches, judgeAttempts) {
+    const problems = [];
+    if (trialLaunches > input.ceilings.trial) {
+        problems.push(`trial launches ${String(trialLaunches)} exceed the ceiling ${String(input.ceilings.trial)}`);
+    }
+    if (judgeAttempts > input.ceilings.judge) {
+        problems.push(`judge attempts ${String(judgeAttempts)} exceed the ceiling ${String(input.ceilings.judge)}`);
+    }
+    const unresolvedTrials = unresolvedReservations(input.trialClaims);
+    if (unresolvedTrials.length > 0) {
+        problems.push(`${String(unresolvedTrials.length)} durable trial reservation(s) never spawned and stay unresolved`);
+    }
+    const unresolvedJudges = unresolvedReservations(input.judgeClaims);
+    if (unresolvedJudges.length > 0) {
+        problems.push(`${String(unresolvedJudges.length)} durable judge reservation(s) never spawned and stay unresolved`);
+    }
+    return problems;
+}
+/** Per-slot disagreements the accounts expose. */
+function observationProblems(observed) {
+    const problems = [];
+    for (const slot of observed) {
+        if (slot.launched && !slot.recorded) {
+            problems.push(`${slot.stamp}: launched without a recorded result`);
+        }
+        for (const attempt of slot.judgeAttempts) {
+            if (attempt.spawned && attempt.steps === null) {
+                problems.push(`${slot.stamp}/${attempt.attemptId}: spawned without a readable trail`);
+            }
+        }
+    }
+    return problems;
+}
+/** A closed campaign with an observed but unrecorded slot. */
+function closureProblems(closed, observed, recorded) {
+    return closed && recorded < observed
+        ? [
+            `campaign is closed with ${String(observed - recorded)} observed slot(s) lacking a recorded result`,
+        ]
+        : [];
+}
+/**
+ * Decide whether a campaign may close now, and with what scope.
+ *
+ * A campaign with no evidence is refused rather than closed empty. A stopped
+ * partial campaign closes with the slots that actually ran; the schedule's
+ * unrun slots are preserved as scope, not invented results.
+ * @param report The derived closure report.
+ * @returns The close event, or every reason closure is refused.
+ */
+export function closeCampaign(report) {
+    if (report.state === "closed") {
+        return {
+            ok: false,
+            problems: ["campaign is already closed; reopening is refused"],
+        };
+    }
+    if (report.observedSlots === 0) {
+        return {
+            ok: false,
+            problems: ["no campaign evidence exists to close"],
+        };
+    }
+    if (report.problems.length > 0) {
+        return { ok: false, problems: report.problems };
+    }
+    return {
+        ok: true,
+        event: {
+            event: "campaign-closed",
+            stoppedAt: report.stoppedAt,
+            scope: report.observedSlots,
+        },
+    };
+}

package/dist/evaluators/docs/analysis/decision.d.ts

@@ -0,0 +1,59 @@
+/**
+ * Decision evaluation for the analysis framework.
+ *
+ * The frozen selection rule is evaluated as three separate limbs — assessment
+ * coverage, the utility threshold and the non-compensable guardrails — and one
+ * overall outcome. Missing rules report "not defined" instead of inventing
+ * defaults; an unknown limb can prevent qualification without proving that
+ * the candidate failed the performance threshold.
+ */
+import type { AccountingCounts } from "./accounting.ts";
+import { type Interval } from "./bounds.ts";
+import type { GuardrailArmSummary } from "./guardrails.ts";
+import type { SelectionSpec } from "./model.ts";
+/** The coverage limb. */
+export interface CoverageLimb {
+    /** Whether the required coverage is complete. */
+    status: "pass" | "fail" | "not-applicable" | "not-defined";
+    /** The observed documented/planned coverage, rendered for reports. */
+    detail: string;
+}
+/** The utility threshold limb. */
+export interface ThresholdLimb {
+    /** Whether the candidate's lead meets the threshold. */
+    status: "pass" | "fail" | "inconclusive" | "not-defined";
+    /** Candidate minus control over the assessed populations, or null. */
+    lead: number | null;
+    /** The full-population candidate-minus-control interval, when available. */
+    interval: Interval | null;
+    /** The threshold comparison, rendered for reports. */
+    detail: string;
+}
+/** The guardrail limb for the candidate arm. */
+export interface GuardrailLimb {
+    /** Whether a non-compensable breach is established or unresolved. */
+    status: "clean" | "breached" | "unresolved" | "not-defined";
+    /** The breached or unresolved guardrail ids, rendered for reports. */
+    detail: string;
+}
+/** The decision result. */
+export interface DecisionResult {
+    /** The overall outcome under the declared rule. */
+    overall: "qualified" | "not-qualified" | "inconclusive" | "not-defined";
+    /** The coverage limb. */
+    coverage: CoverageLimb;
+    /** The threshold limb. */
+    threshold: ThresholdLimb;
+    /** The guardrail limb. */
+    guardrails: GuardrailLimb;
+    /** Human-readable reasons for the non-passing limbs. */
+    reasons: string[];
+}
+/**
+ * Evaluate the declared decision rule for a candidate against its control.
+ * @param selection The frozen selection rule, or undefined when none was frozen.
+ * @param arms The accounting counts keyed by arm id.
+ * @param guardrails The candidate arm's guardrail summaries.
+ * @returns The three limbs and the overall outcome.
+ */
+export declare function evaluateDecision(selection: SelectionSpec | undefined, arms: Readonly<Record<string, AccountingCounts>>, guardrails: readonly GuardrailArmSummary[]): DecisionResult;

package/dist/evaluators/docs/analysis/decision.js

@@ -0,0 +1,161 @@
+import { differenceInterval, intervalFromCounts, } from "./bounds.js";
+/**
+ * Build the result for a campaign without a usable declared rule.
+ * @param detail Why no rule could be evaluated.
+ * @returns The not-defined decision.
+ */
+function notDefined(detail) {
+    return {
+        overall: "not-defined",
+        coverage: { status: "not-defined", detail },
+        threshold: {
+            status: "not-defined",
+            lead: null,
+            interval: null,
+            detail,
+        },
+        guardrails: { status: "not-defined", detail },
+        reasons: [detail],
+    };
+}
+/**
+ * Evaluate the coverage limb against the declared rule.
+ * @param selection The declared selection rule.
+ * @param control The control arm counts.
+ * @param candidate The candidate arm counts.
+ * @returns The coverage limb.
+ */
+function coverageLimb(selection, control, candidate) {
+    if (!selection.requireCompleteCoverage) {
+        return {
+            status: "not-applicable",
+            detail: "the rule does not require complete coverage",
+        };
+    }
+    const complete = control.documented === control.planned &&
+        candidate.documented === candidate.planned;
+    return {
+        status: complete ? "pass" : "fail",
+        detail: `assessment coverage: control ${control.documented}/${control.planned}, candidate ${candidate.documented}/${candidate.planned}`,
+    };
+}
+/**
+ * Evaluate the utility threshold limb against the full-population interval.
+ * The rule is defined on the full design, so a threshold passes only when
+ * every missing-data completion clears it and fails only when none can; a
+ * crossing interval is inconclusive. The assessed lead stays descriptive.
+ * @param selection The declared selection rule.
+ * @param control The control arm counts.
+ * @param candidate The candidate arm counts.
+ * @returns The threshold limb.
+ */
+function thresholdLimb(selection, control, candidate) {
+    const delta = differenceInterval(intervalFromCounts(candidate), intervalFromCounts(control));
+    if (control.assessed === 0 || candidate.assessed === 0) {
+        return {
+            status: "inconclusive",
+            lead: null,
+            interval: delta,
+            detail: "no fully scored trials to compare",
+        };
+    }
+    const lead = candidate.utilityTotal - control.utilityTotal;
+    const status = delta.low >= selection.threshold
+        ? "pass"
+        : delta.high < selection.threshold
+            ? "fail"
+            : "inconclusive";
+    return {
+        status,
+        lead,
+        interval: delta,
+        detail: `lead ${lead === null ? "unknown" : String(lead)}; full-population interval ${delta.low}..${delta.high} vs threshold ${selection.threshold}`,
+    };
+}
+/**
+ * Evaluate the guardrail limb over the candidate arm's summaries.
+ * @param guardrails The candidate arm's guardrail summaries.
+ * @returns The guardrail limb.
+ */
+function guardrailLimb(guardrails) {
+    if (guardrails.length === 0) {
+        return {
+            status: "not-defined",
+            detail: "the campaign declared no guardrails",
+        };
+    }
+    const breached = guardrails
+        .filter((summary) => summary.status === "breached")
+        .map((summary) => summary.guardrailId);
+    if (breached.length > 0) {
+        return {
+            status: "breached",
+            detail: `breached: ${breached.join(", ")}`,
+        };
+    }
+    const unresolved = guardrails
+        .filter((summary) => summary.status === "unresolved")
+        .map((summary) => summary.guardrailId);
+    if (unresolved.length > 0) {
+        return {
+            status: "unresolved",
+            detail: `unresolved: ${unresolved.join(", ")}`,
+        };
+    }
+    return { status: "clean", detail: "no established or unresolved breach" };
+}
+/**
+ * Combine the limbs into the overall outcome.
+ * @param coverage The coverage limb.
+ * @param threshold The threshold limb.
+ * @param guardrails The guardrail limb.
+ * @returns The overall outcome.
+ */
+function overallOf(coverage, threshold, guardrails) {
+    if (coverage.status === "fail")
+        return "inconclusive";
+    if (guardrails.status === "breached")
+        return "not-qualified";
+    if (guardrails.status === "unresolved")
+        return "inconclusive";
+    if (threshold.status === "inconclusive")
+        return "inconclusive";
+    if (threshold.status === "fail")
+        return "not-qualified";
+    return "qualified";
+}
+/**
+ * Evaluate the declared decision rule for a candidate against its control.
+ * @param selection The frozen selection rule, or undefined when none was frozen.
+ * @param arms The accounting counts keyed by arm id.
+ * @param guardrails The candidate arm's guardrail summaries.
+ * @returns The three limbs and the overall outcome.
+ */
+export function evaluateDecision(selection, arms, guardrails) {
+    if (selection === undefined) {
+        return notDefined("the campaign froze no selection rule");
+    }
+    const control = arms[selection.controlArm];
+    const candidate = arms[selection.candidateArm];
+    if (control === undefined || candidate === undefined) {
+        return notDefined(`selection names an arm with no accounting: ${selection.controlArm}/${selection.candidateArm}`);
+    }
+    const coverage = coverageLimb(selection, control, candidate);
+    const threshold = thresholdLimb(selection, control, candidate);
+    const guardrail = guardrailLimb(guardrails);
+    const reasons = [];
+    if (coverage.status === "fail")
+        reasons.push(coverage.detail);
+    if (threshold.status === "fail")
+        reasons.push(threshold.detail);
+    if (guardrail.status === "breached" || guardrail.status === "unresolved") {
+        reasons.push(guardrail.detail);
+    }
+    return {
+        overall: overallOf(coverage, threshold, guardrail),
+        coverage,
+        threshold,
+        guardrails: guardrail,
+        reasons,
+    };
+}

package/dist/evaluators/docs/analysis/freeze.d.ts

@@ -0,0 +1,201 @@
+/**
+ * The experiment freeze: every immutable input, bound before the first launch.
+ *
+ * A freeze is exclusive. It is written once, with `wx`, after checking that no
+ * trial, judge attempt or result exists yet, that both arm trees are clean, and
+ * that each arm actually served the description its registration measured.
+ * Verification later compares the same hashes, so a modified rubric, fixture,
+ * script, spec or swapped description invalidates the premise instead of
+ * silently changing the experiment.
+ */
+/** One immutable input and its SHA-256. */
+export interface FrozenFile {
+    /** The root-relative identity. */
+    path: string;
+    /** The SHA-256 of its bytes. */
+    sha256: string;
+}
+/** One directory inventory: every file under it, path-sorted. */
+export interface FrozenInventory {
+    /** The root-relative directory identity. */
+    path: string;
+    /** Every file beneath it. */
+    files: FrozenFile[];
+}
+/** The current freeze record format. */
+export declare const FREEZE_SCHEMA_VERSION = 2;
+/** One arm's frozen identity. */
+export interface FrozenArm {
+    /** The arm id. */
+    id: string;
+    /** The arm worktree's commit. */
+    commit: string;
+    /** The arm worktree's tree. */
+    tree: string;
+    /** Whether the worktree was clean at freeze time. */
+    clean: boolean;
+    /** The registered native description's character count. */
+    descriptionChars: number;
+    /** The registered native description's SHA-256. */
+    descriptionHash: string;
+    /** The delivered description's character count. */
+    deliveredChars: number;
+    /** The delivered description's SHA-256. */
+    deliveredHash: string;
+    /** The annotation payload's character count. */
+    annotationChars: number;
+    /** The annotation payload's SHA-256. */
+    annotationHash: string;
+}
+/** The frozen record. */
+export interface FreezeRecord {
+    /** The record format; absent means a legacy, read-only freeze. */
+    schemaVersion: number;
+    /** The campaign id. */
+    campaign: string;
+    /** The source revision the campaign was materialised from. */
+    sourceRevision: string;
+    /** The source tree. */
+    sourceTree: string;
+    /** The image identity actually used, if known before launch. */
+    imageId: string;
+    /** The container CLI identity. */
+    containerCli: string;
+    /** The measurement declaration hash. */
+    measurementSha256: string;
+    /** The arms, in id order. */
+    arms: FrozenArm[];
+    /** Every immutable single-file input, path-sorted. */
+    files: FrozenFile[];
+    /** Every immutable input directory and its complete file inventory. */
+    inventories: FrozenInventory[];
+    /** The stopping rules, verbatim. */
+    stoppingRules: string[];
+    /** The trial and judge ceilings. */
+    ceilings: {
+        trial: number;
+        judge: number;
+    };
+}
+/** The freeze outcome. */
+export type FreezeOutcome = {
+    ok: true;
+    record: FreezeRecord;
+} | {
+    ok: false;
+    problems: string[];
+};
+/** The inputs a freeze decision reads. */
+export interface FreezeInput {
+    /** The campaign id. */
+    campaign: string;
+    /** The source revision. */
+    sourceRevision: string;
+    /** The source tree. */
+    sourceTree: string;
+    /** The image identity. */
+    imageId: string | null;
+    /** The container CLI identity. */
+    containerCli: string | null;
+    /** The measurement declaration hash. */
+    measurementSha256: string | null;
+    /** The arms' registered and delivered descriptions. */
+    arms: readonly FrozenArm[];
+    /** Every immutable input. */
+    files: readonly FrozenFile[];
+    /** Every immutable input directory. */
+    inventories: readonly FrozenInventory[];
+    /** The stopping rules. */
+    stoppingRules: readonly string[];
+    /** The trial and judge ceilings. */
+    ceilings: {
+        trial: number;
+        judge: number;
+    } | null;
+    /** Whether any trial or judge evidence already exists. */
+    started: boolean;
+    /** Whether a freeze file already exists. */
+    frozen: boolean;
+}
+/**
+ * SHA-256 of exact text bytes.
+ * @param text The text.
+ * @returns The lowercase hex digest.
+ */
+export declare function sha256Text(text: string): string;
+/** One served tool description. */
+export interface ServedDescription {
+    /** The description text. */
+    description: string;
+    /** Its UTF-16 code-point count, matching the registration measurement. */
+    chars: number;
+    /** The SHA-256 of its UTF-8 bytes. */
+    sha256: string;
+}
+/**
+ * Extract the served description of one tool from a provider request body.
+ *
+ * The delivery probe saves the raw request body, so this is the actual payload
+ * the provider received — never a hand-extracted copy. A body without the tool
+ * is a problem, not an empty description.
+ * @param body The parsed request body.
+ * @param toolName The registered tool name.
+ * @returns The served description, or a problem.
+ */
+export declare function deliveredToolDescription(body: unknown, toolName?: string): {
+    delivered?: ServedDescription;
+    problem?: string;
+};
+/**
+ * Decide whether the experiment may be frozen.
+ *
+ * Refusals happen before anything is written: a started experiment, an
+ * existing freeze, a dirty arm, a missing or contradictory delivery, an absent
+ * input or a ceiling all block. Inputs are deduplicated by path and sorted, so
+ * two runs hash the same immutable set in the same order.
+ * @param input Everything the decision reads.
+ * @returns The record, or every problem.
+ */
+export declare function freezeExperiment(input: FreezeInput): FreezeOutcome;
+/**
+ * Write a verified freeze exclusively. An existing file refuses without
+ * touching its bytes.
+ * @param path The freeze file path.
+ * @param record The verified record.
+ * @returns Whether the write landed.
+ */
+export declare function writeFreeze(path: string, record: FreezeRecord): {
+    ok: true;
+} | {
+    ok: false;
+    reason: string;
+};
+/** The observed state a verification compares against the freeze. */
+export interface FreezeObservation {
+    /** The current immutable single-file inputs. */
+    files: readonly FrozenFile[];
+    /** The current inventory for every declared directory. */
+    inventories: readonly FrozenInventory[];
+    /** The image identity actually executed. */
+    imageId: string;
+    /** The container CLI actually used. */
+    containerCli: string;
+    /** The source revision in use. */
+    sourceRevision: string;
+}
+/**
+ * Why a freeze cannot authorise new spend, or null when it can.
+ * @param record The recorded freeze, or undefined when absent.
+ * @returns The refusal, or null for a modern complete record.
+ */
+export declare function spendFreezeProblem(record: FreezeRecord | null | undefined): string | null;
+/**
+ * Compare observed inputs with the frozen record, including directory
+ * inventories in both directions so an added input is caught too.
+ * @param record The frozen record.
+ * @param observation What is present now.
+ * @returns Every mismatch; empty means the premise still holds.
+ */
+export declare function verifyFreeze(record: FreezeRecord, observation: FreezeObservation): string[];
+/** Whether a path already holds a freeze. */
+export declare function freezeExists(path: string): boolean;