@markstanden/opencode-test-container 0.0.0-stage → 0.1.1

package/dist/evaluators/docs/analysis/bounds.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Interval arithmetic for trial utility bounds.
+ *
+ * A missing component is reported as a closed interval, never as zero and
+ * never as a confidence interval. Arm bounds come from assessed totals plus
+ * the summed missing components; with independent arms the candidate-minus-
+ * control interval is `candidate.low − control.high .. candidate.high −
+ * control.low`.
+ */
+/** A closed numeric interval. */
+export interface Interval {
+    /** The lowest value the quantity can take. */
+    low: number;
+    /** The highest value the quantity can take. */
+    high: number;
+}
+/** The accounting totals an interval can be read from. */
+export interface BoundableTotals {
+    /** Sum of assessed trial utilities. */
+    utilityTotal: number;
+    /** Full-population lower bound. */
+    utilityLow: number;
+    /** Full-population upper bound. */
+    utilityHigh: number;
+}
+/**
+ * Read the interval an accounting record already carries.
+ * @param counts The accounting totals and their derived bounds.
+ * @returns The closed interval.
+ */
+export declare function intervalFromCounts(counts: BoundableTotals): Interval;
+/**
+ * Compute a candidate-minus-control interval for independent bounds.
+ * The worst case pairs the candidate's low with the control's high and the
+ * candidate's high with the control's low.
+ * @param candidate The candidate arm's interval.
+ * @param control The control arm's interval.
+ * @returns The difference interval.
+ */
+export declare function differenceInterval(candidate: Interval, control: Interval): Interval;

package/dist/evaluators/docs/analysis/bounds.js

@@ -0,0 +1,31 @@
+/**
+ * Interval arithmetic for trial utility bounds.
+ *
+ * A missing component is reported as a closed interval, never as zero and
+ * never as a confidence interval. Arm bounds come from assessed totals plus
+ * the summed missing components; with independent arms the candidate-minus-
+ * control interval is `candidate.low − control.high .. candidate.high −
+ * control.low`.
+ */
+/**
+ * Read the interval an accounting record already carries.
+ * @param counts The accounting totals and their derived bounds.
+ * @returns The closed interval.
+ */
+export function intervalFromCounts(counts) {
+    return { low: counts.utilityLow, high: counts.utilityHigh };
+}
+/**
+ * Compute a candidate-minus-control interval for independent bounds.
+ * The worst case pairs the candidate's low with the control's high and the
+ * candidate's high with the control's low.
+ * @param candidate The candidate arm's interval.
+ * @param control The control arm's interval.
+ * @returns The difference interval.
+ */
+export function differenceInterval(candidate, control) {
+    return {
+        low: candidate.low - control.high,
+        high: candidate.high - control.low,
+    };
+}

package/dist/evaluators/docs/analysis/budget.d.ts

@@ -0,0 +1,137 @@
+/**
+ * Durable, exclusive spend claims for a campaign.
+ *
+ * A claim is a small file created with `wx` in a per-kind directory. Because
+ * the filesystem decides the winner, two invocations racing for the last
+ * allowance cannot both succeed, and no lock or clock is consulted.
+ *
+ * A claim counts against its ceiling from the moment it is created. That is
+ * deliberate: a crash between reserving and spawning leaves a durable
+ * reservation that still consumes budget, so recovery can never silently
+ * exceed the authorised number of calls.
+ */
+/** The spend kinds with independent ceilings. */
+export type BudgetKind = "trial" | "judge" | "recovery";
+/** The lifecycle of one durable claim. */
+export type ClaimState = "reserved" | "spawned" | "closed";
+/** One durable claim record. */
+export interface BudgetClaim {
+    /** The spend kind. */
+    kind: BudgetKind;
+    /** The one-based claim index within the kind. */
+    index: number;
+    /** The claim file path. */
+    path: string;
+    /** The current lifecycle state. */
+    state: ClaimState;
+    /** When the claim was created. */
+    at: string | null;
+    /** The trial stamp or analysis selection the claim names. */
+    stamp: string | null;
+    /** The attempt id the claim names. */
+    attemptId: string | null;
+    /** A free-form note written by the caller. */
+    note: string | null;
+    /** The logical attempt this claim owns, when one was named. */
+    key: string | null;
+}
+/** The outcome of claiming one budget slot. */
+export type ClaimResult = {
+    ok: true;
+    claim: BudgetClaim;
+} | {
+    ok: false;
+    reason: string;
+    used: number;
+    ceiling: number;
+};
+/** A claim's caller-supplied identity. */
+export interface ClaimIdentity {
+    /** The trial stamp or analysis selection the claim names. */
+    stamp?: string;
+    /** The attempt id the claim names. */
+    attemptId?: string;
+    /** A free-form note. */
+    note?: string;
+    /** The logical attempt this claim owns; duplicate keys are refused. */
+    key?: string;
+    /** The creation timestamp, injected for determinism in tests. */
+    at?: string;
+}
+/** The claim filename for one index. */
+export declare function claimFileName(kind: BudgetKind, index: number): string;
+/** The key lock filename for one logical attempt. */
+export declare function keyLockName(kind: BudgetKind, key: string): string;
+/** The directory holding one kind's claims. */
+export declare function claimDir(root: string, kind: BudgetKind): string;
+/**
+ * Read every claim of one kind, in index order. A file that matches the claim
+ * name but cannot be parsed is returned as a `reserved` claim so it still
+ * consumes budget: corrupt state fails closed.
+ * @param root The campaign evidence root.
+ * @param kind The spend kind.
+ * @returns The claims, including unreadable ones as reserved.
+ */
+export declare function readClaims(root: string, kind: BudgetKind): BudgetClaim[];
+/**
+ * Claim the next free slot of one kind, or refuse at the ceiling.
+ *
+ * The claim is written with `wx`, so the create is the arbitration point.
+ * A refused claim writes nothing and returns the observed usage.
+ * @param root The campaign evidence root.
+ * @param kind The spend kind.
+ * @param ceiling The authorised calls of this kind.
+ * @param identity The caller's identity fields.
+ * @returns The created claim, or the refusal.
+ */
+export declare function claimBudget(root: string, kind: BudgetKind, ceiling: number, identity?: ClaimIdentity): ClaimResult;
+/**
+ * Release a reservation that provably dispatched nothing.
+ *
+ * Only a `reserved` claim may be released: a spawned claim's call may have
+ * begun and must keep consuming the ceiling.
+ * @param claim The claim to release.
+ * @returns Whether the claim and its logical lock were removed.
+ */
+export declare function releaseClaim(claim: BudgetClaim): boolean;
+/**
+ * Advance one claim's durable state.
+ * @param claim The claim to update.
+ * @param state The new lifecycle state.
+ * @param patch Extra identity fields to record.
+ * @param patch.note The new note.
+ * @param patch.stamp The stamp to record.
+ * @param patch.attemptId The attempt id to record.
+ * @returns The updated claim.
+ */
+export declare function advanceClaim(claim: BudgetClaim, state: ClaimState, patch?: {
+    note?: string;
+    stamp?: string;
+    attemptId?: string;
+}): BudgetClaim;
+/** Per-state claim counts. */
+export interface ClaimUsage {
+    /** Total claims, which is the spend committed against the ceiling. */
+    claimed: number;
+    /** Claims whose call has not yet been observed to start. */
+    reserved: number;
+    /** Claims whose call started and has not closed. */
+    spawned: number;
+    /** Claims whose call reached a terminal record. */
+    closed: number;
+}
+/**
+ * Count one kind's claims by lifecycle state.
+ * @param root The campaign evidence root.
+ * @param kind The spend kind.
+ * @returns The counts.
+ */
+export declare function claimUsage(root: string, kind: BudgetKind): ClaimUsage;
+/**
+ * Classify a claim that has not closed, for a restart decision. A reservation
+ * that never spawned is `unresolved`; a spawned call with no terminal record
+ * is `unknown-outcome`. Both refuse re-entry and preserve the original claim.
+ * @param claim The claim.
+ * @returns The restart classification, or null when the claim is closed.
+ */
+export declare function restartDisposition(claim: BudgetClaim): "unresolved" | "unknown-outcome" | null;

package/dist/evaluators/docs/analysis/budget.js

@@ -0,0 +1,305 @@
+/**
+ * Durable, exclusive spend claims for a campaign.
+ *
+ * A claim is a small file created with `wx` in a per-kind directory. Because
+ * the filesystem decides the winner, two invocations racing for the last
+ * allowance cannot both succeed, and no lock or clock is consulted.
+ *
+ * A claim counts against its ceiling from the moment it is created. That is
+ * deliberate: a crash between reserving and spawning leaves a durable
+ * reservation that still consumes budget, so recovery can never silently
+ * exceed the authorised number of calls.
+ */
+import { createHash } from "node:crypto";
+import { mkdirSync, readFileSync, readdirSync, unlinkSync, writeFileSync, } from "node:fs";
+import { dirname, join } from "node:path";
+/** Whether a value is a plain JSON object. */
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/** The claim filename for one index. */
+export function claimFileName(kind, index) {
+    return `${kind}-${String(index).padStart(4, "0")}.claim`;
+}
+/** The key lock filename for one logical attempt. */
+export function keyLockName(kind, key) {
+    return `${kind}-key-${createHash("sha256").update(key, "utf8").digest("hex").slice(0, 16)}.lock`;
+}
+/** The directory holding one kind's claims. */
+export function claimDir(root, kind) {
+    return join(root, kind);
+}
+/** A recognised claim state, or null. */
+function claimStateOf(value) {
+    if (value === "reserved" || value === "spawned" || value === "closed") {
+        return value;
+    }
+    return null;
+}
+/** A string field, or null. */
+function stringOrNull(value) {
+    return typeof value === "string" ? value : null;
+}
+/** Parse one claim file, or null when its bytes are unusable. */
+function claimOf(kind, index, path, text) {
+    let value;
+    try {
+        value = JSON.parse(text);
+    }
+    catch {
+        return null;
+    }
+    if (!isRecord(value))
+        return null;
+    const state = claimStateOf(value.state);
+    if (state === null)
+        return null;
+    return {
+        kind,
+        index,
+        path,
+        state,
+        at: stringOrNull(value.at),
+        stamp: stringOrNull(value.stamp),
+        attemptId: stringOrNull(value.attemptId),
+        note: stringOrNull(value.note),
+        key: stringOrNull(value.key),
+    };
+}
+/**
+ * Read every claim of one kind, in index order. A file that matches the claim
+ * name but cannot be parsed is returned as a `reserved` claim so it still
+ * consumes budget: corrupt state fails closed.
+ * @param root The campaign evidence root.
+ * @param kind The spend kind.
+ * @returns The claims, including unreadable ones as reserved.
+ */
+export function readClaims(root, kind) {
+    const dir = claimDir(root, kind);
+    let names;
+    try {
+        names = readdirSync(dir);
+    }
+    catch {
+        return [];
+    }
+    const pattern = new RegExp(`^${kind}-(\\d{4})\\.claim$`);
+    const claims = [];
+    for (const name of names.sort()) {
+        const match = pattern.exec(name);
+        if (match === null)
+            continue;
+        const index = Number(match[1]);
+        const path = join(dir, name);
+        const text = readFileSync(path, "utf8");
+        claims.push(claimOf(kind, index, path, text) ?? {
+            kind,
+            index,
+            path,
+            state: "reserved",
+            at: null,
+            stamp: null,
+            attemptId: null,
+            note: "unreadable claim treated as reserved",
+            key: null,
+        });
+    }
+    return claims;
+}
+/**
+ * Claim the next free slot of one kind, or refuse at the ceiling.
+ *
+ * The claim is written with `wx`, so the create is the arbitration point.
+ * A refused claim writes nothing and returns the observed usage.
+ * @param root The campaign evidence root.
+ * @param kind The spend kind.
+ * @param ceiling The authorised calls of this kind.
+ * @param identity The caller's identity fields.
+ * @returns The created claim, or the refusal.
+ */
+export function claimBudget(root, kind, ceiling, identity = {}) {
+    if (!Number.isInteger(ceiling) || ceiling < 1) {
+        return {
+            ok: false,
+            reason: `invalid ${kind} ceiling ${String(ceiling)}`,
+            used: 0,
+            ceiling,
+        };
+    }
+    const dir = claimDir(root, kind);
+    mkdirSync(dir, { recursive: true });
+    const key = identity.key ?? null;
+    if (key !== null && !acquireKeyLock(dir, kind, key)) {
+        return {
+            ok: false,
+            reason: `${kind} logical attempt ${key} already owns a reservation`,
+            used: readClaims(root, kind).length,
+            ceiling,
+        };
+    }
+    for (let index = 1; index <= ceiling; index += 1) {
+        const claim = createClaim(dir, kind, index, identity);
+        if (claim !== undefined)
+            return { ok: true, claim };
+    }
+    if (key !== null)
+        releaseKeyLock(dir, kind, key);
+    return {
+        ok: false,
+        reason: `${kind} ceiling reached: ${String(ceiling)} authorised call(s)`,
+        used: ceiling,
+        ceiling,
+    };
+}
+/**
+ * Try to create one claim file exclusively.
+ * @param dir The kind's claim directory.
+ * @param kind The spend kind.
+ * @param index The claim index.
+ * @param identity The caller's identity fields.
+ * @returns The claim, or undefined when that slot is already taken.
+ */
+function createClaim(dir, kind, index, identity) {
+    const path = join(dir, claimFileName(kind, index));
+    const claim = {
+        kind,
+        index,
+        path,
+        state: "reserved",
+        at: identity.at ?? null,
+        stamp: identity.stamp ?? null,
+        attemptId: identity.attemptId ?? null,
+        note: identity.note ?? null,
+        key: identity.key ?? null,
+    };
+    try {
+        writeFileSync(path, claimBytes(claim, new Date().toISOString()), {
+            encoding: "utf8",
+            flag: "wx",
+        });
+        return { ...claim, at: identity.at ?? new Date().toISOString() };
+    }
+    catch (error) {
+        if (error.code !== "EEXIST")
+            throw error;
+        return undefined;
+    }
+}
+/**
+ * Serialise one claim record.
+ * @param claim The claim.
+ * @param at The creation timestamp.
+ * @returns The JSON bytes with a trailing newline.
+ */
+function claimBytes(claim, at) {
+    return `${JSON.stringify({
+        kind: claim.kind,
+        index: claim.index,
+        state: claim.state,
+        at: claim.at ?? at,
+        stamp: claim.stamp,
+        attemptId: claim.attemptId,
+        note: claim.note,
+        key: claim.key,
+    }, null, 2)}\n`;
+}
+/**
+ * Acquire the exclusive lock for one logical attempt.
+ * @param dir The kind's claim directory.
+ * @param kind The spend kind.
+ * @param key The logical attempt key.
+ * @returns True when this caller now owns the reservation.
+ */
+function acquireKeyLock(dir, kind, key) {
+    try {
+        writeFileSync(join(dir, keyLockName(kind, key)), `${key}\n`, {
+            encoding: "utf8",
+            flag: "wx",
+        });
+        return true;
+    }
+    catch (error) {
+        if (error.code === "EEXIST")
+            return false;
+        throw error;
+    }
+}
+/** Remove one logical attempt's lock. */
+function releaseKeyLock(dir, kind, key) {
+    try {
+        unlinkSync(join(dir, keyLockName(kind, key)));
+    }
+    catch {
+        /* already released */
+    }
+}
+/**
+ * Release a reservation that provably dispatched nothing.
+ *
+ * Only a `reserved` claim may be released: a spawned claim's call may have
+ * begun and must keep consuming the ceiling.
+ * @param claim The claim to release.
+ * @returns Whether the claim and its logical lock were removed.
+ */
+export function releaseClaim(claim) {
+    if (claim.state !== "reserved")
+        return false;
+    try {
+        unlinkSync(claim.path);
+    }
+    catch {
+        return false;
+    }
+    if (claim.key !== null) {
+        releaseKeyLock(dirname(claim.path), claim.kind, claim.key);
+    }
+    return true;
+}
+/**
+ * Advance one claim's durable state.
+ * @param claim The claim to update.
+ * @param state The new lifecycle state.
+ * @param patch Extra identity fields to record.
+ * @param patch.note The new note.
+ * @param patch.stamp The stamp to record.
+ * @param patch.attemptId The attempt id to record.
+ * @returns The updated claim.
+ */
+export function advanceClaim(claim, state, patch = {}) {
+    const updated = {
+        ...claim,
+        state,
+        stamp: patch.stamp ?? claim.stamp,
+        attemptId: patch.attemptId ?? claim.attemptId,
+        note: patch.note ?? claim.note,
+    };
+    writeFileSync(updated.path, claimBytes(updated, new Date().toISOString()), "utf8");
+    return updated;
+}
+/**
+ * Count one kind's claims by lifecycle state.
+ * @param root The campaign evidence root.
+ * @param kind The spend kind.
+ * @returns The counts.
+ */
+export function claimUsage(root, kind) {
+    const claims = readClaims(root, kind);
+    return {
+        claimed: claims.length,
+        reserved: claims.filter((claim) => claim.state === "reserved").length,
+        spawned: claims.filter((claim) => claim.state === "spawned").length,
+        closed: claims.filter((claim) => claim.state === "closed").length,
+    };
+}
+/**
+ * Classify a claim that has not closed, for a restart decision. A reservation
+ * that never spawned is `unresolved`; a spawned call with no terminal record
+ * is `unknown-outcome`. Both refuse re-entry and preserve the original claim.
+ * @param claim The claim.
+ * @returns The restart classification, or null when the claim is closed.
+ */
+export function restartDisposition(claim) {
+    if (claim.state === "closed")
+        return null;
+    return claim.state === "reserved" ? "unresolved" : "unknown-outcome";
+}

package/dist/evaluators/docs/analysis/closure.d.ts

@@ -0,0 +1,175 @@
+/**
+ * Campaign accounting and closure, derived from durable evidence.
+ *
+ * The report never trusts a hard-coded checkpoint. Counts come from the
+ * frozen schedule, the launch claims, the recorded trial directories and each
+ * attempt's own token/duration record. A stopped partial campaign closes with
+ * the real scope it ran, and a closed campaign cannot be reopened by either
+ * the runner or the recovery path.
+ */
+import type { BudgetClaim } from "./budget.ts";
+/** What one scheduled slot's evidence proves. */
+export interface SlotObservation {
+    /** The declared slot number. */
+    slot: number;
+    /** The pair id. */
+    pairId: string | null;
+    /** The trial stamp. */
+    stamp: string;
+    /** The declared arm. */
+    arm: string;
+    /** The declared scenario. */
+    scenario: string;
+    /** Whether a trial launch attempt exists. */
+    launched: boolean;
+    /** Whether the launch reached a recorded result. */
+    recorded: boolean;
+    /** Recorded trial steps, or null when no trail was readable. */
+    trialSteps: number | null;
+    /** Recorded trial token fields; null means unavailable, never zero. */
+    trialTokens: TokenFields | null;
+    /** Recorded trial duration in milliseconds, or null. */
+    trialDurationMs: number | null;
+    /** Judge attempts recorded for this slot. */
+    judgeAttempts: readonly JudgeAttempt[];
+    /** The published selection for this slot, or null when unrecorded. */
+    selectedAttemptId?: string | null;
+}
+/** One token field, kept explicit so a known zero differs from unknown. */
+export type TokenFields = {
+    /** Input-token subtotal, or null when unrecorded on any observation. */
+    input: number | null;
+    /** Output-token subtotal, or null when unrecorded on any observation. */
+    output: number | null;
+    /** Reasoning-token subtotal, or null when unrecorded on any observation. */
+    reasoning: number | null;
+    /** Cache-read-token subtotal, or null when unrecorded on any observation. */
+    cacheRead: number | null;
+};
+/** What one judge attempt proves. */
+export interface JudgeAttempt {
+    /** The attempt id, e.g. `attempt-002`. */
+    attemptId: string;
+    /** Whether the attempt spawned a model call. */
+    spawned: boolean;
+    /** Whether the call reached inference (at least one step). */
+    reachedInference: boolean;
+    /** Recorded steps, or null when no trail was readable. */
+    steps: number | null;
+    /** Recorded token fields, or null when unavailable. */
+    tokens: TokenFields | null;
+    /** The recorded terminal status. */
+    status: "judged" | "invalid-verdict" | "judge-error" | "started" | "unknown";
+    /** Recorded duration, or null. */
+    durationMs: number | null;
+    /** The packet identity the attempt was judged from, or null. */
+    packetSha256?: string | null;
+    /** The machine-readable failure class, when the attempt is unusable. */
+    failureClass?: string | null;
+    /** Whether the capture proves a complete final response, or null. */
+    complete?: boolean | null;
+}
+/** The derivation inputs. */
+export interface ClosureInput {
+    /** The frozen schedule size. */
+    scheduledSlots: number;
+    /** Every slot's observation, in schedule order. */
+    observations: readonly SlotObservation[];
+    /** The durable trial claims, when the runner kept them. */
+    trialClaims?: readonly BudgetClaim[];
+    /** The durable judge claims, when the runner kept them. */
+    judgeClaims?: readonly BudgetClaim[];
+    /** The authorised ceilings. */
+    ceilings: {
+        trial: number;
+        judge: number;
+    };
+    /** Whether the attempt ledger records a terminal close. */
+    closed: boolean;
+}
+/** One model's accounting. */
+export interface ModelAccount {
+    /** Distinct scheduled slots with a spawn. */
+    attempts: number;
+    /** Attempts that reached inference. */
+    inferred: number;
+    /** Summed steps, or null when no trail was readable. */
+    steps: number | null;
+    /** Per-field totals; null where no attempt reported the field. */
+    tokens: TokenFields;
+    /** Steps that reported each field, for coverage. */
+    tokenCoverage: {
+        input: number;
+        output: number;
+        reasoning: number;
+        cacheRead: number;
+    };
+    /** Summed duration, or null when no attempt reported one. */
+    durationMs: number | null;
+}
+/** The closure report. */
+export interface ClosureReport {
+    /** Whether the campaign was already closed. */
+    state: "open" | "closed";
+    /** The scheduled slot count. */
+    scheduledSlots: number;
+    /** Distinct slots with any evidence. */
+    observedSlots: number;
+    /** Slots with no evidence at all. */
+    unrunSlots: number[];
+    /** Distinct slots with a trial spawn. */
+    trialLaunches: number;
+    /** Distinct slots with a recorded result. */
+    recordedTrials: number;
+    /** The trial accounting. */
+    trial: ModelAccount;
+    /** The judge accounting, across every attempt. */
+    judge: ModelAccount & {
+        attempts: number;
+        usableJudgements: number;
+        unusableJudgements: number;
+        /** Recorded first attempts, one per judged slot at most. */
+        initialAttempts: number;
+        /** Recorded recovery attempts beside a first attempt. */
+        recoveryAttempts: number;
+    };
+    /** The checkpoint the campaign actually stopped at, or null. */
+    stoppedAt: string | null;
+    /** Every reconciliation disagreement. */
+    problems: string[];
+}
+/**
+ * Charge a claimed-but-unspawned attempt explicitly. A reservation consumes
+ * budget and appears as an unresolved attempt, never as a launch.
+ * @param claims The durable claims.
+ * @returns The count of claims that never spawned.
+ */
+export declare function unresolvedReservations(claims: readonly BudgetClaim[] | undefined): BudgetClaim[];
+/**
+ * Derive the campaign accounts from durable observations.
+ * @param input The schedule, slot observations, claims and ceilings.
+ * @returns The closure report.
+ */
+export declare function deriveClosure(input: ClosureInput): ClosureReport;
+/** The closure decision. */
+export type ClosureOutcome = {
+    ok: true;
+    event: {
+        event: "campaign-closed";
+        stoppedAt: string | null;
+        scope: number;
+    };
+} | {
+    ok: false;
+    problems: string[];
+};
+/**
+ * Decide whether a campaign may close now, and with what scope.
+ *
+ * A campaign with no evidence is refused rather than closed empty. A stopped
+ * partial campaign closes with the slots that actually ran; the schedule's
+ * unrun slots are preserved as scope, not invented results.
+ * @param report The derived closure report.
+ * @returns The close event, or every reason closure is refused.
+ */
+export declare function closeCampaign(report: ClosureReport): ClosureOutcome;