@markstanden/opencode-test-container 0.0.0-stage → 0.1.1

package/dist/campaign/assess.d.ts

@@ -0,0 +1,45 @@
+import { assessSkillTrial } from "../evaluators/skills/assess.ts";
+import { assessDocsTrial } from "../evaluators/docs/assess.ts";
+/** One stored assessment. */
+export type StoredAssessment = ReturnType<typeof assessSkillTrial> | ReturnType<typeof assessDocsTrial>;
+/** Aggregate counts for a status/report line. */
+export interface AssessmentSummary {
+    /** The assessed value. */
+    assessed: number;
+    /** The pass value. */
+    pass: number;
+    /** The fail value. */
+    fail: number;
+    /** The unknown value. */
+    unknown: number;
+    /** The invalid value. */
+    invalid: number;
+    /** The unrun value. */
+    unrun: number;
+}
+/** The result of one assess invocation. */
+export interface AssessResult {
+    /** The ok value. */
+    ok: boolean;
+    /** The summary value. */
+    summary: AssessmentSummary;
+    /** The problems value. */
+    problems: string[];
+}
+/** Path of one deterministic assessment. */
+export declare function assessmentPath(root: string, slotId: string): string;
+/**
+ * Deterministically assess every scheduled slot from captured evidence.
+ * Deterministic suites make zero model calls.
+ * @param root The frozen campaign root.
+ * @param options Slot selection and injected clock.
+ * @param options.slotIds The selected slots.
+ * @param options.now The injected clock.
+ * @returns Aggregate counts and any refusals.
+ */
+export declare function assessCampaign(root: string, options?: {
+    slotIds?: readonly string[];
+    now?: () => string;
+}): AssessResult;
+/** Read all stored assessments, sorted by slot id. */
+export declare function readAssessments(root: string): StoredAssessment[];

package/dist/campaign/assess.js

@@ -0,0 +1,208 @@
+import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync, } from "node:fs";
+import { join } from "node:path";
+import { canonicalJson, sha256Of } from "../contracts/canonical.js";
+import { isRecord } from "../contracts/validate.js";
+import { loadPreparedManifest } from "./prepare.js";
+import { loadFrozen } from "./freeze.js";
+import { appendLedgerEvent, readLedger } from "./ledger.js";
+import { trialDirectory } from "./run.js";
+import { assessSkillTrial } from "../evaluators/skills/assess.js";
+import { assessDocsTrial } from "../evaluators/docs/assess.js";
+/** An empty aggregate. */
+function emptySummary() {
+    return { assessed: 0, pass: 0, fail: 0, unknown: 0, invalid: 0, unrun: 0 };
+}
+/** Path of one deterministic assessment. */
+export function assessmentPath(root, slotId) {
+    return join(root, "evidence", "assessments", `${slotId}.json`);
+}
+/**
+ * Write an assessment immutably: an existing different record is refused, not
+ * overwritten. Reassessing unchanged evidence is idempotent.
+ * @param path The assessment path.
+ * @param assessment The assessment to store.
+ * @returns Null on success, or the refusal.
+ */
+function storeAssessment(path, assessment) {
+    const text = `${JSON.stringify(assessment, null, 2)}\n`;
+    if (!existsSync(path)) {
+        mkdirSync(join(path, ".."), { recursive: true });
+        writeFileSync(path, text, "utf8");
+        return null;
+    }
+    const existing = readFileSync(path, "utf8");
+    try {
+        const parsed = JSON.parse(existing);
+        if (isRecord(parsed) &&
+            canonicalJson(parsed) === canonicalJson(assessment)) {
+            return null;
+        }
+    }
+    catch {
+        return `${path} is unreadable; refusing to overwrite it`;
+    }
+    return `${path} already holds a different assessment; historical assessments are immutable`;
+}
+/** Tally one assessment. */
+function tally(summary, assessment) {
+    summary.assessed += 1;
+    switch (assessment.outcome) {
+        case "pass":
+            summary.pass += 1;
+            break;
+        case "fail":
+            summary.fail += 1;
+            break;
+        default:
+            summary.unknown += 1;
+    }
+    if (assessment.validity === "invalid")
+        summary.invalid += 1;
+    if (assessment.validity === "unrun")
+        summary.unrun += 1;
+}
+/**
+ * Deterministically assess every scheduled slot from captured evidence.
+ * Deterministic suites make zero model calls.
+ * @param root The frozen campaign root.
+ * @param options Slot selection and injected clock.
+ * @returns Aggregate counts and any refusals.
+ * @param options.slotIds The slotIds value.
+ * @param options.now The now value.
+ */
+/** Why assessment is refused now, or null when it may proceed. */
+function assessmentRefusal(root) {
+    const frozen = loadFrozen(root);
+    if (!frozen.ok)
+        return frozen.problem;
+    const ledger = readLedger(root);
+    return ledger.state === "malformed"
+        ? `attempt ledger is malformed: ${ledger.problems.join("; ")}`
+        : null;
+}
+/** Build the deterministic assessment for one slot. */
+function assessmentForSlot(root, manifest, slot, now) {
+    const testCase = manifest.suite.cases.find((entry) => entry.caseId === slot.caseId);
+    if (testCase === undefined)
+        return null;
+    const trialDir = trialDirectory(root, slot.slotId);
+    const common = {
+        campaignId: manifest.campaignId,
+        slotId: slot.slotId,
+        caseId: slot.caseId,
+        armId: slot.armId,
+        repetitionId: slot.repetitionId,
+        attemptId: `deterministic-${slot.slotId}`,
+        assessedAt: now(),
+    };
+    if (testCase.kind === "skill-trigger") {
+        return assessSkillTrial({ ...common, testCase, trialDir });
+    }
+    const preparedCase = manifest.cases.find((entry) => entry.caseId === slot.caseId);
+    return assessDocsTrial({
+        ...common,
+        testCase,
+        trialDir,
+        workspace: join(trialDir, "workspace"),
+        prepared: new Map((preparedCase?.workspace ?? []).map((entry) => [
+            entry.path,
+            entry.sha256,
+        ])),
+    });
+}
+/**
+ * Deterministically assess every scheduled slot from captured evidence.
+ * Deterministic suites make zero model calls.
+ * @param root The frozen campaign root.
+ * @param options Slot selection and injected clock.
+ * @returns Aggregate counts and any refusals.
+ * @param options.slotIds The slotIds value.
+ * @param options.now The now value.
+ */
+/** Store one assessment and append its ledger event once. */
+function recordAssessment(root, manifest, slot, now, recorded) {
+    const assessment = assessmentForSlot(root, manifest, slot, now);
+    if (assessment === null) {
+        return {
+            assessment: null,
+            problems: [`slot ${slot.slotId} has no case ${slot.caseId}`],
+        };
+    }
+    const refusal = storeAssessment(assessmentPath(root, slot.slotId), assessment);
+    if (refusal !== null) {
+        return { assessment: null, problems: [refusal] };
+    }
+    if (!recorded.has(assessment.attemptId)) {
+        appendLedgerEvent(root, {
+            event: "assessment",
+            campaignId: manifest.campaignId,
+            slotId: slot.slotId,
+            attemptId: assessment.attemptId,
+            outcome: assessment.outcome,
+            validity: assessment.validity,
+            assessmentSha256: sha256Of(assessment),
+        }, assessment.assessedAt);
+        recorded.add(assessment.attemptId);
+    }
+    return { assessment, problems: [] };
+}
+/**
+ * Deterministically assess every scheduled slot from captured evidence.
+ * Deterministic suites make zero model calls.
+ * @param root The frozen campaign root.
+ * @param options Slot selection and injected clock.
+ * @param options.slotIds The selected slots.
+ * @param options.now The injected clock.
+ * @returns Aggregate counts and any refusals.
+ */
+export function assessCampaign(root, options = {}) {
+    const prepared = loadPreparedManifest(root);
+    if (!prepared.ok) {
+        return {
+            ok: false,
+            summary: emptySummary(),
+            problems: [prepared.problem],
+        };
+    }
+    const refusal = assessmentRefusal(root);
+    if (refusal !== null) {
+        return { ok: false, summary: emptySummary(), problems: [refusal] };
+    }
+    const ledger = readLedger(root);
+    const now = options.now ?? (() => new Date().toISOString());
+    const summary = emptySummary();
+    const problems = [];
+    const recorded = new Set(ledger.events
+        .filter((event) => event.event === "assessment")
+        .map((event) => typeof event.attemptId === "string" ? event.attemptId : ""));
+    const manifest = prepared.manifest;
+    const slots = manifest.campaign.schedule.slots.filter((slot) => options.slotIds === undefined ||
+        options.slotIds.includes(slot.slotId));
+    for (const slot of slots) {
+        const result = recordAssessment(root, manifest, slot, now, recorded);
+        problems.push(...result.problems);
+        if (result.assessment !== null)
+            tally(summary, result.assessment);
+    }
+    return { ok: problems.length === 0, summary, problems };
+}
+/** Read all stored assessments, sorted by slot id. */
+export function readAssessments(root) {
+    const directory = join(root, "evidence", "assessments");
+    if (!existsSync(directory))
+        return [];
+    return readdirSync(directory)
+        .filter((name) => name.endsWith(".json"))
+        .sort()
+        .flatMap((name) => {
+        try {
+            const value = JSON.parse(readFileSync(join(directory, name), "utf8"));
+            return isRecord(value)
+                ? [value]
+                : [];
+        }
+        catch {
+            return [];
+        }
+    });
+}

package/dist/campaign/close.d.ts

@@ -0,0 +1,17 @@
+/** The closure outcome. */
+export type CloseResult = {
+    ok: true;
+    alreadyClosed: boolean;
+    closedAt: string;
+} | {
+    ok: false;
+    problems: string[];
+};
+/**
+ * Close a campaign's ledger. Closure is an append-only event and is
+ * idempotent; a malformed ledger or an unresolved launch refusal prevents it.
+ * @param root The campaign root.
+ * @param now Injected clock for tests.
+ * @returns The closure outcome.
+ */
+export declare function closeCampaign(root: string, now?: () => string): CloseResult;

package/dist/campaign/close.js

@@ -0,0 +1,112 @@
+import { appendLedgerEvent, readLedger, trialStarts } from "./ledger.js";
+import { loadPreparedManifest } from "./prepare.js";
+import { loadFrozen } from "./freeze.js";
+import { readAssessments } from "./assess.js";
+import { checkedSelection, packetIntegrityProblems, policyOf, readJudgeAttempts, } from "./judge.js";
+import { readClaims } from "../evaluators/docs/analysis/budget.js";
+import { join } from "node:path";
+/** Unsettled spend claims that must stop closure, or an empty list. */
+function unsettledClaims(root) {
+    return ["judge", "recovery"].flatMap((kind) => readClaims(join(root, "evidence"), kind)
+        .filter((claim) => claim.state !== "closed")
+        .map((claim) => `${kind} claim ${claim.index} is ${claim.state}${claim.stamp === null ? "" : ` for ${claim.stamp}`}`));
+}
+/** Completed slots with neither a selected judgement nor a terminal failure. */
+function unjudgedCompleted(root, manifest, completed) {
+    if (manifest.campaign.assessment.judge === undefined)
+        return [];
+    const policy = policyOf(manifest);
+    return manifest.campaign.schedule.slots.flatMap((slot) => {
+        if (!completed.has(slot.slotId))
+            return [];
+        const checked = policy.ok
+            ? checkedSelection({
+                root,
+                manifest,
+                slot,
+                policy: policy.policy,
+            })
+            : { ok: false, problems: policy.problems };
+        if (checked.ok)
+            return [];
+        const terminal = readJudgeAttempts(root, slot.slotId).some((attempt) => attempt.judgementStatus === "invalid-verdict" ||
+            attempt.judgementStatus === "judge-error");
+        return terminal ? [] : [slot.slotId];
+    });
+}
+/** Every reason closure must refuse now, in reporting order. */
+function closureRefusals(root, manifest, starts, ended) {
+    const unresolved = starts
+        .map((event) => (typeof event.slotId === "string" ? event.slotId : ""))
+        .filter((slotId) => !ended.has(slotId));
+    if (unresolved.length > 0) {
+        return [
+            `unresolved trial launch(es): ${unresolved.join(", ")}; closure would hide unknown spend`,
+        ];
+    }
+    const claims = unsettledClaims(root);
+    if (claims.length > 0) {
+        return [
+            `unsettled judge spend: ${claims.join("; ")}; closure would hide an unknown outcome`,
+        ];
+    }
+    const packetProblems = manifest.campaign.schedule.slots.flatMap((slot) => packetIntegrityProblems(root, manifest, slot).map((problem) => `${slot.slotId}: ${problem}`));
+    if (packetProblems.length > 0) {
+        return ["judge packet integrity failed", ...packetProblems];
+    }
+    const policy = policyOf(manifest);
+    if (!policy.ok)
+        return policy.problems;
+    const pending = unjudgedCompleted(root, manifest, ended);
+    return pending.length > 0
+        ? [
+            `unjudged completed slot(s): ${pending.join(", ")}; run otc judge before closure`,
+        ]
+        : [];
+}
+/**
+ * Close a campaign's ledger. Closure is an append-only event and is
+ * idempotent; a malformed ledger or an unresolved launch refusal prevents it.
+ * @param root The campaign root.
+ * @param now Injected clock for tests.
+ * @returns The closure outcome.
+ */
+export function closeCampaign(root, now = () => new Date().toISOString()) {
+    const prepared = loadPreparedManifest(root);
+    if (!prepared.ok)
+        return { ok: false, problems: [prepared.problem] };
+    const frozen = loadFrozen(root);
+    if (!frozen.ok)
+        return { ok: false, problems: [frozen.problem] };
+    const ledger = readLedger(root);
+    if (ledger.state === "malformed") {
+        return { ok: false, problems: ledger.problems };
+    }
+    if (ledger.state === "closed") {
+        return {
+            ok: true,
+            alreadyClosed: true,
+            closedAt: ledger.closedAt ?? "unknown",
+        };
+    }
+    const starts = trialStarts(ledger.events);
+    const ended = new Set(ledger.events
+        .filter((event) => event.event === "trial-end")
+        .map((event) => typeof event.slotId === "string" ? event.slotId : ""));
+    const refusals = closureRefusals(root, prepared.manifest, starts, ended);
+    if (refusals.length > 0) {
+        return { ok: false, problems: refusals };
+    }
+    const assessments = readAssessments(root);
+    const closedAt = now();
+    appendLedgerEvent(root, {
+        event: "close",
+        campaignId: prepared.manifest.campaignId,
+        closedAt,
+        trialStarts: starts.length,
+        trialEnds: ended.size,
+        assessments: assessments.length,
+        slots: prepared.manifest.campaign.schedule.slots.length,
+    }, closedAt);
+    return { ok: true, alreadyClosed: false, closedAt };
+}

package/dist/campaign/evidence-guard.d.ts

@@ -0,0 +1,24 @@
+/** The outcome of a guarded evidence operation. */
+export interface EvidenceGuardResult {
+    /** Whether the operation was allowed. */
+    ok: boolean;
+    /** Human-readable reason, safe to log. */
+    reason: string;
+    /** Entries that protect the directory when the operation refused. */
+    entries?: string[];
+}
+/** Everything in a directory except the reservation marker. */
+export declare function recordedEntries(directory: string): string[];
+/**
+ * Clear one abandoned reservation without touching recorded evidence.
+ * @param directory The result directory holding the reservation.
+ * @returns The outcome and, on refusal, the entries that protect the result.
+ */
+export declare function clearReservation(directory: string): EvidenceGuardResult;
+/**
+ * Claim a new attempt directory. An existing directory is never reused or
+ * overwritten, even when it is empty.
+ * @param directory The attempt directory to create.
+ * @returns Whether the claim succeeded or why it was refused.
+ */
+export declare function claimAttemptDir(directory: string): EvidenceGuardResult;

package/dist/campaign/evidence-guard.js

@@ -0,0 +1,49 @@
+// Evidence-preservation guards for recorded trial and judge attempts.
+//
+// A reservation is a lock marker, never evidence. Clearing one must remove
+// only `.reservation` and must refuse when any other entry exists. An existing
+// attempt directory is immutable: a second claim never overwrites its bytes.
+import { existsSync, mkdirSync, readdirSync, rmSync } from "node:fs";
+import { join } from "node:path";
+/** Everything in a directory except the reservation marker. */
+export function recordedEntries(directory) {
+    if (!existsSync(directory))
+        return [];
+    return readdirSync(directory).filter((name) => name !== ".reservation");
+}
+/**
+ * Clear one abandoned reservation without touching recorded evidence.
+ * @param directory The result directory holding the reservation.
+ * @returns The outcome and, on refusal, the entries that protect the result.
+ */
+export function clearReservation(directory) {
+    if (!existsSync(directory))
+        return { ok: false, reason: "directory does not exist" };
+    if (!existsSync(join(directory, ".reservation")))
+        return { ok: false, reason: "no reservation present" };
+    const entries = recordedEntries(directory);
+    if (entries.length > 0)
+        return {
+            ok: false,
+            reason: "recorded evidence is present; refusing to clear",
+            entries,
+        };
+    rmSync(join(directory, ".reservation"));
+    return { ok: true, reason: "reservation cleared; no evidence removed" };
+}
+/**
+ * Claim a new attempt directory. An existing directory is never reused or
+ * overwritten, even when it is empty.
+ * @param directory The attempt directory to create.
+ * @returns Whether the claim succeeded or why it was refused.
+ */
+export function claimAttemptDir(directory) {
+    if (existsSync(directory))
+        return {
+            ok: false,
+            reason: "attempt directory already exists; refusing to overwrite",
+            entries: recordedEntries(directory),
+        };
+    mkdirSync(directory, { recursive: false });
+    return { ok: true, reason: "attempt directory created" };
+}

package/dist/campaign/freeze.d.ts

@@ -0,0 +1,94 @@
+import { runCommand } from "../adapters/process.ts";
+import { type PreparedManifest } from "./prepare.ts";
+import { type ToolIdentity } from "../contracts/version.ts";
+/** The immutable freeze receipt. */
+export interface FrozenRecord {
+    /** The schema version value. */
+    schemaVersion: 1;
+    /** The kind value. */
+    kind: "otc-freeze";
+    /** The campaign id value. */
+    campaignId: string;
+    /** The suite id value. */
+    suiteId: string;
+    /** The freeze sha256 value. */
+    freezeSha256: string;
+    /** The manifest sha256 value. */
+    manifestSha256: string;
+    /** The campaign sha256 value. */
+    campaignSha256: string;
+    /** The suite sha256 value. */
+    suiteSha256: string;
+    /** The schedule sha256 value. */
+    scheduleSha256: string;
+    /** The subjects sha256 value. */
+    subjectsSha256: string;
+    /** The workspaces sha256 value. */
+    workspacesSha256: string;
+    /** The prompts sha256 value. */
+    promptsSha256: string;
+    /** The staged primary-check payload sha256 value. */
+    primaryChecksSha256: string;
+    /** The artefact that froze this campaign. */
+    tool: ToolIdentity;
+    /** The image value. */
+    image: string;
+    /** The image id value. */
+    imageId: string;
+    /** The opencode version value. */
+    opencodeVersion: string;
+    /** The provider value. */
+    provider: {
+        route: string;
+        model: string;
+    };
+    /** The frozen at value. */
+    frozenAt: string;
+}
+/** A freeze attempt's outcome. */
+export type FreezeResult = {
+    ok: true;
+    record: FrozenRecord;
+    alreadyFrozen: boolean;
+} | {
+    ok: false;
+    problems: string[];
+};
+/** The freeze fields whose change must refuse a later freeze/run. */
+export type FreezeIdentity = Omit<FrozenRecord, "freezeSha256" | "frozenAt">;
+/** The canonical freeze identity, independent of the freeze timestamp. */
+export declare function freezeIdentity(record: FreezeIdentity): string;
+/** Path of the freeze receipt. */
+export declare function frozenPath(root: string): string;
+/**
+ * Read a freeze receipt without trusting its shape.
+ * @param root The campaign root.
+ * @returns The record, or the reason it is unusable.
+ */
+export declare function loadFrozen(root: string): {
+    ok: true;
+    record: FrozenRecord;
+} | {
+    ok: false;
+    problem: string;
+};
+/** Strip volatile and derived fields from a freeze record. */
+export declare function identityOf(record: FrozenRecord): FreezeIdentity;
+/** Compare prepared inputs with a freeze receipt. */
+export declare function freezeDifferences(manifest: PreparedManifest, frozen: FrozenRecord, current?: ToolIdentity): string[];
+/** Check that prepared files and the stored campaign copy are intact. */
+export declare function inputDifferences(root: string, manifest: PreparedManifest): string[];
+/**
+ * Freeze a prepared campaign: resolve the image to its immutable identity,
+ * verify the CLI version it contains, and record all input identities.
+ * @param root The prepared campaign root.
+ * @param run Process adapter, injected for tests.
+ * @returns The receipt, or every refusal.
+ */
+export declare function freezeCampaign(root: string, run?: typeof runCommand): Promise<FreezeResult>;
+/**
+ * The read-only delivery check: is the frozen payload complete and unchanged?
+ * @param root The campaign root.
+ * @returns Differences; empty means delivery is intact.
+ */
+export declare function deliveryCheck(root: string, current?: ToolIdentity): string[];