@markstanden/opencode-test-container 0.0.0-stage → 0.1.0

package/dist/campaign/run.js

@@ -0,0 +1,562 @@
+import { randomBytes } from "node:crypto";
+import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { sha256Bytes, sha256Of } from "../contracts/canonical.js";
+import { decodeTrail, captureProblems, } from "../adapters/opencode/events.js";
+import { opencodeRunArgs, providerOverlay, sessionEnv, writeConfigHome, } from "../adapters/opencode/runtime.js";
+import { CONTAINER_CONFIG_HOME, CONTAINER_JOURNAL, CONTAINER_SUBJECTS, CONTAINER_WORKSPACE, normaliseMounts, runContainer, } from "../adapters/podman/container.js";
+import { deliveryCheck, loadFrozen } from "./freeze.js";
+import { stageTree } from "./inventory.js";
+import { executePrimaryCheck } from "./primary.js";
+import { appendLedgerEvent, readLedger, startedSlots, trialStarts, } from "./ledger.js";
+import { loadPreparedManifest, } from "./prepare.js";
+/** Path of one trial's evidence directory. */
+export function trialDirectory(root, slotId) {
+    return join(root, "evidence", "trials", slotId);
+}
+/** Find one schedule slot by id. */
+function slotOf(campaign, slotId) {
+    return campaign.schedule.slots.find((slot) => slot.slotId === slotId);
+}
+/** Find one staged arm by id. */
+function armOf(manifest, armId) {
+    return manifest.subjects.find((arm) => arm.armId === armId);
+}
+/** A staged arm in the normalised campaign form. */
+function campaignArm(campaign, armId) {
+    return campaign.subjects.find((arm) => arm.armId === armId);
+}
+/** Copy the arm's instruction overlays into the workspace. */
+function stageInstructions(root, arm, workspace) {
+    const problems = [];
+    for (const instruction of arm.instructions) {
+        const target = join(workspace, instruction.as);
+        if (existsSync(target)) {
+            problems.push(`instruction ${instruction.id} collides with ${instruction.as}`);
+            continue;
+        }
+        mkdirSync(dirname(target), { recursive: true });
+        copyFileSync(join(root, "prepare", instruction.path), target);
+    }
+    return problems;
+}
+/** Copy the arm's skill bundles into the read-only config home. */
+function stageSkills(root, arm, configHome) {
+    const problems = [];
+    for (const skill of arm.skills) {
+        const staged = stageTree(join(root, "prepare", skill.path), join(configHome, "opencode", "skills", skill.id));
+        if (!staged.ok) {
+            problems.push(`skill ${skill.id}: ${staged.problem}`);
+        }
+    }
+    return problems;
+}
+/** The mounts for one trial. */
+function trialMounts(options) {
+    const mounts = [
+        {
+            source: options.workspace,
+            target: CONTAINER_WORKSPACE,
+            readOnly: false,
+        },
+        {
+            source: options.configHome,
+            target: CONTAINER_CONFIG_HOME,
+            readOnly: true,
+        },
+        {
+            source: options.subjects,
+            target: CONTAINER_SUBJECTS,
+            readOnly: true,
+        },
+    ];
+    if (options.journal !== null) {
+        mounts.push({
+            source: options.journal,
+            target: CONTAINER_JOURNAL,
+            readOnly: false,
+        });
+    }
+    return normaliseMounts(mounts);
+}
+/** Write the 0600 trial environment file and return its names. */
+function writeTrialEnvironment(trialDir, campaign, journal, passthrough) {
+    const env = sessionEnv(process.env, {
+        configHome: CONTAINER_CONFIG_HOME,
+        dataHome: "/tmp/agent-home/.local/share",
+        cacheHome: "/tmp/agent-home/.cache",
+        overlay: providerOverlay(campaign),
+        passthrough,
+    });
+    if (journal !== null) {
+        env.DOCS_OPERATION_JOURNAL = `${CONTAINER_JOURNAL}/docs-operations.jsonl`;
+        env.DOCS_OPERATION_JOURNAL_KEY = randomBytes(32).toString("hex");
+        writeFileSync(join(trialDir, "journal-key.txt"), env.DOCS_OPERATION_JOURNAL_KEY, { encoding: "utf8", mode: 0o600 });
+    }
+    const names = [
+        "HOME",
+        "XDG_CONFIG_HOME",
+        "XDG_DATA_HOME",
+        "XDG_CACHE_HOME",
+        "OPENCODE_CONFIG_CONTENT",
+        ...(journal === null
+            ? []
+            : ["DOCS_OPERATION_JOURNAL", "DOCS_OPERATION_JOURNAL_KEY"]),
+        ...passthrough,
+    ];
+    const envFile = join(trialDir, "trial.env");
+    writeFileSync(envFile, `${names
+        .filter((name) => env[name] !== undefined)
+        .map((name) => `${name}=${env[name] ?? ""}`)
+        .join("\n")}\n`, { encoding: "utf8", mode: 0o600 });
+    return { envFile, names };
+}
+/** Stage the workspace, config home and journal for one trial. */
+function stageTrialPayload(root, campaign, armValue, arm, testCase, trialDir) {
+    mkdirSync(trialDir, { recursive: true });
+    const workspace = join(trialDir, "workspace");
+    const stagedWorkspace = stageTree(join(root, "prepare", "workspaces", testCase.caseId), workspace);
+    if (!stagedWorkspace.ok) {
+        return { ok: false, problems: [stagedWorkspace.problem] };
+    }
+    const instructionProblems = stageInstructions(root, arm, workspace);
+    if (instructionProblems.length > 0) {
+        return { ok: false, problems: instructionProblems };
+    }
+    const configHome = writeConfigHome(trialDir, campaign, armValue);
+    const skillProblems = stageSkills(root, arm, configHome);
+    if (skillProblems.length > 0) {
+        return { ok: false, problems: skillProblems };
+    }
+    const journal = testCase.kind === "docs-task" ? join(trialDir, "journal") : null;
+    if (journal !== null)
+        mkdirSync(journal, { recursive: true });
+    return { ok: true, workspace, configHome, journal };
+}
+/**
+ * Build the container plan for one trial without dispatching it. Dry runs and
+ * tests use the same plan the real runner executes.
+ * @param options The resolved trial inputs.
+ * @returns The validated plan, or the problems found while staging.
+ * @param options.root The root value.
+ * @param options.manifest The manifest value.
+ * @param options.frozen The frozen value.
+ * @param options.slot The slot value.
+ * @param options.testCase The testCase value.
+ * @param options.arm The arm value.
+ * @param options.trialDir The trialDir value.
+ * @param options.passthrough The passthrough value.
+ */
+export function planTrial(options) {
+    const { root, manifest, frozen, slot, testCase, arm, trialDir } = options;
+    const campaign = manifest.campaign;
+    const armValue = campaignArm(campaign, slot.armId);
+    if (armValue === undefined) {
+        return {
+            ok: false,
+            problems: [`arm ${slot.armId} is not in the manifest`],
+        };
+    }
+    const payload = stageTrialPayload(root, campaign, armValue, arm, testCase, trialDir);
+    if (!payload.ok)
+        return payload;
+    const { workspace, configHome, journal } = payload;
+    const configPath = join(configHome, "opencode", "opencode.json");
+    const configSha256 = sha256Bytes(readFileSync(configPath));
+    const prompt = readPrompt(root, testCase.caseId);
+    if (prompt === null) {
+        return {
+            ok: false,
+            problems: [`prompt for ${testCase.caseId} is missing`],
+        };
+    }
+    const argv = [
+        "opencode",
+        ...opencodeRunArgs({
+            model: campaign.runtime.provider.model,
+            title: `otc-${campaign.campaignId}-${slot.slotId}`,
+            prompt,
+            printLogs: process.env.OTC_PRINT_LOGS === "1",
+        }),
+    ];
+    const passthrough = [
+        ...(campaign.runtime.passthroughEnv ?? []),
+        ...(options.passthrough ?? []),
+    ];
+    const { envFile, names } = writeTrialEnvironment(trialDir, campaign, journal, passthrough);
+    const plan = {
+        imageId: frozen.imageId,
+        mounts: trialMounts({
+            workspace,
+            configHome,
+            subjects: join(root, "prepare", "subjects", arm.armId),
+            journal,
+        }),
+        workdir: CONTAINER_WORKSPACE,
+        network: campaign.runtime.network,
+        argv,
+        envNames: [...new Set(names)],
+        envFile,
+        timeoutMs: campaign.runtime.timeoutMs,
+    };
+    return { ok: true, plan, argv, configSha256 };
+}
+/** Read one prepared prompt. */
+function readPrompt(root, caseId) {
+    try {
+        return readFileSync(join(root, "prepare", "prompts", `${caseId}.txt`), "utf8");
+    }
+    catch {
+        return null;
+    }
+}
+/** Capture integrity problems from one completed container result. */
+function captureIntegrity(result, trail) {
+    const integrity = captureProblems(trail);
+    if (result.status !== 0) {
+        integrity.push(`container exited ${String(result.status)}`);
+    }
+    if (result.timedOut)
+        integrity.push("container timed out");
+    if (result.error !== null) {
+        integrity.push(`container error: ${result.error}`);
+    }
+    return integrity;
+}
+/** Write all evidence for one completed trial. */
+function writeTrialEvidence(trialDir, result, trail, meta, answerInWorkspace) {
+    writeFileSync(join(trialDir, "events.jsonl"), result.stdout, "utf8");
+    writeFileSync(join(trialDir, "answer.md"), trail.text, "utf8");
+    if (answerInWorkspace) {
+        const answerPath = join(trialDir, "workspace", "answer.md");
+        if (!existsSync(answerPath)) {
+            writeFileSync(answerPath, trail.text, "utf8");
+        }
+    }
+    writeFileSync(join(trialDir, "tools.jsonl"), trail.calls.length === 0
+        ? ""
+        : `${trail.calls.map((call) => JSON.stringify(call)).join("\n")}\n`, "utf8");
+    writeFileSync(join(trialDir, "meta.json"), `${JSON.stringify(meta, null, 2)}\n`, "utf8");
+    writeFileSync(join(trialDir, "usage.json"), `${JSON.stringify(trail.usage, null, 2)}\n`, "utf8");
+    if (result.stderr !== "") {
+        writeFileSync(join(trialDir, "stderr.txt"), result.stderr, "utf8");
+    }
+}
+/** Execute one claimed trial and record its evidence. */
+async function executeSlot(options) {
+    const { root, manifest, frozen, slot, planned, trialDir, execute, now } = options;
+    const startedAt = now();
+    const launch = {
+        schemaVersion: 1,
+        campaignId: manifest.campaignId,
+        slotId: slot.slotId,
+        caseId: slot.caseId,
+        armId: slot.armId,
+        repetitionId: slot.repetitionId,
+        imageId: frozen.imageId,
+        promptSha256: manifest.cases.find((entry) => entry.caseId === slot.caseId)
+            ?.promptSha256 ?? "unknown",
+        configSha256: planned.configSha256,
+        startedAt,
+        argv: planned.argv,
+        envNames: planned.plan.envNames,
+    };
+    writeFileSync(join(trialDir, "launch.json"), `${JSON.stringify(launch, null, 2)}\n`, "utf8");
+    appendLedgerEvent(root, {
+        event: "trial-start",
+        campaignId: manifest.campaignId,
+        slotId: slot.slotId,
+        trialId: slot.slotId,
+        caseId: slot.caseId,
+        armId: slot.armId,
+        repetitionId: slot.repetitionId,
+        imageId: frozen.imageId,
+        promptSha256: launch.promptSha256,
+    }, startedAt);
+    const started = Date.now();
+    const result = await execute(planned.plan);
+    if (planned.plan.envFile !== null) {
+        rmSync(planned.plan.envFile, { force: true });
+    }
+    const completedAt = now();
+    const trail = decodeTrail(result.stdout);
+    const integrity = captureIntegrity(result, trail);
+    const meta = {
+        schemaVersion: 1,
+        campaignId: manifest.campaignId,
+        slotId: slot.slotId,
+        caseId: slot.caseId,
+        armId: slot.armId,
+        repetitionId: slot.repetitionId,
+        exit: result.status,
+        signal: result.signal,
+        timedOut: result.timedOut,
+        error: result.error,
+        durationMs: Date.now() - started,
+        captureValid: integrity.length === 0,
+        captureProblems: integrity,
+        textParts: trail.textParts,
+        lastPartKind: trail.lastPartKind,
+        parseFailures: trail.parseFailures,
+        events: trail.events,
+        promptSha256: launch.promptSha256,
+        answerSha256: sha256Bytes(trail.text),
+        imageId: frozen.imageId,
+        completedAt,
+    };
+    writeTrialEvidence(trialDir, result, trail, meta, options.testCase.kind === "docs-task");
+    appendLedgerEvent(root, {
+        event: "trial-end",
+        campaignId: manifest.campaignId,
+        slotId: slot.slotId,
+        trialId: slot.slotId,
+        captureValid: meta.captureValid,
+        exit: meta.exit,
+    }, completedAt);
+    const preparedCase = manifest.cases.find((entry) => entry.caseId === slot.caseId);
+    if (options.testCase.kind !== "docs-task" ||
+        options.testCase.primaryCheck === undefined ||
+        preparedCase === undefined) {
+        return { meta, primaryProblems: [] };
+    }
+    const checked = await executePrimaryCheck({
+        root,
+        testCase: options.testCase,
+        prepared: preparedCase,
+        trialDir,
+        workspace: join(trialDir, "workspace"),
+        imageId: frozen.imageId,
+        run: (plan) => execute(plan),
+        now,
+    });
+    if (checked.record !== null) {
+        appendLedgerEvent(root, {
+            event: "primary-check",
+            campaignId: manifest.campaignId,
+            slotId: slot.slotId,
+            checkerId: checked.record.id,
+            status: checked.record.status,
+            exit: checked.record.exit,
+            sourceHash: checked.record.sourceHash,
+        }, now());
+    }
+    return {
+        meta,
+        primaryProblems: checked.problem === null ? [] : [checked.problem],
+    };
+}
+/** Resolve the selected slots, refusing an unknown selector. */
+function selectedSlots(campaign, slotIds) {
+    if (slotIds === undefined)
+        return campaign.schedule.slots;
+    return slotIds.map((slotId) => {
+        const found = slotOf(campaign, slotId);
+        if (found === undefined)
+            throw new Error(`unknown slotId ${slotId}`);
+        return found;
+    });
+}
+/** The capture problems a slot's meta records. */
+function metaProblems(slotId, meta) {
+    if (meta.captureValid)
+        return [];
+    return [
+        `slot ${slotId} capture is invalid: ${meta.captureProblems.join("; ")}`,
+    ];
+}
+/** Whether the campaign stops after this trial. */
+function shouldStop(manifest, meta) {
+    return (manifest.campaign.stopOnFirstFailure === true &&
+        (!meta.captureValid || meta.exit !== 0));
+}
+/** Why a slot may not be dispatched now, or null. */
+function slotPrecondition(root, manifest, slot, already) {
+    if (already.has(slot.slotId)) {
+        return {
+            ran: false,
+            refused: true,
+            problems: [`slot ${slot.slotId} has already started`],
+            stop: false,
+        };
+    }
+    if (trialStarts(readLedger(root).events).length >=
+        manifest.campaign.budgets.trials) {
+        return {
+            ran: false,
+            refused: false,
+            problems: [
+                `trial budget exhausted (${manifest.campaign.budgets.trials}) before slot ${slot.slotId}`,
+            ],
+            stop: true,
+        };
+    }
+    if (existsSync(trialDirectory(root, slot.slotId))) {
+        return {
+            ran: false,
+            refused: true,
+            problems: [
+                `slot ${slot.slotId} has a trial directory without a matching ledger start`,
+            ],
+            stop: false,
+        };
+    }
+    return null;
+}
+/** Attempt one slot: preconditions, planning, dispatch and evidence. */
+async function dispatchSlot(options) {
+    const { root, manifest, frozen, slot, already, dryRun, execute, now } = options;
+    const precondition = slotPrecondition(root, manifest, slot, already);
+    if (precondition !== null)
+        return precondition;
+    const testCase = manifest.suite.cases.find((entry) => entry.caseId === slot.caseId);
+    const arm = armOf(manifest, slot.armId);
+    if (testCase === undefined || arm === undefined) {
+        return {
+            ran: false,
+            refused: false,
+            problems: [`slot ${slot.slotId} has no staged case or arm`],
+            stop: false,
+        };
+    }
+    const trialDir = trialDirectory(root, slot.slotId);
+    const planDirectory = dryRun
+        ? mkdtempSync(join(tmpdir(), "otc-plan-"))
+        : trialDir;
+    const planned = planTrial({
+        root,
+        manifest,
+        frozen,
+        slot,
+        testCase,
+        arm,
+        trialDir: planDirectory,
+        ...(options.passthrough === undefined
+            ? {}
+            : { passthrough: options.passthrough }),
+    });
+    if (!planned.ok) {
+        if (dryRun)
+            rmSync(planDirectory, { recursive: true, force: true });
+        return {
+            ran: false,
+            refused: false,
+            problems: planned.problems,
+            stop: false,
+        };
+    }
+    if (dryRun) {
+        rmSync(planDirectory, { recursive: true, force: true });
+        return { ran: true, refused: false, problems: [], stop: false };
+    }
+    const meta = await executeSlot({
+        root,
+        manifest,
+        frozen,
+        slot,
+        testCase,
+        planned,
+        trialDir,
+        execute,
+        now,
+    });
+    return {
+        ran: true,
+        refused: false,
+        problems: [
+            ...metaProblems(slot.slotId, meta.meta),
+            ...meta.primaryProblems,
+        ],
+        stop: shouldStop(manifest, meta.meta),
+    };
+}
+/** The validated state a run needs, or the refusal. */
+function runState(root, current) {
+    const prepared = loadPreparedManifest(root);
+    if (!prepared.ok)
+        return { ok: false, problems: [prepared.problem] };
+    const frozen = loadFrozen(root);
+    if (!frozen.ok)
+        return { ok: false, problems: [frozen.problem] };
+    const differences = current === undefined
+        ? deliveryCheck(root)
+        : deliveryCheck(root, current);
+    if (differences.length > 0) {
+        return {
+            ok: false,
+            problems: ["delivery check failed", ...differences],
+        };
+    }
+    const ledger = readLedger(root);
+    if (ledger.state === "malformed") {
+        return {
+            ok: false,
+            problems: [
+                `attempt ledger is malformed: ${ledger.problems.join("; ")}`,
+            ],
+        };
+    }
+    if (ledger.state === "closed") {
+        return {
+            ok: false,
+            problems: ["campaign is closed; no further dispatch is permitted"],
+        };
+    }
+    return {
+        ok: true,
+        manifest: prepared.manifest,
+        frozen: frozen.record,
+        ledger,
+    };
+}
+/**
+ * Execute one or more scheduled slots. A launch receipt and ledger event are
+ * written before the container starts, so an interrupted dispatch remains
+ * visibly claimed rather than looking blank.
+ * @param root The frozen campaign root.
+ * @param options Slot selection and dry-run choice.
+ * @param options.slotIds The selected slots.
+ * @param options.dryRun Plan without dispatch.
+ * @param options.passthrough Named environment values to pass through.
+ * @param deps Injected process runner and clock.
+ * @returns What ran and what was refused.
+ */
+export async function runCampaign(root, options = {}, deps = {}) {
+    const state = runState(root, deps.tool);
+    if (!state.ok) {
+        return { ok: false, ran: [], refused: [], problems: state.problems };
+    }
+    const manifest = state.manifest;
+    const selected = selectedSlots(manifest.campaign, options.slotIds);
+    const already = new Set(startedSlots(state.ledger.events));
+    const now = deps.now ?? (() => new Date().toISOString());
+    const execute = deps.runContainer ?? runContainer;
+    const ran = [];
+    const refused = [];
+    const problems = [];
+    for (const slot of selected) {
+        const outcome = await dispatchSlot({
+            root,
+            manifest,
+            frozen: state.frozen,
+            slot,
+            already,
+            dryRun: options.dryRun === true,
+            passthrough: options.passthrough,
+            execute,
+            now,
+        });
+        if (outcome.ran)
+            ran.push(slot.slotId);
+        if (outcome.refused)
+            refused.push(slot.slotId);
+        problems.push(...outcome.problems);
+        if (outcome.stop)
+            break;
+    }
+    return { ok: problems.length === 0, ran, refused, problems };
+}
+/** The canonical identity of a trial's immutable launch receipt. */
+export function launchSha256(launch) {
+    return sha256Of(launch);
+}

package/dist/campaign/status.d.ts

@@ -0,0 +1,76 @@
+import type { LifecycleState } from "../contracts/types.ts";
+import { type ToolIdentity } from "../contracts/version.ts";
+/** A suggested next command, derived from checked state. */
+export interface NextAction {
+    /** The command value. */
+    command: string;
+    /** The reason value. */
+    reason: string;
+}
+/** The read-only status document. */
+export interface StatusReport {
+    /** The schema version value. */
+    schemaVersion: 1;
+    /** The state value. */
+    state: LifecycleState;
+    /** The campaign id value. */
+    campaignId: string | null;
+    /** The suite id value. */
+    suiteId: string | null;
+    /** The root value. */
+    root: string;
+    /** The image id value. */
+    imageId: string | null;
+    /** The frozen at value. */
+    frozenAt: string | null;
+    /** The closed at value. */
+    closedAt: string | null;
+    /** The counts value. */
+    counts: {
+        slots: number;
+        started: number;
+        completed: number;
+        assessed: number;
+        pass: number;
+        fail: number;
+        unknown: number;
+    };
+    /** The budgets value. */
+    budgets: {
+        trials: number;
+        used: number;
+        remaining: number;
+    } | null;
+    /** The executing tool identity. */
+    tool: ToolIdentity | null;
+    /** The judgement counts value. */
+    judgement: {
+        /** Slots with a checked selection. */
+        judged: number;
+        /** Completed slots still awaiting a usable judgement. */
+        pending: number;
+        /** Completed slots whose attempts ended without a usable verdict. */
+        failed: number;
+        /** Every recorded judge attempt. */
+        attempts: number;
+        /** Attempts that ended without a usable verdict. */
+        invalid: number;
+        /** Durable judge and recovery claims not yet closed. */
+        unsettled: number;
+        /** Packet and selection integrity diagnostics. */
+        problems: string[];
+    } | null;
+    /** The next actions value. */
+    nextActions: NextAction[];
+    /** The problems value. */
+    problems: string[];
+}
+/**
+ * Observe a campaign root without writing, building, probing or dispatching.
+ * Every next action is derived from the manifest and checked evidence.
+ * @param root The campaign root.
+ * @returns The status document.
+ */
+export declare function statusOf(root: string): StatusReport;
+/** Status for a slot's trial directory, used by reports and diagnostics. */
+export declare function slotTrialExists(root: string, slotId: string): boolean;