@markstanden/opencode-test-container 0.0.0-stage → 0.1.1

package/dist/campaign/status.js

@@ -0,0 +1,237 @@
+import { existsSync } from "node:fs";
+import { join } from "node:path";
+import { loadPreparedManifest } from "./prepare.js";
+import { loadFrozen } from "./freeze.js";
+import { readLedger, trialStarts } from "./ledger.js";
+import { readAssessments, assessmentPath } from "./assess.js";
+import { checkedSelection, packetIntegrityProblems, policyOf, readJudgeAttempts, selectionPath as selectionPathOf, } from "./judge.js";
+import { toolIdentity, toolIdentityProblems, } from "../contracts/version.js";
+import { readClaims } from "../evaluators/docs/analysis/budget.js";
+import { trialDirectory } from "./run.js";
+/** Derive the finite lifecycle state from checked evidence. */
+function lifecycleStateOf(options) {
+    if (options.closed)
+        return "closed";
+    const allAssessed = options.manifest.campaign.schedule.slots.length > 0 &&
+        options.manifest.campaign.schedule.slots.every((slot) => existsSync(assessmentPath(options.root, slot.slotId)));
+    if (allAssessed)
+        return "assessed";
+    if (options.starts > 0)
+        return "running";
+    return options.frozen ? "frozen" : "prepared";
+}
+/** The judgement counts for every scheduled slot. */
+function judgementOf(root, manifest) {
+    if (manifest.campaign.assessment.judge === undefined)
+        return null;
+    const policy = policyOf(manifest);
+    const slots = manifest.campaign.schedule.slots;
+    let judged = 0;
+    let pending = 0;
+    let failed = 0;
+    let attempts = 0;
+    let invalid = 0;
+    const problems = [];
+    for (const slot of slots) {
+        const recorded = readJudgeAttempts(root, slot.slotId);
+        attempts += recorded.length;
+        invalid += recorded.filter((attempt) => attempt.judgementStatus === "invalid-verdict" ||
+            attempt.judgementStatus === "judge-error").length;
+        const checked = policy.ok
+            ? checkedSelection({
+                root,
+                manifest,
+                slot,
+                policy: policy.policy,
+            })
+            : { ok: false, problems: policy.problems };
+        if (checked.ok) {
+            judged += 1;
+        }
+        else if (existsSync(selectionPathOf(root, slot.slotId))) {
+            problems.push(...checked.problems.map((problem) => `${slot.slotId}: ${problem}`));
+        }
+        else if (existsSync(join(trialDirectory(root, slot.slotId), "meta.json"))) {
+            const terminal = recorded.some((attempt) => attempt.judgementStatus === "invalid-verdict" ||
+                attempt.judgementStatus === "judge-error");
+            if (terminal)
+                failed += 1;
+            else
+                pending += 1;
+        }
+    }
+    const unsettled = ["judge", "recovery"].reduce((sum, kind) => sum +
+        readClaims(join(root, "evidence"), kind).filter((claim) => claim.state !== "closed").length, 0);
+    problems.push(...slots.flatMap((slot) => packetIntegrityProblems(root, manifest, slot).map((problem) => `${slot.slotId}: ${problem}`)));
+    return {
+        judged,
+        pending,
+        failed,
+        attempts,
+        invalid,
+        unsettled,
+        problems,
+    };
+}
+/** The judge action when completed slots are awaiting a checked judgement. */
+function judgeAction(root, judgement) {
+    return judgement !== null && judgement.pending > 0
+        ? [
+            {
+                command: `otc judge --root ${JSON.stringify(root)}`,
+                reason: `${judgement.pending} completed slot(s) have no checked judgement`,
+            },
+        ]
+        : [];
+}
+/** Build the actionable next commands from state, never from constants. */
+function nextActionsOf(options) {
+    const quoted = JSON.stringify(options.root);
+    if (!options.frozen) {
+        return [
+            {
+                command: `otc freeze --root ${quoted}`,
+                reason: "the prepared payload is not frozen",
+            },
+        ];
+    }
+    if (options.closed) {
+        return [
+            {
+                command: `otc analyse --root ${quoted} --out ${JSON.stringify(join(options.root, "reports", "final"))}`,
+                reason: "the campaign is closed; render final analysis",
+            },
+        ];
+    }
+    const slots = options.manifest.campaign.schedule.slots;
+    const remaining = options.manifest.campaign.budgets.trials - options.starts;
+    const unstarted = slots.length - options.starts;
+    const actions = [];
+    if (unstarted > 0 && remaining > 0) {
+        actions.push({
+            command: `otc run --root ${quoted}`,
+            reason: `${unstarted} slot(s) unstarted, ${remaining} trial(s) budgeted`,
+        });
+    }
+    else if (unstarted === 0 && options.assessments < slots.length) {
+        actions.push({
+            command: `otc assess --root ${quoted}`,
+            reason: "every started slot has evidence; deterministic assessment is pending",
+        });
+    }
+    actions.push(...judgeAction(options.root, options.judgement));
+    if (options.starts > 0 && options.completed === options.starts) {
+        actions.push({
+            command: `otc close --root ${quoted}`,
+            reason: "all started trials have ended",
+        });
+    }
+    return actions;
+}
+/**
+ * Observe a campaign root without writing, building, probing or dispatching.
+ * Every next action is derived from the manifest and checked evidence.
+ * @param root The campaign root.
+ * @returns The status document.
+ */
+export function statusOf(root) {
+    const prepared = loadPreparedManifest(root);
+    if (!prepared.ok) {
+        return {
+            schemaVersion: 1,
+            state: "blocked",
+            campaignId: null,
+            suiteId: null,
+            root,
+            imageId: null,
+            frozenAt: null,
+            closedAt: null,
+            counts: {
+                slots: 0,
+                started: 0,
+                completed: 0,
+                assessed: 0,
+                pass: 0,
+                fail: 0,
+                unknown: 0,
+            },
+            tool: null,
+            budgets: null,
+            judgement: null,
+            nextActions: [],
+            problems: [prepared.problem],
+        };
+    }
+    const manifest = prepared.manifest;
+    const ledger = readLedger(root);
+    const problems = ledger.state === "malformed" ? [...ledger.problems] : [];
+    const starts = trialStarts(ledger.events);
+    const completed = new Set(ledger.events
+        .filter((event) => event.event === "trial-end")
+        .map((event) => typeof event.slotId === "string" ? event.slotId : ""));
+    const assessments = readAssessments(root);
+    const frozen = loadFrozen(root);
+    if (!frozen.ok)
+        problems.push(frozen.problem);
+    const currentTool = toolIdentity();
+    if (frozen.ok) {
+        problems.push(...toolIdentityProblems(frozen.record.tool, currentTool));
+    }
+    const policy = policyOf(manifest);
+    if (!policy.ok)
+        problems.push(...policy.problems);
+    const judgement = judgementOf(root, manifest);
+    problems.push(...(judgement?.problems ?? []));
+    const state = lifecycleStateOf({
+        root,
+        manifest,
+        frozen: frozen.ok,
+        starts: starts.length,
+        closed: ledger.state === "closed",
+    });
+    const nextActions = nextActionsOf({
+        root,
+        manifest,
+        frozen: frozen.ok,
+        closed: ledger.state === "closed",
+        starts: starts.length,
+        completed: completed.size,
+        assessments: assessments.length,
+        judgement: judgementOf(root, manifest),
+    });
+    return {
+        schemaVersion: 1,
+        state,
+        campaignId: manifest.campaignId,
+        suiteId: manifest.suiteId,
+        root,
+        imageId: frozen.ok ? frozen.record.imageId : null,
+        frozenAt: frozen.ok ? frozen.record.frozenAt : null,
+        closedAt: ledger.closedAt,
+        counts: {
+            slots: manifest.campaign.schedule.slots.length,
+            started: starts.length,
+            completed: completed.size,
+            assessed: assessments.length,
+            pass: assessments.filter((entry) => entry.outcome === "pass")
+                .length,
+            fail: assessments.filter((entry) => entry.outcome === "fail")
+                .length,
+            unknown: assessments.filter((entry) => entry.outcome === "unknown")
+                .length,
+        },
+        tool: currentTool,
+        budgets: {
+            trials: manifest.campaign.budgets.trials,
+            used: starts.length,
+            remaining: manifest.campaign.budgets.trials - starts.length,
+        },
+        judgement,
+        nextActions,
+        problems,
+    };
+}
+/** Status for a slot's trial directory, used by reports and diagnostics. */
+export function slotTrialExists(root, slotId) {
+    return existsSync(trialDirectory(root, slotId));
+}

package/dist/cli/otc.d.ts

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+/** The commands `otc` exposes. */
+export declare const COMMANDS: readonly ["validate", "prepare", "delivery-check", "freeze", "run", "judge", "assess", "close", "analyse", "status", "replay", "help", "version"];
+/** One command name. */
+export type Command = (typeof COMMANDS)[number];
+/** Parsed command-line options. */
+export interface CliOptions {
+    /** The command value. */
+    command: Command;
+    /** The campaign value. */
+    campaign: string | null;
+    /** The root value. */
+    root: string | null;
+    /** The out value. */
+    out: string | null;
+    /** The slots value. */
+    slots: string[];
+    /** The json value. */
+    json: boolean;
+    /** The dry run value. */
+    dryRun: boolean;
+    /** The passthrough value. */
+    passthrough: string[];
+    /** The recorded docs campaign manifest (replay). */
+    manifest: string | null;
+    /** The recorded evidence root (replay). */
+    evidenceRoot: string | null;
+    /** Whether to replace an existing analysis output. */
+    rerender: boolean;
+    /** Whether unbound legacy answers are accepted. */
+    allowLegacyUnbound: boolean;
+    /** The help value. */
+    help: boolean;
+}
+/** Usage text, kept identical to the documented command list. */
+export declare const USAGE = "otc \u2014 an isolated test sandbox for OpenCode plugins, skills and agent instructions\n\nUsage: otc <command> [options]\n\nCommands:\n  validate        Validate a campaign and its suite; no writes\n  prepare         Validate, then stage an immutable campaign root\n  delivery-check  Recheck staged payloads and the freeze receipt\n  freeze          Resolve the image ID and freeze all input identities\n  run             Execute scheduled slots through Podman and OpenCode\n  judge           Blindly judge completed docs trials from checked evidence\n  assess          Deterministically assess captured evidence\n  close           Append closure to the ledger (idempotent)\n  analyse         Render final analysis after closure\n  status          Read-only state and next actions\n  replay          Replay recorded docs evidence through the docs evaluator\n  help            This text\n  version         Print the package version\n\nOptions:\n  --campaign <file>   Campaign JSON (default: $OTC_CAMPAIGN)\n  --root <dir>        Campaign root (default: $OTC_ROOT)\n  --out <dir>         Fresh report directory (analyse)\n  --slot <id>         Select a slot; repeat or comma-separate\n  --passthrough <name> Pass one named environment variable into the trial\n  --manifest <file>   Recorded docs campaign manifest (replay)\n  --evidence-root <dir> Recorded evidence root (replay)\n  --rerender          Replace an existing analysis output (replay)\n  --allow-legacy-unbound Allow unbound legacy answers (replay)\n  --dry-run           Plan a trial without dispatching it\n  --json              Machine-readable output (status, validate, assess)\n  --help              This text\n\nExamples:\n  otc validate --campaign examples/skills/otel-skill-container.json\n  otc prepare --campaign campaign.json --root /tmp/otc-run\n  otc freeze --root /tmp/otc-run\n  otc run --root /tmp/otc-run --dry-run\n  otc run --root /tmp/otc-run\n  otc judge --root /tmp/otc-run\n  otc assess --root /tmp/otc-run\n  otc close --root /tmp/otc-run\n  otc analyse --root /tmp/otc-run --out /tmp/otc-report\n  otc status --root /tmp/otc-run --json\n  otc replay --manifest campaign.json --evidence-root evidence --out report\n";
+/**
+ * Parse argv into a command and its options.
+ * @param argv Arguments after the executable.
+ * @returns The parsed options.
+ */
+export declare function parseArgs(argv: readonly string[]): CliOptions;
+/** A command-line usage refusal. */
+export declare class UsageError extends Error {
+}
+/**
+ * Run one command and return its exit code.
+ * @param argv Arguments after the executable.
+ * @returns The process exit code.
+ */
+export declare function main(argv: readonly string[]): Promise<number>;