@markstanden/opencode-test-container 0.0.0-stage → 0.1.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Mark Standen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,264 @@
-# Temporary Holding Version
+# OpenCode Test Container
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+An isolated test sandbox for OpenCode plugins, skills and agent instructions.
+
+`otc` prepares an immutable campaign, freezes every input identity, runs each
+scheduled slot in a Podman sandbox through the real OpenCode CLI, executes the
+product's primary check, assesses the captured evidence deterministically,
+optionally dispatches an arm-blind judge, closes the ledger and renders a
+report. The judge is the only model call OTC makes; a campaign either uses a
+scripted loopback provider (offline) or the host provider route the runtime
+declares (potentially paid).
+
+```sh
+npm install --global @markstanden/opencode-test-container
+otc help
+```
+
+The installed binary is emitted JavaScript and carries every runtime asset; it
+does not need a checkout, `tsx` or Bun.
+
+## The model
+
+| Term       | Meaning                                                      |
+| ---------- | ------------------------------------------------------------ |
+| suite      | reusable, discriminated case definitions and assessment kind |
+| case       | one prompt/context and its expected behaviour                |
+| campaign   | one frozen scheduled execution                               |
+| arm        | one treatment configuration (subjects + non-secret options)  |
+| slot       | one case × arm × repetition                                  |
+| trial      | one execution of a slot                                      |
+| assessment | deterministic or model-based evaluation                      |
+| report     | derived output, outside immutable evidence                   |
+
+A minimal skill-only campaign needs no docs plugin, no knowledge lake and no
+judge. See [`examples/skills`](examples/skills).
+
+## Quick start (offline, no provider call)
+
+```sh
+otc validate --campaign examples/skills/campaign.json
+otc prepare  --campaign examples/skills/campaign.json --root /tmp/otc-run
+otc freeze   --root /tmp/otc-run
+otc run      --root /tmp/otc-run --dry-run   # inspect the plan, no dispatch
+otc run      --root /tmp/otc-run             # real Podman + OpenCode
+otc judge    --root /tmp/otc-run             # optional blind judge, if declared
+otc assess   --root /tmp/otc-run             # deterministic, zero judge calls
+otc close    --root /tmp/otc-run
+otc analyse  --root /tmp/otc-run --out /tmp/otc-report
+otc status   --root /tmp/otc-run --json
+```
+
+`status` is read-only: it does not build images, probe providers, repair files
+or dispatch. Its next actions are derived from the manifest and checked
+evidence, not from baked-in campaign constants.
+
+## Suite format
+
+Versioned JSON (`schemaVersion: 1`). `assessment` selects one trusted
+evaluator; cases are discriminated.
+
+```jsonc
+{
+    "schemaVersion": 1,
+    "suiteId": "skills-basic",
+    "assessment": "skill-activation",
+    "cases": [
+        {
+            "caseId": "positive-instrument",
+            "kind": "skill-trigger",
+            "targetSkillId": "otel-helper",
+            "expected": "activation",
+            "prompt": "Instrument the checkout service with OpenTelemetry spans.",
+        },
+        {
+            "caseId": "negative-near-miss",
+            "kind": "skill-trigger",
+            "targetSkillId": "otel-helper",
+            "expected": "no-activation",
+            "prompt": "Add a console log; no telemetry is needed.",
+        },
+    ],
+}
+```
+
+Docs cases use `kind: "docs-task"`, `expectedDocs`, an optional `criterion`
+and deterministic `guardrails`. An optional `primaryCheck` names a
+product-supplied checker directory and script; it is staged read-only and run
+in its own no-network container after the trial, never mounted into the trial.
+`truth` is arm-blind judge context and never enters a workspace. Docs evidence
+is assessed from the plugin's signed operation journal, never from a transcript
+claim.
+
+## Campaign format
+
+```jsonc
+{
+    "schemaVersion": 1,
+    "campaignId": "skills-local",
+    "suite": { "path": "suite.json", "sha256": "<optional pinned hash>" },
+    "subjects": [
+        {
+            "armId": "skill-only",
+            "skills": [{ "id": "otel-helper", "path": "skills/otel-helper" }],
+            "plugins": [{ "id": "docs", "path": "/abs/path/to/plugin" }],
+            "instructions": [
+                { "id": "overlay", "path": "AGENTS.md", "as": "AGENTS.md" },
+            ],
+        },
+    ],
+    "schedule": {
+        "kind": "explicit",
+        "slots": [
+            {
+                "slotId": "s1",
+                "caseId": "positive-instrument",
+                "armId": "skill-only",
+                "repetitionId": "r0",
+            },
+        ],
+    },
+    "runtime": {
+        "image": "localhost/otc-sandbox:2.2.6",
+        "imageId": "sha256:…",
+        "opencodeVersion": "2.2.6",
+        "provider": {
+            "route": "scripted",
+            "model": "vendor/model",
+            "overlay": {},
+        },
+        "timeoutMs": 180000,
+        "network": "host",
+        "passthroughEnv": ["PROVIDER_API_KEY"],
+    },
+    "budgets": { "trials": 1, "judgeAttempts": 2 },
+    "assessment": { "kind": "skill-activation" },
+}
+```
+
+A docs campaign may declare a judge and a narrow recovery registration:
+
+```jsonc
+"budgets": { "trials": 4, "judgeAttempts": 8 },
+"assessment": {
+    "kind": "docs-recording",
+    "judge": { "model": "provider/model", "timeoutMs": 180000 },
+    "recovery": {
+        "version": 1,
+        "eligibleFailureClasses": ["json-syntax"],
+        "requiresCompletionEvidence": true,
+        "maxRecoveriesPerSlot": 1,
+        "recoveryCeiling": 4,
+        "totalJudgeAttempts": 8,
+        "providerRequestLimit": null
+    }
+}
+```
+
+`judgeAttempts` counts dispatches, not HTTP requests. Each attempt is claimed
+durably before it starts; a reserved or spawned claim keeps consuming budget.
+"none" in `network` is the only true no-egress mode; without `judgeAttempts`
+a campaign makes no model call of its own.
+
+Subject sources may be local paths; they are copied into
+`prepare/subjects/<armId>` before dispatch. Relative paths resolve against the
+campaign document. Symlinks escaping a source root, path collisions and
+unreadable inventories are refused. Only the staged payload is mounted — never
+the whole OTC repository, the complete subject repository, other cases, truth
+or rubric data.
+
+## Assessment
+
+- `skill-activation` is deterministic and makes zero judge calls. It records
+  the requested target, successful target loads, failed target loads, other
+  skill loads and no-invocation separately. Capture validity is separate from
+  the criterion: a complete run with no activation is an observed negative; a
+  missing/truncated capture or a failed target load is unknown/invalid, never a
+  passing negative. A non-skill route to the target body is contamination.
+- `docs-recording` verifies the plugin's HMAC-signed journal against the host
+  tool trail, then checks the written bytes. A refused write retains zero
+  recording credit; attempts are counted separately.
+- The full recorded-evidence analyser is also available offline:
+
+    ```sh
+    otc replay --manifest <campaign.json> \
+      --evidence-root <recorded-evidence-root> --out <fresh-report-dir>
+    ```
+
+    It validates the declared campaign, loads explicit evidence states, and
+    produces accounting, utility/opportunity counts, retrieval measures,
+    preservation, guardrails, pairing, bounds, screening, decision and stable
+    reports. It makes no model call. `--allow-legacy-unbound` is the explicit
+    legacy-answer escape; `--rerender` replaces an existing output.
+
+- An optional docs judge is arm-blind by construction: the packet contains the
+  task, standing permission, authored truth, before/after lake documents,
+  changed paths, primary-check output, verified docs operations and the final
+  answer. It contains no arm label, campaign identity, evidence root or prior
+  verdict. The established parser and validator classify a usable judgement or
+  an explicit `json-syntax`, `schema`, `semantic`, `empty-or-truncated` or
+  `transport-or-process` failure. Only `json-syntax` is recoverable, and only
+  once, from identical packet bytes under a registered policy and durable
+  ceilings.
+
+Model judging is not a deterministic pass/fail substitute: the deterministic
+assessment remains the criterion, and the judge records the documentation
+outcome and utility. A campaign that declares a judge is not refused.
+
+A selection is accepted only through one checked reader shared by judge,
+status, closure and reporting: attempt set, packet bytes, policy hash,
+judgement hash, outcome/utility and the judged mutation state are re-derived.
+Slot and attempt packet bytes are hashed and compared with the registered
+identity and with the packet reconstructed from current evidence.
+Judge and recovery claims live under `campaign root/evidence`; a recovery
+settles both, and an interrupted claim stays visible and blocks closure. The
+judge's standing-permission block comes from the pristine prepared workspace,
+never the trial's live `AGENTS.md`. Before dispatch, the executing artifact
+hash is compared with the freeze, derived from the executing module's own
+`src`/`dist` tree, so changed code under the same package version is refused.
+
+## Sandbox limits
+
+The container is read-only apart from the trial workspace, `/tmp` and (for
+docs cases) the journal directory. Capabilities are dropped and
+`no-new-privileges` is set. With `network: "host"` the trial shares the host
+network so a loopback provider proxy is reachable; that is **not** network
+isolation and OTC does not describe it as provider-only egress. Credentials
+travel through a 0600 env file, never argv, logs or receipts.
+
+A product primary checker runs in a separate `network: "none"` container with
+only the completed workspace, the pristine workspace and the staged checker
+mounted read-only. The judge runs in another neutral container with no arm
+subjects, no campaign manifest and no evidence root.
+
+## Docs evaluator capability
+
+`src/evaluators/docs` carries the established docs analysis semantics, not a
+narrower recording-only criterion: recording credit, utility and opportunity
+counts, retrieval, preservation, guardrails, pairing/bounds, screening,
+decision and reporting. `docs/capability-matrix.md` maps every capability to
+its original implementation, tests, parity evidence and remaining owner.
+
+## Historical compatibility
+
+`src/legacy/prod014.ts` translates the frozen prod014 schedule and manifest
+into the new vocabulary without renaming a stamp or scenario id. The old
+campaign roots are not resumed by this engine; use a fresh root and keep the
+original for provenance.
+
+## Development
+
+See [AGENTS.md](AGENTS.md) and [TESTING.md](TESTING.md). The offline real
+rehearsal is separate from `npm test`:
+
+```sh
+npm run rehearse:offline
+npm run rehearse:fresh-docs
+npm run rehearse:installed
+```
+
+The second rehearsal exercises the complete fresh docs lifecycle: primary
+check, blind judge, one recovery, an exhausted budget, checked selection,
+closure, analysis and post-close refusal. The third packs and installs the
+tarball and repeats the same lifecycle through the installed binary; see
+[TESTING.md](TESTING.md).

package/container/Containerfile

@@ -0,0 +1,24 @@
+# OTC sandbox image.
+#
+# Both inputs are explicit build arguments: the Node base and the OpenCode CLI
+# version. The recipe does not float. A campaign freezes the resolved image ID
+# and the CLI version it contains before any dispatch.
+FROM ${NODE_BASE}
+
+ARG HOST_UID=1000
+ARG HOST_GID=1000
+ARG OPENCODE_VERSION
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends ca-certificates git \
+    && rm -rf /var/lib/apt/lists/* \
+    && test -n "${OPENCODE_VERSION}" \
+    && npm install -g --silent "@opencode/cli@${OPENCODE_VERSION}" \
+    && mkdir -p /workspace /tmp/agent-home \
+    && chown -R "${HOST_UID}:${HOST_GID}" /workspace /tmp/agent-home
+
+USER ${HOST_UID}:${HOST_GID}
+WORKDIR /workspace
+ENV HOME=/tmp/agent-home

package/container/README.md

@@ -0,0 +1,29 @@
+# OTC container
+
+`Containerfile` builds the sandbox image. Both floating inputs are explicit:
+
+```sh
+podman build \
+  --build-arg NODE_BASE=docker.io/library/node:26.13.0-slim \
+  --build-arg OPENCODE_VERSION=2.2.6 \
+  --build-arg HOST_UID="$(id -u)" \
+  --build-arg HOST_GID="$(id -g)" \
+  -t localhost/otc-sandbox:2.2.6 .
+```
+
+`otc freeze` resolves the tag to an immutable image ID and proves the image
+contains the declared OpenCode version. Campaigns then dispatch the ID, never
+a mutable tag.
+
+## Isolation, stated honestly
+
+The container has a read-only root filesystem, a bounded `/tmp` tmpfs, dropped
+capabilities, no-new-privileges and `--userns=keep-id`. Only the trial
+workspace, the read-only config home and the read-only subject payload are
+mounted.
+
+It shares the host network when `runtime.network` is `host`, because the
+existing provider route lives on the host loopback proxy. That is **not**
+provider-only egress and OTC does not claim it is. Use `network: "none"` when
+the trial needs no provider, or add network enforcement before claiming a
+provider-only boundary.

package/dist/adapters/opencode/events.d.ts

@@ -0,0 +1,90 @@
+/** One deduplicated tool call decoded from the CLI event stream. */
+export interface ToolCall {
+    /** The id value. */
+    id: string;
+    /** The tool value. */
+    tool: string | null;
+    /** The status value. */
+    status: string | null;
+    /** The input value. */
+    input: unknown;
+    /** The output value. */
+    output: string | null;
+    /** The metadata value. */
+    metadata: unknown;
+}
+/** Token and cost usage folded from `step_finish` events. */
+export interface Usage {
+    /** The input tokens value. */
+    inputTokens: number;
+    /** The output tokens value. */
+    outputTokens: number;
+    /** The reasoning tokens value. */
+    reasoningTokens: number;
+    /** The cache read tokens value. */
+    cacheReadTokens: number;
+    /** The cache write tokens value. */
+    cacheWriteTokens: number;
+    /** The cost usd value. */
+    costUsd: number;
+    /** Whether every usage value came from a recorded step-finish event. */
+    reported: boolean;
+    /** The steps value. */
+    steps: number;
+}
+/** A decoded trial event stream. */
+export interface DecodedTrail {
+    /** Parsed events. */
+    events: number;
+    /** Non-empty lines that were not JSON objects. */
+    parseFailures: number;
+    /** The concatenated answer text, newline-terminated. */
+    text: string;
+    /** Number of text parts observed. */
+    textParts: number;
+    /** The last part kind observed, or null. */
+    lastPartKind: "text" | "tool" | null;
+    /** Deduplicated calls in first-seen order. */
+    calls: ToolCall[];
+    /** Session identities observed. */
+    sessions: string[];
+    /** Message identities observed. */
+    messages: string[];
+    /** Known usage, with `reported` false when no step-finish event existed. */
+    usage: Usage;
+}
+/**
+ * Decode `opencode run --format json` JSONL into answer, calls, identity and
+ * usage. Malformed lines are counted, never treated as an empty trail.
+ * @param stdout Raw CLI stdout.
+ * @returns The decoded trail.
+ */
+export declare function decodeTrail(stdout: string): DecodedTrail;
+/** Whether a decoded stream is a complete, readable observation. */
+export declare function captureProblems(trail: DecodedTrail): string[];
+/** One skill tool call observed in a trail. */
+export interface SkillCall {
+    /** The call id value. */
+    callId: string;
+    /** The requested skill identity, when the input names one. */
+    requestedId: string | null;
+    /** The status value. */
+    status: string | null;
+    /** Whether the call completed with a non-empty skill body. */
+    loaded: boolean;
+}
+/**
+ * Every skill invocation in a trail, including other skills and failed loads.
+ * @param trail The decoded trial.
+ * @returns Skill calls in first-seen order.
+ */
+export declare function skillCalls(trail: DecodedTrail): SkillCall[];
+/**
+ * Whether a non-skill tool route names the target skill body. Activation
+ * through the skill tool is the criterion; a `read`, `grep`, `glob` or
+ * `execute` route is contamination evidence, not a successful activation.
+ * @param trail The decoded trial.
+ * @param targetSkillId The target identity.
+ * @returns The tool names that touched the target body, deduplicated.
+ */
+export declare function bodyRoutes(trail: DecodedTrail, targetSkillId: string): string[];

package/dist/adapters/opencode/events.js

@@ -0,0 +1,213 @@
+import { isRecord } from "../../contracts/validate.js";
+/** The empty usage accumulator. */
+function emptyUsage() {
+    return {
+        inputTokens: 0,
+        outputTokens: 0,
+        reasoningTokens: 0,
+        cacheReadTokens: 0,
+        cacheWriteTokens: 0,
+        costUsd: 0,
+        reported: false,
+        steps: 0,
+    };
+}
+/** Parse one JSONL line, or undefined when it is not an object. */
+function parseLine(line) {
+    try {
+        const value = JSON.parse(line);
+        return isRecord(value) ? value : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Fold one event's identity ids into the trail. */
+function foldIdentity(event, sessions, messages) {
+    if (typeof event.sessionID === "string")
+        sessions.add(event.sessionID);
+    if (typeof event.messageID === "string")
+        messages.add(event.messageID);
+    if (!isRecord(event.part))
+        return;
+    if (typeof event.part.sessionID === "string") {
+        sessions.add(event.part.sessionID);
+    }
+    if (typeof event.part.messageID === "string") {
+        messages.add(event.part.messageID);
+    }
+}
+/** The first defined value, or null. */
+function firstDefined(...values) {
+    for (const value of values) {
+        if (value !== undefined)
+            return value;
+    }
+    return null;
+}
+/** Merge one latest-state tool part over its previous call. */
+function mergedToolCall(previous, part, id) {
+    const state = isRecord(part.state)
+        ? part.state
+        : {};
+    const prev = previous ?? {
+        id,
+        tool: null,
+        status: null,
+        input: null,
+        output: null,
+        metadata: null,
+    };
+    return {
+        id,
+        tool: firstDefined(part.tool, prev.tool),
+        status: firstDefined(state.status, prev.status),
+        input: firstDefined(state.input, prev.input),
+        output: firstDefined(state.output, prev.output),
+        metadata: firstDefined(state.metadata, prev.metadata),
+    };
+}
+/** Fold one tool part over the previous call state. */
+function foldToolPart(part, calls, unkeyed) {
+    const id = typeof part.id === "string" && part.id !== ""
+        ? part.id
+        : `unkeyed-${++unkeyed.count}`;
+    calls.set(id, mergedToolCall(calls.get(id), part, id));
+}
+/** Fold one step-finish event's usage into the accumulator. */
+function foldUsage(event, usage) {
+    const part = isRecord(event.part) ? event.part : undefined;
+    if (part?.type !== "step-finish" || !isRecord(part.tokens))
+        return;
+    const tokens = part.tokens;
+    const token = (field) => typeof tokens[field] === "number" ? tokens[field] : 0;
+    const cache = isRecord(tokens.cache) ? tokens.cache : {};
+    usage.inputTokens += token("input");
+    usage.outputTokens += token("output");
+    usage.reasoningTokens += token("reasoning");
+    usage.cacheReadTokens += typeof cache.read === "number" ? cache.read : 0;
+    usage.cacheWriteTokens += typeof cache.write === "number" ? cache.write : 0;
+    usage.costUsd += typeof part.cost === "number" ? part.cost : 0;
+    usage.reported = true;
+    usage.steps += 1;
+}
+/**
+ * Decode `opencode run --format json` JSONL into answer, calls, identity and
+ * usage. Malformed lines are counted, never treated as an empty trail.
+ * @param stdout Raw CLI stdout.
+ * @returns The decoded trail.
+ */
+export function decodeTrail(stdout) {
+    const texts = [];
+    const calls = new Map();
+    const sessions = new Set();
+    const messages = new Set();
+    const usage = emptyUsage();
+    const unkeyed = { count: 0 };
+    let events = 0;
+    let parseFailures = 0;
+    let textParts = 0;
+    let lastPartKind = null;
+    for (const line of stdout.split("\n").filter((entry) => entry !== "")) {
+        const event = parseLine(line);
+        if (event === undefined) {
+            parseFailures += 1;
+            continue;
+        }
+        events += 1;
+        foldIdentity(event, sessions, messages);
+        foldUsage(event, usage);
+        const part = isRecord(event.part) ? event.part : event;
+        if (part.type === "text" && typeof part.text === "string") {
+            texts.push(part.text);
+            textParts += 1;
+            lastPartKind = "text";
+            continue;
+        }
+        if (part.type === "tool") {
+            foldToolPart(part, calls, unkeyed);
+            lastPartKind = "tool";
+        }
+    }
+    return {
+        events,
+        parseFailures,
+        text: `${texts.join("\n").trim()}\n`,
+        textParts,
+        lastPartKind,
+        calls: [...calls.values()],
+        sessions: [...sessions],
+        messages: [...messages],
+        usage,
+    };
+}
+/** Whether a decoded stream is a complete, readable observation. */
+export function captureProblems(trail) {
+    const problems = [];
+    if (trail.events === 0)
+        problems.push("the trial emitted no events");
+    if (trail.parseFailures > 0) {
+        problems.push(`${trail.parseFailures} event line(s) were not readable JSON`);
+    }
+    if (trail.textParts === 0) {
+        problems.push("the trial emitted no answer text");
+    }
+    if (trail.lastPartKind !== "text") {
+        problems.push("the trial stream did not end on an answer");
+    }
+    return problems;
+}
+/** Fold the input/output of one skill call into its observation. */
+function skillCallOf(call) {
+    if (call.tool !== "skill")
+        return null;
+    const input = isRecord(call.input) ? call.input : {};
+    const requested = typeof input.id === "string"
+        ? input.id
+        : typeof input.name === "string"
+            ? input.name
+            : null;
+    const output = call.output?.trim() ?? "";
+    return {
+        callId: call.id,
+        requestedId: requested,
+        status: call.status,
+        loaded: call.status === "completed" &&
+            output !== "" &&
+            !output.startsWith("Rejected:"),
+    };
+}
+/**
+ * Every skill invocation in a trail, including other skills and failed loads.
+ * @param trail The decoded trial.
+ * @returns Skill calls in first-seen order.
+ */
+export function skillCalls(trail) {
+    return trail.calls
+        .map(skillCallOf)
+        .filter((call) => call !== null);
+}
+/**
+ * Whether a non-skill tool route names the target skill body. Activation
+ * through the skill tool is the criterion; a `read`, `grep`, `glob` or
+ * `execute` route is contamination evidence, not a successful activation.
+ * @param trail The decoded trial.
+ * @param targetSkillId The target identity.
+ * @returns The tool names that touched the target body, deduplicated.
+ */
+export function bodyRoutes(trail, targetSkillId) {
+    const routes = new Set();
+    const needles = [
+        `skills/${targetSkillId}/SKILL.md`,
+        `skill://${targetSkillId}`,
+    ];
+    for (const call of trail.calls) {
+        if (call.tool === "skill")
+            continue;
+        const text = JSON.stringify(call.input ?? null);
+        if (needles.some((needle) => text.includes(needle))) {
+            routes.add(call.tool ?? "unknown");
+        }
+    }
+    return [...routes].sort();
+}