@markstanden/opencode-test-container 0.0.0-stage → 0.1.1

package/dist/evaluators/docs/campaign.js

@@ -0,0 +1,222 @@
+/**
+ * Pure primitives for the arm-blind automatic-notetaking campaign.
+ *
+ * These are the parts that must be identical across every review and rerun:
+ * the seeded schedule, arm-to-directory resolution, the documentation-change
+ * classification, the packet label guard and the tolerant JSON extraction.
+ * They are pure so a test can pin them without spawning anything.
+ */
+import { blindingFindings } from "./analysis/blinding.js";
+/** The frozen schedule seed for the 2026-11-10 six-arm campaign. */
+export const CAMPAIGN_SEED = 20261110;
+/** Native-description arms, in the frozen arm-id order. */
+export const ARMS = ["A", "B", "C", "D", "E", "F"];
+/** Screening scenarios (the four NOTE cases). */
+export const NOTE_CASES = ["NOTE-01", "NOTE-02", "NOTE-03", "NOTE-04"];
+/** Confirmation scenarios, sealed for the 2026-11-10 campaign only. */
+export const HELD_OUT = ["H01", "H02", "H03", "H04"];
+/**
+ * A deterministic 32-bit PRNG (mulberry32), so a schedule is reproducible
+ * from its seed alone.
+ * @param seed The integer seed.
+ * @returns A function yielding successive floats in [0, 1).
+ */
+export function mulberry32(seed) {
+    let state = seed >>> 0;
+    return () => {
+        state = (state + 0x6d2b79f5) >>> 0;
+        let t = state;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+/**
+ * Fisher–Yates shuffle with an injected random source.
+ * @param random The random source.
+ * @param items The items to shuffle.
+ * @returns A shuffled copy.
+ */
+export function shuffle(random, items) {
+    const copy = [...items];
+    for (let index = copy.length - 1; index > 0; index -= 1) {
+        const swap = Math.floor(random() * (index + 1));
+        const carried = copy[index];
+        copy[index] = copy[swap];
+        copy[swap] = carried;
+    }
+    return copy;
+}
+/**
+ * Rotate a list left by `by` positions.
+ * @param items The items.
+ * @param by The rotation offset.
+ * @returns The rotated list.
+ */
+export function rotate(items, by) {
+    return items.map((_, index) => items[(index + by) % items.length]);
+}
+/**
+ * Build both frozen stages of the campaign schedule.
+ *
+ * Screening runs every arm once in a seeded order inside each
+ * scenario/repetition block, with scenario order rotated across repetitions.
+ * Confirmation alternates the finalist/control pair on (scenario index +
+ * repetition) mod 2 and rotates scenario order by repetition.
+ * @param seed The schedule seed; the campaign froze 20261110.
+ * @returns The screening and confirmation slot lists.
+ */
+export function buildSchedule(seed = CAMPAIGN_SEED) {
+    const random = mulberry32(seed);
+    const screening = [];
+    let slot = 0;
+    for (let repetition = 1; repetition <= 2; repetition += 1) {
+        for (const scenario of rotate(NOTE_CASES, repetition - 1)) {
+            for (const arm of shuffle(random, ARMS)) {
+                slot += 1;
+                screening.push({
+                    stage: "screening",
+                    slot,
+                    block: `${repetition}-${scenario}`,
+                    repetition,
+                    scenario,
+                    arm,
+                    stamp: `fr-${arm}-s${repetition}-${scenario}`,
+                });
+            }
+        }
+    }
+    const confirmation = [];
+    for (let repetition = 0; repetition <= 4; repetition += 1) {
+        for (const scenario of rotate(HELD_OUT, repetition)) {
+            const scenarioIndex = HELD_OUT.indexOf(scenario);
+            const roles = (scenarioIndex + repetition) % 2 === 0
+                ? ["F", "A"]
+                : ["A", "F"];
+            for (const role of roles) {
+                slot += 1;
+                confirmation.push({
+                    stage: "confirmation",
+                    slot,
+                    block: `${repetition}-${scenario}`,
+                    repetition,
+                    scenario,
+                    role,
+                    stamp: `fc-${role}-r${repetition}-${scenario}`,
+                });
+            }
+        }
+    }
+    return { seed, screening, confirmation };
+}
+/**
+ * Resolve one arm letter to its external worktree directory.
+ *
+ * The arm letter is case-insensitive and must belong to the frozen set; a
+ * runner that silently accepted an unknown letter or mixed case in a path is
+ * exactly the defect that invalidated 20 confirmation slots in the original
+ * campaign.
+ * @param root The campaign root holding the arm worktrees.
+ * @param arm The arm letter, either case.
+ * @returns The arm's absolute-ish directory path.
+ */
+export function armDir(root, arm) {
+    const letter = String(arm).toLowerCase();
+    if (!/^[a-f]$/.test(letter)) {
+        throw new Error(`unknown arm ${JSON.stringify(arm)}: expected A-F`);
+    }
+    return `${root}/arms/${letter}`;
+}
+/**
+ * Classify documentation changes between two knowledge-lake snapshots.
+ * @param beforeDocs Markdown documents before the trial ({path, contents}).
+ * @param afterDocs Markdown documents after the trial ({path, contents}).
+ * @returns The created, modified and removed paths and a change flag.
+ */
+export function docChanges(beforeDocs, afterDocs) {
+    const before = new Map(beforeDocs.map((doc) => [doc.path, doc.contents]));
+    const after = new Map(afterDocs.map((doc) => [doc.path, doc.contents]));
+    const createdPaths = [...after.keys()].filter((path) => !before.has(path));
+    const modifiedPaths = [...after.keys()].filter((path) => before.has(path) && before.get(path) !== after.get(path));
+    const removedPaths = [...before.keys()].filter((path) => !after.has(path));
+    return {
+        documentsChanged: createdPaths.length > 0 ||
+            modifiedPaths.length > 0 ||
+            removedPaths.length > 0,
+        createdPaths,
+        modifiedPaths,
+        removedPaths,
+    };
+}
+/**
+ * Patterns that must never reach a judge: campaign and host paths, and the
+ * words that frame an experiment. Arm and stamp detection is shared with the
+ * trial boundary through `blindingFindings`.
+ */
+export const LABEL_PATTERNS = [
+    /docs-desc-fresh/,
+    /recording-confirm/,
+    /\/home\/[^/\s]+/,
+    /fr-[A-F]-[sr]\d/,
+    /fc-[AF]-r\d/,
+    /cc-[AB]-r\d/,
+    /(?:^|\W)arm[_ -]?[A-F](?:\W|$)/i,
+    /candidate/i,
+    /control arm/i,
+];
+/**
+ * Refuse a judge request that carries any experiment label.
+ * @param surfaces Every judge-bound text surface: packet, neutral paths, the
+ *   request context. No surface is exempt.
+ * @returns The matched pattern sources; empty means arm-blind.
+ */
+export function labelLeaks(...surfaces) {
+    const text = surfaces.join("\n");
+    const generic = LABEL_PATTERNS.filter((pattern) => pattern.test(text)).map((pattern) => String(pattern));
+    const arm = blindingFindings("judge", text).map((finding) => finding.fragment);
+    return [...new Set([...generic, ...arm])];
+}
+/**
+ * Extract the first balanced JSON object from text, string- and
+ * escape-aware, so trailing prose containing braces cannot corrupt the parse.
+ * @param text The judge answer.
+ * @returns The raw object text, or undefined when there is none.
+ */
+export function firstBalancedObject(text) {
+    const start = text.indexOf("{");
+    if (start === -1)
+        return undefined;
+    let depth = 0;
+    let inString = false;
+    let escaped = false;
+    for (let index = start; index < text.length; index += 1) {
+        const char = text[index];
+        if (inString) {
+            ({ inString, escaped } = stepInString(char, escaped));
+            continue;
+        }
+        if (char === '"')
+            inString = true;
+        else if (char === "{")
+            depth += 1;
+        else if (char === "}") {
+            depth -= 1;
+            if (depth === 0)
+                return text.slice(start, index + 1);
+        }
+    }
+    return undefined;
+}
+/**
+ * Advance the string-literal state by one character.
+ * @param char The current character.
+ * @param escaped Whether the previous character was an escape.
+ * @returns The next string and escape state.
+ */
+function stepInString(char, escaped) {
+    if (escaped)
+        return { inString: true, escaped: false };
+    if (char === "\\")
+        return { inString: true, escaped: true };
+    return { inString: char !== '"', escaped: false };
+}

package/dist/evaluators/docs/index.d.ts

@@ -0,0 +1,25 @@
+/**
+ * The docs evaluator: checked recording, utility, retrieval, preservation,
+ * guardrails, pairing, bounds, screening, decision and rendering semantics.
+ *
+ * These modules are the established analysis implementation, migrated from
+ * the docs plugin with their tests. They consume recorded evidence only; none
+ * of them calls a model.
+ */
+export { runAnalysis } from "./analysis/replay.ts";
+export * from "./analysis/accounting.ts";
+export * from "./analysis/bounds.ts";
+export * from "./analysis/decision.ts";
+export * from "./analysis/guardrails.ts";
+export * from "./analysis/measures.ts";
+export * from "./analysis/model.ts";
+export * from "./analysis/pairing.ts";
+export * from "./analysis/records.ts";
+export * from "./analysis/render.ts";
+export * from "./analysis/screening/rules.ts";
+export * from "./analysis/screening/scan.ts";
+export * from "./analysis/screening/inventory.ts";
+export * from "./analysis/screening/reconcile.ts";
+export { DOCUMENTATION_OUTCOMES, PRESERVATION_CASE, buildAcPrompt, finalistQualifies, judgeFailureClass, parseAcJudgement, rankCandidates, sensitivity, summariseArm, validateAcJudgement, type DocumentationOutcome, type JudgeFailureClass, type PreservationState, } from "./scoring.ts";
+export { assessDocsTrial } from "./assess.ts";
+export { verifyOperationsJournal } from "./journal.ts";

package/dist/evaluators/docs/index.js

@@ -0,0 +1,25 @@
+/**
+ * The docs evaluator: checked recording, utility, retrieval, preservation,
+ * guardrails, pairing, bounds, screening, decision and rendering semantics.
+ *
+ * These modules are the established analysis implementation, migrated from
+ * the docs plugin with their tests. They consume recorded evidence only; none
+ * of them calls a model.
+ */
+export { runAnalysis } from "./analysis/replay.js";
+export * from "./analysis/accounting.js";
+export * from "./analysis/bounds.js";
+export * from "./analysis/decision.js";
+export * from "./analysis/guardrails.js";
+export * from "./analysis/measures.js";
+export * from "./analysis/model.js";
+export * from "./analysis/pairing.js";
+export * from "./analysis/records.js";
+export * from "./analysis/render.js";
+export * from "./analysis/screening/rules.js";
+export * from "./analysis/screening/scan.js";
+export * from "./analysis/screening/inventory.js";
+export * from "./analysis/screening/reconcile.js";
+export { DOCUMENTATION_OUTCOMES, PRESERVATION_CASE, buildAcPrompt, finalistQualifies, judgeFailureClass, parseAcJudgement, rankCandidates, sensitivity, summariseArm, validateAcJudgement, } from "./scoring.js";
+export { assessDocsTrial } from "./assess.js";
+export { verifyOperationsJournal } from "./journal.js";

package/dist/evaluators/docs/journal.d.ts

@@ -0,0 +1,150 @@
+/**
+ * The one checked evidence boundary for the plugin's operation journal.
+ *
+ * The plugin can write an opt-in, HMAC-signed journal of served docs
+ * operations. That journal is the only record of a nested Code Mode call, so
+ * it is also the easiest evidence to forge. This module decides what the
+ * journal proves, and nothing else in the harness or analyser may credit a
+ * journal line it did not verify.
+ *
+ * Trust boundary: the signing key and the journal are both harness output,
+ * copied after the trial process exits. A trial tool cannot acquire the key
+ * during the run, so a correctly signed line with this run's session, message
+ * and execute identities, matching a host-recorded dispatch one-to-one, is
+ * evidence the plugin served that call. The key is held locally beside the
+ * evidence; possession of that key is possession of the ability to forge, so
+ * the verification is tamper-evident, not tamper-proof.
+ */
+/** The session and message identities observed in one run's CLI trail. */
+export interface JournalIdentity {
+    /** Session ids the run emitted. */
+    sessions: ReadonlySet<string>;
+    /** Message ids the run emitted. */
+    messages: ReadonlySet<string>;
+}
+/** One docs call the host recorded, native or nested. */
+export interface JournalDispatch {
+    /** The journal's registered tool name (`docs` or `docs_search`). */
+    tool: string;
+    /** The dispatch status the host recorded. */
+    status: string;
+    /** The dispatch input the host recorded. */
+    input: unknown;
+    /** The host call id (native) or parent execute id (nested). */
+    callId: string | null;
+}
+/** One credited journal line. */
+export interface CreditedJournalRecord {
+    /** The zero-based line index. */
+    index: number;
+    /** The verified record, including its MAC. */
+    record: Record<string, unknown>;
+}
+/** One rejected journal line and why. */
+export interface RejectedJournalRecord {
+    /** The zero-based line index. */
+    index: number;
+    /** A stable reason slug, suitable for diagnostics. */
+    reason: "not-json" | "not-served-twin" | "bad-mac" | "unattributed" | "no-host-dispatch";
+}
+/** The explicit result of verifying a journal. */
+export type JournalVerification = {
+    /** No journal was captured: the pre-instrumentation status quo. */
+    state: "absent";
+    credited: [];
+    rejected: [];
+    problems: string[];
+} | {
+    /** A journal exists but its signing key does not. Nothing is credited. */
+    state: "unverifiable-key";
+    credited: [];
+    rejected: RejectedJournalRecord[];
+    problems: string[];
+} | {
+    /** The journal was verified; `credited` is the only usable evidence. */
+    state: "verified";
+    credited: CreditedJournalRecord[];
+    rejected: RejectedJournalRecord[];
+    problems: string[];
+};
+/** Whether a record names a served docs call on either registration surface. */
+export declare function isServedRecord(record: Record<string, unknown>): boolean;
+/**
+ * Whether a record's MAC matches the given key. The MAC covers every field
+ * except `mac`, in the record's own member order, exactly as the plugin writes
+ * it.
+ * @param record The journal record.
+ * @param key The run's signing key.
+ * @returns True when the MAC is well formed and matches.
+ */
+export declare function journalMacValid(record: Record<string, unknown>, key: string): boolean;
+/**
+ * Whether a record carries this run's observed identity and names an execute
+ * call present in its trail.
+ * @param record The journal record.
+ * @param identity The run's observed session/message ids.
+ * @param callIds The execute call ids in the run's trail.
+ * @returns True when every identity matches exactly.
+ */
+export declare function journalRecordAttributed(record: Record<string, unknown>, identity: JournalIdentity, callIds: ReadonlySet<string>): boolean;
+/**
+ * Verify one captured journal against this run's identity and host dispatches.
+ *
+ * Order matters: shape, then MAC, then identity, then one-to-one dispatch
+ * matching. A line failing any step is rejected with its reason and is never
+ * credited. If a journal exists but no usable key does, nothing is credited
+ * and the state says why.
+ * @param input The captured lines, key and run observations.
+ * @param input.lines The captured journal lines, or undefined when absent.
+ * @param input.key The run's signing key, or null when unavailable.
+ * @param input.identity The run's observed session and message ids.
+ * @param input.callIds The execute call ids in the run's trail.
+ * @param input.nativeCalls The docs calls the host recorded directly.
+ * @param input.dispatches The nested docs dispatches the host recorded.
+ * @returns The explicit verification result.
+ */
+export declare function verifyOperationsJournal(input: {
+    lines: readonly string[] | undefined;
+    key: string | null | undefined;
+    identity: JournalIdentity;
+    callIds: ReadonlySet<string>;
+    nativeCalls?: readonly JournalDispatch[];
+    dispatches: readonly JournalDispatch[];
+}): JournalVerification;
+/**
+ * Collect the session and message identities a saved CLI trail observed.
+ * @param stdoutText The saved `stdout.jsonl` bytes.
+ * @returns The identity sets; empty when the trail is unreadable.
+ */
+export declare function journalIdentityFromTrail(stdoutText: string): JournalIdentity;
+/** The host half of journal attribution, read from a saved tools trail. */
+export interface HostDispatches {
+    /** Host call ids present in the trail, native and execute. */
+    callIds: Set<string>;
+    /** Native docs calls the host recorded directly. */
+    nativeCalls: JournalDispatch[];
+    /** Nested docs dispatches the host recorded inside execute results. */
+    dispatches: JournalDispatch[];
+}
+/**
+ * Extract the host dispatches and execute call ids from a saved `tools.jsonl`.
+ * @param toolsText The saved tools trail bytes.
+ * @returns The host half of attribution; empty when the trail is unreadable.
+ */
+export declare function hostDispatchesFromTrail(toolsText: string): HostDispatches;
+/**
+ * Check a captured answer against the producer's `meta.answerHash`.
+ *
+ * The harness hashes the exact final-answer text as bytes (`sha256(text)`),
+ * so the analyser must hash the recorded `final.md` bytes, never a normalised
+ * rendering. A missing hash leaves the answer unbound rather than trusted.
+ * @param answer The recorded answer bytes.
+ * @param answerHash The producer's recorded hash, when present.
+ * @param settings The optional binding settings.
+ * @param settings.allowUnbound Explicit read-only coverage for legacy evidence
+ *   written before the producer recorded a hash.
+ * @returns A problem, or null when the answer is bound.
+ */
+export declare function answerIdentityProblem(answer: string, answerHash: unknown, settings?: {
+    allowUnbound?: boolean;
+}): string | null;