@markstanden/opencode-test-container 0.0.0-stage → 0.1.1

package/dist/evaluators/docs/assess.js

@@ -0,0 +1,433 @@
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { sha256Bytes } from "../../contracts/canonical.js";
+import { isRecord } from "../../contracts/validate.js";
+import { decodeTrail } from "../../adapters/opencode/events.js";
+import { hostDispatchesFromTrail, journalIdentityFromTrail, verifyOperationsJournal, } from "./journal.js";
+import { readPrimaryCheck, } from "../../campaign/primary.js";
+/** Determine which docs verb one host input names. */
+export function docsVerbOf(input) {
+    if (!isRecord(input))
+        return "other";
+    if (input.add !== undefined)
+        return "add";
+    if (input.search !== undefined)
+        return "search";
+    if (input.read !== undefined)
+        return "read";
+    if (input.check !== undefined)
+        return "check";
+    if (input.help !== undefined)
+        return "help";
+    return "other";
+}
+/** The metadata a served docs result carried. */
+function resultMetadata(record) {
+    const result = record.result;
+    return isRecord(result) && isRecord(result.metadata) ? result.metadata : {};
+}
+/** Whether a credited record proves a completed write and where. */
+export function creditedWriteOf(record) {
+    if (resultMetadata(record).written !== true)
+        return null;
+    const metadata = resultMetadata(record);
+    const input = isRecord(record.input) ? record.input : {};
+    const add = input.add;
+    const mode = typeof add === "string"
+        ? "create"
+        : isRecord(add) && typeof add.mode === "string"
+            ? add.mode
+            : null;
+    return {
+        path: typeof metadata.path === "string" ? metadata.path : null,
+        mode,
+    };
+}
+/** Read meta without trusting it. */
+function readMeta(path) {
+    if (!existsSync(path))
+        return null;
+    try {
+        const value = JSON.parse(readFileSync(path, "utf8"));
+        return isRecord(value) ? value : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Read usage without trusting it. */
+function readUsage(path) {
+    if (!existsSync(path))
+        return null;
+    try {
+        const value = JSON.parse(readFileSync(path, "utf8"));
+        return isRecord(value) ? value : null;
+    }
+    catch {
+        return null;
+    }
+}
+/** Read the captured journal lines, or undefined when no file was written. */
+function journalLines(path) {
+    if (!existsSync(path))
+        return undefined;
+    return readFileSync(path, "utf8").split("\n");
+}
+/** Read the per-run signing key, or null when unavailable. */
+function journalKey(path) {
+    if (!existsSync(path))
+        return null;
+    const value = readFileSync(path, "utf8").trim();
+    return value === "" ? null : value;
+}
+/** Check retained and omitted text in one written file. */
+function textReasons(target, text, testCase) {
+    const retained = (testCase.expectedDocs.retains ?? []).flatMap((value) => text.includes(value)
+        ? []
+        : [`${target} does not retain ${JSON.stringify(value)}`]);
+    const omitted = (testCase.expectedDocs.omits ?? []).flatMap((value) => text.includes(value)
+        ? [`${target} contains forbidden ${JSON.stringify(value)}`]
+        : []);
+    const link = testCase.expectedDocs.link;
+    return [
+        ...retained,
+        ...omitted,
+        ...(link !== undefined && !text.includes(`](${link})`)
+            ? [`${target} does not link ${link}`]
+            : []),
+    ];
+}
+/** Check one written file's required and forbidden text. */
+function contentReasons(workspace, testCase) {
+    const target = testCase.expectedDocs.target;
+    if (target === undefined)
+        return [];
+    const path = join(workspace, testCase.lakeRoot ?? "docs", target);
+    if (!existsSync(path))
+        return [`expected written file ${target} is missing`];
+    return textReasons(target, readFileSync(path, "utf8"), testCase);
+}
+/** Check paths that must not exist. */
+function absentReasons(workspace, testCase) {
+    return (testCase.guardrails?.absent ?? []).flatMap((path) => existsSync(join(workspace, path))
+        ? [`guardrail path ${path} exists`]
+        : []);
+}
+/** Check paths whose prepared identity must survive unchanged. */
+function unchangedReasons(workspace, testCase, prepared) {
+    return (testCase.guardrails?.unchanged ?? []).flatMap((path) => {
+        const expected = prepared.get(path);
+        if (expected === undefined) {
+            return [`guardrail path ${path} has no prepared identity`];
+        }
+        const current = join(workspace, path);
+        if (!existsSync(current))
+            return [`guardrail path ${path} was removed`];
+        return sha256Bytes(readFileSync(current)) === expected
+            ? []
+            : [`guardrail path ${path} changed`];
+    });
+}
+/** Check guardrails against the final workspace and prepared inventory. */
+function guardrailReasons(workspace, testCase, prepared) {
+    return [
+        ...absentReasons(workspace, testCase),
+        ...unchangedReasons(workspace, testCase, prepared),
+    ];
+}
+/** The readable parts of one docs operation input. */
+function operationInputParts(input) {
+    const parts = [];
+    if (typeof input.search === "string")
+        parts.push(`search "${input.search}"`);
+    if (typeof input.read === "string")
+        parts.push(`read ${input.read}`);
+    if (typeof input.check === "string")
+        parts.push(`check ${input.check}`);
+    const add = input.add;
+    if (typeof add === "object" && add !== null) {
+        const declaration = add;
+        parts.push(`${String(declaration.mode ?? "create")} ${String(declaration.path ?? "(body)")}`);
+    }
+    return parts;
+}
+/** One verified docs operation, rendered for the judge. */
+function operationLine(entry) {
+    const surface = entry.surface === "codemode-twin" ? "twin" : "native";
+    const input = typeof entry.input === "object" && entry.input !== null
+        ? entry.input
+        : {};
+    const parts = operationInputParts(input);
+    return `${String(entry.status ?? "unknown")} ${surface} ${parts.join("; ") || "help"}`;
+}
+/** Verify the captured journal against the host trail. */
+function verificationOf(trialDir, events, tools) {
+    const host = hostDispatchesFromTrail(tools);
+    return verifyOperationsJournal({
+        lines: journalLines(join(trialDir, "journal", "docs-operations.jsonl")),
+        key: journalKey(join(trialDir, "journal-key.txt")),
+        identity: journalIdentityFromTrail(events),
+        callIds: host.callIds,
+        nativeCalls: host.nativeCalls,
+        dispatches: host.dispatches,
+    });
+}
+/** Whether an absent journal contradicts the case's declared operation need. */
+function journalAbsenceProblem(testCase, verification, host) {
+    if (verification.state !== "absent") {
+        return verification.state === "verified"
+            ? null
+            : `docs journal is ${verification.state}; no operation is credited`;
+    }
+    const dispatched = host.nativeCalls.length + host.dispatches.length;
+    const expectsOperations = (testCase.criterion?.minVerifiedCalls ?? 0) > 0 ||
+        testCase.expectedDocs.kind !== "none";
+    return dispatched > 0 || expectsOperations
+        ? "docs journal is absent; no operation is credited"
+        : null;
+}
+/** Capture integrity problems for a docs trial. */
+function docsIntegrity(testCase, meta, verification, host) {
+    const integrity = [];
+    if (meta.captureValid !== true)
+        integrity.push(...(meta.captureProblems ?? []));
+    if (meta.exit !== 0)
+        integrity.push(`container exited ${String(meta.exit)}`);
+    const absence = journalAbsenceProblem(testCase, verification, host);
+    if (absence !== null)
+        integrity.push(absence);
+    if (verification.rejected.length > 0) {
+        integrity.push(`${verification.rejected.length} journal line(s) were rejected: ${verification.rejected
+            .map((entry) => entry.reason)
+            .join(", ")}`);
+    }
+    return integrity;
+}
+/** Why a declared primary check stops assessment before the criterion. */
+function primarySeparation(testCase, record) {
+    if (testCase.primaryCheck === undefined || record === null) {
+        return testCase.primaryCheck === undefined
+            ? null
+            : {
+                validity: "invalid",
+                reasons: [
+                    `primary check ${testCase.primaryCheck.id} has no recorded result`,
+                ],
+            };
+    }
+    return record.status === "timeout" || record.status === "error"
+        ? {
+            validity: "unknown",
+            reasons: [
+                `primary check ${record.id} ${record.status}: ${record.reason ?? record.stderr.trim()}`,
+            ],
+        }
+        : null;
+}
+/** A failed primary check adds one non-compensable criterion reason. */
+function primaryFailure(record) {
+    return record !== null && record.status === "fail"
+        ? [`primary check ${record.id} failed (exit ${String(record.exit)})`]
+        : [];
+}
+/** Apply the docs criterion to credited evidence and the final workspace. */
+function docsReasons(testCase, credited, writes, workspace, prepared) {
+    const criterion = testCase.criterion ?? { minVerifiedCalls: 0 };
+    const verbs = new Set(criterion.verbs ?? []);
+    const creditedForCriterion = credited.filter((record) => verbs.size === 0 || verbs.has(docsVerbOf(record.input))).length;
+    const reasons = [];
+    if (creditedForCriterion < criterion.minVerifiedCalls) {
+        reasons.push(`verified calls ${creditedForCriterion} below required ${criterion.minVerifiedCalls}`);
+    }
+    if (testCase.expectedDocs.kind === "none") {
+        if (writes.length > 0) {
+            reasons.push(`${writes.length} write(s) were credited; this case expects refusal`);
+        }
+    }
+    else {
+        const target = testCase.expectedDocs.target;
+        const match = writes.find((write) => write.path === target &&
+            (testCase.expectedDocs.kind === "create"
+                ? write.mode === "create" || write.mode === null
+                : write.mode === "replace"));
+        if (match === undefined) {
+            reasons.push(`no credited ${testCase.expectedDocs.kind} write to ${String(target)}`);
+        }
+        reasons.push(...contentReasons(workspace, testCase));
+    }
+    reasons.push(...guardrailReasons(workspace, testCase, prepared));
+    return reasons;
+}
+/**
+ * Assess one docs-recording trial from the signed journal and the uploaded
+ * workspace. The journal is the only route that credits a served operation;
+ * an attempted but refused write retains zero recording credit.
+ * @param options Trial identity, case definition, evidence and workspace.
+ * @returns The assessment.
+ * @param options.campaignId The campaignId value.
+ * @param options.slotId The slotId value.
+ * @param options.caseId The caseId value.
+ * @param options.armId The armId value.
+ * @param options.repetitionId The repetitionId value.
+ * @param options.testCase The testCase value.
+ * @param options.trialDir The trialDir value.
+ * @param options.workspace The workspace value.
+ * @param options.prepared The prepared value.
+ * @param options.attemptId The attemptId value.
+ * @param options.assessedAt The assessedAt value.
+ */
+export function assessDocsTrial(options) {
+    const { testCase, trialDir } = options;
+    const meta = readMeta(join(trialDir, "meta.json"));
+    const primaryRecord = readPrimaryCheck(trialDir);
+    const primary = primaryRecord === null
+        ? null
+        : {
+            id: primaryRecord.id,
+            status: primaryRecord.status,
+            exit: primaryRecord.exit,
+        };
+    const base = {
+        schemaVersion: 1,
+        kind: "docs-recording",
+        campaignId: options.campaignId,
+        slotId: options.slotId,
+        caseId: options.caseId,
+        armId: options.armId,
+        repetitionId: options.repetitionId,
+        attemptId: options.attemptId,
+        expected: testCase.expectedDocs.kind,
+    };
+    const empty = {
+        creditedCalls: 0,
+        creditedWrites: 0,
+        attemptedWrites: 0,
+        rejectedLines: 0,
+        searchCalls: 0,
+    };
+    if (meta === null || !existsSync(join(trialDir, "events.jsonl"))) {
+        return {
+            ...base,
+            validity: "unrun",
+            outcome: "unknown",
+            counts: empty,
+            journal: {
+                state: "absent",
+                credited: 0,
+                rejected: 0,
+                problems: ["trial evidence is missing"],
+            },
+            target: null,
+            primary,
+            reasons: ["trial evidence is missing"],
+            usage: null,
+            answered: false,
+            assessedAt: options.assessedAt,
+        };
+    }
+    const events = readFileSync(join(trialDir, "events.jsonl"), "utf8");
+    const trail = decodeTrail(events);
+    const tools = existsSync(join(trialDir, "tools.jsonl"))
+        ? readFileSync(join(trialDir, "tools.jsonl"), "utf8")
+        : "";
+    const host = hostDispatchesFromTrail(tools);
+    const verification = verificationOf(trialDir, events, tools);
+    const credited = verification.credited.map((entry) => entry.record);
+    const writes = credited
+        .map(creditedWriteOf)
+        .filter((entry) => entry !== null);
+    const counts = {
+        creditedCalls: credited.length,
+        creditedWrites: writes.length,
+        attemptedWrites: countAttemptedWrites(host, verification),
+        rejectedLines: verification.rejected.length,
+        searchCalls: credited.filter((record) => docsVerbOf(record.input) === "search").length,
+    };
+    const integrity = docsIntegrity(testCase, meta, verification, host);
+    if (integrity.length > 0) {
+        return {
+            ...base,
+            validity: "invalid",
+            outcome: "unknown",
+            counts,
+            journal: journalSummary(verification),
+            target: null,
+            primary,
+            reasons: integrity,
+            usage: readUsage(join(trialDir, "usage.json")),
+            answered: trail.text.trim() !== "",
+            assessedAt: options.assessedAt,
+        };
+    }
+    const separation = primarySeparation(testCase, primaryRecord);
+    if (separation !== null) {
+        return {
+            ...base,
+            validity: separation.validity,
+            outcome: "unknown",
+            counts,
+            journal: journalSummary(verification),
+            target: null,
+            primary,
+            reasons: separation.reasons,
+            usage: readUsage(join(trialDir, "usage.json")),
+            answered: trail.text.trim() !== "",
+            assessedAt: options.assessedAt,
+        };
+    }
+    const reasons = docsReasons(testCase, credited, writes, options.workspace, options.prepared);
+    reasons.push(...primaryFailure(primaryRecord));
+    const outcome = reasons.length === 0 ? "pass" : "fail";
+    return {
+        ...base,
+        validity: "valid",
+        outcome,
+        counts,
+        journal: journalSummary(verification),
+        target: testCase.expectedDocs.target ?? null,
+        primary,
+        reasons,
+        usage: readUsage(join(trialDir, "usage.json")),
+        answered: trail.text.trim() !== "",
+        assessedAt: options.assessedAt,
+    };
+}
+/** Count add attempts, including refused ones and rejected journal lines. */
+function countAttemptedWrites(host, verification) {
+    const native = host.nativeCalls.filter((dispatch) => docsVerbOf(dispatch.input) === "add").length;
+    const nested = host.dispatches.filter((dispatch) => docsVerbOf(dispatch.input) === "add").length;
+    return Math.max(native + nested, verification.rejected.length);
+}
+/** The compact journal summary a report carries. */
+export function journalSummary(verification) {
+    return {
+        state: verification.state,
+        credited: verification.credited.length,
+        rejected: verification.rejected.length,
+        problems: verification.problems,
+    };
+}
+/**
+ * The judge-visible docs operations for one trial: credited journal records
+ * only. An unattributable journal yields no lines and an explicit problem.
+ * @param trialDir The trial evidence directory.
+ * @returns The rendered operations and the verification state.
+ */
+export function creditedOperationLines(trialDir) {
+    if (!existsSync(join(trialDir, "journal", "docs-operations.jsonl"))) {
+        return { state: "absent", lines: [], problems: [] };
+    }
+    const events = existsSync(join(trialDir, "events.jsonl"))
+        ? readFileSync(join(trialDir, "events.jsonl"), "utf8")
+        : "";
+    const tools = existsSync(join(trialDir, "tools.jsonl"))
+        ? readFileSync(join(trialDir, "tools.jsonl"), "utf8")
+        : "";
+    const verification = verificationOf(trialDir, events, tools);
+    const lines = verification.credited.map((entry) => operationLine(entry.record));
+    return {
+        state: verification.state,
+        lines,
+        problems: verification.problems,
+    };
+}

package/dist/evaluators/docs/campaign.d.ts

@@ -0,0 +1,133 @@
+/**
+ * Pure primitives for the arm-blind automatic-notetaking campaign.
+ *
+ * These are the parts that must be identical across every review and rerun:
+ * the seeded schedule, arm-to-directory resolution, the documentation-change
+ * classification, the packet label guard and the tolerant JSON extraction.
+ * They are pure so a test can pin them without spawning anything.
+ */
+/** The frozen schedule seed for the 2026-11-10 six-arm campaign. */
+export declare const CAMPAIGN_SEED = 20261110;
+/** Native-description arms, in the frozen arm-id order. */
+export declare const ARMS: string[];
+/** One knowledge-lake document snapshot entry. */
+export interface LakeDocument {
+    /** Lake-relative document path. */
+    path: string;
+    /** Full document bytes. */
+    contents: string;
+}
+/** The classification of a trial's documentation change. */
+export interface DocumentationChange {
+    /** Whether any knowledge-lake document changed at all. */
+    documentsChanged: boolean;
+    /** New Markdown documents. */
+    createdPaths: string[];
+    /** Changed pre-existing Markdown documents. */
+    modifiedPaths: string[];
+    /** Removed Markdown documents. */
+    removedPaths: string[];
+}
+/** One scheduled trial slot. */
+export interface ScheduleSlot {
+    /** Stage name. */
+    stage: "screening" | "confirmation";
+    /** One-based position in the frozen execution order. */
+    slot: number;
+    /** Scenario/repetition block. */
+    block: string;
+    /** Screening repetition (1-based) or confirmation repetition (0-based). */
+    repetition: number;
+    /** Scenario id. */
+    scenario: string;
+    /** Screening arm letter. */
+    arm?: string;
+    /** Confirmation role: finalist or control. */
+    role?: "F" | "A";
+    /** Result stamp. */
+    stamp: string;
+}
+/** Both frozen stages. */
+export interface CampaignSchedule {
+    /** The seed that reproduces the schedule. */
+    seed: number;
+    /** Screening slots. */
+    screening: ScheduleSlot[];
+    /** Confirmation slots. */
+    confirmation: ScheduleSlot[];
+}
+/** Screening scenarios (the four NOTE cases). */
+export declare const NOTE_CASES: string[];
+/** Confirmation scenarios, sealed for the 2026-11-10 campaign only. */
+export declare const HELD_OUT: string[];
+/**
+ * A deterministic 32-bit PRNG (mulberry32), so a schedule is reproducible
+ * from its seed alone.
+ * @param seed The integer seed.
+ * @returns A function yielding successive floats in [0, 1).
+ */
+export declare function mulberry32(seed: number): () => number;
+/**
+ * Fisher–Yates shuffle with an injected random source.
+ * @param random The random source.
+ * @param items The items to shuffle.
+ * @returns A shuffled copy.
+ */
+export declare function shuffle<T>(random: () => number, items: T[]): T[];
+/**
+ * Rotate a list left by `by` positions.
+ * @param items The items.
+ * @param by The rotation offset.
+ * @returns The rotated list.
+ */
+export declare function rotate<T>(items: T[], by: number): T[];
+/**
+ * Build both frozen stages of the campaign schedule.
+ *
+ * Screening runs every arm once in a seeded order inside each
+ * scenario/repetition block, with scenario order rotated across repetitions.
+ * Confirmation alternates the finalist/control pair on (scenario index +
+ * repetition) mod 2 and rotates scenario order by repetition.
+ * @param seed The schedule seed; the campaign froze 20261110.
+ * @returns The screening and confirmation slot lists.
+ */
+export declare function buildSchedule(seed?: number): CampaignSchedule;
+/**
+ * Resolve one arm letter to its external worktree directory.
+ *
+ * The arm letter is case-insensitive and must belong to the frozen set; a
+ * runner that silently accepted an unknown letter or mixed case in a path is
+ * exactly the defect that invalidated 20 confirmation slots in the original
+ * campaign.
+ * @param root The campaign root holding the arm worktrees.
+ * @param arm The arm letter, either case.
+ * @returns The arm's absolute-ish directory path.
+ */
+export declare function armDir(root: string, arm: string): string;
+/**
+ * Classify documentation changes between two knowledge-lake snapshots.
+ * @param beforeDocs Markdown documents before the trial ({path, contents}).
+ * @param afterDocs Markdown documents after the trial ({path, contents}).
+ * @returns The created, modified and removed paths and a change flag.
+ */
+export declare function docChanges(beforeDocs: LakeDocument[], afterDocs: LakeDocument[]): DocumentationChange;
+/**
+ * Patterns that must never reach a judge: campaign and host paths, and the
+ * words that frame an experiment. Arm and stamp detection is shared with the
+ * trial boundary through `blindingFindings`.
+ */
+export declare const LABEL_PATTERNS: RegExp[];
+/**
+ * Refuse a judge request that carries any experiment label.
+ * @param surfaces Every judge-bound text surface: packet, neutral paths, the
+ *   request context. No surface is exempt.
+ * @returns The matched pattern sources; empty means arm-blind.
+ */
+export declare function labelLeaks(...surfaces: string[]): string[];
+/**
+ * Extract the first balanced JSON object from text, string- and
+ * escape-aware, so trailing prose containing braces cannot corrupt the parse.
+ * @param text The judge answer.
+ * @returns The raw object text, or undefined when there is none.
+ */
+export declare function firstBalancedObject(text: string): string | undefined;