@markstanden/opencode-test-container 0.0.0-stage → 0.1.1

package/dist/evaluators/docs/analysis/freeze.js

@@ -0,0 +1,395 @@
+/**
+ * The experiment freeze: every immutable input, bound before the first launch.
+ *
+ * A freeze is exclusive. It is written once, with `wx`, after checking that no
+ * trial, judge attempt or result exists yet, that both arm trees are clean, and
+ * that each arm actually served the description its registration measured.
+ * Verification later compares the same hashes, so a modified rubric, fixture,
+ * script, spec or swapped description invalidates the premise instead of
+ * silently changing the experiment.
+ */
+import { createHash } from "node:crypto";
+import { existsSync, writeFileSync } from "node:fs";
+/** The current freeze record format. */
+export const FREEZE_SCHEMA_VERSION = 2;
+/**
+ * SHA-256 of exact text bytes.
+ * @param text The text.
+ * @returns The lowercase hex digest.
+ */
+export function sha256Text(text) {
+    return createHash("sha256").update(text, "utf8").digest("hex");
+}
+/**
+ * Extract the served description of one tool from a provider request body.
+ *
+ * The delivery probe saves the raw request body, so this is the actual payload
+ * the provider received — never a hand-extracted copy. A body without the tool
+ * is a problem, not an empty description.
+ * @param body The parsed request body.
+ * @param toolName The registered tool name.
+ * @returns The served description, or a problem.
+ */
+export function deliveredToolDescription(body, toolName = "docs") {
+    if (typeof body !== "object" || body === null) {
+        return { problem: "delivery request body is not an object" };
+    }
+    const tools = body.tools;
+    if (!Array.isArray(tools)) {
+        return { problem: "delivery request body has no tools array" };
+    }
+    for (const entry of tools) {
+        const served = servedDescription(entry, toolName);
+        if (served !== undefined)
+            return served;
+    }
+    return { problem: `delivery request body does not serve tool ${toolName}` };
+}
+/**
+ * Read one served tool's description.
+ * @param entry The request body's tool entry.
+ * @param toolName The registered tool name.
+ * @returns The served description, a problem for a malformed match, or
+ *   undefined when the entry is not the named tool.
+ */
+function servedDescription(entry, toolName) {
+    if (typeof entry !== "object" || entry === null)
+        return undefined;
+    const fn = entry.function;
+    if (typeof fn !== "object" || fn === null)
+        return undefined;
+    if (fn.name !== toolName)
+        return undefined;
+    const description = fn.description;
+    if (typeof description !== "string" || description.trim() === "") {
+        return {
+            problem: `served tool ${toolName} has no nonblank description`,
+        };
+    }
+    return {
+        delivered: {
+            description,
+            chars: [...description].length,
+            sha256: sha256Text(description),
+        },
+    };
+}
+/**
+ * Check one arm's registration against what the provider was served.
+ * @param arm The arm identity.
+ * @returns Problems; empty means registration and delivery agree.
+ */
+function armProblems(arm) {
+    const problems = [];
+    if (!arm.clean)
+        problems.push(`arm ${arm.id} worktree is dirty`);
+    if (arm.descriptionHash !== arm.deliveredHash) {
+        problems.push(`arm ${arm.id} delivered description differs from the registered one`);
+    }
+    if (arm.descriptionChars !== arm.deliveredChars) {
+        problems.push(`arm ${arm.id} delivered description length differs from the registered one`);
+    }
+    return problems;
+}
+/**
+ * Decide whether the experiment may be frozen.
+ *
+ * Refusals happen before anything is written: a started experiment, an
+ * existing freeze, a dirty arm, a missing or contradictory delivery, an absent
+ * input or a ceiling all block. Inputs are deduplicated by path and sorted, so
+ * two runs hash the same immutable set in the same order.
+ * @param input Everything the decision reads.
+ * @returns The record, or every problem.
+ */
+export function freezeExperiment(input) {
+    const problems = [
+        ...stateProblems(input),
+        ...armSetProblems(input.arms),
+        ...identityProblems(input),
+        ...inputFileProblems(input.files),
+        ...inventoryProblems(input.inventories),
+    ];
+    if (problems.length > 0)
+        return { ok: false, problems };
+    const byPath = new Map();
+    for (const file of input.files)
+        byPath.set(file.path, file);
+    return {
+        ok: true,
+        record: {
+            schemaVersion: FREEZE_SCHEMA_VERSION,
+            campaign: input.campaign,
+            sourceRevision: input.sourceRevision,
+            sourceTree: input.sourceTree,
+            imageId: input.imageId,
+            containerCli: input.containerCli,
+            measurementSha256: input.measurementSha256,
+            arms: [...input.arms].sort((left, right) => left.id.localeCompare(right.id)),
+            files: [...byPath.values()].sort((left, right) => left.path.localeCompare(right.path)),
+            inventories: [...input.inventories]
+                .map((inventory) => ({
+                path: inventory.path,
+                files: [...inventory.files].sort((left, right) => left.path.localeCompare(right.path)),
+            }))
+                .sort((left, right) => left.path.localeCompare(right.path)),
+            stoppingRules: [...input.stoppingRules],
+            ceilings: input.ceilings,
+        },
+    };
+}
+/**
+ * The inventory preconditions.
+ * @param inventories The declared directory inventories.
+ * @returns Problems; empty means each inventory is usable.
+ */
+function inventoryProblems(inventories) {
+    const problems = [];
+    for (const inventory of inventories) {
+        if (inventory.path.trim() === "") {
+            problems.push("every inventory needs a directory path");
+        }
+        if (inventory.files.length === 0) {
+            problems.push(`inventory ${inventory.path} is empty`);
+        }
+        for (const file of inventory.files) {
+            if (!file.path.startsWith(`${inventory.path}/`)) {
+                problems.push(`inventory file ${file.path} is outside ${inventory.path}`);
+            }
+        }
+    }
+    return problems;
+}
+/**
+ * The start/existing-freeze preconditions.
+ * @param input The freeze input.
+ * @returns Problems; empty means the experiment has not begun.
+ */
+function stateProblems(input) {
+    const problems = [];
+    if (input.started) {
+        problems.push("refusing to freeze after a launch, judge attempt or recorded result exists");
+    }
+    if (input.frozen) {
+        problems.push("refusing to overwrite an existing freeze");
+    }
+    return problems;
+}
+/**
+ * The arm-set preconditions: at least two distinct arms, clean, delivered as
+ * registered, and not serving identical descriptions.
+ * @param arms The arms.
+ * @returns Problems; empty means the arm set is coherent.
+ */
+function armSetProblems(arms) {
+    const problems = [];
+    if (arms.length < 2)
+        problems.push("a comparison needs at least two arms");
+    const ids = arms.map((arm) => arm.id).sort();
+    if (new Set(ids).size !== ids.length) {
+        problems.push("arm ids must be distinct");
+    }
+    for (const arm of arms)
+        problems.push(...armProblems(arm));
+    const sorted = [...arms].sort((left, right) => left.id.localeCompare(right.id));
+    const first = sorted[0];
+    const second = sorted[1];
+    if (first !== undefined &&
+        second !== undefined &&
+        first.descriptionHash === second.descriptionHash) {
+        problems.push("the arms serve an identical description");
+    }
+    return problems;
+}
+/**
+ * The run-identity preconditions.
+ * @param input The freeze input.
+ * @returns Problems; empty means identity and rules are recorded.
+ */
+function identityProblems(input) {
+    return [
+        ...requiredIdentities(input),
+        ...ruleProblems(input.ceilings, input.stoppingRules),
+    ];
+}
+/**
+ * The image, CLI and measurement identity checks.
+ * @param input The freeze input.
+ * @returns Problems; empty means those identities are recorded.
+ */
+function requiredIdentities(input) {
+    const problems = [];
+    if (input.imageId === null || input.imageId.trim() === "") {
+        problems.push("no container image identity was observed");
+    }
+    if (input.containerCli === null || input.containerCli.trim() === "") {
+        problems.push("no container CLI identity was observed");
+    }
+    if (input.measurementSha256 === null ||
+        input.measurementSha256.trim() === "") {
+        problems.push("no measurement declaration was frozen");
+    }
+    return problems;
+}
+/**
+ * The ceiling and stopping-rule checks.
+ * @param ceilings The declared ceilings, when present.
+ * @param stoppingRules The declared stopping rules.
+ * @returns Problems; empty means the rules are recorded.
+ */
+function ruleProblems(ceilings, stoppingRules) {
+    const problems = [];
+    if (ceilings === null || ceilings.trial < 1 || ceilings.judge < 1) {
+        problems.push("the trial and judge ceilings must both be positive");
+    }
+    if (stoppingRules.length === 0) {
+        problems.push("the stopping rules must be recorded");
+    }
+    return problems;
+}
+/**
+ * The immutable-input preconditions.
+ * @param files The hashed inputs.
+ * @returns Problems; empty means every input is identified.
+ */
+function inputFileProblems(files) {
+    const problems = [];
+    if (files.length === 0)
+        problems.push("no immutable inputs were hashed");
+    for (const file of files) {
+        if (file.path.trim() === "" || file.sha256.trim() === "") {
+            problems.push("every immutable input needs a path and a hash");
+        }
+    }
+    return problems;
+}
+/**
+ * Write a verified freeze exclusively. An existing file refuses without
+ * touching its bytes.
+ * @param path The freeze file path.
+ * @param record The verified record.
+ * @returns Whether the write landed.
+ */
+export function writeFreeze(path, record) {
+    try {
+        writeFileSync(path, `${JSON.stringify(record, null, 2)}\n`, {
+            encoding: "utf8",
+            flag: "wx",
+        });
+        return { ok: true };
+    }
+    catch (error) {
+        return {
+            ok: false,
+            reason: error.code === "EEXIST"
+                ? "refusing to overwrite an existing freeze"
+                : `cannot write freeze: ${String(error)}`,
+        };
+    }
+}
+/**
+ * Why a freeze cannot authorise new spend, or null when it can.
+ * @param record The recorded freeze, or undefined when absent.
+ * @returns The refusal, or null for a modern complete record.
+ */
+export function spendFreezeProblem(record) {
+    if (record === null || record === undefined) {
+        return "no freeze record; new spend requires a validated freeze";
+    }
+    if (record.schemaVersion !== FREEZE_SCHEMA_VERSION) {
+        return `freeze format ${String(record.schemaVersion)} is legacy and read-only; re-freeze before spending`;
+    }
+    return (inventoryRefusal(record) ??
+        ceilingRefusal(record.ceilings) ??
+        imageRefusal(record.imageId));
+}
+/** The inventory requirement, or null. */
+function inventoryRefusal(record) {
+    return Array.isArray(record.inventories) && record.inventories.length > 0
+        ? null
+        : "the freeze carries no input inventory; it cannot detect added inputs";
+}
+/** The ceiling requirement, or null. */
+function ceilingRefusal(ceilings) {
+    if (ceilings === null)
+        return "the freeze carries no valid trial and judge ceilings";
+    const valid = Number.isInteger(ceilings.trial) &&
+        Number.isInteger(ceilings.judge) &&
+        ceilings.trial > 0 &&
+        ceilings.judge > 0;
+    return valid
+        ? null
+        : "the freeze carries no valid trial and judge ceilings";
+}
+/** The image requirement, or null. */
+function imageRefusal(imageId) {
+    return imageId.trim() === ""
+        ? "the freeze carries no container image identity"
+        : null;
+}
+/**
+ * Compare observed inputs with the frozen record, including directory
+ * inventories in both directions so an added input is caught too.
+ * @param record The frozen record.
+ * @param observation What is present now.
+ * @returns Every mismatch; empty means the premise still holds.
+ */
+export function verifyFreeze(record, observation) {
+    const problems = [];
+    if (observation.sourceRevision !== record.sourceRevision) {
+        problems.push(`source revision changed: ${record.sourceRevision} to ${observation.sourceRevision}`);
+    }
+    if (observation.imageId !== record.imageId) {
+        problems.push(`container image changed: ${record.imageId} to ${observation.imageId}`);
+    }
+    if (observation.containerCli !== record.containerCli) {
+        problems.push(`container CLI changed: ${record.containerCli} to ${observation.containerCli}`);
+    }
+    const frozen = new Map(record.files.map((file) => [file.path, file.sha256]));
+    for (const file of observation.files) {
+        if (frozen.get(file.path) !== file.sha256) {
+            problems.push(`immutable input changed: ${file.path}`);
+        }
+        frozen.delete(file.path);
+    }
+    for (const path of [...frozen.keys()].sort()) {
+        problems.push(`immutable input missing: ${path}`);
+    }
+    problems.push(...inventoryProblemsOf(record.inventories, observation.inventories));
+    return problems;
+}
+/**
+ * Compare every declared directory inventory with what is present.
+ * @param frozen The recorded inventories.
+ * @param observed The inventories read now.
+ * @returns Added, changed and missing file problems.
+ */
+function inventoryProblemsOf(frozen, observed) {
+    const problems = [];
+    const byPath = new Map(observed.map((entry) => [entry.path, entry]));
+    for (const inventory of frozen) {
+        const current = byPath.get(inventory.path);
+        if (current === undefined) {
+            problems.push(`inventory missing: ${inventory.path}`);
+            continue;
+        }
+        byPath.delete(inventory.path);
+        const expected = new Map(inventory.files.map((file) => [file.path, file.sha256]));
+        for (const file of current.files) {
+            if (expected.get(file.path) !== file.sha256) {
+                problems.push(`immutable input changed: ${file.path}`);
+            }
+            expected.delete(file.path);
+        }
+        for (const path of [...expected.keys()].sort()) {
+            problems.push(`immutable input missing: ${path}`);
+        }
+    }
+    for (const path of [...byPath.keys()].sort()) {
+        problems.push(`unexpected inventory: ${path}`);
+    }
+    return problems;
+}
+/** Whether a path already holds a freeze. */
+export function freezeExists(path) {
+    return existsSync(path);
+}

package/dist/evaluators/docs/analysis/guardrails.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Non-compensable guardrail evaluation.
+ *
+ * Each declared guardrail is evaluated per trial into pass, fail, unknown or
+ * not-applicable with its evidence, then summarised per arm. A missing
+ * policy record stays unknown, never a false pass; and a textual gate reason
+ * can raise an unresolved guardrail but can never establish semantic loss on
+ * its own.
+ */
+import type { GuardrailKind, GuardrailSpec, ScenarioIdentity } from "./model.ts";
+import type { TrialRecord } from "./records.ts";
+/** One guardrail's state for one trial. */
+export type GuardrailStatus = "pass" | "fail" | "unknown" | "not-applicable";
+/** One trial's guardrail finding with its evidence. */
+export interface GuardrailTrialFinding {
+    /** The evaluated status. */
+    status: GuardrailStatus;
+    /** Evidence lines supporting a failure or an unresolved state. */
+    evidence: string[];
+}
+/** One guardrail's summary for an arm or population. */
+export interface GuardrailArmSummary {
+    /** The declared guardrail id. */
+    guardrailId: string;
+    /** The declared guardrail kind. */
+    kind: GuardrailKind;
+    /** The declared requirement, quoted verbatim. */
+    requirement: string;
+    /** The population status: breached, unresolved, clean or not applicable. */
+    status: "clean" | "breached" | "unresolved" | "not-applicable";
+    /** Per-status trial counts. */
+    coverage: Record<GuardrailStatus, number>;
+    /** Stamp-prefixed evidence lines for failures and unresolved trials. */
+    evidence: string[];
+}
+/**
+ * Evaluate one declared guardrail for one trial.
+ * @param guardrail The declared guardrail.
+ * @param record The loaded trial record.
+ * @param scenario The trial's declared scenario, when known.
+ * @returns The trial finding.
+ */
+export declare function evaluateGuardrail(guardrail: GuardrailSpec, record: TrialRecord, scenario: ScenarioIdentity | undefined): GuardrailTrialFinding;
+/**
+ * Summarise every declared guardrail over one population.
+ * @param guardrails The declared guardrails.
+ * @param scenarios The declared scenarios, for inventory lookup.
+ * @param trials The loaded trial records in the population.
+ * @returns One summary per guardrail, in declared order.
+ */
+export declare function summariseGuardrails(guardrails: readonly GuardrailSpec[], scenarios: readonly ScenarioIdentity[], trials: readonly TrialRecord[]): GuardrailArmSummary[];

package/dist/evaluators/docs/analysis/guardrails.js

@@ -0,0 +1,194 @@
+/** Wording that marks a gate block as touching a protected path. */
+const PRESERVATION_BLOCK = /\bpreserv/i;
+/** States that keep a still-valid fact alive. */
+const RETAINED_STATES = ["retained", "corrected"];
+/**
+ * Evaluate a mutation-policy guardrail from the recorded run metadata.
+ * @param record The loaded trial record.
+ * @returns Pass with no violations, fail with the recorded violations, or unknown.
+ */
+function mutationFinding(record) {
+    if (record.run.state !== "available") {
+        return {
+            status: "unknown",
+            evidence: ["no recorded run metadata for the mutation policy"],
+        };
+    }
+    const violations = record.run.value.policyViolations;
+    if (!Array.isArray(violations)) {
+        return {
+            status: "unknown",
+            evidence: ["no recorded policyViolations field (limited evidence)"],
+        };
+    }
+    if (violations.length === 0)
+        return { status: "pass", evidence: [] };
+    return {
+        status: "fail",
+        evidence: violations.map((one) => `outside policy: ${String(one)}`),
+    };
+}
+/**
+ * Evaluate one inventory fact's observation into a guardrail outcome.
+ * @param fact The configured fact.
+ * @param entry The loaded observation, when present.
+ * @returns The outcome and an evidence line for failures and unknowns.
+ */
+function factOutcome(fact, entry) {
+    if (entry === undefined || entry.state === "missing") {
+        return {
+            status: "unknown",
+            line: `${fact.id}: no evidence-backed assessment`,
+        };
+    }
+    if (entry.state !== "available") {
+        return {
+            status: "unknown",
+            line: `${fact.id}: unreadable assessment (${entry.state})`,
+        };
+    }
+    const { state, quote } = entry.value;
+    const obsolete = fact.obsoleteCurrentPatterns !== undefined;
+    if (obsolete) {
+        if (state === "removed")
+            return { status: "pass", line: "" };
+        if (state === "retained") {
+            return {
+                status: "fail",
+                line: `${fact.id}: obsolete claim retained: ${quote}`,
+            };
+        }
+    }
+    else {
+        if (RETAINED_STATES.includes(state))
+            return { status: "pass", line: "" };
+        if (state === "lost") {
+            return {
+                status: "fail",
+                line: `${fact.id}: valid fact lost: ${quote}`,
+            };
+        }
+    }
+    return {
+        status: "unknown",
+        line: `${fact.id}: unrecognised preservation state ${JSON.stringify(state)}`,
+    };
+}
+/**
+ * Evaluate a preservation inventory against the recorded observations.
+ * A failure dominates an unknown; an unknown dominates a pass.
+ * @param inventory The configured inventory.
+ * @param observed The per-fact observations.
+ * @returns The aggregate finding.
+ */
+function inventoryFinding(inventory, observed) {
+    const failures = [];
+    const unknowns = [];
+    for (const fact of inventory.facts) {
+        const outcome = factOutcome(fact, observed[fact.id]);
+        if (outcome.status === "fail")
+            failures.push(outcome.line);
+        else if (outcome.status === "unknown")
+            unknowns.push(outcome.line);
+    }
+    if (failures.length > 0)
+        return { status: "fail", evidence: failures };
+    if (unknowns.length > 0)
+        return { status: "unknown", evidence: unknowns };
+    return { status: "pass", evidence: [] };
+}
+/**
+ * Evaluate a destructive-loss guardrail for one trial.
+ *
+ * With a configured inventory the semantic assessment decides. Without one,
+ * the guardrail is not applicable — unless a gate block names a preserved
+ * path, which raises an unresolved state: the text cannot prove loss, and an
+ * authorised rewrite cannot prove safety.
+ * @param record The loaded trial record.
+ * @param scenario The trial's declared scenario.
+ * @returns The finding.
+ */
+function lossFinding(record, scenario) {
+    const inventory = scenario?.preservationInventory;
+    if (inventory === undefined) {
+        const gate = record.gate;
+        const blocked = gate.state === "available" &&
+            gate.value.status === "blocked" &&
+            gate.value.reason !== null &&
+            PRESERVATION_BLOCK.test(gate.value.reason);
+        return blocked && gate.state === "available"
+            ? {
+                status: "unknown",
+                evidence: [gate.value.reason ?? "preserve gate block"],
+            }
+            : { status: "not-applicable", evidence: [] };
+    }
+    const observed = record.assessment.preservation;
+    if (observed.state !== "available") {
+        return {
+            status: "unknown",
+            evidence: ["no preservation assessment recorded"],
+        };
+    }
+    return inventoryFinding(inventory, observed.value);
+}
+/**
+ * Evaluate one declared guardrail for one trial.
+ * @param guardrail The declared guardrail.
+ * @param record The loaded trial record.
+ * @param scenario The trial's declared scenario, when known.
+ * @returns The trial finding.
+ */
+export function evaluateGuardrail(guardrail, record, scenario) {
+    if (guardrail.kind === "mutation-policy")
+        return mutationFinding(record);
+    return lossFinding(record, scenario);
+}
+/**
+ * Derive a population's guardrail status from its coverage counts.
+ * @param coverage The per-status counts.
+ * @returns Breached on any failure, unresolved on any unknown, else clean.
+ */
+function populationStatus(coverage) {
+    if (coverage.fail > 0)
+        return "breached";
+    if (coverage.unknown > 0)
+        return "unresolved";
+    if (coverage.pass > 0)
+        return "clean";
+    return "not-applicable";
+}
+/**
+ * Summarise every declared guardrail over one population.
+ * @param guardrails The declared guardrails.
+ * @param scenarios The declared scenarios, for inventory lookup.
+ * @param trials The loaded trial records in the population.
+ * @returns One summary per guardrail, in declared order.
+ */
+export function summariseGuardrails(guardrails, scenarios, trials) {
+    const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
+    return guardrails.map((guardrail) => {
+        const coverage = {
+            pass: 0,
+            fail: 0,
+            unknown: 0,
+            "not-applicable": 0,
+        };
+        const evidence = [];
+        for (const trial of trials) {
+            const finding = evaluateGuardrail(guardrail, trial, scenarioById.get(trial.identity.scenario));
+            coverage[finding.status] += 1;
+            if (finding.status === "fail" || finding.status === "unknown") {
+                evidence.push(...finding.evidence.map((line) => `${trial.identity.stamp}: ${line}`));
+            }
+        }
+        return {
+            guardrailId: guardrail.id,
+            kind: guardrail.kind,
+            requirement: guardrail.requirement,
+            status: populationStatus(coverage),
+            coverage,
+            evidence,
+        };
+    });
+}