@markstanden/opencode-test-container 0.0.0-stage → 0.1.0

package/dist/evaluators/docs/scoring.d.ts

@@ -0,0 +1,367 @@
+/**
+ * Campaign-level structured scoring for automatic-notetaking trials.
+ *
+ * The harness's semantic judge answers a case rubric and its deterministic
+ * gate answers "was a required write made". A prose comparison needs a
+ * third, separate measure: one mutually exclusive documentation outcome per
+ * trial, scored by declared utility weights, plus the independent
+ * primary-task result. This module is the pure part — prompt construction,
+ * strict parsing/validation against the mutation evidence, scoring,
+ * sensitivity and the frozen selection rule — so a campaign cannot silently
+ * turn a malformed judge answer or a note in a restraint case into a pass.
+ *
+ * Nothing here calls a model or touches the filesystem.
+ */
+/** The one documentation outcome a trial receives. */
+export type DocumentationOutcome = "useful-create" | "useful-update" | "transient" | "duplicate" | "misleading" | "none";
+/** The scenario whose correction needs a semantic preservation assessment. */
+export declare const PRESERVATION_CASE = "R02";
+/** The assessed state of one still-valid fact. */
+export type PreservationState = "retained" | "corrected" | "lost";
+/** The assessed state of the superseded TLS claim. */
+export type ObsoleteClaimState = "removed" | "retained";
+/** One fact's evidence-backed preservation finding. */
+export interface PreservationFinding {
+    /** Whether the fact survived, was corrected, or was lost. */
+    state: PreservationState;
+    /** The decisive excerpt from the after-trial documents. */
+    quote: string;
+}
+/** The required R02 preservation assessment, one entry per fixed fact. */
+export interface PreservationAssessment {
+    /** The 10-connection pool cap. */
+    poolSize: PreservationFinding;
+    /** The 5-second acquire timeout. */
+    acquireTimeout: PreservationFinding;
+    /** Startup migrations. */
+    startupMigrations: PreservationFinding;
+    /** The obsolete `sslmode=disable` claim. */
+    obsoleteTlsClaim: {
+        state: ObsoleteClaimState;
+        quote: string;
+    };
+}
+/** One configured fact in a scenario-keyed preservation inventory. */
+export interface PreservationFactSpec {
+    /** Stable fact id, used as the judgement field name. */
+    id: string;
+    /** Human-readable label for the prompt. */
+    label: string;
+    /** True when the fact is the superseded claim rather than a still-valid one. */
+    obsolete?: boolean;
+}
+/** A scenario-keyed preservation inventory the judge must assess. */
+export interface PreservationSpec {
+    /** The facts to assess, in prompt order. */
+    facts: PreservationFactSpec[];
+}
+/** The seeded note and distinctive detail a retrieval assessment is about. */
+export interface RetrievalSpec {
+    /** Lake-relative path of the seeded note. */
+    note: string;
+    /** The declared lake holding the note, when the declaration names one. */
+    lake?: string;
+    /** The actionable detail the task must apply. */
+    detail: string;
+}
+/** The assessed state of a trial's use of a seeded note. */
+export type RetrievalState = "applied" | "read-not-applied" | "search-only" | "not-retrieved" | "unknown";
+/** One trial's retrieval-application finding. */
+export interface RetrievalApplication {
+    /** Whether the note was retrieved and applied, read only, searched only, etc. */
+    state: RetrievalState;
+    /** The decisive excerpt, or "(none)". */
+    quote: string;
+}
+/** Everything a judge must state about one trial's documentation. */
+export interface AcJudgement {
+    /** The one outcome the trial earns. */
+    documentationOutcome: DocumentationOutcome;
+    /** Why that outcome, naming every mixed component. */
+    documentationReason: string;
+    /** Decisive excerpts from the resulting documents/evidence. */
+    evidence: string[];
+    /** The R02 preservation assessment, required only for that scenario. */
+    preservation?: PreservationAssessment;
+    /** The retrieval-application assessment, when the scenario declares one. */
+    retrievalApplication?: RetrievalApplication;
+}
+/** The mutation facts the judgement is validated against. */
+export interface DocState {
+    /** Whether any file under the documentation surface changed at all. */
+    documentsChanged: boolean;
+    /** New Markdown document paths under the documentation surface. */
+    createdPaths: string[];
+    /** Changed pre-existing Markdown document paths. */
+    modifiedPaths: string[];
+}
+/** Utility weights, frozen before scored sessions. */
+export interface UtilityWeights {
+    /** Reward for a useful create and a useful update. */
+    useful: number;
+    /** Penalty for a supported but valueless transient note. */
+    transient: number;
+    /** Penalty for a duplicate note. */
+    duplicate: number;
+    /** Penalty for misleading or unsupported documentation. */
+    misleading: number;
+    /** Penalty for a failed primary task. */
+    taskFailure: number;
+}
+/** The prospective defaults declared by the campaign prompt. */
+export declare const DEFAULT_WEIGHTS: UtilityWeights;
+/** The one documentation outcome a trial receives. */
+export declare const DOCUMENTATION_OUTCOMES: readonly DocumentationOutcome[];
+/** One trial's resolved inputs for scoring. */
+export interface ScoredTrial {
+    /** The arm the trial belongs to. */
+    arm: string;
+    /** The scenario it ran. */
+    scenario: string;
+    /** Whether the trial was planned (a slot in the frozen schedule). */
+    planned: boolean;
+    /** Whether the session completed with usable evidence. */
+    usable: boolean;
+    /** The documentation outcome, absent when not assessed. */
+    outcome?: DocumentationOutcome | undefined;
+    /** The independent primary-check result. */
+    primary: "pass" | "fail" | "unknown";
+    /** A non-compensable guardrail breach (unauthorised/destructive write). */
+    guardrailBreach: boolean;
+}
+/**
+ * Build the arm-blind judge prompt for one trial.
+ *
+ * Every input is arm-neutral by construction: the task the agent was given,
+ * the standing permission, the authoring truth for the scenario, the
+ * knowledge-lake documents before and after, the changed paths, the
+ * independent primary-check result, the observed docs operations and the
+ * final answer. No arm label, candidate description, hypothesis, prior
+ * verdict or aggregate result is accepted by this function.
+ * @param input The labelled evidence blocks.
+ * @param input.taskPrompt The exact task the agent was given.
+ * @param input.standingPermission The workspace permission file's contents.
+ * @param input.scenarioTruth The scenario's durable-finding facts.
+ * @param input.beforeDocs Knowledge-lake documents before the trial.
+ * @param input.afterDocs Knowledge-lake documents after the trial.
+ * @param input.changedPaths Paths the trial changed.
+ * @param input.primary The independent primary-check result.
+ * @param input.primary.status The checker's status.
+ * @param input.primary.exit Its exit code, or null.
+ * @param input.primary.stdout Its captured stdout.
+ * @param input.primary.stderr Its captured stderr.
+ * @param input.docsOperations Verified docs operations observed.
+ * @param input.answer The trial's final answer.
+ * @param input.requirePreservation Require the R02 semantic preservation block.
+ * @param input.preservationSpec A scenario-keyed preservation inventory.
+ * @param input.retrievalSpec A seeded-note retrieval-application assessment.
+ * @returns The complete judge prompt.
+ */
+export declare function buildAcPrompt(input: {
+    taskPrompt: string;
+    standingPermission: string;
+    scenarioTruth: string;
+    beforeDocs: Array<{
+        path: string;
+        contents: string;
+    }>;
+    afterDocs: Array<{
+        path: string;
+        contents: string;
+    }>;
+    changedPaths: string[];
+    primary: {
+        status: string;
+        exit: number | null;
+        stdout: string;
+        stderr: string;
+    };
+    docsOperations: string[];
+    answer: string;
+    requirePreservation?: boolean;
+    preservationSpec?: PreservationSpec;
+    retrievalSpec?: RetrievalSpec;
+}): string;
+/** Pull the one JSON object out of a judge's prose, if any. */
+export declare function parseAcJudgement(text: string): {
+    value?: Record<string, unknown>;
+    problems: string[];
+};
+/**
+ * The machine-readable reason one judge attempt produced no usable verdict.
+ *
+ * A complete-but-malformed answer (`json-syntax`, `schema`, `semantic`) is a
+ * model-output failure; `empty-or-truncated` means the capture itself cannot
+ * establish a final assessed response, so it is never eligible for a
+ * registered recovery. `transport-or-process` is assigned by the caller that
+ * observed the process exit.
+ */
+export type JudgeFailureClass = "transport-or-process" | "empty-or-truncated" | "json-syntax" | "schema" | "semantic";
+/**
+ * Classify a non-usable parsed answer.
+ *
+ * Completion refers to the final assessed response: the caller must establish
+ * that the trail terminated on the final text part and that the text is
+ * nonblank. When completion cannot be established the class is
+ * `empty-or-truncated` regardless of the parse failure, because a recovery
+ * must never be dispatched against unknown capture.
+ * @param input The recorded facts and problems.
+ * @param input.complete Whether the capture proves a complete final response.
+ * @param input.value The parsed value, absent on a parse failure.
+ * @param input.problems The recorded problems.
+ * @returns The failure class.
+ */
+export declare function judgeFailureClass(input: {
+    complete: boolean;
+    value?: Record<string, unknown> | undefined;
+    problems: string[];
+}): Exclude<JudgeFailureClass, "transport-or-process">;
+/** Schema problems for one parsed judgement: fields, enums and evidence. */
+export declare function judgementSchemaProblems(value: Record<string, unknown>): string[];
+/** Allowed states for a retrieval-application finding. */
+export declare const RETRIEVAL_STATES: readonly RetrievalState[];
+/**
+ * Check a parsed judgement against its schema AND the recorded mutation
+ * evidence. A judgement that contradicts the evidence is invalid, never
+ * silently corrected: "none" with a changed document, or a create with no
+ * created document, means the judge did not read the evidence.
+ * @param value The parsed judge object.
+ * @param state The mutation facts for the trial.
+ * @param options Preservation and retrieval requirements.
+ * @param options.requirePreservation True for the R02 correction scenario.
+ * @param options.preservationSpec A scenario-keyed preservation inventory.
+ * @param options.retrieval True when the scenario needs a retrieval finding.
+ * @returns The problems; empty means a usable judgement.
+ */
+export declare function validateAcJudgement(value: Record<string, unknown>, state: DocState, options?: {
+    requirePreservation?: boolean;
+    preservationSpec?: PreservationSpec;
+    retrieval?: boolean;
+}): string[];
+/** The documentation component of one trial's utility. */
+export declare function documentationScore(outcome: DocumentationOutcome, weights?: UtilityWeights): number;
+/**
+ * The closed documentation interval for an unassessed outcome: the lowest and
+ * highest score any allowed outcome can earn under the declared weights.
+ * Derived from the policy, never assumed — a custom duplicate penalty can sit
+ * below the misleading penalty, and the interval must follow it.
+ * @param weights The frozen utility weights.
+ * @returns The lowest and highest possible documentation scores.
+ */
+export declare function documentationRange(weights?: UtilityWeights): {
+    low: number;
+    high: number;
+};
+/**
+ * One trial's utility, or a closed interval when a component is unknown.
+ *
+ * An unassessed documentation outcome contributes nothing and is reported as
+ * missing, never as zero. The documentation interval is the full range of
+ * allowed outcome scores under the declared weights; the task interval
+ * retains whatever the primary check already established, so a known pass
+ * does not widen the bound as though it too were unknown.
+ */
+export declare function trialUtility(trial: ScoredTrial, weights?: UtilityWeights): {
+    score: number;
+    low: number;
+    high: number;
+    assessed: boolean;
+};
+/** Per-arm aggregate of the frozen trial population. */
+export interface ArmSummary {
+    /** The arm this summary describes. */
+    arm: string;
+    /** Planned slots. */
+    planned: number;
+    /** Slots whose session started. */
+    attempted: number;
+    /** Slots that completed. */
+    completed: number;
+    /** Slots with usable evidence. */
+    usable: number;
+    /** Slots with a usable documentation judgement. */
+    assessed: number;
+    /** Count of trials per documentation outcome. */
+    outcomes: Record<DocumentationOutcome, number>;
+    /** Primary checks that passed. */
+    primaryPass: number;
+    /** Primary checks that failed. */
+    primaryFail: number;
+    /** Primary checks without a usable result. */
+    primaryUnknown: number;
+    /** Trials flagged for a non-compensable guardrail breach. */
+    guardrailBreaches: number;
+    /** Sum of assessed trial utilities. */
+    total: number;
+    /** Mean over planned slots, or null without a denominator. */
+    meanPerPlanned: number | null;
+    /** Mean over assessed slots, or null without a denominator. */
+    meanPerAssessed: number | null;
+    /** Sum of the low bounds of every unassessed planned trial. */
+    missingLow: number;
+    /** Sum of the high bounds of every unassessed planned trial. */
+    missingHigh: number;
+}
+/**
+ * Aggregate one arm's trials. Means are reported per planned trial (the
+ * pre-registered denominator, with missing trials visible in the bounds) and
+ * per assessed trial separately, so an incomplete arm cannot hide behind a
+ * smaller denominator.
+ * @param trials The arm's planned slots.
+ * @param weights The frozen utility weights.
+ * @returns The arm summary.
+ */
+export declare function summariseArm(trials: ScoredTrial[], weights?: UtilityWeights): ArmSummary;
+/**
+ * The frozen finalist rule: a candidate qualifies only with complete
+ * assessments for both arms, no guardrail breach, and a default-utility lead
+ * of at least the threshold across the compared trials.
+ * @param control The control arm summary.
+ * @param candidate The candidate arm summary.
+ * @param threshold The required lead.
+ * @returns Whether the candidate qualifies, and why not when it does not.
+ */
+export declare function finalistQualifies(control: ArmSummary, candidate: ArmSummary, threshold: number): {
+    qualifies: boolean;
+    reason: string;
+};
+/**
+ * Rank the eligible candidates by the frozen order: default utility, then
+ * fewer primary-task failures, then fewer misleading outcomes, then fewer
+ * description characters, then arm id.
+ * @param candidates The candidate summaries with their description lengths.
+ * @returns The same list, best first.
+ */
+export declare function rankCandidates(candidates: Array<{
+    summary: ArmSummary;
+    characters: number;
+    arm: string;
+}>): Array<{
+    summary: ArmSummary;
+    characters: number;
+    arm: string;
+}>;
+/** One sensitivity cell: useful weight × transient penalty. */
+export interface SensitivityCell {
+    /** The useful create/update weight in this cell. */
+    useful: number;
+    /** The transient penalty (positive magnitude) in this cell. */
+    transient: number;
+    /** Total utility per arm. */
+    totals: Record<string, number>;
+    /** Arms best first. */
+    ranking: string[];
+}
+/**
+ * Re-score every arm under the requested weight grid, holding the other
+ * penalties fixed at the campaign's declared weights. This explores
+ * preferences; it is not nine experiments.
+ * @param arms The planned trials, keyed by arm.
+ * @param usefulWeights The useful create/update rewards to try.
+ * @param transientPenalties The transient penalties to try, as positive
+ *   magnitudes applied as negative scores.
+ * @param baseWeights The campaign's non-grid weights; defaults to the
+ *   prospective defaults so legacy callers keep their frozen baseline.
+ * @returns One cell per combination, with totals and the resulting ranking.
+ */
+export declare function sensitivity(arms: Record<string, ScoredTrial[]>, usefulWeights: number[], transientPenalties: number[], baseWeights?: UtilityWeights): SensitivityCell[];