@crewhaus/grader-provability 0.0.0-stage → 0.8.0

package/dist/graders.js

@@ -0,0 +1,435 @@
+import { claimReasons, levelRank, maxRung, meetsFloor, } from "@crewhaus/provability";
+import { gateSpendOf } from "./cost";
+import { expectedSourceKey, sampleProvabilityOf } from "./metadata";
+/** The release thresholds (design §9.1's table). */
+export const PROVABILITY_METRIC_THRESHOLDS = Object.freeze({
+    /** met / (claims − assumptions) at the floor. */
+    claimCoverage: 0.95,
+    /** defined markers / markers. */
+    citationValidity: 0.99,
+    /** declared assumptions / claims — an UPPER bound. */
+    assumptionRate: 0.1,
+    /** matched replays / replays — every replay must match. */
+    replayIntegrity: 1,
+});
+/** The reason a provability grader returns when the run carried no bundle. */
+export const NO_PROOF_BUNDLE = "no proof bundle (is provability: declared?)";
+/** The pinned prefix of the result for a bundle whose verifier threw (§5.1 fail-open). */
+export const UNVERIFIED_BUNDLE = "the gate could not verify this answer";
+/** The pinned prefix of the result for a malformed `metadata.provability`. */
+export const INVALID_SAMPLE_METADATA = "invalid metadata.provability";
+/** The pinned prefix of a floor-dependent result whose floor is above the bundle's. */
+export const FLOOR_ABOVE_BUNDLE = "not gradable at this floor";
+/**
+ * Every prefix a result carries when it REPLACES a measurement (no score was
+ * measured; `score` is a placeholder 0). A roll-up keeps these out of a
+ * lower-is-better mean, where a placeholder 0 would read as the best value.
+ */
+export const UNMEASURED_PREFIXES = Object.freeze([
+    NO_PROOF_BUNDLE,
+    UNVERIFIED_BUNDLE,
+    INVALID_SAMPLE_METADATA,
+    FLOOR_ABOVE_BUNDLE,
+]);
+/** Whether a provability grade replaced its measurement with a failure (see {@link UNMEASURED_PREFIXES}). */
+export function isUnmeasured(result) {
+    return UNMEASURED_PREFIXES.some((prefix) => result.rationale.startsWith(prefix));
+}
+/** The pinned prefix of an infinite `costPerProvenClaim` rationale (see the grader). */
+export const INFINITE_COST_PREFIX = "∞ per proven claim";
+/** The pinned prefix of a `costPerProvenClaim` whose spend is a lower bound (see the grader). */
+export const UNPRICED_COST_PREFIX = "≥ (partly unpriced) per proven claim";
+const fail = (rationale) => ({
+    result: { passed: false, score: 0, rationale },
+});
+/**
+ * The bundle and floor a grader measures, or the failing result that replaces
+ * the measurement. `floorDependent` graders also refuse a floor above the
+ * bundle's (see the module doc).
+ */
+function standing(sample, run, floorDependent) {
+    const meta = sampleProvabilityOf(sample);
+    // `meta.error` names the field: "metadata.provability… must be …".
+    if (!meta.ok)
+        return fail(`invalid ${meta.error}`);
+    const bundle = run.artifacts?.proof;
+    if (bundle === undefined) {
+        return fail(`${NO_PROOF_BUNDLE} — this sample carries no provability verdict: the run was not gated (declare provability: on the spec, or run \`crewhaus eval --provability <level>\`), or the sample ended before the gate graded an answer`);
+    }
+    if (bundle.verdict === "unverified") {
+        return fail(`${UNVERIFIED_BUNDLE} (${bundle.error ?? "the verifier failed"}) — it stood unverified, so nothing was graded`);
+    }
+    const floor = meta.value.level ?? bundle.level;
+    if (floorDependent && levelRank(floor) > levelRank(bundle.level)) {
+        return fail(`${FLOOR_ABOVE_BUNDLE}: the sample's ${floor} floor is above the ${bundle.level} floor the gate graded at, and rungs above a gate's floor are recorded without the checks only those floors run — gate the sample at ${floor} (the eval runner does, per sample) for a grade`);
+    }
+    return { bundle, floor, meta: meta.value };
+}
+/** The claims that count toward the score: everything but accepted assumptions. */
+function counted(bundle) {
+    return bundle.claims.filter((c) => c.status !== "assumption");
+}
+/** The highest floor one recorded claim would meet (see the module doc). */
+function claimCeiling(claim) {
+    if (claim.bindings.some((b) => b.reason === "no_definition"))
+        return "none";
+    return maxRung(...claim.bindings.filter((b) => b.kind !== "assumption").map((b) => b.rung));
+}
+/** Does `claim` meet `floor`? Exact at the bundle's own floor, re-read from rungs elsewhere. */
+function claimMeets(bundle, claim, floor) {
+    if (floor === bundle.level)
+        return claim.status === "met";
+    if (claim.status === "assumption")
+        return false;
+    return meetsFloor(claimCeiling(claim), floor);
+}
+/** met / counted at `floor`; 1 for the vacuous zero-counted answer. */
+function coverageAt(bundle, floor) {
+    const claims = counted(bundle);
+    if (floor === bundle.level) {
+        return { met: bundle.met, counted: claims.length, score: bundle.score };
+    }
+    const met = claims.filter((c) => claimMeets(bundle, c, floor)).length;
+    return { met, counted: claims.length, score: claims.length === 0 ? 1 : met / claims.length };
+}
+/** `reason ×n` tally over the claims that miss `floor`, most frequent first. */
+function reasonTally(bundle, floor) {
+    const tally = new Map();
+    for (const claim of counted(bundle)) {
+        if (claimMeets(bundle, claim, floor))
+            continue;
+        const reasons = claimReasons(claim.status === "unmet" ? claim : { ...claim, status: "unmet" });
+        for (const reason of reasons.length > 0 ? reasons : ["below_floor"]) {
+            tally.set(reason, (tally.get(reason) ?? 0) + 1);
+        }
+    }
+    return [...tally]
+        .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+        .map(([reason, n]) => (n > 1 ? `${reason} ×${n}` : reason))
+        .join(", ");
+}
+/** The highest rung a source binding can reach (design §3: a source grounds; it never checks). */
+const SOURCE_CEILING = "grounded";
+/**
+ * Whether a source binding grounded the answer at `floor`. At `cited` /
+ * `grounded` that is the floor itself — the gate's own `ok` at its floor
+ * (which also applies `grounded.independent`), the recorded rung elsewhere.
+ * Above `grounded` no source is ever `ok` (`below_floor`: a figure at
+ * `checked` needs a tool), yet a source the claim was grounded in still
+ * grounded the answer, so the bar there is the source ceiling, `grounded`.
+ */
+function sourceGrounds(bundle, b, floor) {
+    if (levelRank(floor) > levelRank(SOURCE_CEILING))
+        return meetsFloor(b.rung, SOURCE_CEILING);
+    return floor === bundle.level ? b.ok : meetsFloor(b.rung, floor);
+}
+/**
+ * Every source origin that grounded the answer at `floor` (see
+ * {@link sourceGrounds}) in text the run did not write there itself: an
+ * expected source is one the answer was meant to READ, and a binding whose
+ * grounding text an earlier call of the run wrote to that origin
+ * (`writtenBy`, design §10) shows the run authored it instead.
+ */
+function okSourceOrigins(bundle, floor) {
+    const out = new Set();
+    for (const claim of bundle.claims) {
+        for (const b of claim.bindings) {
+            if (b.kind !== "source" || b.origin === undefined || b.writtenBy !== undefined)
+                continue;
+            if (sourceGrounds(bundle, b, floor))
+                out.add(b.origin);
+        }
+    }
+    return out;
+}
+/** Every source origin a binding grounded in text the run wrote there itself (`writtenBy`). */
+function selfWrittenOrigins(bundle) {
+    const out = new Set();
+    for (const claim of bundle.claims) {
+        for (const b of claim.bindings) {
+            if (b.kind === "source" && b.origin !== undefined && b.writtenBy !== undefined) {
+                out.add(b.origin);
+            }
+        }
+    }
+    return out;
+}
+/** The `expected_sources` no ok binding grounded, as written. */
+function missingSources(bundle, floor, expected) {
+    const origins = okSourceOrigins(bundle, floor);
+    return expected.filter((source) => !origins.has(expectedSourceKey(source)));
+}
+const plural = (n, one, many = `${one}s`) => (n === 1 ? one : many);
+// ---------------------------------------------------------------------------
+// 1. provability.claimCoverage
+// ---------------------------------------------------------------------------
+/**
+ * met / (claims − assumptions) at the floor, the gate's own score when the
+ * floor is the run's. Passes at ≥ 0.95 AND when every
+ * `metadata.provability.expected_sources` origin grounded the answer — an ok
+ * source binding, or at `checked` / `proven` (where a source is never `ok`)
+ * one that reached `grounded` (see `sourceGrounds`). A missing one fails the
+ * sample; the score stays the coverage, the rationale names the source.
+ */
+export const claimCoverage = async (sample, run) => {
+    const s = standing(sample, run, true);
+    if ("result" in s)
+        return s.result;
+    const { bundle, floor } = s;
+    const { met, counted: n, score } = coverageAt(bundle, floor);
+    const threshold = PROVABILITY_METRIC_THRESHOLDS.claimCoverage;
+    const missing = missingSources(bundle, floor, s.meta.expectedSources ?? []);
+    const parts = [
+        n === 0
+            ? `no claims to prove at the ${floor} floor (vacuous: ${bundle.vacuous ?? "no counted claims"})`
+            : `${met}/${n} ${plural(n, "claim")} meet the ${floor} floor (threshold ≥${threshold})`,
+    ];
+    const tally = reasonTally(bundle, floor);
+    if (tally !== "")
+        parts.push(`unmet: ${tally}`);
+    if (bundle.assumptions > 0) {
+        parts.push(`${bundle.assumptions} ${plural(bundle.assumptions, "assumption")} excluded`);
+    }
+    // Design §10: a source the run wrote itself before reading it back still
+    // grounds at the gate; the grader names it, so a sample graded on such a
+    // run says so.
+    const own = selfWrittenOrigins(bundle);
+    if (own.size > 0) {
+        parts.push(`${own.size} ${plural(own.size, "source")} the run wrote itself before reading ${own.size === 1 ? "it" : "them"} back: ${[...own].map((o) => JSON.stringify(o)).join(", ")}`);
+    }
+    if (missing.length > 0) {
+        parts.push(`expected source(s) not grounding the answer: ${missing
+            .map((m) => `${JSON.stringify(m)}${own.has(expectedSourceKey(m)) ? " (the run wrote it itself)" : ""}`)
+            .join(", ")}`);
+    }
+    return { passed: score >= threshold && missing.length === 0, score, rationale: parts.join("; ") };
+};
+// ---------------------------------------------------------------------------
+// 2. provability.rungAchieved
+// ---------------------------------------------------------------------------
+/**
+ * The highest floor the answer would have passed (`bundle.achieved`, the
+ * gate ran every rung for it), normalized 0..1 as rank/4 — a per-sample
+ * ladder score that shows HOW FAR an answer falls short. Passes when the
+ * achieved rung is at or above the floor.
+ */
+export const rungAchieved = async (sample, run) => {
+    const s = standing(sample, run, true);
+    if ("result" in s)
+        return s.result;
+    const { bundle, floor } = s;
+    const rank = levelRank(bundle.achieved);
+    const passed = rank >= levelRank(floor);
+    return {
+        passed,
+        score: rank / 4,
+        rationale: `achieved ${bundle.achieved} (rung ${rank}/4) against the ${floor} floor — ${passed
+            ? "at or above it"
+            : `${levelRank(floor) - rank} ${plural(levelRank(floor) - rank, "rung")} short`}`,
+    };
+};
+// ---------------------------------------------------------------------------
+// 3. provability.citationValidity
+// ---------------------------------------------------------------------------
+/**
+ * defined markers / markers — the `cited` sub-check alone: every `[n]` in a
+ * claim needs its `[n]: …` definition. One binding is one marker occurrence;
+ * a marker with no definition is a binding whose reason is `no_definition`.
+ * No markers at all is a vacuous pass (claimCoverage is what fails an
+ * uncited answer). Passes at ≥ 0.99.
+ *
+ * Design §9.1 counts the markers the ANSWER wrote. A claim carrying
+ * `definition` is text a definition line shows after what it names, read
+ * as citing that definition: its one binding is the verifier's synthetic
+ * self-binding, defined by construction and never typed as a marker. So it
+ * is skipped — counted, a gloss sentence (or a long bibliography) would add
+ * a defined marker to both sides and lift a dangling-marker answer over the
+ * bar.
+ */
+export const citationValidity = async (sample, run) => {
+    const s = standing(sample, run, false);
+    if ("result" in s)
+        return s.result;
+    const { bundle } = s;
+    let markers = 0;
+    let dangling = 0;
+    const undefinedRefs = [];
+    for (const claim of bundle.claims) {
+        if (claim.definition !== undefined)
+            continue;
+        for (const b of claim.bindings) {
+            markers += 1;
+            if (b.reason === "no_definition") {
+                dangling += 1;
+                if (!undefinedRefs.includes(b.ref))
+                    undefinedRefs.push(b.ref);
+            }
+        }
+    }
+    const threshold = PROVABILITY_METRIC_THRESHOLDS.citationValidity;
+    if (markers === 0) {
+        const uncited = bundle.claims.filter((c) => c.bindings.length === 0).length;
+        return {
+            passed: true,
+            score: 1,
+            rationale: `no citation markers to check (vacuous)${uncited > 0 ? ` — ${uncited} uncited ${plural(uncited, "claim")} (see claimCoverage)` : ""}`,
+        };
+    }
+    const score = (markers - dangling) / markers;
+    return {
+        passed: score >= threshold,
+        score,
+        rationale: `${markers - dangling}/${markers} ${plural(markers, "marker")} defined (threshold ≥${threshold})${undefinedRefs.length > 0 ? ` — no definition for ${undefinedRefs.join(" ")}` : ""}`,
+    };
+};
+// ---------------------------------------------------------------------------
+// 4. provability.assumptionRate
+// ---------------------------------------------------------------------------
+/**
+ * declared assumptions / claims — the assumption dodge made visible. LOWER is
+ * better (the continuity.reAskRate precedent): passes at ≤ 0.1. Declared
+ * counts the accepted assumptions, the ones the config refused
+ * (`assumptions: forbid`, past `max_assumptions`) — a refused assumption is
+ * still the answer leaning on one — and the hypotheses accepted proofs rest
+ * on (`hypotheses`, design §6.4): a proof that assumes its own conclusion is
+ * not "0 assumptions". Zero claims is a vacuous 0.
+ */
+export const assumptionRate = async (sample, run) => {
+    const s = standing(sample, run, false);
+    if ("result" in s)
+        return s.result;
+    const { bundle } = s;
+    const refused = bundle.assumptionsRefused ?? 0;
+    const hypotheses = bundle.hypotheses ?? 0;
+    const declared = bundle.assumptions + refused + hypotheses;
+    const threshold = PROVABILITY_METRIC_THRESHOLDS.assumptionRate;
+    if (bundle.claimCount === 0) {
+        return { passed: true, score: 0, rationale: "no claims (assumption rate 0, vacuous)" };
+    }
+    const rate = declared / bundle.claimCount;
+    return {
+        passed: rate <= threshold,
+        score: rate,
+        rationale: `${declared}/${bundle.claimCount} ${plural(bundle.claimCount, "claim")} rest on a declared assumption (upper-bound ≤${threshold})${refused > 0 ? ` — ${refused} refused by the config` : ""}${hypotheses > 0
+            ? ` — ${hypotheses} ${plural(hypotheses, "proof hypothesis", "proof hypotheses")} counted`
+            : ""}`,
+    };
+};
+// ---------------------------------------------------------------------------
+// 5. provability.replayIntegrity
+// ---------------------------------------------------------------------------
+/**
+ * replayed executions whose output digest matched / replayed executions.
+ * An execution is one `tool:<toolUseId>` (bindings citing the same call share
+ * its one replay). Must be exactly 1: a mismatch means a cited result is not
+ * what the tool returns for that input. None replayed is a vacuous pass —
+ * replay runs only at a `checked` floor, for pure tools, with
+ * `execution.replay` on.
+ */
+export const replayIntegrity = async (sample, run) => {
+    const s = standing(sample, run, true);
+    if ("result" in s)
+        return s.result;
+    const replays = new Map();
+    s.bundle.claims.forEach((claim, i) => {
+        claim.bindings.forEach((b, j) => {
+            if (b.kind !== "execution" || b.replayed !== true)
+                return;
+            const key = b.origin ?? `${i}:${j}`;
+            const matched = b.reason !== "replay_mismatch";
+            // One mismatching citation of a call is the call's verdict.
+            replays.set(key, (replays.get(key) ?? true) && matched);
+        });
+    });
+    if (replays.size === 0) {
+        return {
+            passed: true,
+            score: 1,
+            rationale: "no replayed executions (vacuous) — replay runs at the checked floor, for pure tools, with execution.replay on",
+        };
+    }
+    const matched = [...replays.values()].filter(Boolean).length;
+    const mismatched = [...replays].filter(([, ok]) => !ok).map(([id]) => id);
+    const score = matched / replays.size;
+    return {
+        passed: score === PROVABILITY_METRIC_THRESHOLDS.replayIntegrity,
+        score,
+        rationale: `${matched}/${replays.size} replayed ${plural(replays.size, "execution")} matched (must be 1.0)${mismatched.length > 0 ? ` — replay mismatch: ${mismatched.join(", ")}` : ""}`,
+    };
+};
+// ---------------------------------------------------------------------------
+// 6. provability.costPerProvenClaim
+// ---------------------------------------------------------------------------
+const usd = (micros) => `$${(micros / 1_000_000).toFixed(4)}`;
+/**
+ * Gate-caused spend per met claim (see `./cost` for exactly what counts and
+ * how it is priced) — the `continuity.costPerProvenOutcome` precedent, but
+ * REPORTED, with no threshold: it never fails a measured sample (only a
+ * missing or unverified bundle does, like every grader here). Score is USD
+ * per claim met at the floor. Three outcomes are told apart by a pinned
+ * rationale prefix, which the roll-up reads:
+ *   - {@link INFINITE_COST_PREFIX} — spend (priced, or tokens nothing could
+ *     price) with zero met claims: the infinite ratio, kept JSON-safe as
+ *     score 0;
+ *   - {@link UNPRICED_COST_PREFIX} — some re-run tokens had no price, so the
+ *     score is the PRICED part per met claim, a lower bound, never a $0 that
+ *     reads as free;
+ *   - otherwise the exact ratio.
+ *
+ * `pricing` prices re-run calls on a run no cost-tracker metered (the eval
+ * runner passes its own `RunEvalOptions.pricing`); without it those calls
+ * are unpriced.
+ */
+export function createCostPerProvenClaim(opts = {}) {
+    return async (sample, run) => {
+        const s = standing(sample, run, true);
+        if ("result" in s)
+            return s.result;
+        const { bundle, floor } = s;
+        const { met } = coverageAt(bundle, floor);
+        const spend = gateSpendOf(run.events, bundle, opts);
+        const breakdown = [
+            `re-runs ${usd(spend.rerunUsdMicros)} over ${spend.reruns} gate ${plural(spend.reruns, "pass", "passes")} after the first`,
+            `waiver judge ${usd(spend.judgeUsdMicros)}`,
+            ...(spend.unpriced
+                ? [
+                    `${spend.unpricedTokens} of ${spend.rerunTokens} re-run tokens unpriced (no cost_accrual priced them, and no pricing row for their model)`,
+                ]
+                : []),
+            ...(spend.checks > 0
+                ? [
+                    `checker time ${spend.checkerMs}ms over ${spend.checks} ${plural(spend.checks, "check")}`,
+                ]
+                : []),
+        ].join("; ");
+        if (met === 0) {
+            // Unpriced tokens are real spend: with nothing met, the ratio is ∞.
+            return spend.totalUsdMicros > 0 || spend.unpriced
+                ? {
+                    passed: true,
+                    score: 0,
+                    rationale: `${INFINITE_COST_PREFIX} — spent ${usd(spend.totalUsdMicros)}${spend.unpriced ? ` + ${spend.unpricedTokens} unpriced tokens` : ""} with zero claims met at the ${floor} floor (${breakdown}; reported, no threshold)`,
+                }
+                : {
+                    passed: true,
+                    score: 0,
+                    rationale: `$0.0000 per proven claim — no gate spend and no claims met at the ${floor} floor (${breakdown}; reported, no threshold)`,
+                };
+        }
+        const ratio = spend.totalUsdMicros / 1_000_000 / met;
+        if (spend.unpriced) {
+            return {
+                passed: true,
+                score: ratio,
+                rationale: `${UNPRICED_COST_PREFIX} — at least $${ratio.toFixed(4)} (${usd(spend.totalUsdMicros)} priced gate spend across ${met} met ${plural(met, "claim")}: ${breakdown}; reported, no threshold)`,
+            };
+        }
+        return {
+            passed: true,
+            score: ratio,
+            rationale: `$${ratio.toFixed(4)} per proven claim (${usd(spend.totalUsdMicros)} gate spend across ${met} met ${plural(met, "claim")}: ${breakdown}; reported, no threshold)`,
+        };
+    };
+}
+/** {@link createCostPerProvenClaim} with no pricing seam (the registry's instance). */
+export const costPerProvenClaim = createCostPerProvenClaim();

package/dist/index.d.ts

@@ -0,0 +1,71 @@
+/**
+ * `@crewhaus/grader-provability` — the 0.8.0 provability release's eval layer
+ * (design/0.8.0-provability-release.md §9.1): six DETERMINISTIC graders over
+ * the proof bundle the eval runner captured for each sample
+ * (`RunResult.artifacts.proof`), following the grader-continuity /
+ * grader-12-metric-rubric conventions — one installer, per-sample scores,
+ * a cross-sample roll-up.
+ *
+ *   provability.claimCoverage      met / (claims − assumptions) at the floor   ≥ 0.95
+ *                                  (+ metadata.provability.expected_sources)
+ *   provability.rungAchieved       the highest floor passed, rank/4            ≥ the floor
+ *   provability.citationValidity   defined markers / markers                   ≥ 0.99
+ *   provability.assumptionRate     declared assumptions / claims (LOWER wins)  ≤ 0.1
+ *   provability.replayIntegrity    matched replays / replays                   = 1.0
+ *   provability.costPerProvenClaim gate-caused spend per met claim             reported
+ *
+ * A sample's floor is `metadata.provability.level` when it sets one, else the
+ * bundle's (the spec's floor, or `crewhaus eval --provability <level>`). A run
+ * with no bundle fails every grader with {@link NO_PROOF_BUNDLE} — see
+ * `./graders` for why a missing verdict is never a pass, and for the other
+ * pinned "unmeasured" outcomes ({@link isUnmeasured}).
+ *
+ * Graders read only what the runner hands them; none calls a model, none
+ * re-runs the verifier, none touches the harness's live stores (Pillar 2: the
+ * eval layer measures end states, offline).
+ */
+import type { Grader } from "@crewhaus/eval-grader";
+import type { GraderRegistry } from "@crewhaus/grader-registry";
+import type { GatePricing } from "./cost";
+export { FLOOR_ABOVE_BUNDLE, INFINITE_COST_PREFIX, INVALID_SAMPLE_METADATA, NO_PROOF_BUNDLE, PROVABILITY_METRIC_THRESHOLDS, UNMEASURED_PREFIXES, UNPRICED_COST_PREFIX, UNVERIFIED_BUNDLE, assumptionRate, citationValidity, claimCoverage, costPerProvenClaim, createCostPerProvenClaim, isUnmeasured, replayIntegrity, rungAchieved, } from "./graders";
+export { type GatePricing, type GateSpend, gateSpendOf } from "./cost";
+export { type SampleProvability, type SampleProvabilityRead, expectedSourceKey, sampleProvabilityOf, } from "./metadata";
+export { PROVABILITY_METRIC_SPECS, type ProvabilityCostSummary, type ProvabilityMetricSpec, type ProvabilityMetricSummary, type ProvabilitySummary, renderProvabilitySummaryLines, summarizeCostPerProvenClaim, summarizeProvabilityMetrics, } from "./summary";
+/** The six graders under their canonical names, in the design's table order. */
+export declare const PROVABILITY_GRADERS: ReadonlyArray<readonly [string, Grader]>;
+/**
+ * The six graders, with `costPerProvenClaim` pricing re-runs through
+ * `pricing` on a run no cost-tracker metered (see `./cost`). The eval runner
+ * builds its set here from its own `RunEvalOptions.pricing`; without a seam
+ * this is {@link PROVABILITY_GRADERS}.
+ */
+export declare function createProvabilityGraders(opts?: {
+    readonly pricing?: GatePricing;
+}): ReadonlyArray<readonly [string, Grader]>;
+/**
+ * The graders that are REPORTED, never a pass/fail bar: their verdict is
+ * always a pass when measured, so a combination that let them vote would
+ * pass a sample on a dollar figure.
+ */
+export declare const PROVABILITY_REPORTED_GRADERS: ReadonlyArray<string>;
+/** The canonical names, in order. */
+export declare const PROVABILITY_GRADER_NAMES: ReadonlyArray<string>;
+/**
+ * How one of the six joins a sample's overall grade (design §9.1), keyed on
+ * its canonical name — the one answer for every path that adds it, the
+ * `--provability` flag and a graders file's `type: registry` entry alike:
+ *   - `"requirement"` — the five with a bar: the sample must pass each on top
+ *     of its own `combine:` policy, never as a vote inside it (a vacuous pass
+ *     or a lower-is-better rate cannot carry a `combine: any` or skew a
+ *     `weighted` score);
+ *   - `"report"` — {@link PROVABILITY_REPORTED_GRADERS}: recorded, never part
+ *     of the verdict (more gate spend can never turn a fail into a pass).
+ * `undefined` for any other name.
+ */
+export declare function provabilityGraderRole(name: string): "requirement" | "report" | undefined;
+/**
+ * Register all six graders. Skips names already present (like
+ * `registerContinuityGraders`, so a plugin registered first keeps its name)
+ * and returns the canonical names.
+ */
+export declare function registerProvabilityGraders(registry: GraderRegistry): ReadonlyArray<string>;

package/dist/index.js

@@ -0,0 +1,65 @@
+import { assumptionRate, citationValidity, claimCoverage, costPerProvenClaim, createCostPerProvenClaim, replayIntegrity, rungAchieved, } from "./graders";
+export { FLOOR_ABOVE_BUNDLE, INFINITE_COST_PREFIX, INVALID_SAMPLE_METADATA, NO_PROOF_BUNDLE, PROVABILITY_METRIC_THRESHOLDS, UNMEASURED_PREFIXES, UNPRICED_COST_PREFIX, UNVERIFIED_BUNDLE, assumptionRate, citationValidity, claimCoverage, costPerProvenClaim, createCostPerProvenClaim, isUnmeasured, replayIntegrity, rungAchieved, } from "./graders";
+export { gateSpendOf } from "./cost";
+export { expectedSourceKey, sampleProvabilityOf, } from "./metadata";
+export { PROVABILITY_METRIC_SPECS, renderProvabilitySummaryLines, summarizeCostPerProvenClaim, summarizeProvabilityMetrics, } from "./summary";
+/** The six graders under their canonical names, in the design's table order. */
+export const PROVABILITY_GRADERS = Object.freeze([
+    ["provability.claimCoverage", claimCoverage],
+    ["provability.rungAchieved", rungAchieved],
+    ["provability.citationValidity", citationValidity],
+    ["provability.assumptionRate", assumptionRate],
+    ["provability.replayIntegrity", replayIntegrity],
+    ["provability.costPerProvenClaim", costPerProvenClaim],
+]);
+/**
+ * The six graders, with `costPerProvenClaim` pricing re-runs through
+ * `pricing` on a run no cost-tracker metered (see `./cost`). The eval runner
+ * builds its set here from its own `RunEvalOptions.pricing`; without a seam
+ * this is {@link PROVABILITY_GRADERS}.
+ */
+export function createProvabilityGraders(opts = {}) {
+    if (opts.pricing === undefined)
+        return PROVABILITY_GRADERS;
+    const cost = createCostPerProvenClaim({ pricing: opts.pricing });
+    return Object.freeze(PROVABILITY_GRADERS.map(([name, grader]) => Object.freeze([name, name === "provability.costPerProvenClaim" ? cost : grader])));
+}
+/**
+ * The graders that are REPORTED, never a pass/fail bar: their verdict is
+ * always a pass when measured, so a combination that let them vote would
+ * pass a sample on a dollar figure.
+ */
+export const PROVABILITY_REPORTED_GRADERS = Object.freeze([
+    "provability.costPerProvenClaim",
+]);
+/** The canonical names, in order. */
+export const PROVABILITY_GRADER_NAMES = Object.freeze(PROVABILITY_GRADERS.map(([name]) => name));
+/**
+ * How one of the six joins a sample's overall grade (design §9.1), keyed on
+ * its canonical name — the one answer for every path that adds it, the
+ * `--provability` flag and a graders file's `type: registry` entry alike:
+ *   - `"requirement"` — the five with a bar: the sample must pass each on top
+ *     of its own `combine:` policy, never as a vote inside it (a vacuous pass
+ *     or a lower-is-better rate cannot carry a `combine: any` or skew a
+ *     `weighted` score);
+ *   - `"report"` — {@link PROVABILITY_REPORTED_GRADERS}: recorded, never part
+ *     of the verdict (more gate spend can never turn a fail into a pass).
+ * `undefined` for any other name.
+ */
+export function provabilityGraderRole(name) {
+    if (PROVABILITY_REPORTED_GRADERS.includes(name))
+        return "report";
+    return PROVABILITY_GRADER_NAMES.includes(name) ? "requirement" : undefined;
+}
+/**
+ * Register all six graders. Skips names already present (like
+ * `registerContinuityGraders`, so a plugin registered first keeps its name)
+ * and returns the canonical names.
+ */
+export function registerProvabilityGraders(registry) {
+    for (const [name, grader] of PROVABILITY_GRADERS) {
+        if (!registry.has(name))
+            registry.register(name, grader);
+    }
+    return PROVABILITY_GRADER_NAMES;
+}

package/dist/metadata.d.ts

@@ -0,0 +1,47 @@
+/**
+ * A dataset sample's `metadata.provability` (design/0.8.0-provability-release.md
+ * §9.1): the per-sample floor override and the origins that must ground the
+ * answer.
+ *
+ *   metadata:
+ *     provability: grounded                  # shorthand ≡ { level: grounded }
+ *     provability:
+ *       level: checked                       # grade THIS sample at checked
+ *       expected_sources:                    # each must be an ok source origin
+ *         - https://acme.example/policy
+ *         - docs/pricing.md
+ *
+ * `eval-dataset` keeps `metadata` free-form (`z.record(z.unknown())`), so the
+ * reading lives here, beside the only graders that use it. A malformed value
+ * is reported, never ignored: the graders turn it into a failing result that
+ * names the field, because a floor the author meant and the grader dropped
+ * would grade the sample against the wrong bar in silence.
+ */
+import type { Sample } from "@crewhaus/eval-dataset";
+import { type ProvabilityLevel } from "@crewhaus/provability";
+/** The parsed `metadata.provability`; both fields optional. */
+export type SampleProvability = {
+    /** The floor this sample is graded at, overriding the run's. */
+    readonly level?: ProvabilityLevel;
+    /** Origins (URLs, `doi:` refs, workspace paths) that must ground the answer, as written. */
+    readonly expectedSources?: ReadonlyArray<string>;
+};
+export type SampleProvabilityRead = {
+    readonly ok: true;
+    readonly value: SampleProvability;
+} | {
+    readonly ok: false;
+    readonly error: string;
+};
+/** Read `sample.metadata.provability`. Absent ⇒ `{ ok: true, value: {} }`. */
+export declare function sampleProvabilityOf(sample: Pick<Sample, "metadata">): SampleProvabilityRead;
+/**
+ * The key an expected source is compared under — the normalization the
+ * bundle's binding origins went through (`@crewhaus/provability`'s origin
+ * rules): an `http(s)` URL normalized, a `doi:` ref as its resolver URL, and
+ * anything else as a workspace-relative path measured from `workspaceRoot`
+ * (the eval's cwd, where the run's tools resolved paths). A path that lands
+ * outside the workspace keeps its written form (minus a leading `./`), so it
+ * can only match an origin spelled exactly that way.
+ */
+export declare function expectedSourceKey(source: string, workspaceRoot?: string): string;