open-multi-agent-kit 1.0.0 → 1.2.0

package/dist/metacognition/calibration-selective.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Probability calibration and selective execution (Jev audit algorithm A3).
+ *
+ * A provider's `confidence` is a statistic over its own option distribution,
+ * not a calibrated probability of any event we care about. Before it can gate
+ * execution, the predicted event has to be named — "the chosen candidate was
+ * appropriate" is a different event from "the driver executed it" and from
+ * "the task succeeded" — and the score has to be measured on held-out data.
+ *
+ * These are the measurement primitives, deliberately small: distribution
+ * features, a post-hoc temperature rule, scoring rules, and a selective gate
+ * that always reports coverage next to risk. None of them makes a score
+ * trustworthy; they make it measurable.
+ */
+/** Floor for numeric zeros in temperature scaling; recorded so results are reproducible. */
+export declare const TEMPERATURE_EPSILON = 1e-12;
+/** Clip for log loss so one confident miss does not return Infinity. */
+export declare const LOG_LOSS_CLIP = 1e-15;
+export interface DistributionFeatures {
+    readonly maxProbability: number;
+    /** p(1) − p(2). Undefined for a single candidate: there is nothing to compare. */
+    readonly margin: number | undefined;
+    /** Undefined for a single candidate, where entropy carries no information. */
+    readonly normalizedEntropy: number | undefined;
+    readonly candidateCount: number;
+    readonly logCandidateCount: number;
+}
+/**
+ * Summary features of a candidate distribution.
+ *
+ * A single candidate can show probability 1 while telling you nothing about
+ * whether that candidate is correct, so its margin and entropy are undefined
+ * rather than maximally confident.
+ */
+export declare function distributionFeatures(probabilities: readonly number[]): DistributionFeatures;
+/**
+ * Post-hoc temperature rule: p̃(a) ∝ max(p(a), ε)^(1/T).
+ *
+ * T is fitted on a calibration split, never on the data being scored. This
+ * does not assume the input came from a softmax; whether it helps has to be
+ * measured separately.
+ */
+export declare function temperatureScale(probabilities: readonly number[], temperature: number, epsilon?: number): number[];
+export declare function brierScore(forecasts: readonly number[], outcomes: readonly number[]): number;
+export declare function negativeLogLoss(forecasts: readonly number[], outcomes: readonly number[], clip?: number): number;
+export interface CalibrationBin {
+    readonly lower: number;
+    readonly upper: number;
+    readonly count: number;
+    readonly accuracy: number | undefined;
+    readonly confidence: number | undefined;
+}
+export interface CalibrationReport {
+    readonly error: number;
+    readonly bins: readonly CalibrationBin[];
+    readonly samples: number;
+}
+/**
+ * Expected calibration error with its bin occupancy.
+ *
+ * ECE is sensitive to binning, so the bins and their counts are returned with
+ * it: a near-zero error over three samples is not evidence of calibration.
+ */
+export declare function expectedCalibrationError(forecasts: readonly number[], outcomes: readonly number[], binCount: number): CalibrationReport;
+export interface SelectiveExecutionReport {
+    readonly admitted: number;
+    readonly coverage: number;
+    /** Undefined when nothing was admitted: that is unmeasurable, not risk-free. */
+    readonly risk: number | undefined;
+}
+/**
+ * Selective execution gate: a(τ) = 1[q̂ ≥ τ] · 1[G = 1].
+ *
+ * Risk and coverage are returned together because refusing everything drives
+ * measured risk to zero while delivering nothing.
+ */
+export declare function selectiveExecution(scores: readonly number[], appropriate: readonly number[], threshold: number, policyGates: readonly boolean[]): SelectiveExecutionReport;
+//# sourceMappingURL=calibration-selective.d.ts.map

package/dist/metacognition/calibration-selective.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibration-selective.d.ts","sourceRoot":"","sources":["../../src/metacognition/calibration-selective.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,4FAA4F;AAC5F,eAAO,MAAM,mBAAmB,QAAQ,CAAC;AACzC,wEAAwE;AACxE,eAAO,MAAM,aAAa,QAAQ,CAAC;AAEnC,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,oFAAkF;IAClF,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,8EAA8E;IAC9E,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/C,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,iBAAiB,EAAE,MAAM,CAAC;CACnC;AAYD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,GAAG,oBAAoB,CAwB3F;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC/B,aAAa,EAAE,SAAS,MAAM,EAAE,EAChC,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,MAA4B,GACnC,MAAM,EAAE,CAWV;AAYD,wBAAgB,UAAU,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAK5F;AAED,wBAAgB,eAAe,CAC9B,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,IAAI,GAAE,MAAsB,GAC1B,MAAM,CASR;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,CAAC;IACzC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CACzB;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,CACvC,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,QAAQ,EAAE,SAAS,MAAM,EAAE,EAC3B,QAAQ,EAAE,MAAM,GACd,iBAAiB,CAsBnB;AAED,MAAM,WAAW,wBAAwB;IACxC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,gFAAgF;IAChF,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CAClC;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CACjC,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,WAAW,EAAE,SAAS,MAAM,EAAE,EAC9B,SAAS,EAAE,MAAM,EACjB,WAAW,EAAE,SAAS,OAAO,EAAE,GAC7B,wBAAwB,CAmB1B","sourcesContent":["/**\n * Probability calibration and selective execution (Jev audit algorithm A3).\n *\n * A provider's `confidence` is a statistic over its own option distribution,\n * not a calibrated probability of any event we care about. Before it can gate\n * execution, the predicted event has to be named — \"the chosen candidate was\n * appropriate\" is a different event from \"the driver executed it\" and from\n * \"the task succeeded\" — and the score has to be measured on held-out data.\n *\n * These are the measurement primitives, deliberately small: distribution\n * features, a post-hoc temperature rule, scoring rules, and a selective gate\n * that always reports coverage next to risk. None of them makes a score\n * trustworthy; they make it measurable.\n */\n\nimport { ensure } from \"./validation.ts\";\n\nconst SUM_TOLERANCE = 1e-9;\n/** Floor for numeric zeros in temperature scaling; recorded so results are reproducible. */\nexport const TEMPERATURE_EPSILON = 1e-12;\n/** Clip for log loss so one confident miss does not return Infinity. */\nexport const LOG_LOSS_CLIP = 1e-15;\n\nexport interface DistributionFeatures {\n\treadonly maxProbability: number;\n\t/** p(1) − p(2). Undefined for a single candidate: there is nothing to compare. */\n\treadonly margin: number | undefined;\n\t/** Undefined for a single candidate, where entropy carries no information. */\n\treadonly normalizedEntropy: number | undefined;\n\treadonly candidateCount: number;\n\treadonly logCandidateCount: number;\n}\n\nfunction assertDistribution(probabilities: readonly number[]): void {\n\tensure(Array.isArray(probabilities) && probabilities.length > 0, \"distribution must be non-empty\");\n\tlet total = 0;\n\tfor (const p of probabilities) {\n\t\tensure(typeof p === \"number\" && Number.isFinite(p) && p >= 0 && p <= 1, \"each probability must be in [0,1]\");\n\t\ttotal += p;\n\t}\n\tensure(Math.abs(total - 1) <= SUM_TOLERANCE, \"distribution must sum to one\");\n}\n\n/**\n * Summary features of a candidate distribution.\n *\n * A single candidate can show probability 1 while telling you nothing about\n * whether that candidate is correct, so its margin and entropy are undefined\n * rather than maximally confident.\n */\nexport function distributionFeatures(probabilities: readonly number[]): DistributionFeatures {\n\tassertDistribution(probabilities);\n\tconst sorted = [...probabilities].sort((a, b) => b - a);\n\tconst count = probabilities.length;\n\tconst maxProbability = sorted[0] ?? 0;\n\tif (count === 1) {\n\t\treturn {\n\t\t\tmaxProbability,\n\t\t\tmargin: undefined,\n\t\t\tnormalizedEntropy: undefined,\n\t\t\tcandidateCount: 1,\n\t\t\tlogCandidateCount: 0,\n\t\t};\n\t}\n\t// 0·log0 is defined as 0; skipping the term is exactly that definition.\n\tlet entropy = 0;\n\tfor (const p of probabilities) if (p > 0) entropy -= p * Math.log(p);\n\treturn {\n\t\tmaxProbability,\n\t\tmargin: maxProbability - (sorted[1] ?? 0),\n\t\tnormalizedEntropy: entropy / Math.log(count),\n\t\tcandidateCount: count,\n\t\tlogCandidateCount: Math.log(count),\n\t};\n}\n\n/**\n * Post-hoc temperature rule: p̃(a) ∝ max(p(a), ε)^(1/T).\n *\n * T is fitted on a calibration split, never on the data being scored. This\n * does not assume the input came from a softmax; whether it helps has to be\n * measured separately.\n */\nexport function temperatureScale(\n\tprobabilities: readonly number[],\n\ttemperature: number,\n\tepsilon: number = TEMPERATURE_EPSILON,\n): number[] {\n\tassertDistribution(probabilities);\n\tensure(\n\t\ttypeof temperature === \"number\" && Number.isFinite(temperature) && temperature > 0,\n\t\t\"temperature must be positive\",\n\t);\n\tensure(typeof epsilon === \"number\" && epsilon > 0 && epsilon < 1, \"epsilon must be in (0,1)\");\n\tconst raised = probabilities.map((p) => Math.max(p, epsilon) ** (1 / temperature));\n\tconst total = raised.reduce((sum, value) => sum + value, 0);\n\tensure(total > 0 && Number.isFinite(total), \"temperature scaling produced a degenerate distribution\");\n\treturn raised.map((value) => value / total);\n}\n\nfunction assertBinaryPaired(forecasts: readonly number[], outcomes: readonly number[]): void {\n\tensure(Array.isArray(forecasts) && Array.isArray(outcomes), \"forecasts and outcomes must be arrays\");\n\tensure(forecasts.length === outcomes.length, \"forecasts and outcomes must align\");\n\tensure(forecasts.length > 0, \"at least one paired observation is required\");\n\tfor (const q of forecasts) {\n\t\tensure(typeof q === \"number\" && Number.isFinite(q) && q >= 0 && q <= 1, \"each forecast must be in [0,1]\");\n\t}\n\tfor (const y of outcomes) ensure(y === 0 || y === 1, \"each outcome must be 0 or 1\");\n}\n\nexport function brierScore(forecasts: readonly number[], outcomes: readonly number[]): number {\n\tassertBinaryPaired(forecasts, outcomes);\n\tlet total = 0;\n\tfor (const [i, q] of forecasts.entries()) total += (q - outcomes[i]!) ** 2;\n\treturn total / forecasts.length;\n}\n\nexport function negativeLogLoss(\n\tforecasts: readonly number[],\n\toutcomes: readonly number[],\n\tclip: number = LOG_LOSS_CLIP,\n): number {\n\tassertBinaryPaired(forecasts, outcomes);\n\tensure(typeof clip === \"number\" && clip > 0 && clip < 0.5, \"clip must be in (0,0.5)\");\n\tlet total = 0;\n\tfor (const [i, raw] of forecasts.entries()) {\n\t\tconst q = Math.min(1 - clip, Math.max(clip, raw));\n\t\ttotal -= outcomes[i] === 1 ? Math.log(q) : Math.log(1 - q);\n\t}\n\treturn total / forecasts.length;\n}\n\nexport interface CalibrationBin {\n\treadonly lower: number;\n\treadonly upper: number;\n\treadonly count: number;\n\treadonly accuracy: number | undefined;\n\treadonly confidence: number | undefined;\n}\n\nexport interface CalibrationReport {\n\treadonly error: number;\n\treadonly bins: readonly CalibrationBin[];\n\treadonly samples: number;\n}\n\n/**\n * Expected calibration error with its bin occupancy.\n *\n * ECE is sensitive to binning, so the bins and their counts are returned with\n * it: a near-zero error over three samples is not evidence of calibration.\n */\nexport function expectedCalibrationError(\n\tforecasts: readonly number[],\n\toutcomes: readonly number[],\n\tbinCount: number,\n): CalibrationReport {\n\tassertBinaryPaired(forecasts, outcomes);\n\tensure(Number.isSafeInteger(binCount) && binCount > 0, \"binCount must be a positive integer\");\n\tconst sums = Array.from({ length: binCount }, () => ({ count: 0, outcome: 0, forecast: 0 }));\n\tfor (const [i, q] of forecasts.entries()) {\n\t\tconst index = Math.min(binCount - 1, Math.floor(q * binCount));\n\t\tconst bin = sums[index]!;\n\t\tbin.count += 1;\n\t\tbin.outcome += outcomes[i]!;\n\t\tbin.forecast += q;\n\t}\n\tconst samples = forecasts.length;\n\tlet error = 0;\n\tconst bins = sums.map((bin, index) => {\n\t\tconst accuracy = bin.count > 0 ? bin.outcome / bin.count : undefined;\n\t\tconst confidence = bin.count > 0 ? bin.forecast / bin.count : undefined;\n\t\tif (accuracy !== undefined && confidence !== undefined) {\n\t\t\terror += (bin.count / samples) * Math.abs(accuracy - confidence);\n\t\t}\n\t\treturn { lower: index / binCount, upper: (index + 1) / binCount, count: bin.count, accuracy, confidence };\n\t});\n\treturn { error, bins, samples };\n}\n\nexport interface SelectiveExecutionReport {\n\treadonly admitted: number;\n\treadonly coverage: number;\n\t/** Undefined when nothing was admitted: that is unmeasurable, not risk-free. */\n\treadonly risk: number | undefined;\n}\n\n/**\n * Selective execution gate: a(τ) = 1[q̂ ≥ τ] · 1[G = 1].\n *\n * Risk and coverage are returned together because refusing everything drives\n * measured risk to zero while delivering nothing.\n */\nexport function selectiveExecution(\n\tscores: readonly number[],\n\tappropriate: readonly number[],\n\tthreshold: number,\n\tpolicyGates: readonly boolean[],\n): SelectiveExecutionReport {\n\tassertBinaryPaired(scores, appropriate);\n\tensure(policyGates.length === scores.length, \"policy gates must align with scores\");\n\tensure(\n\t\ttypeof threshold === \"number\" && Number.isFinite(threshold) && threshold >= 0 && threshold <= 1,\n\t\t\"threshold must be in [0,1]\",\n\t);\n\tlet admitted = 0;\n\tlet errors = 0;\n\tfor (const [i, score] of scores.entries()) {\n\t\tif (score < threshold || policyGates[i] !== true) continue;\n\t\tadmitted += 1;\n\t\tif (appropriate[i] === 0) errors += 1;\n\t}\n\treturn {\n\t\tadmitted,\n\t\tcoverage: admitted / scores.length,\n\t\trisk: admitted > 0 ? errors / admitted : undefined,\n\t};\n}\n"]}

package/dist/metacognition/calibration-selective.js

@@ -0,0 +1,161 @@
+/**
+ * Probability calibration and selective execution (Jev audit algorithm A3).
+ *
+ * A provider's `confidence` is a statistic over its own option distribution,
+ * not a calibrated probability of any event we care about. Before it can gate
+ * execution, the predicted event has to be named — "the chosen candidate was
+ * appropriate" is a different event from "the driver executed it" and from
+ * "the task succeeded" — and the score has to be measured on held-out data.
+ *
+ * These are the measurement primitives, deliberately small: distribution
+ * features, a post-hoc temperature rule, scoring rules, and a selective gate
+ * that always reports coverage next to risk. None of them makes a score
+ * trustworthy; they make it measurable.
+ */
+import { ensure } from "./validation.js";
+const SUM_TOLERANCE = 1e-9;
+/** Floor for numeric zeros in temperature scaling; recorded so results are reproducible. */
+export const TEMPERATURE_EPSILON = 1e-12;
+/** Clip for log loss so one confident miss does not return Infinity. */
+export const LOG_LOSS_CLIP = 1e-15;
+function assertDistribution(probabilities) {
+    ensure(Array.isArray(probabilities) && probabilities.length > 0, "distribution must be non-empty");
+    let total = 0;
+    for (const p of probabilities) {
+        ensure(typeof p === "number" && Number.isFinite(p) && p >= 0 && p <= 1, "each probability must be in [0,1]");
+        total += p;
+    }
+    ensure(Math.abs(total - 1) <= SUM_TOLERANCE, "distribution must sum to one");
+}
+/**
+ * Summary features of a candidate distribution.
+ *
+ * A single candidate can show probability 1 while telling you nothing about
+ * whether that candidate is correct, so its margin and entropy are undefined
+ * rather than maximally confident.
+ */
+export function distributionFeatures(probabilities) {
+    assertDistribution(probabilities);
+    const sorted = [...probabilities].sort((a, b) => b - a);
+    const count = probabilities.length;
+    const maxProbability = sorted[0] ?? 0;
+    if (count === 1) {
+        return {
+            maxProbability,
+            margin: undefined,
+            normalizedEntropy: undefined,
+            candidateCount: 1,
+            logCandidateCount: 0,
+        };
+    }
+    // 0·log0 is defined as 0; skipping the term is exactly that definition.
+    let entropy = 0;
+    for (const p of probabilities)
+        if (p > 0)
+            entropy -= p * Math.log(p);
+    return {
+        maxProbability,
+        margin: maxProbability - (sorted[1] ?? 0),
+        normalizedEntropy: entropy / Math.log(count),
+        candidateCount: count,
+        logCandidateCount: Math.log(count),
+    };
+}
+/**
+ * Post-hoc temperature rule: p̃(a) ∝ max(p(a), ε)^(1/T).
+ *
+ * T is fitted on a calibration split, never on the data being scored. This
+ * does not assume the input came from a softmax; whether it helps has to be
+ * measured separately.
+ */
+export function temperatureScale(probabilities, temperature, epsilon = TEMPERATURE_EPSILON) {
+    assertDistribution(probabilities);
+    ensure(typeof temperature === "number" && Number.isFinite(temperature) && temperature > 0, "temperature must be positive");
+    ensure(typeof epsilon === "number" && epsilon > 0 && epsilon < 1, "epsilon must be in (0,1)");
+    const raised = probabilities.map((p) => Math.max(p, epsilon) ** (1 / temperature));
+    const total = raised.reduce((sum, value) => sum + value, 0);
+    ensure(total > 0 && Number.isFinite(total), "temperature scaling produced a degenerate distribution");
+    return raised.map((value) => value / total);
+}
+function assertBinaryPaired(forecasts, outcomes) {
+    ensure(Array.isArray(forecasts) && Array.isArray(outcomes), "forecasts and outcomes must be arrays");
+    ensure(forecasts.length === outcomes.length, "forecasts and outcomes must align");
+    ensure(forecasts.length > 0, "at least one paired observation is required");
+    for (const q of forecasts) {
+        ensure(typeof q === "number" && Number.isFinite(q) && q >= 0 && q <= 1, "each forecast must be in [0,1]");
+    }
+    for (const y of outcomes)
+        ensure(y === 0 || y === 1, "each outcome must be 0 or 1");
+}
+export function brierScore(forecasts, outcomes) {
+    assertBinaryPaired(forecasts, outcomes);
+    let total = 0;
+    for (const [i, q] of forecasts.entries())
+        total += (q - outcomes[i]) ** 2;
+    return total / forecasts.length;
+}
+export function negativeLogLoss(forecasts, outcomes, clip = LOG_LOSS_CLIP) {
+    assertBinaryPaired(forecasts, outcomes);
+    ensure(typeof clip === "number" && clip > 0 && clip < 0.5, "clip must be in (0,0.5)");
+    let total = 0;
+    for (const [i, raw] of forecasts.entries()) {
+        const q = Math.min(1 - clip, Math.max(clip, raw));
+        total -= outcomes[i] === 1 ? Math.log(q) : Math.log(1 - q);
+    }
+    return total / forecasts.length;
+}
+/**
+ * Expected calibration error with its bin occupancy.
+ *
+ * ECE is sensitive to binning, so the bins and their counts are returned with
+ * it: a near-zero error over three samples is not evidence of calibration.
+ */
+export function expectedCalibrationError(forecasts, outcomes, binCount) {
+    assertBinaryPaired(forecasts, outcomes);
+    ensure(Number.isSafeInteger(binCount) && binCount > 0, "binCount must be a positive integer");
+    const sums = Array.from({ length: binCount }, () => ({ count: 0, outcome: 0, forecast: 0 }));
+    for (const [i, q] of forecasts.entries()) {
+        const index = Math.min(binCount - 1, Math.floor(q * binCount));
+        const bin = sums[index];
+        bin.count += 1;
+        bin.outcome += outcomes[i];
+        bin.forecast += q;
+    }
+    const samples = forecasts.length;
+    let error = 0;
+    const bins = sums.map((bin, index) => {
+        const accuracy = bin.count > 0 ? bin.outcome / bin.count : undefined;
+        const confidence = bin.count > 0 ? bin.forecast / bin.count : undefined;
+        if (accuracy !== undefined && confidence !== undefined) {
+            error += (bin.count / samples) * Math.abs(accuracy - confidence);
+        }
+        return { lower: index / binCount, upper: (index + 1) / binCount, count: bin.count, accuracy, confidence };
+    });
+    return { error, bins, samples };
+}
+/**
+ * Selective execution gate: a(τ) = 1[q̂ ≥ τ] · 1[G = 1].
+ *
+ * Risk and coverage are returned together because refusing everything drives
+ * measured risk to zero while delivering nothing.
+ */
+export function selectiveExecution(scores, appropriate, threshold, policyGates) {
+    assertBinaryPaired(scores, appropriate);
+    ensure(policyGates.length === scores.length, "policy gates must align with scores");
+    ensure(typeof threshold === "number" && Number.isFinite(threshold) && threshold >= 0 && threshold <= 1, "threshold must be in [0,1]");
+    let admitted = 0;
+    let errors = 0;
+    for (const [i, score] of scores.entries()) {
+        if (score < threshold || policyGates[i] !== true)
+            continue;
+        admitted += 1;
+        if (appropriate[i] === 0)
+            errors += 1;
+    }
+    return {
+        admitted,
+        coverage: admitted / scores.length,
+        risk: admitted > 0 ? errors / admitted : undefined,
+    };
+}
+//# sourceMappingURL=calibration-selective.js.map

package/dist/metacognition/calibration-selective.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibration-selective.js","sourceRoot":"","sources":["../../src/metacognition/calibration-selective.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,MAAM,aAAa,GAAG,IAAI,CAAC;AAC3B,4FAA4F;AAC5F,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,CAAC;AACzC,wEAAwE;AACxE,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,CAAC;AAYnC,SAAS,kBAAkB,CAAC,aAAgC,EAAQ;IACnE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,gCAAgC,CAAC,CAAC;IACnG,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;QAC/B,MAAM,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,mCAAmC,CAAC,CAAC;QAC7G,KAAK,IAAI,CAAC,CAAC;IACZ,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,IAAI,aAAa,EAAE,8BAA8B,CAAC,CAAC;AAAA,CAC7E;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,aAAgC,EAAwB;IAC5F,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC;IACnC,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACtC,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO;YACN,cAAc;YACd,MAAM,EAAE,SAAS;YACjB,iBAAiB,EAAE,SAAS;YAC5B,cAAc,EAAE,CAAC;YACjB,iBAAiB,EAAE,CAAC;SACpB,CAAC;IACH,CAAC;IACD,yEAAwE;IACxE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,aAAa;QAAE,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACrE,OAAO;QACN,cAAc;QACd,MAAM,EAAE,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACzC,iBAAiB,EAAE,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;QAC5C,cAAc,EAAE,KAAK;QACrB,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;KAClC,CAAC;AAAA,CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC/B,aAAgC,EAChC,WAAmB,EACnB,OAAO,GAAW,mBAAmB,EAC1B;IACX,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAClC,MAAM,CACL,OAAO,WAAW,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,EAClF,8BAA8B,CAC9B,CAAC;IACF,MAAM,CAAC,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,EAAE,0BAA0B,CAAC,CAAC;IAC9F,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5D,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,wDAAwD,CAAC,CAAC;IACtG,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC;AAAA,CAC5C;AAED,SAAS,kBAAkB,CAAC,SAA4B,EAAE,QAA2B,EAAQ;IAC5F,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,uCAAuC,CAAC,CAAC;IACrG,MAAM,CAAC,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,mCAAmC,CAAC,CAAC;IAClF,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,6CAA6C,CAAC,CAAC;IAC5E,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,gCAAgC,CAAC,CAAC;IAC3G,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,QAAQ;QAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,6BAA6B,CAAC,CAAC;AAAA,CACpF;AAED,MAAM,UAAU,UAAU,CAAC,SAA4B,EAAE,QAA2B,EAAU;IAC7F,kBAAkB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACxC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE;QAAE,KAAK,IAAI,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC;IAC3E,OAAO,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;AAAA,CAChC;AAED,MAAM,UAAU,eAAe,CAC9B,SAA4B,EAC5B,QAA2B,EAC3B,IAAI,GAAW,aAAa,EACnB;IACT,kBAAkB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACxC,MAAM,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,GAAG,EAAE,yBAAyB,CAAC,CAAC;IACtF,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QAClD,KAAK,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,OAAO,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC;AAAA,CAChC;AAgBD;;;;;GAKG;AACH,MAAM,UAAU,wBAAwB,CACvC,SAA4B,EAC5B,QAA2B,EAC3B,QAAgB,EACI;IACpB,kBAAkB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IACxC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,qCAAqC,CAAC,CAAC;IAC9F,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7F,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC;QAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAE,CAAC;QACzB,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;QACf,GAAG,CAAC,OAAO,IAAI,QAAQ,CAAC,CAAC,CAAE,CAAC;QAC5B,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;IACnB,CAAC;IACD,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC;IACjC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC;QACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACrE,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACxE,IAAI,QAAQ,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YACxD,KAAK,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,KAAK,GAAG,QAAQ,EAAE,KAAK,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC;IAAA,CAC1G,CAAC,CAAC;IACH,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAAA,CAChC;AASD;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CACjC,MAAyB,EACzB,WAA8B,EAC9B,SAAiB,EACjB,WAA+B,EACJ;IAC3B,kBAAkB,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACxC,MAAM,CAAC,WAAW,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,qCAAqC,CAAC,CAAC;IACpF,MAAM,CACL,OAAO,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,SAAS,IAAI,CAAC,IAAI,SAAS,IAAI,CAAC,EAC/F,4BAA4B,CAC5B,CAAC;IACF,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3C,IAAI,KAAK,GAAG,SAAS,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,IAAI;YAAE,SAAS;QAC3D,QAAQ,IAAI,CAAC,CAAC;QACd,IAAI,WAAW,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,MAAM,IAAI,CAAC,CAAC;IACvC,CAAC;IACD,OAAO;QACN,QAAQ;QACR,QAAQ,EAAE,QAAQ,GAAG,MAAM,CAAC,MAAM;QAClC,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,SAAS;KAClD,CAAC;AAAA,CACF","sourcesContent":["/**\n * Probability calibration and selective execution (Jev audit algorithm A3).\n *\n * A provider's `confidence` is a statistic over its own option distribution,\n * not a calibrated probability of any event we care about. Before it can gate\n * execution, the predicted event has to be named — \"the chosen candidate was\n * appropriate\" is a different event from \"the driver executed it\" and from\n * \"the task succeeded\" — and the score has to be measured on held-out data.\n *\n * These are the measurement primitives, deliberately small: distribution\n * features, a post-hoc temperature rule, scoring rules, and a selective gate\n * that always reports coverage next to risk. None of them makes a score\n * trustworthy; they make it measurable.\n */\n\nimport { ensure } from \"./validation.ts\";\n\nconst SUM_TOLERANCE = 1e-9;\n/** Floor for numeric zeros in temperature scaling; recorded so results are reproducible. */\nexport const TEMPERATURE_EPSILON = 1e-12;\n/** Clip for log loss so one confident miss does not return Infinity. */\nexport const LOG_LOSS_CLIP = 1e-15;\n\nexport interface DistributionFeatures {\n\treadonly maxProbability: number;\n\t/** p(1) − p(2). Undefined for a single candidate: there is nothing to compare. */\n\treadonly margin: number | undefined;\n\t/** Undefined for a single candidate, where entropy carries no information. */\n\treadonly normalizedEntropy: number | undefined;\n\treadonly candidateCount: number;\n\treadonly logCandidateCount: number;\n}\n\nfunction assertDistribution(probabilities: readonly number[]): void {\n\tensure(Array.isArray(probabilities) && probabilities.length > 0, \"distribution must be non-empty\");\n\tlet total = 0;\n\tfor (const p of probabilities) {\n\t\tensure(typeof p === \"number\" && Number.isFinite(p) && p >= 0 && p <= 1, \"each probability must be in [0,1]\");\n\t\ttotal += p;\n\t}\n\tensure(Math.abs(total - 1) <= SUM_TOLERANCE, \"distribution must sum to one\");\n}\n\n/**\n * Summary features of a candidate distribution.\n *\n * A single candidate can show probability 1 while telling you nothing about\n * whether that candidate is correct, so its margin and entropy are undefined\n * rather than maximally confident.\n */\nexport function distributionFeatures(probabilities: readonly number[]): DistributionFeatures {\n\tassertDistribution(probabilities);\n\tconst sorted = [...probabilities].sort((a, b) => b - a);\n\tconst count = probabilities.length;\n\tconst maxProbability = sorted[0] ?? 0;\n\tif (count === 1) {\n\t\treturn {\n\t\t\tmaxProbability,\n\t\t\tmargin: undefined,\n\t\t\tnormalizedEntropy: undefined,\n\t\t\tcandidateCount: 1,\n\t\t\tlogCandidateCount: 0,\n\t\t};\n\t}\n\t// 0·log0 is defined as 0; skipping the term is exactly that definition.\n\tlet entropy = 0;\n\tfor (const p of probabilities) if (p > 0) entropy -= p * Math.log(p);\n\treturn {\n\t\tmaxProbability,\n\t\tmargin: maxProbability - (sorted[1] ?? 0),\n\t\tnormalizedEntropy: entropy / Math.log(count),\n\t\tcandidateCount: count,\n\t\tlogCandidateCount: Math.log(count),\n\t};\n}\n\n/**\n * Post-hoc temperature rule: p̃(a) ∝ max(p(a), ε)^(1/T).\n *\n * T is fitted on a calibration split, never on the data being scored. This\n * does not assume the input came from a softmax; whether it helps has to be\n * measured separately.\n */\nexport function temperatureScale(\n\tprobabilities: readonly number[],\n\ttemperature: number,\n\tepsilon: number = TEMPERATURE_EPSILON,\n): number[] {\n\tassertDistribution(probabilities);\n\tensure(\n\t\ttypeof temperature === \"number\" && Number.isFinite(temperature) && temperature > 0,\n\t\t\"temperature must be positive\",\n\t);\n\tensure(typeof epsilon === \"number\" && epsilon > 0 && epsilon < 1, \"epsilon must be in (0,1)\");\n\tconst raised = probabilities.map((p) => Math.max(p, epsilon) ** (1 / temperature));\n\tconst total = raised.reduce((sum, value) => sum + value, 0);\n\tensure(total > 0 && Number.isFinite(total), \"temperature scaling produced a degenerate distribution\");\n\treturn raised.map((value) => value / total);\n}\n\nfunction assertBinaryPaired(forecasts: readonly number[], outcomes: readonly number[]): void {\n\tensure(Array.isArray(forecasts) && Array.isArray(outcomes), \"forecasts and outcomes must be arrays\");\n\tensure(forecasts.length === outcomes.length, \"forecasts and outcomes must align\");\n\tensure(forecasts.length > 0, \"at least one paired observation is required\");\n\tfor (const q of forecasts) {\n\t\tensure(typeof q === \"number\" && Number.isFinite(q) && q >= 0 && q <= 1, \"each forecast must be in [0,1]\");\n\t}\n\tfor (const y of outcomes) ensure(y === 0 || y === 1, \"each outcome must be 0 or 1\");\n}\n\nexport function brierScore(forecasts: readonly number[], outcomes: readonly number[]): number {\n\tassertBinaryPaired(forecasts, outcomes);\n\tlet total = 0;\n\tfor (const [i, q] of forecasts.entries()) total += (q - outcomes[i]!) ** 2;\n\treturn total / forecasts.length;\n}\n\nexport function negativeLogLoss(\n\tforecasts: readonly number[],\n\toutcomes: readonly number[],\n\tclip: number = LOG_LOSS_CLIP,\n): number {\n\tassertBinaryPaired(forecasts, outcomes);\n\tensure(typeof clip === \"number\" && clip > 0 && clip < 0.5, \"clip must be in (0,0.5)\");\n\tlet total = 0;\n\tfor (const [i, raw] of forecasts.entries()) {\n\t\tconst q = Math.min(1 - clip, Math.max(clip, raw));\n\t\ttotal -= outcomes[i] === 1 ? Math.log(q) : Math.log(1 - q);\n\t}\n\treturn total / forecasts.length;\n}\n\nexport interface CalibrationBin {\n\treadonly lower: number;\n\treadonly upper: number;\n\treadonly count: number;\n\treadonly accuracy: number | undefined;\n\treadonly confidence: number | undefined;\n}\n\nexport interface CalibrationReport {\n\treadonly error: number;\n\treadonly bins: readonly CalibrationBin[];\n\treadonly samples: number;\n}\n\n/**\n * Expected calibration error with its bin occupancy.\n *\n * ECE is sensitive to binning, so the bins and their counts are returned with\n * it: a near-zero error over three samples is not evidence of calibration.\n */\nexport function expectedCalibrationError(\n\tforecasts: readonly number[],\n\toutcomes: readonly number[],\n\tbinCount: number,\n): CalibrationReport {\n\tassertBinaryPaired(forecasts, outcomes);\n\tensure(Number.isSafeInteger(binCount) && binCount > 0, \"binCount must be a positive integer\");\n\tconst sums = Array.from({ length: binCount }, () => ({ count: 0, outcome: 0, forecast: 0 }));\n\tfor (const [i, q] of forecasts.entries()) {\n\t\tconst index = Math.min(binCount - 1, Math.floor(q * binCount));\n\t\tconst bin = sums[index]!;\n\t\tbin.count += 1;\n\t\tbin.outcome += outcomes[i]!;\n\t\tbin.forecast += q;\n\t}\n\tconst samples = forecasts.length;\n\tlet error = 0;\n\tconst bins = sums.map((bin, index) => {\n\t\tconst accuracy = bin.count > 0 ? bin.outcome / bin.count : undefined;\n\t\tconst confidence = bin.count > 0 ? bin.forecast / bin.count : undefined;\n\t\tif (accuracy !== undefined && confidence !== undefined) {\n\t\t\terror += (bin.count / samples) * Math.abs(accuracy - confidence);\n\t\t}\n\t\treturn { lower: index / binCount, upper: (index + 1) / binCount, count: bin.count, accuracy, confidence };\n\t});\n\treturn { error, bins, samples };\n}\n\nexport interface SelectiveExecutionReport {\n\treadonly admitted: number;\n\treadonly coverage: number;\n\t/** Undefined when nothing was admitted: that is unmeasurable, not risk-free. */\n\treadonly risk: number | undefined;\n}\n\n/**\n * Selective execution gate: a(τ) = 1[q̂ ≥ τ] · 1[G = 1].\n *\n * Risk and coverage are returned together because refusing everything drives\n * measured risk to zero while delivering nothing.\n */\nexport function selectiveExecution(\n\tscores: readonly number[],\n\tappropriate: readonly number[],\n\tthreshold: number,\n\tpolicyGates: readonly boolean[],\n): SelectiveExecutionReport {\n\tassertBinaryPaired(scores, appropriate);\n\tensure(policyGates.length === scores.length, \"policy gates must align with scores\");\n\tensure(\n\t\ttypeof threshold === \"number\" && Number.isFinite(threshold) && threshold >= 0 && threshold <= 1,\n\t\t\"threshold must be in [0,1]\",\n\t);\n\tlet admitted = 0;\n\tlet errors = 0;\n\tfor (const [i, score] of scores.entries()) {\n\t\tif (score < threshold || policyGates[i] !== true) continue;\n\t\tadmitted += 1;\n\t\tif (appropriate[i] === 0) errors += 1;\n\t}\n\treturn {\n\t\tadmitted,\n\t\tcoverage: admitted / scores.length,\n\t\trisk: admitted > 0 ? errors / admitted : undefined,\n\t};\n}\n"]}

package/dist/metacognition/calibration.d.ts

@@ -0,0 +1,62 @@
+export interface ConditionKey {
+    readonly modelRevision: string;
+    readonly skillHashes: readonly string[];
+    readonly toolchain: string;
+    readonly taskBand: string;
+    readonly checkKind: string;
+    readonly checkHealth: "healthy" | "degraded" | "unverified";
+    readonly budgetBand: string;
+    readonly policyVersion: string;
+}
+export type BucketState = "insufficient-data" | "active" | "demoted";
+export interface CalibrationBucket {
+    readonly key: ConditionKey;
+    readonly keyHash: string;
+    readonly priorAlpha: number;
+    readonly priorBeta: number;
+    successes: number;
+    failures: number;
+    drift: number;
+    state: BucketState;
+    readonly demotedReason?: string;
+}
+export interface CalibrationStore {
+    readonly buckets: Readonly<Record<string, CalibrationBucket>>;
+    readonly minSamples: number;
+    readonly referenceMean: number;
+    readonly slack: number;
+    readonly threshold: number;
+}
+export declare function conditionKeyHash(key: ConditionKey): string;
+export declare function createCalibrationStore(options: {
+    readonly minSamples: number;
+    readonly priorAlpha: number;
+    readonly priorBeta: number;
+    readonly referenceMean: number;
+    readonly slack: number;
+    readonly threshold: number;
+}): CalibrationStore;
+/**
+ * Record one independent task outcome. Never count 100 runs of one task as
+ * 100 independent tasks; the caller supplies one pre-registered binary outcome
+ * per task. Environment failures belong in delivery metrics, not here.
+ */
+export declare function recordOutcome(store: CalibrationStore, key: ConditionKey, outcome: "success" | "failure", loss: number): CalibrationStore;
+/**
+ * Demote every bucket whose condition changed underneath it: new model
+ * revision, new check definition, or new major toolchain version. Explicit
+ * condition changes never wait for statistical drift detection.
+ */
+export declare function demoteOnConditionChange(store: CalibrationStore, change: {
+    modelRevision?: string;
+    checkKind?: string;
+    toolchain?: string;
+    policyVersion?: string;
+}): CalibrationStore;
+/** Posterior mean success rate for one bucket, or `insufficient-data`. */
+export declare function conditionalSuccessRate(store: CalibrationStore, key: ConditionKey): {
+    state: BucketState;
+    rate?: number;
+    samples: number;
+};
+//# sourceMappingURL=calibration.d.ts.map

package/dist/metacognition/calibration.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibration.d.ts","sourceRoot":"","sources":["../../src/metacognition/calibration.ts"],"names":[],"mappings":"AAcA,MAAM,WAAW,YAAY;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,EAAE,SAAS,GAAG,UAAU,GAAG,YAAY,CAAC;IAC5D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;CAC/B;AACD,MAAM,MAAM,WAAW,GAAG,mBAAmB,GAAG,QAAQ,GAAG,SAAS,CAAC;AACrE,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,WAAW,CAAC;IACnB,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CAChC;AACD,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC;IAC9D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC3B;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,YAAY,GAAG,MAAM,CAkB1D;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE;IAC/C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC3B,GAAG,gBAAgB,CAenB;AAkBD;;;;GAIG;AACH,wBAAgB,aAAa,CAC5B,KAAK,EAAE,gBAAgB,EACvB,GAAG,EAAE,YAAY,EACjB,OAAO,EAAE,SAAS,GAAG,SAAS,EAC9B,IAAI,EAAE,MAAM,GACV,gBAAgB,CAoBlB;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CACtC,KAAK,EAAE,gBAAgB,EACvB,MAAM,EAAE;IAAE,aAAa,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,aAAa,CAAC,EAAE,MAAM,CAAA;CAAE,GAChG,gBAAgB,CAclB;AAED,0EAA0E;AAC1E,wBAAgB,sBAAsB,CACrC,KAAK,EAAE,gBAAgB,EACvB,GAAG,EAAE,YAAY,GACf;IAAE,KAAK,EAAE,WAAW,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAUxD","sourcesContent":["/**\n * Algorithms E and G — conditional capability/calibration records and\n * drift-triggered demotion. Spec §8, §10.3.\n *\n * Records are per condition bucket: model revision + loaded skill content\n * fingerprints + toolchain version + task/effect/difficulty band + check\n * type/health + budget band + selection policy version. A bucket with too few\n * samples stays `insufficient-data`; a new provider revision, check\n * definition, or major library version demotes the bucket immediately,\n * without waiting for statistical detection.\n */\nimport { betaPosteriorMean, driftStatistic } from \"./decision.ts\";\nimport { canonical, ensure, finite, integer, lexical, member, text } from \"./validation.ts\";\n\nexport interface ConditionKey {\n\treadonly modelRevision: string;\n\treadonly skillHashes: readonly string[];\n\treadonly toolchain: string;\n\treadonly taskBand: string;\n\treadonly checkKind: string;\n\treadonly checkHealth: \"healthy\" | \"degraded\" | \"unverified\";\n\treadonly budgetBand: string;\n\treadonly policyVersion: string;\n}\nexport type BucketState = \"insufficient-data\" | \"active\" | \"demoted\";\nexport interface CalibrationBucket {\n\treadonly key: ConditionKey;\n\treadonly keyHash: string;\n\treadonly priorAlpha: number;\n\treadonly priorBeta: number;\n\tsuccesses: number;\n\tfailures: number;\n\tdrift: number;\n\tstate: BucketState;\n\treadonly demotedReason?: string;\n}\nexport interface CalibrationStore {\n\treadonly buckets: Readonly<Record<string, CalibrationBucket>>;\n\treadonly minSamples: number;\n\treadonly referenceMean: number;\n\treadonly slack: number;\n\treadonly threshold: number;\n}\n\nexport function conditionKeyHash(key: ConditionKey): string {\n\ttext(key.modelRevision, \"modelRevision\", 128);\n\ttext(key.toolchain, \"toolchain\", 128);\n\ttext(key.taskBand, \"taskBand\", 128);\n\ttext(key.checkKind, \"checkKind\", 128);\n\tmember(key.checkHealth, [\"healthy\", \"degraded\", \"unverified\"], \"checkHealth\");\n\ttext(key.budgetBand, \"budgetBand\", 128);\n\ttext(key.policyVersion, \"policyVersion\", 64);\n\treturn canonical([\n\t\tkey.modelRevision,\n\t\t[...key.skillHashes].sort(lexical),\n\t\tkey.toolchain,\n\t\tkey.taskBand,\n\t\tkey.checkKind,\n\t\tkey.checkHealth,\n\t\tkey.budgetBand,\n\t\tkey.policyVersion,\n\t]);\n}\n\nexport function createCalibrationStore(options: {\n\treadonly minSamples: number;\n\treadonly priorAlpha: number;\n\treadonly priorBeta: number;\n\treadonly referenceMean: number;\n\treadonly slack: number;\n\treadonly threshold: number;\n}): CalibrationStore {\n\tinteger(options.minSamples, \"minSamples\", 100_000);\n\tfinite(options.priorAlpha, \"priorAlpha\", 1e9);\n\tfinite(options.priorBeta, \"priorBeta\", 1e9);\n\tensure(options.priorAlpha > 0 && options.priorBeta > 0, \"beta prior must be positive\");\n\tfinite(options.referenceMean, \"referenceMean\");\n\tfinite(options.slack, \"slack\");\n\tfinite(options.threshold, \"threshold\");\n\treturn {\n\t\tbuckets: {},\n\t\tminSamples: options.minSamples,\n\t\treferenceMean: options.referenceMean,\n\t\tslack: options.slack,\n\t\tthreshold: options.threshold,\n\t};\n}\n\nfunction bucketFor(store: CalibrationStore, key: ConditionKey): CalibrationBucket {\n\tconst keyHash = conditionKeyHash(key);\n\treturn (\n\t\tstore.buckets[keyHash] ?? {\n\t\t\tkey,\n\t\t\tkeyHash,\n\t\t\tpriorAlpha: 1,\n\t\t\tpriorBeta: 1,\n\t\t\tsuccesses: 0,\n\t\t\tfailures: 0,\n\t\t\tdrift: 0,\n\t\t\tstate: \"insufficient-data\",\n\t\t}\n\t);\n}\n\n/**\n * Record one independent task outcome. Never count 100 runs of one task as\n * 100 independent tasks; the caller supplies one pre-registered binary outcome\n * per task. Environment failures belong in delivery metrics, not here.\n */\nexport function recordOutcome(\n\tstore: CalibrationStore,\n\tkey: ConditionKey,\n\toutcome: \"success\" | \"failure\",\n\tloss: number,\n): CalibrationStore {\n\tmember(outcome, [\"success\", \"failure\"], \"outcome\");\n\tfinite(loss, \"loss\");\n\tconst bucket = bucketFor(store, key);\n\tif (bucket.state === \"demoted\") return store;\n\tconst next: CalibrationBucket = {\n\t\t...bucket,\n\t\tsuccesses: bucket.successes + (outcome === \"success\" ? 1 : 0),\n\t\tfailures: bucket.failures + (outcome === \"failure\" ? 1 : 0),\n\t\tdrift: driftStatistic(bucket.drift, loss, store.referenceMean, store.slack),\n\t\tstate: bucket.successes + bucket.failures + 1 >= store.minSamples ? \"active\" : \"insufficient-data\",\n\t};\n\tconst demoted = next.drift > store.threshold;\n\treturn {\n\t\t...store,\n\t\tbuckets: {\n\t\t\t...store.buckets,\n\t\t\t[next.keyHash]: demoted ? { ...next, state: \"demoted\", demotedReason: \"drift-threshold\" } : next,\n\t\t},\n\t};\n}\n\n/**\n * Demote every bucket whose condition changed underneath it: new model\n * revision, new check definition, or new major toolchain version. Explicit\n * condition changes never wait for statistical drift detection.\n */\nexport function demoteOnConditionChange(\n\tstore: CalibrationStore,\n\tchange: { modelRevision?: string; checkKind?: string; toolchain?: string; policyVersion?: string },\n): CalibrationStore {\n\tconst buckets: Record<string, CalibrationBucket> = {};\n\tfor (const [hash, bucket] of Object.entries(store.buckets)) {\n\t\tconst changed =\n\t\t\t(change.modelRevision !== undefined && bucket.key.modelRevision !== change.modelRevision) ||\n\t\t\t(change.checkKind !== undefined && bucket.key.checkKind !== change.checkKind) ||\n\t\t\t(change.toolchain !== undefined && bucket.key.toolchain !== change.toolchain) ||\n\t\t\t(change.policyVersion !== undefined && bucket.key.policyVersion !== change.policyVersion);\n\t\tbuckets[hash] =\n\t\t\tchanged && bucket.state !== \"demoted\"\n\t\t\t\t? { ...bucket, state: \"demoted\", demotedReason: \"condition-change\" }\n\t\t\t\t: bucket;\n\t}\n\treturn { ...store, buckets };\n}\n\n/** Posterior mean success rate for one bucket, or `insufficient-data`. */\nexport function conditionalSuccessRate(\n\tstore: CalibrationStore,\n\tkey: ConditionKey,\n): { state: BucketState; rate?: number; samples: number } {\n\tconst bucket = store.buckets[conditionKeyHash(key)];\n\tif (!bucket) return { state: \"insufficient-data\", samples: 0 };\n\tconst samples = bucket.successes + bucket.failures;\n\tif (bucket.state !== \"active\") return { state: bucket.state, samples };\n\treturn {\n\t\tstate: \"active\",\n\t\tsamples,\n\t\trate: betaPosteriorMean(bucket.priorAlpha, bucket.priorBeta, bucket.successes, bucket.failures),\n\t};\n}\n"]}

package/dist/metacognition/calibration.js

@@ -0,0 +1,122 @@
+/**
+ * Algorithms E and G — conditional capability/calibration records and
+ * drift-triggered demotion. Spec §8, §10.3.
+ *
+ * Records are per condition bucket: model revision + loaded skill content
+ * fingerprints + toolchain version + task/effect/difficulty band + check
+ * type/health + budget band + selection policy version. A bucket with too few
+ * samples stays `insufficient-data`; a new provider revision, check
+ * definition, or major library version demotes the bucket immediately,
+ * without waiting for statistical detection.
+ */
+import { betaPosteriorMean, driftStatistic } from "./decision.js";
+import { canonical, ensure, finite, integer, lexical, member, text } from "./validation.js";
+export function conditionKeyHash(key) {
+    text(key.modelRevision, "modelRevision", 128);
+    text(key.toolchain, "toolchain", 128);
+    text(key.taskBand, "taskBand", 128);
+    text(key.checkKind, "checkKind", 128);
+    member(key.checkHealth, ["healthy", "degraded", "unverified"], "checkHealth");
+    text(key.budgetBand, "budgetBand", 128);
+    text(key.policyVersion, "policyVersion", 64);
+    return canonical([
+        key.modelRevision,
+        [...key.skillHashes].sort(lexical),
+        key.toolchain,
+        key.taskBand,
+        key.checkKind,
+        key.checkHealth,
+        key.budgetBand,
+        key.policyVersion,
+    ]);
+}
+export function createCalibrationStore(options) {
+    integer(options.minSamples, "minSamples", 100_000);
+    finite(options.priorAlpha, "priorAlpha", 1e9);
+    finite(options.priorBeta, "priorBeta", 1e9);
+    ensure(options.priorAlpha > 0 && options.priorBeta > 0, "beta prior must be positive");
+    finite(options.referenceMean, "referenceMean");
+    finite(options.slack, "slack");
+    finite(options.threshold, "threshold");
+    return {
+        buckets: {},
+        minSamples: options.minSamples,
+        referenceMean: options.referenceMean,
+        slack: options.slack,
+        threshold: options.threshold,
+    };
+}
+function bucketFor(store, key) {
+    const keyHash = conditionKeyHash(key);
+    return (store.buckets[keyHash] ?? {
+        key,
+        keyHash,
+        priorAlpha: 1,
+        priorBeta: 1,
+        successes: 0,
+        failures: 0,
+        drift: 0,
+        state: "insufficient-data",
+    });
+}
+/**
+ * Record one independent task outcome. Never count 100 runs of one task as
+ * 100 independent tasks; the caller supplies one pre-registered binary outcome
+ * per task. Environment failures belong in delivery metrics, not here.
+ */
+export function recordOutcome(store, key, outcome, loss) {
+    member(outcome, ["success", "failure"], "outcome");
+    finite(loss, "loss");
+    const bucket = bucketFor(store, key);
+    if (bucket.state === "demoted")
+        return store;
+    const next = {
+        ...bucket,
+        successes: bucket.successes + (outcome === "success" ? 1 : 0),
+        failures: bucket.failures + (outcome === "failure" ? 1 : 0),
+        drift: driftStatistic(bucket.drift, loss, store.referenceMean, store.slack),
+        state: bucket.successes + bucket.failures + 1 >= store.minSamples ? "active" : "insufficient-data",
+    };
+    const demoted = next.drift > store.threshold;
+    return {
+        ...store,
+        buckets: {
+            ...store.buckets,
+            [next.keyHash]: demoted ? { ...next, state: "demoted", demotedReason: "drift-threshold" } : next,
+        },
+    };
+}
+/**
+ * Demote every bucket whose condition changed underneath it: new model
+ * revision, new check definition, or new major toolchain version. Explicit
+ * condition changes never wait for statistical drift detection.
+ */
+export function demoteOnConditionChange(store, change) {
+    const buckets = {};
+    for (const [hash, bucket] of Object.entries(store.buckets)) {
+        const changed = (change.modelRevision !== undefined && bucket.key.modelRevision !== change.modelRevision) ||
+            (change.checkKind !== undefined && bucket.key.checkKind !== change.checkKind) ||
+            (change.toolchain !== undefined && bucket.key.toolchain !== change.toolchain) ||
+            (change.policyVersion !== undefined && bucket.key.policyVersion !== change.policyVersion);
+        buckets[hash] =
+            changed && bucket.state !== "demoted"
+                ? { ...bucket, state: "demoted", demotedReason: "condition-change" }
+                : bucket;
+    }
+    return { ...store, buckets };
+}
+/** Posterior mean success rate for one bucket, or `insufficient-data`. */
+export function conditionalSuccessRate(store, key) {
+    const bucket = store.buckets[conditionKeyHash(key)];
+    if (!bucket)
+        return { state: "insufficient-data", samples: 0 };
+    const samples = bucket.successes + bucket.failures;
+    if (bucket.state !== "active")
+        return { state: bucket.state, samples };
+    return {
+        state: "active",
+        samples,
+        rate: betaPosteriorMean(bucket.priorAlpha, bucket.priorBeta, bucket.successes, bucket.failures),
+    };
+}
+//# sourceMappingURL=calibration.js.map

package/dist/metacognition/calibration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"calibration.js","sourceRoot":"","sources":["../../src/metacognition/calibration.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AACH,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAClE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAC;AAgC5F,MAAM,UAAU,gBAAgB,CAAC,GAAiB,EAAU;IAC3D,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC;IACtC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC;IACpC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC;IACtC,MAAM,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,YAAY,CAAC,EAAE,aAAa,CAAC,CAAC;IAC9E,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,aAAa,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;IAC7C,OAAO,SAAS,CAAC;QAChB,GAAG,CAAC,aAAa;QACjB,CAAC,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;QAClC,GAAG,CAAC,SAAS;QACb,GAAG,CAAC,QAAQ;QACZ,GAAG,CAAC,SAAS;QACb,GAAG,CAAC,WAAW;QACf,GAAG,CAAC,UAAU;QACd,GAAG,CAAC,aAAa;KACjB,CAAC,CAAC;AAAA,CACH;AAED,MAAM,UAAU,sBAAsB,CAAC,OAOtC,EAAoB;IACpB,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IACnD,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC;IAC9C,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC;IAC5C,MAAM,CAAC,OAAO,CAAC,UAAU,GAAG,CAAC,IAAI,OAAO,CAAC,SAAS,GAAG,CAAC,EAAE,6BAA6B,CAAC,CAAC;IACvF,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IAC/C,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IACvC,OAAO;QACN,OAAO,EAAE,EAAE;QACX,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC5B,CAAC;AAAA,CACF;AAED,SAAS,SAAS,CAAC,KAAuB,EAAE,GAAiB,EAAqB;IACjF,MAAM,OAAO,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO,CACN,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI;QACzB,GAAG;QACH,OAAO;QACP,UAAU,EAAE,CAAC;QACb,SAAS,EAAE,CAAC;QACZ,SAAS,EAAE,CAAC;QACZ,QAAQ,EAAE,CAAC;QACX,KAAK,EAAE,CAAC;QACR,KAAK,EAAE,mBAAmB;KAC1B,CACD,CAAC;AAAA,CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC5B,KAAuB,EACvB,GAAiB,EACjB,OAA8B,EAC9B,IAAY,EACO;IACnB,MAAM,CAAC,OAAO,EAAE,CAAC,SAAS,EAAE,SAAS,CAAC,EAAE,SAAS,CAAC,CAAC;IACnD,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACrB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACrC,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IAC7C,MAAM,IAAI,GAAsB;QAC/B,GAAG,MAAM;QACT,SAAS,EAAE,MAAM,CAAC,SAAS,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7D,QAAQ,EAAE,MAAM,CAAC,QAAQ,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,KAAK,EAAE,cAAc,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,KAAK,CAAC;QAC3E,KAAK,EAAE,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,mBAAmB;KAClG,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,SAAS,CAAC;IAC7C,OAAO;QACN,GAAG,KAAK;QACR,OAAO,EAAE;YACR,GAAG,KAAK,CAAC,OAAO;YAChB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,IAAI;SAChG;KACD,CAAC;AAAA,CACF;AAED;;;;GAIG;AACH,MAAM,UAAU,uBAAuB,CACtC,KAAuB,EACvB,MAAkG,EAC/E;IACnB,MAAM,OAAO,GAAsC,EAAE,CAAC;IACtD,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5D,MAAM,OAAO,GACZ,CAAC,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,CAAC,aAAa,CAAC;YACzF,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS,CAAC;YAC7E,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS,CAAC;YAC7E,CAAC,MAAM,CAAC,aAAa,KAAK,SAAS,IAAI,MAAM,CAAC,GAAG,CAAC,aAAa,KAAK,MAAM,CAAC,aAAa,CAAC,CAAC;QAC3F,OAAO,CAAC,IAAI,CAAC;YACZ,OAAO,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS;gBACpC,CAAC,CAAC,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,kBAAkB,EAAE;gBACpE,CAAC,CAAC,MAAM,CAAC;IACZ,CAAC;IACD,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,CAAC;AAAA,CAC7B;AAED,0EAA0E;AAC1E,MAAM,UAAU,sBAAsB,CACrC,KAAuB,EACvB,GAAiB,EACwC;IACzD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,mBAAmB,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC/D,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,GAAG,MAAM,CAAC,QAAQ,CAAC;IACnD,IAAI,MAAM,CAAC,KAAK,KAAK,QAAQ;QAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC;IACvE,OAAO;QACN,KAAK,EAAE,QAAQ;QACf,OAAO;QACP,IAAI,EAAE,iBAAiB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC;KAC/F,CAAC;AAAA,CACF","sourcesContent":["/**\n * Algorithms E and G — conditional capability/calibration records and\n * drift-triggered demotion. Spec §8, §10.3.\n *\n * Records are per condition bucket: model revision + loaded skill content\n * fingerprints + toolchain version + task/effect/difficulty band + check\n * type/health + budget band + selection policy version. A bucket with too few\n * samples stays `insufficient-data`; a new provider revision, check\n * definition, or major library version demotes the bucket immediately,\n * without waiting for statistical detection.\n */\nimport { betaPosteriorMean, driftStatistic } from \"./decision.ts\";\nimport { canonical, ensure, finite, integer, lexical, member, text } from \"./validation.ts\";\n\nexport interface ConditionKey {\n\treadonly modelRevision: string;\n\treadonly skillHashes: readonly string[];\n\treadonly toolchain: string;\n\treadonly taskBand: string;\n\treadonly checkKind: string;\n\treadonly checkHealth: \"healthy\" | \"degraded\" | \"unverified\";\n\treadonly budgetBand: string;\n\treadonly policyVersion: string;\n}\nexport type BucketState = \"insufficient-data\" | \"active\" | \"demoted\";\nexport interface CalibrationBucket {\n\treadonly key: ConditionKey;\n\treadonly keyHash: string;\n\treadonly priorAlpha: number;\n\treadonly priorBeta: number;\n\tsuccesses: number;\n\tfailures: number;\n\tdrift: number;\n\tstate: BucketState;\n\treadonly demotedReason?: string;\n}\nexport interface CalibrationStore {\n\treadonly buckets: Readonly<Record<string, CalibrationBucket>>;\n\treadonly minSamples: number;\n\treadonly referenceMean: number;\n\treadonly slack: number;\n\treadonly threshold: number;\n}\n\nexport function conditionKeyHash(key: ConditionKey): string {\n\ttext(key.modelRevision, \"modelRevision\", 128);\n\ttext(key.toolchain, \"toolchain\", 128);\n\ttext(key.taskBand, \"taskBand\", 128);\n\ttext(key.checkKind, \"checkKind\", 128);\n\tmember(key.checkHealth, [\"healthy\", \"degraded\", \"unverified\"], \"checkHealth\");\n\ttext(key.budgetBand, \"budgetBand\", 128);\n\ttext(key.policyVersion, \"policyVersion\", 64);\n\treturn canonical([\n\t\tkey.modelRevision,\n\t\t[...key.skillHashes].sort(lexical),\n\t\tkey.toolchain,\n\t\tkey.taskBand,\n\t\tkey.checkKind,\n\t\tkey.checkHealth,\n\t\tkey.budgetBand,\n\t\tkey.policyVersion,\n\t]);\n}\n\nexport function createCalibrationStore(options: {\n\treadonly minSamples: number;\n\treadonly priorAlpha: number;\n\treadonly priorBeta: number;\n\treadonly referenceMean: number;\n\treadonly slack: number;\n\treadonly threshold: number;\n}): CalibrationStore {\n\tinteger(options.minSamples, \"minSamples\", 100_000);\n\tfinite(options.priorAlpha, \"priorAlpha\", 1e9);\n\tfinite(options.priorBeta, \"priorBeta\", 1e9);\n\tensure(options.priorAlpha > 0 && options.priorBeta > 0, \"beta prior must be positive\");\n\tfinite(options.referenceMean, \"referenceMean\");\n\tfinite(options.slack, \"slack\");\n\tfinite(options.threshold, \"threshold\");\n\treturn {\n\t\tbuckets: {},\n\t\tminSamples: options.minSamples,\n\t\treferenceMean: options.referenceMean,\n\t\tslack: options.slack,\n\t\tthreshold: options.threshold,\n\t};\n}\n\nfunction bucketFor(store: CalibrationStore, key: ConditionKey): CalibrationBucket {\n\tconst keyHash = conditionKeyHash(key);\n\treturn (\n\t\tstore.buckets[keyHash] ?? {\n\t\t\tkey,\n\t\t\tkeyHash,\n\t\t\tpriorAlpha: 1,\n\t\t\tpriorBeta: 1,\n\t\t\tsuccesses: 0,\n\t\t\tfailures: 0,\n\t\t\tdrift: 0,\n\t\t\tstate: \"insufficient-data\",\n\t\t}\n\t);\n}\n\n/**\n * Record one independent task outcome. Never count 100 runs of one task as\n * 100 independent tasks; the caller supplies one pre-registered binary outcome\n * per task. Environment failures belong in delivery metrics, not here.\n */\nexport function recordOutcome(\n\tstore: CalibrationStore,\n\tkey: ConditionKey,\n\toutcome: \"success\" | \"failure\",\n\tloss: number,\n): CalibrationStore {\n\tmember(outcome, [\"success\", \"failure\"], \"outcome\");\n\tfinite(loss, \"loss\");\n\tconst bucket = bucketFor(store, key);\n\tif (bucket.state === \"demoted\") return store;\n\tconst next: CalibrationBucket = {\n\t\t...bucket,\n\t\tsuccesses: bucket.successes + (outcome === \"success\" ? 1 : 0),\n\t\tfailures: bucket.failures + (outcome === \"failure\" ? 1 : 0),\n\t\tdrift: driftStatistic(bucket.drift, loss, store.referenceMean, store.slack),\n\t\tstate: bucket.successes + bucket.failures + 1 >= store.minSamples ? \"active\" : \"insufficient-data\",\n\t};\n\tconst demoted = next.drift > store.threshold;\n\treturn {\n\t\t...store,\n\t\tbuckets: {\n\t\t\t...store.buckets,\n\t\t\t[next.keyHash]: demoted ? { ...next, state: \"demoted\", demotedReason: \"drift-threshold\" } : next,\n\t\t},\n\t};\n}\n\n/**\n * Demote every bucket whose condition changed underneath it: new model\n * revision, new check definition, or new major toolchain version. Explicit\n * condition changes never wait for statistical drift detection.\n */\nexport function demoteOnConditionChange(\n\tstore: CalibrationStore,\n\tchange: { modelRevision?: string; checkKind?: string; toolchain?: string; policyVersion?: string },\n): CalibrationStore {\n\tconst buckets: Record<string, CalibrationBucket> = {};\n\tfor (const [hash, bucket] of Object.entries(store.buckets)) {\n\t\tconst changed =\n\t\t\t(change.modelRevision !== undefined && bucket.key.modelRevision !== change.modelRevision) ||\n\t\t\t(change.checkKind !== undefined && bucket.key.checkKind !== change.checkKind) ||\n\t\t\t(change.toolchain !== undefined && bucket.key.toolchain !== change.toolchain) ||\n\t\t\t(change.policyVersion !== undefined && bucket.key.policyVersion !== change.policyVersion);\n\t\tbuckets[hash] =\n\t\t\tchanged && bucket.state !== \"demoted\"\n\t\t\t\t? { ...bucket, state: \"demoted\", demotedReason: \"condition-change\" }\n\t\t\t\t: bucket;\n\t}\n\treturn { ...store, buckets };\n}\n\n/** Posterior mean success rate for one bucket, or `insufficient-data`. */\nexport function conditionalSuccessRate(\n\tstore: CalibrationStore,\n\tkey: ConditionKey,\n): { state: BucketState; rate?: number; samples: number } {\n\tconst bucket = store.buckets[conditionKeyHash(key)];\n\tif (!bucket) return { state: \"insufficient-data\", samples: 0 };\n\tconst samples = bucket.successes + bucket.failures;\n\tif (bucket.state !== \"active\") return { state: bucket.state, samples };\n\treturn {\n\t\tstate: \"active\",\n\t\tsamples,\n\t\trate: betaPosteriorMean(bucket.priorAlpha, bucket.priorBeta, bucket.successes, bucket.failures),\n\t};\n}\n"]}

package/dist/metacognition/checkpoint.d.ts

@@ -0,0 +1,60 @@
+/**
+ * §9.4 checkpoint ordering: refresh obligations from host observations,
+ * evaluate knowledge gaps, classify prediction mismatches, evaluate verifier
+ * blind spots and runner health, then select one bounded action.
+ *
+ * The host calls this at a safe decision boundary — never mid-stream, and
+ * never as a path for model reasoning to remove a `required` obligation.
+ */
+import type { ChangeAtom, ObligationRule } from "./obligations.ts";
+import { type ActionConstraints, type MetaAction, type MetaActionKind } from "./policy.ts";
+import type { MetaState } from "./state.ts";
+import { type FinishState } from "./state.ts";
+import { type MutantRecord, type VerifierEvaluation } from "./verifier.ts";
+export interface CheckpointInput {
+    readonly state: MetaState;
+    readonly atoms?: readonly ChangeAtom[];
+    readonly rules?: readonly ObligationRule[];
+    readonly nowMs: number;
+    readonly constraints: ActionConstraints;
+    readonly mutants?: readonly MutantRecord[];
+    readonly receipt?: {
+        receiptId: string;
+        candidateHash: string;
+        scope: string;
+    } | null;
+    readonly options?: {
+        readonly maxRepetitions?: number;
+        readonly switchCost?: number;
+        readonly estimatedSwitchGain?: number;
+        readonly voiCandidates?: readonly {
+            kind: MetaActionKind;
+            voi: number;
+        }[];
+        readonly progress?: {
+            readonly goalScope: string;
+            readonly candidateFamily: string;
+            readonly errorClass: string;
+            readonly approach: string;
+            readonly checkMethod: string;
+            readonly sourceFamily: string;
+        };
+    };
+}
+export interface CheckpointResult {
+    readonly state: MetaState;
+    readonly action: MetaAction;
+    readonly finish: FinishState;
+    readonly newObligations: number;
+    readonly newPredictions: number;
+    readonly mismatches: number;
+    readonly verifierEvaluations: readonly VerifierEvaluation[];
+}
+/**
+ * One checkpoint evaluation. Refreshes obligations from observed atoms when
+ * provided, resolves registered predictions against fresh observations when
+ * provided, evaluates verifier mutants when provided, then selects the next
+ * action under the §14 priority table.
+ */
+export declare function checkpoint(input: CheckpointInput): CheckpointResult;
+//# sourceMappingURL=checkpoint.d.ts.map

package/dist/metacognition/checkpoint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint.d.ts","sourceRoot":"","sources":["../../src/metacognition/checkpoint.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,KAAK,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAEnE,OAAO,EAAE,KAAK,iBAAiB,EAAE,KAAK,UAAU,EAAE,KAAK,cAAc,EAAgB,MAAM,aAAa,CAAC;AACzG,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,KAAK,WAAW,EAAkC,MAAM,YAAY,CAAC;AAE9E,OAAO,EAAoB,KAAK,YAAY,EAAE,KAAK,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAE7F,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,UAAU,EAAE,CAAC;IACvC,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAC3C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,iBAAiB,CAAC;IACxC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IAC3C,QAAQ,CAAC,OAAO,CAAC,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACtF,QAAQ,CAAC,OAAO,CAAC,EAAE;QAClB,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QACjC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,mBAAmB,CAAC,EAAE,MAAM,CAAC;QACtC,QAAQ,CAAC,aAAa,CAAC,EAAE,SAAS;YAAE,IAAI,EAAE,cAAc,CAAC;YAAC,GAAG,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAC1E,QAAQ,CAAC,QAAQ,CAAC,EAAE;YACnB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;YAC3B,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;YACjC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;YAC5B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;YAC1B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;YAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;SAC9B,CAAC;KACF,CAAC;CACF;AACD,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,mBAAmB,EAAE,SAAS,kBAAkB,EAAE,CAAC;CAC5D;AAED;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,eAAe,GAAG,gBAAgB,CA4CnE","sourcesContent":["/**\n * §9.4 checkpoint ordering: refresh obligations from host observations,\n * evaluate knowledge gaps, classify prediction mismatches, evaluate verifier\n * blind spots and runner health, then select one bounded action.\n *\n * The host calls this at a safe decision boundary — never mid-stream, and\n * never as a path for model reasoning to remove a `required` obligation.\n */\nimport type { ChangeAtom, ObligationRule } from \"./obligations.ts\";\nimport { instantiateObligations } from \"./obligations.ts\";\nimport { type ActionConstraints, type MetaAction, type MetaActionKind, selectAction } from \"./policy.ts\";\nimport type { MetaState } from \"./state.ts\";\nimport { type FinishState, finishState, validateMetaState } from \"./state.ts\";\nimport { integer } from \"./validation.ts\";\nimport { evaluateVerifier, type MutantRecord, type VerifierEvaluation } from \"./verifier.ts\";\n\nexport interface CheckpointInput {\n\treadonly state: MetaState;\n\treadonly atoms?: readonly ChangeAtom[];\n\treadonly rules?: readonly ObligationRule[];\n\treadonly nowMs: number;\n\treadonly constraints: ActionConstraints;\n\treadonly mutants?: readonly MutantRecord[];\n\treadonly receipt?: { receiptId: string; candidateHash: string; scope: string } | null;\n\treadonly options?: {\n\t\treadonly maxRepetitions?: number;\n\t\treadonly switchCost?: number;\n\t\treadonly estimatedSwitchGain?: number;\n\t\treadonly voiCandidates?: readonly { kind: MetaActionKind; voi: number }[];\n\t\treadonly progress?: {\n\t\t\treadonly goalScope: string;\n\t\t\treadonly candidateFamily: string;\n\t\t\treadonly errorClass: string;\n\t\t\treadonly approach: string;\n\t\t\treadonly checkMethod: string;\n\t\t\treadonly sourceFamily: string;\n\t\t};\n\t};\n}\nexport interface CheckpointResult {\n\treadonly state: MetaState;\n\treadonly action: MetaAction;\n\treadonly finish: FinishState;\n\treadonly newObligations: number;\n\treadonly newPredictions: number;\n\treadonly mismatches: number;\n\treadonly verifierEvaluations: readonly VerifierEvaluation[];\n}\n\n/**\n * One checkpoint evaluation. Refreshes obligations from observed atoms when\n * provided, resolves registered predictions against fresh observations when\n * provided, evaluates verifier mutants when provided, then selects the next\n * action under the §14 priority table.\n */\nexport function checkpoint(input: CheckpointInput): CheckpointResult {\n\tinteger(input.nowMs, \"nowMs\");\n\tvalidateMetaState(input.state);\n\tlet state = input.state;\n\tlet newObligations = 0;\n\tif (input.atoms && input.rules) {\n\t\tconst report = instantiateObligations(input.atoms, input.rules, input.nowMs);\n\t\tconst merged = [...state.obligations, ...report.required, ...report.candidates];\n\t\tnewObligations = report.required.length + report.candidates.length;\n\t\tstate = { ...state, obligations: merged };\n\t}\n\tlet mismatches = 0;\n\tfor (const p of state.predictions) {\n\t\tif (p.status === \"resolved\" && p.mismatch !== undefined && p.mismatch !== \"none\") mismatches += 1;\n\t}\n\tconst verifierEvaluations: VerifierEvaluation[] = [...state.verifier.evaluations];\n\tif (input.mutants) {\n\t\tfor (const obligation of state.obligations) {\n\t\t\tconst scoped = input.mutants.filter((m) => m.obligationId === obligation.id);\n\t\t\tif (scoped.length === 0) continue;\n\t\t\tverifierEvaluations.push(\n\t\t\t\tevaluateVerifier(obligation.id, input.mutants, state.verifier.runnerHealth === \"healthy\"),\n\t\t\t);\n\t\t}\n\t\tstate = { ...state, verifier: { ...state.verifier, evaluations: verifierEvaluations } };\n\t}\n\tconst action = selectAction(state, input.constraints, {\n\t\treceipt: input.receipt ?? null,\n\t\tmaxRepetitions: input.options?.maxRepetitions,\n\t\tswitchCost: input.options?.switchCost,\n\t\testimatedSwitchGain: input.options?.estimatedSwitchGain,\n\t\tvoiCandidates: input.options?.voiCandidates,\n\t\tprogress: input.options?.progress,\n\t});\n\tconst finish = finishState(state, input.receipt ?? null);\n\treturn {\n\t\tstate: { ...state, checkpointCount: state.checkpointCount + 1 },\n\t\taction,\n\t\tfinish,\n\t\tnewObligations,\n\t\tnewPredictions: 0,\n\t\tmismatches,\n\t\tverifierEvaluations,\n\t};\n}\n"]}