attention-manifest-protocol 0.4.2 → 0.4.3

package/src/bayes/calibration.ts

@@ -1,204 +0,0 @@
-/**
- * Post-hoc calibration utilities.
- *
- * The model's posterior is not trusted as-is; we measure ECE/Brier and fit a
- * monotone isotonic map only for evaluation. Cross-validation is mandatory on
- * small samples so the "improvement" is not just in-sample overfit.
- */
-export interface CalibrationPoint {
-    predicted: number;
-    actual: boolean;
-}
-
-export interface IsotonicModel {
-    /**
-     * Upper predicted bound of each block.
-     */
-    thresholds: number[];
-    /**
-     * Observed frequency of each block.
-     */
-    values: number[];
-}
-
-/**
- * Calibration statistics for one set of predictions.
- */
-type CalibrationReport = {
-    ece: number;
-    brier: number;
-    rawEce: number;
-    rawBrier: number;
-};
-
-/**
- * One isotonic block: summed outcomes, sample count and upper bound.
- */
-type Block = {
-    sum: number;
-    n: number;
-    max: number;
-};
-
-/**
- * One reliability bin.
- */
-type Bucket = {
-    n: number;
-    confidence: number;
-    accuracy: number;
-};
-
-/**
- * Expected calibration error over optionally binned predictions.
- */
-type EceScore = (points: readonly CalibrationPoint[], bins?: number) => number;
-
-/**
- * Fit an isotonic map to the given points.
- */
-type FitIsotonic = (points: readonly CalibrationPoint[]) => IsotonicModel;
-
-/**
- * Fit a temperature scalar to the given points.
- */
-type FitTemperature = (points: readonly CalibrationPoint[], grid?: readonly number[]) => TemperatureModel;
-
-/**
- * Apply a temperature scalar to one prediction.
- */
-type ApplyTemperature = (model: TemperatureModel, predicted: number) => number;
-
-/**
- * Cross-validated calibration report.
- */
-type CrossValidate = (points: readonly CalibrationPoint[], folds?: number) => CalibrationReport;
-
-const DefaultGrid: readonly number[] = [0.25, 0.5, 0.75, 1, 1.25, 1.5, 2, 2.5, 3, 4, 5];
-
-const clamp01: (value: number) => number = (value: number): number => Math.max(0, Math.min(1, value));
-
-export const brier: (points: readonly CalibrationPoint[]) => number = (points: readonly CalibrationPoint[]): number => {
-    if (points.length === 0) {
-        return 0;
-    }
-    return points.reduce((sum, point) => sum + (point.predicted - (point.actual ? 1 : 0)) ** 2, 0) / points.length;
-};
-
-export const ece: EceScore = (points: readonly CalibrationPoint[], bins: number = 10): number => {
-    if (points.length === 0) {
-        return 0;
-    }
-    const buckets: Bucket[] = Array.from({ length: bins }, () => ({ n: 0, confidence: 0, accuracy: 0 }));
-    for (let i: number = 0; i < points.length; i += 1) {
-        const point: CalibrationPoint = points[i];
-        const index: number = Math.min(bins - 1, Math.max(0, Math.floor(clamp01(point.predicted) * bins)));
-        buckets[index].n += 1;
-        buckets[index].confidence += point.predicted;
-        buckets[index].accuracy += point.actual ? 1 : 0;
-    }
-    return buckets.reduce((sum, bucket) => {
-        if (bucket.n === 0) {
-            return sum;
-        }
-        return sum + (bucket.n / points.length) * Math.abs(bucket.accuracy / bucket.n - bucket.confidence / bucket.n);
-    }, 0);
-};
-
-/**
- * Pool-adjacent-violators isotonic regression.
- */
-export const fitIsotonic: FitIsotonic = (points: readonly CalibrationPoint[]): IsotonicModel => {
-    const sorted: CalibrationPoint[] = [...points].sort((left, right) => left.predicted - right.predicted);
-    const blocks: Block[] = [];
-    for (let i: number = 0; i < sorted.length; i += 1) {
-        const point: CalibrationPoint = sorted[i];
-        blocks.push({ sum: point.actual ? 1 : 0, n: 1, max: point.predicted });
-        while (blocks.length >= 2) {
-            const current: Block = blocks[blocks.length - 1];
-            const previous: Block = blocks[blocks.length - 2];
-            if (previous.sum / previous.n <= current.sum / current.n) {
-                break;
-            }
-            blocks.splice(blocks.length - 2, 2, {
-                sum: previous.sum + current.sum,
-                n: previous.n + current.n,
-                max: current.max,
-            });
-        }
-    }
-    return {
-        thresholds: blocks.map((block) => block.max),
-        values: blocks.map((block) => block.sum / block.n),
-    };
-};
-
-export const applyIsotonic: (model: IsotonicModel, predicted: number) => number = (model: IsotonicModel, predicted: number): number => {
-    if (model.thresholds.length === 0) {
-        return clamp01(predicted);
-    }
-    const index: number = model.thresholds.findIndex((threshold) => predicted <= threshold);
-    return model.values[index === -1 ? model.values.length - 1 : index];
-};
-
-export interface TemperatureModel {
-    temperature: number;
-}
-
-const logit: (value: number) => number = (value: number): number => {
-    const bounded: number = Math.max(1e-6, Math.min(1 - 1e-6, value));
-    return Math.log(bounded / (1 - bounded));
-};
-
-const sigmoid: (value: number) => number = (value: number): number => 1 / (1 + Math.exp(-value));
-
-/**
- * Grid-fit temperature by NLL; T<1 sharpens, T>1 softens.
- */
-export const fitTemperature: FitTemperature = (points: readonly CalibrationPoint[], grid: readonly number[] = DefaultGrid): TemperatureModel => {
-    if (points.length === 0) {
-        return { temperature: 1 };
-    }
-    let best: number = 1;
-    let bestLoss: number = Number.POSITIVE_INFINITY;
-    for (let i: number = 0; i < grid.length; i += 1) {
-        const temperature: number = grid[i];
-        const loss: number = points.reduce((sum, point) => {
-            const calibrated: number = sigmoid(logit(point.predicted) / temperature);
-            const target: number = point.actual ? 1 : 0;
-            return sum - (target * Math.log(Math.max(1e-9, calibrated)) + (1 - target) * Math.log(Math.max(1e-9, 1 - calibrated)));
-        }, 0);
-        if (loss < bestLoss) {
-            bestLoss = loss;
-            best = temperature;
-        }
-    }
-    return { temperature: best };
-};
-
-export const applyTemperature: ApplyTemperature = (model: TemperatureModel, predicted: number): number => sigmoid(logit(predicted) / model.temperature);
-
-export const crossValidateCalibration: CrossValidate = (points: readonly CalibrationPoint[], folds: number = 5): CalibrationReport => {
-    if (points.length === 0) {
-        return { ece: 0, brier: 0, rawEce: 0, rawBrier: 0 };
-    }
-    const predictions: CalibrationPoint[] = [];
-    for (let fold: number = 0; fold < folds; fold += 1) {
-        const train: CalibrationPoint[] = points.filter((_point, index) => index % folds !== fold);
-        const test: CalibrationPoint[] = points.filter((_point, index) => index % folds === fold);
-        if (train.length === 0 || test.length === 0) {
-            continue;
-        }
-        const model: IsotonicModel = fitIsotonic(train);
-        for (let i: number = 0; i < test.length; i += 1) {
-            const point: CalibrationPoint = test[i];
-            predictions.push({ predicted: applyIsotonic(model, point.predicted), actual: point.actual });
-        }
-    }
-    return {
-        ece: ece(predictions),
-        brier: brier(predictions),
-        rawEce: ece(points),
-        rawBrier: brier(points),
-    };
-};

package/src/bayes/calibrationRegistry.ts

@@ -1,111 +0,0 @@
-/**
- * Calibration registry.
- *
- * A fitted map is activated only when there are enough points AND its
- * cross-validated ECE beats raw by `minImprovement`. Otherwise the scope stays
- * raw. This prevents the "isotonic made it worse" failure from shipping.
- */
-import { applyIsotonic, applyTemperature, fitIsotonic, fitTemperature } from "./calibration.ts";
-import type { CalibrationPoint, IsotonicModel, TemperatureModel } from "./calibration.ts";
-import { selectCalibration, bootstrapEceDelta } from "./calibrationSelection.ts";
-import type { CalibrationMethod, CalibrationSelection } from "./calibrationSelection.ts";
-
-export interface CalibrationModel {
-    scope: string;
-    method: CalibrationMethod;
-    samples: number;
-    rawEce: number;
-    cvEce: number;
-    thresholds: number[];
-    values: number[];
-    temperature?: number;
-    ciLower: number;
-    ciUpper: number;
-    activatedAt: string;
-}
-
-export interface RegisterOptions {
-    minSamples?: number;
-    minImprovement?: number;
-    folds?: number;
-    /**
-     * Require the bootstrap 95% upper bound to be below zero.
-     */
-    requireCi?: boolean;
-    bootstrapSamples?: number;
-    seed?: number;
-    at?: string;
-}
-
-export interface RegisterResult {
-    status: "raw" | "activated";
-    selection: CalibrationSelection;
-    model?: CalibrationModel;
-}
-
-/**
- * Bootstrap confidence interval for the ECE delta.
- */
-type BootstrapInterval = {
-    mean: number;
-    lower: number;
-    upper: number;
-};
-
-const clamp01: (value: number) => number = (value: number): number => Math.max(0, Math.min(1, value));
-
-export class CalibrationRegistry {
-    private readonly models = new Map<string, CalibrationModel>();
-
-    register(scope: string, points: readonly CalibrationPoint[], options: RegisterOptions = {}): RegisterResult {
-        const minSamples: number = options.minSamples ?? 30;
-        const minImprovement: number = options.minImprovement ?? 0.01;
-        const selection: CalibrationSelection = selectCalibration(points, options.folds ?? 5);
-        const candidate: boolean = points.length >= minSamples && selection.method !== "raw" && selection.rawEce - selection.ece >= minImprovement;
-        const ci: BootstrapInterval = candidate
-            ? bootstrapEceDelta(points, selection.method, options.folds ?? 5, options.bootstrapSamples ?? 500, options.seed ?? 42)
-            : { mean: 0, lower: 0, upper: 0 };
-        const improves: boolean = candidate && (options.requireCi === false || ci.upper < 0);
-        if (!improves) {
-            this.models.delete(scope);
-            return { status: "raw", selection };
-        }
-        const model: IsotonicModel = fitIsotonic(points);
-        const temperature: TemperatureModel = fitTemperature(points);
-        const record: CalibrationModel = {
-            scope,
-            method: selection.method,
-            samples: points.length,
-            rawEce: selection.rawEce,
-            cvEce: selection.ece,
-            thresholds: model.thresholds,
-            values: model.values,
-            ...(selection.method === "temperature" ? { temperature: temperature.temperature } : {}),
-            ciLower: ci.lower,
-            ciUpper: ci.upper,
-            activatedAt: options.at ?? new Date().toISOString(),
-        };
-        this.models.set(scope, record);
-        return { status: "activated", selection, model: record };
-    }
-
-    calibrate(scope: string, predicted: number): number {
-        const model: CalibrationModel | undefined = this.models.get(scope);
-        if (model === (void 0)) {
-            return clamp01(predicted);
-        }
-        if (model.method === "temperature") {
-            return applyTemperature({ temperature: model.temperature ?? 1 }, predicted);
-        }
-        const calibrated: number = applyIsotonic({ thresholds: model.thresholds, values: model.values }, predicted);
-        return model.method === "isotonic" ? calibrated : 0.5 * calibrated + 0.5 * clamp01(predicted);
-    }
-
-    model(scope: string): CalibrationModel | undefined {
-        return this.models.get(scope);
-    }
-
-    scopes(): string[] {
-        return [...this.models.keys()];
-    }
-}

package/src/bayes/calibrationSelection.ts

@@ -1,154 +0,0 @@
-/**
- * Calibration model selection. Fits raw / isotonic / shrunk isotonic /
- * temperature on each cross-validation fold and picks the lowest ECE; ties
- * prefer raw.
- */
-import { applyIsotonic, applyTemperature, brier, ece, fitIsotonic, fitTemperature } from "./calibration.ts";
-import type { CalibrationPoint, IsotonicModel, TemperatureModel } from "./calibration.ts";
-
-export type CalibrationMethod = "raw" | "isotonic" | "shrunk" | "temperature";
-
-export interface CalibrationSelection {
-    method: CalibrationMethod;
-    ece: number;
-    brier: number;
-    rawEce: number;
-    rawBrier: number;
-}
-
-/**
- * Maps a raw probability through one calibration method.
- */
-type Predictor = (method: CalibrationMethod, isotonic: IsotonicModel, temperature: TemperatureModel, value: number) => number;
-
-const clamp01: (value: number) => number = (value: number): number => Math.max(0, Math.min(1, value));
-
-const predict: Predictor = (method: CalibrationMethod, isotonic: IsotonicModel, temperature: TemperatureModel, value: number): number => {
-    if (method === "raw") {
-        return clamp01(value);
-    }
-    if (method === "temperature") {
-        return applyTemperature(temperature, value);
-    }
-    const calibrated: number = applyIsotonic(isotonic, value);
-    return method === "isotonic" ? calibrated : 0.5 * calibrated + 0.5 * value;
-};
-
-export interface CvPrediction {
-    predicted: number;
-    actual: boolean;
-}
-
-/**
- * Emits one out-of-fold prediction per calibration point.
- */
-type CrossValidatedPredictor = (points: readonly CalibrationPoint[], method: CalibrationMethod, folds: number) => CvPrediction[];
-
-export const crossValidatedPredictions: CrossValidatedPredictor = (points: readonly CalibrationPoint[], method: CalibrationMethod, folds: number) => {
-    const predictions: CvPrediction[] = [];
-    for (let fold: number = 0; fold < folds; fold += 1) {
-        const train: CalibrationPoint[] = points.filter((_, index) => index % folds !== fold);
-        const test: CalibrationPoint[] = points.filter((_, index) => index % folds === fold);
-        if (train.length === 0 || test.length === 0) {
-            continue;
-        }
-        const isotonic: IsotonicModel = fitIsotonic(train);
-        const temperature: TemperatureModel = fitTemperature(train);
-        for (let index: number = 0; index < test.length; index += 1) {
-            const point: CalibrationPoint = test[index];
-            predictions.push({ predicted: predict(method, isotonic, temperature, point.predicted), actual: point.actual });
-        }
-    }
-    return predictions;
-};
-
-/**
- * Cross-validated ECE and Brier score for one method.
- */
-type CalibrationScore = {
-    ece: number;
-    brier: number;
-};
-
-/**
- * Scores one calibration method by cross-validated ECE and Brier.
- */
-type CrossValidator = (points: readonly CalibrationPoint[], method: CalibrationMethod, folds: number) => CalibrationScore;
-
-const crossValidate: CrossValidator = (points: readonly CalibrationPoint[], method: CalibrationMethod, folds: number): CalibrationScore => {
-    const predictions: CvPrediction[] = crossValidatedPredictions(points, method, folds);
-    return { ece: ece(predictions), brier: brier(predictions) };
-};
-
-const mulberry32: (seed: number) => () => number = (seed: number): (() => number) => {
-    let state: number = seed >>> 0;
-    return () => {
-        state += 0x6d2b79f5;
-        let value: number = Math.imul(state ^ (state >>> 15), 1 | state);
-        value ^= value + Math.imul(value ^ (value >>> 7), 61 | value);
-        return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
-    };
-};
-
-/**
- * Percentile interval for the paired ECE delta.
- */
-type BootstrapInterval = {
-    mean: number;
-    lower: number;
-    upper: number;
-};
-
-/**
- * Bootstrap signature for the ECE delta estimate.
- */
-type BootstrapDelta = (points: readonly CalibrationPoint[], method: CalibrationMethod, folds?: number, samples?: number, seed?: number) => BootstrapInterval;
-
-/**
- * Paired bootstrap of ECE(calibrated) - ECE(raw); negative means improvement.
- */
-export const bootstrapEceDelta: BootstrapDelta = (points: readonly CalibrationPoint[], method: CalibrationMethod, folds = 5, samples = 500, seed = 42) => {
-    const raw: CvPrediction[] = crossValidatedPredictions(points, "raw", folds);
-    const calibrated: CvPrediction[] = crossValidatedPredictions(points, method, folds);
-    if (raw.length === 0 || raw.length !== calibrated.length) {
-        return { mean: 0, lower: 0, upper: 0 };
-    }
-    const random: () => number = mulberry32(seed);
-    const deltas: number[] = [];
-    for (let sample: number = 0; sample < samples; sample += 1) {
-        const left: CvPrediction[] = [];
-        const right: CvPrediction[] = [];
-        for (let index: number = 0; index < raw.length; index += 1) {
-            const pick: number = Math.floor(random() * raw.length);
-            left.push(raw[pick]);
-            right.push(calibrated[pick]);
-        }
-        deltas.push(ece(right) - ece(left));
-    }
-    deltas.sort((a, b) => a - b);
-    const mean: number = deltas.reduce((sum, value) => sum + value, 0) / deltas.length;
-    return {
-        mean,
-        lower: deltas[Math.floor(deltas.length * 0.025)] ?? mean,
-        upper: deltas[Math.floor(deltas.length * 0.975)] ?? mean,
-    };
-};
-
-/**
- * Picks the calibration method with the lowest cross-validated ECE.
- */
-type CalibrationSelector = (points: readonly CalibrationPoint[], folds?: number) => CalibrationSelection;
-
-export const selectCalibration: CalibrationSelector = (points: readonly CalibrationPoint[], folds: number = 5): CalibrationSelection => {
-    const rawEce: number = ece(points);
-    const rawBrier: number = brier(points);
-    const candidates: CalibrationSelection[] = [{ method: "raw", ece: rawEce, brier: rawBrier, rawEce, rawBrier }];
-    const methods: readonly CalibrationMethod[] = ["isotonic", "shrunk", "temperature"];
-    for (let index: number = 0; index < methods.length; index += 1) {
-        const method: CalibrationMethod = methods[index];
-        const score: CalibrationScore = crossValidate(points, method, folds);
-        candidates.push({ method, ece: score.ece, brier: score.brier, rawEce, rawBrier });
-    }
-    candidates.sort((left, right) => left.ece - right.ece || (left.method === "raw" ? -1 : 1));
-    return candidates[0];
-};