jevkit-calibrate 0.1.0

package/README.md

@@ -0,0 +1,93 @@
+# jevkit-calibrate
+
+Calibration is Jev's central claim: the probabilities are meant to track real
+frequencies, so that among answers given 0.8, about 80% are right. TypeSafe
+measures this across groups of predictions and says plainly that it does not
+guarantee any individual answer.
+
+This package checks the claim on **your** data, which is the part nobody else
+can do for you, and turns the result into a threshold you can defend.
+
+> Unofficial and unaffiliated with TypeSafe.
+
+```bash
+pip install jevkit-calibrate
+```
+
+Pure standard library. No numpy, no pandas, no plotting stack. A calibration
+check that needs a build toolchain is a calibration check that does not get run.
+
+## Measure
+
+```python
+from jevkit_core import read_records
+from jevkit_calibrate import calibrate, observations_from_records
+
+observations = observations_from_records(read_records("labeled.jevl"))
+report = calibrate(observations)
+
+print(report.summary())
+print(report.diagram())
+```
+
+```
+observations: 4000
+accuracy:     0.7485
+mean claimed: 0.7469  (underconfident by 0.0016)
+ECE:          0.0094  (over 10 bins)
+MCE:          0.0167
+Brier:        0.1681
+log loss:     0.5043
+```
+
+`ECE` is the average gap between claimed and observed, weighted by how many
+observations fall in each bin. `MCE` is the worst gap in any one bin, which is
+what catches a healthy-looking ECE hiding one badly wrong region. `Brier` and
+`log loss` are proper scoring rules: they reward being calibrated **and**
+decisive, so a model that always says 0.5 scores badly even though it is
+perfectly calibrated.
+
+## Pick a threshold
+
+TypeSafe's confidence page recommends three bands and says where you draw them
+depends on your data. This draws them from the data.
+
+```python
+from jevkit_calibrate import recommend_for_accuracy
+
+point = recommend_for_accuracy(observations, target_accuracy=0.95)
+if point is None:
+    print("no threshold reaches 95% on this data")
+else:
+    print(f"threshold {point.threshold:.2f}: "
+          f"covers {point.coverage:.1%} at {point.accuracy:.1%}, "
+          f"{point.errors} wrong answers acted on")
+```
+
+`None` is a real answer, not a failure. It means this question cannot be
+automated at that bar, and the honest move is to change the question rather
+than lower the threshold.
+
+## CLI
+
+```bash
+jevkit-calibrate labeled.jevl
+jevkit-calibrate labeled.jevl --target-accuracy 0.95
+jevkit-calibrate labeled.jevl --min-coverage 0.80
+jevkit-calibrate labeled.jevl --max-ece 0.05     # CI gate
+jevkit-calibrate labeled.jevl --format json
+```
+
+## Which quantity gets calibrated
+
+By default, the probability mass on the chosen outcome, which is what "when it
+says 0.8, is it right 80% of the time" means.
+
+`--use-confidence` calibrates the API's `confidence` statistic instead. That is
+a different question, and the one to ask when you want to know whether your
+*routing* threshold sits in the right place. Nouls carry no confidence, so they
+fall back to probability either way.
+
+## License
+
+MIT

package/dist/cli.d.ts

@@ -0,0 +1,7 @@
+#!/usr/bin/env node
+/** `jevkit-calibrate` command line interface. */
+export declare const EXIT_OK = 0;
+export declare const EXIT_FAILED_GATE = 1;
+export declare const EXIT_USAGE = 2;
+export declare function main(argv?: string[]): number;
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,iDAAiD;AAQjD,eAAO,MAAM,OAAO,IAAI,CAAC;AACzB,eAAO,MAAM,gBAAgB,IAAI,CAAC;AAClC,eAAO,MAAM,UAAU,IAAI,CAAC;AAgB5B,wBAAgB,IAAI,CAAC,IAAI,GAAE,MAAM,EAA0B,GAAG,MAAM,CAuGnE"}

package/dist/cli.js

@@ -0,0 +1,150 @@
+#!/usr/bin/env node
+/** `jevkit-calibrate` command line interface. */
+import { RecordFormatError, readRecords } from "jevkit-core";
+import { calibrate } from "./metrics.js";
+import { observationsFromRecords } from "./records.js";
+import { recommendForAccuracy, recommendForCoverage } from "./thresholds.js";
+export const EXIT_OK = 0;
+export const EXIT_FAILED_GATE = 1;
+export const EXIT_USAGE = 2;
+const USAGE = `usage: jevkit-calibrate <labeled.jevl...> [options]
+
+Measure how well Jev's probabilities match outcomes on your labeled data, and
+pick a confidence threshold from it. Never calls the API.
+
+  --bins N               reliability bins (default 10)
+  --question ID          restrict to this question id (repeatable)
+  --use-confidence       calibrate the API's confidence, not the probability
+  --target-accuracy F    recommend the lowest threshold reaching this accuracy
+  --min-coverage F       recommend the highest threshold still covering this
+  --max-ece F            exit non-zero if ECE exceeds this
+  --format text|json
+  --no-diagram`;
+export function main(argv = process.argv.slice(2)) {
+    const files = [];
+    const questions = [];
+    let bins = 10;
+    let useConfidence = false;
+    let targetAccuracy = null;
+    let minCoverage = null;
+    let maxEce = null;
+    let format = "text";
+    let noDiagram = false;
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        switch (arg) {
+            case "--bins":
+                bins = Number(argv[++i]);
+                break;
+            case "--question":
+                questions.push(String(argv[++i]));
+                break;
+            case "--use-confidence":
+                useConfidence = true;
+                break;
+            case "--target-accuracy":
+                targetAccuracy = Number(argv[++i]);
+                break;
+            case "--min-coverage":
+                minCoverage = Number(argv[++i]);
+                break;
+            case "--max-ece":
+                maxEce = Number(argv[++i]);
+                break;
+            case "--no-diagram":
+                noDiagram = true;
+                break;
+            case "--format": {
+                const v = argv[++i];
+                if (v !== "text" && v !== "json") {
+                    console.error("jevkit-calibrate: --format expects text or json");
+                    return EXIT_USAGE;
+                }
+                format = v;
+                break;
+            }
+            case "-h":
+            case "--help":
+                console.log(USAGE);
+                return EXIT_OK;
+            default:
+                if (arg.startsWith("--")) {
+                    console.error(`jevkit-calibrate: unknown option ${arg}`);
+                    return EXIT_USAGE;
+                }
+                files.push(arg);
+        }
+    }
+    if (!files.length) {
+        console.error(USAGE);
+        return EXIT_USAGE;
+    }
+    const records = [];
+    try {
+        for (const path of files)
+            records.push(...readRecords(path));
+    }
+    catch (err) {
+        const message = err instanceof RecordFormatError ? err.message : err.message;
+        console.error(`jevkit-calibrate: ${message}`);
+        return EXIT_USAGE;
+    }
+    const observations = observationsFromRecords(records, {
+        questionIds: questions.length ? questions : undefined,
+        useConfidence,
+    });
+    if (!observations.length) {
+        console.error("jevkit-calibrate: no labeled observations found. Records need a 'label' object " +
+            "keyed by question id.");
+        return EXIT_USAGE;
+    }
+    const report = calibrate(observations, bins);
+    const recommendation = targetAccuracy !== null
+        ? { kind: "accuracy", target: targetAccuracy, point: recommendForAccuracy(observations, targetAccuracy) }
+        : minCoverage !== null
+            ? { kind: "coverage", target: minCoverage, point: recommendForCoverage(observations, minCoverage) }
+            : null;
+    if (format === "json") {
+        const payload = { ...report.toJSON() };
+        if (recommendation) {
+            const p = recommendation.point;
+            payload["recommendation"] = {
+                for: recommendation.kind, target: recommendation.target,
+                threshold: p?.threshold ?? null, coverage: p?.coverage ?? null,
+                accuracy: p?.accuracy ?? null, errors: p?.errors ?? null,
+            };
+        }
+        console.log(JSON.stringify(payload, null, 2));
+    }
+    else {
+        console.log(report.summary());
+        if (!noDiagram) {
+            console.log();
+            console.log(report.diagram());
+        }
+        if (recommendation) {
+            console.log();
+            const { kind, target, point } = recommendation;
+            if (!point) {
+                console.log(kind === "accuracy"
+                    ? `No threshold reaches ${(target * 100).toFixed(1)}% accuracy on this data. The ` +
+                        `question cannot be automated at that bar; change the question rather than the threshold.`
+                    : `No threshold covers ${(target * 100).toFixed(1)}% of cases.`);
+            }
+            else {
+                console.log(`threshold ${point.threshold.toFixed(2)}: covers ${(point.coverage * 100).toFixed(1)}% ` +
+                    `(${point.covered}/${point.total}) at ${(point.accuracy * 100).toFixed(1)}% accuracy, ` +
+                    `${point.errors} wrong answer(s) acted on, ${point.escalated} escalated`);
+            }
+        }
+    }
+    if (maxEce !== null && report.ece > maxEce) {
+        console.error(`jevkit-calibrate: ECE ${report.ece.toFixed(4)} exceeds ${maxEce}`);
+        return EXIT_FAILED_GATE;
+    }
+    return EXIT_OK;
+}
+const invokedDirectly = process.argv[1] !== undefined && import.meta.url === `file://${process.argv[1]}`;
+if (invokedDirectly)
+    process.exit(main());
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,iDAAiD;AAEjD,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAA6B,MAAM,aAAa,CAAC;AAExF,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAE7E,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC;AACzB,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAClC,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAC;AAE5B,MAAM,KAAK,GAAG;;;;;;;;;;;;eAYC,CAAC;AAEhB,MAAM,UAAU,IAAI,CAAC,OAAiB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,IAAI,cAAc,GAAkB,IAAI,CAAC;IACzC,IAAI,WAAW,GAAkB,IAAI,CAAC;IACtC,IAAI,MAAM,GAAkB,IAAI,CAAC;IACjC,IAAI,MAAM,GAAoB,MAAM,CAAC;IACrC,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACrB,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,QAAQ;gBAAE,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YAC/C,KAAK,YAAY;gBAAE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YAC5D,KAAK,kBAAkB;gBAAE,aAAa,GAAG,IAAI,CAAC;gBAAC,MAAM;YACrD,KAAK,mBAAmB;gBAAE,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YACpE,KAAK,gBAAgB;gBAAE,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YAC9D,KAAK,WAAW;gBAAE,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBAAC,MAAM;YACpD,KAAK,cAAc;gBAAE,SAAS,GAAG,IAAI,CAAC;gBAAC,MAAM;YAC7C,KAAK,UAAU,CAAC,CAAC,CAAC;gBAChB,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;gBACpB,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,iDAAiD,CAAC,CAAC;oBAAC,OAAO,UAAU,CAAC;gBAAC,CAAC;gBAC1H,MAAM,GAAG,CAAC,CAAC;gBAAC,MAAM;YACpB,CAAC;YACD,KAAK,IAAI,CAAC;YAAC,KAAK,QAAQ;gBAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAAC,OAAO,OAAO,CAAC;YAC7D;gBACE,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;oBAAC,OAAO,CAAC,KAAK,CAAC,oCAAoC,GAAG,EAAE,CAAC,CAAC;oBAAC,OAAO,UAAU,CAAC;gBAAC,CAAC;gBAC1G,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAAC,OAAO,UAAU,CAAC;IAAC,CAAC;IAE/D,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,IAAI,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK;YAAE,OAAO,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAE,GAAa,CAAC,OAAO,CAAC;QACxF,OAAO,CAAC,KAAK,CAAC,qBAAqB,OAAO,EAAE,CAAC,CAAC;QAC9C,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,MAAM,YAAY,GAAG,uBAAuB,CAAC,OAAO,EAAE;QACpD,WAAW,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;QACrD,aAAa;KACd,CAAC,CAAC;IACH,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CACX,iFAAiF;YACjF,uBAAuB,CACxB,CAAC;QACF,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,MAAM,MAAM,GAAG,SAAS,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAC7C,MAAM,cAAc,GAClB,cAAc,KAAK,IAAI;QACrB,CAAC,CAAC,EAAE,IAAI,EAAE,UAAmB,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,oBAAoB,CAAC,YAAY,EAAE,cAAc,CAAC,EAAE;QAClH,CAAC,CAAC,WAAW,KAAK,IAAI;YACpB,CAAC,CAAC,EAAE,IAAI,EAAE,UAAmB,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,oBAAoB,CAAC,YAAY,EAAE,WAAW,CAAC,EAAE;YAC5G,CAAC,CAAC,IAAI,CAAC;IAEb,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,MAAM,OAAO,GAA4B,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QAChE,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,CAAC,GAAG,cAAc,CAAC,KAAK,CAAC;YAC/B,OAAO,CAAC,gBAAgB,CAAC,GAAG;gBAC1B,GAAG,EAAE,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,cAAc,CAAC,MAAM;gBACvD,SAAS,EAAE,CAAC,EAAE,SAAS,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,IAAI,IAAI;gBAC9D,QAAQ,EAAE,CAAC,EAAE,QAAQ,IAAI,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,IAAI,IAAI;aACzD,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9B,IAAI,CAAC,SAAS,EAAE,CAAC;YAAC,OAAO,CAAC,GAAG,EAAE,CAAC;YAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAAC,CAAC;QACjE,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,CAAC,GAAG,EAAE,CAAC;YACd,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,cAAc,CAAC;YAC/C,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO,CAAC,GAAG,CACT,IAAI,KAAK,UAAU;oBACjB,CAAC,CAAC,wBAAwB,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,+BAA+B;wBAChF,0FAA0F;oBAC5F,CAAC,CAAC,uBAAuB,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAClE,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CACT,aAAa,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;oBACxF,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,QAAQ,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc;oBACvF,GAAG,KAAK,CAAC,MAAM,8BAA8B,KAAK,CAAC,SAAS,YAAY,CACzE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC;QAC3C,OAAO,CAAC,KAAK,CAAC,yBAAyB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,MAAM,EAAE,CAAC,CAAC;QAClF,OAAO,gBAAgB,CAAC;IAC1B,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,eAAe,GACnB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,UAAU,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;AACnF,IAAI,eAAe;IAAE,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,16 @@
+/**
+ * Verify Jev's calibration on your own data, and pick thresholds from it.
+ *
+ * Calibration is the central claim behind a System One model, and the one thing
+ * a user cannot check without tooling. This package computes reliability
+ * diagrams, ECE, MCE, Brier and log loss over labeled answers, and turns a
+ * labeled set into a defensible confidence threshold.
+ *
+ * No dependencies beyond jevkit-core. A calibration check that drags in a
+ * numerical stack is a calibration check that does not get run.
+ */
+export { Bin, CalibrationReport, Observation, brierScore, calibrate, expectedCalibrationError, logLoss, maximumCalibrationError, reliabilityBins, type ObservationInit, } from "./metrics.js";
+export { observationsFromRecords, type ExtractOptions } from "./records.js";
+export { ThresholdPoint, recommendForAccuracy, recommendForCoverage, sweep, } from "./thresholds.js";
+export declare const VERSION = "0.1.0";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,GAAG,EAAE,iBAAiB,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAC1D,wBAAwB,EAAE,OAAO,EAAE,uBAAuB,EAAE,eAAe,EAC3E,KAAK,eAAe,GACrB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,uBAAuB,EAAE,KAAK,cAAc,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EACL,cAAc,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,KAAK,GAClE,MAAM,iBAAiB,CAAC;AAEzB,eAAO,MAAM,OAAO,UAAU,CAAC"}

package/dist/index.js

@@ -0,0 +1,16 @@
+/**
+ * Verify Jev's calibration on your own data, and pick thresholds from it.
+ *
+ * Calibration is the central claim behind a System One model, and the one thing
+ * a user cannot check without tooling. This package computes reliability
+ * diagrams, ECE, MCE, Brier and log loss over labeled answers, and turns a
+ * labeled set into a defensible confidence threshold.
+ *
+ * No dependencies beyond jevkit-core. A calibration check that drags in a
+ * numerical stack is a calibration check that does not get run.
+ */
+export { Bin, CalibrationReport, Observation, brierScore, calibrate, expectedCalibrationError, logLoss, maximumCalibrationError, reliabilityBins, } from "./metrics.js";
+export { observationsFromRecords } from "./records.js";
+export { ThresholdPoint, recommendForAccuracy, recommendForCoverage, sweep, } from "./thresholds.js";
+export const VERSION = "0.1.0";
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,GAAG,EAAE,iBAAiB,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAC1D,wBAAwB,EAAE,OAAO,EAAE,uBAAuB,EAAE,eAAe,GAE5E,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,uBAAuB,EAAuB,MAAM,cAAc,CAAC;AAC5E,OAAO,EACL,cAAc,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,KAAK,GAClE,MAAM,iBAAiB,CAAC;AAEzB,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC"}

package/dist/metrics.d.ts

@@ -0,0 +1,131 @@
+/**
+ * Calibration metrics over labeled jev answers.
+ *
+ * Calibration is Jev's central claim: the probabilities are meant to reflect
+ * real-world frequencies, so that among answers given 0.8, about 80% are right.
+ * TypeSafe measures this across groups of predictions and says plainly that it
+ * does not guarantee any individual answer. This module is how you check the
+ * claim on *your* data, which is the part nobody else can do for you.
+ *
+ * Everything here is pure arithmetic over recorded answers. Nothing calls the
+ * API and there are no dependencies: a calibration check that drags in a
+ * numerical stack is a calibration check that does not get run.
+ */
+export interface ObservationInit {
+    probability: number;
+    correct: boolean;
+    questionId?: string;
+    requestId?: string;
+}
+/**
+ * One labeled prediction.
+ *
+ * `probability` is the model's stated probability for the outcome it chose;
+ * `correct` is whether that outcome matched the label.
+ */
+export declare class Observation {
+    readonly probability: number;
+    readonly correct: boolean;
+    readonly questionId: string;
+    readonly requestId: string;
+    constructor(init: ObservationInit);
+}
+/** One bucket of a reliability diagram. */
+export declare class Bin {
+    readonly lower: number;
+    readonly upper: number;
+    readonly observations: Observation[];
+    constructor(lower: number, upper: number);
+    get count(): number;
+    /** What the model claimed, on average. */
+    get meanProbability(): number;
+    /** What actually happened. */
+    get accuracy(): number;
+    /** Claimed minus observed. Positive means overconfident. */
+    get gap(): number;
+    label(): string;
+}
+/**
+ * Bucket observations into equal-width probability bins.
+ *
+ * Equal-width rather than equal-count, because the question being asked is
+ * "when the model says 0.9, is it right 90% of the time", and that question is
+ * about fixed probability ranges. Empty bins are kept so the diagram does not
+ * silently mislead about coverage.
+ */
+export declare function reliabilityBins(observations: Observation[], nBins?: number): Bin[];
+/**
+ * ECE: average gap between claimed and observed, weighted by bin population.
+ *
+ * 0 is perfect. ECE depends on the bin count, so compare values only when they
+ * were computed with the same `nBins`.
+ */
+export declare function expectedCalibrationError(observations: Observation[], nBins?: number): number;
+/**
+ * MCE: the worst gap in any populated bin.
+ *
+ * ECE can look healthy while one region is badly wrong. MCE catches that, and
+ * it is the number that matters when one region is where your high-stakes
+ * decisions live.
+ */
+export declare function maximumCalibrationError(observations: Observation[], nBins?: number): number;
+/**
+ * Mean squared error between probability and outcome. Lower is better.
+ *
+ * Unlike ECE this is a proper scoring rule: it rewards being both calibrated
+ * and decisive, so a model that always says 0.5 scores poorly even though it is
+ * perfectly calibrated.
+ */
+export declare function brierScore(observations: Observation[]): number;
+/**
+ * Mean negative log likelihood. Lower is better.
+ *
+ * Punishes confident mistakes far harder than Brier does. `eps` clamps the
+ * probability away from 0 and 1, since an unclamped confident miss is infinite
+ * and would swamp every other observation.
+ */
+export declare function logLoss(observations: Observation[], eps?: number): number;
+export declare class CalibrationReport {
+    readonly observations: Observation[];
+    readonly nBins: number;
+    constructor(observations: Observation[], nBins?: number);
+    get count(): number;
+    get accuracy(): number;
+    get meanProbability(): number;
+    get bins(): Bin[];
+    get ece(): number;
+    get mce(): number;
+    get brier(): number;
+    get logLoss(): number;
+    /** Claimed probability exceeds observed accuracy overall. */
+    get overconfident(): boolean;
+    /**
+     * A text reliability diagram.
+     *
+     * Each row shows a bin's claimed probability against what actually happened,
+     * so a miscalibrated region is visible without plotting.
+     */
+    diagram(width?: number): string;
+    summary(): string;
+    toJSON(): {
+        count: number;
+        accuracy: number;
+        meanProbability: number;
+        overconfident: boolean;
+        ece: number;
+        mce: number;
+        brier: number;
+        logLoss: number;
+        nBins: number;
+        bins: {
+            lower: number;
+            upper: number;
+            count: number;
+            meanProbability: number;
+            accuracy: number;
+            gap: number;
+        }[];
+    };
+}
+export declare function calibrate(observations: Iterable<Observation>, nBins?: number): CalibrationReport;
+//# sourceMappingURL=metrics.d.ts.map

package/dist/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,qBAAa,WAAW;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;gBAEf,IAAI,EAAE,eAAe;CASlC;AAED,2CAA2C;AAC3C,qBAAa,GAAG;IAGF,QAAQ,CAAC,KAAK,EAAE,MAAM;IAAE,QAAQ,CAAC,KAAK,EAAE,MAAM;IAF1D,QAAQ,CAAC,YAAY,EAAE,WAAW,EAAE,CAAM;gBAErB,KAAK,EAAE,MAAM,EAAW,KAAK,EAAE,MAAM;IAE1D,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,0CAA0C;IAC1C,IAAI,eAAe,IAAI,MAAM,CAG5B;IAED,8BAA8B;IAC9B,IAAI,QAAQ,IAAI,MAAM,CAGrB;IAED,4DAA4D;IAC5D,IAAI,GAAG,IAAI,MAAM,CAEhB;IAED,KAAK,IAAI,MAAM;CAGhB;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,YAAY,EAAE,WAAW,EAAE,EAAE,KAAK,SAAK,GAAG,GAAG,EAAE,CAU9E;AAED;;;;;GAKG;AACH,wBAAgB,wBAAwB,CAAC,YAAY,EAAE,WAAW,EAAE,EAAE,KAAK,SAAK,GAAG,MAAM,CAMxF;AAED;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,WAAW,EAAE,EAAE,KAAK,SAAK,GAAG,MAAM,CAIvF;AAED;;;;;;GAMG;AACH,wBAAgB,UAAU,CAAC,YAAY,EAAE,WAAW,EAAE,GAAG,MAAM,CAM9D;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,YAAY,EAAE,WAAW,EAAE,EAAE,GAAG,SAAQ,GAAG,MAAM,CAQxE;AAED,qBAAa,iBAAiB;IAChB,QAAQ,CAAC,YAAY,EAAE,WAAW,EAAE;IAAE,QAAQ,CAAC,KAAK;gBAA3C,YAAY,EAAE,WAAW,EAAE,EAAW,KAAK,SAAK;IAErE,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,IAAI,QAAQ,IAAI,MAAM,CAGrB;IAED,IAAI,eAAe,IAAI,MAAM,CAG5B;IAED,IAAI,IAAI,IAAI,GAAG,EAAE,CAEhB;IAED,IAAI,GAAG,IAAI,MAAM,CAEhB;IAED,IAAI,GAAG,IAAI,MAAM,CAEhB;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,6DAA6D;IAC7D,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED;;;;;OAKG;IACH,OAAO,CAAC,KAAK,SAAK,GAAG,MAAM;IA6B3B,OAAO,IAAI,MAAM;IAejB,MAAM;;;;;;;;;;;;;;;;;;;CAiBP;AAED,wBAAgB,SAAS,CAAC,YAAY,EAAE,QAAQ,CAAC,WAAW,CAAC,EAAE,KAAK,SAAK,GAAG,iBAAiB,CAE5F"}

package/dist/metrics.js

@@ -0,0 +1,251 @@
+/**
+ * Calibration metrics over labeled jev answers.
+ *
+ * Calibration is Jev's central claim: the probabilities are meant to reflect
+ * real-world frequencies, so that among answers given 0.8, about 80% are right.
+ * TypeSafe measures this across groups of predictions and says plainly that it
+ * does not guarantee any individual answer. This module is how you check the
+ * claim on *your* data, which is the part nobody else can do for you.
+ *
+ * Everything here is pure arithmetic over recorded answers. Nothing calls the
+ * API and there are no dependencies: a calibration check that drags in a
+ * numerical stack is a calibration check that does not get run.
+ */
+/**
+ * One labeled prediction.
+ *
+ * `probability` is the model's stated probability for the outcome it chose;
+ * `correct` is whether that outcome matched the label.
+ */
+export class Observation {
+    probability;
+    correct;
+    questionId;
+    requestId;
+    constructor(init) {
+        if (!(init.probability >= 0 && init.probability <= 1)) {
+            throw new RangeError(`probability must be in [0, 1], got ${init.probability}`);
+        }
+        this.probability = init.probability;
+        this.correct = init.correct;
+        this.questionId = init.questionId ?? "";
+        this.requestId = init.requestId ?? "";
+    }
+}
+/** One bucket of a reliability diagram. */
+export class Bin {
+    lower;
+    upper;
+    observations = [];
+    constructor(lower, upper) {
+        this.lower = lower;
+        this.upper = upper;
+    }
+    get count() {
+        return this.observations.length;
+    }
+    /** What the model claimed, on average. */
+    get meanProbability() {
+        if (!this.count)
+            return 0;
+        return this.observations.reduce((a, o) => a + o.probability, 0) / this.count;
+    }
+    /** What actually happened. */
+    get accuracy() {
+        if (!this.count)
+            return 0;
+        return this.observations.filter((o) => o.correct).length / this.count;
+    }
+    /** Claimed minus observed. Positive means overconfident. */
+    get gap() {
+        return this.meanProbability - this.accuracy;
+    }
+    label() {
+        return `[${this.lower.toFixed(2)}, ${this.upper.toFixed(2)}${this.upper === 1 ? "]" : ")"}`;
+    }
+}
+/**
+ * Bucket observations into equal-width probability bins.
+ *
+ * Equal-width rather than equal-count, because the question being asked is
+ * "when the model says 0.9, is it right 90% of the time", and that question is
+ * about fixed probability ranges. Empty bins are kept so the diagram does not
+ * silently mislead about coverage.
+ */
+export function reliabilityBins(observations, nBins = 10) {
+    if (nBins < 1)
+        throw new RangeError("nBins must be at least 1");
+    const bins = [];
+    for (let i = 0; i < nBins; i++)
+        bins.push(new Bin(i / nBins, (i + 1) / nBins));
+    for (const obs of observations) {
+        // The top bin is closed so a probability of exactly 1 lands in it.
+        const index = Math.min(Math.floor(obs.probability * nBins), nBins - 1);
+        bins[index].observations.push(obs);
+    }
+    return bins;
+}
+/**
+ * ECE: average gap between claimed and observed, weighted by bin population.
+ *
+ * 0 is perfect. ECE depends on the bin count, so compare values only when they
+ * were computed with the same `nBins`.
+ */
+export function expectedCalibrationError(observations, nBins = 10) {
+    if (!observations.length)
+        return 0;
+    const total = observations.length;
+    return reliabilityBins(observations, nBins)
+        .filter((b) => b.count)
+        .reduce((sum, b) => sum + (b.count / total) * Math.abs(b.gap), 0);
+}
+/**
+ * MCE: the worst gap in any populated bin.
+ *
+ * ECE can look healthy while one region is badly wrong. MCE catches that, and
+ * it is the number that matters when one region is where your high-stakes
+ * decisions live.
+ */
+export function maximumCalibrationError(observations, nBins = 10) {
+    if (!observations.length)
+        return 0;
+    const populated = reliabilityBins(observations, nBins).filter((b) => b.count);
+    return populated.reduce((m, b) => Math.max(m, Math.abs(b.gap)), 0);
+}
+/**
+ * Mean squared error between probability and outcome. Lower is better.
+ *
+ * Unlike ECE this is a proper scoring rule: it rewards being both calibrated
+ * and decisive, so a model that always says 0.5 scores poorly even though it is
+ * perfectly calibrated.
+ */
+export function brierScore(observations) {
+    if (!observations.length)
+        return 0;
+    return (observations.reduce((sum, o) => sum + (o.probability - (o.correct ? 1 : 0)) ** 2, 0) /
+        observations.length);
+}
+/**
+ * Mean negative log likelihood. Lower is better.
+ *
+ * Punishes confident mistakes far harder than Brier does. `eps` clamps the
+ * probability away from 0 and 1, since an unclamped confident miss is infinite
+ * and would swamp every other observation.
+ */
+export function logLoss(observations, eps = 1e-15) {
+    if (!observations.length)
+        return 0;
+    let total = 0;
+    for (const o of observations) {
+        const p = Math.min(Math.max(o.probability, eps), 1 - eps);
+        total -= o.correct ? Math.log(p) : Math.log(1 - p);
+    }
+    return total / observations.length;
+}
+export class CalibrationReport {
+    observations;
+    nBins;
+    constructor(observations, nBins = 10) {
+        this.observations = observations;
+        this.nBins = nBins;
+    }
+    get count() {
+        return this.observations.length;
+    }
+    get accuracy() {
+        if (!this.count)
+            return 0;
+        return this.observations.filter((o) => o.correct).length / this.count;
+    }
+    get meanProbability() {
+        if (!this.count)
+            return 0;
+        return this.observations.reduce((a, o) => a + o.probability, 0) / this.count;
+    }
+    get bins() {
+        return reliabilityBins(this.observations, this.nBins);
+    }
+    get ece() {
+        return expectedCalibrationError(this.observations, this.nBins);
+    }
+    get mce() {
+        return maximumCalibrationError(this.observations, this.nBins);
+    }
+    get brier() {
+        return brierScore(this.observations);
+    }
+    get logLoss() {
+        return logLoss(this.observations);
+    }
+    /** Claimed probability exceeds observed accuracy overall. */
+    get overconfident() {
+        return this.meanProbability > this.accuracy;
+    }
+    /**
+     * A text reliability diagram.
+     *
+     * Each row shows a bin's claimed probability against what actually happened,
+     * so a miscalibrated region is visible without plotting.
+     */
+    diagram(width = 28) {
+        const lines = [
+            `${"bin".padEnd(14)} ${"n".padStart(5)} ${"claimed".padStart(8)} ` +
+                `${"actual".padStart(8)} ${"gap".padStart(7)}`,
+            "-".repeat(46),
+        ];
+        for (const b of this.bins) {
+            if (!b.count) {
+                lines.push(`${b.label().padEnd(14)} ${String(0).padStart(5)} ${"-".padStart(8)} ` +
+                    `${"-".padStart(8)} ${"-".padStart(7)}`);
+                continue;
+            }
+            const barLen = Math.floor(b.accuracy * width);
+            const marker = Math.floor(b.meanProbability * width);
+            const cells = Array.from({ length: width }, (_, i) => (i < barLen ? "#" : " "));
+            if (marker >= 0 && marker < width)
+                cells[marker] = cells[marker] === " " ? "|" : "+";
+            const gap = `${b.gap >= 0 ? "+" : ""}${b.gap.toFixed(3)}`;
+            lines.push(`${b.label().padEnd(14)} ${String(b.count).padStart(5)} ` +
+                `${b.meanProbability.toFixed(3).padStart(8)} ${b.accuracy.toFixed(3).padStart(8)} ` +
+                `${gap.padStart(7)}  ${cells.join("")}`);
+        }
+        lines.push("", "  # observed accuracy, | claimed probability, + both");
+        return lines.join("\n");
+    }
+    summary() {
+        if (!this.count)
+            return "no labeled observations";
+        const direction = this.overconfident ? "overconfident" : "underconfident";
+        const gap = Math.abs(this.meanProbability - this.accuracy);
+        return [
+            `observations: ${this.count}`,
+            `accuracy:     ${this.accuracy.toFixed(4)}`,
+            `mean claimed: ${this.meanProbability.toFixed(4)}  (${direction} by ${gap.toFixed(4)})`,
+            `ECE:          ${this.ece.toFixed(4)}  (over ${this.nBins} bins)`,
+            `MCE:          ${this.mce.toFixed(4)}`,
+            `Brier:        ${this.brier.toFixed(4)}`,
+            `log loss:     ${this.logLoss.toFixed(4)}`,
+        ].join("\n");
+    }
+    toJSON() {
+        return {
+            count: this.count,
+            accuracy: this.accuracy,
+            meanProbability: this.meanProbability,
+            overconfident: this.overconfident,
+            ece: this.ece,
+            mce: this.mce,
+            brier: this.brier,
+            logLoss: this.logLoss,
+            nBins: this.nBins,
+            bins: this.bins.map((b) => ({
+                lower: b.lower, upper: b.upper, count: b.count,
+                meanProbability: b.meanProbability, accuracy: b.accuracy, gap: b.gap,
+            })),
+        };
+    }
+}
+export function calibrate(observations, nBins = 10) {
+    return new CalibrationReport([...observations], nBins);
+}
+//# sourceMappingURL=metrics.js.map

package/dist/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH;;;;;GAKG;AACH,MAAM,OAAO,WAAW;IACb,WAAW,CAAS;IACpB,OAAO,CAAU;IACjB,UAAU,CAAS;IACnB,SAAS,CAAS;IAE3B,YAAY,IAAqB;QAC/B,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,IAAI,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC;YACtD,MAAM,IAAI,UAAU,CAAC,sCAAsC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACpC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IACxC,CAAC;CACF;AAED,2CAA2C;AAC3C,MAAM,OAAO,GAAG;IAGO;IAAwB;IAFpC,YAAY,GAAkB,EAAE,CAAC;IAE1C,YAAqB,KAAa,EAAW,KAAa;QAArC,UAAK,GAAL,KAAK,CAAQ;QAAW,UAAK,GAAL,KAAK,CAAQ;IAAG,CAAC;IAE9D,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;IAClC,CAAC;IAED,0CAA0C;IAC1C,IAAI,eAAe;QACjB,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;IAC/E,CAAC;IAED,8BAA8B;IAC9B,IAAI,QAAQ;QACV,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;IACxE,CAAC;IAED,4DAA4D;IAC5D,IAAI,GAAG;QACL,OAAO,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC9C,CAAC;IAED,KAAK;QACH,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;IAC9F,CAAC;CACF;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,YAA2B,EAAE,KAAK,GAAG,EAAE;IACrE,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,0BAA0B,CAAC,CAAC;IAChE,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE;QAAE,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;IAC/E,KAAK,MAAM,GAAG,IAAI,YAAY,EAAE,CAAC;QAC/B,mEAAmE;QACnE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,WAAW,GAAG,KAAK,CAAC,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;QACvE,IAAI,CAAC,KAAK,CAAE,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,wBAAwB,CAAC,YAA2B,EAAE,KAAK,GAAG,EAAE;IAC9E,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC;IAClC,OAAO,eAAe,CAAC,YAAY,EAAE,KAAK,CAAC;SACxC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;SACtB,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AACtE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CAAC,YAA2B,EAAE,KAAK,GAAG,EAAE;IAC7E,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACnC,MAAM,SAAS,GAAG,eAAe,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9E,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,UAAU,CAAC,YAA2B;IACpD,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACnC,OAAO,CACL,YAAY,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpF,YAAY,CAAC,MAAM,CACpB,CAAC;AACJ,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,OAAO,CAAC,YAA2B,EAAE,GAAG,GAAG,KAAK;IAC9D,IAAI,CAAC,YAAY,CAAC,MAAM;QAAE,OAAO,CAAC,CAAC;IACnC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;QAC1D,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC;AACrC,CAAC;AAED,MAAM,OAAO,iBAAiB;IACP;IAAsC;IAA3D,YAAqB,YAA2B,EAAW,QAAQ,EAAE;QAAhD,iBAAY,GAAZ,YAAY,CAAe;QAAW,UAAK,GAAL,KAAK,CAAK;IAAG,CAAC;IAEzE,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;IAClC,CAAC;IAED,IAAI,QAAQ;QACV,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;IACxE,CAAC;IAED,IAAI,eAAe;QACjB,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,CAAC;QAC1B,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC;IAC/E,CAAC;IAED,IAAI,IAAI;QACN,OAAO,eAAe,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,GAAG;QACL,OAAO,wBAAwB,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,GAAG;QACL,OAAO,uBAAuB,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,KAAK;QACP,OAAO,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACvC,CAAC;IAED,IAAI,OAAO;QACT,OAAO,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACpC,CAAC;IAED,6DAA6D;IAC7D,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC;IAC9C,CAAC;IAED;;;;;OAKG;IACH,OAAO,CAAC,KAAK,GAAG,EAAE;QAChB,MAAM,KAAK,GAAG;YACZ,GAAG,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;gBAChE,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;YAChD,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;SACf,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1B,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;gBACb,KAAK,CAAC,IAAI,CACR,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;oBACpE,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAC1C,CAAC;gBACF,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAC;YAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,GAAG,KAAK,CAAC,CAAC;YACrD,MAAM,KAAK,GAAa,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1F,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,GAAG,KAAK;gBAAE,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACrF,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1D,KAAK,CAAC,IAAI,CACR,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;gBACvD,GAAG,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG;gBACnF,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAC1C,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,sDAAsD,CAAC,CAAC;QACvE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,OAAO;QACL,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,yBAAyB,CAAC;QAClD,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,gBAAgB,CAAC;QAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3D,OAAO;YACL,iBAAiB,IAAI,CAAC,KAAK,EAAE;YAC7B,iBAAiB,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YAC3C,iBAAiB,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,SAAS,OAAO,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;YACvF,iBAAiB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,KAAK,QAAQ;YACjE,iBAAiB,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACtC,iBAAiB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YACxC,iBAAiB,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;SAC3C,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAED,MAAM;QACJ,OAAO;YACL,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,aAAa,EAAE,IAAI,CAAC,aAAa;YACjC,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC1B,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK;gBAC9C,eAAe,EAAE,CAAC,CAAC,eAAe,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG;aACrE,CAAC,CAAC;SACJ,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,SAAS,CAAC,YAAmC,EAAE,KAAK,GAAG,EAAE;IACvE,OAAO,IAAI,iBAAiB,CAAC,CAAC,GAAG,YAAY,CAAC,EAAE,KAAK,CAAC,CAAC;AACzD,CAAC"}

package/dist/records.d.ts

@@ -0,0 +1,26 @@
+/** Turning labeled `.jevl` records into calibration observations. */
+import { type Record as JevlRecord } from "jevkit-core";
+import { Observation } from "./metrics.js";
+export interface ExtractOptions {
+    questionIds?: Iterable<string>;
+    /**
+     * Which quantity to calibrate.
+     *
+     * The default, `false`, uses the probability mass on the chosen outcome,
+     * which is what "when it says 0.8, is it right 80% of the time" means.
+     * `true` calibrates the API's `confidence` statistic instead, which is a
+     * different question and answers whether your *routing* threshold is well
+     * placed. Nouls have no confidence, so they fall back to probability either
+     * way.
+     */
+    useConfidence?: boolean;
+}
+/**
+ * Extract one observation per labeled answer.
+ *
+ * Records with no `label` are skipped: calibration needs ground truth, and
+ * silently treating an unlabeled record as correct or incorrect would poison
+ * every number downstream.
+ */
+export declare function observationsFromRecords(records: Iterable<JevlRecord>, options?: ExtractOptions): Observation[];
+//# sourceMappingURL=records.d.ts.map

package/dist/records.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"records.d.ts","sourceRoot":"","sources":["../src/records.ts"],"names":[],"mappings":"AAAA,qEAAqE;AAErE,OAAO,EAAE,KAAK,MAAM,IAAI,UAAU,EAAgB,MAAM,aAAa,CAAC;AAEtE,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,MAAM,WAAW,cAAc;IAC7B,WAAW,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/B;;;;;;;;;OASG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,EAC7B,OAAO,GAAE,cAAmB,GAC3B,WAAW,EAAE,CA0Bf"}

package/dist/records.js

@@ -0,0 +1,37 @@
+/** Turning labeled `.jevl` records into calibration observations. */
+import { parseAnswers } from "jevkit-core";
+import { Observation } from "./metrics.js";
+/**
+ * Extract one observation per labeled answer.
+ *
+ * Records with no `label` are skipped: calibration needs ground truth, and
+ * silently treating an unlabeled record as correct or incorrect would poison
+ * every number downstream.
+ */
+export function observationsFromRecords(records, options = {}) {
+    const wanted = options.questionIds ? new Set(options.questionIds) : null;
+    const out = [];
+    for (const record of records) {
+        if (!record.label || !Object.keys(record.label).length)
+            continue;
+        const answers = parseAnswers(record.answers);
+        for (const [qid, label] of Object.entries(record.label)) {
+            if (wanted && !wanted.has(qid))
+                continue;
+            const answer = answers[qid];
+            if (!answer)
+                continue;
+            const probability = options.useConfidence && answer.confidence !== null
+                ? answer.confidence
+                : answer.topProbability;
+            out.push(new Observation({
+                probability,
+                correct: answer.isCorrect(label),
+                questionId: qid,
+                requestId: record.requestId,
+            }));
+        }
+    }
+    return out;
+}
+//# sourceMappingURL=records.js.map

package/dist/records.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"records.js","sourceRoot":"","sources":["../src/records.ts"],"names":[],"mappings":"AAAA,qEAAqE;AAErE,OAAO,EAA6B,YAAY,EAAE,MAAM,aAAa,CAAC;AAEtE,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAiB3C;;;;;;GAMG;AACH,MAAM,UAAU,uBAAuB,CACrC,OAA6B,EAC7B,UAA0B,EAAE;IAE5B,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzE,MAAM,GAAG,GAAkB,EAAE,CAAC;IAE9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM;YAAE,SAAS;QACjE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC7C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACzC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,MAAM;gBAAE,SAAS;YACtB,MAAM,WAAW,GACf,OAAO,CAAC,aAAa,IAAI,MAAM,CAAC,UAAU,KAAK,IAAI;gBACjD,CAAC,CAAC,MAAM,CAAC,UAAU;gBACnB,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC;YAC5B,GAAG,CAAC,IAAI,CACN,IAAI,WAAW,CAAC;gBACd,WAAW;gBACX,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;gBAChC,UAAU,EAAE,GAAG;gBACf,SAAS,EAAE,MAAM,CAAC,SAAS;aAC5B,CAAC,CACH,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/thresholds.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Choosing a confidence threshold from labeled data instead of guessing.
+ *
+ * TypeSafe's confidence page recommends three bands: act automatically, proceed
+ * with caution, escalate. It also says plainly that where you draw those lines
+ * depends on your domain and your data. This module draws them from the data.
+ *
+ * The trade is always the same. Raising the threshold means acting on fewer
+ * cases but being right more often on the ones you do act on. That is a curve,
+ * not a number, so `sweep` returns the whole curve and the `recommendFor*`
+ * functions pick a point on it against a constraint you state.
+ */
+import type { Observation } from "./metrics.js";
+/** What happens if you auto-handle everything at or above `threshold`. */
+export declare class ThresholdPoint {
+    readonly threshold: number;
+    readonly covered: number;
+    readonly total: number;
+    readonly correct: number;
+    constructor(threshold: number, covered: number, total: number, correct: number);
+    /** Fraction of cases handled automatically. */
+    get coverage(): number;
+    /**
+     * Accuracy on the automatically handled cases.
+     *
+     * Defined as 1 when nothing is covered: a threshold that acts on nothing is
+     * never wrong. Read it together with `coverage`, never alone.
+     */
+    get accuracy(): number;
+    get escalated(): number;
+    /** Wrong answers you acted on. Usually the number that actually costs. */
+    get errors(): number;
+}
+/**
+ * Coverage and accuracy at every threshold from 0 to 1.
+ *
+ * Uses the probability the model put on the outcome it chose, so this works for
+ * Choice and Score confidence and for a Noul's distance from 0.5 alike, as long
+ * as the caller is consistent about which it fed in.
+ */
+export declare function sweep(observations: Observation[], steps?: number): ThresholdPoint[];
+/**
+ * Lowest threshold reaching `targetAccuracy`, so coverage stays highest.
+ *
+ * Returns `null` when no threshold reaches the target, which is a real answer:
+ * it means this question cannot be automated at that accuracy and the honest
+ * move is to change the question rather than the threshold.
+ */
+export declare function recommendForAccuracy(observations: Observation[], targetAccuracy: number, steps?: number): ThresholdPoint | null;
+/**
+ * Highest threshold still covering `minCoverage`, so accuracy is best.
+ *
+ * The mirror of `recommendForAccuracy`: use it when throughput is the binding
+ * constraint and you want the most accurate threshold that still keeps enough
+ * volume out of the review queue.
+ */
+export declare function recommendForCoverage(observations: Observation[], minCoverage: number, steps?: number): ThresholdPoint | null;
+//# sourceMappingURL=thresholds.d.ts.map

package/dist/thresholds.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"thresholds.d.ts","sourceRoot":"","sources":["../src/thresholds.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,0EAA0E;AAC1E,qBAAa,cAAc;IAEvB,QAAQ,CAAC,SAAS,EAAE,MAAM;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM;gBAHf,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,MAAM;IAG1B,+CAA+C;IAC/C,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED;;;;;OAKG;IACH,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,IAAI,SAAS,IAAI,MAAM,CAEtB;IAED,0EAA0E;IAC1E,IAAI,MAAM,IAAI,MAAM,CAEnB;CACF;AAED;;;;;;GAMG;AACH,wBAAgB,KAAK,CAAC,YAAY,EAAE,WAAW,EAAE,EAAE,KAAK,SAAM,GAAG,cAAc,EAAE,CAiBhF;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,YAAY,EAAE,WAAW,EAAE,EAC3B,cAAc,EAAE,MAAM,EACtB,KAAK,SAAM,GACV,cAAc,GAAG,IAAI,CASvB;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,YAAY,EAAE,WAAW,EAAE,EAC3B,WAAW,EAAE,MAAM,EACnB,KAAK,SAAM,GACV,cAAc,GAAG,IAAI,CAOvB"}

package/dist/thresholds.js

@@ -0,0 +1,97 @@
+/**
+ * Choosing a confidence threshold from labeled data instead of guessing.
+ *
+ * TypeSafe's confidence page recommends three bands: act automatically, proceed
+ * with caution, escalate. It also says plainly that where you draw those lines
+ * depends on your domain and your data. This module draws them from the data.
+ *
+ * The trade is always the same. Raising the threshold means acting on fewer
+ * cases but being right more often on the ones you do act on. That is a curve,
+ * not a number, so `sweep` returns the whole curve and the `recommendFor*`
+ * functions pick a point on it against a constraint you state.
+ */
+/** What happens if you auto-handle everything at or above `threshold`. */
+export class ThresholdPoint {
+    threshold;
+    covered;
+    total;
+    correct;
+    constructor(threshold, covered, total, correct) {
+        this.threshold = threshold;
+        this.covered = covered;
+        this.total = total;
+        this.correct = correct;
+    }
+    /** Fraction of cases handled automatically. */
+    get coverage() {
+        return this.total ? this.covered / this.total : 0;
+    }
+    /**
+     * Accuracy on the automatically handled cases.
+     *
+     * Defined as 1 when nothing is covered: a threshold that acts on nothing is
+     * never wrong. Read it together with `coverage`, never alone.
+     */
+    get accuracy() {
+        return this.covered ? this.correct / this.covered : 1;
+    }
+    get escalated() {
+        return this.total - this.covered;
+    }
+    /** Wrong answers you acted on. Usually the number that actually costs. */
+    get errors() {
+        return this.covered - this.correct;
+    }
+}
+/**
+ * Coverage and accuracy at every threshold from 0 to 1.
+ *
+ * Uses the probability the model put on the outcome it chose, so this works for
+ * Choice and Score confidence and for a Noul's distance from 0.5 alike, as long
+ * as the caller is consistent about which it fed in.
+ */
+export function sweep(observations, steps = 101) {
+    if (steps < 2)
+        throw new RangeError("steps must be at least 2");
+    const total = observations.length;
+    const points = [];
+    for (let i = 0; i < steps; i++) {
+        const threshold = i / (steps - 1);
+        const covered = observations.filter((o) => o.probability >= threshold);
+        points.push(new ThresholdPoint(threshold, covered.length, total, covered.filter((o) => o.correct).length));
+    }
+    return points;
+}
+/**
+ * Lowest threshold reaching `targetAccuracy`, so coverage stays highest.
+ *
+ * Returns `null` when no threshold reaches the target, which is a real answer:
+ * it means this question cannot be automated at that accuracy and the honest
+ * move is to change the question rather than the threshold.
+ */
+export function recommendForAccuracy(observations, targetAccuracy, steps = 101) {
+    if (!(targetAccuracy >= 0 && targetAccuracy <= 1)) {
+        throw new RangeError("targetAccuracy must be in [0, 1]");
+    }
+    const candidates = sweep(observations, steps).filter((p) => p.covered > 0 && p.accuracy >= targetAccuracy);
+    if (!candidates.length)
+        return null;
+    return candidates.reduce((best, p) => (p.threshold < best.threshold ? p : best));
+}
+/**
+ * Highest threshold still covering `minCoverage`, so accuracy is best.
+ *
+ * The mirror of `recommendForAccuracy`: use it when throughput is the binding
+ * constraint and you want the most accurate threshold that still keeps enough
+ * volume out of the review queue.
+ */
+export function recommendForCoverage(observations, minCoverage, steps = 101) {
+    if (!(minCoverage >= 0 && minCoverage <= 1)) {
+        throw new RangeError("minCoverage must be in [0, 1]");
+    }
+    const candidates = sweep(observations, steps).filter((p) => p.coverage >= minCoverage);
+    if (!candidates.length)
+        return null;
+    return candidates.reduce((best, p) => (p.threshold > best.threshold ? p : best));
+}
+//# sourceMappingURL=thresholds.js.map

package/dist/thresholds.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"thresholds.js","sourceRoot":"","sources":["../src/thresholds.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH,0EAA0E;AAC1E,MAAM,OAAO,cAAc;IAEd;IACA;IACA;IACA;IAJX,YACW,SAAiB,EACjB,OAAe,EACf,KAAa,EACb,OAAe;QAHf,cAAS,GAAT,SAAS,CAAQ;QACjB,YAAO,GAAP,OAAO,CAAQ;QACf,UAAK,GAAL,KAAK,CAAQ;QACb,YAAO,GAAP,OAAO,CAAQ;IACvB,CAAC;IAEJ,+CAA+C;IAC/C,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;IAED;;;;;OAKG;IACH,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC;IACnC,CAAC;IAED,0EAA0E;IAC1E,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IACrC,CAAC;CACF;AAED;;;;;;GAMG;AACH,MAAM,UAAU,KAAK,CAAC,YAA2B,EAAE,KAAK,GAAG,GAAG;IAC5D,IAAI,KAAK,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,0BAA0B,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC;IAClC,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,IAAI,SAAS,CAAC,CAAC;QACvE,MAAM,CAAC,IAAI,CACT,IAAI,cAAc,CAChB,SAAS,EACT,OAAO,CAAC,MAAM,EACd,KAAK,EACL,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CACxC,CACF,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAClC,YAA2B,EAC3B,cAAsB,EACtB,KAAK,GAAG,GAAG;IAEX,IAAI,CAAC,CAAC,cAAc,IAAI,CAAC,IAAI,cAAc,IAAI,CAAC,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,UAAU,CAAC,kCAAkC,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,MAAM,CAClD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,IAAI,cAAc,CACrD,CAAC;IACF,IAAI,CAAC,UAAU,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACpC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AACnF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAClC,YAA2B,EAC3B,WAAmB,EACnB,KAAK,GAAG,GAAG;IAEX,IAAI,CAAC,CAAC,WAAW,IAAI,CAAC,IAAI,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,UAAU,CAAC,+BAA+B,CAAC,CAAC;IACxD,CAAC;IACD,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,IAAI,WAAW,CAAC,CAAC;IACvF,IAAI,CAAC,UAAU,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACpC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;AACnF,CAAC"}

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "jevkit-calibrate",
+  "version": "0.1.0",
+  "description": "Verify TypeSafe Jev's calibration on your own data. Reliability diagrams, ECE, Brier, and confidence thresholds derived from labeled outcomes.",
+  "keywords": [
+    "jev",
+    "typesafe",
+    "system-one",
+    "calibration",
+    "ece",
+    "reliability",
+    "thresholds"
+  ],
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "jevkit-calibrate": "./dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/pjdurden/jevkit-js.git"
+  },
+  "dependencies": {
+    "jevkit-core": "^0.2.0"
+  },
+  "scripts": {
+    "build": "tsc -b"
+  }
+}