akm-cli 0.9.0-beta.2 → 0.9.0-beta.26

package/dist/commands/improve/calibration.js

@@ -0,0 +1,161 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+// ---------------------------------------------------------------------------
+// Reliability computation
+// ---------------------------------------------------------------------------
+/** Number of fixed-width reliability buckets over [0, 1]. */
+export const CALIBRATION_BUCKET_COUNT = 10;
+function roundRate(value) {
+    return Number(value.toFixed(4));
+}
+/**
+ * Assign a confidence in [0, 1] to one of {@link CALIBRATION_BUCKET_COUNT}
+ * fixed-width buckets. The final bucket is closed on the right so confidence
+ * === 1 lands in the top bucket rather than overflowing.
+ */
+function bucketIndex(confidence) {
+    const clamped = Math.min(1, Math.max(0, confidence));
+    const idx = Math.floor(clamped * CALIBRATION_BUCKET_COUNT);
+    return Math.min(CALIBRATION_BUCKET_COUNT - 1, idx);
+}
+/**
+ * Compute a deterministic calibration summary from a list of acted-on gate
+ * decisions. Pure: identical input always yields identical output, with no
+ * dependency on wall-clock time or randomness.
+ */
+export function summarizeCalibration(samples) {
+    const buckets = [];
+    const bucketCounts = new Array(CALIBRATION_BUCKET_COUNT).fill(0);
+    const bucketAccepted = new Array(CALIBRATION_BUCKET_COUNT).fill(0);
+    const bucketConfSum = new Array(CALIBRATION_BUCKET_COUNT).fill(0);
+    let accepted = 0;
+    let confSum = 0;
+    for (const sample of samples) {
+        const idx = bucketIndex(sample.confidence);
+        bucketCounts[idx] = (bucketCounts[idx] ?? 0) + 1;
+        bucketConfSum[idx] = (bucketConfSum[idx] ?? 0) + sample.confidence;
+        confSum += sample.confidence;
+        if (sample.outcome === "auto-accepted") {
+            accepted += 1;
+            bucketAccepted[idx] = (bucketAccepted[idx] ?? 0) + 1;
+        }
+    }
+    const total = samples.length;
+    const width = 1 / CALIBRATION_BUCKET_COUNT;
+    for (let i = 0; i < CALIBRATION_BUCKET_COUNT; i += 1) {
+        const count = bucketCounts[i] ?? 0;
+        const acc = bucketAccepted[i] ?? 0;
+        buckets.push({
+            lower: roundRate(i * width),
+            upper: roundRate((i + 1) * width),
+            count,
+            accepted: acc,
+            acceptRate: count > 0 ? roundRate(acc / count) : 0,
+            meanConfidence: count > 0 ? roundRate((bucketConfSum[i] ?? 0) / count) : 0,
+        });
+    }
+    const overallAcceptRate = total > 0 ? roundRate(accepted / total) : 0;
+    const meanConfidence = total > 0 ? roundRate(confSum / total) : 0;
+    return {
+        samples: total,
+        accepted,
+        rejected: total - accepted,
+        overallAcceptRate,
+        meanConfidence,
+        calibrationGap: total > 0 ? roundRate(meanConfidence - overallAcceptRate) : 0,
+        buckets,
+    };
+}
+/**
+ * Project a list of `gateDecision` records (read from the proposal store) into
+ * the acted-on calibration samples within an optional `[since, until)` window.
+ *
+ * Only `auto-accepted` / `auto-rejected` decisions with a finite confidence in
+ * [0, 1] contribute. `deferred` decisions and decisions missing a confidence
+ * are excluded (no realized accept/reject signal). The window filter uses each
+ * decision's `decidedAt` timestamp; decisions with an unparseable timestamp are
+ * kept only when no window is supplied.
+ *
+ * Exploration-budget promotions (`reason === "exploration-budget"`) are EXCLUDED:
+ * they are accepted regardless of confidence, so they carry no reliability signal
+ * about the gate threshold. Counting them would inflate the apparent accept-rate
+ * and bias the auto-tuner downward — they are exempt from auto-tune by design
+ * (WS-4 exploration budget).
+ */
+export function gateDecisionsToSamples(decisions, window) {
+    const sinceMs = window?.since ? new Date(window.since).getTime() : undefined;
+    const untilMs = window?.until ? new Date(window.until).getTime() : undefined;
+    const samples = [];
+    for (const decision of decisions) {
+        if (!decision)
+            continue;
+        if (decision.outcome !== "auto-accepted" && decision.outcome !== "auto-rejected")
+            continue;
+        if (decision.reason === "exploration-budget")
+            continue;
+        const confidence = decision.confidence;
+        if (typeof confidence !== "number" || !Number.isFinite(confidence) || confidence < 0 || confidence > 1)
+            continue;
+        if (sinceMs !== undefined || untilMs !== undefined) {
+            const ts = new Date(decision.decidedAt).getTime();
+            if (!Number.isFinite(ts))
+                continue;
+            if (sinceMs !== undefined && ts < sinceMs)
+                continue;
+            if (untilMs !== undefined && ts >= untilMs)
+                continue;
+        }
+        samples.push({ confidence, outcome: decision.outcome });
+    }
+    return samples;
+}
+/**
+ * Compute a bounded, opt-in threshold adjustment from a calibration summary.
+ * PURE and deterministic — does not mutate config or read the clock. The
+ * caller is responsible for persisting `newThreshold` and logging the result.
+ *
+ * Algorithm (deliberately simple and bounded):
+ *   - When `autoTune` is false → no-op (`disabled`).
+ *   - When samples < `minSamples` → no-op (`insufficient-samples`).
+ *   - Otherwise nudge by at most `maxStep` toward `targetAcceptRate`, then
+ *     clamp into `[minThreshold, maxThreshold]`. The step size scales with the
+ *     gap from target but is capped, so a single run can never make a large
+ *     swing.
+ */
+export function computeThresholdAutoTune(currentThreshold, summary, config) {
+    const previousThreshold = Math.round(currentThreshold);
+    const noop = (reason) => ({
+        adjusted: false,
+        previousThreshold,
+        newThreshold: previousThreshold,
+        delta: 0,
+        reason,
+    });
+    if (!config.autoTune)
+        return noop("disabled");
+    if (summary.samples < config.minSamples)
+        return noop("insufficient-samples");
+    const gap = config.targetAcceptRate - summary.overallAcceptRate;
+    // A small dead-band so tiny noise doesn't churn the threshold every run.
+    const DEAD_BAND = 0.01;
+    if (Math.abs(gap) <= DEAD_BAND)
+        return noop("within-target");
+    // gap > 0 ⇒ realized below target ⇒ raise threshold (be stricter).
+    // gap < 0 ⇒ realized above target ⇒ lower threshold (be more permissive).
+    const direction = gap > 0 ? 1 : -1;
+    // Scale the step with the gap magnitude (in points) but cap at maxStep.
+    const desiredMagnitude = Math.min(config.maxStep, Math.max(1, Math.round(Math.abs(gap) * 100)));
+    const proposed = previousThreshold + direction * desiredMagnitude;
+    const clamped = Math.min(config.maxThreshold, Math.max(config.minThreshold, proposed));
+    const delta = clamped - previousThreshold;
+    if (delta === 0)
+        return noop("clamped-at-bound");
+    return {
+        adjusted: true,
+        previousThreshold,
+        newThreshold: clamped,
+        delta,
+        reason: direction > 0 ? "below-target-raise" : "above-target-lower",
+    };
+}