@harperfast/harper 5.3.0 → 5.3.1

package/dist/resources/models/calibrationStore.js

@@ -0,0 +1,986 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CALIBRATION_ATTRIBUTES = exports.CALIBRATIONS_TABLE = void 0;
+exports.getCalibrationsTable = getCalibrationsTable;
+exports.resetCalibrationsTable = resetCalibrationsTable;
+exports.declareCalibrationsTableAtBoot = declareCalibrationsTableAtBoot;
+exports.isCalibrationEnabled = isCalibrationEnabled;
+exports.configureCalibration = configureCalibration;
+exports.calibrationJob = calibrationJob;
+exports.resetCalibrationCache = resetCalibrationCache;
+exports.outstandingCalibrationReads = outstandingCalibrationReads;
+exports.setCalibrationReadForTests = setCalibrationReadForTests;
+exports.applyFits = applyFits;
+exports.runCalibration = runCalibration;
+exports.processingOrder = processingOrder;
+exports.listCalibrations = listCalibrations;
+const databases_ts_1 = require("../databases.js");
+const transaction_ts_1 = require("../transaction.js");
+const harper_logger_ts_1 = __importDefault(require("../../utility/logging/harper_logger.js"));
+const manageThreads_js_1 = require("../../server/threads/manageThreads.js");
+const write_ts_1 = require("../analytics/write.js");
+const engine_ts_1 = require("../scheduler/engine.js");
+const decision_ts_1 = require("./decision.js");
+const decisionStore_ts_1 = require("./decisionStore.js");
+const calibration_ts_1 = require("./calibration.js");
+const log = harper_logger_ts_1.default.forComponent('models').conditional;
+exports.CALIBRATIONS_TABLE = 'hdb_model_calibrations';
+const JOB_NAME = 'models-calibration';
+const CACHE_LIMIT = 1024;
+const CACHE_FRESH_MS = 60_000;
+const LOAD_TIMEOUT_MS = 2_000;
+const FAULT_LOG_INTERVAL_MS = 60_000;
+const SCAN_PAGE = 500;
+const YIELD_EVERY = 500;
+const EXAMPLE_OVERHEAD_BYTES = 200;
+const POPULATION_OVERHEAD_BYTES = 1_000;
+const DAY_MS = 24 * 60 * 60 * 1000;
+const RANK_WIDTH = 16;
+const CURSOR_ID = 'discovery/cursor';
+const LIST_BATCH = 16;
+const BUDGET_KEYS = ['maxDecisions', 'maxPopulations', 'maxExamplesPerKey', 'maxBytes', 'maxRunMs'];
+function budgetsOnly(budgets, configured) {
+    const out = {};
+    for (const key of BUDGET_KEYS) {
+        const value = budgets?.[key];
+        if (typeof value === 'number' && Number.isSafeInteger(value) && value > 0)
+            out[key] = Math.min(value, configured[key]);
+    }
+    return out;
+}
+const DEFAULTS = {
+    interval: DAY_MS,
+    maxDecisions: 100_000,
+    maxPopulations: 1_000,
+    maxExamplesPerKey: 5_000,
+    maxBytes: 64 * 1024 * 1024,
+    maxLoads: 8,
+    maxRunMs: 60_000,
+    minReport: 20,
+    minTrain: 100,
+    minHeldOut: 100,
+    heldOutShare: 0.3,
+    eceMargin: 0.01,
+    maxAgeMs: 30 * DAY_MS,
+};
+exports.CALIBRATION_ATTRIBUTES = [
+    { name: 'id', isPrimaryKey: true },
+    { name: 'kind', type: 'string', indexed: true },
+    { name: 'key', type: 'string' },
+    { name: 'rank', type: 'string', indexed: true },
+    { name: 'population', type: 'string' },
+    { name: 'field', type: 'string' },
+    { name: 'tenant', type: 'string' },
+    { name: 'app', type: 'string' },
+    { name: 'model', type: 'string' },
+    { name: 'entry', type: 'string' },
+    { name: 'signature', type: 'string' },
+    { name: 'instructionsHash', type: 'string' },
+    { name: 'schemaHash', type: 'string' },
+    { name: 'schema' },
+    { name: 't', type: 'number' },
+    { name: 'epsilon', type: 'number' },
+    { name: 'cutoff', type: 'number' },
+    { name: 'evidenceAt', type: 'number' },
+    { name: 'fittedAt', type: 'number' },
+    { name: 'lastFittedAt', type: 'number', indexed: true },
+    { name: 'owner', type: 'string', indexed: true },
+    { name: 'boundary' },
+    { name: 'before', type: 'number' },
+    { name: 'applyUntil', type: 'number' },
+    { name: 'expiresAt', expiresAt: true, indexed: true },
+    { name: 'eligible', type: 'boolean' },
+    { name: 'reason', type: 'string' },
+    { name: 'policy' },
+    { name: 'policyDigest', type: 'string' },
+    { name: 'inputDigest', type: 'string' },
+    { name: 'decisions', type: 'number' },
+    { name: 'labelled', type: 'number' },
+    { name: 'report' },
+];
+let calibrations;
+/** Declared like the decision tables: a read-only node takes what the catalog holds and declares nothing. */
+function getCalibrationsTable(readOnly = (0, databases_ts_1.isReadOnlyMode)()) {
+    if (calibrations)
+        return calibrations;
+    const handle = readOnly
+        ? (0, databases_ts_1.getDatabases)().system?.[exports.CALIBRATIONS_TABLE]
+        : (0, databases_ts_1.table)({
+            table: exports.CALIBRATIONS_TABLE,
+            database: 'system',
+            // Declared positively, like the decision tables: every node must apply the same versions.
+            replicate: true,
+            audit: true,
+            attributes: exports.CALIBRATION_ATTRIBUTES,
+        });
+    if (handle) {
+        handle.loadAsInstance = false;
+        calibrations = handle;
+    }
+    return handle;
+}
+function resetCalibrationsTable() {
+    calibrations = undefined;
+    resetCalibrationCache();
+}
+function declareCalibrationsTableAtBoot() {
+    if ((0, databases_ts_1.isReadOnlyMode)())
+        return;
+    try {
+        getCalibrationsTable();
+    }
+    catch (err) {
+        log.warn?.(`models: calibration table could not be declared at boot: ${(0, engine_ts_1.safeErrorMessage)(err)}`);
+    }
+}
+function freshContext() {
+    const user = transaction_ts_1.contextStorage.getStore()?.user;
+    return (user ? { user } : {});
+}
+let settings;
+let activePolicyDigest;
+let registeredBlock;
+function isCalibrationEnabled() {
+    return settings !== undefined;
+}
+function policyOf(config) {
+    return {
+        minReport: config.minReport,
+        minTrain: config.minTrain,
+        minHeldOut: config.minHeldOut,
+        heldOutShare: config.heldOutShare,
+        eceMargin: config.eceMargin,
+        maxAgeMs: config.maxAgeMs,
+    };
+}
+/**
+ * Apply the `models.calibration` block. Present: fits apply on every worker, and the application primary
+ * worker registers the cluster-once job. Absent: nothing applies and the job is removed.
+ */
+function configureCalibration(block, primary = (0, manageThreads_js_1.isApplicationPrimaryWorker)(undefined)) {
+    if (!block) {
+        resetCalibrationCache();
+        settings = undefined;
+        activePolicyDigest = undefined;
+        registeredBlock = undefined;
+        if (primary)
+            (0, engine_ts_1.unregisterInternalJobs)(JOB_NAME);
+        return;
+    }
+    const next = { ...DEFAULTS, ...definedOnly(block) };
+    const digest = (0, decision_ts_1.canonicalJson)(next);
+    if (settings && (0, decision_ts_1.canonicalJson)(settings) === digest && (registeredBlock === digest || !primary))
+        return;
+    resetCalibrationCache();
+    settings = next;
+    activePolicyDigest = (0, calibration_ts_1.policyDigest)(policyOf(settings));
+    if (!primary || (0, databases_ts_1.isReadOnlyMode)())
+        return;
+    registeredBlock = digest;
+    (0, engine_ts_1.registerInternalJobs)(JOB_NAME, [
+        {
+            name: 'calibrate',
+            componentName: (0, engine_ts_1.internalJobOwner)(JOB_NAME),
+            intervalMs: settings.interval,
+            handler: calibrationJob,
+        },
+    ]);
+    (0, engine_ts_1.startSchedulerEngine)();
+}
+async function calibrationJob() {
+    const run = await runCalibration();
+    if (run.status === 'failed')
+        throw new Error(`calibration run failed: ${run.error}`);
+}
+function definedOnly(block) {
+    return Object.fromEntries(Object.entries(block).filter(([, value]) => value !== undefined));
+}
+const cache = new Map();
+const loading = new Map();
+let generation = 0;
+let outstandingReads = 0;
+let lastFaultLog = 0;
+function resetCalibrationCache() {
+    generation++;
+    cache.clear();
+    loading.clear();
+}
+function invalidate(keys) {
+    if (keys.length === 0)
+        return;
+    generation++;
+    for (const key of keys) {
+        cache.delete(key);
+        loading.delete(key);
+    }
+}
+function outstandingCalibrationReads() {
+    return outstandingReads;
+}
+function remember(key, fit) {
+    cache.delete(key);
+    cache.set(key, { fit, loadedAt: Date.now() });
+    while (cache.size > CACHE_LIMIT)
+        cache.delete(cache.keys().next().value);
+}
+function logFault(what, err) {
+    const now = Date.now();
+    if (now - lastFaultLog < FAULT_LOG_INTERVAL_MS)
+        return;
+    lastFaultLog = now;
+    log.warn?.(`models: ${what}: ${(0, engine_ts_1.safeErrorMessage)(err)}`);
+}
+function schemaBytes(schema) {
+    try {
+        return ((0, decision_ts_1.canonicalJson)(schema ?? null) ?? '').length;
+    }
+    catch {
+        return 0;
+    }
+}
+function ownerOf(tenant) {
+    return tenant == null ? 'none' : `t:${tenant}`;
+}
+function pad(n) {
+    return String(Math.max(0, Math.floor(n))).padStart(RANK_WIDTH, '0');
+}
+function rankOf(key, evidenceAt, fittedAt, id) {
+    return `${key}|${pad(evidenceAt)}|${pad(fittedAt)}|${id}`;
+}
+/** The newest version of a key: the greatest rank within the key's prefix, one descending index read. */
+async function readNewest(key) {
+    const rows = getCalibrationsTable();
+    if (!rows)
+        return null;
+    return (0, transaction_ts_1.transaction)(freshContext(), async () => {
+        for await (const row of rows.search({
+            conditions: [{ attribute: 'rank', comparator: 'starts_with', value: `${key}|`, descending: true }],
+            limit: 1,
+        })) {
+            return row;
+        }
+        return null;
+    });
+}
+function scheduleLoad(key, read = readNewest) {
+    if (loading.has(key) || !settings || outstandingReads >= settings.maxLoads)
+        return;
+    const token = {};
+    const startedAt = generation;
+    loading.set(key, token);
+    outstandingReads++;
+    const owns = () => loading.get(key) === token && generation === startedAt;
+    const timer = setTimeout(() => {
+        if (!owns())
+            return;
+        remember(key, null);
+        loading.delete(key);
+    }, LOAD_TIMEOUT_MS);
+    timer.unref?.();
+    let pending;
+    try {
+        pending = read(key);
+    }
+    catch (err) {
+        pending = Promise.reject(err);
+    }
+    pending
+        .then((row) => {
+        if (owns())
+            remember(key, row);
+    }, (err) => {
+        logFault('calibration lookup failed', err);
+        if (owns())
+            remember(key, null);
+    })
+        .catch(() => { })
+        .finally(() => {
+        clearTimeout(timer);
+        outstandingReads--;
+        if (loading.get(key) === token)
+            loading.delete(key);
+    });
+}
+/** @internal — testing only: replace the storage read the cache loads through. */
+let readForLoads;
+function setCalibrationReadForTests(read) {
+    readForLoads = read;
+}
+function isUsable(row, now, key) {
+    return (!!row &&
+        row.kind === 'fit' &&
+        row.key === key &&
+        typeof row.rank === 'string' &&
+        row.rank.startsWith(`${key}|`) &&
+        typeof row.population === 'string' &&
+        (0, calibration_ts_1.calibrationKey)(row.population, row.field) === key &&
+        row.eligible === true &&
+        row.applyUntil > now &&
+        row.policyDigest === activePolicyDigest &&
+        (0, calibration_ts_1.isValidParams)(row));
+}
+function cachedFit(key, now) {
+    const entry = cache.get(key);
+    if (!entry) {
+        scheduleLoad(key, readForLoads);
+        return undefined;
+    }
+    cache.delete(key);
+    cache.set(key, entry);
+    if (now - entry.loadedAt >= CACHE_FRESH_MS)
+        scheduleLoad(key, readForLoads);
+    return isUsable(entry.fit, now, key) ? entry.fit : undefined;
+}
+function hasNoMatchLeaf(schema) {
+    if ((0, decision_ts_1.isObjectSchema)(schema))
+        return Object.values(schema.properties).some((leaf) => leaf.noMatch === true);
+    return schema.noMatch === true;
+}
+/**
+ * The decision with every distribution replaced by its population's fit, or undefined when any field lacks
+ * a cached usable fit or the order would change: all fields or none. Never waits on storage. A schema with
+ * a no-match leaf is never fit-calibrated, because its no-match score would stay raw under `calibrated: true`.
+ */
+function applyFits(schema, decision, population, now = Date.now()) {
+    if (hasNoMatchLeaf(schema))
+        return undefined;
+    if ((0, decision_ts_1.isObjectSchema)(schema)) {
+        const fields = decision.fields;
+        if (!fields)
+            return undefined;
+        const names = Object.keys(schema.properties);
+        const fits = names.map((name) => cachedFit((0, calibration_ts_1.calibrationKey)(population, name), now));
+        if (fits.some((fit) => !fit))
+            return undefined;
+        const calibrated = {};
+        const snapshot = [];
+        for (let i = 0; i < names.length; i++) {
+            const fit = fits[i];
+            const marginal = calibrateMarginal(schema.properties[names[i]], fields[names[i]], fit);
+            if (!marginal)
+                return undefined;
+            calibrated[names[i]] = marginal;
+            snapshot.push({ field: names[i], fitId: fit.id, t: fit.t, epsilon: fit.epsilon });
+        }
+        return { decision: { ...decision, fields: calibrated, calibrated: true }, rawFields: fields, snapshot };
+    }
+    const fit = cachedFit((0, calibration_ts_1.calibrationKey)(population, undefined), now);
+    if (!fit || !decision.distribution)
+        return undefined;
+    const marginal = calibrateMarginal(schema, { value: decision.value, probability: decision.probability, distribution: decision.distribution }, fit);
+    if (!marginal)
+        return undefined;
+    return {
+        decision: { ...decision, probability: marginal.probability, distribution: marginal.distribution, calibrated: true },
+        rawDistribution: decision.distribution,
+        snapshot: [{ fitId: fit.id, t: fit.t, epsilon: fit.epsilon }],
+    };
+}
+function calibrateMarginal(leaf, marginal, fit) {
+    if (!marginal)
+        return undefined;
+    const values = (0, decision_ts_1.allowedValues)(leaf);
+    const vector = (0, calibration_ts_1.toVector)(marginal.distribution, values);
+    if (!vector)
+        return undefined;
+    const out = (0, calibration_ts_1.applyCalibration)(vector, fit);
+    if (!out.every(Number.isFinite))
+        return undefined;
+    const indexByValue = new Map(values.map((value, i) => [(0, decision_ts_1.canonicalJson)(value), i]));
+    const distribution = marginal.distribution.map((entry) => ({
+        value: entry.value,
+        probability: out[indexByValue.get((0, decision_ts_1.canonicalJson)(entry.value))],
+    }));
+    for (let i = 1; i < distribution.length; i++)
+        if (distribution[i].probability > distribution[i - 1].probability)
+            return undefined;
+    if ((0, decision_ts_1.canonicalJson)(distribution[0].value) !== (0, decision_ts_1.canonicalJson)(marginal.value))
+        return undefined;
+    return { value: marginal.value, probability: distribution[0].probability, distribution };
+}
+let queue = Promise.resolve();
+/** One run at a time per process. Resolves with the run's result, an unexpected fault included as a failed run. */
+function runCalibration(budgets = {}, deps = {}) {
+    const next = queue.then(() => runOnce(budgets, deps)).catch((err) => failedRun(err));
+    queue = next;
+    return next;
+}
+function failedRun(err) {
+    const error = `unexpected fault: ${(0, engine_ts_1.safeErrorMessage)(err)}`;
+    log.error?.(`models: calibration run failed ${error}`);
+    return {
+        status: 'failed',
+        error,
+        scanned: 0,
+        read: 0,
+        discovered: 0,
+        processed: 0,
+        pending: 0,
+        written: 0,
+        eligible: 0,
+        skipped: 0,
+        failed: 0,
+        durationMs: 0,
+    };
+}
+async function runOnce(budgets, deps) {
+    const now = deps.now ?? Date.now;
+    const configured = { ...DEFAULTS, ...settings };
+    const config = { ...configured, ...budgetsOnly(budgets, configured) };
+    const policy = policyOf(config);
+    const digestOfPolicy = (0, calibration_ts_1.policyDigest)(policy);
+    const started = now();
+    const deadline = started + config.maxRunMs;
+    const result = {
+        status: 'completed',
+        scanned: 0,
+        read: 0,
+        discovered: 0,
+        processed: 0,
+        pending: 0,
+        written: 0,
+        eligible: 0,
+        skipped: 0,
+        failed: 0,
+        durationMs: 0,
+    };
+    const { decisions, outcomes } = (0, decisionStore_ts_1.getDecisionTables)();
+    const store = getCalibrationsTable();
+    if (!decisions || !outcomes || !store)
+        return finish();
+    const populations = new Map();
+    const writtenKeys = [];
+    let bytes = 0;
+    await discover();
+    try {
+        const discovered = [...populations.values()];
+        const known = await (0, transaction_ts_1.transaction)(freshContext(), async () => {
+            for (const found of discovered) {
+                const head = await store.get(found.head.id);
+                if (head?.lastFittedAt !== undefined)
+                    found.lastFittedAt = head.lastFittedAt;
+                else
+                    await store.put({ ...found.head, lastFittedAt: 0, expiresAt: found.expiresAt ?? now() + policy.maxAgeMs });
+            }
+            const out = [];
+            for await (const row of store.search({
+                conditions: [{ attribute: 'kind', value: 'population' }],
+                sort: { attribute: 'lastFittedAt' },
+                limit: config.maxPopulations,
+            }))
+                out.push(row);
+            return out;
+        });
+        for (const head of known) {
+            if (populations.has(head.population))
+                continue;
+            const size = POPULATION_OVERHEAD_BYTES + schemaBytes(head.schema);
+            if (bytes + size > config.maxBytes) {
+                result.stoppedBy ??= 'maxBytes';
+                break;
+            }
+            bytes += size;
+            const { lastFittedAt, expiresAt, ...rest } = head;
+            populations.set(head.population, { head: rest, lastFittedAt, expiresAt, bytes: size });
+        }
+    }
+    catch (err) {
+        fail('reading known populations', err);
+    }
+    result.discovered = populations.size;
+    const ordered = processingOrder([...populations.values()]);
+    const order = ordered.slice(0, config.maxPopulations);
+    if (ordered.length > order.length) {
+        result.stoppedBy ??= 'maxPopulations';
+        result.pending = ordered.length - order.length;
+    }
+    let allowance = config.maxDecisions - result.scanned;
+    for (let i = 0; i < order.length; i++) {
+        if (allowance <= 0) {
+            result.stoppedBy ??= 'maxDecisions';
+            result.pending += order.length - i;
+            break;
+        }
+        if (now() >= deadline) {
+            result.stoppedBy ??= 'maxRunMs';
+            result.pending += order.length - i;
+            break;
+        }
+        const outcome = await processPopulation(order[i]);
+        if (outcome === 'deferred') {
+            await rotate(order[i]);
+            result.stoppedBy ??= 'maxDecisions';
+            result.pending += order.length - i;
+            break;
+        }
+        if (outcome === 'budget') {
+            await rotate(order[i]);
+            result.stoppedBy ??= 'maxBytes';
+            result.pending += order.length - i;
+            break;
+        }
+        if (outcome === 'deadline') {
+            await rotate(order[i]);
+            result.stoppedBy ??= 'maxRunMs';
+            result.pending += order.length - i;
+            break;
+        }
+        result.processed++;
+    }
+    invalidate(writtenKeys);
+    return finish();
+    async function discover() {
+        const decisionBudget = Math.max(1, Math.floor(config.maxDecisions / 2));
+        const newest = await scan(undefined, [], Math.floor(decisionBudget / 2), Math.floor(config.maxPopulations / 2));
+        if (newest.fault || newest.stopped)
+            return;
+        if (newest.end)
+            return saveCursor(undefined, []);
+        let cursor;
+        try {
+            cursor = await (0, transaction_ts_1.transaction)(freshContext(), () => store.get(CURSOR_ID));
+        }
+        catch (err) {
+            logFault('calibration could not read its discovery cursor', err);
+        }
+        const saved = typeof cursor?.before === 'number' ? cursor.before : undefined;
+        const savedSeen = Array.isArray(cursor?.boundary) ? cursor.boundary : [];
+        const resume = saved !== undefined && (newest.before === undefined || saved <= newest.before);
+        const from = resume ? saved : newest.before;
+        const seen = !resume
+            ? newest.boundary
+            : saved === newest.before
+                ? [...new Set([...savedSeen, ...newest.boundary])]
+                : savedSeen;
+        const rest = await scan(from, seen, decisionBudget - result.scanned, config.maxPopulations - populations.size);
+        if (rest.fault)
+            return;
+        if (rest.capped)
+            result.stoppedBy ??= rest.capped;
+        else if (!rest.end && !rest.stopped)
+            result.stoppedBy ??= 'maxDecisions';
+        return rest.end ? saveCursor(undefined, []) : saveCursor(rest.before, rest.boundary);
+    }
+    async function saveCursor(before, boundary) {
+        try {
+            await (0, transaction_ts_1.transaction)(freshContext(), () => store.put({ id: CURSOR_ID, kind: 'cursor', before, boundary, expiresAt: now() + decisionStore_ts_1.DECISION_RETENTION_MS }));
+        }
+        catch (err) {
+            logFault('calibration could not save its discovery cursor', err);
+        }
+    }
+    async function scan(start, startSeen, budget, populationBudget) {
+        let before = start;
+        let taken = 0;
+        let added = 0;
+        const seenAtBoundary = new Set(start === undefined ? [] : startSeen);
+        while (true) {
+            if (taken >= budget)
+                return { before, boundary: [...seenAtBoundary], end: false, stopped: false };
+            let page;
+            const requested = Math.min(SCAN_PAGE, budget - taken) + seenAtBoundary.size;
+            let returned = 0;
+            try {
+                page = await (0, transaction_ts_1.transaction)(freshContext(), async () => {
+                    const rows = [];
+                    const conditions = before === undefined
+                        ? [{ attribute: 'expiresAt', comparator: 'greater_than', value: 0, descending: true }]
+                        : [{ attribute: 'expiresAt', comparator: 'less_than_equal', value: before, descending: true }];
+                    for await (const row of decisions.search({
+                        conditions,
+                        limit: requested,
+                    })) {
+                        returned++;
+                        if (!seenAtBoundary.has(row.id))
+                            rows.push(row);
+                    }
+                    return rows;
+                });
+            }
+            catch (err) {
+                fail('discovering populations', err);
+                return { before, boundary: [...seenAtBoundary], end: false, stopped: true, fault: true };
+            }
+            if (page.length === 0)
+                return { before, boundary: [...seenAtBoundary], end: true, stopped: false };
+            for (const row of page) {
+                if (taken >= budget)
+                    return { before, boundary: [...seenAtBoundary], end: false, stopped: false };
+                if (now() >= deadline) {
+                    result.stoppedBy ??= 'maxRunMs';
+                    return { before, boundary: [...seenAtBoundary], end: false, stopped: true };
+                }
+                const isNew = Boolean(row.population && row.signature) && !populations.has(row.population);
+                let size = 0;
+                if (isNew) {
+                    if (added >= populationBudget)
+                        return { before, boundary: [...seenAtBoundary], end: false, stopped: false, capped: 'maxPopulations' };
+                    size = POPULATION_OVERHEAD_BYTES + schemaBytes(row.schema);
+                    if (bytes + size > config.maxBytes)
+                        return { before, boundary: [...seenAtBoundary], end: false, stopped: false, capped: 'maxBytes' };
+                }
+                taken++;
+                result.scanned++;
+                result.reachedAt = Math.min(result.reachedAt ?? row.at, row.at);
+                if (row.expiresAt !== before)
+                    seenAtBoundary.clear();
+                before = row.expiresAt;
+                seenAtBoundary.add(row.id);
+                if (result.scanned % YIELD_EVERY === 0)
+                    await new Promise((resolve) => setImmediate(resolve));
+                if (!isNew)
+                    continue;
+                added++;
+                bytes += size;
+                populations.set(row.population, {
+                    bytes: size,
+                    expiresAt: row.expiresAt,
+                    head: {
+                        id: `population/${row.population}`,
+                        kind: 'population',
+                        population: row.population,
+                        owner: ownerOf(row.tenant),
+                        tenant: row.tenant,
+                        model: row.model,
+                        entry: row.entry ?? `registered:${row.backend}`,
+                        signature: row.signature,
+                        instructionsHash: row.instructionsHash,
+                        schemaHash: row.schemaHash,
+                        schema: row.schema,
+                    },
+                });
+            }
+            if (returned < requested)
+                return { before, boundary: [...seenAtBoundary], end: true, stopped: false };
+        }
+    }
+    async function processPopulation(found) {
+        const { head } = found;
+        const schema = head.schema;
+        let perKey;
+        let noMatchSchema;
+        try {
+            const leaves = (0, decision_ts_1.isObjectSchema)(schema)
+                ? Object.entries(schema.properties)
+                : [[undefined, schema]];
+            perKey = leaves.map(([field, leaf]) => ({
+                field,
+                values: (0, decision_ts_1.allowedValues)(leaf),
+                examples: [],
+                digester: (0, calibration_ts_1.inputDigester)(),
+                evidenceAt: 0,
+                cutoff: 0,
+            }));
+            noMatchSchema = hasNoMatchLeaf(schema);
+        }
+        catch (err) {
+            result.failed++;
+            logFault('calibration found a malformed population', err);
+            try {
+                const touchedAt = now();
+                await (0, transaction_ts_1.transaction)(freshContext(), () => store.put({ ...head, lastFittedAt: touchedAt, expiresAt: found.expiresAt ?? touchedAt + policy.maxAgeMs }));
+            }
+            catch { }
+            return 'failed';
+        }
+        const rowBytes = perKey.reduce((sum, key) => sum + EXAMPLE_OVERHEAD_BYTES + key.values.length * 8, 0);
+        const sample = Math.min(config.maxExamplesPerKey, Math.max(1, Math.floor((configured.maxBytes - found.bytes) / rowBytes)));
+        const limit = Math.min(sample, allowance + 1);
+        let rows;
+        let used = found.bytes;
+        let newestExpiry = 0;
+        try {
+            rows = await (0, transaction_ts_1.transaction)(freshContext(), async () => {
+                const out = [];
+                for await (const row of decisions.search({
+                    conditions: [
+                        { attribute: 'populationRank', comparator: 'starts_with', value: `${head.population}|`, descending: true },
+                    ],
+                    limit,
+                })) {
+                    out.push(row);
+                }
+                return out;
+            });
+            const available = allowance;
+            allowance -= rows.length;
+            result.read += rows.length;
+            if (rows.length > available && available < sample)
+                return 'deferred';
+            rows.sort((a, b) => a.at - b.at || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+            for (let start = 0; start < rows.length; start += YIELD_EVERY) {
+                if (start > 0) {
+                    await new Promise((resolve) => setImmediate(resolve));
+                    if (now() >= deadline)
+                        return 'deadline';
+                }
+                const chunk = rows.slice(start, start + YIELD_EVERY);
+                const chunkFacts = await (0, transaction_ts_1.transaction)(freshContext(), () => Promise.all(chunk.flatMap((row) => perKey.map((key) => outcomes.get((0, decisionStore_ts_1.truthKey)(row.id, key.field))))));
+                for (let r = 0; r < chunk.length; r++) {
+                    const row = chunk[r];
+                    if (row.expiresAt > newestExpiry)
+                        newestExpiry = row.expiresAt;
+                    const facts = chunkFacts.slice(r * perKey.length, (r + 1) * perKey.length);
+                    for (let i = 0; i < perKey.length; i++) {
+                        const key = perKey[i];
+                        const distribution = key.field === undefined
+                            ? (row.rawDistribution ?? row.distribution)
+                            : (row.rawFields ?? row.fields)?.[key.field]?.distribution;
+                        const probabilities = (0, calibration_ts_1.toVector)(distribution, key.values);
+                        const fact = facts[i];
+                        key.digester.add([row.id, probabilities ?? null, fact?.state ?? null, fact?.at ?? null]);
+                        key.evidenceAt = Math.max(key.evidenceAt, row.at, fact?.at ?? 0);
+                        if (!probabilities || !fact?.state || fact.state.kind === 'unknown')
+                            continue;
+                        let truth;
+                        if (fact.state.kind === 'noMatch')
+                            truth = calibration_ts_1.NO_MATCH_TRUTH;
+                        else {
+                            const expected = (0, decision_ts_1.canonicalJson)(fact.state.value);
+                            truth = key.values.findIndex((value) => (0, decision_ts_1.canonicalJson)(value) === expected);
+                            if (truth < 0)
+                                continue;
+                        }
+                        used += EXAMPLE_OVERHEAD_BYTES + probabilities.length * 8;
+                        if (used > config.maxBytes)
+                            return 'budget';
+                        const returned = (0, decision_ts_1.canonicalJson)(key.field === undefined ? row.value : row.value?.[key.field]);
+                        const chosen = key.values.findIndex((value) => (0, decision_ts_1.canonicalJson)(value) === returned);
+                        key.examples.push({ probabilities, truth, at: row.at, chosen: chosen >= 0 ? chosen : undefined });
+                        key.cutoff = row.at;
+                    }
+                }
+            }
+        }
+        catch (err) {
+            result.failed++;
+            logFault('calibration could not read a population', err);
+            return 'failed';
+        }
+        const epsilon = (0, calibration_ts_1.smoothingEpsilon)(head.signature);
+        const fittedAt = now();
+        const written = [];
+        try {
+            for (const key of perKey) {
+                const keyId = (0, calibration_ts_1.calibrationKey)(head.population, key.field);
+                const inputDigest = key.digester.digest();
+                const previous = await readNewest(keyId);
+                if (previous && previous.inputDigest === inputDigest && previous.policyDigest === digestOfPolicy) {
+                    result.skipped++;
+                    continue;
+                }
+                const labelled = key.examples.length;
+                if (!previous && labelled < policy.minReport) {
+                    result.skipped++;
+                    continue;
+                }
+                const inSet = key.examples.filter((example) => example.truth !== calibration_ts_1.NO_MATCH_TRUTH);
+                const { train, heldOut } = (0, calibration_ts_1.splitByTime)(key.examples, policy.heldOutShare);
+                const trainInSet = train.filter((example) => example.truth !== calibration_ts_1.NO_MATCH_TRUTH);
+                const heldOutInSet = heldOut.filter((example) => example.truth !== calibration_ts_1.NO_MATCH_TRUTH);
+                const enough = trainInSet.length >= policy.minTrain && heldOutInSet.length >= policy.minHeldOut;
+                let params;
+                if (enough && !noMatchSchema) {
+                    params = await (0, calibration_ts_1.fitCalibration)(trainInSet, epsilon, deadline, now);
+                    if (!params)
+                        return 'deadline';
+                }
+                const identity = (p) => p;
+                let report;
+                let improves = false;
+                if (params) {
+                    const fitted = params;
+                    const raw = (0, calibration_ts_1.reliability)(heldOut, identity);
+                    const calibrated = (0, calibration_ts_1.reliability)(heldOut, (p) => (0, calibration_ts_1.applyCalibration)(p, fitted));
+                    improves = calibrated.ece <= raw.ece - policy.eceMargin && calibrated.nll <= raw.nll;
+                    report = {
+                        window: 'heldOut',
+                        labelled,
+                        noMatchCount: labelled - inSet.length,
+                        trainCount: trainInSet.length,
+                        heldOutCount: heldOutInSet.length,
+                        raw,
+                        calibrated,
+                    };
+                }
+                else if (labelled >= policy.minReport) {
+                    report = {
+                        window: 'all',
+                        labelled,
+                        noMatchCount: labelled - inSet.length,
+                        trainCount: 0,
+                        heldOutCount: 0,
+                        raw: (0, calibration_ts_1.reliability)(key.examples, identity),
+                    };
+                }
+                const eligible = Boolean(params) && improves;
+                const reason = noMatchSchema
+                    ? 'no-match-schema'
+                    : !params
+                        ? 'too-few-labels'
+                        : improves
+                            ? undefined
+                            : 'no-improvement';
+                const t = params?.t ?? 1;
+                const id = (0, calibration_ts_1.fitId)(keyId, inputDigest, policy, { t, epsilon, eligible, reason: reason ?? null });
+                if (previous?.id === id) {
+                    result.skipped++;
+                    continue;
+                }
+                const applyUntil = fittedAt + policy.maxAgeMs;
+                const evidenceAt = Math.max(key.evidenceAt, (previous?.evidenceAt ?? 0) + 1);
+                const row = {
+                    id,
+                    kind: 'fit',
+                    key: keyId,
+                    rank: rankOf(keyId, evidenceAt, fittedAt, id),
+                    population: head.population,
+                    field: key.field,
+                    tenant: head.tenant,
+                    model: head.model,
+                    entry: head.entry,
+                    signature: head.signature,
+                    instructionsHash: head.instructionsHash,
+                    schemaHash: head.schemaHash,
+                    t,
+                    epsilon,
+                    cutoff: key.cutoff,
+                    evidenceAt,
+                    fittedAt,
+                    applyUntil,
+                    expiresAt: applyUntil + decisionStore_ts_1.DECISION_RETENTION_MS,
+                    eligible,
+                    reason,
+                    policy,
+                    policyDigest: digestOfPolicy,
+                    inputDigest,
+                    decisions: rows.length,
+                    labelled,
+                    report,
+                };
+                written.push(row);
+            }
+            const headRow = {
+                ...head,
+                owner: ownerOf(head.tenant),
+                lastFittedAt: fittedAt,
+                expiresAt: rows.length
+                    ? Math.max(newestExpiry, fittedAt + policy.maxAgeMs)
+                    : (found.expiresAt ?? fittedAt + policy.maxAgeMs),
+            };
+            await (0, transaction_ts_1.transaction)(freshContext(), async () => {
+                for (const row of written)
+                    await store.put(row);
+                await store.put(headRow);
+            });
+        }
+        catch (err) {
+            result.failed++;
+            logFault('calibration could not fit a population', err);
+            return 'failed';
+        }
+        for (const row of written)
+            writtenKeys.push(row.key);
+        result.written += written.length;
+        result.eligible += written.filter((row) => row.eligible).length;
+        return 'done';
+    }
+    /** A population this run could not fit goes to the back of the queue, so it cannot hold up the ones behind it. */
+    async function rotate(found) {
+        try {
+            const at = now();
+            await (0, transaction_ts_1.transaction)(freshContext(), () => store.put({ ...found.head, lastFittedAt: at, expiresAt: Math.max(found.expiresAt ?? 0, at + policy.maxAgeMs) }));
+        }
+        catch (err) {
+            logFault('calibration could not requeue a population', err);
+        }
+    }
+    function fail(what, err) {
+        result.status = 'failed';
+        result.error = `${what}: ${(0, engine_ts_1.safeErrorMessage)(err)}`;
+        log.error?.(`models: calibration run failed ${result.error}`);
+    }
+    function finish() {
+        if (result.failed > 0 && result.status === 'completed') {
+            result.status = 'failed';
+            result.error = `${result.failed} population(s) could not be processed`;
+        }
+        result.durationMs = now() - started;
+        try {
+            (0, write_ts_1.recordAction)(result.durationMs, 'model-calibrate', undefined, result.status);
+        }
+        catch { }
+        log.info?.(`models: calibration run ${result.status}: scanned ${result.scanned} decisions, read ${result.read} for fitting, ${result.discovered} populations, processed ${result.processed}, pending ${result.pending}, wrote ${result.written} (${result.eligible} eligible), skipped ${result.skipped}, failed ${result.failed}${result.stoppedBy ? `, stopped by ${result.stoppedBy}` : ''} in ${result.durationMs} ms`);
+        return result;
+    }
+}
+function processingOrder(populations) {
+    return [...populations].sort((a, b) => (a.lastFittedAt ?? -1) - (b.lastFittedAt ?? -1) || a.head.population.localeCompare(b.head.population));
+}
+async function listCalibrations(tenant, filter = {}) {
+    const store = getCalibrationsTable();
+    if (!store)
+        return [];
+    const now = Date.now();
+    const out = [];
+    const owned = await (0, transaction_ts_1.transaction)(freshContext(), async () => {
+        const rows = [];
+        for await (const row of store.search({
+            conditions: [
+                { attribute: 'owner', value: ownerOf(tenant) },
+                { attribute: 'kind', value: 'population' },
+                ...(filter.model === undefined ? [] : [{ attribute: 'model', value: filter.model }]),
+            ],
+        }))
+            rows.push(row);
+        return rows;
+    });
+    const keys = [];
+    for (const head of owned) {
+        if ((head.tenant ?? null) !== (tenant ?? null))
+            continue;
+        if (filter.model !== undefined && head.model !== filter.model)
+            continue;
+        try {
+            const fields = (0, decision_ts_1.isObjectSchema)(head.schema)
+                ? Object.keys(head.schema.properties)
+                : [undefined];
+            for (const field of fields)
+                keys.push((0, calibration_ts_1.calibrationKey)(head.population, field));
+        }
+        catch { }
+    }
+    const newest = [];
+    for (let i = 0; i < keys.length; i += LIST_BATCH) {
+        newest.push(...(await Promise.all(keys.slice(i, i + LIST_BATCH).map((key) => readNewest(key)))));
+    }
+    {
+        for (let i = 0; i < newest.length; i++) {
+            const row = newest[i];
+            if (!row)
+                continue;
+            out.push({
+                model: row.model,
+                field: row.field,
+                population: row.population,
+                signature: row.signature,
+                instructionsHash: row.instructionsHash,
+                schemaHash: row.schemaHash,
+                fittedAt: row.fittedAt,
+                applyUntil: row.applyUntil,
+                eligible: row.eligible,
+                applied: isUsable(row, now, keys[i]),
+                reason: row.reason,
+                decisions: row.decisions,
+                labelled: row.labelled,
+                t: row.t,
+                report: row.report,
+            });
+        }
+    }
+    return out.sort((a, b) => b.fittedAt - a.fittedAt);
+}
+//# sourceMappingURL=calibrationStore.js.map