@harperfast/harper 5.3.0 → 5.3.1

package/resources/models/Models.ts

@@ -3,9 +3,25 @@ import { contextStorage } from '../transaction.ts';
 import {
 	defineBackend,
 	getBackend,
+	getBackendSource,
 	ModelBackendNotFoundError,
 	registerBackend as registerBackendImpl,
+	SERVED_SOURCE,
+	type ServedSourceHook,
 } from './backendRegistry.ts';
+import {
+	type AppliedCalibration,
+	applyFits,
+	type CalibrationBudgets,
+	type CalibrationRunResult,
+	type CalibrationSummary,
+	isCalibrationEnabled,
+	listCalibrations,
+	runCalibration,
+} from './calibrationStore.ts';
+import { populationKey, populationRank } from './calibration.ts';
+import { safeErrorMessage } from '../scheduler/engine.ts';
+import harperLogger from '../../utility/logging/harper_logger.ts';
 import { getRouter, registerRouter as registerRouterImpl } from './routing.ts';
 import { getModelCallAnalyticsWriter, type ModelCallAnalyticsWriter, type ModelCallRecord } from './analyticsTable.ts';
 import { recordAction } from '../analytics/write.ts';
@@ -214,6 +230,7 @@ export class Models implements ModelsContract {
 		let firstError: unknown = undefined;
 		let hasError = false;
 		for (const backend of resolved.candidates) {
+			let completed: GenerateResult | undefined;
 			// Don't spend a backend call on an already-cancelled request — between
 			// candidates the caller may have aborted (and at entry it may already be).
 			signal?.throwIfAborted();
@@ -227,7 +244,7 @@ export class Models implements ModelsContract {
 				// Propagate usage onto the returned GenerateResult so callers (notably the
 				// `toolMode: 'auto'` loop's budget tracker) can read cumulative tokens without
 				// re-querying analytics. Pure pass-through — backend usage is the source of truth.
-				return result.usage ? { ...result.output, usage: result.usage } : result.output;
+				completed = result.usage ? { ...result.output, usage: result.usage } : result.output;
 			} catch (err) {
 				this.#recordFailure(backend, 'generate', opts.model, accounting, opts, attemptStart, err);
 				if (!hasError) {
@@ -236,6 +253,10 @@ export class Models implements ModelsContract {
 				}
 				if (signal?.aborted) throw err; // caller cancelled — stop, surface the abort
 			}
+			if (completed) {
+				reportServed(opts, backend);
+				return completed;
+			}
 		}
 		throw firstError;
 	}
@@ -389,6 +410,17 @@ export class Models implements ModelsContract {
 				continue;
 			}
 			const callId = this.#record(backend, 'decide', callOpts.model, accounting, undefined, result, attemptStart);
+			const calibration = calibrate(
+				call,
+				decision,
+				backend,
+				callOpts.model,
+				instructions,
+				accounting,
+				result.output,
+				identity
+			);
+			if (calibration.applied) decision = calibration.applied.decision;
 			if (!identity) return result.usage ? { ...decision, usage: result.usage } : decision;
 			const id = await this.#persistDecision(
 				callId,
@@ -398,7 +430,8 @@ export class Models implements ModelsContract {
 				accounting,
 				result.output,
 				identity,
-				decision
+				decision,
+				calibration
 			);
 			return result.usage ? { id, ...decision, usage: result.usage } : { id, ...decision };
 		}
@@ -437,6 +470,7 @@ export class Models implements ModelsContract {
 		let firstFailure: unknown = undefined;
 		let hasFailure = false;
 		for (const backend of resolved.candidates) {
+			let completed: (ChoiceScores & { usage?: TokenUsage }) | undefined;
 			signal?.throwIfAborted();
 			const attemptStart = performance.now();
 			try {
@@ -462,7 +496,7 @@ export class Models implements ModelsContract {
 						`Backend '${backend.name}' did not return one finite log-likelihood per choice (${choices.length})`
 					);
 				this.#record(backend, 'scoreChoices', opts.model, accounting, undefined, result, attemptStart);
-				return result.usage ? { logLikelihoods, usage: result.usage } : { logLikelihoods };
+				completed = result.usage ? { logLikelihoods, usage: result.usage } : { logLikelihoods };
 			} catch (err) {
 				// Only this attempt's own decline bills its tokens: the error may travel on as an abort
 				// reason or through the decision adapter, and those rows must not count it again.
@@ -486,6 +520,10 @@ export class Models implements ModelsContract {
 				}
 				if (signal?.aborted) throw err;
 			}
+			if (completed) {
+				reportServed(opts, backend);
+				return completed;
+			}
 		}
 		throw hasFailure ? firstFailure : firstError;
 	}
@@ -496,6 +534,20 @@ export class Models implements ModelsContract {
 			DecisionRecord<T> | undefined;
 	}
 
+	/**
+	 * Fit calibrations from recorded outcomes now, as the periodic job does, over every tenant's decisions:
+	 * a maintenance operation for trusted code. `budgets` may narrow this run; the fit policy always comes
+	 * from configuration. Resolves with the run's result; a data fault is reported there, never thrown.
+	 */
+	calibrate(budgets?: CalibrationBudgets): Promise<CalibrationRunResult> {
+		return runCalibration(budgets);
+	}
+
+	/** The newest calibration of each field of each population the caller's tenant owns. */
+	getCalibrations(filter?: { model?: string }): Promise<CalibrationSummary[]> {
+		return listCalibrations(resolveCallContext().accounting.tenantId, filter);
+	}
+
 	async recordOutcome<T = unknown>(id: string, outcome: OutcomeReport): Promise<DecisionRecord<T>> {
 		checkDecisionId(id);
 		return (await this.#decisionStore.recordOutcome(
@@ -513,7 +565,8 @@ export class Models implements ModelsContract {
 		accounting: AccountingContext,
 		output: DecisionOutput<unknown>,
 		identity: CallIdentity,
-		decision: Omit<Decision<T>, 'id' | 'usage'>
+		decision: Omit<Decision<T>, 'id' | 'usage'>,
+		calibration: Calibration<T>
 	): Promise<string> {
 		const id = newDecisionId();
 		const at = Date.now();
@@ -538,6 +591,16 @@ export class Models implements ModelsContract {
 			calibrated: decision.calibrated,
 		};
 		if (decision.noMatch !== undefined) row.noMatch = decision.noMatch;
+		if (calibration.entry !== undefined) row.entry = calibration.entry;
+		if (calibration.population !== undefined) {
+			row.population = calibration.population;
+			row.populationRank = populationRank(calibration.population, row.expiresAt, row.id);
+		}
+		if (calibration.applied) {
+			row.rawDistribution = calibration.applied.rawDistribution;
+			row.rawFields = calibration.applied.rawFields;
+			row.calibration = calibration.applied.snapshot;
+		}
 		try {
 			await this.#decisionStore.persist(row);
 		} catch (err) {
@@ -714,9 +777,69 @@ function toBackendOpts<TOpts extends { model?: string; signal?: AbortSignal }>(
 ): BackendOpts<TOpts> {
 	const backendOpts = { ...opts, signal, accounting };
 	delete backendOpts.model;
+	if (SERVED_SOURCE in backendOpts) delete (backendOpts as Record<symbol, unknown>)[SERVED_SOURCE];
 	return backendOpts;
 }
 
+const calibrationLog = harperLogger.forComponent('models').conditional;
+let lastCalibrationFaultLog = 0;
+
+function logCalibrationFault(what: string, err: unknown): void {
+	const now = Date.now();
+	if (now - lastCalibrationFaultLog < 60_000) return;
+	lastCalibrationFaultLog = now;
+	calibrationLog.warn?.(`models: ${what}: ${safeErrorMessage(err)}`);
+}
+
+function reportServed(opts: object, backend: ModelBackend): void {
+	const hook = (opts as Record<symbol, unknown>)[SERVED_SOURCE];
+	if (typeof hook !== 'function') return;
+	try {
+		(hook as ServedSourceHook)(getBackendSource(backend));
+	} catch (err) {
+		logCalibrationFault('a served-source hook failed', err);
+	}
+}
+
+interface Calibration<T> {
+	entry?: string;
+	population?: string;
+	applied?: AppliedCalibration<T>;
+}
+
+const NOT_CALIBRATING: Calibration<never> = {};
+
+function calibrate<T>(
+	schema: DecisionSchema,
+	decision: Omit<Decision<T>, 'id' | 'usage'>,
+	backend: ModelBackend,
+	model: string | undefined,
+	instructions: string | undefined,
+	accounting: AccountingContext,
+	output: DecisionOutput<unknown>,
+	identity: CallIdentity | undefined
+): Calibration<T> {
+	if (!isCalibrationEnabled()) return NOT_CALIBRATING as Calibration<T>;
+	try {
+		const signature = typeof output.signature === 'string' && output.signature ? output.signature : undefined;
+		const entry = getBackendSource(backend) ?? `registered:${backend.name}`;
+		if (!signature) return { entry };
+		const population = populationKey({
+			tenant: accounting.tenantId,
+			model: model ?? 'default',
+			entry,
+			signature,
+			instructionsHash: instructions ? hashText(instructions) : undefined,
+			schemaHash: identity?.hash ?? hashSchema(schema),
+		});
+		if (decision.calibrated) return { entry, population };
+		return { entry, population, applied: applyFits(schema, decision, population) };
+	} catch (err) {
+		logCalibrationFault('calibration was not applied', err);
+		return NOT_CALIBRATING as Calibration<T>;
+	}
+}
+
 function resolveCandidates(kind: ModelKind, model: string | undefined, requires: Capability[]): Resolution {
 	const logicalName = model ?? 'default';
 	const candidates = getRouter().route({ kind, logicalName, requires: [...requires] });

package/resources/models/backendRegistry.ts

@@ -51,6 +51,23 @@ interface CaptureSlot {
 	active: boolean;
 }
 
+const sources = new WeakMap<ModelBackend, string>();
+
+/**
+ * An internal option key: the decision adapter passes a hook under it on inner calls, and the facade
+ * reports each successful attempt's source through it. A symbol, so it is never part of the public options.
+ */
+export const SERVED_SOURCE = Symbol('models.servedSource');
+export type ServedSourceHook = (source: string | undefined) => void;
+
+export function setBackendSource(backend: ModelBackend, fingerprint: string): void {
+	sources.set(backend, fingerprint);
+}
+
+export function getBackendSource(backend: ModelBackend): string | undefined {
+	return sources.get(backend);
+}
+
 // Async-context scoped: a module-global slot would divert unrelated registrations and collide
 // concurrent constructions.
 const captureScope = new AsyncLocalStorage<CaptureSlot>();

package/resources/models/bootstrap.ts

@@ -34,7 +34,8 @@ import { registerOllamaBackend, type OllamaBackendConfig } from '../../component
 import { registerOpenAIBackend, type OpenAIBackendConfig } from '../../components/openai/index.ts';
 import { registerAnthropicBackend, type AnthropicBackendConfig } from '../../components/anthropic/index.ts';
 import { registerBedrockBackend, type BedrockBackendConfig } from '../../components/bedrock/index.ts';
-import { setModelsConfigHash } from './decisionStore.ts';
+import { setModelsConfigHash, sourceFingerprint } from './decisionStore.ts';
+import { type CalibrationConfig, configureCalibration } from './calibrationStore.ts';
 import { isAbsolute, resolve as resolvePath } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { createRequire } from 'node:module';
@@ -45,6 +46,7 @@ import {
 	getBackend,
 	removeIfCurrent,
 	replaceIfCurrent,
+	setBackendSource,
 	type CapturedInstall,
 } from './backendRegistry.ts';
 import { getSharedRootConfigWatcher, RootConfigWatcher } from '../../config/RootConfigWatcher.ts';
@@ -86,6 +88,7 @@ interface ModelsConfig {
 	embedding?: Record<string, ModelEntry>;
 	generative?: Record<string, ModelEntry>;
 	decision?: Record<string, ModelEntry>;
+	calibration?: CalibrationConfig | null;
 }
 
 interface RootConfig {
@@ -288,6 +291,10 @@ function publishEntry(
 ): void {
 	const { kind, logicalName, entry, entryJson } = desiredEntry;
 	const key = slotKey(kind, logicalName);
+	const sourceConfig = desiredEntry.configJson ? JSON.parse(desiredEntry.configJson) : entry;
+	setBackendSource(backend, sourceFingerprint(kind, logicalName, sourceConfig));
+	for (const extra of extras)
+		setBackendSource(extra.backend, sourceFingerprint(extra.kind, extra.logicalName, sourceConfig));
 	// Boot overwrites occupants (the documented contract); a reload replaces only what this
 	// projection installed — or a helper the projection installed under this name, which a config
 	// entry outranks — so genuine application overrides survive.
@@ -503,6 +510,7 @@ async function applyModels(block: ModelsConfig | null | undefined, isBoot: boole
 		}
 	}
 	setModelsConfigHash(installedConfiguration());
+	configureCalibration(block?.calibration);
 }
 
 function installedConfiguration(): Record<string, unknown> | undefined {

package/resources/models/calibration.ts

@@ -0,0 +1,298 @@
+import { createHash } from 'node:crypto';
+import { setImmediate as yieldToEventLoop } from 'node:timers/promises';
+import { canonicalJson } from './decision.ts';
+import type { DecisionOutcome } from './types.ts';
+
+export const SELECTIVE_THRESHOLDS = [0.5, 0.6, 0.7, 0.8, 0.9, 0.95] as const;
+const ECE_BINS = 10;
+const SCORED_EPSILON = 1e-9;
+const DEFAULT_VOTE_SAMPLES = 5;
+const WILSON_Z = 1.96;
+const GOLDEN_ITERATIONS = 60;
+export const MIN_TEMPERATURE = 0.05;
+export const MAX_TEMPERATURE = 20;
+/** The `truth` of an example whose recorded truth is `noMatch`: counted by operational risk only. */
+export const NO_MATCH_TRUTH = -1;
+
+export interface CalibrationParams {
+	t: number;
+	epsilon: number;
+}
+
+export interface Example {
+	probabilities: number[];
+	truth: number;
+	/** Index of the value the decision returned; among tied scores it need not be the first. */
+	chosen?: number;
+}
+
+export interface SelectivePoint {
+	threshold: number;
+	count: number;
+	coverage: number;
+	risk: number | null;
+	riskUpper: number | null;
+}
+
+export interface ReliabilityBin {
+	count: number;
+	confidence: number | null;
+	accuracy: number | null;
+}
+
+export interface Reliability {
+	ece: number;
+	nll: number;
+	bins: ReliabilityBin[];
+	/** Among examples whose truth is an allowed value: the population a fit is trained on. */
+	conditional: SelectivePoint[];
+	/** Among all examples, where a `noMatch` truth above the threshold counts as an error. */
+	operational: SelectivePoint[];
+}
+
+/** What produced a decision's scores and whose outcomes it answers to; any change is a new population. */
+export interface Population {
+	tenant?: string;
+	app?: string;
+	model: string;
+	entry: string;
+	signature: string;
+	instructionsHash?: string;
+	schemaHash: string;
+}
+
+export interface FitPolicy {
+	minReport: number;
+	minTrain: number;
+	minHeldOut: number;
+	heldOutShare: number;
+	eceMargin: number;
+	maxAgeMs: number;
+}
+
+function digest(value: unknown): string {
+	return createHash('sha256').update(canonicalJson(value)).digest('hex');
+}
+
+export function populationKey(population: Population): string {
+	return digest({
+		tenant: population.tenant ?? null,
+		app: population.app ?? null,
+		model: population.model,
+		entry: population.entry,
+		signature: population.signature,
+		instructionsHash: population.instructionsHash ?? null,
+		schemaHash: population.schemaHash,
+	});
+}
+
+/** Orders a population's decisions newest first under one indexed prefix, so a run reads only the newest it needs. */
+export function populationRank(population: string, expiresAt: number, id: string): string {
+	return `${population}|${String(Math.max(0, Math.floor(expiresAt))).padStart(16, '0')}|${id}`;
+}
+
+export function calibrationKey(population: string, field: string | undefined): string {
+	return digest({ population, field: field ?? null });
+}
+
+/** Identical inputs, policy and result give an identical id, so a repeated run writes nothing new. */
+export function fitId(key: string, inputDigest: string, policy: FitPolicy, result: object): string {
+	return digest({ key, inputDigest, policy, result });
+}
+
+export function policyDigest(policy: FitPolicy): string {
+	return digest(policy);
+}
+
+export function inputDigester(): { add(part: unknown): void; digest(): string } {
+	const hash = createHash('sha256');
+	return {
+		add(part: unknown) {
+			hash.update(canonicalJson(part));
+			hash.update('\n');
+		},
+		digest: () => hash.digest('hex'),
+	};
+}
+
+/**
+ * Smoothing mass for a distribution: half a vote per value for a voted distribution, so a value no
+ * sample chose keeps a finite log; a numerical guard otherwise.
+ */
+export function smoothingEpsilon(signature: string | undefined): number {
+	if (!signature || !/(^|;)mode=vote(;|$)/.test(signature)) return SCORED_EPSILON;
+	const samples = Number(/(?:^|;)samples=(\d+)(?:;|$)/.exec(signature)?.[1]);
+	return 0.5 / (Number.isInteger(samples) && samples > 0 ? samples : DEFAULT_VOTE_SAMPLES);
+}
+
+/** A replicated row is not trusted to be well formed. */
+export function isValidParams(params: unknown): params is CalibrationParams {
+	if (!params || typeof params !== 'object') return false;
+	const { t, epsilon } = params as { t?: unknown; epsilon?: unknown };
+	return (
+		typeof t === 'number' &&
+		Number.isFinite(t) &&
+		t >= MIN_TEMPERATURE &&
+		t <= MAX_TEMPERATURE &&
+		typeof epsilon === 'number' &&
+		Number.isFinite(epsilon) &&
+		epsilon > 0 &&
+		epsilon < 1
+	);
+}
+
+/** Strictly increasing in each value's raw score, so ties stay tied and the winner never moves. */
+export function applyCalibration(probabilities: number[], params: CalibrationParams): number[] {
+	const k = probabilities.length;
+	const inverse = 1 / params.t;
+	const logs = new Array<number>(k);
+	let max = -Infinity;
+	for (let i = 0; i < k; i++) {
+		logs[i] = Math.log((Math.max(0, probabilities[i]) + params.epsilon) / (1 + k * params.epsilon)) * inverse;
+		if (logs[i] > max) max = logs[i];
+	}
+	let total = 0;
+	for (let i = 0; i < k; i++) {
+		logs[i] = Math.exp(logs[i] - max);
+		total += logs[i];
+	}
+	for (let i = 0; i < k; i++) logs[i] /= total;
+	return logs;
+}
+
+function meanNll(examples: Example[], params: CalibrationParams): number {
+	let total = 0;
+	for (const example of examples) {
+		total -= Math.log(Math.max(applyCalibration(example.probabilities, params)[example.truth], 1e-15));
+	}
+	return total / examples.length;
+}
+
+/**
+ * The temperature minimizing negative log-likelihood on `train` (examples with an allowed-value truth),
+ * by golden-section search over `log t`. Yields between iterations; undefined when `deadline` passes first.
+ */
+export async function fitCalibration(
+	train: Example[],
+	epsilon: number,
+	deadline: number,
+	now: () => number = Date.now
+): Promise<CalibrationParams | undefined> {
+	const f = (x: number) => meanNll(train, { t: Math.exp(x), epsilon });
+	const ratio = (Math.sqrt(5) - 1) / 2;
+	let a = Math.log(MIN_TEMPERATURE);
+	let b = Math.log(MAX_TEMPERATURE);
+	let c = b - ratio * (b - a);
+	let d = a + ratio * (b - a);
+	let fc = f(c);
+	let fd = f(d);
+	for (let i = 0; i < GOLDEN_ITERATIONS; i++) {
+		await yieldToEventLoop();
+		if (now() >= deadline) return undefined;
+		if (fc < fd) {
+			b = d;
+			d = c;
+			fd = fc;
+			c = b - ratio * (b - a);
+			fc = f(c);
+		} else {
+			a = c;
+			c = d;
+			fc = fd;
+			d = a + ratio * (b - a);
+			fd = f(d);
+		}
+	}
+	return { t: Math.exp((a + b) / 2), epsilon };
+}
+
+function argmax(p: number[]): number {
+	let top = 0;
+	for (let i = 1; i < p.length; i++) if (p[i] > p[top]) top = i;
+	return top;
+}
+
+export function wilsonUpper(errors: number, count: number): number | null {
+	if (count === 0) return null;
+	const p = errors / count;
+	const z2 = WILSON_Z * WILSON_Z;
+	const centre = p + z2 / (2 * count);
+	const margin = WILSON_Z * Math.sqrt((p * (1 - p)) / count + z2 / (4 * count * count));
+	return Math.min(1, (centre + margin) / (1 + z2 / count));
+}
+
+function selective(tops: Array<{ confidence: number; correct: boolean }>, total: number): SelectivePoint[] {
+	return SELECTIVE_THRESHOLDS.map((threshold) => {
+		let count = 0;
+		let errors = 0;
+		for (const top of tops) {
+			if (top.confidence < threshold) continue;
+			count++;
+			if (!top.correct) errors++;
+		}
+		return {
+			threshold,
+			count,
+			coverage: total ? count / total : 0,
+			risk: count ? errors / count : null,
+			riskUpper: wilsonUpper(errors, count),
+		};
+	});
+}
+
+export function reliability(examples: Example[], transform: (p: number[]) => number[]): Reliability {
+	const sums = Array.from({ length: ECE_BINS }, () => ({ count: 0, confidence: 0, correct: 0 }));
+	const inSet: Array<{ confidence: number; correct: boolean }> = [];
+	const all: Array<{ confidence: number; correct: boolean }> = [];
+	let nll = 0;
+	for (const example of examples) {
+		const p = transform(example.probabilities);
+		const top = example.chosen ?? argmax(p);
+		const confidence = p[top];
+		if (example.truth === NO_MATCH_TRUTH) {
+			all.push({ confidence, correct: false });
+			continue;
+		}
+		const correct = top === example.truth;
+		const bin = sums[Math.min(ECE_BINS - 1, Math.floor(confidence * ECE_BINS))];
+		bin.count++;
+		bin.confidence += confidence;
+		bin.correct += correct ? 1 : 0;
+		inSet.push({ confidence, correct });
+		all.push({ confidence, correct });
+		nll -= Math.log(Math.max(p[example.truth], 1e-15));
+	}
+	const n = inSet.length;
+	let ece = 0;
+	for (const bin of sums) if (bin.count && n) ece += Math.abs(bin.confidence - bin.correct) / n;
+	return {
+		ece,
+		nll: n ? nll / n : 0,
+		bins: sums.map(({ count, confidence, correct }) => ({
+			count,
+			confidence: count ? confidence / count : null,
+			accuracy: count ? correct / count : null,
+		})),
+		conditional: selective(inSet, n),
+		operational: selective(all, all.length),
+	};
+}
+
+/** Oldest first; the newest `heldOutShare` is held out, so a refit is judged on decisions newer than any it trained on. */
+export function splitByTime<E>(ordered: E[], heldOutShare: number): { train: E[]; heldOut: E[] } {
+	const heldOutCount = Math.floor(ordered.length * heldOutShare);
+	return {
+		train: ordered.slice(0, ordered.length - heldOutCount),
+		heldOut: ordered.slice(ordered.length - heldOutCount),
+	};
+}
+
+export function toVector(
+	distribution: DecisionOutcome[] | undefined,
+	values: readonly unknown[]
+): number[] | undefined {
+	if (!Array.isArray(distribution)) return undefined;
+	const byValue = new Map(distribution.map((entry) => [canonicalJson(entry.value), entry.probability]));
+	const vector = values.map((value) => byValue.get(canonicalJson(value)));
+	return vector.every((p) => typeof p === 'number' && Number.isFinite(p)) ? (vector as number[]) : undefined;
+}