@harperfast/harper 5.3.0 → 5.3.1

package/resources/models/decisionStore.ts

@@ -45,6 +45,16 @@ export interface DecisionRow {
 	fields?: DecisionRecord['fields'];
 	noMatch?: number;
 	calibrated: boolean;
+	/** The backend's scores, kept when a fitted calibration replaced them, so a refit never reads its own output. */
+	rawDistribution?: DecisionRecord['distribution'];
+	rawFields?: DecisionRecord['fields'];
+	/** The fit applied to each field (`field` absent for a leaf schema), with its parameters as applied. */
+	calibration?: Array<{ field?: string; fitId: string; t: number; epsilon: number }>;
+	/** The decision entry's source fingerprint, or `registered:<name>` for a backend registered from code. */
+	entry?: string;
+	/** The calibration population, for a decision whose score source is identified. */
+	population?: string;
+	populationRank?: string;
 }
 
 type Fact = 'truth' | 'action';
@@ -84,6 +94,12 @@ export const DECISION_ATTRIBUTES = [
 	{ name: 'fields' },
 	{ name: 'noMatch', type: 'number' },
 	{ name: 'calibrated', type: 'boolean' },
+	{ name: 'rawDistribution' },
+	{ name: 'rawFields' },
+	{ name: 'calibration' },
+	{ name: 'entry', type: 'string' },
+	{ name: 'population', type: 'string' },
+	{ name: 'populationRank', type: 'string', indexed: true },
 ];
 
 export const OUTCOME_ATTRIBUTES = [
@@ -279,6 +295,10 @@ function visibleTo(row: DecisionRow, tenant: string | undefined): boolean {
 	return tenant == null || row.tenant == null || row.tenant === tenant;
 }
 
+export function truthKey(id: string, field?: string): string {
+	return factKey(id, 'truth', field);
+}
+
 function factKey(id: string, fact: Fact, field?: string): string {
 	return field == null ? `${id}/${fact}` : `${id}/${fact}/${field}`;
 }
@@ -394,6 +414,43 @@ export function getModelsConfigHash(): string | undefined {
 	return modelsConfigHash;
 }
 
+/** A configured entry's identity without its credentials: the same fingerprint for the same visible configuration. */
+export function sourceFingerprint(kind: string, logicalName: string | undefined, config: unknown): string {
+	return createHash('sha256')
+		.update(
+			canonicalJson({
+				kind,
+				logicalName: logicalName ?? null,
+				config: withoutNamedCredentials(withoutFallback(config)),
+			})
+		)
+		.digest('hex');
+}
+
+/**
+ * Only fields known to hold credentials. A setting that merely ends in `key` or `token` can change what a source
+ * scores, so it stays in the fingerprint; rotating an unrecognized secret therefore starts calibration over.
+ */
+const NAMED_CREDENTIAL =
+	/^(apiKey|apiSecret|accessKeyId|secretAccessKey|sessionToken|authorization|password|token|bearerToken|credentials?)$/i;
+
+function withoutNamedCredentials(value: unknown): unknown {
+	if (Array.isArray(value)) return value.map(withoutNamedCredentials);
+	if (!value || typeof value !== 'object') return value;
+	const kept: Record<string, unknown> = {};
+	for (const [key, entry] of Object.entries(value as Record<string, unknown>)) {
+		if (!NAMED_CREDENTIAL.test(key)) kept[key] = withoutNamedCredentials(entry);
+	}
+	return kept;
+}
+
+/** Which backend serves after this one is routing, not what produces this one's scores. */
+function withoutFallback(config: unknown): unknown {
+	if (!config || typeof config !== 'object' || Array.isArray(config)) return config;
+	const { fallback: _fallback, ...rest } = config as Record<string, unknown>;
+	return rest;
+}
+
 const CREDENTIAL_KEY = /^(apiKey|accessKeyId|authorization|.*secret.*|.*password.*|.*credential.*|.*key|.*token)$/i;
 
 function withoutCredentials(value: unknown): unknown {

package/resources/models/generativeDecision.ts

@@ -11,7 +11,8 @@
  */
 import { ServerError } from '../../utility/errors/hdbError.ts';
 import { composeSignal } from './backendHelpers.ts';
-import { setDecision } from './backendRegistry.ts';
+import { SERVED_SOURCE, type ServedSourceHook, setDecision } from './backendRegistry.ts';
+import { isCalibrationEnabled } from './calibrationStore.ts';
 import {
 	allowedValues,
 	isObjectSchema,
@@ -143,7 +144,8 @@ export function createGenerativeDecisionBackend(
 		state: DecideInput,
 		schema: DecisionSchema,
 		instructions: string | undefined,
-		signal: AbortSignal | undefined
+		signal: AbortSignal | undefined,
+		served: object
 	): Promise<DecisionOutput<unknown>> => {
 		const input = buildInput(
 			state,
@@ -161,6 +163,7 @@ export function createGenerativeDecisionBackend(
 					temperature,
 					signal: sampleSignal,
 					requires,
+					...served,
 				});
 				return parseSample(schema, result.content, logicalName);
 			}),
@@ -176,17 +179,18 @@ export function createGenerativeDecisionBackend(
 		field: string | undefined,
 		leaf: DecisionLeaf,
 		instructions: string | undefined,
-		signal: AbortSignal
+		signal: AbortSignal,
+		served: object
 	): Promise<LeafScore> => {
 		const values = allowedValues(leaf);
 		const choices = values.map((value) => String(value));
 		if (leaf.noMatch !== true) {
 			const input = buildInput(state, schema, instructions, SCORE_SYSTEM_PROMPT, field);
-			const scored = await score(input, choices, { model: logicalName, signal });
+			const scored = await score(input, choices, { model: logicalName, signal, ...served });
 			return { distribution: softmax(values, scored?.logLikelihoods, logicalName) };
 		}
 		const input = buildInput(state, schema, instructions, SCORE_NO_MATCH_SYSTEM_PROMPT, field);
-		const scored = await score(input, [...choices, noMatchChoice(choices)], { model: logicalName, signal });
+		const scored = await score(input, [...choices, noMatchChoice(choices)], { model: logicalName, signal, ...served });
 		const all = softmax([...values, NO_MATCH], scored?.logLikelihoods, logicalName);
 		// Normalized over the allowed values' own scores, not by dividing out the none share, which can underflow to zero.
 		const distribution = softmax(values, (scored!.logLikelihoods as number[]).slice(0, values.length), logicalName);
@@ -199,11 +203,12 @@ export function createGenerativeDecisionBackend(
 		state: DecideInput,
 		schema: DecisionSchema,
 		instructions: string | undefined,
-		signal: AbortSignal | undefined
+		signal: AbortSignal | undefined,
+		served: object
 	): Promise<DecisionOutput<unknown>> => {
 		if (!isObjectSchema(schema)) {
 			const [scored] = await runPool(
-				[(leafSignal: AbortSignal) => scoreLeaf(state, schema, undefined, schema, instructions, leafSignal)],
+				[(leafSignal: AbortSignal) => scoreLeaf(state, schema, undefined, schema, instructions, leafSignal, served)],
 				concurrency,
 				signal
 			);
@@ -214,7 +219,7 @@ export function createGenerativeDecisionBackend(
 			entries.map(
 				([name, leaf]) =>
 					(leafSignal: AbortSignal) =>
-						scoreLeaf(state, schema, name, leaf, instructions, leafSignal)
+						scoreLeaf(state, schema, name, leaf, instructions, leafSignal, served)
 			),
 			concurrency,
 			signal
@@ -225,6 +230,12 @@ export function createGenerativeDecisionBackend(
 	const signatureFor = (mode: ScoringMode) =>
 		`generative=${logicalName};mode=${mode};samples=${samples};temperature=${temperature ?? 'default'}`;
 
+	const signed = (mode: ScoringMode, collector: SourceCollector | undefined): string | undefined => {
+		if (!collector) return signatureFor(mode);
+		if (collector.mixed || collector.source === null) return undefined;
+		return `${signatureFor(mode)};source=${collector.source}`;
+	};
+
 	const requires = config.requireStructuredOutput === false ? undefined : (STRUCTURED_OUTPUT as Capability[]);
 	return {
 		name: 'generative',
@@ -238,23 +249,50 @@ export function createGenerativeDecisionBackend(
 			const mode: ScoringMode =
 				scoring === 'auto' ? (canScore(logicalName, largestLeaf(schema)) ? 'score' : 'vote') : scoring;
 			if (mode === 'score') {
+				const collector = isCalibrationEnabled() ? sourceCollector() : undefined;
 				try {
-					return {
-						status: 'completed',
-						output: { ...(await scoreAll(state, schema, opts.instructions, signal)), signature: signatureFor('score') },
-					};
+					const output = await scoreAll(state, schema, opts.instructions, signal, servedOpts(collector));
+					return { status: 'completed', output: withSignature(output, signed('score', collector)) };
 				} catch (err) {
 					if (scoring !== 'auto' || !isScoringUnsupported(err)) throw err;
 				}
 			}
-			return {
-				status: 'completed',
-				output: { ...(await vote(state, schema, opts.instructions, signal)), signature: signatureFor('vote') },
-			};
+			const collector = isCalibrationEnabled() ? sourceCollector() : undefined;
+			const output = await vote(state, schema, opts.instructions, signal, servedOpts(collector));
+			return { status: 'completed', output: withSignature(output, signed('vote', collector)) };
 		},
 	};
 }
 
+const NOT_COLLECTING = Object.freeze({});
+
+interface SourceCollector {
+	source: string | null;
+	mixed: boolean;
+	hook: ServedSourceHook;
+}
+
+function sourceCollector(): SourceCollector {
+	const collector: SourceCollector = {
+		source: null,
+		mixed: false,
+		hook: (source) => {
+			if (source === undefined) collector.mixed = true;
+			else if (collector.source === null) collector.source = source;
+			else if (collector.source !== source) collector.mixed = true;
+		},
+	};
+	return collector;
+}
+
+function servedOpts(collector: SourceCollector | undefined): object {
+	return collector ? { [SERVED_SOURCE]: collector.hook } : NOT_COLLECTING;
+}
+
+function withSignature(output: DecisionOutput<unknown>, signature: string | undefined): DecisionOutput<unknown> {
+	return signature === undefined ? output : { ...output, signature };
+}
+
 // A candidate that advertises scoring without implementing it counts as able, so the facade reports
 // that contract failure instead of the decision quietly voting.
 function routesToScorer(logicalName: string, largestLeafSize: number): boolean {

package/resources/models/types.ts

@@ -7,6 +7,8 @@
  * Reference: planning artifact `tmp/harper-510-phase-1-detail.md` (canonical shapes).
  */
 
+import type { CalibrationBudgets, CalibrationRunResult, CalibrationSummary } from './calibrationStore.ts';
+
 export interface Models {
 	embed(input: string | string[], opts?: EmbedOpts): Promise<Float32Array[]>;
 	generate(input: GenerateInput, opts?: GenerateOpts): Promise<GenerateResult>;
@@ -22,6 +24,10 @@ export interface Models {
 	getDecision<T = unknown>(id: string): Promise<DecisionRecord<T> | undefined>;
 	/** Record what actually happened for a decision: its truth, the action taken, or both. See #2840. */
 	recordOutcome<T = unknown>(id: string, outcome: OutcomeReport): Promise<DecisionRecord<T>>;
+	/** Fit calibrations from recorded outcomes now, as the periodic job does. See #2841. */
+	calibrate(budgets?: CalibrationBudgets): Promise<CalibrationRunResult>;
+	/** The newest calibration of each field of each population the caller's tenant owns. See #2841. */
+	getCalibrations(filter?: { model?: string }): Promise<CalibrationSummary[]>;
 	/** Register a custom backend under a logical id, selectable via `opts.model`. See #1325. */
 	registerBackend(kind: ModelKind, id: string, backend: ModelBackend): void;
 	/** Build a `ModelBackend` from a spec; pair with `registerBackend`. See #1325. */