okengine 0.22.0 → 0.23.1

package/src/elements/ai/decisions/certificate.ts

@@ -0,0 +1,377 @@
+/**
+ * Decision certificates. The lockfile is the only autonomy grant.
+ * The runtime can force review or abstain; it never raises a threshold.
+ */
+
+import { createHash } from "node:crypto";
+
+/** Temperature scaling for choice and score. */
+export interface TemperatureCalibrator {
+  readonly kind: "temperature";
+  readonly t: number;
+}
+
+/** Platt scaling for a boolean `noul` probability. */
+export interface PlattCalibrator {
+  readonly kind: "platt";
+  readonly a: number;
+  readonly b: number;
+}
+
+/** Beta scaling for a boolean `noul` probability. `c` is the scale. */
+export interface BetaCalibrator {
+  readonly kind: "beta";
+  readonly a: number;
+  readonly b: number;
+  readonly c?: number;
+}
+
+/** Calibrator stored on one certificate slice. */
+export type DecisionCalibrator = TemperatureCalibrator | PlattCalibrator | BetaCalibrator;
+
+/** One locale slice. Absent `locale` is the empty-string slice. */
+export interface DecisionCertSlice {
+  readonly hash: string;
+  readonly calibrator: DecisionCalibrator;
+  readonly threshold: number;
+  readonly metrics?: Readonly<Record<string, number>>;
+}
+
+/** One decision inside `oke-decisions.lock.json`. */
+export interface DecisionLockEntry {
+  readonly model: string;
+  /** Epoch ms the certificate was written. Drift counts labels after this. */
+  readonly certifiedAt?: number;
+  readonly questions: Readonly<Record<string, Readonly<Record<string, DecisionCertSlice>>>>;
+}
+
+/** App lockfile. Lives next to that app's OKE config. */
+export interface DecisionLockfile {
+  readonly decisions: Readonly<Record<string, DecisionLockEntry>>;
+}
+
+/** Why a question is not auto. */
+export type DecisionUncertainty =
+  | "low-confidence"
+  | "missing-lock"
+  | "stale-hash"
+  | "version-mismatch"
+  | "uncertified-locale"
+  | "drift"
+  | "outage";
+
+/** Label row written when a review resolves. */
+export interface DecisionLabel {
+  readonly decision: string;
+  readonly question: string;
+  readonly value: unknown;
+  readonly propensity: number;
+  readonly reviewer: string;
+  readonly locale?: string;
+  readonly tenant?: string;
+  readonly model?: string;
+  /** Calibrated confidence used by Learn-then-Test. */
+  readonly score?: number;
+  /** Raw provider distribution the candidate fits. */
+  readonly raw?: unknown;
+  /** 1 when the label disagrees with the model, else 0. */
+  readonly loss?: number;
+  /** Epoch ms the label was written. */
+  readonly at?: number;
+  /** Calling input, with secret and redacted fields already masked. */
+  readonly input?: unknown;
+  /** Review id this label belongs to. Rows from one review share it. */
+  readonly reviewId?: string;
+}
+
+/** App-wide candidate. A full lock entry, not a count. */
+export interface DecisionCandidate {
+  readonly model: string;
+  readonly questions: DecisionLockEntry["questions"];
+}
+
+/** Lockfile name at the app project root. */
+export const DECISION_LOCK_FILENAME = "oke-decisions.lock.json";
+
+const labels: DecisionLabel[] = [];
+let lockfile: DecisionLockfile | undefined;
+const drifted = new Map<string, number>();
+
+/**
+ * Replace the in-memory lock. `undefined` is a missing lockfile.
+ *
+ * @param next - Lockfile, or none
+ */
+export function setDecisionLock(next: DecisionLockfile | undefined): void {
+  lockfile = next;
+}
+
+/**
+ * Current lockfile, if one was loaded or promoted.
+ */
+export function getDecisionLock(): DecisionLockfile | undefined {
+  return lockfile;
+}
+
+/**
+ * Set or clear drift for one decision.
+ *
+ * @param name - Decision name
+ * @param next - Suspend that decision when true
+ */
+export function setDecisionDrift(name: string, next: boolean): void {
+  if (next) drifted.set(name, drifted.get(name) ?? 0);
+  else drifted.delete(name);
+}
+
+/**
+ * Whether drift has suspended this decision.
+ *
+ * @param name - Decision name
+ */
+export function decisionDriftSuspended(name: string): boolean {
+  return drifted.has(name);
+}
+
+/**
+ * Decision names currently suspended.
+ */
+export function decisionDriftNames(): readonly string[] {
+  return [...drifted.keys()];
+}
+
+/**
+ * Labels collected by review resolution, including audit reviews.
+ */
+export function decisionLabels(): readonly DecisionLabel[] {
+  return labels;
+}
+
+/**
+ * Drop lock, labels, and drift. Tests only.
+ */
+export function resetDecisionCertificates(): void {
+  labels.length = 0;
+  lockfile = undefined;
+  drifted.clear();
+}
+
+/**
+ * Stable hash of a question's instructions and options.
+ *
+ * @param question - Author question
+ */
+export function questionHash(question: {
+  readonly instructions: string;
+  readonly options?: Readonly<Record<string, string | null>>;
+  readonly levels?: readonly string[];
+}): string {
+  const body = JSON.stringify({
+    instructions: question.instructions,
+    options: question.options ?? null,
+    levels: question.levels ?? null,
+  });
+  return createHash("sha256").update(body).digest("hex");
+}
+
+/**
+ * Temperature-scale a distribution. `t = 1` leaves it unchanged.
+ *
+ * @param probs - Probabilities that sum to about 1
+ * @param t - Temperature
+ */
+export function applyTemperature(probs: readonly number[], t: number): number[] {
+  const temp = t > 0 ? t : 1;
+  const scaled = probs.map((p) => Math.pow(Math.max(p, 1e-12), 1 / temp));
+  const z = scaled.reduce((sum, p) => sum + p, 0);
+  return scaled.map((p) => p / z);
+}
+
+/**
+ * Calibrated P(true). Confidence is `max(p, 1 - p)` at the call site.
+ *
+ * @param noul - Provider probability of the true class
+ * @param calibrator - Platt or beta
+ */
+export function calibrateBoolean(
+  noul: number,
+  calibrator: PlattCalibrator | BetaCalibrator,
+): number {
+  const p = Math.min(1, Math.max(0, noul));
+  if (calibrator.kind === "platt") {
+    const logit = Math.log(Math.max(p, 1e-12) / Math.max(1 - p, 1e-12));
+    const z = calibrator.a * logit + calibrator.b;
+    return 1 / (1 + Math.exp(-z));
+  }
+  const a = calibrator.a;
+  const b = calibrator.b;
+  const c = calibrator.c ?? 0;
+  const ratio = Math.pow(Math.max(1 - p, 1e-12), b) / Math.pow(Math.max(p, 1e-12), a);
+  return 1 / (1 + Math.exp(-c) * ratio);
+}
+
+/** Fixed Learn-then-Test grid: 0.50 through 0.99, step 0.01. */
+export const LEARN_THEN_TEST_GRID: readonly number[] = Array.from(
+  { length: 50 },
+  (_, i) => (50 + i) / 100,
+);
+
+/**
+ * One-sided exact binomial cdf, `P(X ≤ k)` for `X ~ Binomial(n, p)`.
+ * `k < 0` is 0. The sum is in log space.
+ *
+ * @param k - Observed errors
+ * @param n - Accepted labels
+ * @param p - Boundary error rate
+ */
+export function binomialCdf(k: number, n: number, p: number): number {
+  if (k < 0) return 0;
+  if (n <= 0) return 1;
+  if (p <= 0) return 1;
+  if (p >= 1) return k >= n ? 1 : 0;
+  if (k >= n) return 1;
+  const last = Math.floor(k);
+  let logTerm = n * Math.log(1 - p);
+  let maxLog = logTerm;
+  const logs = [logTerm];
+  for (let i = 0; i < last; i++) {
+    logTerm += Math.log(n - i) - Math.log(i + 1) + Math.log(p) - Math.log(1 - p);
+    logs.push(logTerm);
+    if (logTerm > maxLog) maxLog = logTerm;
+  }
+  let sum = 0;
+  for (const logP of logs) sum += Math.exp(logP - maxLog);
+  const total = Math.exp(maxLog) * sum;
+  return Math.min(1, Number.isFinite(total) ? total : 0);
+}
+
+/**
+ * Learn-then-Test on a fixed threshold grid. A threshold passes only when
+ * the error count is low under Binomial(n, maxError): `binomialCdf(errors, n, maxError) ≤ δ/m`
+ * (Bonferroni). The loosest passing threshold wins. A grid with no pass returns null.
+ *
+ * @param rows - Score and 0/1 loss
+ * @param maxError - Risk cap
+ * @param delta - Family-wise level. Default `0.1` (`autonomy.risk`)
+ */
+export function learnThenTest(
+  rows: readonly { readonly score: number; readonly loss: number }[],
+  maxError: number,
+  delta = 0.1,
+): number | null {
+  if (rows.length === 0) return null;
+  const m = LEARN_THEN_TEST_GRID.length;
+  const alpha = delta / m;
+  let chosen: number | null = null;
+  for (const threshold of LEARN_THEN_TEST_GRID) {
+    const accepted = rows.filter((row) => row.score >= threshold);
+    if (accepted.length === 0) continue;
+    const errors = accepted.reduce((sum, row) => sum + (row.loss > 0 ? 1 : 0), 0);
+    const pValue = binomialCdf(errors, accepted.length, maxError);
+    if (pValue > alpha) continue;
+    if (chosen === null || threshold < chosen) chosen = threshold;
+  }
+  return chosen;
+}
+
+/**
+ * Record a label. The reviewer Flow owns this write.
+ *
+ * @param label - Resolved label
+ */
+export function recordDecisionLabel(label: DecisionLabel): void {
+  labels.push(label);
+}
+
+/**
+ * Build one lock entry from stored labels. The same fit and test as certify.
+ *
+ * @param decision - Decision name
+ * @param fit - Turns that decision's labels into a lock entry
+ */
+export function aggregateDecisionCandidate(
+  decision: string,
+  fit: (rows: readonly DecisionLabel[]) => DecisionCandidate = () => ({
+    model: "",
+    questions: {},
+  }),
+): DecisionCandidate {
+  const rows = labels.filter((label) => label.decision === decision);
+  return fit(rows);
+}
+
+/**
+ * Write a fetched candidate into the lockfile slot. Does not recompute it.
+ *
+ * @param name - Decision name
+ * @param candidate - Body returned by the admin endpoint
+ * @param current - Lockfile to merge into
+ */
+export function lockFromCandidate(
+  name: string,
+  candidate: unknown,
+  current: DecisionLockfile | undefined,
+): DecisionLockfile {
+  const entry = parseDecisionLockEntry(candidate);
+  if (!entry) throw new TypeError("promote: candidate is not a lock entry");
+  setDecisionDrift(name, false);
+  return { decisions: { ...(current?.decisions ?? {}), [name]: entry } };
+}
+
+/**
+ * Accept a lock entry with a model and a questions map.
+ *
+ * @param body - Candidate or file slot
+ */
+export function parseDecisionLockEntry(body: unknown): DecisionLockEntry | undefined {
+  if (!body || typeof body !== "object") return undefined;
+  const record = body as Record<string, unknown>;
+  if (typeof record.model !== "string") return undefined;
+  if (
+    !record.questions ||
+    typeof record.questions !== "object" ||
+    Array.isArray(record.questions)
+  ) {
+    return undefined;
+  }
+  return {
+    model: record.model,
+    ...(typeof record.certifiedAt === "number" ? { certifiedAt: record.certifiedAt } : {}),
+    questions: record.questions as DecisionLockEntry["questions"],
+  };
+}
+
+/**
+ * Accept a lockfile object.
+ *
+ * @param body - Parsed JSON
+ */
+export function parseDecisionLockfile(body: unknown): DecisionLockfile | undefined {
+  if (!body || typeof body !== "object") return undefined;
+  const decisions = (body as { decisions?: unknown }).decisions;
+  if (!decisions || typeof decisions !== "object" || Array.isArray(decisions)) return undefined;
+  const next: Record<string, DecisionLockEntry> = {};
+  for (const [name, entry] of Object.entries(decisions)) {
+    const parsed = parseDecisionLockEntry(entry);
+    if (!parsed) return undefined;
+    next[name] = parsed;
+  }
+  return { decisions: next };
+}
+
+/**
+ * Read `oke-decisions.lock.json` from an app root and install it.
+ * A missing file clears the in-memory lock.
+ *
+ * @param root - Directory that holds the app config
+ */
+export async function loadDecisionLockfile(root: string): Promise<DecisionLockfile | undefined> {
+  const file = Bun.file(`${root}/${DECISION_LOCK_FILENAME}`);
+  if (!(await file.exists())) {
+    setDecisionLock(undefined);
+    return undefined;
+  }
+  const parsed = parseDecisionLockfile(await file.json());
+  setDecisionLock(parsed);
+  return parsed;
+}

package/src/elements/ai/decisions/certify.ts

@@ -0,0 +1,314 @@
+/**
+ * Build a decision certificate from a seed JSONL file.
+ * `{ id?, input, expect: { <question>: value }, locale? }`.
+ */
+
+import type { AiDecisionQuestion } from "../declare.ts";
+import {
+  applyTemperature,
+  learnThenTest,
+  questionHash,
+  type DecisionCalibrator,
+  type DecisionCertSlice,
+  type DecisionLabel,
+  type DecisionLockEntry,
+} from "./certificate.ts";
+import type { DecisionResponse } from "./provider.ts";
+
+/** One seed row. */
+export interface DecisionSeedCase {
+  readonly id?: string;
+  readonly input: unknown;
+  readonly expect: Readonly<Record<string, unknown>>;
+  readonly locale?: string;
+}
+
+/** One provider call used while certifying. */
+export type DecisionEvaluate = (input: unknown) => Promise<DecisionResponse>;
+
+/**
+ * Parse decision seed JSONL. Blank lines are skipped.
+ *
+ * @param text - File contents
+ */
+export function parseDecisionSeed(text: string): DecisionSeedCase[] {
+  const cases: DecisionSeedCase[] = [];
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    const row = JSON.parse(trimmed) as {
+      id?: unknown;
+      input?: unknown;
+      expect?: unknown;
+      locale?: unknown;
+    };
+    if (!row.expect || typeof row.expect !== "object" || Array.isArray(row.expect)) {
+      throw new TypeError("decision seed row needs expect");
+    }
+    cases.push({
+      ...(typeof row.id === "string" ? { id: row.id } : {}),
+      input: row.input,
+      expect: row.expect as Readonly<Record<string, unknown>>,
+      ...(typeof row.locale === "string" ? { locale: row.locale } : {}),
+    });
+  }
+  return cases;
+}
+
+/** One labeled probability used to fit a slice. */
+interface FitRow {
+  readonly probs: readonly number[];
+  readonly labelIndex: number;
+  readonly noul?: number;
+  readonly booleanLabel?: boolean;
+  /** Inverse propensity. Audit rows count more than a full review. */
+  readonly weight: number;
+}
+
+/**
+ * Certificate from a seed file. Calls the provider, fits a calibrator per
+ * question, and keeps a threshold only when Learn-then-Test passes.
+ *
+ * @param options - Decision, risk, seed, and provider
+ */
+export async function certifySeed(options: {
+  readonly model: string;
+  readonly maxError: number;
+  readonly delta?: number;
+  readonly jsonl: string;
+  readonly ask: Readonly<Record<string, AiDecisionQuestion>>;
+  readonly evaluate: DecisionEvaluate;
+}): Promise<DecisionLockEntry> {
+  const cases = parseDecisionSeed(options.jsonl);
+  const buckets = new Map<string, FitRow[]>();
+  let resolved = options.model;
+  for (const row of cases) {
+    const response = await options.evaluate(row.input);
+    if (response.model) resolved = response.model;
+    const locale = row.locale ?? "";
+    for (const [id, expected] of Object.entries(row.expect)) {
+      const question = options.ask[id];
+      if (!question) continue;
+      const answer = response.answers[id];
+      const fit = fitRow(question, answer, expected);
+      if (!fit) continue;
+      const key = `${id}\0${locale}`;
+      const list = buckets.get(key) ?? [];
+      list.push(fit);
+      buckets.set(key, list);
+    }
+  }
+  const entry = entryFromBuckets(
+    resolved,
+    options.ask,
+    buckets,
+    options.maxError,
+    options.delta ?? 0.1,
+  );
+  return { ...entry, certifiedAt: Date.now() };
+}
+
+/**
+ * The same certificate, built from persisted labels that already carry a score.
+ *
+ * @param options - Model, questions, labels, and risk
+ */
+export function certifyLabels(options: {
+  readonly model: string;
+  readonly maxError: number;
+  readonly delta?: number;
+  readonly ask: Readonly<Record<string, AiDecisionQuestion>>;
+  readonly labels: readonly DecisionLabel[];
+}): DecisionLockEntry {
+  const groups = new Map<string, DecisionLabel[]>();
+  for (const label of options.labels) {
+    if (label.raw === undefined) continue;
+    if (options.model && label.model !== options.model) continue;
+    const key = `${label.question}\0${label.locale ?? ""}`;
+    const list = groups.get(key) ?? [];
+    list.push(label);
+    groups.set(key, list);
+  }
+  const questions: Record<string, Record<string, DecisionCertSlice>> = {};
+  const model = options.labels.find((label) => label.model)?.model ?? options.model;
+  const delta = options.delta ?? 0.1;
+  for (const [key, rows] of groups) {
+    const split = key.indexOf("\0");
+    const id = key.slice(0, split);
+    const locale = key.slice(split + 1);
+    const question = options.ask[id];
+    if (!question) continue;
+    const fitted = rows.flatMap((row) => {
+      const fit = fitRow(question, row.raw, row.value);
+      return fit ? [{ ...fit, weight: propensityWeight(row.propensity) }] : [];
+    });
+    const slice = sliceFromRows(question, fitted, options.maxError, delta);
+    if (!slice) continue;
+    const bucket = questions[id] ?? {};
+    bucket[locale] = slice;
+    questions[id] = bucket;
+  }
+  return { model, certifiedAt: Date.now(), questions };
+}
+
+function entryFromBuckets(
+  model: string,
+  ask: Readonly<Record<string, AiDecisionQuestion>>,
+  buckets: ReadonlyMap<string, readonly FitRow[]>,
+  maxError: number,
+  delta: number,
+): DecisionLockEntry {
+  const questions: Record<string, Record<string, DecisionCertSlice>> = {};
+  for (const [key, rows] of buckets) {
+    const split = key.indexOf("\0");
+    const id = key.slice(0, split);
+    const locale = key.slice(split + 1);
+    const question = ask[id];
+    if (!question) continue;
+    const slice = sliceFromRows(question, rows, maxError, delta);
+    if (!slice) continue;
+    const bucket = questions[id] ?? {};
+    bucket[locale] = slice;
+    questions[id] = bucket;
+  }
+  return { model, questions };
+}
+
+function sliceFromRows(
+  question: AiDecisionQuestion,
+  rows: readonly FitRow[],
+  maxError: number,
+  delta: number,
+): DecisionCertSlice | undefined {
+  const calibrator = fitCalibrator(question, rows);
+  const scored = rows.map((row) => scoreFit(question, row, calibrator));
+  const threshold = learnThenTest(scored, maxError, delta);
+  if (threshold === null) return undefined;
+  const errors = scored.filter((row) => row.score >= threshold && row.loss > 0).length;
+  const accepted = scored.filter((row) => row.score >= threshold).length;
+  return {
+    hash: questionHash(question),
+    calibrator,
+    threshold,
+    metrics: { labels: rows.length, accepted, errors, maxError, delta },
+  };
+}
+
+function fitCalibrator(question: AiDecisionQuestion, rows: readonly FitRow[]): DecisionCalibrator {
+  if (question.kind === "boolean") return fitPlatt(rows);
+  return { kind: "temperature", t: fitTemperature(rows) };
+}
+
+function fitTemperature(rows: readonly FitRow[]): number {
+  let bestT = 1;
+  let best = Number.POSITIVE_INFINITY;
+  for (let step = 1; step <= 20; step++) {
+    const t = step / 10;
+    let nll = 0;
+    for (const row of rows) {
+      const scaled = applyTemperature(row.probs, t);
+      nll -= row.weight * Math.log(Math.max(scaled[row.labelIndex] ?? 1e-12, 1e-12));
+    }
+    if (nll < best) {
+      best = nll;
+      bestT = t;
+    }
+  }
+  return bestT;
+}
+
+function fitPlatt(rows: readonly FitRow[]): DecisionCalibrator {
+  let best = { a: 1, b: 0, nll: Number.POSITIVE_INFINITY };
+  for (let a = -2; a <= 2.01; a += 0.5) {
+    for (let b = -2; b <= 2.01; b += 0.5) {
+      let nll = 0;
+      for (const row of rows) {
+        const p = platt(row.noul ?? 0.5, a, b);
+        const y = row.booleanLabel ? 1 : 0;
+        nll -=
+          row.weight *
+          (y * Math.log(Math.max(p, 1e-12)) + (1 - y) * Math.log(Math.max(1 - p, 1e-12)));
+      }
+      if (nll < best.nll) best = { a, b, nll };
+    }
+  }
+  return { kind: "platt", a: best.a, b: best.b };
+}
+
+function platt(noul: number, a: number, b: number): number {
+  const p = Math.min(1, Math.max(0, noul));
+  const logit = Math.log(Math.max(p, 1e-12) / Math.max(1 - p, 1e-12));
+  return 1 / (1 + Math.exp(-(a * logit + b)));
+}
+
+function scoreFit(
+  question: AiDecisionQuestion,
+  row: FitRow,
+  calibrator: DecisionCalibrator,
+): { readonly score: number; readonly loss: number } {
+  if (question.kind === "boolean" && calibrator.kind !== "temperature") {
+    const p = platt(
+      row.noul ?? 0.5,
+      calibrator.kind === "platt" ? calibrator.a : 1,
+      calibrator.kind === "platt" ? calibrator.b : 0,
+    );
+    const predicted = p >= 0.5;
+    return { score: Math.max(p, 1 - p), loss: predicted === row.booleanLabel ? 0 : 1 };
+  }
+  const t = calibrator.kind === "temperature" ? calibrator.t : 1;
+  const scaled = applyTemperature(row.probs, t);
+  let best = 0;
+  for (let i = 1; i < scaled.length; i++) {
+    if ((scaled[i] ?? 0) > (scaled[best] ?? 0)) best = i;
+  }
+  return { score: scaled[best] ?? 0, loss: best === row.labelIndex ? 0 : 1 };
+}
+
+/**
+ * Provider answers wrap the distribution in `probabilities`.
+ * A stored label writes that same distribution as `raw`.
+ *
+ * @param record - Answer or stored raw
+ */
+function storedDistribution(record: Record<string, unknown>): unknown {
+  if (record.probabilities !== undefined) return record.probabilities;
+  return record;
+}
+
+function fitRow(
+  question: AiDecisionQuestion,
+  answer: unknown,
+  expected: unknown,
+): FitRow | undefined {
+  const record = answer && typeof answer === "object" ? (answer as Record<string, unknown>) : {};
+  if (question.kind === "boolean") {
+    if (typeof expected !== "boolean") return undefined;
+    return {
+      probs: [],
+      labelIndex: expected ? 1 : 0,
+      noul: typeof record.noul === "number" ? record.noul : 0,
+      booleanLabel: expected,
+      weight: 1,
+    };
+  }
+  const keys =
+    question.kind === "choice"
+      ? [...Object.keys(question.options), "none_of_these"]
+      : [...question.levels];
+  const labelIndex = keys.indexOf(String(expected));
+  if (labelIndex < 0) return undefined;
+  const raw = storedDistribution(record);
+  const probs = keys.map((key, index) => {
+    if (raw && typeof raw === "object" && !Array.isArray(raw)) {
+      const value = (raw as Record<string, unknown>)[key];
+      return typeof value === "number" ? value : 0;
+    }
+    return Array.isArray(raw) && typeof raw[index] === "number" ? raw[index] : 0;
+  });
+  return { probs, labelIndex, weight: 1 };
+}
+
+function propensityWeight(propensity: number): number {
+  if (!(propensity > 0)) return 1;
+  return 1 / propensity;
+}