okengine 0.23.2 → 0.24.0

package/src/elements/ai/decisions/certificate.ts

@@ -37,26 +37,43 @@ export interface DecisionCertSlice {
   readonly metrics?: Readonly<Record<string, number>>;
 }
 
-/** One decision inside `oke-decisions.lock.json`. */
-export interface DecisionLockEntry {
+/** One decider's certificate inside a decision. */
+export interface DecisionDeciderCert {
   readonly model: string;
+  /** False when the provider only echoes an alias. */
+  readonly pinned: boolean;
+  /** Epoch ms an alias certificate stops granting auto. */
+  readonly expiresAt?: number;
   /** Epoch ms the certificate was written. Drift counts labels after this. */
   readonly certifiedAt?: number;
   readonly questions: Readonly<Record<string, Readonly<Record<string, DecisionCertSlice>>>>;
 }
 
-/** App lockfile. Lives next to that app's OKE config. */
+/** One decision inside `oke-decisions.lock.json`. */
+export interface DecisionLockEntry {
+  readonly deciders: Readonly<Record<string, DecisionDeciderCert>>;
+}
+
+/** App lockfile. Version 1 is rejected. */
 export interface DecisionLockfile {
+  readonly version: 2;
   readonly decisions: Readonly<Record<string, DecisionLockEntry>>;
 }
 
+/** A lockfile that is not version 2. */
+export class DecisionLockVersionError extends Error {
+  constructor() {
+    super("oke-decisions.lock.json is not version 2. Run `oke decide certify` to recertify.");
+    this.name = "DecisionLockVersionError";
+  }
+}
+
 /** Why a question is not auto. */
-export type DecisionUncertainty =
-  | "low-confidence"
-  | "missing-lock"
-  | "stale-hash"
-  | "version-mismatch"
-  | "uncertified-locale"
+export type DecisionWhy =
+  | "uncertain"
+  | "refused"
+  | "none_of_these"
+  | "uncertified"
   | "drift"
   | "outage";
 
@@ -84,10 +101,14 @@ export interface DecisionLabel {
   readonly reviewId?: string;
 }
 
-/** App-wide candidate. A full lock entry, not a count. */
+/** App-wide candidate for one decider. */
 export interface DecisionCandidate {
+  readonly decider: string;
   readonly model: string;
-  readonly questions: DecisionLockEntry["questions"];
+  readonly pinned: boolean;
+  readonly expiresAt?: number;
+  readonly certifiedAt?: number;
+  readonly questions: DecisionDeciderCert["questions"];
 }
 
 /** Lockfile name at the app project root. */
@@ -292,7 +313,9 @@ export function recordDecisionLabel(label: DecisionLabel): void {
 export function aggregateDecisionCandidate(
   decision: string,
   fit: (rows: readonly DecisionLabel[]) => DecisionCandidate = () => ({
+    decider: "",
     model: "",
+    pinned: true,
     questions: {},
   }),
 ): DecisionCandidate {
@@ -312,21 +335,39 @@ export function lockFromCandidate(
   candidate: unknown,
   current: DecisionLockfile | undefined,
 ): DecisionLockfile {
-  const entry = parseDecisionLockEntry(candidate);
-  if (!entry) throw new TypeError("promote: candidate is not a lock entry");
+  const parsed = parseDecisionCandidate(candidate);
+  if (!parsed) throw new TypeError("promote: candidate is not a lock entry");
   setDecisionDrift(name, false);
-  return { decisions: { ...(current?.decisions ?? {}), [name]: entry } };
+  const prev = current?.decisions[name]?.deciders ?? {};
+  return {
+    version: 2,
+    decisions: {
+      ...(current?.decisions ?? {}),
+      [name]: { deciders: { ...prev, [parsed.decider]: certFromCandidate(parsed) } },
+    },
+  };
+}
+
+function certFromCandidate(candidate: DecisionCandidate): DecisionDeciderCert {
+  return {
+    model: candidate.model,
+    pinned: candidate.pinned,
+    ...(candidate.expiresAt !== undefined ? { expiresAt: candidate.expiresAt } : {}),
+    ...(candidate.certifiedAt !== undefined ? { certifiedAt: candidate.certifiedAt } : {}),
+    questions: candidate.questions,
+  };
 }
 
 /**
- * Accept a lock entry with a model and a questions map.
+ * Accept a candidate for one decider.
  *
- * @param body - Candidate or file slot
+ * @param body - Candidate body
  */
-export function parseDecisionLockEntry(body: unknown): DecisionLockEntry | undefined {
+export function parseDecisionCandidate(body: unknown): DecisionCandidate | undefined {
   if (!body || typeof body !== "object") return undefined;
   const record = body as Record<string, unknown>;
-  if (typeof record.model !== "string") return undefined;
+  if (typeof record.decider !== "string" || typeof record.model !== "string") return undefined;
+  if (typeof record.pinned !== "boolean") return undefined;
   if (
     !record.questions ||
     typeof record.questions !== "object" ||
@@ -335,28 +376,70 @@ export function parseDecisionLockEntry(body: unknown): DecisionLockEntry | undef
     return undefined;
   }
   return {
+    decider: record.decider,
     model: record.model,
+    pinned: record.pinned,
+    ...(typeof record.expiresAt === "number" ? { expiresAt: record.expiresAt } : {}),
     ...(typeof record.certifiedAt === "number" ? { certifiedAt: record.certifiedAt } : {}),
-    questions: record.questions as DecisionLockEntry["questions"],
+    questions: record.questions as DecisionCandidate["questions"],
   };
 }
 
 /**
- * Accept a lockfile object.
+ * Accept one decision slot (`deciders` map).
+ *
+ * @param body - File slot
+ */
+export function parseDecisionLockEntry(body: unknown): DecisionLockEntry | undefined {
+  if (!body || typeof body !== "object") return undefined;
+  const record = body as Record<string, unknown>;
+  if (typeof record.model === "string" && record.deciders === undefined) {
+    throw new DecisionLockVersionError();
+  }
+  if (!record.deciders || typeof record.deciders !== "object" || Array.isArray(record.deciders)) {
+    return undefined;
+  }
+  const deciders: Record<string, DecisionDeciderCert> = {};
+  for (const [name, entry] of Object.entries(record.deciders)) {
+    const parsed = parseDecisionCandidate({
+      ...(entry as object),
+      decider: name,
+      pinned: (entry as { pinned?: unknown }).pinned ?? true,
+    });
+    if (!parsed) return undefined;
+    deciders[name] = certFromCandidate(parsed);
+  }
+  return { deciders };
+}
+
+/**
+ * Accept a version 2 lockfile. Version 1 throws {@link DecisionLockVersionError}.
  *
  * @param body - Parsed JSON
  */
 export function parseDecisionLockfile(body: unknown): DecisionLockfile | undefined {
   if (!body || typeof body !== "object") return undefined;
-  const decisions = (body as { decisions?: unknown }).decisions;
-  if (!decisions || typeof decisions !== "object" || Array.isArray(decisions)) return undefined;
+  const record = body as { version?: unknown; decisions?: unknown };
+  if (record.version !== 2) {
+    if (record.decisions && typeof record.decisions === "object") {
+      throw new DecisionLockVersionError();
+    }
+    return undefined;
+  }
+  if (
+    !record.decisions ||
+    typeof record.decisions !== "object" ||
+    Array.isArray(record.decisions)
+  ) {
+    return undefined;
+  }
   const next: Record<string, DecisionLockEntry> = {};
-  for (const [name, entry] of Object.entries(decisions)) {
+  for (const [name, entry] of Object.entries(record.decisions)) {
     const parsed = parseDecisionLockEntry(entry);
     if (!parsed) return undefined;
     next[name] = parsed;
   }
-  return { decisions: next };
+  return { version: 2, decisions: next };
 }
 
 /**

package/src/elements/ai/decisions/certify.ts

@@ -10,8 +10,8 @@ import {
   questionHash,
   type DecisionCalibrator,
   type DecisionCertSlice,
+  type DecisionDeciderCert,
   type DecisionLabel,
-  type DecisionLockEntry,
 } from "./certificate.ts";
 import type { DecisionResponse } from "./provider.ts";
 
@@ -26,6 +26,24 @@ export interface DecisionSeedCase {
 /** One provider call used while certifying. */
 export type DecisionEvaluate = (input: unknown) => Promise<DecisionResponse>;
 
+/** Printed for one question after certify. */
+export interface CertifyQuestionReport {
+  readonly accuracy: number;
+  readonly ece: number;
+  readonly coverage: number;
+  readonly p50Ms: number;
+  readonly p95Ms: number;
+  readonly cost: number;
+}
+
+/** Certificate plus the comparison row. */
+export interface CertifySeedResult {
+  readonly cert: DecisionDeciderCert;
+  readonly report: Readonly<Record<string, CertifyQuestionReport>>;
+  /** True when the certificate is an expiring alias. */
+  readonly unpinned: boolean;
+}
+
 /**
  * Parse decision seed JSONL. Blank lines are skipped.
  *
@@ -73,29 +91,51 @@ interface FitRow {
  */
 export async function certifySeed(options: {
   readonly model: string;
+  readonly pinned?: boolean;
+  readonly expiresAt?: number;
   readonly maxError: number;
   readonly delta?: number;
   readonly jsonl: string;
   readonly ask: Readonly<Record<string, AiDecisionQuestion>>;
   readonly evaluate: DecisionEvaluate;
-}): Promise<DecisionLockEntry> {
+  readonly now?: () => number;
+}): Promise<CertifySeedResult> {
   const cases = parseDecisionSeed(options.jsonl);
   const buckets = new Map<string, FitRow[]>();
+  const samples = new Map<string, { correct: boolean; confidence: number }[]>();
+  const latencies: number[] = [];
+  let cost = 0;
   let resolved = options.model;
+  const now = options.now ?? Date.now;
   for (const row of cases) {
+    const started = now();
     const response = await options.evaluate(row.input);
+    latencies.push(Math.max(0, now() - started));
+    cost += response.usage.cost ?? 0;
     if (response.model) resolved = response.model;
     const locale = row.locale ?? "";
     for (const [id, expected] of Object.entries(row.expect)) {
       const question = options.ask[id];
       if (!question) continue;
-      const answer = response.answers[id];
+      const answers = response.answers;
+      const answer = Array.isArray(answers)
+        ? undefined
+        : (answers as Readonly<Record<string, unknown>>)[id];
       const fit = fitRow(question, answer, expected);
       if (!fit) continue;
       const key = `${id}\0${locale}`;
       const list = buckets.get(key) ?? [];
       list.push(fit);
       buckets.set(key, list);
+      const confidence =
+        fit.noul !== undefined ? Math.max(fit.noul, 1 - fit.noul) : Math.max(...fit.probs, 0);
+      const correct =
+        fit.booleanLabel !== undefined
+          ? fit.noul !== undefined && fit.noul >= 0.5 === fit.booleanLabel
+          : fit.probs[fit.labelIndex] === Math.max(...fit.probs);
+      const bag = samples.get(id) ?? [];
+      bag.push({ correct, confidence });
+      samples.set(id, bag);
     }
   }
   const entry = entryFromBuckets(
@@ -105,7 +145,30 @@ export async function certifySeed(options: {
     options.maxError,
     options.delta ?? 0.1,
   );
-  return { ...entry, certifiedAt: Date.now() };
+  const pinned = options.pinned !== false;
+  const cert: DecisionDeciderCert = {
+    ...entry,
+    pinned,
+    ...(options.expiresAt !== undefined ? { expiresAt: options.expiresAt } : {}),
+    certifiedAt: Date.now(),
+  };
+  const report: Record<string, CertifyQuestionReport> = {};
+  for (const [id, rows] of samples) {
+    const threshold = cert.questions[id]?.[""]?.threshold;
+    const covered =
+      threshold === undefined
+        ? 0
+        : rows.filter((row) => row.confidence >= threshold).length / rows.length;
+    report[id] = {
+      accuracy: rows.filter((row) => row.correct).length / rows.length,
+      ece: expectedCalibrationError(rows),
+      coverage: covered,
+      p50Ms: percentile(latencies, 0.5),
+      p95Ms: percentile(latencies, 0.95),
+      cost,
+    };
+  }
+  return { cert, report, unpinned: !pinned };
 }
 
 /**
@@ -119,7 +182,7 @@ export function certifyLabels(options: {
   readonly delta?: number;
   readonly ask: Readonly<Record<string, AiDecisionQuestion>>;
   readonly labels: readonly DecisionLabel[];
-}): DecisionLockEntry {
+}): DecisionDeciderCert {
   const groups = new Map<string, DecisionLabel[]>();
   for (const label of options.labels) {
     if (label.raw === undefined) continue;
@@ -148,7 +211,7 @@ export function certifyLabels(options: {
     bucket[locale] = slice;
     questions[id] = bucket;
   }
-  return { model, certifiedAt: Date.now(), questions };
+  return { model, pinned: true, certifiedAt: Date.now(), questions };
 }
 
 function entryFromBuckets(
@@ -157,7 +220,7 @@ function entryFromBuckets(
   buckets: ReadonlyMap<string, readonly FitRow[]>,
   maxError: number,
   delta: number,
-): DecisionLockEntry {
+): DecisionDeciderCert {
   const questions: Record<string, Record<string, DecisionCertSlice>> = {};
   for (const [key, rows] of buckets) {
     const split = key.indexOf("\0");
@@ -171,7 +234,7 @@ function entryFromBuckets(
     bucket[locale] = slice;
     questions[id] = bucket;
   }
-  return { model, questions };
+  return { model, pinned: true, questions };
 }
 
 function sliceFromRows(
@@ -270,6 +333,32 @@ function scoreFit(
  *
  * @param record - Answer or stored raw
  */
+function percentile(values: readonly number[], p: number): number {
+  if (values.length === 0) return 0;
+  const sorted = [...values].sort((a, b) => a - b);
+  const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1));
+  return sorted[index] ?? 0;
+}
+
+function expectedCalibrationError(
+  rows: readonly { readonly confidence: number; readonly correct: boolean }[],
+): number {
+  if (rows.length === 0) return 0;
+  const bins = Array.from({ length: 10 }, () => ({ n: 0, confidence: 0, correct: 0 }));
+  for (const row of rows) {
+    const index = Math.min(9, Math.floor(row.confidence * 10));
+    const bin = bins[index];
+    if (!bin) continue;
+    bin.n += 1;
+    bin.confidence += row.confidence;
+    bin.correct += row.correct ? 1 : 0;
+  }
+  return bins.reduce((sum, bin) => {
+    if (bin.n === 0) return sum;
+    return sum + (bin.n / rows.length) * Math.abs(bin.correct / bin.n - bin.confidence / bin.n);
+  }, 0);
+}
+
 function storedDistribution(record: Record<string, unknown>): unknown {
   if (record.probabilities !== undefined) return record.probabilities;
   return record;
@@ -281,12 +370,18 @@ function fitRow(
   expected: unknown,
 ): FitRow | undefined {
   const record = answer && typeof answer === "object" ? (answer as Record<string, unknown>) : {};
+  if (record.type === "refusal" || record.type === "malformed") return undefined;
   if (question.kind === "boolean") {
     if (typeof expected !== "boolean") return undefined;
     return {
       probs: [],
       labelIndex: expected ? 1 : 0,
-      noul: typeof record.noul === "number" ? record.noul : 0,
+      noul:
+        typeof record.probability === "number"
+          ? record.probability
+          : typeof record.noul === "number"
+            ? record.noul
+            : 0,
       booleanLabel: expected,
       weight: 1,
     };

package/src/elements/ai/decisions/codec.test.ts

@@ -0,0 +1,142 @@
+/**
+ * Codec round-trips for the verified System One body and the OpenAI decisions wire.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { ai } from "../../ai.ts";
+import { encodeDecisionRequest, normalizeDecisionResponse, wireQuestions } from "./codec.ts";
+
+const openRouterBody = {
+  model: "typesafe/jev-1.13-20260917",
+  answers: {
+    team: {
+      type: "choice",
+      choice: "technical",
+      probabilities: { billing: 0.49, none_of_these: 0.02, technical: 0.49 },
+      confidence: 0.23,
+    },
+    urgent: { type: "noul", noul: 0.54 },
+  },
+  usage: { input_tokens: 348, output_tokens: 58, cost: 0.000014616 },
+  id: "gen-dec-1791389894-FvrNcwqrVRdAEvlEDRh9",
+  provider: "TypeSafe",
+};
+
+describe("decision codecs", () => {
+  test("System One normalizes the verified OpenRouter body", () => {
+    const response = normalizeDecisionResponse("systemone", openRouterBody);
+    expect(response.model).toBe("typesafe/jev-1.13-20260917");
+    expect(response.provider).toBe("TypeSafe");
+    expect(response.usage).toEqual({ inputTokens: 348, outputTokens: 58, cost: 0.000014616 });
+    expect(response.answers).toMatchObject({
+      team: {
+        type: "choice",
+        choice: "technical",
+        probabilities: { billing: 0.49, none_of_these: 0.02, technical: 0.49 },
+      },
+      urgent: { type: "boolean", probability: 0.54 },
+    });
+  });
+
+  test("a choice without a choice string uses the highest probability", () => {
+    const response = normalizeDecisionResponse("systemone", {
+      model: "m",
+      answers: {
+        team: { type: "choice", probabilities: { billing: 0.2, technical: 0.8 } },
+      },
+    });
+    expect(response.answers).toMatchObject({
+      team: { type: "choice", choice: "technical" },
+    });
+  });
+
+  test("refusal drops the text and a partial body is malformed", () => {
+    const response = normalizeDecisionResponse("systemone", {
+      model: "m",
+      answers: {
+        team: { type: "refusal", text: "not a label" },
+        urgent: { type: "choice" },
+        empty: { nope: true },
+      },
+    });
+    expect(response.answers).toEqual({
+      team: { type: "refusal" },
+      urgent: { type: "malformed" },
+      empty: { type: "malformed" },
+    });
+    expect(JSON.stringify(response.answers)).not.toContain("not a label");
+  });
+
+  test("OpenAI decisions encode and normalize predicate, choice, score, and refusal", () => {
+    const questions = wireQuestions({
+      urgent: ai.boolean("Does this need a person today?"),
+      team: ai.choice("Which team owns this ticket?", {
+        billing: "Billing",
+        technical: "Technical",
+      }),
+      severity: ai.score("How severe is this?", ["low", "high"]),
+    });
+    const encoded = encodeDecisionRequest("openai-decisions", {
+      model: "gpt-6-luna",
+      state: { ticket: "Checkout is blank" },
+      questions,
+    }) as {
+      model: string;
+      input: string;
+      questions: {
+        name: string;
+        type: string;
+        choices?: { value: string }[];
+        levels?: { label: string }[];
+      }[];
+    };
+    expect(encoded.model).toBe("gpt-6-luna");
+    expect(encoded.input).toBe(JSON.stringify({ ticket: "Checkout is blank" }));
+    expect(encoded.questions.map((question) => question.type)).toEqual([
+      "predicate",
+      "choice",
+      "score",
+    ]);
+    expect(encoded.questions[1]?.choices?.map((choice) => choice.value)).toEqual([
+      "billing",
+      "technical",
+      "none_of_these",
+    ]);
+    expect(encoded.questions[2]?.levels).toEqual([{ label: "low" }, { label: "high" }]);
+
+    const response = normalizeDecisionResponse("openai-decisions", {
+      model: "gpt-6-luna",
+      answers: [
+        { name: "urgent", type: "predicate", probability: 0.12 },
+        {
+          name: "team",
+          type: "choice",
+          choice: "none_of_these",
+          probabilities: [
+            { value: "billing", probability: 0.1 },
+            { value: "technical", probability: 0.2 },
+            { value: "none_of_these", probability: 0.7 },
+          ],
+        },
+        {
+          name: "severity",
+          type: "score",
+          score: "low",
+          probabilities: [
+            { label: "low", probability: 0.8 },
+            { label: "high", probability: 0.2 },
+          ],
+        },
+        { name: "held", type: "refusal" },
+      ],
+      usage: { input_tokens: 20, output_tokens: 8 },
+    });
+    expect(response.model).toBe("gpt-6-luna");
+    expect(response.answers).toMatchObject({
+      urgent: { type: "boolean", probability: 0.12 },
+      team: { type: "choice", choice: "none_of_these" },
+      severity: { type: "score", score: "low", probabilities: { low: 0.8, high: 0.2 } },
+      held: { type: "refusal" },
+    });
+  });
+});