okengine 0.23.2 → 0.24.0

package/src/elements/ai/decisions/codec.ts

@@ -0,0 +1,302 @@
+/**
+ * System One and OpenAI Decisions codecs.
+ * Both become one internal answer. Loaded only from the decision module.
+ */
+
+import type { AiDecisionQuestion } from "../declare.ts";
+import type { DeciderProtocol } from "../deciders/presets.ts";
+import type { DecisionRequest, DecisionResponse, DecisionUsage, WireQuestion } from "./provider.ts";
+
+/** One question after the codec. Refusal carries no text. */
+export type NormalizedAnswer =
+  | { readonly type: "boolean"; readonly probability: number }
+  | {
+      readonly type: "choice";
+      readonly choice: string;
+      readonly probabilities: Readonly<Record<string, number>>;
+      readonly confidence?: number;
+    }
+  | {
+      readonly type: "score";
+      readonly score: string;
+      readonly probabilities: Readonly<Record<string, number>>;
+      readonly confidence?: number;
+    }
+  | { readonly type: "refusal" }
+  | { readonly type: "malformed" };
+
+/**
+ * Token estimate used before a call. One token per four JSON characters.
+ *
+ * @param state - Calling input
+ */
+export function estimateDecisionTokens(state: unknown): number {
+  const text = typeof state === "string" ? state : JSON.stringify(state ?? null);
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * Encode a request for the decider's protocol.
+ *
+ * @param protocol - `systemone` or `openai-decisions`
+ * @param request - Model, state, and questions
+ */
+export function encodeDecisionRequest(
+  protocol: DeciderProtocol,
+  request: DecisionRequest,
+): unknown {
+  if (protocol === "systemone") {
+    return {
+      model: request.model,
+      state: request.state,
+      questions: request.questions,
+    };
+  }
+  const input =
+    typeof request.state === "string" ? request.state : JSON.stringify(request.state ?? null);
+  return {
+    model: request.model,
+    input,
+    questions: Object.entries(request.questions).map(([name, question]) =>
+      encodeOpenAIQuestion(name, question),
+    ),
+  };
+}
+
+function encodeOpenAIQuestion(name: string, question: WireQuestion): unknown {
+  if (question.type === "noul") {
+    return { name, type: "predicate", instructions: question.instructions };
+  }
+  if (question.type === "choice") {
+    return {
+      name,
+      type: "choice",
+      instructions: question.instructions,
+      choices: Object.entries(question.criteria).map(([value, description]) =>
+        description == null || description === "" ? { value } : { value, description },
+      ),
+    };
+  }
+  return {
+    name,
+    type: "score",
+    instructions: question.instructions,
+    levels: question.criteria.map((label) => ({ label })),
+  };
+}
+
+/**
+ * Normalize a parsed provider body. Malformed answers stay on the question id.
+ *
+ * @param protocol - Decider protocol
+ * @param body - Parsed JSON, or an already parsed {@link DecisionResponse}
+ */
+export function normalizeDecisionResponse(
+  protocol: DeciderProtocol,
+  body: unknown,
+): DecisionResponse {
+  if (!body || typeof body !== "object") {
+    return { model: "", answers: {}, usage: {} };
+  }
+  const record = body as {
+    model?: unknown;
+    provider?: unknown;
+    answers?: unknown;
+    usage?: unknown;
+  };
+  const model = typeof record.model === "string" ? record.model : "";
+  const answers =
+    protocol === "openai-decisions"
+      ? normalizeOpenAIAnswers(record.answers)
+      : normalizeSystemOneAnswers(record.answers);
+  return {
+    model,
+    ...(typeof record.provider === "string" ? { provider: record.provider } : {}),
+    answers,
+    usage: usageFrom(record.usage),
+  };
+}
+
+function normalizeSystemOneAnswers(raw: unknown): Record<string, NormalizedAnswer> {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return {};
+  const answers: Record<string, NormalizedAnswer> = {};
+  for (const [id, value] of Object.entries(raw)) {
+    answers[id] = normalizeSystemOneAnswer(value);
+  }
+  return answers;
+}
+
+function normalizeSystemOneAnswer(value: unknown): NormalizedAnswer {
+  if (!value || typeof value !== "object") return { type: "malformed" };
+  const record = value as Record<string, unknown>;
+  if (record.type === "noul" && typeof record.noul === "number") {
+    return { type: "boolean", probability: record.noul };
+  }
+  if (record.type === "choice") {
+    const probabilities = numberRecord(record.probabilities);
+    const choice =
+      typeof record.choice === "string"
+        ? record.choice
+        : probabilities
+          ? argmax(probabilities)
+          : "";
+    if (!probabilities || !choice) return { type: "malformed" };
+    return {
+      type: "choice",
+      choice,
+      probabilities,
+      ...(typeof record.confidence === "number" ? { confidence: record.confidence } : {}),
+    };
+  }
+  if (record.type === "score") {
+    const probabilities = numberRecord(record.probabilities);
+    if (!probabilities) return { type: "malformed" };
+    const score =
+      typeof record.score === "string"
+        ? record.score
+        : typeof record.score === "number"
+          ? String(record.score)
+          : argmax(probabilities);
+    return {
+      type: "score",
+      score,
+      probabilities,
+      ...(typeof record.confidence === "number" ? { confidence: record.confidence } : {}),
+    };
+  }
+  if (record.type === "refusal") return { type: "refusal" };
+  return { type: "malformed" };
+}
+
+function normalizeOpenAIAnswers(raw: unknown): Record<string, NormalizedAnswer> {
+  if (!Array.isArray(raw)) return {};
+  const answers: Record<string, NormalizedAnswer> = {};
+  for (const value of raw) {
+    if (!value || typeof value !== "object") continue;
+    const record = value as Record<string, unknown>;
+    const name = typeof record.name === "string" ? record.name : "";
+    if (!name) continue;
+    answers[name] = normalizeOpenAIAnswer(record);
+  }
+  return answers;
+}
+
+function normalizeOpenAIAnswer(record: Record<string, unknown>): NormalizedAnswer {
+  if (record.type === "refusal") return { type: "refusal" };
+  if (record.type === "predicate" && typeof record.probability === "number") {
+    return { type: "boolean", probability: record.probability };
+  }
+  if (record.type === "choice" && typeof record.choice === "string") {
+    const probabilities = probabilityList(record.probabilities, "value");
+    if (!probabilities) return { type: "malformed" };
+    return {
+      type: "choice",
+      choice: String(record.choice),
+      probabilities,
+      ...(typeof record.confidence === "number" ? { confidence: record.confidence } : {}),
+    };
+  }
+  if (record.type === "score") {
+    const probabilities = probabilityList(record.probabilities, "label");
+    if (!probabilities) return { type: "malformed" };
+    const score = typeof record.score === "string" ? record.score : argmax(probabilities);
+    return {
+      type: "score",
+      score,
+      probabilities,
+      ...(typeof record.confidence === "number" ? { confidence: record.confidence } : {}),
+    };
+  }
+  return { type: "malformed" };
+}
+
+function probabilityList(raw: unknown, key: "value" | "label"): Record<string, number> | undefined {
+  if (!Array.isArray(raw)) return undefined;
+  const probabilities: Record<string, number> = {};
+  for (const item of raw) {
+    if (!item || typeof item !== "object") return undefined;
+    const row = item as Record<string, unknown>;
+    const name = row[key];
+    if (
+      (typeof name !== "string" && typeof name !== "boolean" && typeof name !== "number") ||
+      typeof row.probability !== "number"
+    ) {
+      return undefined;
+    }
+    probabilities[String(name)] = row.probability;
+  }
+  return probabilities;
+}
+
+function numberRecord(raw: unknown): Record<string, number> | undefined {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined;
+  const probabilities: Record<string, number> = {};
+  for (const [key, value] of Object.entries(raw)) {
+    if (typeof value !== "number") return undefined;
+    probabilities[key] = value;
+  }
+  return probabilities;
+}
+
+function argmax(probabilities: Readonly<Record<string, number>>): string {
+  let best = "";
+  let score = -1;
+  for (const [key, value] of Object.entries(probabilities)) {
+    if (value > score) {
+      best = key;
+      score = value;
+    }
+  }
+  return best;
+}
+
+function usageFrom(raw: unknown): DecisionUsage {
+  if (!raw || typeof raw !== "object") return {};
+  const usage = raw as {
+    input_tokens?: unknown;
+    output_tokens?: unknown;
+    cost?: unknown;
+    inputTokens?: unknown;
+    outputTokens?: unknown;
+  };
+  const input = usage.input_tokens ?? usage.inputTokens;
+  const output = usage.output_tokens ?? usage.outputTokens;
+  return {
+    ...(typeof input === "number" ? { inputTokens: input } : {}),
+    ...(typeof output === "number" ? { outputTokens: output } : {}),
+    ...(typeof usage.cost === "number" ? { cost: usage.cost } : {}),
+  };
+}
+
+/**
+ * Wire questions for one decision. Choice always injects `none_of_these`.
+ *
+ * @param ask - Declared questions
+ */
+export function wireQuestions(
+  ask: Readonly<Record<string, AiDecisionQuestion>>,
+): Record<string, WireQuestion> {
+  const questions: Record<string, WireQuestion> = {};
+  for (const [id, question] of Object.entries(ask)) {
+    questions[id] = wireQuestion(question);
+  }
+  return questions;
+}
+
+function wireQuestion(question: AiDecisionQuestion): WireQuestion {
+  if (question.kind === "choice") {
+    return {
+      type: "choice",
+      instructions: question.instructions,
+      criteria: { ...question.options, none_of_these: null },
+    };
+  }
+  if (question.kind === "score") {
+    return { type: "score", instructions: question.instructions, criteria: question.levels };
+  }
+  return {
+    type: "noul",
+    instructions: question.instructions,
+    ...(question.criteria !== undefined ? { criteria: question.criteria } : {}),
+  };
+}

package/src/elements/ai/decisions/decide.test.ts

@@ -54,6 +54,44 @@ function choice() {
   return ai.choice("which team", { billing: "Billing", technical: "Technical" });
 }
 
+function jev(name = "jev") {
+  return ai.decider(name, { provider: "openrouter", model: "typesafe/jev-1.13.0" });
+}
+
+function setLock(
+  body:
+    | {
+        decisions: Record<
+          string,
+          { model: string; certifiedAt?: number; questions: Record<string, unknown> }
+        >;
+      }
+    | undefined,
+) {
+  if (!body) {
+    setDecisionLock(undefined);
+    return;
+  }
+  setDecisionLock({
+    version: 2,
+    decisions: Object.fromEntries(
+      Object.entries(body.decisions).map(([name, entry]) => [
+        name,
+        {
+          deciders: {
+            jev: {
+              model: entry.model,
+              pinned: true as const,
+              ...(entry.certifiedAt !== undefined ? { certifiedAt: entry.certifiedAt } : {}),
+              questions: entry.questions as never,
+            },
+          },
+        },
+      ]),
+    ),
+  });
+}
+
 function provider() {
   setDecisionProvider(async () => ({
     model: "typesafe/jev-1.13.0",
@@ -74,11 +112,12 @@ describe("fx.decide", () => {
     provider();
     const question = choice();
     const decl = ai.decision("triage", {
-      onUncertain: "abstain",
+      decider: jev(),
+      otherwise: "abstain",
       autonomy: { maxError: 0.05, audit: 0 },
       ask: { team: question },
     });
-    setDecisionLock({
+    setLock({
       decisions: {
         triage: {
           model: "typesafe/jev-1.13.0",
@@ -122,7 +161,8 @@ describe("fx.decide", () => {
       throw new DecisionOutageError("down");
     });
     const decl = ai.decision("triage", {
-      onUncertain: "abstain",
+      decider: jev(),
+      otherwise: "abstain",
       ask: { team: choice() },
     });
     const fx = createFx({ flow: "run", effects: { decides: ["triage"] }, now: () => 1 });
@@ -139,11 +179,12 @@ describe("fx.decide", () => {
     provider();
     const question = choice();
     const decl = ai.decision("triage", {
-      onUncertain: "abstain",
+      decider: jev(),
+      otherwise: "abstain",
       autonomy: { maxError: 0.05, audit: 1 },
       ask: { team: question },
     });
-    setDecisionLock({
+    setLock({
       decisions: {
         triage: {
           model: "typesafe/jev-1.13.0",
@@ -187,7 +228,8 @@ describe("fx.decide", () => {
     provider();
     gate.policy("ops", (ctx) => ctx.operator.id !== null);
     const decl = ai.decision("triage", {
-      review: "ops",
+      decider: jev(),
+      otherwise: "ops",
       ask: { team: choice() },
     });
     const store = await createPostgresJournalStore({ sql: createPostgresJournalFake() });
@@ -232,7 +274,8 @@ describe("fx.decide", () => {
   test("the parked review masks secret input and keeps the rest", async () => {
     provider();
     const decl = ai.decision("triage", {
-      review: "ops",
+      decider: jev(),
+      otherwise: "ops",
       in: z.object({
         ticket: z.string(),
         token: z.string().describe("secret"),
@@ -298,11 +341,12 @@ describe("fx.decide", () => {
     provider();
     const question = choice();
     const decl = ai.decision("triage", {
-      onUncertain: "abstain",
+      decider: jev(),
+      otherwise: "abstain",
       autonomy: { maxError: 0.05, audit: 0 },
       ask: { team: question },
     });
-    setDecisionLock({
+    setLock({
       decisions: {
         triage: {
           model: "typesafe/jev-1.13.0",
@@ -340,7 +384,11 @@ describe("fx.decide", () => {
   test("the same decision can park twice in one run", async () => {
     provider();
     gate.policy("ops", (ctx) => ctx.operator.id !== null);
-    const decl = ai.decision("triage", { review: "ops", ask: { team: choice() } });
+    const decl = ai.decision("triage", {
+      decider: jev(),
+      otherwise: "ops",
+      ask: { team: choice() },
+    });
     const store = createMemoryJournalStore();
     const journal = createJournal({ store, now: () => 1_000_000 });
     const session = await journal.start("run", {});
@@ -413,11 +461,12 @@ describe("fx.decide", () => {
     }));
     const question = choice();
     const decl = ai.decision("triage", {
-      onUncertain: "abstain",
+      decider: jev(),
+      otherwise: "abstain",
       autonomy: { maxError: 0.05, audit: 0 },
       ask: { team: question },
     });
-    setDecisionLock({
+    setLock({
       decisions: {
         triage: {
           model: "typesafe/jev-1.13.0",
@@ -456,11 +505,12 @@ describe("fx.decide", () => {
     const team = choice();
     const urgent = ai.boolean("urgent?");
     const decl = ai.decision("triage", {
-      onUncertain: "abstain",
+      decider: jev(),
+      otherwise: "abstain",
       autonomy: { maxError: 0.05, audit: 0 },
       ask: { team, urgent },
     });
-    setDecisionLock({
+    setLock({
       decisions: {
         triage: {
           model: "typesafe/jev-1.13.0",
@@ -498,7 +548,11 @@ describe("fx.decide", () => {
   test("a reviewer may resolve a choice as none_of_these", async () => {
     provider();
     gate.policy("ops", () => true);
-    const decl = ai.decision("triage", { review: "ops", ask: { team: choice() } });
+    const decl = ai.decision("triage", {
+      decider: jev(),
+      otherwise: "ops",
+      ask: { team: choice() },
+    });
     const store = await createPostgresJournalStore({ sql: createPostgresJournalFake() });
     const journal = createJournal({ store, now: () => 1 });
     const session = await journal.start("run", {});
@@ -553,11 +607,12 @@ describe("fx.decide", () => {
     }));
     const question = ai.boolean("urgent?");
     const decl = ai.decision("triage", {
-      onUncertain: "abstain",
+      decider: jev(),
+      otherwise: "abstain",
       autonomy: { maxError: 0.05, audit: 0 },
       ask: { urgent: question },
     });
-    setDecisionLock({
+    setLock({
       decisions: {
         triage: {
           model: "typesafe/jev-1.13.0",
@@ -581,7 +636,11 @@ describe("fx.decide", () => {
   test("resolve checks the gate, the tenant, and every open question", async () => {
     provider();
     gate.policy("ops", (ctx) => ctx.operator.id === "reviewer");
-    const decl = ai.decision("triage", { review: "ops", ask: { team: choice() } });
+    const decl = ai.decision("triage", {
+      decider: jev(),
+      otherwise: "ops",
+      ask: { team: choice() },
+    });
     const store = await createPostgresJournalStore({ sql: createPostgresJournalFake() });
     const session = await createJournal({ store, now: () => 1 }).start("run", {});
     const fx = createFx({
@@ -636,7 +695,11 @@ describe("fx.decide", () => {
 
   test("pending review stamps the tenant", async () => {
     provider();
-    const decl = ai.decision("triage", { review: "ops", ask: { team: choice() } });
+    const decl = ai.decision("triage", {
+      decider: jev(),
+      otherwise: "ops",
+      ask: { team: choice() },
+    });
     const store = createMemoryJournalStore();
     const session = await createJournal({ store, now: () => 1 }).start("run", {});
     const fx = createFx({
@@ -683,12 +746,12 @@ describe("fx.decide", () => {
     try {
       const question = choice();
       const decl = ai.decision("triage", {
-        onUncertain: "abstain",
-        model: "typesafe/jev-1.13",
+        decider: jev(),
+        otherwise: "abstain",
         autonomy: { maxError: 0.05, audit: 0 },
         ask: { team: question },
       });
-      setDecisionLock({
+      setLock({
         decisions: {
           triage: {
             model: "typesafe/jev-1.13.0",
@@ -743,11 +806,12 @@ describe("fx.decide", () => {
     const team = choice();
     const urgent = ai.boolean("urgent");
     const decl = ai.decision("triage", {
-      onUncertain: "abstain",
+      decider: jev(),
+      otherwise: "abstain",
       autonomy: { maxError: 0.05, audit: 1 },
       ask: { team, urgent },
     });
-    setDecisionLock({
+    setLock({
       decisions: {
         triage: {
           model: "typesafe/jev-1.13.0",
@@ -824,7 +888,11 @@ describe("fx.decide", () => {
   test("an operator resolves another tenant and a tenant reviewer cannot", async () => {
     provider();
     gate.policy("ops", (ctx) => ctx.auth.userId === "ada" && ctx.operator.id === "reviewer");
-    const decl = ai.decision("triage", { review: "ops", ask: { team: choice() } });
+    const decl = ai.decision("triage", {
+      decider: jev(),
+      otherwise: "ops",
+      ask: { team: choice() },
+    });
     const store = await createPostgresJournalStore({ sql: createPostgresJournalFake() });
     const session = await createJournal({ store, now: () => 1 }).start("run", {});
     const fx = createFx({
@@ -872,8 +940,12 @@ describe("fx.decide", () => {
       throw new DecisionOutageError("down");
     });
     const question = choice();
-    const decl = ai.decision("triage", { review: "ops", ask: { team: question } });
-    setDecisionLock({
+    const decl = ai.decision("triage", {
+      decider: jev(),
+      otherwise: "ops",
+      ask: { team: question },
+    });
+    setLock({
       decisions: {
         triage: {
           model: "typesafe/jev-1.13.0",
@@ -908,9 +980,9 @@ describe("fx.decide", () => {
       (entry) => entry.kind === "step" && entry.name.startsWith("ai-decision:"),
     );
     expect(
-      (step && step.kind === "step" ? step.value : undefined) as { reason?: string },
+      (step && step.kind === "step" ? step.value : undefined) as { why?: string },
     ).toMatchObject({
-      reason: "outage",
+      why: "outage",
     });
   });
 
@@ -930,7 +1002,8 @@ describe("fx.decide", () => {
       usage: {},
     }));
     const decl = ai.decision("severity", {
-      onUncertain: "abstain",
+      decider: jev(),
+      otherwise: "abstain",
       ask: { severity: ai.score("How severe is this?", ["low", "medium", "high"]) },
     });
     const fx = createFx({ flow: "run", effects: { decides: ["severity"] } });
@@ -942,7 +1015,11 @@ describe("fx.decide", () => {
   });
 
   test("a missing secret is a config error", async () => {
-    const decl = ai.decision("triage", { onUncertain: "abstain", ask: { team: choice() } });
+    const decl = ai.decision("triage", {
+      decider: jev(),
+      otherwise: "abstain",
+      ask: { team: choice() },
+    });
     const fx = createFx({
       flow: "run",
       effects: { decides: ["triage"], secrets: ["OPENROUTER_API_KEY"] },