okengine 0.22.0 → 0.23.1

package/src/elements/ai/decisions/labels.test.ts

@@ -0,0 +1,282 @@
+/**
+ * Labels and the drift flag live on the journal driver.
+ */
+
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createFileJournalStore, createMemoryJournalStore } from "../../../kernel/journal.ts";
+import {
+  DecisionLabelStoreError,
+  createFileDecisionLabelStore,
+  createPostgresDecisionLabelStore,
+  setDeclaredDriftDecisions,
+} from "../../../kernel/decision-label-store.ts";
+import {
+  decisionDriftNames,
+  decisionDriftSuspended,
+  setDecisionDrift,
+  setDecisionLock,
+} from "./certificate.ts";
+import {
+  auditDriftExceeded,
+  closeDecisionLabelStore,
+  flushDecisionLabels,
+  loadDecisionLabels,
+  openDecisionLabelStore,
+  persistDecisionDrift,
+  decisionLabelWriteFailures,
+  persistDecisionLabel,
+} from "./labels.ts";
+
+afterEach(() => {
+  closeDecisionLabelStore();
+  for (const name of decisionDriftNames()) setDecisionDrift(name, false);
+  setDecisionLock(undefined);
+});
+
+describe("decision labels", () => {
+  test("labels and the suspension flag survive a reopened journal", async () => {
+    const root = await mkdtemp(join(tmpdir(), "oke-labels-"));
+    const path = join(root, "journal.json");
+    const first = createFileJournalStore(path);
+    await openDecisionLabelStore(first, setDecisionDrift);
+    persistDecisionLabel({
+      decision: "triage",
+      question: "team",
+      value: "billing",
+      propensity: 1,
+      reviewer: "a",
+      tenant: "acme",
+      model: "jev",
+      score: 0.4,
+      loss: 1,
+      at: 1,
+    });
+    persistDecisionDrift("triage", true);
+    await flushDecisionLabels();
+    closeDecisionLabelStore();
+    setDecisionDrift("triage", false);
+    const second = createFileJournalStore(path);
+    await openDecisionLabelStore(second, setDecisionDrift);
+    const saved = await loadDecisionLabels("triage", "acme");
+    expect(saved).toHaveLength(1);
+    expect(saved[0]?.at).toBe(1);
+    expect(await loadDecisionLabels("triage", "other")).toHaveLength(0);
+    expect(await loadDecisionLabels("triage")).toHaveLength(1);
+    expect(decisionDriftSuspended("triage")).toBe(true);
+  });
+
+  test("one correct audit label never suspends", async () => {
+    const journal = createMemoryJournalStore();
+    await openDecisionLabelStore(journal, setDecisionDrift);
+    persistDecisionLabel({
+      decision: "triage",
+      question: "team",
+      value: "technical",
+      propensity: 0.1,
+      reviewer: "audit",
+      model: "typesafe/jev-1.13.0",
+      loss: 0,
+      at: Date.now(),
+    });
+    await flushDecisionLabels();
+    expect(
+      auditDriftExceeded({
+        maxError: 0.05,
+        model: "typesafe/jev-1.13.0",
+        since: 0,
+        labels: await loadDecisionLabels("triage"),
+      }),
+    ).toBe(false);
+  });
+
+  test("audit drift is a one-sided binomial on the pinned model since the certificate", async () => {
+    const journal = createMemoryJournalStore();
+    await openDecisionLabelStore(journal, setDecisionDrift);
+    const now = Date.now();
+    for (let i = 0; i < 30; i++) {
+      persistDecisionLabel({
+        decision: "triage",
+        question: "team",
+        value: "billing",
+        propensity: 0.1,
+        reviewer: "audit",
+        model: "typesafe/jev-1.13.0",
+        loss: 1,
+        at: now,
+      });
+    }
+    persistDecisionLabel({
+      decision: "triage",
+      question: "team",
+      value: "billing",
+      propensity: 0.1,
+      reviewer: "audit",
+      model: "other-model",
+      loss: 1,
+      at: now,
+    });
+    await flushDecisionLabels();
+    expect(
+      auditDriftExceeded({
+        maxError: 0.05,
+        model: "typesafe/jev-1.13.0",
+        since: now - 1000,
+        now,
+        labels: await loadDecisionLabels("triage"),
+      }),
+    ).toBe(true);
+    expect(
+      auditDriftExceeded({
+        maxError: 0.05,
+        model: "typesafe/jev-1.13.0",
+        since: now + 1,
+        now,
+        labels: await loadDecisionLabels("triage"),
+      }),
+    ).toBe(false);
+  });
+
+  test("a newer certificate ignores a drift flag raised against the previous one", async () => {
+    const journal = createMemoryJournalStore();
+    await openDecisionLabelStore(journal, setDecisionDrift);
+    persistDecisionDrift("triage", true, 100);
+    await flushDecisionLabels();
+    closeDecisionLabelStore();
+    setDecisionDrift("triage", false);
+    setDecisionLock({
+      decisions: {
+        triage: { model: "typesafe/jev-1.13.0", certifiedAt: 200, questions: {} },
+      },
+    });
+    await openDecisionLabelStore(journal, setDecisionDrift);
+    expect(decisionDriftSuspended("triage")).toBe(false);
+  });
+
+  test("promoting one decision clears only that decision", async () => {
+    const journal = createMemoryJournalStore();
+    await openDecisionLabelStore(journal, setDecisionDrift);
+    persistDecisionDrift("ship", true, 1);
+    persistDecisionDrift("triage", true, 1);
+    await flushDecisionLabels();
+    persistDecisionDrift("ship", false);
+    await flushDecisionLabels();
+    closeDecisionLabelStore();
+    for (const name of decisionDriftNames()) setDecisionDrift(name, false);
+    await openDecisionLabelStore(journal, setDecisionDrift);
+    expect(decisionDriftSuspended("ship")).toBe(false);
+    expect(decisionDriftSuspended("triage")).toBe(true);
+  });
+
+  test("one failed insert does not drop the next write", async () => {
+    const journal = createMemoryJournalStore();
+    const decisions = journal.decisions;
+    if (!decisions) throw new Error("expected a label store");
+    let calls = 0;
+    const original = decisions.insert.bind(decisions);
+    decisions.insert = async (label, at) => {
+      calls += 1;
+      if (calls === 1) throw new Error("insert failed");
+      await original(label, at);
+    };
+    await openDecisionLabelStore(journal, setDecisionDrift);
+    persistDecisionLabel({
+      decision: "triage",
+      question: "team",
+      value: "a",
+      propensity: 1,
+      reviewer: "a",
+    });
+    persistDecisionLabel({
+      decision: "triage",
+      question: "team",
+      value: "b",
+      propensity: 1,
+      reviewer: "a",
+    });
+    await flushDecisionLabels();
+    const rows = await loadDecisionLabels("triage");
+    expect(rows.map((row) => row.value)).toEqual(["b"]);
+    expect(decisionLabelWriteFailures().map((event) => event.message)).toEqual(["insert failed"]);
+  });
+
+  test("a postgres label store throws when init fails", async () => {
+    const sql = {
+      async query(): Promise<Record<string, unknown>[]> {
+        throw new Error("query failed");
+      },
+      async exec(): Promise<{ changes: number }> {
+        throw new Error("exec failed");
+      },
+    };
+    await expect(createPostgresDecisionLabelStore(sql)).rejects.toBeInstanceOf(
+      DecisionLabelStoreError,
+    );
+  });
+
+  test("postgres null tenant matches, and an older drift table gains certified_at", async () => {
+    const { connectPglite } = await import("../../../drivers/pglite.ts");
+    const db = await connectPglite({ url: "memory://decision-labels" });
+    try {
+      await db.exec(`CREATE TABLE oke_decision_drift (
+        id INTEGER PRIMARY KEY,
+        suspended INTEGER NOT NULL
+      )`);
+      await db.exec(`INSERT INTO oke_decision_drift (id, suspended) VALUES (1, 1)`);
+      setDeclaredDriftDecisions(["ship", "triage"]);
+      const store = await createPostgresDecisionLabelStore(db);
+      await store.insert(
+        {
+          decision: "triage",
+          question: "team",
+          value: "billing",
+          propensity: 1,
+          reviewer: "a",
+        },
+        1,
+      );
+      await store.insert(
+        {
+          decision: "triage",
+          question: "team",
+          value: "technical",
+          propensity: 1,
+          reviewer: "a",
+          tenant: "acme",
+        },
+        2,
+      );
+      const unlabeled = await store.list("triage", null);
+      const acme = await store.list("triage", "acme");
+      expect(unlabeled.map((row) => row.value)).toEqual(["billing"]);
+      expect(acme.map((row) => row.value)).toEqual(["technical"]);
+      expect(await store.drift()).toEqual({
+        ship: { suspended: true, certifiedAt: 0 },
+        triage: { suspended: true, certifiedAt: 0 },
+      });
+    } finally {
+      await db.close();
+    }
+  });
+
+  test("a file store copies a legacy star flag onto every declared decision", async () => {
+    const root = await mkdtemp(join(tmpdir(), "oke-drift-"));
+    const path = join(root, "decision-labels.json");
+    await Bun.write(
+      path,
+      JSON.stringify({
+        rows: [],
+        drifts: { "*": { suspended: true, certifiedAt: 4 } },
+      }),
+    );
+    setDeclaredDriftDecisions(["triage", "ship"]);
+    const store = createFileDecisionLabelStore(path);
+    expect(await store.drift()).toEqual({
+      triage: { suspended: true, certifiedAt: 4 },
+      ship: { suspended: true, certifiedAt: 4 },
+    });
+    setDeclaredDriftDecisions([]);
+  });
+});

package/src/elements/ai/decisions/labels.ts

@@ -0,0 +1,257 @@
+/**
+ * Decision labels and the app drift flag on the journal driver.
+ * Boot opens the store only when the app declares decisions.
+ */
+
+import { AsyncLocalStorage } from "node:async_hooks";
+import type { JournalStore } from "../../../kernel/journal.ts";
+import type { RunTelemetry } from "../../../kernel/run-telemetry.ts";
+import type {
+  DecisionDriftRecord,
+  DecisionLabelStore,
+} from "../../../kernel/decision-label-store.ts";
+import {
+  binomialCdf,
+  decisionLabels,
+  getDecisionLock,
+  type DecisionCandidate,
+  type DecisionLabel,
+} from "./certificate.ts";
+
+/** Rolling window for audit drift. Labels older than this are ignored. */
+export const DECISION_DRIFT_WINDOW_MS = 7 * 24 * 60 * 60 * 1000;
+
+let store: DecisionLabelStore | undefined;
+let writes: Promise<void> = Promise.resolve();
+
+/** Trace log attached to the run that queued the write. */
+const telemetryScope = new AsyncLocalStorage<RunTelemetry>();
+
+/** A failed label write. The chain continues; the error is not dropped. */
+export interface DecisionLabelWriteFailure {
+  readonly decision: string;
+  readonly question: string;
+  readonly message: string;
+  readonly at: number;
+}
+
+const writeFailures: DecisionLabelWriteFailure[] = [];
+
+/**
+ * Recent label-write failures. Console reads this for the decisions page.
+ */
+export function decisionLabelWriteFailures(): readonly DecisionLabelWriteFailure[] {
+  return writeFailures;
+}
+
+/**
+ * Attach this run's telemetry so a failed label write lands on its trace.
+ *
+ * @param telemetry - Collector for the current invocation
+ */
+export function bindDecisionLabelTelemetry(telemetry: RunTelemetry): void {
+  telemetryScope.enterWith(telemetry);
+}
+
+/**
+ * Bind the journal's decision store and load the drift flag.
+ *
+ * @param journal - Journal that owns the label table
+ * @param setDrift - Install the loaded flag
+ */
+export async function openDecisionLabelStore(
+  journal: JournalStore,
+  setDrift: (name: string, suspended: boolean) => void,
+): Promise<void> {
+  await writes;
+  store = journal.decisions;
+  if (!store) return;
+  const flags = await store.drift();
+  for (const [name, flag] of Object.entries(flags)) {
+    const pinned = pinnedDecision(name);
+    const suspended = flag.suspended && (pinned?.since ?? 0) <= flag.certifiedAt;
+    setDrift(name, suspended);
+  }
+}
+
+/**
+ * Wait until label and drift writes have reached the journal driver.
+ */
+export function flushDecisionLabels(): Promise<void> {
+  return writes;
+}
+
+/**
+ * Drop the bound store. Tests use this.
+ */
+export function closeDecisionLabelStore(): void {
+  store = undefined;
+  writeFailures.length = 0;
+}
+
+/**
+ * Write one label. Tenant scopes the row. The candidate job reads every tenant.
+ *
+ * @param label - Resolved label. `label.at` is the stored time when set.
+ * @param at - Epoch ms used only when `label.at` is absent
+ */
+export function persistDecisionLabel(label: DecisionLabel, at?: number): void {
+  const stamped = label.at ?? at ?? Date.now();
+  const row = { ...label, at: stamped };
+  if (!store) return;
+  const target = store;
+  const write = (): Promise<void> =>
+    target.insert(row, stamped).then(
+      () => undefined,
+      (error: unknown) => {
+        noteLabelWriteFailure(row, error);
+      },
+    );
+  writes = writes.then(write, write);
+}
+
+/**
+ * Write a candidate onto the journal driver.
+ *
+ * @param decision - Decision name
+ * @param entry - Fitted lock entry
+ */
+export async function persistDecisionCandidate(
+  decision: string,
+  entry: DecisionCandidate,
+): Promise<void> {
+  await store?.putCandidate(decision, entry);
+}
+
+/**
+ * Read one candidate from the journal driver.
+ *
+ * @param decision - Decision name
+ */
+export async function loadDecisionCandidate(
+  decision: string,
+): Promise<DecisionCandidate | undefined> {
+  return store?.getCandidate(decision);
+}
+
+/**
+ * Decision names that have a candidate on the journal driver.
+ */
+export async function listDecisionCandidates(): Promise<readonly string[]> {
+  return (await store?.listCandidates()) ?? [];
+}
+
+/**
+ * Record a failed label write on the run trace and for the decisions page.
+ * The returned promise still fulfills so the next write runs.
+ *
+ * @param label - Row that did not persist
+ * @param error - Driver error
+ */
+function noteLabelWriteFailure(label: DecisionLabel, error: unknown): void {
+  const message = error instanceof Error ? error.message : "decision label write failed";
+  const at = Date.now();
+  writeFailures.push({
+    decision: label.decision,
+    question: label.question,
+    message,
+    at,
+  });
+  const telemetry = telemetryScope.getStore();
+  if (!telemetry) return;
+  telemetry.logs.push({
+    level: "error",
+    message: "decision label write failed",
+    data: { decision: label.decision, question: label.question, error: message },
+    at,
+  });
+}
+
+/**
+ * Labels for one decision. Omit `tenant` to read every tenant.
+ * Reads the journal mirror when one is open, otherwise in-memory review labels.
+ *
+ * @param decision - Decision name
+ * @param tenant - Tenant filter
+ */
+export async function loadDecisionLabels(
+  decision?: string,
+  tenant?: string | null,
+): Promise<DecisionLabel[]> {
+  if (store) return store.list(decision, tenant);
+  return decisionLabels().filter((label) => {
+    if (decision !== undefined && label.decision !== decision) return false;
+    if (tenant !== undefined && (label.tenant ?? null) !== tenant) return false;
+    return true;
+  });
+}
+
+/**
+ * Compare audit labels for the pinned model to the certified error cap.
+ * Only labels since the certificate, inside the rolling window, count.
+ * One-sided exact binomial. A single correct label does not suspend.
+ *
+ * @param options - Cap, pinned model, and certificate time
+ */
+export function auditDriftExceeded(options: {
+  readonly maxError: number;
+  readonly delta?: number;
+  readonly model: string;
+  readonly since: number;
+  readonly now?: number;
+  readonly windowMs?: number;
+  readonly labels: readonly DecisionLabel[];
+}): boolean {
+  const delta = options.delta ?? 0.1;
+  const now = options.now ?? Date.now();
+  const windowMs = options.windowMs ?? DECISION_DRIFT_WINDOW_MS;
+  const floor = Math.max(options.since, now - windowMs);
+  const audits = options.labels.filter((label) => {
+    if (!(label.propensity < 1) || label.loss === undefined) return false;
+    if (label.model !== options.model) return false;
+    const at = label.at ?? 0;
+    return at >= floor;
+  });
+  if (audits.length === 0) return false;
+  const errors = audits.filter((label) => (label.loss ?? 0) > 0).length;
+  const tail = 1 - binomialCdf(errors - 1, audits.length, options.maxError);
+  return tail <= delta;
+}
+
+/**
+ * Persist one decision's suspension flag.
+ *
+ * @param name - Decision name
+ * @param suspended - Drift detected for that decision
+ * @param certifiedAt - Certificate time the flag was raised against
+ */
+export function persistDecisionDrift(name: string, suspended: boolean, certifiedAt = 0): void {
+  if (!store) return;
+  const target = store;
+  const record: DecisionDriftRecord = { suspended, certifiedAt };
+  const write = async (): Promise<void> => {
+    try {
+      await target.setDrift(name, record);
+    } catch (error) {
+      noteLabelWriteFailure(
+        { decision: name, question: "drift", value: null, propensity: 0, reviewer: "drift" },
+        error,
+      );
+      throw error;
+    }
+  };
+  writes = writes.then(write, write);
+}
+
+/**
+ * Pinned model and certificate time for one decision, when a lock exists.
+ *
+ * @param name - Decision name
+ */
+export function pinnedDecision(
+  name: string,
+): { readonly model: string; readonly since: number } | undefined {
+  const entry = getDecisionLock()?.decisions[name];
+  if (!entry) return undefined;
+  return { model: entry.model, since: entry.certifiedAt ?? 0 };
+}

package/src/elements/ai/decisions/openrouter.live.test.ts

@@ -0,0 +1,136 @@
+/**
+ * Opt-in OpenRouter decision smoke check.
+ *
+ * Skipped unless `OPENROUTER_API_KEY` is set. `bun run test` (CI) ignores
+ * `*.live.test.ts`. The key is never printed, logged, or written.
+ */
+
+import { afterEach, describe, expect, test } from "bun:test";
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { ai, resetAiDecls } from "../../ai.ts";
+import { createFx } from "../../../kernel/fx.ts";
+import { resetDecisionCertificates } from "./certificate.ts";
+import { OPENROUTER_DECISION_URL } from "./openrouter.ts";
+import { resetDecisionProvider } from "../../../kernel/fx-decide.ts";
+
+function openRouterKey(): string | undefined {
+  const fromEnv = process.env["OPENROUTER_API_KEY"];
+  if (fromEnv && fromEnv.length > 0) return fromEnv;
+  try {
+    const text = readFileSync(resolve(import.meta.dir, "../../../../.env.local"), "utf8");
+    for (const line of text.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed.startsWith("OPENROUTER_API_KEY=")) continue;
+      const value = trimmed.slice("OPENROUTER_API_KEY=".length).replace(/^["']|["']$/g, "");
+      if (value.length > 0) return value;
+    }
+  } catch {
+    return undefined;
+  }
+  return undefined;
+}
+
+const apiKey = process.env["CI"] === "true" ? undefined : openRouterKey();
+
+afterEach(() => {
+  resetAiDecls();
+  resetDecisionCertificates();
+  resetDecisionProvider();
+});
+
+function decision() {
+  return ai.decision("triage-live", {
+    onUncertain: "abstain",
+    ask: {
+      team: ai.choice("Which team should own this ticket?", {
+        billing: "Payments and refunds",
+        technical: "Bugs and outages",
+      }),
+      severity: ai.score("How severe is this?", ["low", "medium", "high"]),
+      urgent: ai.boolean("Does this need a person today?"),
+    },
+  });
+}
+
+function fx() {
+  return createFx({
+    flow: "triage.live",
+    effects: { decides: ["triage-live"], secrets: ["OPENROUTER_API_KEY"] },
+    secrets: { OPENROUTER_API_KEY: apiKey ?? "" },
+  });
+}
+
+describe.skipIf(apiKey === undefined)("openrouter decisions live", () => {
+  test("fx.decide hits /api/alpha/decisions and the body matches the fixture fields", async () => {
+    const original = globalThis.fetch;
+    let seenUrl = "";
+    let raw: unknown;
+    globalThis.fetch = (async (input, init) => {
+      seenUrl = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
+      const res = await original(input, init);
+      const text = await res.clone().text();
+      raw = JSON.parse(text) as unknown;
+      return res;
+    }) as typeof fetch;
+    try {
+      const decl = decision();
+      const result = (await fx().decide(decl, {
+        ticket: "Checkout is blank after Pay.",
+      })) as {
+        $: {
+          team: { raw: unknown };
+          severity: { raw: unknown };
+          urgent: { raw: unknown };
+        };
+      };
+      expect(seenUrl).toBe(OPENROUTER_DECISION_URL);
+      expect(raw && typeof raw === "object").toBe(true);
+      const body = raw as {
+        model?: unknown;
+        answers?: Record<string, Record<string, unknown>>;
+        usage?: Record<string, unknown>;
+      };
+      expect(typeof body.model).toBe("string");
+      expect(String(body.model)).toMatch(/\d+\.\d+/);
+      const team = body.answers?.team;
+      const severity = body.answers?.severity;
+      const urgent = body.answers?.urgent;
+      expect(team?.type).toBe("choice");
+      expect(team?.probabilities && typeof team.probabilities).toBe("object");
+      expect(typeof team?.confidence).toBe("number");
+      expect(severity?.type).toBe("score");
+      expect(severity?.probabilities && typeof severity.probabilities).toBe("object");
+      expect(severity?.legend && typeof severity.legend).toBe("object");
+      expect(typeof severity?.confidence).toBe("number");
+      expect(urgent?.type).toBe("noul");
+      expect(typeof urgent?.noul).toBe("number");
+      expect(body.usage && typeof body.usage).toBe("object");
+      expect(result.$.team.raw).toBeDefined();
+
+      const english = (await fx().decide(decl, {
+        ticket: "Checkout is blank after Pay.",
+      })) as typeof result;
+      const arabic = (await fx().decide(decl, {
+        ticket: "صفحة الدفع فاضية بعد ما دفعت.",
+      })) as typeof result;
+      console.log(
+        "openrouter probabilities",
+        JSON.stringify({
+          en: {
+            team: english.$.team.raw,
+            severity: english.$.severity.raw,
+            urgent: english.$.urgent.raw,
+          },
+          ar: {
+            team: arabic.$.team.raw,
+            severity: arabic.$.severity.raw,
+            urgent: arabic.$.urgent.raw,
+          },
+        }),
+      );
+    } finally {
+      globalThis.fetch = original;
+    }
+  }, 120_000);
+});