okengine 0.21.1 → 0.23.0

package/src/cli/eval.ts

@@ -4,6 +4,7 @@
 
 import { resolve } from "node:path";
 import { parseEvalJsonl, runPromptEvals, type EvalCase } from "../elements/ai/eval.ts";
+import type { DecisionResponse } from "../elements/ai/decisions/provider.ts";
 import type { Manifest } from "../manifest/types.ts";
 
 /** Options for {@link runOkeEval}. */
@@ -96,17 +97,157 @@ export async function runOkeEval(options: OkeEvalOptions = {}): Promise<number>
  */
 export async function evalCli(args: string[]): Promise<number> {
   let manifestPath: string | undefined;
+  let certify = false;
   for (let i = 0; i < args.length; i++) {
     const a = args[i]!;
     if (a === "--manifest" || a === "-m") {
       manifestPath = args[++i];
+    } else if (a === "--certify") {
+      certify = true;
     } else if (a === "--help" || a === "-h") {
-      console.log(`oke eval [--manifest oke.manifest.json]
+      console.log(`oke eval [--manifest oke.manifest.json] [--certify]
 
 Run prompt eval sets declared in the Manifest. Fails CI on regression.
+--certify builds a decision certificate from each decision seed file.
 `);
       return 0;
     }
   }
+  if (certify) return runOkeCertify({ manifestPath });
   return runOkeEval({ manifestPath });
 }
+
+/**
+ * Build certificates from decision seed files. Prompt evals are not run.
+ *
+ * @param options - Manifest path
+ */
+export async function runOkeCertify(
+  options: {
+    readonly manifestPath?: string;
+    readonly manifest?: Manifest;
+    readonly root?: string;
+    readonly evaluate?: (input: unknown) => Promise<DecisionResponse>;
+  } = {},
+): Promise<number> {
+  const { certifySeed } = await import("../elements/ai/decisions/certify.ts");
+  const { aiDecisionRegistry } = await import("../kernel/element-registries.ts");
+  const { DECISION_LOCK_FILENAME, parseDecisionLockfile } =
+    await import("../elements/ai/decisions/certificate.ts");
+  const { resolveStartEntry } = await import("./start.ts");
+  const root = resolve(options.root ?? process.env["OKE_ROOT_DIR"] ?? ".");
+  process.env["OKE_ROOT_DIR"] ??= root;
+  let manifest = options.manifest;
+  if (!manifest) {
+    try {
+      const entry = await resolveStartEntry(root);
+      await import(entry);
+    } catch (err) {
+      console.error(
+        `oke eval: failed to load the app: ${err instanceof Error ? err.message : String(err)}`,
+      );
+      return 1;
+    }
+    const path = resolve(root, options.manifestPath ?? "oke.manifest.json");
+    const file = Bun.file(path);
+    if (!(await file.exists())) {
+      console.error(`oke eval: manifest not found: ${path}`);
+      return 1;
+    }
+    manifest = (await file.json()) as Manifest;
+  }
+  const decisions = manifest.ai?.decisions ?? {};
+  const names = Object.keys(decisions);
+  if (names.length === 0) {
+    process.stdout.write("oke eval: no decisions declared\n");
+    return 0;
+  }
+  const injected = options.evaluate;
+  const lockPath = resolve(root, DECISION_LOCK_FILENAME);
+  const existingFile = Bun.file(lockPath);
+  const current = (await existingFile.exists())
+    ? parseDecisionLockfile(await existingFile.json())
+    : undefined;
+  const next = { decisions: { ...(current?.decisions ?? {}) } };
+  for (const name of names) {
+    const decision = decisions[name];
+    if (!decision?.evals) {
+      process.stdout.write(`oke eval: skip ${name} (no evals)\n`);
+      continue;
+    }
+    const decl = aiDecisionRegistry.find((item) => item.name === name);
+    if (!decl) {
+      console.error(`oke eval: decision "${name}" is not loaded`);
+      return 1;
+    }
+    const text = await Bun.file(resolve(root, decision.evals)).text();
+    const evaluate = injected ?? (await providerForDecision(decl));
+    next.decisions[name] = await certifySeed({
+      model: decl.model ?? decision.model ?? "",
+      maxError: decision.autonomy?.maxError ?? 0.05,
+      delta: decision.autonomy?.risk ?? decl.autonomy?.risk ?? 0.1,
+      jsonl: text,
+      ask: decl.ask,
+      evaluate,
+    });
+  }
+  await Bun.write(lockPath, `${JSON.stringify(next, null, 2)}\n`);
+  const { setDecisionDrift } = await import("../elements/ai/decisions/certificate.ts");
+  const { persistDecisionDrift } = await import("../elements/ai/decisions/labels.ts");
+  for (const name of names) {
+    setDecisionDrift(name, false);
+    persistDecisionDrift(name, false);
+  }
+  return 0;
+}
+
+async function providerForDecision(
+  decl: import("../elements/ai/declare.ts").AiDecisionDecl,
+): Promise<(input: unknown) => Promise<DecisionResponse>> {
+  const typesafe = decl.driverId === "typesafe";
+  const keyName = typesafe ? "TYPESAFE_API_KEY" : "OPENROUTER_API_KEY";
+  const apiKey = process.env[keyName];
+  if (!apiKey) throw new Error(`oke eval: secret "${keyName}" is not configured`);
+  const { createTypesafeDecisionProvider, TYPESAFE_JEV_MODEL } =
+    await import("../elements/ai/decisions/typesafe.ts");
+  const { createOpenRouterDecisionProvider, OPENROUTER_JEV_MODEL } =
+    await import("../elements/ai/decisions/openrouter.ts");
+  const provider = typesafe
+    ? createTypesafeDecisionProvider(apiKey)
+    : createOpenRouterDecisionProvider(apiKey);
+  const model = decl.model ?? (typesafe ? TYPESAFE_JEV_MODEL : OPENROUTER_JEV_MODEL);
+  return (input) =>
+    provider.evaluate({
+      model,
+      state: input,
+      questions: wireFromDecl(decl),
+    });
+}
+
+function wireFromDecl(
+  decl: import("../elements/ai/declare.ts").AiDecisionDecl,
+): import("../elements/ai/decisions/provider.ts").DecisionRequest["questions"] {
+  const questions: Record<string, import("../elements/ai/decisions/provider.ts").WireQuestion> = {};
+  for (const [id, question] of Object.entries(decl.ask)) {
+    if (question.kind === "choice") {
+      questions[id] = {
+        type: "choice",
+        instructions: question.instructions,
+        criteria: { ...question.options, none_of_these: null },
+      };
+    } else if (question.kind === "score") {
+      questions[id] = {
+        type: "score",
+        instructions: question.instructions,
+        criteria: question.levels,
+      };
+    } else {
+      questions[id] = {
+        type: "noul",
+        instructions: question.instructions,
+        ...(question.criteria !== undefined ? { criteria: question.criteria } : {}),
+      };
+    }
+  }
+  return questions;
+}

package/src/cli/index.ts

@@ -5,6 +5,7 @@
  * Bare `oke` (TTY) → interactive Ink TUI. `oke <command>` runs directly.
  */
 
+import type { VaultAuditConfig } from "../elements/vault/types.ts";
 import { aiCli } from "./ai.ts";
 import { branchCli } from "./branch.ts";
 import { buildCli } from "./build.ts";
@@ -15,6 +16,7 @@ import { dbCli } from "./db.ts";
 import { devCli } from "./dev.ts";
 import { doctorCli } from "./doctor.ts";
 import { dockerCli } from "./docker.ts";
+import { decideCli } from "./decide.ts";
 import { evalCli } from "./eval.ts";
 import { EXIT_CODE_HELP, EXIT_OK, EXIT_USAGE } from "./exit.ts";
 import { gatesListCli } from "./gates-list.ts";
@@ -28,6 +30,7 @@ import { stackCli } from "./stack.ts";
 import { startCli } from "./start.ts";
 import { testCli } from "./test.ts";
 import { upgradeCli } from "./upgrade.ts";
+import { loadOkeConfig } from "./load-config.ts";
 import { vaultCli } from "./vault-cmd.ts";
 
 const rawArgv = process.argv.slice(2);
@@ -90,7 +93,13 @@ if (cmd === "client" && sub === "add") {
 }
 
 if (cmd === "vault") {
-  process.exit(await vaultCli(sub ? [sub, ...rest] : rest));
+  let audit: VaultAuditConfig | undefined;
+  try {
+    audit = (await loadOkeConfig()).config.vault?.audit;
+  } catch {
+    audit = undefined;
+  }
+  process.exit(await vaultCli(sub ? [sub, ...rest] : rest, audit === undefined ? {} : { audit }));
 }
 
 if (cmd === "docker") {
@@ -109,6 +118,10 @@ if (cmd === "eval") {
   process.exit(await evalCli(sub ? [sub, ...rest] : rest));
 }
 
+if (cmd === "decide") {
+  process.exit(await decideCli(sub ? [sub, ...rest] : rest));
+}
+
 if (cmd === "ai") {
   process.exit(await aiCli(sub ? [sub, ...rest] : rest));
 }

package/src/cli/load-config.images.test.ts

@@ -18,19 +18,19 @@ describe("flattenImagesConfig", () => {
           sql: "postgres:18-alpine",
           kv: "redis:8-alpine",
           files: "rustfs/rustfs:1.0.0",
-          index: "getmeili/meilisearch:v1.53",
+          index: "getmeili/meilisearch:v1.54",
         },
-        channel: { email: "axllent/mailpit:v1.31.1" },
-        pgdog: "ghcr.io/pgdogdev/pgdog:v0.1.57",
+        channel: { email: "axllent/mailpit:v1.31.2" },
+        pgdog: "ghcr.io/pgdogdev/pgdog:v0.1.59",
         proxy: "caddy:2-alpine",
       }),
     ).toEqual({
       "store.sql": "postgres:18-alpine",
       "store.kv": "redis:8-alpine",
       "store.files": "rustfs/rustfs:1.0.0",
-      "store.index": "getmeili/meilisearch:v1.53",
-      "channel.email": "axllent/mailpit:v1.31.1",
-      pgdog: "ghcr.io/pgdogdev/pgdog:v0.1.57",
+      "store.index": "getmeili/meilisearch:v1.54",
+      "channel.email": "axllent/mailpit:v1.31.2",
+      pgdog: "ghcr.io/pgdogdev/pgdog:v0.1.59",
       proxy: "caddy:2-alpine",
     });
   });
@@ -53,8 +53,8 @@ describe("defaultImagesFromConfig", () => {
     });
     expect(images).toEqual({
       "store.sql": "postgres:18-alpine",
-      pgdog: "ghcr.io/pgdogdev/pgdog:v0.1.57",
-      "store.kv": "redis:8-alpine",
+      pgdog: "ghcr.io/pgdogdev/pgdog:v0.1.59",
+      "store.kv": "docker.dragonflydb.io/dragonflydb/dragonfly:v2.0.0",
     });
   });
 
@@ -68,7 +68,7 @@ describe("defaultImagesFromConfig", () => {
       },
     });
     expect(images["store.sql"]).toBe("pgvector/pgvector:pg17");
-    expect(images.pgdog).toBe("ghcr.io/pgdogdev/pgdog:v0.1.57");
+    expect(images.pgdog).toBe("ghcr.io/pgdogdev/pgdog:v0.1.59");
   });
 
   test("clock postgres alone pulls store.sql image", () => {
@@ -79,7 +79,7 @@ describe("defaultImagesFromConfig", () => {
     });
     expect(images).toEqual({
       "store.sql": "postgres:18-alpine",
-      pgdog: "ghcr.io/pgdogdev/pgdog:v0.1.57",
+      pgdog: "ghcr.io/pgdogdev/pgdog:v0.1.59",
     });
   });
 
@@ -91,7 +91,7 @@ describe("defaultImagesFromConfig", () => {
     });
     expect(images).toEqual({
       "store.sql": "postgres:18-alpine",
-      pgdog: "ghcr.io/pgdogdev/pgdog:v0.1.57",
+      pgdog: "ghcr.io/pgdogdev/pgdog:v0.1.59",
     });
   });
 
@@ -129,7 +129,7 @@ describe("resolveImages", () => {
       },
     });
     expect(images["store.sql"]).toBe("postgres:18-alpine");
-    expect(images["store.kv"]).toBe("redis:8-alpine");
+    expect(images["store.kv"]).toBe("docker.dragonflydb.io/dragonflydb/dragonfly:v2.0.0");
     expect(images["store.kv.durable"]).toBeUndefined();
   });
 });

package/src/cli/load-config.ts

@@ -15,8 +15,8 @@ import type { Manifest } from "../manifest/types.ts";
 /** Default image pins when `images` is omitted but prod drivers need containers. */
 const DEFAULT_SQL_IMAGE = "postgres:18-alpine";
 const DEFAULT_PGVECTOR_IMAGE = "pgvector/pgvector:pg17";
-const DEFAULT_PGDOG_IMAGE = "ghcr.io/pgdogdev/pgdog:v0.1.57";
-const DEFAULT_KV_IMAGE = "redis:8-alpine";
+const DEFAULT_PGDOG_IMAGE = "ghcr.io/pgdogdev/pgdog:v0.1.59";
+const DEFAULT_KV_IMAGE = "docker.dragonflydb.io/dragonflydb/dragonfly:v2.0.0";
 
 /**
  * Prefer `dev` then `prod` for a driver map (compose infra ≈ production).

package/src/cli/mcp-from-console.ts

@@ -22,6 +22,10 @@ export interface McpConsoleSurface {
   readonly cwd: string;
   /** Clock shared with Console. */
   readonly now: () => number;
+  /** Journal that holds agent events and parked approvals. */
+  readonly journalStore?: import("../kernel/journal.ts").JournalStore | null;
+  /** In-process agent ledger. The follow log is the durable source. */
+  readonly aiRuntime?: import("../elements/ai.ts").AiRuntime | null;
 }
 
 /**
@@ -33,6 +37,50 @@ export function mcpContextFromConsole(state: McpConsoleSurface): McpContext {
   return {
     getManifest: () => state.manifest,
     listRuns: () => state.listRuns(),
+    listAgentRuns: async () => {
+      const { loadConsoleAgentRuns } = await import("../console/server/ai-runs.ts");
+      return loadConsoleAgentRuns({
+        runtime: state.aiRuntime ?? null,
+        store: state.journalStore ?? null,
+        now: state.now(),
+      });
+    },
+    getAgentRun: async (runId) => {
+      const { loadConsoleAgentRun } = await import("../console/server/ai-runs.ts");
+      const run = await loadConsoleAgentRun(
+        {
+          runtime: state.aiRuntime ?? null,
+          store: state.journalStore ?? null,
+          now: state.now(),
+        },
+        runId,
+      );
+      if (!run) return undefined;
+      return { run, events: run.events };
+    },
+    listApprovals: async () => {
+      const { loadApprovalQueue } = await import("../console/server/ai-runs.ts");
+      return loadApprovalQueue(state.journalStore ?? null, state.now());
+    },
+    listDecisions: async (tenant) => {
+      const { loadDecisionQueue, projectMcpDecisions, candidateMetrics } =
+        await import("../console/server/decisions.ts");
+      const { listDecisionCandidates, loadDecisionCandidate } =
+        await import("../elements/ai/decisions/labels.ts");
+      const names = await listDecisionCandidates();
+      const fitted: Record<string, Record<string, number>> = {};
+      for (const name of names) {
+        const metrics = candidateMetrics(await loadDecisionCandidate(name));
+        if (metrics) fitted[name] = metrics;
+      }
+      const queue = await loadDecisionQueue(state.journalStore ?? null, state.now(), tenant);
+      const pending: Record<string, number> = {};
+      for (const row of queue) {
+        if (row.status !== "pending" || row.labelOnly) continue;
+        pending[row.decision] = (pending[row.decision] ?? 0) + 1;
+      }
+      return projectMcpDecisions(state.manifest, pending, fitted);
+    },
     proposeStructural: async (input) =>
       emitStructuralDiff({
         cwd: state.cwd,

package/src/cli/registry.ts

@@ -550,7 +550,41 @@ export const OKE_COMMANDS: readonly CliCommand[] = [
     name: "eval",
     summary: "prompt eval sets (CI gate)",
     leaf: true,
-    flags: [MANIFEST, HELP],
+    flags: [
+      MANIFEST,
+      {
+        long: "--certify",
+        takesValue: false,
+        summary: "Build a decision certificate from the seed file",
+      },
+      HELP,
+    ],
+  },
+  {
+    name: "decide",
+    summary: "promote a decision certificate",
+    subcommands: [
+      {
+        name: "promote",
+        summary: "Fetch the operator candidate and write the app lockfile",
+        positionals: "<name>",
+        flags: [
+          {
+            long: "--origin",
+            takesValue: true,
+            valueName: "url",
+            summary: "App origin that serves the candidate",
+          },
+          {
+            long: "--lock",
+            takesValue: true,
+            valueName: "path",
+            summary: "Lockfile path (default oke-decisions.lock.json)",
+          },
+          HELP,
+        ],
+      },
+    ],
   },
   {
     name: "ai",

package/src/cli/vault-cmd.test.ts

@@ -282,6 +282,29 @@ describe("oke vault audit", () => {
       await h.close();
     }
   });
+
+  test("audit verify refuses a webhook sink", async () => {
+    const sql = createMemoryVaultSql();
+    const errors: string[] = [];
+    const original = console.error;
+    console.error = (...args: unknown[]) => {
+      errors.push(args.map((part) => String(part)).join(" "));
+    };
+    try {
+      const code = await vaultCli(["audit", "verify"], {
+        sql,
+        env: {},
+        stdinIsTTY: false,
+        audit: { sink: "webhook", webhookUrl: "https://audit.example/hook" },
+        write: () => undefined,
+      });
+      expect(code).toBe(1);
+      expect(errors.join("\n")).toContain('audit.sink "db"');
+    } finally {
+      console.error = original;
+      await sql.close();
+    }
+  });
 });
 
 describe("oke vault backup / restore", () => {

package/src/cli/vault-cmd.ts

@@ -24,6 +24,7 @@ import type { SqlConnection } from "../drivers/types.ts";
 import { openBuiltinVaultAdapter, VAULT_MASTER_KEY_ENV } from "../drivers/vault-builtin.ts";
 import { parseDotenv, formatDotenv } from "../drivers/vault-dotenv-parse.ts";
 import type { BuiltinVaultAdapter } from "../elements/vault/builtin-adapter.ts";
+import type { VaultAuditConfig } from "../elements/vault/types.ts";
 import { promptHidden, readStdinSecure } from "./vault-secure-input.ts";
 
 /** Env var for the *new* master key when resuming an interrupted `rotate-master`. */
@@ -70,6 +71,11 @@ export interface VaultCmdOptions {
    * closes a connection it did not open.
    */
   readonly sql?: SqlConnection;
+  /**
+   * `vault.audit` from `oke.config.ts`. The `oke` binary loads this; tests
+   * pass it directly. Omitted means the SQL hash chain.
+   */
+  readonly audit?: VaultAuditConfig;
 }
 
 /**
@@ -332,6 +338,7 @@ async function withVault<T>(
     masterKey,
     ...(flags.url === undefined ? {} : { url: flags.url }),
     ...(options.sql === undefined ? {} : { connection: options.sql }),
+    ...(options.audit === undefined ? {} : { audit: options.audit }),
   });
   try {
     return await run(opened.adapter);