okengine 0.1.4

package/src/elements/ai/schema.ts

@@ -0,0 +1,194 @@
+/**
+ * Prompt output schema validation — distinct from provider errors.
+ *
+ * The model answered; the shape was wrong. This rate is the metric that
+ * rises first when a prompt regresses (console §9.10).
+ */
+
+/** Structured mismatch between a model answer and the declared `out` schema. */
+export interface AiSchemaMismatch {
+  readonly missing: readonly string[];
+  readonly extra: readonly string[];
+  readonly typeMismatches: readonly string[];
+}
+
+/**
+ * Error raised when a model returns a payload that fails the prompt's
+ * declared output schema. Distinct from a provider / transport failure.
+ */
+export class AiSchemaValidationError extends Error {
+  readonly code = "AiSchemaInvalid" as const;
+  readonly prompt: string;
+  readonly version: number | undefined;
+  readonly mismatch: AiSchemaMismatch;
+  readonly raw: unknown;
+
+  /**
+   * @param prompt - Prompt name
+   * @param version - Prompt version
+   * @param mismatch - Field-level mismatches
+   * @param raw - Raw model payload
+   */
+  constructor(
+    prompt: string,
+    version: number | undefined,
+    mismatch: AiSchemaMismatch,
+    raw: unknown,
+  ) {
+    const parts: string[] = [];
+    if (mismatch.missing.length > 0) {
+      parts.push(`missing [${mismatch.missing.join(", ")}]`);
+    }
+    if (mismatch.extra.length > 0) {
+      parts.push(`extra [${mismatch.extra.join(", ")}]`);
+    }
+    if (mismatch.typeMismatches.length > 0) {
+      parts.push(`types [${mismatch.typeMismatches.join(", ")}]`);
+    }
+    super(
+      `ai: schema validation failed for prompt "${prompt}"` +
+        (version !== undefined ? `@${version}` : "") +
+        (parts.length > 0 ? `: ${parts.join("; ")}` : ""),
+    );
+    this.name = "AiSchemaValidationError";
+    this.prompt = prompt;
+    this.version = version;
+    this.mismatch = mismatch;
+    this.raw = raw;
+  }
+}
+
+/**
+ * Whether a value is a JSON-schema-like object with `properties`.
+ *
+ * @param schema - Declared `out`
+ */
+export function isObjectSchema(
+  schema: unknown,
+): schema is {
+  readonly properties?: Record<string, unknown>;
+  readonly required?: readonly string[];
+  readonly type?: string;
+} {
+  return (
+    schema !== null &&
+    typeof schema === "object" &&
+    !Array.isArray(schema) &&
+    ("properties" in schema || "required" in schema || "type" in schema)
+  );
+}
+
+/**
+ * Coerce a model completion into a plain object (pre-validation).
+ *
+ * @param value - Raw completion
+ */
+export function coerceModelObject(value: unknown): Record<string, unknown> {
+  if (value && typeof value === "object" && !Array.isArray(value)) {
+    return value as Record<string, unknown>;
+  }
+  if (typeof value === "string") {
+    try {
+      const parsed = JSON.parse(value) as unknown;
+      if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+        return parsed as Record<string, unknown>;
+      }
+    } catch {
+      return { text: value };
+    }
+    return { text: value };
+  }
+  return { value };
+}
+
+/**
+ * Diff a payload against a JSON-schema-like `out` declaration.
+ *
+ * @param schema - Declared output schema
+ * @param value - Coerced object
+ */
+export function matchOutSchema(
+  schema: unknown,
+  value: Record<string, unknown>,
+): AiSchemaMismatch | null {
+  if (!isObjectSchema(schema)) return null;
+
+  const props = (schema.properties ?? {}) as Record<string, unknown>;
+  const required = Array.isArray(schema.required)
+    ? [...schema.required]
+    : Object.keys(props);
+  const keys = new Set(Object.keys(value));
+  const missing = required.filter((k) => !keys.has(k));
+  const extra =
+    Object.keys(props).length > 0
+      ? [...keys].filter((k) => !(k in props))
+      : [];
+  const typeMismatches: string[] = [];
+  for (const [key, propSchema] of Object.entries(props)) {
+    if (!(key in value)) continue;
+    const expected =
+      propSchema &&
+      typeof propSchema === "object" &&
+      !Array.isArray(propSchema) &&
+      "type" in propSchema
+        ? String((propSchema as { type?: unknown }).type)
+        : undefined;
+    if (!expected) continue;
+    const actual = value[key];
+    if (!matchesJsonType(actual, expected)) {
+      typeMismatches.push(`${key}:expected ${expected}`);
+    }
+  }
+
+  if (
+    missing.length === 0 &&
+    extra.length === 0 &&
+    typeMismatches.length === 0
+  ) {
+    return null;
+  }
+  return { missing, extra, typeMismatches };
+}
+
+/**
+ * Validate a model payload against `out`. Returns the object or throws
+ * {@link AiSchemaValidationError}.
+ *
+ * @param prompt - Prompt name
+ * @param version - Prompt version
+ * @param schema - Declared `out`
+ * @param raw - Raw model payload
+ */
+export function validatePromptOut(
+  prompt: string,
+  version: number | undefined,
+  schema: unknown,
+  raw: unknown,
+): Record<string, unknown> {
+  const object = coerceModelObject(raw);
+  const mismatch = matchOutSchema(schema, object);
+  if (mismatch) {
+    throw new AiSchemaValidationError(prompt, version, mismatch, raw);
+  }
+  return object;
+}
+
+function matchesJsonType(value: unknown, type: string): boolean {
+  switch (type) {
+    case "string":
+      return typeof value === "string";
+    case "number":
+    case "integer":
+      return typeof value === "number" && Number.isFinite(value);
+    case "boolean":
+      return typeof value === "boolean";
+    case "object":
+      return value !== null && typeof value === "object" && !Array.isArray(value);
+    case "array":
+      return Array.isArray(value);
+    case "null":
+      return value === null;
+    default:
+      return true;
+  }
+}

package/src/elements/ai.test.ts

@@ -0,0 +1,393 @@
+/**
+ * AI element acceptance:
+ * - an agent calling a flow it lacks the gate for is denied and the denial is recorded
+ * - a flow sending a pii field to a model fails the build
+ * - nondeterministic ⇒ journaling forced, auto-cache disabled
+ * - prompts as versioned artifacts with eval sets
+ * - embeddings into store.index
+ * - fallback chain records both model attempts
+ */
+
+import { describe, expect, test } from "bun:test";
+import {
+  createMockAiDriver,
+  memoryIndexDriver,
+  mockAiDriver,
+} from "../drivers/index.ts";
+import { createFx } from "../kernel/fx.ts";
+import {
+  ai,
+  AiPiiBuildError,
+  AiSchemaValidationError,
+  assertAllowPiiForAsk,
+  createAiRuntime,
+  parseEvalJsonl,
+  runPromptEvals,
+} from "./ai.ts";
+import { createGateRuntime, gate } from "./gate.ts";
+
+describe("ai declaration", () => {
+  test("model.prompt / agent / embed shapes", () => {
+    const smart = ai.model("smart", { provider: "anthropic", tier: "opus" });
+    const triage = smart.prompt("ticket-triage", {
+      version: 3,
+      evals: "./evals/triage.jsonl",
+      budget: { maxCostPerCall: 0.02 },
+    });
+    expect(triage.name).toBe("ticket-triage");
+    expect(triage.version).toBe(3);
+    expect(triage.model).toBe("smart");
+
+    const agent = ai.agent("support", {
+      tools: [{ name: "bookings.getBooking" }, "bookings.refundBooking"],
+      maxSteps: 6,
+      model: smart,
+    });
+    expect(agent.tools).toEqual([
+      "bookings.getBooking",
+      "bookings.refundBooking",
+    ]);
+
+    const embed = ai.embed("docs", {
+      model: smart,
+      into: { name: "kb", facet: "index" },
+    });
+    expect(embed.into).toBe("kb");
+  });
+});
+
+describe("agent gate denial is recorded", () => {
+  test("tool lacking gate is denied and denial is recorded", async () => {
+    const member = gate.policy("member", ({ auth }) => !!auth.verified);
+    const gates = createGateRuntime({ gates: [member] });
+
+    const refund = ai.agent("support", {
+      tools: ["bookings.refundBooking"],
+      maxSteps: 2,
+    });
+
+    const called: string[] = [];
+    const runtime = createAiRuntime({
+      agents: [refund],
+      gates,
+      gatesForFlow: (name) =>
+        name === "bookings.refundBooking" ? ["member"] : [],
+      callFlow: async (name) => {
+        called.push(name);
+        return { ok: true };
+      },
+    });
+
+    const result = await runtime.runAgent("support", {
+      message: "refund please",
+      auth: { userId: "u1", scopes: new Set(), verified: false },
+    });
+
+    expect(result.ok).toBe(false);
+    expect(result.denials).toHaveLength(1);
+    expect(result.denials[0]!.tool).toBe("bookings.refundBooking");
+    expect(result.denials[0]!.gate).toBe("member");
+    expect(runtime.denials).toHaveLength(1);
+    expect(called).toHaveLength(0);
+  });
+
+  test("tool with satisfied gate is called", async () => {
+    const member = gate.policy("member", ({ auth }) => !!auth.verified);
+    const gates = createGateRuntime({ gates: [member] });
+    const runtime = createAiRuntime({
+      agents: [
+        ai.agent("support", { tools: ["bookings.getBooking"], maxSteps: 1 }),
+      ],
+      gates,
+      gatesForFlow: () => ["member"],
+      callFlow: async () => ({ booking: "B1" }),
+    });
+    const result = await runtime.runAgent("support", {
+      message: "status?",
+      auth: { userId: "u1", scopes: new Set(), verified: true },
+    });
+    expect(result.ok).toBe(true);
+    expect(result.denials).toHaveLength(0);
+    expect(result.output).toEqual({ booking: "B1" });
+  });
+});
+
+describe("pii to third-party model fails the build", () => {
+  test("pii field without allowPii throws AiPiiBuildError", () => {
+    expect(() =>
+      assertAllowPiiForAsk({
+        flow: "support.createTicket",
+        askFields: ["subject", "email", "body"],
+        classifications: {
+          "users.email": { pii: true },
+          email: { pii: true },
+        },
+        provider: "anthropic",
+        pii: "masked",
+      }),
+    ).toThrow(AiPiiBuildError);
+
+    try {
+      assertAllowPiiForAsk({
+        flow: "support.createTicket",
+        askFields: ["email"],
+        classifications: { email: { pii: true } },
+        provider: "openai-compatible",
+      });
+    } catch (err) {
+      expect(err).toBeInstanceOf(AiPiiBuildError);
+      expect((err as AiPiiBuildError).fields).toContain("email");
+      expect((err as AiPiiBuildError).message).toContain("allowPii");
+    }
+  });
+
+  test("allowPii or mock provider permits", () => {
+    expect(() =>
+      assertAllowPiiForAsk({
+        flow: "support.createTicket",
+        askFields: ["email"],
+        classifications: { email: { pii: true } },
+        provider: "anthropic",
+        allowPii: true,
+      }),
+    ).not.toThrow();
+
+    expect(() =>
+      assertAllowPiiForAsk({
+        flow: "support.createTicket",
+        askFields: ["email"],
+        classifications: { email: { pii: true } },
+        provider: "mock",
+      }),
+    ).not.toThrow();
+  });
+});
+
+describe("journaling forced · auto-cache disabled", () => {
+  test("ask is journaled and replayed without re-calling the model", async () => {
+    let calls = 0;
+    const driver = createMockAiDriver({
+      "*": { urgency: "high", team: "ops" },
+    });
+    // Wrap to count
+    const base = await driver.open({ model: "mock" });
+    const counting = {
+      ...base,
+      async complete(opts: Parameters<typeof base.complete>[0]) {
+        calls++;
+        return base.complete(opts);
+      },
+    };
+
+    const smart = ai.model("smart", { provider: "mock" });
+    const triage = smart.prompt("ticket-triage", { version: 1 });
+    const runtime = createAiRuntime({
+      models: [smart],
+      prompts: [triage],
+      clients: { smart: counting },
+    });
+
+    expect(runtime.journalingForced).toBe(true);
+    expect(runtime.autoCacheDisabled).toBe(true);
+
+    const a = await runtime.ask("ticket-triage", { subject: "x" });
+    const b = await runtime.ask("ticket-triage", { subject: "x" });
+    expect(a).toEqual(b);
+    expect(calls).toBe(1);
+    expect(runtime.journal).toHaveLength(1);
+  });
+
+  test("fx.cache is disabled when aiRuntime is bound", async () => {
+    const smart = ai.model("smart", { provider: "mock" });
+    const client = await mockAiDriver.open({ model: "mock" });
+    const aiRuntime = createAiRuntime({
+      models: [smart],
+      clients: { smart: client },
+    });
+    const fx = createFx({
+      flow: "support.createTicket",
+      effects: {},
+      aiRuntime,
+    });
+    await fx.cache.set("k", "v");
+    expect(await fx.cache.get("k")).toBeUndefined();
+    const produced = await fx.cache.getOrSet("k", "1m", () => "fresh");
+    expect(produced).toBe("fresh");
+    expect(await fx.cache.get("k")).toBeUndefined();
+  });
+});
+
+describe("prompt evals gate CI", () => {
+  test("runPromptEvals fails on mismatch", async () => {
+    const smart = ai.model("smart", { provider: "mock" });
+    const triage = smart.prompt("ticket-triage", { version: 3 });
+    const client = await createMockAiDriver({
+      "*": { urgency: "high", team: "ops", summary: "ok" },
+    }).open();
+    const runtime = createAiRuntime({
+      models: [smart],
+      prompts: [triage],
+      clients: { smart: client },
+    });
+
+    const cases = parseEvalJsonl(
+      [
+        JSON.stringify({
+          id: "1",
+          input: { subject: "x" },
+          expect: { urgency: "high", team: "ops", summary: "ok" },
+        }),
+        JSON.stringify({
+          id: "2",
+          input: { subject: "y" },
+          expect: { urgency: "low" },
+        }),
+      ].join("\n"),
+    );
+
+    const suite = await runPromptEvals({
+      prompt: "ticket-triage",
+      version: 3,
+      cases,
+      ask: (input) => runtime.ask("ticket-triage", input),
+    });
+    expect(suite.ok).toBe(false);
+    expect(suite.failed).toBe(1);
+    expect(suite.passed).toBe(1);
+  });
+});
+
+describe("embeddings into store.index", () => {
+  test("embed upserts into index driver", async () => {
+    const index = await memoryIndexDriver.open({ name: "kb", dims: 8 });
+    const smart = ai.model("smart", { provider: "mock" });
+    const embed = ai.embed("docs", { model: smart, into: "kb" });
+    const client = await mockAiDriver.open({ model: "mock" });
+    const runtime = createAiRuntime({
+      models: [smart],
+      embeds: [embed],
+      clients: { smart: client },
+      indexes: { kb: index },
+    });
+    await runtime.embed("docs", "doc-1", "hello world");
+    const hits = await index.search(
+      (await client.embed!({ input: "hello world" })).vectors[0]!,
+      1,
+    );
+    expect(hits[0]?.id).toBe("doc-1");
+  });
+});
+
+describe("model fallback chain", () => {
+  test("records both attempts when first model fails", async () => {
+    const failing = {
+      driverId: "mock" as const,
+      model: "smart",
+      async complete() {
+        throw new Error("smart down");
+      },
+    };
+    const ok = await createMockAiDriver({
+      "*": { urgency: "low" },
+    }).open({ model: "fast" });
+
+    const smart = ai.model("smart", { provider: "mock" });
+    const fast = ai.model("fast", { provider: "mock" });
+    const triage = smart.prompt("ticket-triage", { version: 1 });
+    const runtime = createAiRuntime({
+      models: [smart, fast],
+      prompts: [triage],
+      clients: { smart: failing, fast: ok },
+    });
+
+    const out = await runtime.ask("ticket-triage", { subject: "x" }, {
+      via: ["smart", "fast"],
+    });
+    expect(out.urgency).toBe("low");
+    const entry = runtime.journal[0]!;
+    expect(entry.attempts).toHaveLength(2);
+    expect(entry.attempts[0]).toMatchObject({ model: "smart", ok: false });
+    expect(entry.attempts[1]).toMatchObject({ model: "fast", ok: true });
+    expect(entry.outcome).toBe("ok");
+  });
+});
+
+describe("schema-validation is its own class", () => {
+  test("model answered but shape wrong → AiSchemaValidationError", async () => {
+    const smart = ai.model("smart", { provider: "mock" });
+    const triage = smart.prompt("ticket-triage", {
+      version: 3,
+      out: {
+        type: "object",
+        properties: {
+          urgency: { type: "string" },
+          team: { type: "string" },
+        },
+        required: ["urgency", "team"],
+      },
+    });
+    const client = await createMockAiDriver({
+      "*": { urgency: "high" },
+    }).open();
+    const runtime = createAiRuntime({
+      models: [smart],
+      prompts: [triage],
+      clients: { smart: client },
+    });
+
+    let thrown: unknown;
+    try {
+      await runtime.ask("ticket-triage", { subject: "x" });
+    } catch (err) {
+      thrown = err;
+    }
+    expect(thrown).toBeInstanceOf(AiSchemaValidationError);
+    expect((thrown as AiSchemaValidationError).code).toBe("AiSchemaInvalid");
+    expect((thrown as AiSchemaValidationError).mismatch.missing).toContain(
+      "team",
+    );
+    expect(runtime.journal[0]!.outcome).toBe("schema_invalid");
+    expect(runtime.journal[0]!.outcome).not.toBe("provider_error");
+  });
+});
+
+describe("agent tool trail carries effects; denials are not errors", () => {
+  test("denied tool is a denial line with Manifest effects", async () => {
+    const member = gate.policy("member", ({ auth }) => !!auth.verified);
+    const gates = createGateRuntime({ gates: [member] });
+    const runtime = createAiRuntime({
+      agents: [
+        ai.agent("support", {
+          tools: ["bookings.refundBooking"],
+          maxSteps: 1,
+        }),
+      ],
+      gates,
+      gatesForFlow: () => ["member"],
+      effectsForFlow: (name) =>
+        name === "bookings.refundBooking"
+          ? [
+              { kind: "write", resource: "sql:bookings" },
+              { kind: "send", resource: "refund-notice" },
+            ]
+          : [],
+      callFlow: async () => ({ ok: true }),
+    });
+
+    const result = await runtime.runAgent("support", {
+      message: "refund",
+      auth: { userId: "u1", scopes: new Set(), verified: false },
+    });
+
+    expect(result.ok).toBe(false);
+    expect(result.trail).toHaveLength(1);
+    expect(result.trail[0]!.status).toBe("denied");
+    expect(result.trail[0]!.denial?.gate).toBe("member");
+    expect(result.trail[0]!.effects).toEqual([
+      { kind: "write", resource: "sql:bookings" },
+      { kind: "send", resource: "refund-notice" },
+    ]);
+    expect(runtime.agentRuns).toHaveLength(1);
+    expect(runtime.denials).toHaveLength(1);
+  });
+});

package/src/elements/ai.ts

@@ -0,0 +1,56 @@
+/**
+ * AI element — reaching machine intelligence.
+ *
+ * Physics: inference · prompts · embeddings · agents.
+ * Drivers: `mock` (dev) · `anthropic` · `openai-compatible` · `bedrock` ·
+ * `vertex` · `ollama`. No production default — prod must declare.
+ *
+ * Prompts are versioned artifacts with validated output shapes and eval sets
+ * (`oke eval` gates CI). Agents' tools are the app's own flows (gates apply).
+ * PII fields cannot reach a third-party model without explicit `allowPii`.
+ * Nondeterministic ⇒ journaling forced, auto-cache disabled.
+ */
+
+export { ai } from "./ai/declare.ts";
+export type {
+  AiAgentDecl,
+  AiAgentOptions,
+  AiBudgetDecl,
+  AiEmbedDecl,
+  AiEmbedOptions,
+  AiModelDecl,
+  AiModelOptions,
+  AiPromptDecl,
+  AiPromptOptions,
+} from "./ai/declare.ts";
+
+export { createAiRuntime, AiSchemaValidationError } from "./ai/runtime.ts";
+export type {
+  AgentDenial,
+  AgentRunRecord,
+  AgentToolEffect,
+  AgentToolStep,
+  AiAgentRunOptions,
+  AiAskOptions,
+  AiAskOutcome,
+  AiFallbackAttempt,
+  AiJournalEntry,
+  AiRuntime,
+  AiSchemaMismatch,
+  CreateAiRuntimeOptions,
+} from "./ai/runtime.ts";
+
+export {
+  assertAllowPiiForAsk,
+  AiPiiBuildError,
+  type PiiAskCheckInput,
+} from "./ai/pii.ts";
+
+export {
+  runPromptEvals,
+  parseEvalJsonl,
+  type EvalCase,
+  type EvalCaseResult,
+  type EvalSuiteResult,
+  type RunPromptEvalsOptions,
+} from "./ai/eval.ts";