okengine 0.1.4

package/src/console/ui/ai/fixture.ts

@@ -0,0 +1,182 @@
+/**
+ * AI panel fixtures for unit tests and the axe gate (console §9.10).
+ */
+
+import type { AiListResponse, PromptVersionMetrics } from "./types.ts";
+
+function dist(
+  samples: readonly number[],
+): PromptVersionMetrics["cost"] {
+  const sorted = [...samples].sort((a, b) => a - b);
+  const mean =
+    sorted.length === 0
+      ? 0
+      : sorted.reduce((a, b) => a + b, 0) / sorted.length;
+  const p50 = sorted[Math.floor(sorted.length * 0.5)] ?? 0;
+  const p95 = sorted[Math.floor(sorted.length * 0.95)] ?? p50;
+  const min = sorted[0] ?? 0;
+  const max = sorted[sorted.length - 1] ?? 0;
+  return {
+    samples: sorted,
+    mean,
+    p50,
+    p95,
+    buckets:
+      sorted.length === 0
+        ? []
+        : [
+            { min, max: min + (max - min) / 2 || min + 0.001, count: Math.ceil(sorted.length / 2) },
+            {
+              min: min + (max - min) / 2 || min + 0.001,
+              max: max + 0.0001,
+              count: Math.floor(sorted.length / 2),
+            },
+          ],
+  };
+}
+
+/** Baseline v2 — healthy. */
+export const VERSION_V2: PromptVersionMetrics = {
+  prompt: "ticket-triage",
+  version: 2,
+  sampleCount: 100,
+  cost: dist([0.008, 0.009, 0.01, 0.011, 0.012]),
+  latencyMs: dist([120, 140, 160, 180, 200]),
+  evalScore: dist([0.82, 0.84, 0.85, 0.86, 0.88]),
+  schemaInvalidRate: 0.02,
+  providerErrorRate: 0.01,
+  okRate: 0.97,
+  overBudgetRate: 0.0,
+  budgetMaxCostPerCall: 0.02,
+  outcomeCounts: { ok: 97, provider_error: 1, schema_invalid: 2 },
+};
+
+/** Candidate v3 — higher eval, worse schema + budget. */
+export const VERSION_V3: PromptVersionMetrics = {
+  prompt: "ticket-triage",
+  version: 3,
+  sampleCount: 100,
+  cost: dist([0.015, 0.018, 0.022, 0.025, 0.03]),
+  latencyMs: dist([150, 170, 190, 220, 250]),
+  evalScore: dist([0.9, 0.91, 0.92, 0.93, 0.94]),
+  schemaInvalidRate: 0.086,
+  providerErrorRate: 0.01,
+  okRate: 0.904,
+  overBudgetRate: 0.12,
+  budgetMaxCostPerCall: 0.02,
+  outcomeCounts: { ok: 90, provider_error: 1, schema_invalid: 9 },
+};
+
+/** Full AI list fixture. */
+export const AI_LIST_FIXTURE: AiListResponse = {
+  prompts: [
+    {
+      name: "ticket-triage",
+      version: 3,
+      model: "smart",
+      evals: "./evals/triage.jsonl",
+      budgetMaxCostPerCall: 0.02,
+      manifestDiffPath: "/ai/prompts/ticket-triage/version",
+    },
+  ],
+  agents: [
+    {
+      name: "support",
+      tools: ["bookings.getBooking", "bookings.refundBooking"],
+      maxSteps: 6,
+      model: "smart",
+      budgetMaxCostPerRun: 0.25,
+    },
+  ],
+  versions: [VERSION_V2, VERSION_V3],
+  allowPii: [
+    {
+      flowId: "support.createTicket",
+      asks: ["ticket-triage"],
+      pii: "allow",
+      allowPii: true,
+      source: "src/flows/support/index.ts",
+    },
+  ],
+  fallbackChains: [
+    {
+      prompt: "ticket-triage",
+      version: 3,
+      attempts: [
+        {
+          model: "smart",
+          ok: false,
+          error: "timeout",
+          cost: 0,
+          latencyMs: 800,
+          at: 1_700_000_000_000,
+        },
+        {
+          model: "fast",
+          ok: true,
+          cost: 0.004,
+          latencyMs: 90,
+          at: 1_700_000_000_100,
+        },
+      ],
+      actualCost: 0.004,
+      primaryOnlyCost: null,
+      costConsequence: 0.004,
+      at: 1_700_000_000_100,
+    },
+  ],
+  agentRuns: [
+    {
+      id: "agent-run-1",
+      agent: "support",
+      message: "refund booking B1",
+      ok: false,
+      steps: 2,
+      cost: 0,
+      at: 1_700_000_000_200,
+      trail: [
+        {
+          tool: "bookings.getBooking",
+          status: "ok",
+          effects: [{ kind: "read", resource: "sql:bookings" }],
+          denial: null,
+          at: 1_700_000_000_210,
+        },
+        {
+          tool: "bookings.refundBooking",
+          status: "denied",
+          effects: [
+            { kind: "write", resource: "sql:bookings" },
+            { kind: "send", resource: "refund-notice" },
+          ],
+          denial: {
+            agent: "support",
+            tool: "bookings.refundBooking",
+            gate: "member",
+            reason: "not verified",
+            at: 1_700_000_000_220,
+          },
+          at: 1_700_000_000_220,
+        },
+      ],
+      denials: [
+        {
+          agent: "support",
+          tool: "bookings.refundBooking",
+          gate: "member",
+          reason: "not verified",
+          at: 1_700_000_000_220,
+        },
+      ],
+    },
+  ],
+  denials: [
+    {
+      agent: "support",
+      tool: "bookings.refundBooking",
+      gate: "member",
+      reason: "not verified",
+      at: 1_700_000_000_220,
+    },
+  ],
+};

package/src/console/ui/ai/format.ts

@@ -0,0 +1,90 @@
+/**
+ * Display helpers for AI panel distributions and trails (console §9.10).
+ */
+
+import type { AgentToolEffect, AiDistributionBucket } from "./types.ts";
+
+/**
+ * Format a rate as a percentage string.
+ *
+ * @param rate - 0..1
+ */
+export function formatRate(rate: number): string {
+  return `${(rate * 100).toFixed(1)}%`;
+}
+
+/**
+ * Format USD cost.
+ *
+ * @param cost - Dollars
+ */
+export function formatCost(cost: number): string {
+  return `$${cost.toFixed(4)}`;
+}
+
+/**
+ * Format latency.
+ *
+ * @param ms - Milliseconds
+ */
+export function formatLatency(ms: number): string {
+  if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`;
+  return `${Math.round(ms)}ms`;
+}
+
+/**
+ * Format eval score.
+ *
+ * @param score - 0..1
+ */
+export function formatEval(score: number): string {
+  return `${(score * 100).toFixed(1)}%`;
+}
+
+/**
+ * Accessible summary of a distribution (no axes — text first).
+ *
+ * @param label - Metric name
+ * @param mean - Mean
+ * @param p50 - Median
+ * @param p95 - p95
+ * @param format - Value formatter
+ */
+export function distributionSummary(
+  label: string,
+  mean: number,
+  p50: number,
+  p95: number,
+  format: (n: number) => string,
+): string {
+  return `${label}: mean ${format(mean)}, p50 ${format(p50)}, p95 ${format(p95)}`;
+}
+
+/**
+ * Max count in buckets (for bar width).
+ *
+ * @param buckets - Histogram
+ */
+export function maxBucketCount(
+  buckets: readonly AiDistributionBucket[],
+): number {
+  return buckets.reduce((m, b) => Math.max(m, b.count), 0);
+}
+
+/**
+ * Format an effect for the trail (tier vocabulary).
+ *
+ * @param effect - Tool effect
+ */
+export function formatEffect(effect: AgentToolEffect): string {
+  return `${effect.kind} ${effect.resource}`;
+}
+
+/**
+ * Label for a trail line — denial is never "error".
+ *
+ * @param status - ok | denied
+ */
+export function trailStatusLabel(status: "ok" | "denied"): string {
+  return status === "denied" ? "DENIED" : "ok";
+}

package/src/console/ui/ai/group.test.ts

@@ -0,0 +1,37 @@
+/**
+ * AI catalogue filter / group tests (console §9.10).
+ */
+
+import { describe, expect, test } from "bun:test";
+import { AI_LIST_FIXTURE } from "./fixture.ts";
+import {
+  allowPiiStanding,
+  filterAgents,
+  filterPrompts,
+  runsForAgent,
+  versionsForPrompt,
+} from "./group.ts";
+
+describe("AI group helpers", () => {
+  test("filterPrompts matches name", () => {
+    expect(filterPrompts(AI_LIST_FIXTURE.prompts, "triage")).toHaveLength(1);
+    expect(filterPrompts(AI_LIST_FIXTURE.prompts, "nope")).toHaveLength(0);
+  });
+
+  test("versionsForPrompt sorts ascending", () => {
+    const vs = versionsForPrompt(AI_LIST_FIXTURE.versions, "ticket-triage");
+    expect(vs.map((v) => v.version)).toEqual([2, 3]);
+  });
+
+  test("allowPiiStanding keeps only acknowledgements", () => {
+    expect(allowPiiStanding(AI_LIST_FIXTURE.allowPii)).toHaveLength(1);
+    expect(allowPiiStanding(AI_LIST_FIXTURE.allowPii)[0]!.flowId).toBe(
+      "support.createTicket",
+    );
+  });
+
+  test("runsForAgent + filterAgents", () => {
+    expect(filterAgents(AI_LIST_FIXTURE.agents, "support")).toHaveLength(1);
+    expect(runsForAgent(AI_LIST_FIXTURE.agentRuns, "support")).toHaveLength(1);
+  });
+});

package/src/console/ui/ai/group.ts

@@ -0,0 +1,88 @@
+/**
+ * Filter / group AI catalogue rows (console §9.10).
+ */
+
+import type {
+  AgentCatalogueRow,
+  AgentRunRow,
+  AllowPiiRow,
+  PromptCatalogueRow,
+  PromptVersionMetrics,
+} from "./types.ts";
+
+/**
+ * Filter prompts by substring.
+ *
+ * @param prompts - Catalogue
+ * @param q - Query
+ */
+export function filterPrompts(
+  prompts: readonly PromptCatalogueRow[],
+  q: string | undefined,
+): readonly PromptCatalogueRow[] {
+  const needle = q?.trim().toLowerCase();
+  if (!needle) return prompts;
+  return prompts.filter(
+    (p) =>
+      p.name.toLowerCase().includes(needle) ||
+      (p.model?.toLowerCase().includes(needle) ?? false),
+  );
+}
+
+/**
+ * Filter agents by substring.
+ *
+ * @param agents - Catalogue
+ * @param q - Query
+ */
+export function filterAgents(
+  agents: readonly AgentCatalogueRow[],
+  q: string | undefined,
+): readonly AgentCatalogueRow[] {
+  const needle = q?.trim().toLowerCase();
+  if (!needle) return agents;
+  return agents.filter(
+    (a) =>
+      a.name.toLowerCase().includes(needle) ||
+      a.tools.some((t) => t.toLowerCase().includes(needle)),
+  );
+}
+
+/**
+ * Versions for one prompt, sorted ascending.
+ *
+ * @param versions - All version metrics
+ * @param prompt - Prompt name
+ */
+export function versionsForPrompt(
+  versions: readonly PromptVersionMetrics[],
+  prompt: string,
+): readonly PromptVersionMetrics[] {
+  return versions
+    .filter((v) => v.prompt === prompt)
+    .sort((a, b) => a.version - b.version);
+}
+
+/**
+ * Agent runs for one agent.
+ *
+ * @param runs - All runs
+ * @param agent - Agent name
+ */
+export function runsForAgent(
+  runs: readonly AgentRunRow[],
+  agent: string,
+): readonly AgentRunRow[] {
+  return runs.filter((r) => r.agent === agent);
+}
+
+/**
+ * Rows that explicitly allow PII egress (standing security review).
+ *
+ * @param rows - All allowPii projections
+ */
+export function allowPiiStanding(
+  rows: readonly AllowPiiRow[],
+): readonly AllowPiiRow[] {
+  return rows.filter((r) => r.allowPii);
+}

package/src/console/ui/ai/index.ts

@@ -0,0 +1,66 @@
+/**
+ * AI panel pure modules (console §9.10).
+ */
+
+export type {
+  AgentCatalogueRow,
+  AgentDenial,
+  AgentRunRow,
+  AgentToolEffect,
+  AiAskOutcome,
+  AiDistributionBucket,
+  AiFallbackAttempt,
+  AiListResponse,
+  AiMetricDistribution,
+  AllowPiiRow,
+  FallbackChainRow,
+  PromptCatalogueRow,
+  PromptVersionMetrics,
+} from "./types.ts";
+
+export {
+  parseAiSearch,
+  serializeAiSearch,
+  openPromptVersion,
+  openAgentRun,
+  manifestDiffHref,
+  type AiSearch,
+} from "./search.ts";
+
+export {
+  promotionDecision,
+  formatPromotionBlockers,
+  type PromotionBlocker,
+  type PromotionDecision,
+  type PromotionNumbers,
+} from "./promotion.ts";
+
+export {
+  filterPrompts,
+  filterAgents,
+  versionsForPrompt,
+  runsForAgent,
+  allowPiiStanding,
+} from "./group.ts";
+
+export {
+  formatRate,
+  formatCost,
+  formatLatency,
+  formatEval,
+  distributionSummary,
+  maxBucketCount,
+  formatEffect,
+  trailStatusLabel,
+} from "./format.ts";
+
+export {
+  AI_LIST_FIXTURE,
+  VERSION_V2,
+  VERSION_V3,
+} from "./fixture.ts";
+
+export {
+  overBudgetFindings,
+  type OverBudgetFinding,
+} from "./findings.ts";

package/src/console/ui/ai/promotion.test.ts

@@ -0,0 +1,77 @@
+/**
+ * Promotion gate tests (console §9.10).
+ */
+
+import { describe, expect, test } from "bun:test";
+import { VERSION_V2, VERSION_V3 } from "./fixture.ts";
+import {
+  formatPromotionBlockers,
+  promotionDecision,
+} from "./promotion.ts";
+import type { PromptVersionMetrics } from "./types.ts";
+
+function withRates(
+  base: PromptVersionMetrics,
+  patch: Partial<PromptVersionMetrics>,
+): PromptVersionMetrics {
+  return { ...base, ...patch };
+}
+
+describe("promotionDecision", () => {
+  test("blocks when schema-invalid rises even if eval improves", () => {
+    const decision = promotionDecision(VERSION_V2, VERSION_V3);
+    expect(decision.allowed).toBe(false);
+    if (decision.allowed) return;
+    expect(decision.blockers).toContain("schema_validity");
+    expect(decision.blockers).toContain("budget");
+    expect(decision.evalImproved).toBe(true);
+    expect(decision.numbers.toSchemaInvalidRate).toBe(0.086);
+    expect(decision.numbers.fromSchemaInvalidRate).toBe(0.02);
+    const lines = formatPromotionBlockers(decision);
+    expect(lines.some((l) => l.includes("schema-invalid"))).toBe(true);
+    expect(lines.some((l) => l.includes("improved — still blocked"))).toBe(
+      true,
+    );
+  });
+
+  test("allows when schema and budget do not regress", () => {
+    const better = withRates(VERSION_V3, {
+      version: 4,
+      schemaInvalidRate: 0.01,
+      overBudgetRate: 0,
+      cost: {
+        ...VERSION_V3.cost,
+        p95: 0.015,
+        mean: 0.01,
+      },
+      evalScore: {
+        ...VERSION_V3.evalScore,
+        mean: 0.95,
+      },
+    });
+    const decision = promotionDecision(VERSION_V2, better);
+    expect(decision.allowed).toBe(true);
+    expect(decision.evalImproved).toBe(true);
+  });
+
+  test("blocks budget when p95 crosses maxCostPerCall", () => {
+    const candidate = withRates(VERSION_V2, {
+      version: 5,
+      schemaInvalidRate: 0.02,
+      overBudgetRate: 0,
+      cost: {
+        ...VERSION_V2.cost,
+        p95: 0.05,
+        mean: 0.03,
+      },
+      evalScore: {
+        ...VERSION_V2.evalScore,
+        mean: 0.99,
+      },
+    });
+    const decision = promotionDecision(VERSION_V2, candidate);
+    expect(decision.allowed).toBe(false);
+    if (decision.allowed) return;
+    expect(decision.blockers).toEqual(["budget"]);
+  });
+});

package/src/console/ui/ai/promotion.ts

@@ -0,0 +1,150 @@
+/**
+ * Automatic promotion gate between prompt versions (console §9.10).
+ *
+ * A version can score higher on evals and still be blocked when schema
+ * validity or budget would regress. Numbers are read from metrics — never
+ * hand-waved.
+ */
+
+import type { PromptVersionMetrics } from "./types.ts";
+
+/** Why promotion is blocked. */
+export type PromotionBlocker = "budget" | "schema_validity";
+
+/** Promotion decision with the real numbers that drove it. */
+export type PromotionDecision =
+  | {
+      readonly allowed: true;
+      readonly from: PromptVersionMetrics;
+      readonly to: PromptVersionMetrics;
+      readonly evalImproved: boolean;
+      readonly numbers: PromotionNumbers;
+    }
+  | {
+      readonly allowed: false;
+      readonly from: PromptVersionMetrics;
+      readonly to: PromptVersionMetrics;
+      readonly blockers: readonly PromotionBlocker[];
+      readonly evalImproved: boolean;
+      readonly numbers: PromotionNumbers;
+    };
+
+/** Concrete rates / costs compared. */
+export interface PromotionNumbers {
+  readonly fromEvalMean: number;
+  readonly toEvalMean: number;
+  readonly fromSchemaInvalidRate: number;
+  readonly toSchemaInvalidRate: number;
+  readonly fromOverBudgetRate: number;
+  readonly toOverBudgetRate: number;
+  readonly fromP95Cost: number;
+  readonly toP95Cost: number;
+  readonly budgetMaxCostPerCall: number | null;
+}
+
+/**
+ * Decide whether promoting `to` over `from` is allowed.
+ *
+ * Blocked when schema-invalid rate rises or budget adherence regresses,
+ * even if eval score improves.
+ *
+ * @param from - Current / baseline version metrics
+ * @param to - Candidate version metrics
+ */
+export function promotionDecision(
+  from: PromptVersionMetrics,
+  to: PromptVersionMetrics,
+): PromotionDecision {
+  if (from.prompt !== to.prompt) {
+    throw new Error(
+      `promotionDecision: prompt mismatch (${from.prompt} vs ${to.prompt})`,
+    );
+  }
+
+  const numbers: PromotionNumbers = {
+    fromEvalMean: from.evalScore.mean,
+    toEvalMean: to.evalScore.mean,
+    fromSchemaInvalidRate: from.schemaInvalidRate,
+    toSchemaInvalidRate: to.schemaInvalidRate,
+    fromOverBudgetRate: from.overBudgetRate,
+    toOverBudgetRate: to.overBudgetRate,
+    fromP95Cost: from.cost.p95,
+    toP95Cost: to.cost.p95,
+    budgetMaxCostPerCall:
+      to.budgetMaxCostPerCall ?? from.budgetMaxCostPerCall,
+  };
+
+  const blockers: PromotionBlocker[] = [];
+
+  // Schema validity regresses when the invalid rate rises.
+  if (to.schemaInvalidRate > from.schemaInvalidRate + 1e-12) {
+    blockers.push("schema_validity");
+  }
+
+  // Budget regresses when over-budget rate rises, or p95 crosses the budget
+  // while the baseline stayed under.
+  const budget = numbers.budgetMaxCostPerCall;
+  if (to.overBudgetRate > from.overBudgetRate + 1e-12) {
+    blockers.push("budget");
+  } else if (
+    budget !== null &&
+    to.cost.p95 > budget &&
+    from.cost.p95 <= budget
+  ) {
+    blockers.push("budget");
+  }
+
+  const evalImproved = to.evalScore.mean > from.evalScore.mean + 1e-12;
+
+  if (blockers.length > 0) {
+    return {
+      allowed: false,
+      from,
+      to,
+      blockers,
+      evalImproved,
+      numbers,
+    };
+  }
+
+  return {
+    allowed: true,
+    from,
+    to,
+    evalImproved,
+    numbers,
+  };
+}
+
+/**
+ * Format a blocked promotion for the operator (real numbers, no fluff).
+ *
+ * @param decision - Blocked decision
+ */
+export function formatPromotionBlockers(
+  decision: Extract<PromotionDecision, { allowed: false }>,
+): readonly string[] {
+  const lines: string[] = [];
+  const n = decision.numbers;
+  if (decision.blockers.includes("schema_validity")) {
+    lines.push(
+      `schema-invalid ${(n.fromSchemaInvalidRate * 100).toFixed(1)}% → ${(n.toSchemaInvalidRate * 100).toFixed(1)}%`,
+    );
+  }
+  if (decision.blockers.includes("budget")) {
+    const budget =
+      n.budgetMaxCostPerCall !== null
+        ? `budget $${n.budgetMaxCostPerCall.toFixed(4)}`
+        : "budget";
+    lines.push(
+      `over-budget ${(n.fromOverBudgetRate * 100).toFixed(1)}% → ${(n.toOverBudgetRate * 100).toFixed(1)}%` +
+        ` · p95 $${n.fromP95Cost.toFixed(4)} → $${n.toP95Cost.toFixed(4)} (${budget})`,
+    );
+  }
+  if (decision.evalImproved) {
+    lines.push(
+      `eval mean ${(n.fromEvalMean * 100).toFixed(1)}% → ${(n.toEvalMean * 100).toFixed(1)}% (improved — still blocked)`,
+    );
+  }
+  return lines;
+}