okengine 0.11.2 → 0.12.0

package/src/console/ui/ai/AiA11yView.tsx

@@ -1,254 +0,0 @@
-/**
- * Static AI markup for the axe gate — same landmarks as the live panel.
- */
-
-import {
-  allowPiiStanding,
-  distributionSummary,
-  filterAgents,
-  filterPrompts,
-  formatCost,
-  formatEval,
-  formatLatency,
-  formatPromotionBlockers,
-  formatRate,
-  manifestDiffHref,
-  maxBucketCount,
-  promotionDecision,
-  trailStatusLabel,
-  versionsForPrompt,
-  AI_LIST_FIXTURE,
-} from "./index.ts";
-
-/** Props for {@link AiA11yView}. */
-export interface AiA11yViewProps {
-  /** Prompt to open. */
-  readonly openPrompt?: string;
-  /** Version to open. */
-  readonly openVersion?: number;
-  /** Agent to open. */
-  readonly openAgent?: string;
-}
-
-/**
- * Accessible AI panel for CI.
- *
- * @param props - Open prompt / agent
- */
-export function AiA11yView(props: AiA11yViewProps) {
-  const data = AI_LIST_FIXTURE;
-  const promptName = props.openPrompt ?? "ticket-triage";
-  const version = props.openVersion ?? 3;
-  const agentName = props.openAgent ?? "support";
-  const prompts = filterPrompts(data.prompts, undefined);
-  const agents = filterAgents(data.agents, undefined);
-  const versions = versionsForPrompt(data.versions, promptName);
-  const selected = versions.find((v) => v.version === version) ?? versions[versions.length - 1];
-  const baseline = versions.find((v) => v.version === 2) ?? versions[0];
-  const decision =
-    baseline && selected && baseline.version !== selected.version
-      ? promotionDecision(baseline, selected)
-      : null;
-  const pii = allowPiiStanding(data.allowPii);
-  const run = data.agentRuns.find((r) => r.agent === agentName);
-
-  return (
-    <div className="ai-a11y">
-      <a href="#ai-main">Skip to main content</a>
-      <header>
-        <h1>AI</h1>
-        <p>Distributions per prompt version — cost, latency, eval score</p>
-        <label>
-          Filter
-          <input aria-label="Filter prompts and agents" defaultValue="" />
-        </label>
-      </header>
-      <main id="ai-main">
-        <section aria-label="Prompts">
-          <h2>Prompts</h2>
-          <ul>
-            {prompts.map((p) => (
-              <li key={p.name}>
-                <button
-                  type="button"
-                  aria-pressed={p.name === promptName}
-                  style={{ minHeight: 32 }}
-                >
-                  {p.name}
-                  {p.version !== undefined ? ` @${p.version}` : ""}
-                </button>
-                <a href={manifestDiffHref(p.manifestDiffPath)}>Manifest Diff</a>
-              </li>
-            ))}
-          </ul>
-        </section>
-
-        {selected ? (
-          <section aria-label="Version distributions" aria-live="polite">
-            <h2>
-              {selected.prompt} @{selected.version}
-            </h2>
-            <p role="status">
-              {distributionSummary(
-                "Cost",
-                selected.cost.mean,
-                selected.cost.p50,
-                selected.cost.p95,
-                formatCost,
-              )}
-            </p>
-            <div
-              role="img"
-              aria-label={distributionSummary(
-                "Cost distribution",
-                selected.cost.mean,
-                selected.cost.p50,
-                selected.cost.p95,
-                formatCost,
-              )}
-            >
-              {selected.cost.buckets.map((b, i) => (
-                <div
-                  key={i}
-                  style={{
-                    width: `${maxBucketCount(selected.cost.buckets) === 0 ? 0 : (b.count / maxBucketCount(selected.cost.buckets)) * 100}%`,
-                    minHeight: 8,
-                  }}
-                />
-              ))}
-            </div>
-            <p>
-              {distributionSummary(
-                "Latency",
-                selected.latencyMs.mean,
-                selected.latencyMs.p50,
-                selected.latencyMs.p95,
-                formatLatency,
-              )}
-            </p>
-            <p>
-              {distributionSummary(
-                "Eval score",
-                selected.evalScore.mean,
-                selected.evalScore.p50,
-                selected.evalScore.p95,
-                formatEval,
-              )}
-            </p>
-            <table>
-              <caption>Outcome classes</caption>
-              <thead>
-                <tr>
-                  <th scope="col">Class</th>
-                  <th scope="col">Rate</th>
-                </tr>
-              </thead>
-              <tbody>
-                <tr>
-                  <td>schema-invalid</td>
-                  <td>{formatRate(selected.schemaInvalidRate)}</td>
-                </tr>
-                <tr>
-                  <td>provider-error</td>
-                  <td>{formatRate(selected.providerErrorRate)}</td>
-                </tr>
-                <tr>
-                  <td>ok</td>
-                  <td>{formatRate(selected.okRate)}</td>
-                </tr>
-              </tbody>
-            </table>
-          </section>
-        ) : null}
-
-        {decision && !decision.allowed ? (
-          <section aria-label="Promotion gate" role="status">
-            <h2>Promotion blocked</h2>
-            <ul>
-              {formatPromotionBlockers(decision).map((line) => (
-                <li key={line}>{line}</li>
-              ))}
-            </ul>
-          </section>
-        ) : null}
-
-        <section aria-label="Agents">
-          <h2>Agents</h2>
-          <ul>
-            {agents.map((a) => (
-              <li key={a.name}>
-                <button type="button" aria-pressed={a.name === agentName} style={{ minHeight: 32 }}>
-                  {a.name}
-                </button>
-              </li>
-            ))}
-          </ul>
-        </section>
-
-        {run ? (
-          <section aria-label="Agent run trail">
-            <h2>Agent run {run.id}</h2>
-            <ol>
-              {run.trail.map((step, i) => (
-                <li key={`${step.tool}-${i}`}>
-                  <span>{trailStatusLabel(step.status)}</span> <span>{step.tool}</span>
-                  {step.status === "denied" && step.denial ? (
-                    <p role="status">
-                      DENIED by gate {step.denial.gate}: {step.denial.reason}
-                    </p>
-                  ) : null}
-                  <ul>
-                    {step.effects.map((e) => (
-                      <li key={`${e.kind}:${e.resource}`}>
-                        {e.kind} {e.resource}
-                      </li>
-                    ))}
-                  </ul>
-                </li>
-              ))}
-            </ol>
-          </section>
-        ) : null}
-
-        <section aria-label="Fallback chains">
-          <h2>Fallback chains</h2>
-          <ul>
-            {data.fallbackChains.map((chain, i) => (
-              <li key={i}>
-                {chain.prompt}: {chain.attempts.map((a) => a.model).join(" → ")}
-                {chain.costConsequence !== null
-                  ? ` · cost consequence ${formatCost(chain.costConsequence)}`
-                  : ""}
-              </li>
-            ))}
-          </ul>
-        </section>
-
-        <section aria-label="allowPii security review">
-          <h2>allowPii</h2>
-          <table>
-            <caption>Standing security review — PII egress acknowledgements</caption>
-            <thead>
-              <tr>
-                <th scope="col">Flow</th>
-                <th scope="col">Asks</th>
-                <th scope="col">allowPii</th>
-                <th scope="col">Source</th>
-              </tr>
-            </thead>
-            <tbody>
-              {pii.map((row) => (
-                <tr key={row.flowId}>
-                  <td>{row.flowId}</td>
-                  <td>{row.asks.join(", ") || "—"}</td>
-                  <td>{row.allowPii ? "yes" : "no"}</td>
-                  <td>{row.source ?? "—"}</td>
-                </tr>
-              ))}
-            </tbody>
-          </table>
-        </section>
-      </main>
-    </div>
-  );
-}

package/src/console/ui/ai/a11y.test.tsx

@@ -1,96 +0,0 @@
-/**
- * Accessibility gate — axe reports zero violations at WCAG 2.2 AA
- * (console §7.5 · §9.10).
- */
-
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import axe from "axe-core";
-import { Window } from "happy-dom";
-import { createRoot, type Root } from "react-dom/client";
-import { act } from "react";
-import { AiA11yView } from "./AiA11yView.tsx";
-
-describe("AI panel accessibility", () => {
-  let window: Window;
-  let root: Root;
-  let container: HTMLElement;
-
-  beforeEach(() => {
-    (globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true;
-    window = new Window({ url: "http://console.test/ai" });
-    Object.defineProperty(globalThis, "window", {
-      value: window,
-      configurable: true,
-      writable: true,
-    });
-    Object.defineProperty(globalThis, "document", {
-      value: window.document,
-      configurable: true,
-      writable: true,
-    });
-    Object.defineProperty(globalThis, "HTMLElement", {
-      value: window.HTMLElement,
-      configurable: true,
-      writable: true,
-    });
-    Object.defineProperty(globalThis, "Node", {
-      value: window.Node,
-      configurable: true,
-      writable: true,
-    });
-    Object.defineProperty(globalThis, "navigator", {
-      value: window.navigator,
-      configurable: true,
-      writable: true,
-    });
-    Object.defineProperty(globalThis, "getComputedStyle", {
-      value: window.getComputedStyle.bind(window),
-      configurable: true,
-      writable: true,
-    });
-    Object.defineProperty(globalThis, "requestAnimationFrame", {
-      value: (cb: FrameRequestCallback) => window.setTimeout(cb, 0),
-      configurable: true,
-      writable: true,
-    });
-
-    container = document.createElement("div");
-    document.body.appendChild(container);
-    root = createRoot(container);
-  });
-
-  afterEach(() => {
-    act(() => {
-      root.unmount();
-    });
-    container.remove();
-    window.close();
-  });
-
-  test("axe reports zero violations at WCAG 2.2 AA", async () => {
-    await act(async () => {
-      root.render(<AiA11yView openPrompt="ticket-triage" openVersion={3} openAgent="support" />);
-    });
-
-    await new Promise((r) => setTimeout(r, 10));
-
-    const results = await axe.run(container, {
-      runOnly: {
-        type: "tag",
-        values: ["wcag2a", "wcag2aa", "wcag22aa"],
-      },
-    });
-
-    if (results.violations.length > 0) {
-      const summary = results.violations
-        .map(
-          (v) =>
-            `${v.id}: ${v.help} (${v.nodes.length} nodes) — ${v.nodes.map((n) => n.html).join("; ")}`,
-        )
-        .join("\n");
-      console.error(summary);
-    }
-
-    expect(results.violations).toEqual([]);
-  });
-});

package/src/console/ui/ai/findings.ts

@@ -1,41 +0,0 @@
-/**
- * Model budget overrun findings for Overview aggregation (console §9.10 · §9.16).
- *
- * Detection is the AI panel's `overBudgetRate` metric — not recomputed here.
- */
-
-import type { PromptVersionMetrics } from "./types.ts";
-
-/** One over-budget prompt version from the AI panel. */
-export interface OverBudgetFinding {
-  readonly prompt: string;
-  readonly version: number;
-  readonly overBudgetRate: number;
-  readonly budgetMaxCostPerCall: number | null;
-  readonly p95Cost: number;
-}
-
-/**
- * Prompt versions whose projected metrics already report over-budget spend.
- *
- * @param versions - AI panel version metrics
- */
-export function overBudgetFindings(
-  versions: readonly PromptVersionMetrics[],
-): readonly OverBudgetFinding[] {
-  return versions
-    .filter((v) => v.overBudgetRate > 0)
-    .map((v) => ({
-      prompt: v.prompt,
-      version: v.version,
-      overBudgetRate: v.overBudgetRate,
-      budgetMaxCostPerCall: v.budgetMaxCostPerCall,
-      p95Cost: v.cost.p95,
-    }))
-    .sort(
-      (a, b) =>
-        b.overBudgetRate - a.overBudgetRate ||
-        a.prompt.localeCompare(b.prompt) ||
-        b.version - a.version,
-    );
-}

package/src/console/ui/ai/fixture.ts

@@ -1,177 +0,0 @@
-/**
- * AI panel fixtures for unit tests and the axe gate (console §9.10).
- */
-
-import type { AiListResponse, PromptVersionMetrics } from "./types.ts";
-
-function dist(samples: readonly number[]): PromptVersionMetrics["cost"] {
-  const sorted = [...samples].sort((a, b) => a - b);
-  const mean = sorted.length === 0 ? 0 : sorted.reduce((a, b) => a + b, 0) / sorted.length;
-  const p50 = sorted[Math.floor(sorted.length * 0.5)] ?? 0;
-  const p95 = sorted[Math.floor(sorted.length * 0.95)] ?? p50;
-  const min = sorted[0] ?? 0;
-  const max = sorted[sorted.length - 1] ?? 0;
-  return {
-    samples: sorted,
-    mean,
-    p50,
-    p95,
-    buckets:
-      sorted.length === 0
-        ? []
-        : [
-            { min, max: min + (max - min) / 2 || min + 0.001, count: Math.ceil(sorted.length / 2) },
-            {
-              min: min + (max - min) / 2 || min + 0.001,
-              max: max + 0.0001,
-              count: Math.floor(sorted.length / 2),
-            },
-          ],
-  };
-}
-
-/** Baseline v2 — healthy. */
-export const VERSION_V2: PromptVersionMetrics = {
-  prompt: "ticket-triage",
-  version: 2,
-  sampleCount: 100,
-  cost: dist([0.008, 0.009, 0.01, 0.011, 0.012]),
-  latencyMs: dist([120, 140, 160, 180, 200]),
-  evalScore: dist([0.82, 0.84, 0.85, 0.86, 0.88]),
-  schemaInvalidRate: 0.02,
-  providerErrorRate: 0.01,
-  okRate: 0.97,
-  overBudgetRate: 0.0,
-  budgetMaxCostPerCall: 0.02,
-  outcomeCounts: { ok: 97, provider_error: 1, schema_invalid: 2 },
-};
-
-/** Candidate v3 — higher eval, worse schema + budget. */
-export const VERSION_V3: PromptVersionMetrics = {
-  prompt: "ticket-triage",
-  version: 3,
-  sampleCount: 100,
-  cost: dist([0.015, 0.018, 0.022, 0.025, 0.03]),
-  latencyMs: dist([150, 170, 190, 220, 250]),
-  evalScore: dist([0.9, 0.91, 0.92, 0.93, 0.94]),
-  schemaInvalidRate: 0.086,
-  providerErrorRate: 0.01,
-  okRate: 0.904,
-  overBudgetRate: 0.12,
-  budgetMaxCostPerCall: 0.02,
-  outcomeCounts: { ok: 90, provider_error: 1, schema_invalid: 9 },
-};
-
-/** Full AI list fixture. */
-export const AI_LIST_FIXTURE: AiListResponse = {
-  prompts: [
-    {
-      name: "ticket-triage",
-      version: 3,
-      model: "smart",
-      evals: "./evals/triage.jsonl",
-      budgetMaxCostPerCall: 0.02,
-      manifestDiffPath: "/ai/prompts/ticket-triage/version",
-    },
-  ],
-  agents: [
-    {
-      name: "support",
-      tools: ["bookings.getBooking", "bookings.refundBooking"],
-      maxSteps: 6,
-      model: "smart",
-      budgetMaxCostPerRun: 0.25,
-    },
-  ],
-  versions: [VERSION_V2, VERSION_V3],
-  allowPii: [
-    {
-      flowId: "support.createTicket",
-      asks: ["ticket-triage"],
-      pii: "allow",
-      allowPii: true,
-      source: "src/flows/support/index.ts",
-    },
-  ],
-  fallbackChains: [
-    {
-      prompt: "ticket-triage",
-      version: 3,
-      attempts: [
-        {
-          model: "smart",
-          ok: false,
-          error: "timeout",
-          cost: 0,
-          latencyMs: 800,
-          at: 1_700_000_000_000,
-        },
-        {
-          model: "fast",
-          ok: true,
-          cost: 0.004,
-          latencyMs: 90,
-          at: 1_700_000_000_100,
-        },
-      ],
-      actualCost: 0.004,
-      primaryOnlyCost: null,
-      costConsequence: 0.004,
-      at: 1_700_000_000_100,
-    },
-  ],
-  agentRuns: [
-    {
-      id: "agent-run-1",
-      agent: "support",
-      message: "refund booking B1",
-      ok: false,
-      steps: 2,
-      cost: 0,
-      at: 1_700_000_000_200,
-      trail: [
-        {
-          tool: "bookings.getBooking",
-          status: "ok",
-          effects: [{ kind: "read", resource: "sql:bookings" }],
-          denial: null,
-          at: 1_700_000_000_210,
-        },
-        {
-          tool: "bookings.refundBooking",
-          status: "denied",
-          effects: [
-            { kind: "write", resource: "sql:bookings" },
-            { kind: "send", resource: "refund-notice" },
-          ],
-          denial: {
-            agent: "support",
-            tool: "bookings.refundBooking",
-            gate: "member",
-            reason: "not verified",
-            at: 1_700_000_000_220,
-          },
-          at: 1_700_000_000_220,
-        },
-      ],
-      denials: [
-        {
-          agent: "support",
-          tool: "bookings.refundBooking",
-          gate: "member",
-          reason: "not verified",
-          at: 1_700_000_000_220,
-        },
-      ],
-    },
-  ],
-  denials: [
-    {
-      agent: "support",
-      tool: "bookings.refundBooking",
-      gate: "member",
-      reason: "not verified",
-      at: 1_700_000_000_220,
-    },
-  ],
-};

package/src/console/ui/ai/format.ts

@@ -1,88 +0,0 @@
-/**
- * Display helpers for AI panel distributions and trails (console §9.10).
- */
-
-import type { AgentToolEffect, AiDistributionBucket } from "./types.ts";
-
-/**
- * Format a rate as a percentage string.
- *
- * @param rate - 0..1
- */
-export function formatRate(rate: number): string {
-  return `${(rate * 100).toFixed(1)}%`;
-}
-
-/**
- * Format USD cost.
- *
- * @param cost - Dollars
- */
-export function formatCost(cost: number): string {
-  return `$${cost.toFixed(4)}`;
-}
-
-/**
- * Format latency.
- *
- * @param ms - Milliseconds
- */
-export function formatLatency(ms: number): string {
-  if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`;
-  return `${Math.round(ms)}ms`;
-}
-
-/**
- * Format eval score.
- *
- * @param score - 0..1
- */
-export function formatEval(score: number): string {
-  return `${(score * 100).toFixed(1)}%`;
-}
-
-/**
- * Accessible summary of a distribution (no axes — text first).
- *
- * @param label - Metric name
- * @param mean - Mean
- * @param p50 - Median
- * @param p95 - p95
- * @param format - Value formatter
- */
-export function distributionSummary(
-  label: string,
-  mean: number,
-  p50: number,
-  p95: number,
-  format: (n: number) => string,
-): string {
-  return `${label}: mean ${format(mean)}, p50 ${format(p50)}, p95 ${format(p95)}`;
-}
-
-/**
- * Max count in buckets (for bar width).
- *
- * @param buckets - Histogram
- */
-export function maxBucketCount(buckets: readonly AiDistributionBucket[]): number {
-  return buckets.reduce((m, b) => Math.max(m, b.count), 0);
-}
-
-/**
- * Format an effect for the trail (tier vocabulary).
- *
- * @param effect - Tool effect
- */
-export function formatEffect(effect: AgentToolEffect): string {
-  return `${effect.kind} ${effect.resource}`;
-}
-
-/**
- * Label for a trail line — denial is never "error".
- *
- * @param status - ok | denied
- */
-export function trailStatusLabel(status: "ok" | "denied"): string {
-  return status === "denied" ? "DENIED" : "ok";
-}

package/src/console/ui/ai/group.test.ts

@@ -1,35 +0,0 @@
-/**
- * AI catalogue filter / group tests (console §9.10).
- */
-
-import { describe, expect, test } from "bun:test";
-import { AI_LIST_FIXTURE } from "./fixture.ts";
-import {
-  allowPiiStanding,
-  filterAgents,
-  filterPrompts,
-  runsForAgent,
-  versionsForPrompt,
-} from "./group.ts";
-
-describe("AI group helpers", () => {
-  test("filterPrompts matches name", () => {
-    expect(filterPrompts(AI_LIST_FIXTURE.prompts, "triage")).toHaveLength(1);
-    expect(filterPrompts(AI_LIST_FIXTURE.prompts, "nope")).toHaveLength(0);
-  });
-
-  test("versionsForPrompt sorts ascending", () => {
-    const vs = versionsForPrompt(AI_LIST_FIXTURE.versions, "ticket-triage");
-    expect(vs.map((v) => v.version)).toEqual([2, 3]);
-  });
-
-  test("allowPiiStanding keeps only acknowledgements", () => {
-    expect(allowPiiStanding(AI_LIST_FIXTURE.allowPii)).toHaveLength(1);
-    expect(allowPiiStanding(AI_LIST_FIXTURE.allowPii)[0]!.flowId).toBe("support.createTicket");
-  });
-
-  test("runsForAgent + filterAgents", () => {
-    expect(filterAgents(AI_LIST_FIXTURE.agents, "support")).toHaveLength(1);
-    expect(runsForAgent(AI_LIST_FIXTURE.agentRuns, "support")).toHaveLength(1);
-  });
-});