okengine 0.9.1 → 0.10.0

package/src/console/ui/runs/errors.ts

@@ -0,0 +1,88 @@
+/**
+ * Error-pattern aggregation over Runs (console §9.11).
+ *
+ * Reuses {@link groupByDimension} on `error` — no parallel tracking store.
+ */
+
+import { groupByDimension } from "./group.ts";
+import type { GroupAggregate, RunRecord } from "./types.ts";
+
+/** Default lookback for "errors in the last hour". */
+export const ERROR_WINDOW_1H_MS = 60 * 60 * 1000;
+
+/**
+ * Runs that failed inside `[now - windowMs, now]`.
+ *
+ * @param runs - Population
+ * @param now - Epoch-ms
+ * @param windowMs - Lookback window
+ */
+export function failedRunsInWindow(
+  runs: readonly RunRecord[],
+  now: number,
+  windowMs: number,
+): RunRecord[] {
+  const from = now - Math.max(0, windowMs);
+  return runs.filter(
+    (r) => r.startedAt >= from && r.startedAt <= now && r.error != null && r.error !== "",
+  );
+}
+
+/**
+ * Aggregate "this error occurred N times in the window".
+ *
+ * @param runs - Population
+ * @param now - Epoch-ms
+ * @param windowMs - Lookback (default 1h)
+ */
+export function errorPatterns(
+  runs: readonly RunRecord[],
+  now: number,
+  windowMs: number = ERROR_WINDOW_1H_MS,
+): GroupAggregate[] {
+  return groupByDimension(failedRunsInWindow(runs, now, windowMs), "error");
+}
+
+/**
+ * Filter a population to runs started at or after `sinceMs`.
+ *
+ * @param runs - Population
+ * @param sinceMs - Inclusive lower bound (epoch-ms); omitted = no filter
+ */
+export function filterRunsSince(
+  runs: readonly RunRecord[],
+  sinceMs: number | undefined,
+): RunRecord[] {
+  if (sinceMs === undefined) return [...runs];
+  return runs.filter((r) => r.startedAt >= sinceMs);
+}
+
+/**
+ * Parse a lookback duration (`1h`, `5m`, `30m`, `24h`) or epoch-ms string.
+ *
+ * @param raw - Search param value
+ */
+export function parseSinceWindowMs(raw: string | undefined): number | undefined {
+  if (!raw || !raw.trim()) return undefined;
+  const trimmed = raw.trim();
+  if (/^\d+$/.test(trimmed)) {
+    const n = Number(trimmed);
+    return Number.isFinite(n) && n > 0 ? n : undefined;
+  }
+  const match = /^(\d+(?:\.\d+)?)\s*(ms|s|m|h|d)$/i.exec(trimmed);
+  if (!match) return undefined;
+  const n = Number(match[1]);
+  if (!Number.isFinite(n) || n <= 0) return undefined;
+  const unit = match[2]!.toLowerCase();
+  const mult =
+    unit === "ms"
+      ? 1
+      : unit === "s"
+        ? 1_000
+        : unit === "m"
+          ? 60_000
+          : unit === "h"
+            ? 3_600_000
+            : 86_400_000;
+  return Math.round(n * mult);
+}

package/src/console/ui/runs/index.ts

@@ -5,6 +5,13 @@
  * without mounting the full SPA.
  */
 
+export {
+  ERROR_WINDOW_1H_MS,
+  errorPatterns,
+  failedRunsInWindow,
+  filterRunsSince,
+  parseSinceWindowMs,
+} from "./errors.ts";
 export { explainDurationOutliers, type OutlierFinding } from "./explain.ts";
 export { RUNS_CHAIN_FIXTURE, RUNS_FIXTURE_T0, runsOutlierFixture } from "./fixture.ts";
 export { discoverDimensions, groupByDimension } from "./group.ts";
@@ -38,6 +45,7 @@ export {
   serializeRunsSearch,
   setDurationRange,
   setGroup,
+  setSince,
   setWhere,
   type RunsSearch,
 } from "./search.ts";

package/src/console/ui/runs/search.ts

@@ -20,6 +20,11 @@ export const RunsSearchSchema = z.object({
   durMin: z.coerce.number().optional(),
   /** Duration brush upper bound (ms). */
   durMax: z.coerce.number().optional(),
+  /**
+   * Lookback window: duration (`1h`, `5m`, `24h`) or epoch-ms lower bound.
+   * Filters the population by `startedAt`.
+   */
+  since: z.string().optional(),
 });
 
 /** Parsed, typed Runs search state. */
@@ -48,9 +53,20 @@ export function serializeRunsSearch(search: RunsSearch): Record<string, string |
   if (search.run) out.run = search.run;
   if (search.durMin !== undefined) out.durMin = search.durMin;
   if (search.durMax !== undefined) out.durMax = search.durMax;
+  if (search.since) out.since = search.since;
   return out;
 }
 
+/**
+ * Set or clear the lookback window (`1h`, `5m`, …).
+ *
+ * @param prev - Current search
+ * @param since - Window string or null to clear
+ */
+export function setSince(prev: RunsSearch, since: string | null): RunsSearch {
+  return { ...prev, since: since ?? undefined };
+}
+
 /**
  * Dimension query derived from search.
  *

package/src/console/ui/shell/panels/runs/RunsPanel.tsx

@@ -13,18 +13,22 @@ import {
   discoverDimensions,
   durationHistogram,
   durationRangeOf,
+  errorPatterns,
   explainDurationOutliers,
   filterRuns,
+  filterRunsSince,
   formatDurationMs,
   groupByDimension,
   normalizeRange,
   openRun,
   parseDimensionQuery,
+  parseSinceWindowMs,
   removeClause,
   rowToRun,
   serializeRunsSearch,
   setDurationRange,
   setGroup,
+  setSince,
   setWhere,
   rootIdOf,
   shouldOfferTracesLink,
@@ -63,7 +67,16 @@ export function RunsPanel() {
 
   const allRuns = runsQuery.data ?? [];
   const query = dimensionQueryOf(search);
-  const filtered = useMemo(() => filterRuns(allRuns, query), [allRuns, query]);
+  const sinceParam = search.since ?? "1h";
+  const windowMs = parseSinceWindowMs(sinceParam === "" ? undefined : sinceParam);
+  const sinceMs =
+    windowMs === undefined
+      ? undefined
+      : /^\d+$/.test(sinceParam.trim())
+        ? windowMs
+        : Date.now() - windowMs;
+  const windowed = useMemo(() => filterRunsSince(allRuns, sinceMs), [allRuns, sinceMs]);
+  const filtered = useMemo(() => filterRuns(windowed, query), [windowed, query]);
   const range = durationRangeOf(search);
   const buckets = useMemo(() => durationHistogram(filtered), [filtered]);
   const maxBucket = Math.max(1, ...buckets.map((b) => b.count));
@@ -71,6 +84,10 @@ export function RunsPanel() {
     () => (search.group ? groupByDimension(filtered, search.group) : []),
     [filtered, search.group],
   );
+  const patterns = useMemo(
+    () => errorPatterns(windowed, Date.now(), windowMs ?? 60 * 60 * 1000),
+    [windowed, windowMs],
+  );
   const findings = useMemo(
     () => (range ? explainDurationOutliers(filtered, range) : []),
     [filtered, range],
@@ -89,6 +106,23 @@ export function RunsPanel() {
           </p>
         </div>
         <QueryBuilder search={search} dimensions={dimensions} onChange={setSearch} />
+        <label className="flex flex-col gap-1 text-sm">
+          <span className="text-[var(--oke-muted)]">Since</span>
+          <select
+            aria-label="Lookback window"
+            className="min-h-8 border border-[var(--oke-line)] bg-transparent px-2 text-sm"
+            value={sinceParam}
+            onChange={(e) => {
+              const v = e.target.value;
+              setSearch(setSince(search, v === "" ? "" : v));
+            }}
+          >
+            <option value="5m">Last 5m</option>
+            <option value="1h">Last 1h</option>
+            <option value="24h">Last 24h</option>
+            <option value="">All time</option>
+          </select>
+        </label>
         <label className="flex flex-col gap-1 text-sm">
           <span className="text-[var(--oke-muted)]">Group by</span>
           <select
@@ -162,6 +196,43 @@ export function RunsPanel() {
             </section>
           ) : null}
 
+          <section aria-label="Error patterns" className="mb-8">
+            <h2 className="mb-2 text-sm font-medium">Error patterns</h2>
+            <p className="mb-3 text-xs text-[var(--oke-muted)]">
+              Failed runs in the selected window, grouped by error code.
+            </p>
+            {patterns.length === 0 ? (
+              <p className="text-sm text-[var(--oke-muted)]">No errors in this window.</p>
+            ) : (
+              <table className="w-full border-collapse text-left text-sm">
+                <thead>
+                  <tr className="border-b border-[var(--oke-line)] text-[var(--oke-muted)]">
+                    <th scope="col" className="py-2 pr-3 font-medium">
+                      Error
+                    </th>
+                    <th scope="col" className="py-2 pr-3 font-medium">
+                      Count
+                    </th>
+                    <th scope="col" className="py-2 font-medium">
+                      p99
+                    </th>
+                  </tr>
+                </thead>
+                <tbody className="divide-y divide-[var(--oke-line)]">
+                  {patterns.map((p) => (
+                    <tr key={p.key}>
+                      <td className="py-2 pr-3 font-mono text-xs text-[var(--oke-danger)]">
+                        {p.key}
+                      </td>
+                      <td className="py-2 pr-3">{p.count}</td>
+                      <td className="py-2">{formatDurationMs(p.p99DurationMs)}</td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            )}
+          </section>
+
           {search.group ? (
             <section aria-label="Group aggregates" className="mb-8">
               <h2 className="mb-2 text-sm font-medium">Group by {search.group}</h2>

package/src/docker/ai-model-status.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, test } from "bun:test";
+import {
+  aiModelIdsMatch,
+  formatAiModelStatusMessage,
+  normalizeAiProbeUrl,
+  probeAiModelStatus,
+  startAiModelWatch,
+  statusFromOllamaTags,
+  statusFromOpenAiModels,
+} from "./ai-model-status.ts";
+
+describe("ai-model-status", () => {
+  test("normalizeAiProbeUrl keeps a single /v1 for openai-compatible", () => {
+    expect(normalizeAiProbeUrl("http://127.0.0.1:8080", "openai-compatible")).toBe(
+      "http://127.0.0.1:8080/v1",
+    );
+    expect(normalizeAiProbeUrl("http://127.0.0.1:8080/v1/", "openai-compatible")).toBe(
+      "http://127.0.0.1:8080/v1",
+    );
+    expect(normalizeAiProbeUrl("http://127.0.0.1:11434/v1", "ollama")).toBe(
+      "http://127.0.0.1:11434",
+    );
+  });
+
+  test("aiModelIdsMatch tolerates llama.cpp quant aliasing", () => {
+    expect(aiModelIdsMatch("gemma4:e4b-q4_K_M", "gemma4:Q4_K_M")).toBe(true);
+    expect(aiModelIdsMatch("smollm2", "smollm2")).toBe(true);
+    expect(aiModelIdsMatch("gemma4:e4b-q4_K_M", "qwen3:8b")).toBe(false);
+  });
+
+  test("statusFromOpenAiModels maps loading → ready", () => {
+    const loading = statusFromOpenAiModels("gemma4:e4b-q4_K_M", {
+      data: [{ id: "gemma4:Q4_K_M", status: { value: "loading" } }],
+    });
+    expect(loading.phase).toBe("loading");
+    expect(loading.reportedId).toBe("gemma4:Q4_K_M");
+
+    const ready = statusFromOpenAiModels("smollm2", {
+      data: [{ id: "smollm2", status: { value: "ready" } }],
+    });
+    expect(ready.phase).toBe("ready");
+  });
+
+  test("statusFromOpenAiModels treats empty list as starting", () => {
+    expect(statusFromOpenAiModels("smollm2", { data: [] }).phase).toBe("starting");
+  });
+
+  test("statusFromOllamaTags ready when present", () => {
+    expect(statusFromOllamaTags("qwen3.5:9b", { models: [{ name: "qwen3.5:9b" }] }).phase).toBe(
+      "ready",
+    );
+    expect(statusFromOllamaTags("qwen3.5:9b", { models: [] }).phase).toBe("loading");
+  });
+
+  test("probeAiModelStatus uses injectable fetch", async () => {
+    const status = await probeAiModelStatus({
+      url: "http://127.0.0.1:9/v1",
+      model: "smollm2",
+      kind: "openai-compatible",
+      fetch: async () =>
+        new Response(JSON.stringify({ data: [{ id: "smollm2", status: { value: "ready" } }] }), {
+          status: 200,
+          headers: { "content-type": "application/json" },
+        }),
+    });
+    expect(status.phase).toBe("ready");
+    expect(formatAiModelStatusMessage(status)).toBe("AI smollm2 — ready");
+  });
+
+  test("startAiModelWatch stops on ready and cancels cleanly", async () => {
+    const messages: string[] = [];
+    let calls = 0;
+    const phases: string[] = [];
+    const stop = startAiModelWatch({
+      url: "http://127.0.0.1:9/v1",
+      model: "smollm2",
+      kind: "openai-compatible",
+      intervalMs: 1,
+      timeoutMs: 5_000,
+      onStatus: (m, status) => {
+        messages.push(m);
+        phases.push(status.phase);
+      },
+      sleep: async () => {},
+      fetch: async () => {
+        calls += 1;
+        const phase = calls < 2 ? "loading" : "ready";
+        return new Response(
+          JSON.stringify({ data: [{ id: "smollm2", status: { value: phase } }] }),
+          { status: 200, headers: { "content-type": "application/json" } },
+        );
+      },
+    });
+    await Bun.sleep(20);
+    expect(messages).toContain("AI smollm2 — loading…");
+    expect(messages).toContain("AI smollm2 — ready");
+    expect(phases).toContain("loading");
+    expect(phases).toContain("ready");
+    stop();
+  });
+});

package/src/docker/ai-model-status.ts

@@ -0,0 +1,276 @@
+/**
+ * Probe local AI servers for configured-model readiness (llama.cpp / Ollama /
+ * OpenAI-compatible). Used by `oke dev` to show model id + phase without
+ * blocking boot.
+ */
+
+/** Normalized readiness phase for a configured model. */
+export type AiModelPhase = "unreachable" | "starting" | "loading" | "ready" | "error";
+
+/** Snapshot of one model’s readiness. */
+export type AiModelStatus = {
+  /** Requested model id (`OKE_AI_MODEL`). */
+  readonly model: string;
+  readonly phase: AiModelPhase;
+  /** Optional short detail (error text, reported id). */
+  readonly detail?: string;
+  /** Id reported by the server when it differs from {@link model}. */
+  readonly reportedId?: string;
+};
+
+/** Which HTTP surface to probe. */
+export type AiModelProbeKind = "openai-compatible" | "ollama";
+
+/** Fetch contract (injectable for tests). */
+export type AiModelFetch = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+/** Options for {@link probeAiModelStatus}. */
+export type ProbeAiModelStatusOptions = {
+  /** Base URL — with or without trailing `/v1` for OpenAI-compatible. */
+  readonly url: string;
+  readonly model: string;
+  readonly kind: AiModelProbeKind;
+  readonly fetch?: AiModelFetch;
+};
+
+/**
+ * Strip trailing slashes; for OpenAI-compatible keep a single `/v1` suffix.
+ *
+ * @param url - Raw base URL
+ * @param kind - Probe kind
+ */
+export function normalizeAiProbeUrl(url: string, kind: AiModelProbeKind): string {
+  let base = url.trim().replace(/\/+$/, "");
+  if (kind === "openai-compatible") {
+    if (!base.endsWith("/v1")) base = `${base}/v1`;
+  } else if (base.endsWith("/v1")) {
+    base = base.slice(0, -3);
+  }
+  return base;
+}
+
+/**
+ * Whether a reported model id covers the requested `OKE_AI_MODEL`.
+ *
+ * Tolerates llama.cpp alias normalization (`gemma4:e4b-q4_K_M` ↔ `gemma4:Q4_K_M`).
+ *
+ * @param want - Requested id
+ * @param reported - Server id
+ */
+export function aiModelIdsMatch(want: string, reported: string): boolean {
+  const a = want.trim();
+  const b = reported.trim();
+  if (!a || !b) return false;
+  if (a === b) return true;
+  const aLower = a.toLowerCase();
+  const bLower = b.toLowerCase();
+  if (aLower === bLower) return true;
+  const aBase = aLower.split(":")[0] ?? aLower;
+  const bBase = bLower.split(":")[0] ?? bLower;
+  if (aBase !== bBase) return false;
+  const aTag = aLower.slice(aBase.length + 1);
+  const bTag = bLower.slice(bBase.length + 1);
+  if (!aTag || !bTag) return true;
+  if (aTag === bTag) return true;
+  // Quant suffixes: e4b-q4_k_m vs q4_k_m
+  if (aTag.endsWith(bTag) || bTag.endsWith(aTag)) return true;
+  const norm = (t: string) => t.replace(/[^a-z0-9]/g, "");
+  return norm(aTag).includes(norm(bTag)) || norm(bTag).includes(norm(aTag));
+}
+
+/**
+ * Parse OpenAI-compatible `GET /v1/models` body into a status for `model`.
+ *
+ * @param model - Requested id
+ * @param json - Parsed JSON
+ */
+export function statusFromOpenAiModels(model: string, json: unknown): AiModelStatus {
+  if (!json || typeof json !== "object") {
+    return { model, phase: "starting", detail: "invalid /v1/models body" };
+  }
+  const data = (json as { data?: unknown }).data;
+  if (!Array.isArray(data)) {
+    return { model, phase: "starting", detail: "no model list" };
+  }
+  if (data.length === 0) {
+    return { model, phase: "starting", detail: "no models registered yet" };
+  }
+  for (const row of data) {
+    if (!row || typeof row !== "object") continue;
+    const id = (row as { id?: unknown }).id;
+    if (typeof id !== "string" || !aiModelIdsMatch(model, id)) continue;
+    const status = (row as { status?: unknown }).status;
+    const value =
+      typeof status === "string"
+        ? status
+        : status &&
+            typeof status === "object" &&
+            typeof (status as { value?: unknown }).value === "string"
+          ? (status as { value: string }).value
+          : undefined;
+    const phase = mapServerPhase(value);
+    return {
+      model,
+      phase,
+      reportedId: id,
+      ...(phase === "error" && value ? { detail: value } : {}),
+    };
+  }
+  return {
+    model,
+    phase: "starting",
+    detail: `waiting for ${model} (server has ${data.length} other model(s))`,
+  };
+}
+
+/**
+ * Parse Ollama `GET /api/tags` into a status for `model`.
+ *
+ * @param model - Requested id
+ * @param json - Parsed JSON
+ */
+export function statusFromOllamaTags(model: string, json: unknown): AiModelStatus {
+  if (!json || typeof json !== "object") {
+    return { model, phase: "starting", detail: "invalid /api/tags body" };
+  }
+  const models = (json as { models?: unknown }).models;
+  if (!Array.isArray(models)) {
+    return { model, phase: "starting" };
+  }
+  for (const row of models) {
+    if (!row || typeof row !== "object") continue;
+    const name = (row as { name?: unknown }).name;
+    if (typeof name === "string" && aiModelIdsMatch(model, name)) {
+      return { model, phase: "ready", reportedId: name };
+    }
+  }
+  return { model, phase: "loading", detail: "not in /api/tags yet" };
+}
+
+/**
+ * Map a server-native status string to {@link AiModelPhase}.
+ *
+ * @param value - Raw status
+ */
+function mapServerPhase(value: string | undefined): AiModelPhase {
+  if (!value) return "ready";
+  const v = value.trim().toLowerCase();
+  if (v === "ready" || v === "running" || v === "loaded" || v === "idle") return "ready";
+  if (v === "loading" || v === "pulling" || v === "downloading" || v === "starting") {
+    return v === "starting" ? "starting" : "loading";
+  }
+  if (v === "error" || v === "failed" || v === "unhealthy") return "error";
+  return "loading";
+}
+
+/**
+ * Probe the AI HTTP API once.
+ *
+ * @param opts - URL + model + kind
+ */
+export async function probeAiModelStatus(opts: ProbeAiModelStatusOptions): Promise<AiModelStatus> {
+  const model = opts.model.trim();
+  if (!model) return { model: "", phase: "error", detail: "OKE_AI_MODEL is empty" };
+  const fetchFn = opts.fetch ?? globalThis.fetch;
+  const base = normalizeAiProbeUrl(opts.url, opts.kind);
+  const path = opts.kind === "ollama" ? `${base}/api/tags` : `${base}/models`;
+  try {
+    const res = await fetchFn(path, { method: "GET" });
+    if (!res.ok) {
+      return {
+        model,
+        phase: res.status >= 500 ? "starting" : "error",
+        detail: `HTTP ${res.status}`,
+      };
+    }
+    const json: unknown = await res.json().catch(() => null);
+    return opts.kind === "ollama"
+      ? statusFromOllamaTags(model, json)
+      : statusFromOpenAiModels(model, json);
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    return { model, phase: "unreachable", detail: msg };
+  }
+}
+
+/**
+ * One-line status for `oke dev` logs.
+ *
+ * @param status - Probe result
+ */
+export function formatAiModelStatusMessage(status: AiModelStatus): string {
+  const id =
+    status.reportedId && status.reportedId !== status.model
+      ? `${status.model} (${status.reportedId})`
+      : status.model || "(unset)";
+  switch (status.phase) {
+    case "unreachable":
+      return `AI ${id} — waiting for server…`;
+    case "starting":
+      return `AI ${id} — starting…`;
+    case "loading":
+      return `AI ${id} — loading…`;
+    case "ready":
+      return `AI ${id} — ready`;
+    case "error":
+      return `AI ${id} — error${status.detail ? ` (${status.detail})` : ""}`;
+  }
+}
+
+/** Options for {@link startAiModelWatch}. */
+export type StartAiModelWatchOptions = ProbeAiModelStatusOptions & {
+  /** Called when the phase (or message) changes. */
+  readonly onStatus: (message: string, status: AiModelStatus) => void;
+  /** Poll interval (ms). Default 2000. */
+  readonly intervalMs?: number;
+  /** Give up after this many ms. Default 30 minutes. */
+  readonly timeoutMs?: number;
+  /** Injectable clock (tests). */
+  readonly now?: () => number;
+  /** Injectable sleep (tests). */
+  readonly sleep?: (ms: number) => Promise<void>;
+};
+
+/**
+ * Poll model readiness in the background until ready, error, timeout, or stop.
+ *
+ * @param opts - Probe + callbacks
+ * @returns Stop function
+ */
+export function startAiModelWatch(opts: StartAiModelWatchOptions): () => void {
+  const intervalMs = opts.intervalMs ?? 2_000;
+  const timeoutMs = opts.timeoutMs ?? 30 * 60_000;
+  const now = opts.now ?? Date.now;
+  const sleep = opts.sleep ?? ((ms: number) => Bun.sleep(ms));
+  let stopped = false;
+  let lastMessage = "";
+  const startedAt = now();
+
+  const tick = async (): Promise<void> => {
+    while (!stopped) {
+      const status = await probeAiModelStatus(opts);
+      if (stopped) return;
+      const message = formatAiModelStatusMessage(status);
+      if (message !== lastMessage) {
+        lastMessage = message;
+        opts.onStatus(message, status);
+      }
+      if (status.phase === "ready" || status.phase === "error") return;
+      if (now() - startedAt >= timeoutMs) {
+        const timed: AiModelStatus = {
+          model: opts.model,
+          phase: "error",
+          detail: `still ${status.phase} after ${Math.round(timeoutMs / 60_000)}m`,
+        };
+        opts.onStatus(formatAiModelStatusMessage(timed), timed);
+        return;
+      }
+      await sleep(intervalMs);
+    }
+  };
+
+  void tick();
+  return () => {
+    stopped = true;
+  };
+}