okengine 0.1.4

package/src/runs/duckdb.ts

@@ -0,0 +1,99 @@
+/**
+ * Thin DuckDB adapter — adopted via `@duckdb/node-api`, not reinvented.
+ */
+
+import {
+  DuckDBInstance,
+  type DuckDBConnection,
+} from "@duckdb/node-api";
+
+import type { RunsRow } from "./types.ts";
+
+/** DuckDB session — instance kept alive with the connection. */
+export interface DuckSession {
+  /** DuckDB instance (must outlive the connection). */
+  readonly instance: DuckDBInstance;
+  /** Active connection. */
+  readonly conn: DuckDBConnection;
+  /** Close connection + instance. */
+  close(): void;
+}
+
+/** Open an in-memory DuckDB session. */
+export async function openDuckDB(): Promise<DuckSession> {
+  const instance = await DuckDBInstance.create(":memory:");
+  const conn = await instance.connect();
+  return {
+    instance,
+    conn,
+    close() {
+      try {
+        (conn as { closeSync?: () => void }).closeSync?.();
+      } catch {
+        /* ignore */
+      }
+      try {
+        (instance as { closeSync?: () => void }).closeSync?.();
+      } catch {
+        /* ignore */
+      }
+    },
+  };
+}
+
+/**
+ * Run a query and return plain JSON-friendly row objects.
+ *
+ * @param conn - DuckDB connection
+ * @param sql - SQL text
+ */
+export async function duckQuery(
+  conn: DuckDBConnection,
+  sql: string,
+): Promise<RunsRow[]> {
+  const reader = await conn.runAndReadAll(sql);
+  const rows = reader.getRowObjectsJson() as RunsRow[];
+  return rows.map(normalizeRow);
+}
+
+/**
+ * Escape a filesystem path for use inside a DuckDB string literal.
+ *
+ * @param path - Absolute path
+ */
+export function duckPath(path: string): string {
+  return path.replaceAll("'", "''");
+}
+
+/**
+ * Quote an identifier.
+ *
+ * @param id - Column / table name
+ */
+export function duckIdent(id: string): string {
+  return `"${id.replaceAll('"', '""')}"`;
+}
+
+/**
+ * SQL literal for a JS value.
+ *
+ * @param value - Value to embed
+ */
+export function duckLiteral(value: unknown): string {
+  if (value === null || value === undefined) return "NULL";
+  if (typeof value === "number") {
+    return Number.isFinite(value) ? String(value) : "NULL";
+  }
+  if (typeof value === "boolean") return value ? "TRUE" : "FALSE";
+  if (typeof value === "bigint") return `${value}`;
+  const s = String(value).replaceAll("'", "''");
+  return `'${s}'`;
+}
+
+function normalizeRow(row: RunsRow): RunsRow {
+  const out: RunsRow = {};
+  for (const [k, v] of Object.entries(row)) {
+    out[k] = typeof v === "bigint" ? Number(v) : v;
+  }
+  return out;
+}

package/src/runs/index.ts

@@ -0,0 +1,82 @@
+/**
+ * Runs store — wide events, Parquet + DuckDB, outlier explanation, crypto-shred.
+ *
+ * Not a ninth element: observability derived from the effect system.
+ * Subpath: planned as part of the Console / runtime surface.
+ */
+
+export type {
+  RunsDriverId,
+  RunLogLine,
+  RunError,
+  RunDimensions,
+  WideEvent,
+  RunsRetention,
+  RunsRow,
+  RunsStore,
+  RunsDriver,
+  RunsOpenOptions,
+} from "./types.ts";
+
+export {
+  createRunTelemetry,
+  cacheDimensionOf,
+  type RunTelemetry,
+} from "./telemetry.ts";
+
+export {
+  collectWideEvent,
+  type CollectWideEventInput,
+} from "./collect.ts";
+
+export {
+  explainOutliers,
+  seedOutlierDataset,
+  type OutlierFinding,
+  type ExplainOutliersOptions,
+} from "./outlier.ts";
+
+export {
+  SUBJECT_KEY_PREFIX,
+  SHREDDED,
+  subjectKeyName,
+  subjectKeysFromVault,
+  createMemorySubjectKeys,
+  ensureSubjectKey,
+  archiveFields,
+  revealArchived,
+  eraseSubject,
+  type SubjectKeyVault,
+} from "./shred.ts";
+
+export {
+  wideEventToRow,
+  rowToWideEvent,
+  writeParquet,
+  readParquet,
+  partitionKey,
+  partitionObjectKey,
+  type ParquetRow,
+} from "./parquet.ts";
+
+export { openDuckDB, duckQuery, duckPath, duckLiteral } from "./duckdb.ts";
+
+export {
+  createRunsRuntime,
+  resolveRunsDriver,
+  filesRunsDriver,
+  memoryRunsDriver,
+  postgresRunsDriver,
+  clickhouseRunsDriver,
+  type CreateRunsRuntimeOptions,
+  type RunsRuntime,
+} from "./runtime.ts";
+
+export {
+  privacyErase,
+  type PrivacyEraseOptions,
+  type PrivacyEraseResult,
+} from "./privacy.ts";
+
+export { createRunsPostgresFake } from "./drivers/postgres.ts";
+export { createRunsClickHouseFake } from "./drivers/clickhouse.ts";

package/src/runs/outlier.ts

@@ -0,0 +1,180 @@
+/**
+ * Automatic outlier explanation — compare every dimension between a selected
+ * population and the rest (console §9.11).
+ *
+ * "94% cache=miss vs 6%" — the system surfaces separating dimensions nobody
+ * asked about, because all dimensions are declared and typed.
+ */
+
+import type { RunDimensions, WideEvent } from "./types.ts";
+
+/** One dimension that separates outliers from the baseline. */
+export interface OutlierFinding {
+  /** Dimension name. */
+  readonly dimension: string;
+  /** Value enriched in the outlier population. */
+  readonly value: string;
+  /** Share of outliers carrying this value (0–1). */
+  readonly outlierShare: number;
+  /** Share of baseline carrying this value (0–1). */
+  readonly baselineShare: number;
+  /** Absolute lift (`outlierShare - baselineShare`). */
+  readonly lift: number;
+  /** Human-readable line matching the Console voice. */
+  readonly explanation: string;
+}
+
+/** Options for {@link explainOutliers}. */
+export interface ExplainOutliersOptions {
+  /**
+   * Predicate selecting the outlier population (e.g. slow runs).
+   *
+   * @param event - Wide event
+   */
+  readonly select: (event: WideEvent) => boolean;
+  /** Max findings to return (default 10). */
+  readonly limit?: number;
+  /** Minimum absolute lift to report (default 0.15). */
+  readonly minLift?: number;
+  /** Dimensions to ignore. */
+  readonly ignore?: readonly string[];
+}
+
+/**
+ * Explain what separates a selected population from the rest.
+ *
+ * @param events - Seeded or queried wide events
+ * @param options - Selection + thresholds
+ */
+export function explainOutliers(
+  events: readonly WideEvent[],
+  options: ExplainOutliersOptions,
+): OutlierFinding[] {
+  const outliers = events.filter(options.select);
+  const baseline = events.filter((e) => !options.select(e));
+  if (outliers.length === 0 || baseline.length === 0) return [];
+
+  const ignore = new Set(options.ignore ?? ["duration_ms", "flow"]);
+  const minLift = options.minLift ?? 0.15;
+  const limit = options.limit ?? 10;
+
+  const dimNames = new Set<string>();
+  for (const e of events) {
+    for (const k of Object.keys(e.dimensions)) {
+      if (!ignore.has(k)) dimNames.add(k);
+    }
+  }
+
+  const findings: OutlierFinding[] = [];
+  for (const dim of dimNames) {
+    const outlierCounts = valueCounts(outliers, dim);
+    const baselineCounts = valueCounts(baseline, dim);
+    const values = new Set([
+      ...outlierCounts.keys(),
+      ...baselineCounts.keys(),
+    ]);
+    for (const value of values) {
+      const outlierShare = (outlierCounts.get(value) ?? 0) / outliers.length;
+      const baselineShare =
+        (baselineCounts.get(value) ?? 0) / baseline.length;
+      const lift = outlierShare - baselineShare;
+      if (lift < minLift) continue;
+      findings.push({
+        dimension: dim,
+        value,
+        outlierShare,
+        baselineShare,
+        lift,
+        explanation: formatExplanation(dim, value, outlierShare, baselineShare),
+      });
+    }
+  }
+
+  findings.sort((a, b) => b.lift - a.lift);
+  return findings.slice(0, limit);
+}
+
+/**
+ * Seed a dataset where one dimension cleanly separates slow runs.
+ *
+ * @param opts - Size + separating dimension
+ */
+export function seedOutlierDataset(opts?: {
+  readonly n?: number;
+  readonly slowShare?: number;
+  readonly separatingDimension?: string;
+  readonly separatingValue?: string;
+  readonly slowDurationMs?: number;
+  readonly fastDurationMs?: number;
+}): WideEvent[] {
+  const n = opts?.n ?? 1000;
+  const slowShare = opts?.slowShare ?? 0.1;
+  const dim = opts?.separatingDimension ?? "cache";
+  const sepValue = opts?.separatingValue ?? "miss";
+  const slowMs = opts?.slowDurationMs ?? 2000;
+  const fastMs = opts?.fastDurationMs ?? 50;
+  const slowCount = Math.floor(n * slowShare);
+
+  const events: WideEvent[] = [];
+  for (let i = 0; i < n; i++) {
+    const slow = i < slowCount;
+    // 90% of slow runs carry the separating value; 5% of fast runs do.
+    const carriesSep = slow ? i % 10 !== 0 : i % 20 === 0;
+    const dimValue = carriesSep ? sepValue : dim === "cache" ? "hit" : "other";
+    const durationMs = slow ? slowMs : fastMs;
+    const startedAt = 1_700_000_000_000 + i;
+    const dimensions: RunDimensions = {
+      flow: "book",
+      trigger: "http",
+      plane: "user",
+      tenant: i % 7 === 0 ? "org_a41" : `org_${i % 50}`,
+      cache: dim === "cache" ? dimValue : i % 3 === 0 ? "miss" : "hit",
+      [dim]: dimValue,
+      duration_ms: durationMs,
+    };
+    events.push({
+      id: `run_${i}`,
+      flow: "book",
+      trigger: "http",
+      plane: "user",
+      tenant: String(dimensions.tenant),
+      principal: `user_${i % 100}`,
+      subjectId: `user_${i % 100}`,
+      gates: ["member"],
+      cache: (dimensions.cache as "hit" | "miss") ?? "none",
+      effects: [],
+      logs: [],
+      durationMs,
+      startedAt,
+      endedAt: startedAt + durationMs,
+      dimensions,
+      error: null,
+    });
+  }
+  return events;
+}
+
+function valueCounts(
+  events: readonly WideEvent[],
+  dim: string,
+): Map<string, number> {
+  const m = new Map<string, number>();
+  for (const e of events) {
+    const raw = e.dimensions[dim];
+    if (raw === undefined || raw === null) continue;
+    const key = String(raw);
+    m.set(key, (m.get(key) ?? 0) + 1);
+  }
+  return m;
+}
+
+function formatExplanation(
+  dim: string,
+  value: string,
+  outlierShare: number,
+  baselineShare: number,
+): string {
+  const o = Math.round(outlierShare * 100);
+  const b = Math.round(baselineShare * 100);
+  return `${o}% ${dim}=${value} vs ${b}%`;
+}

package/src/runs/parquet.ts

@@ -0,0 +1,210 @@
+/**
+ * Parquet partition writer — DuckDB `COPY … (FORMAT PARQUET)`.
+ *
+ * Locality (local vs object storage) is decided by the files driver; this
+ * module only serialises wide events into columnar files.
+ */
+
+import { mkdir, unlink, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
+import { duckPath, duckQuery, openDuckDB } from "./duckdb.ts";
+import type { WideEvent } from "./types.ts";
+
+/** Flattened Parquet / SQL row. */
+export type ParquetRow = Record<string, string | number | boolean | null>;
+
+/**
+ * Flatten a wide event into a columnar row.
+ *
+ * Structured fields become JSON strings; scalar dimensions are promoted to
+ * top-level columns for outlier / filter queries.
+ *
+ * @param event - Wide event
+ */
+export function wideEventToRow(event: WideEvent): ParquetRow {
+  const row: ParquetRow = {
+    id: event.id,
+    parent_id: event.parentId ?? null,
+    flow: event.flow,
+    unit: event.unit ?? null,
+    trigger: event.trigger,
+    plane: event.plane,
+    tenant: event.tenant ?? null,
+    principal: event.principal ?? null,
+    subject_id: event.subjectId ?? null,
+    gates: JSON.stringify(event.gates),
+    cache: event.cache,
+    replica: event.replica ?? null,
+    replica_lag_ms: event.replicaLagMs ?? null,
+    cost: event.cost ?? null,
+    prompt_version: event.promptVersion ?? null,
+    build_version: event.buildVersion ?? null,
+    error_code: event.error?.code ?? null,
+    error_message: event.error?.message ?? null,
+    effects: JSON.stringify(event.effects),
+    logs: JSON.stringify(event.logs),
+    duration_ms: event.durationMs,
+    started_at: event.startedAt,
+    ended_at: event.endedAt,
+    archived: JSON.stringify(event.archived ?? {}),
+    dimensions: JSON.stringify(event.dimensions),
+  };
+  for (const [k, v] of Object.entries(event.dimensions)) {
+    const col = `dim_${k}`;
+    if (row[col] !== undefined) continue;
+    if (v === undefined) continue;
+    row[col] =
+      typeof v === "string" || typeof v === "number" || typeof v === "boolean"
+        ? v
+        : v === null
+          ? null
+          : String(v);
+  }
+  return row;
+}
+
+/**
+ * Reconstruct a {@link WideEvent} from a Parquet / SQL row.
+ *
+ * @param row - Flat row
+ */
+export function rowToWideEvent(row: Record<string, unknown>): WideEvent {
+  const dimensions =
+    typeof row.dimensions === "string"
+      ? (JSON.parse(row.dimensions) as WideEvent["dimensions"])
+      : ((row.dimensions as WideEvent["dimensions"]) ?? {});
+  const archivedRaw =
+    typeof row.archived === "string"
+      ? (JSON.parse(row.archived) as Record<string, string>)
+      : ((row.archived as Record<string, string>) ?? {});
+  const errorCode = row.error_code;
+  return {
+    id: String(row.id),
+    ...(row.parent_id != null ? { parentId: String(row.parent_id) } : {}),
+    flow: String(row.flow),
+    ...(row.unit != null ? { unit: String(row.unit) } : {}),
+    trigger: String(row.trigger),
+    plane: (row.plane as WideEvent["plane"]) ?? "user",
+    tenant: row.tenant != null ? String(row.tenant) : null,
+    principal: row.principal != null ? String(row.principal) : null,
+    subjectId: row.subject_id != null ? String(row.subject_id) : null,
+    gates: parseJsonArray(row.gates) as string[],
+    cache: (row.cache as WideEvent["cache"]) ?? "none",
+    ...(row.replica != null
+      ? { replica: row.replica as "primary" | "replica" }
+      : {}),
+    ...(row.replica_lag_ms != null
+      ? { replicaLagMs: Number(row.replica_lag_ms) }
+      : {}),
+    ...(row.cost != null ? { cost: Number(row.cost) } : {}),
+    ...(row.prompt_version != null
+      ? { promptVersion: Number(row.prompt_version) }
+      : {}),
+    ...(row.build_version != null
+      ? { buildVersion: String(row.build_version) }
+      : {}),
+    error:
+      errorCode != null
+        ? {
+            code: String(errorCode),
+            ...(row.error_message != null
+              ? { message: String(row.error_message) }
+              : {}),
+          }
+        : null,
+    effects: parseJsonArray(row.effects) as WideEvent["effects"],
+    logs: parseJsonArray(row.logs) as WideEvent["logs"],
+    durationMs: Number(row.duration_ms ?? 0),
+    startedAt: Number(row.started_at ?? 0),
+    endedAt: Number(row.ended_at ?? 0),
+    ...(Object.keys(archivedRaw).length > 0 ? { archived: archivedRaw } : {}),
+    dimensions,
+  };
+}
+
+/**
+ * Write rows to a Parquet file via DuckDB (JSONL → COPY).
+ *
+ * @param path - Absolute output path
+ * @param rows - Flattened rows
+ */
+export async function writeParquet(
+  path: string,
+  rows: readonly ParquetRow[],
+): Promise<void> {
+  if (rows.length === 0) return;
+  await mkdir(dirname(path), { recursive: true });
+  const jsonl = `${path}.jsonl`;
+  await writeFile(
+    jsonl,
+    rows.map((r) => JSON.stringify(r)).join("\n"),
+  );
+  const session = await openDuckDB();
+  try {
+    await session.conn.run(
+      `COPY (SELECT * FROM read_json_auto('${duckPath(jsonl)}')) TO '${duckPath(path)}' (FORMAT PARQUET, COMPRESSION ZSTD)`,
+    );
+  } finally {
+    session.close();
+    await unlink(jsonl).catch(() => undefined);
+  }
+}
+
+/**
+ * Read all rows from one or more Parquet files.
+ *
+ * @param paths - Absolute Parquet paths
+ */
+export async function readParquet(
+  paths: readonly string[],
+): Promise<ParquetRow[]> {
+  if (paths.length === 0) return [];
+  const session = await openDuckDB();
+  try {
+    const list = paths.map((p) => `'${duckPath(p)}'`).join(", ");
+    const rows = await duckQuery(
+      session.conn,
+      `SELECT * FROM read_parquet([${list}], union_by_name = true)`,
+    );
+    return rows as ParquetRow[];
+  } finally {
+    session.close();
+  }
+}
+
+/**
+ * Partition key from an event timestamp (day UTC).
+ *
+ * @param startedAt - Epoch-ms
+ */
+export function partitionKey(startedAt: number): string {
+  const d = new Date(startedAt);
+  const y = d.getUTCFullYear();
+  const m = String(d.getUTCMonth() + 1).padStart(2, "0");
+  const day = String(d.getUTCDate()).padStart(2, "0");
+  return `${y}-${m}-${day}`;
+}
+
+/**
+ * Build a partition object key.
+ *
+ * @param day - Partition day
+ * @param id - Unique file id
+ */
+export function partitionObjectKey(day: string, id: string): string {
+  return join("runs", `day=${day}`, `${id}.parquet`);
+}
+
+function parseJsonArray(value: unknown): unknown[] {
+  if (Array.isArray(value)) return value;
+  if (typeof value === "string" && value.length > 0) {
+    try {
+      const parsed: unknown = JSON.parse(value);
+      return Array.isArray(parsed) ? parsed : [];
+    } catch {
+      return [];
+    }
+  }
+  return [];
+}

package/src/runs/privacy.ts

@@ -0,0 +1,49 @@
+/**
+ * Privacy erase — crypto-shred a subject across the runs store (and later
+ * journal / channel receipts). Deletes the Vault key, not the terabytes.
+ */
+
+import {
+  eraseSubject,
+  subjectKeyName,
+  type SubjectKeyVault,
+} from "./shred.ts";
+
+/** Options for {@link privacyErase}. */
+export interface PrivacyEraseOptions {
+  /** Subject to erase. */
+  readonly subjectId: string;
+  /** Subject-key vault (required). */
+  readonly subjectKeys: SubjectKeyVault;
+  /** Write stdout. */
+  readonly write?: (text: string) => void;
+}
+
+/** Result of a privacy erase. */
+export interface PrivacyEraseResult {
+  /** Subject id. */
+  readonly subjectId: string;
+  /** Vault key that was targeted. */
+  readonly key: string;
+  /** Whether the key existed and was deleted. */
+  readonly deleted: boolean;
+}
+
+/**
+ * Crypto-shred a subject by deleting their per-subject Vault key.
+ *
+ * @param options - Subject + vault
+ */
+export function privacyErase(
+  options: PrivacyEraseOptions,
+): PrivacyEraseResult {
+  const key = subjectKeyName(options.subjectId);
+  const deleted = eraseSubject(options.subjectKeys, options.subjectId);
+  const write = options.write ?? ((t) => process.stdout.write(t));
+  write(
+    deleted
+      ? `oke privacy erase: shredded subject ${options.subjectId} (deleted ${key})\n`
+      : `oke privacy erase: no key for subject ${options.subjectId} (${key})\n`,
+  );
+  return { subjectId: options.subjectId, key, deleted };
+}