okengine 0.1.4

package/src/runs/runs.test.ts

@@ -0,0 +1,324 @@
+/**
+ * Runs store acceptance:
+ * - a run carries all declared dimensions with zero instrumentation
+ * - outlier explanation correctly identifies the separating dimension on a seeded dataset
+ * - a query spanning local and object-storage partitions returns one result set
+ * - erasure renders a subject's archived fields unreadable without rewriting partitions
+ * - fx.log lines are a field on the run, not a stream
+ * - optional postgres / clickhouse drivers append + query
+ */
+
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { fsDriver, memoryFilesDriver } from "../drivers/index.ts";
+import { oke } from "../kernel/app.ts";
+import { flow } from "../kernel/flow.ts";
+import { on, resetBindings } from "../kernel/on.ts";
+import { http } from "../kernel/triggers.ts";
+import {
+  SHREDDED,
+  archiveFields,
+  createMemorySubjectKeys,
+  createRunsRuntime,
+  explainOutliers,
+  privacyErase,
+  revealArchived,
+  seedOutlierDataset,
+  subjectKeyName,
+} from "./index.ts";
+
+const temps: string[] = [];
+
+afterEach(async () => {
+  resetBindings();
+  for (const t of temps.splice(0)) {
+    await rm(t, { recursive: true, force: true }).catch(() => undefined);
+  }
+});
+
+async function tmpDir(): Promise<string> {
+  const d = await mkdtemp(join(tmpdir(), "oke-runs-test-"));
+  temps.push(d);
+  return d;
+}
+
+describe("zero-instrumentation dimensions", () => {
+  test("a run carries declared dimensions without flow instrumentation", async () => {
+    resetBindings();
+    const runs = createRunsRuntime({
+      driver: "memory",
+      buildVersion: "1.2.3",
+    });
+    await runs.open();
+
+    on(
+      http.get("/book").gate("member"),
+      flow({
+        name: "book.create",
+        unit: "bookings",
+        plane: "user",
+        effects: { reads: ["sql:bookings"] },
+        do: async (_input, fx) => {
+          fx.log.info("booking started", { step: 1 });
+          await fx.cache.set("k", "v");
+          const hit = await fx.cache.get("k");
+          expect(hit).toBe("v");
+          return { ok: true };
+        },
+      }),
+    );
+
+    const app = oke({
+      name: "runs-dims",
+      runs,
+      fx: {
+        auth: { userId: "user_42", scopes: new Set(["book:write"]) },
+        tenant: { id: "org_a41" },
+      },
+    });
+
+    const matched = app.router.match("GET", "/book");
+    expect(matched).toBeTruthy();
+    await app.execute(matched!.value.flow, {}, matched!.value.trigger);
+
+    await runs.flush();
+    const events = await runs.all();
+    expect(events.length).toBe(1);
+    const e = events[0]!;
+    expect(e.flow).toBe("book.create");
+    expect(e.unit).toBe("bookings");
+    expect(e.trigger).toBe("http");
+    expect(e.plane).toBe("user");
+    expect(e.tenant).toBe("org_a41");
+    expect(e.principal).toBe("user_42");
+    expect(e.gates).toContain("member");
+    expect(e.cache).toBe("hit");
+    expect(e.buildVersion).toBe("1.2.3");
+    expect(e.logs.length).toBe(1);
+    expect(e.logs[0]!.message).toBe("booking started");
+    expect(e.dimensions.flow).toBe("book.create");
+    expect(e.dimensions.tenant).toBe("org_a41");
+    expect(e.dimensions.cache).toBe("hit");
+
+    await runs.close();
+  });
+});
+
+describe("outlier explanation", () => {
+  test("identifies the separating dimension on a seeded dataset", () => {
+    const events = seedOutlierDataset({
+      n: 1000,
+      slowShare: 0.1,
+      separatingDimension: "cache",
+      separatingValue: "miss",
+      slowDurationMs: 2000,
+      fastDurationMs: 40,
+    });
+    const findings = explainOutliers(events, {
+      select: (e) => e.durationMs >= 1000,
+      minLift: 0.2,
+    });
+    expect(findings.length).toBeGreaterThan(0);
+    const top = findings[0]!;
+    expect(top.dimension).toBe("cache");
+    expect(top.value).toBe("miss");
+    expect(top.lift).toBeGreaterThan(0.5);
+    expect(top.explanation).toMatch(/cache=miss/);
+  });
+});
+
+describe("locality — local + object storage", () => {
+  test("query spanning local and remote partitions returns one result set", async () => {
+    const root = await tmpDir();
+    const remoteBucket = await memoryFilesDriver.open({ name: "runs-remote" });
+    // hot window 1 hour — events older than that go remote
+    const hotWindowMs = 60 * 60 * 1000;
+    const runs = createRunsRuntime({
+      driver: "files",
+      localRoot: root,
+      remote: { bucket: remoteBucket },
+      hotWindowMs,
+    });
+    await runs.open();
+
+    const now = Date.now();
+    const base = {
+      flow: "ping",
+      trigger: "http" as const,
+      plane: "user" as const,
+      gates: [] as string[],
+      cache: "none" as const,
+      effects: [],
+      logs: [],
+      error: null,
+      principal: "u1",
+      tenant: "t1",
+      subjectId: "u1",
+    };
+
+    await runs.append({
+      ...base,
+      id: "recent",
+      durationMs: 10,
+      startedAt: now - 1000,
+      endedAt: now,
+      dimensions: { flow: "ping", locality: "local", duration_ms: 10 },
+    });
+    await runs.append({
+      ...base,
+      id: "archived",
+      durationMs: 20,
+      startedAt: now - hotWindowMs * 2,
+      endedAt: now - hotWindowMs * 2 + 20,
+      dimensions: { flow: "ping", locality: "remote", duration_ms: 20 },
+    });
+    await runs.flush();
+
+    // Prove bytes landed in both buckets
+    const localBucket = await fsDriver.open({ name: "check", root: root });
+    const localKeys = await localBucket.list("runs/");
+    const remoteKeys = await remoteBucket.list("runs/");
+    expect(localKeys.some((k) => k.endsWith(".parquet"))).toBe(true);
+    expect(remoteKeys.some((k) => k.endsWith(".parquet"))).toBe(true);
+
+    const rows = await runs.query(
+      "SELECT id, dim_locality AS locality FROM runs ORDER BY id",
+    );
+    expect(rows.length).toBe(2);
+    const ids = rows.map((r) => String(r.id)).sort();
+    expect(ids).toEqual(["archived", "recent"]);
+
+    await runs.close();
+    await localBucket.close();
+    await remoteBucket.close();
+  });
+});
+
+describe("crypto-shredding erasure", () => {
+  test("erasure renders archived fields unreadable without rewriting partitions", async () => {
+    const root = await tmpDir();
+    const keys = createMemorySubjectKeys();
+    const runs = createRunsRuntime({
+      driver: "files",
+      localRoot: root,
+      subjectKeys: keys,
+    });
+    await runs.open();
+
+    const subjectId = "user_erase_1";
+    const archived = await archiveFields(keys, subjectId, {
+      email: "alice@example.com",
+      phone: "+966500000000",
+    });
+    expect(keys.has(subjectKeyName(subjectId))).toBe(true);
+
+    const beforeReveal = await revealArchived(keys, subjectId, archived);
+    expect(beforeReveal.email).toBe("alice@example.com");
+
+    const startedAt = Date.now();
+    await runs.append({
+      id: "run_erase",
+      flow: "profile.update",
+      trigger: "http",
+      plane: "user",
+      tenant: "org_1",
+      principal: subjectId,
+      subjectId,
+      gates: [],
+      cache: "none",
+      effects: [],
+      logs: [],
+      durationMs: 5,
+      startedAt,
+      endedAt: startedAt + 5,
+      archived,
+      dimensions: { flow: "profile.update", subject_id: subjectId },
+      error: null,
+    });
+    await runs.flush();
+
+    // Snapshot partition bytes before erase
+    const bucket = await fsDriver.open({ name: "snap", root: root });
+    const keysBefore = (await bucket.list("runs/")).filter((k) =>
+      k.endsWith(".parquet"),
+    );
+    expect(keysBefore.length).toBeGreaterThan(0);
+    const bytesBefore = await bucket.get(keysBefore[0]!);
+    expect(bytesBefore).toBeTruthy();
+
+    const result = privacyErase({ subjectId, subjectKeys: keys, write: () => {} });
+    expect(result.deleted).toBe(true);
+    expect(keys.has(subjectKeyName(subjectId))).toBe(false);
+
+    const after = await revealArchived(keys, subjectId, archived);
+    expect(after.email).toBe(SHREDDED);
+    expect(after.phone).toBe(SHREDDED);
+
+    // Partitions unchanged
+    const bytesAfter = await bucket.get(keysBefore[0]!);
+    expect(bytesAfter).toEqual(bytesBefore);
+
+    // Query still returns the row (operational dims intact)
+    const rows = await runs.query(
+      "SELECT id, subject_id FROM runs WHERE id = 'run_erase'",
+    );
+    expect(rows.length).toBe(1);
+    expect(rows[0]!.subject_id).toBe(subjectId);
+
+    await runs.close();
+    await bucket.close();
+  });
+});
+
+describe("optional drivers", () => {
+  test("postgres driver appends and queries", async () => {
+    const runs = createRunsRuntime({ driver: "postgres" });
+    await runs.open();
+    const t = Date.now();
+    await runs.append({
+      id: "pg1",
+      flow: "x",
+      trigger: "internal",
+      plane: "user",
+      gates: [],
+      cache: "none",
+      effects: [],
+      logs: [],
+      durationMs: 1,
+      startedAt: t,
+      endedAt: t + 1,
+      dimensions: { flow: "x" },
+      error: null,
+    });
+    const rows = await runs.query("SELECT count(*) AS count FROM oke_runs");
+    expect(Number(rows[0]!.count)).toBe(1);
+    await runs.close();
+  });
+
+  test("clickhouse driver appends and queries", async () => {
+    const runs = createRunsRuntime({ driver: "clickhouse" });
+    await runs.open();
+    const t = Date.now();
+    await runs.append({
+      id: "ch1",
+      flow: "y",
+      trigger: "internal",
+      plane: "user",
+      gates: [],
+      cache: "none",
+      effects: [],
+      logs: [],
+      durationMs: 2,
+      startedAt: t,
+      endedAt: t + 2,
+      dimensions: { flow: "y" },
+      error: null,
+    });
+    const rows = await runs.query("SELECT count() AS count FROM oke_runs");
+    expect(Number(rows[0]!.count)).toBe(1);
+    await runs.close();
+  });
+});

package/src/runs/runtime.ts

@@ -0,0 +1,229 @@
+/**
+ * Runs runtime — append wide events from flow executions, query, erase.
+ */
+
+import { collectWideEvent, type CollectWideEventInput } from "./collect.ts";
+import { clickhouseRunsDriver } from "./drivers/clickhouse.ts";
+import { filesRunsDriver } from "./drivers/files.ts";
+import { memoryRunsDriver } from "./drivers/memory.ts";
+import { postgresRunsDriver } from "./drivers/postgres.ts";
+import {
+  explainOutliers,
+  type ExplainOutliersOptions,
+  type OutlierFinding,
+} from "./outlier.ts";
+import {
+  archiveFields,
+  eraseSubject,
+  revealArchived,
+  type SubjectKeyVault,
+} from "./shred.ts";
+import type {
+  RunsDriver,
+  RunsDriverId,
+  RunsOpenOptions,
+  RunsRow,
+  RunsStore,
+  WideEvent,
+} from "./types.ts";
+
+/** Options for {@link createRunsRuntime}. */
+export interface CreateRunsRuntimeOptions extends RunsOpenOptions {
+  /**
+   * Driver id or instance. Default: `files` (Parquet + DuckDB).
+   * Config example uses `memory` under `test`.
+   */
+  readonly driver?: RunsDriverId | RunsDriver;
+  /** Subject-key vault for crypto-shredding. */
+  readonly subjectKeys?: SubjectKeyVault;
+  /** Build version stamped on collected events. */
+  readonly buildVersion?: string;
+}
+
+/** Runs runtime surface. */
+export interface RunsRuntime {
+  /** Underlying store (after {@link open}). */
+  readonly store: RunsStore | undefined;
+  /** Open the configured driver. */
+  open(): Promise<RunsStore>;
+  /**
+   * Collect + optionally archive PII + append one execution.
+   *
+   * @param input - Collect input (ledger, telemetry, flow, …)
+   * @param archiveCleartext - Personal fields to crypto-shred
+   */
+  record(
+    input: CollectWideEventInput,
+    archiveCleartext?: Readonly<Record<string, string>>,
+  ): Promise<WideEvent>;
+  /**
+   * Append a fully-built wide event.
+   *
+   * @param event - Wide event
+   */
+  append(event: WideEvent): Promise<void>;
+  /** Flush buffers. */
+  flush(): Promise<void>;
+  /**
+   * Query across all partitions.
+   *
+   * @param sql - SQL (`FROM runs` for files/memory)
+   */
+  query(sql: string): Promise<RunsRow[]>;
+  /** All events (small stores / tests). */
+  all(): Promise<WideEvent[]>;
+  /**
+   * Outlier explanation over stored (or provided) events.
+   *
+   * @param options - Selection predicate
+   * @param events - Optional explicit dataset (defaults to `all()`)
+   */
+  explain(
+    options: ExplainOutliersOptions,
+    events?: readonly WideEvent[],
+  ): Promise<OutlierFinding[]>;
+  /**
+   * Reveal archived fields for a subject (or shredded markers).
+   *
+   * @param subjectId - Subject id
+   * @param archived - Ciphertext map
+   */
+  reveal(
+    subjectId: string,
+    archived: Readonly<Record<string, string>>,
+  ): Promise<Record<string, string>>;
+  /**
+   * Crypto-shred a subject — deletes the Vault key, not the Parquet bytes.
+   *
+   * @param subjectId - Subject id
+   */
+  erase(subjectId: string): boolean;
+  /** Close the store. */
+  close(): Promise<void>;
+}
+
+const DRIVER_BY_ID: Record<RunsDriverId, RunsDriver> = {
+  files: filesRunsDriver,
+  memory: memoryRunsDriver,
+  postgres: postgresRunsDriver,
+  clickhouse: clickhouseRunsDriver,
+};
+
+/**
+ * Resolve a driver id or instance.
+ *
+ * @param driver - Id or driver
+ */
+export function resolveRunsDriver(
+  driver?: RunsDriverId | RunsDriver,
+): RunsDriver {
+  if (!driver) return filesRunsDriver;
+  if (typeof driver === "string") {
+    const d = DRIVER_BY_ID[driver];
+    if (!d) throw new Error(`unknown runs driver: ${driver}`);
+    return d;
+  }
+  return driver;
+}
+
+/**
+ * Create a runs runtime.
+ *
+ * @param options - Driver + locality + subject keys
+ */
+export function createRunsRuntime(
+  options: CreateRunsRuntimeOptions = {},
+): RunsRuntime {
+  const driver = resolveRunsDriver(options.driver);
+  let store: RunsStore | undefined;
+  const subjectKeys = options.subjectKeys;
+
+  async function ensureStore(): Promise<RunsStore> {
+    if (!store) {
+      store = await driver.open(options);
+    }
+    return store;
+  }
+
+  return {
+    get store() {
+      return store;
+    },
+    async open() {
+      return ensureStore();
+    },
+    async record(input, archiveCleartext) {
+      const s = await ensureStore();
+      let archived = input.archived;
+      const subjectId =
+        input.telemetry.subjectId ??
+        input.fx.auth.userId ??
+        input.fx.tenant.id ??
+        null;
+      if (
+        archiveCleartext &&
+        subjectId &&
+        subjectKeys &&
+        Object.keys(archiveCleartext).length > 0
+      ) {
+        archived = await archiveFields(
+          subjectKeys,
+          subjectId,
+          archiveCleartext,
+        );
+      }
+      const event = collectWideEvent({
+        ...input,
+        buildVersion: input.buildVersion ?? options.buildVersion,
+        ...(archived !== undefined ? { archived } : {}),
+      });
+      await s.append(event);
+      return event;
+    },
+    async append(event) {
+      const s = await ensureStore();
+      await s.append(event);
+    },
+    async flush() {
+      const s = await ensureStore();
+      await s.flush();
+    },
+    async query(sql) {
+      const s = await ensureStore();
+      return s.query(sql);
+    },
+    async all() {
+      const s = await ensureStore();
+      return s.all();
+    },
+    async explain(explainOptions, events) {
+      const dataset = events ?? (await this.all());
+      return explainOutliers(dataset, explainOptions);
+    },
+    async reveal(subjectId, archived) {
+      if (!subjectKeys) {
+        const out: Record<string, string> = {};
+        for (const k of Object.keys(archived)) out[k] = "[shredded]";
+        return out;
+      }
+      return revealArchived(subjectKeys, subjectId, archived);
+    },
+    erase(subjectId) {
+      if (!subjectKeys) return false;
+      return eraseSubject(subjectKeys, subjectId);
+    },
+    async close() {
+      if (store) {
+        await store.close();
+        store = undefined;
+      }
+    },
+  };
+}
+
+export {
+  filesRunsDriver,
+  memoryRunsDriver,
+  postgresRunsDriver,
+  clickhouseRunsDriver,
+};