okengine 0.11.2 → 0.12.0

package/src/console/server/runs-list-nesting.test.ts

@@ -0,0 +1,97 @@
+/**
+ * Regression: `console.runs.list` must not nest its own projection into the
+ * runs store (exponential payload growth → Console kernel OOM on poll).
+ */
+
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdtemp } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { WideEvent } from "../../runs/types.ts";
+import { serveConsole, type ConsoleServerHandle } from "./serve.ts";
+
+const HOST_RUN: WideEvent = {
+  id: "host-run-1",
+  flow: "bookings.create",
+  unit: "bookings",
+  trigger: "http",
+  plane: "user",
+  gates: [],
+  cache: "none",
+  effects: [],
+  logs: [],
+  durationMs: 10,
+  startedAt: 1_700_000_000_000,
+  endedAt: 1_700_000_000_010,
+  dimensions: { flow: "bookings.create" },
+  input: { flightId: "SK-101", seats: 1 },
+  output: { id: "b1" },
+};
+
+describe("console.runs.list nesting", () => {
+  let cwd: string;
+  let server: ConsoleServerHandle;
+  let token: string;
+
+  beforeAll(async () => {
+    cwd = await mkdtemp(join(tmpdir(), "oke-runs-list-nest-"));
+    server = await serveConsole({
+      port: 0,
+      hostname: "127.0.0.1",
+      cwd,
+      persist: false,
+      silentClaim: true,
+      env: "test",
+      secret: "test-runs-list-nest-secret",
+    });
+    const claim = await fetch(`${server.url}/console/setup/claim`, {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body: JSON.stringify({
+        claimCode: server.console.state.claim.code,
+        email: "ops@example.com",
+        name: "Ops",
+        password: "Password1234!",
+      }),
+    });
+    expect(claim.status).toBe(200);
+    const body = (await claim.json()) as { data: { accessToken: string } };
+    token = body.data.accessToken;
+
+    const runs = server.console.app.bootResult?.runs;
+    expect(runs).toBeDefined();
+    await runs!.append(HOST_RUN);
+  });
+
+  afterAll(() => {
+    server.stop(true);
+  });
+
+  test("repeated GET /console/runs does not explode stored event size", async () => {
+    const sizes: number[] = [];
+    let lastCount: number | null = null;
+
+    for (let i = 0; i < 12; i++) {
+      const res = await fetch(`${server.url}/console/runs`, {
+        headers: { authorization: `Bearer ${token}` },
+      });
+      expect(res.status).toBe(200);
+      const json = (await res.json()) as {
+        data: { runs: readonly { readonly flow: string }[] };
+      };
+      expect(json.data.runs.some((r) => r.flow === "console.runs.list")).toBe(false);
+      if (lastCount === null) lastCount = json.data.runs.length;
+      else expect(json.data.runs.length).toBe(lastCount);
+
+      const all = await server.console.state.listRuns();
+      const max = Math.max(...all.map((r) => JSON.stringify(r).length));
+      sizes.push(max);
+    }
+
+    const first = sizes[0]!;
+    const last = sizes[sizes.length - 1]!;
+    // Exponential nesting would be ~2^11×; allow modest linear noise only.
+    expect(last).toBeLessThan(first * 4);
+    expect(last).toBeLessThan(200_000);
+  });
+});

package/src/console/server/runs-pii.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, test } from "bun:test";
+import { PII_MASK } from "../../elements/store/classify.ts";
+import { maskPiiValue, piiFieldNamesFromManifest } from "./runs-pii.ts";
+import type { Manifest } from "../../manifest/types.ts";
+
+describe("maskPiiValue", () => {
+  test("walks nested objects and arrays", () => {
+    const pii = new Set(["email", "assignee_email"]);
+    const masked = maskPiiValue(
+      {
+        id: "iss_1",
+        email: "a@oke.com",
+        items: [{ id: "c1", assignee_email: "b@oke.com" }],
+      },
+      pii,
+    );
+    expect(masked).toEqual({
+      id: "iss_1",
+      email: PII_MASK,
+      items: [{ id: "c1", assignee_email: PII_MASK }],
+    });
+  });
+
+  test("leaves values alone when no fields are classified", () => {
+    const row = { email: "a@oke.com" };
+    expect(maskPiiValue(row, new Set())).toBe(row);
+  });
+});
+
+describe("piiFieldNamesFromManifest", () => {
+  test("collects table column and classify-map names", () => {
+    const manifest = {
+      oke: "1.0",
+      app: "t",
+      stores: {
+        db: {
+          facet: "sql",
+          tables: {
+            members: { columns: { email: { pii: true }, name: {} } },
+          },
+          classifications: { "issues.assignee_email": { pii: true } },
+        },
+      },
+    } as unknown as Manifest;
+    const names = piiFieldNamesFromManifest(manifest);
+    expect(names.has("email")).toBe(true);
+    expect(names.has("assignee_email")).toBe(true);
+    expect(names.has("assigneeEmail")).toBe(true);
+    expect(names.has("name")).toBe(false);
+  });
+
+  test("aliases camelCase JS keys used by store-backed handlers", () => {
+    const manifest = {
+      oke: "1.0",
+      app: "t",
+      stores: {
+        db: {
+          facet: "sql",
+          tables: {
+            views: { columns: { ownerEmail: { pii: true } } },
+          },
+        },
+      },
+    } as unknown as Manifest;
+    const names = piiFieldNamesFromManifest(manifest);
+    expect(names.has("ownerEmail")).toBe(true);
+    expect(names.has("owner_email")).toBe(true);
+  });
+});

package/src/console/server/runs-pii.ts

@@ -7,7 +7,7 @@
  * leak in every panel that reads `console.runs.list` (console §6 · §9.5 · §9.11).
  */
 
-import { PII_MASK } from "../../elements/store/classify.ts";
+import { addPiiFieldName, PII_MASK } from "../../elements/store/classify.ts";
 import type { Manifest } from "../../manifest/types.ts";
 import type { WideEvent } from "../../runs/types.ts";
 
@@ -28,7 +28,7 @@ export function piiFieldNamesFromManifest(
   for (const store of Object.values(manifest.stores)) {
     for (const table of Object.values(store.tables ?? {})) {
       for (const [col, tags] of Object.entries(table.columns ?? {})) {
-        if (tags?.pii) names.add(col);
+        if (tags?.pii) addPiiFieldName(names, col);
       }
     }
     for (const [key, tags] of Object.entries(store.classifications ?? {})) {
@@ -40,7 +40,7 @@ export function piiFieldNamesFromManifest(
         (tags as { pii?: boolean }).pii
       ) {
         const col = key.includes(".") ? key.slice(key.lastIndexOf(".") + 1) : key;
-        names.add(col);
+        addPiiFieldName(names, col);
       }
     }
   }
@@ -67,6 +67,32 @@ export function maskPiiRecord<T extends Record<string, unknown>>(
   return out as T;
 }
 
+/**
+ * Recursively mask PII-classified keys in a JSON value (Call API responses).
+ *
+ * @param value - Handler output or failure data
+ * @param piiFields - Classified field names
+ * @param mask - Mask token
+ */
+export function maskPiiValue(
+  value: unknown,
+  piiFields: ReadonlySet<string>,
+  mask: string = PII_MASK,
+): unknown {
+  if (piiFields.size === 0 || value === null || value === undefined) return value;
+  if (Array.isArray(value)) {
+    return value.map((item) => maskPiiValue(item, piiFields, mask));
+  }
+  if (typeof value === "object") {
+    const out: Record<string, unknown> = {};
+    for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
+      out[key] = piiFields.has(key) ? mask : maskPiiValue(child, piiFields, mask);
+    }
+    return out;
+  }
+  return value;
+}
+
 /**
  * Apply schema-classification PII masking to a wide event for Console projection.
  *
@@ -89,5 +115,25 @@ export function maskWideEventForConsole(
       data: maskPiiRecord(line.data as Record<string, unknown>, piiFields),
     };
   });
-  return { ...event, dimensions, logs };
+  const input =
+    event.input !== undefined &&
+    event.input !== null &&
+    typeof event.input === "object" &&
+    !Array.isArray(event.input)
+      ? maskPiiRecord(event.input as Record<string, unknown>, piiFields)
+      : event.input;
+  const output =
+    event.output !== undefined &&
+    event.output !== null &&
+    typeof event.output === "object" &&
+    !Array.isArray(event.output)
+      ? maskPiiRecord(event.output as Record<string, unknown>, piiFields)
+      : event.output;
+  return {
+    ...event,
+    dimensions,
+    logs,
+    ...(input !== undefined ? { input } : {}),
+    ...(output !== undefined ? { output } : {}),
+  };
 }

package/src/console/server/runs-query-guard.test.ts

@@ -0,0 +1,75 @@
+import { describe, expect, test } from "bun:test";
+import { guardRunsQuerySql, RUNS_QUERY_ROW_LIMIT } from "./runs-query-guard.ts";
+
+describe("guardRunsQuerySql", () => {
+  test("allows SELECT / WITH / FROM / EXPLAIN / DESCRIBE", () => {
+    expect(guardRunsQuerySql("SELECT id FROM runs LIMIT 10").ok).toBe(true);
+    expect(guardRunsQuerySql("WITH x AS (SELECT 1 AS n) SELECT n FROM x").ok).toBe(true);
+    expect(guardRunsQuerySql("FROM runs SELECT id LIMIT 5").ok).toBe(true);
+    expect(guardRunsQuerySql("EXPLAIN SELECT 1").ok).toBe(true);
+    expect(guardRunsQuerySql("DESCRIBE runs").ok).toBe(true);
+  });
+
+  test("injects LIMIT on unbounded SELECT/FROM/WITH", () => {
+    const result = guardRunsQuerySql("SELECT id FROM runs");
+    expect(result.ok).toBe(true);
+    if (result.ok) {
+      expect(result.injectedLimit).toBe(true);
+      expect(result.sql).toContain(`LIMIT ${RUNS_QUERY_ROW_LIMIT}`);
+    }
+  });
+
+  test("does not inject LIMIT when one is present", () => {
+    const result = guardRunsQuerySql("SELECT id FROM runs LIMIT 3");
+    expect(result.ok).toBe(true);
+    if (result.ok) expect(result.injectedLimit).toBe(false);
+  });
+
+  test("rejects FROM-first filesystem reads", () => {
+    const result = guardRunsQuerySql("FROM read_csv('/etc/passwd')");
+    expect(result).toEqual({ ok: false, reason: "external:read_csv" });
+  });
+
+  test("rejects CTE-wrapped writes", () => {
+    const result = guardRunsQuerySql("WITH x AS (SELECT 1 AS n) INSERT INTO runs SELECT * FROM x");
+    expect(result.ok).toBe(false);
+    if (!result.ok) expect(result.reason).toBe("keyword:insert");
+  });
+
+  test("rejects INSTALL / LOAD / ATTACH / PRAGMA / SET", () => {
+    for (const sql of [
+      "INSTALL httpfs",
+      "LOAD httpfs",
+      "ATTACH '/tmp/other.db'",
+      "PRAGMA show_tables",
+      "SET enable_external_access = true",
+    ]) {
+      const result = guardRunsQuerySql(sql);
+      expect(result.ok).toBe(false);
+    }
+  });
+
+  test("rejects read_parquet on an arbitrary path", () => {
+    const result = guardRunsQuerySql("SELECT * FROM read_parquet('/tmp/other.parquet')");
+    expect(result).toEqual({ ok: false, reason: "external:read_parquet" });
+  });
+
+  test("rejects COPY / CREATE / DELETE / EXPLAIN ANALYZE / multi-statement", () => {
+    expect(guardRunsQuerySql("COPY runs TO '/tmp/out.csv'").ok).toBe(false);
+    expect(guardRunsQuerySql("CREATE TABLE t AS SELECT 1").ok).toBe(false);
+    expect(guardRunsQuerySql("DELETE FROM runs").ok).toBe(false);
+    expect(guardRunsQuerySql("EXPLAIN ANALYZE SELECT 1")).toEqual({
+      ok: false,
+      reason: "explain-analyze",
+    });
+    expect(guardRunsQuerySql("SELECT 1; SELECT 2")).toEqual({
+      ok: false,
+      reason: "multi-statement",
+    });
+  });
+
+  test("does not treat quoted INSERT as a write", () => {
+    const result = guardRunsQuerySql("SELECT 'INSERT' AS label FROM runs LIMIT 1");
+    expect(result.ok).toBe(true);
+  });
+});

package/src/console/server/runs-query-guard.ts

@@ -0,0 +1,224 @@
+/**
+ * Read-only statement guard for Console `POST /console/runs/query`.
+ *
+ * First-token checks are not enough (DuckDB FROM-first, CTE-wrapped DML).
+ * This scans unquoted, comment-stripped text and rejects writes / external access.
+ */
+
+/** Named PII limitation for free-form SELECT output. */
+export const RUNS_QUERY_PII_GAP = "RunsQueryPiiProjectionGap" as const;
+
+/** Hard result ceiling (also injected as LIMIT on unbounded SELECT/FROM). */
+export const RUNS_QUERY_ROW_LIMIT = 1_000;
+
+/** Wall-clock timeout for a Console runs query. */
+export const RUNS_QUERY_TIMEOUT_MS = 5_000;
+
+/** Accepted first keywords after comment strip. */
+const ALLOWED_HEADS = new Set([
+  "SELECT",
+  "WITH",
+  "EXPLAIN",
+  "DESCRIBE",
+  "SUMMARIZE",
+  "FROM",
+  "PIVOT",
+  "UNPIVOT",
+  "TABLE",
+]);
+
+/** Write / session / extension keywords — rejected in the unquoted body. */
+const FORBIDDEN_WORDS = new Set([
+  "INSERT",
+  "UPDATE",
+  "DELETE",
+  "DROP",
+  "ALTER",
+  "CREATE",
+  "TRUNCATE",
+  "REPLACE",
+  "COPY",
+  "MERGE",
+  "CALL",
+  "GRANT",
+  "REVOKE",
+  "VACUUM",
+  "COMMENT",
+  "REFRESH",
+  "INSTALL",
+  "LOAD",
+  "ATTACH",
+  "DETACH",
+  "EXPORT",
+  "IMPORT",
+  "PRAGMA",
+  "SET",
+  "RESET",
+  "ANALYZE",
+  "CHECKPOINT",
+  "PREPARE",
+  "EXECUTE",
+  "DEALLOCATE",
+]);
+
+/** DuckDB filesystem / extension entry points. */
+const EXTERNAL_FNS = [
+  "read_csv",
+  "read_csv_auto",
+  "read_json",
+  "read_json_auto",
+  "read_json_objects",
+  "read_ndjson",
+  "read_parquet",
+  "read_text",
+  "read_blob",
+  "read_xlsx",
+  "httpfs",
+  "copy_database",
+];
+
+/** Successful guard — SQL ready to execute (LIMIT may be injected). */
+export interface RunsQueryGuardOk {
+  readonly ok: true;
+  readonly sql: string;
+  readonly injectedLimit: boolean;
+}
+
+/** Rejected statement. */
+export interface RunsQueryGuardReject {
+  readonly ok: false;
+  readonly reason: string;
+}
+
+/** Result of {@link guardRunsQuerySql}. */
+export type RunsQueryGuardResult = RunsQueryGuardOk | RunsQueryGuardReject;
+
+/**
+ * Validate operator SQL for the runs query console.
+ *
+ * @param sql - Raw editor buffer
+ */
+export function guardRunsQuerySql(sql: string): RunsQueryGuardResult {
+  const trimmed = sql.trim();
+  if (trimmed.length === 0) return reject("empty");
+
+  const statements = splitUnquotedStatements(trimmed);
+  if (statements.length === 0) return reject("empty");
+  if (statements.length > 1) return reject("multi-statement");
+
+  const statement = statements[0]!;
+  const unquoted = unquotedText(statement);
+  const stripped = stripSqlComments(unquoted).replace(/\s+/g, " ").trim();
+  if (stripped.length === 0) return reject("empty");
+
+  const head = leadingKeyword(stripped);
+  if (head === "EXPLAIN" && /\bANALYZE\b/i.test(stripped)) {
+    return reject("explain-analyze");
+  }
+  if (!ALLOWED_HEADS.has(head)) return reject(`head:${head || "unknown"}`);
+
+  const words = stripped.toUpperCase().match(/[A-Z_][A-Z0-9_]*/g) ?? [];
+  for (const word of words) {
+    if (FORBIDDEN_WORDS.has(word)) return reject(`keyword:${word.toLowerCase()}`);
+  }
+
+  const lower = stripped.toLowerCase();
+  for (const fn of EXTERNAL_FNS) {
+    if (lower.includes(fn)) return reject(`external:${fn}`);
+  }
+
+  const inject =
+    (head === "SELECT" || head === "FROM" || head === "WITH" || head === "TABLE") &&
+    !/\bLIMIT\b/i.test(stripped);
+  const prepared = inject
+    ? `${statement.replace(/;+\s*$/, "")} LIMIT ${RUNS_QUERY_ROW_LIMIT}`
+    : statement;
+  return { ok: true, sql: prepared, injectedLimit: inject };
+}
+
+function reject(reason: string): RunsQueryGuardReject {
+  return { ok: false, reason };
+}
+
+function leadingKeyword(sql: string): string {
+  const match = /^([A-Za-z]+)/.exec(sql);
+  return match?.[1]?.toUpperCase() ?? "";
+}
+
+function splitUnquotedStatements(sql: string): string[] {
+  const out: string[] = [];
+  let start = 0;
+  let quote: "'" | '"' | "`" | null = null;
+  for (let i = 0; i < sql.length; i += 1) {
+    const ch = sql[i];
+    if (quote) {
+      if (ch === quote) {
+        if (sql[i + 1] === quote) {
+          i += 1;
+          continue;
+        }
+        quote = null;
+      }
+      continue;
+    }
+    if (ch === "'" || ch === '"' || ch === "`") {
+      quote = ch;
+      continue;
+    }
+    if (ch === ";") {
+      const part = sql.slice(start, i).trim();
+      if (part.replace(/;+/g, "").trim().length > 0) out.push(part);
+      start = i + 1;
+    }
+  }
+  const tail = sql
+    .slice(start)
+    .trim()
+    .replace(/;+\s*$/, "");
+  if (tail.length > 0) out.push(tail);
+  return out;
+}
+
+function unquotedText(sql: string): string {
+  let out = "";
+  let quote: "'" | '"' | "`" | null = null;
+  for (let i = 0; i < sql.length; i += 1) {
+    const ch = sql[i];
+    if (quote) {
+      if (ch === quote) {
+        if (sql[i + 1] === quote) {
+          i += 1;
+          continue;
+        }
+        quote = null;
+      }
+      continue;
+    }
+    if (ch === "'" || ch === '"' || ch === "`") {
+      quote = ch;
+      continue;
+    }
+    out += ch;
+  }
+  return out;
+}
+
+function stripSqlComments(sql: string): string {
+  let out = "";
+  for (let i = 0; i < sql.length; i += 1) {
+    if (sql[i] === "-" && sql[i + 1] === "-") {
+      while (i < sql.length && sql[i] !== "\n") i += 1;
+      out += " ";
+      continue;
+    }
+    if (sql[i] === "/" && sql[i + 1] === "*") {
+      i += 2;
+      while (i < sql.length && !(sql[i] === "*" && sql[i + 1] === "/")) i += 1;
+      i += 1;
+      out += " ";
+      continue;
+    }
+    out += sql[i];
+  }
+  return out;
+}

package/src/console/server/runs-query-mask.test.ts

@@ -0,0 +1,37 @@
+import { describe, expect, test } from "bun:test";
+import { PII_MASK } from "../../elements/store/classify.ts";
+import { maskRunsQueryRows } from "./runs-query-mask.ts";
+
+const PII = new Set(["email", "owner_email"]);
+
+describe("maskRunsQueryRows", () => {
+  test("masks dim_* promotions and JSON blobs", () => {
+    const rows = maskRunsQueryRows(
+      [
+        {
+          id: "r1",
+          dim_email: "a@oke.com",
+          input: JSON.stringify({ email: "a@oke.com", id: "b1" }),
+          flow: "bookings.create",
+        },
+      ],
+      PII,
+    );
+    expect(rows[0]!.dim_email).toBe(PII_MASK);
+    expect(rows[0]!.flow).toBe("bookings.create");
+    const input = JSON.parse(String(rows[0]!.input)) as { email: string; id: string };
+    expect(input.email).toBe(PII_MASK);
+    expect(input.id).toBe("b1");
+  });
+
+  test("RunsQueryPiiProjectionGap — alias leak is real, not silently closed", () => {
+    const rows = maskRunsQueryRows([{ e: "a@oke.com", id: "r1" }], PII);
+    expect(rows[0]!.e).toBe("a@oke.com");
+    expect(rows[0]!.id).toBe("r1");
+  });
+
+  test("revealPii returns cleartext", () => {
+    const rows = maskRunsQueryRows([{ dim_email: "a@oke.com" }], PII, true);
+    expect(rows[0]!.dim_email).toBe("a@oke.com");
+  });
+});

package/src/console/server/runs-query-mask.ts

@@ -0,0 +1,82 @@
+/**
+ * Conservative PII mask for free-form runs SQL rows.
+ *
+ * Matches result column names (and `dim_<name>` promotions) plus known JSON
+ * blobs. Aliases and expressions are the named {@link RUNS_QUERY_PII_GAP}.
+ */
+
+import { addPiiFieldName, PII_MASK } from "../../elements/store/classify.ts";
+import type { RunsRow } from "../../runs/types.ts";
+import { maskPiiValue } from "./runs-pii.ts";
+import { RUNS_QUERY_PII_GAP } from "./runs-query-guard.ts";
+
+/** JSON columns written by `wideEventToRow`. */
+const JSON_COLUMNS = new Set(["input", "output", "logs", "dimensions"]);
+
+/**
+ * Mask a SQL result set using Manifest PII field names.
+ *
+ * @param rows - DuckDB rows
+ * @param piiFields - Classified names (plus aliases)
+ * @param revealPii - When true, return a shallow copy unmasked
+ */
+export function maskRunsQueryRows(
+  rows: readonly RunsRow[],
+  piiFields: ReadonlySet<string>,
+  revealPii = false,
+): RunsRow[] {
+  if (revealPii || piiFields.size === 0) {
+    return rows.map((row) => ({ ...row }));
+  }
+  const names = expandDimAliases(piiFields);
+  return rows.map((row) => maskOneRow(row, names));
+}
+
+/**
+ * Expand classified names with `dim_` promotions used on the Parquet row.
+ *
+ * @param piiFields - Manifest field names
+ */
+export function expandDimAliases(piiFields: ReadonlySet<string>): ReadonlySet<string> {
+  const names = new Set<string>();
+  for (const field of piiFields) {
+    addPiiFieldName(names, field);
+    names.add(`dim_${field}`);
+    const snake = field
+      .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+      .replace(/([A-Z])([A-Z][a-z])/g, "$1_$2")
+      .toLowerCase();
+    names.add(`dim_${snake}`);
+  }
+  return names;
+}
+
+function maskOneRow(row: RunsRow, piiFields: ReadonlySet<string>): RunsRow {
+  const out: RunsRow = {};
+  for (const [key, value] of Object.entries(row)) {
+    if (piiFields.has(key)) {
+      out[key] = PII_MASK;
+      continue;
+    }
+    if (JSON_COLUMNS.has(key)) {
+      out[key] = maskJsonColumn(value, piiFields);
+      continue;
+    }
+    out[key] = value;
+  }
+  return out;
+}
+
+function maskJsonColumn(value: unknown, piiFields: ReadonlySet<string>): unknown {
+  if (typeof value !== "string") {
+    return maskPiiValue(value, piiFields);
+  }
+  try {
+    const parsed: unknown = JSON.parse(value);
+    return JSON.stringify(maskPiiValue(parsed, piiFields));
+  } catch {
+    return value;
+  }
+}
+
+export { RUNS_QUERY_PII_GAP };