okengine 0.11.2 → 0.13.0

package/src/elements/store/cache.ts

@@ -162,6 +162,182 @@ export function createStoreCache(now: () => number = () => Date.now()): StoreCac
   };
 }
 
+/** Store resource refs that participate in the automatic cache cycle. */
+const STORE_REF = /^(sql|kv|files|index):/;
+
+/**
+ * Whether `ref` is a store resource the tier-1 cycle can key off.
+ *
+ * @param ref - Effect resource
+ */
+export function isStoreResourceRef(ref: string): ref is ResourceRef {
+  return STORE_REF.test(ref) && ref !== "runs";
+}
+
+/**
+ * Store reads / writes / asks recorded on one invocation's ledger.
+ *
+ * Used when the flow has no stamped `effects` (open capability token) so
+ * the cache cycle still runs from what `fx.store` actually touched.
+ *
+ * @param entries - Ledger entries from the invocation
+ */
+export function effectsFromLedger(
+  entries: readonly { readonly kind: string; readonly resource: string }[],
+): Effects {
+  const reads: ResourceRef[] = [];
+  const writes: ResourceRef[] = [];
+  const asks: string[] = [];
+  const seenRead = new Set<string>();
+  const seenWrite = new Set<string>();
+  for (const entry of entries) {
+    if (entry.kind === "ask") {
+      asks.push(entry.resource);
+      continue;
+    }
+    if (!isStoreResourceRef(entry.resource)) continue;
+    if (entry.kind === "read" && !seenRead.has(entry.resource)) {
+      seenRead.add(entry.resource);
+      reads.push(entry.resource);
+    }
+    if (entry.kind === "write" && !seenWrite.has(entry.resource)) {
+      seenWrite.add(entry.resource);
+      writes.push(entry.resource);
+    }
+  }
+  return {
+    ...(reads.length > 0 ? { reads } : {}),
+    ...(writes.length > 0 ? { writes } : {}),
+    ...(asks.length > 0 ? { asks } : {}),
+  };
+}
+
+/**
+ * Effects the auto-cache lookup should use: stamped reads when present,
+ * otherwise reads learned from a previous run's ledger.
+ *
+ * @param declared - Flow or capability effects (empty when the token is open)
+ * @param learnedReads - Store reads observed on an earlier invocation
+ */
+export function resolveCacheEffects(
+  declared: Effects | undefined,
+  learnedReads: readonly ResourceRef[] | undefined,
+): Effects {
+  const declaredReads = (declared?.reads ?? []).filter((r) => r !== "runs");
+  if (declaredReads.length > 0 || (declared?.writes?.length ?? 0) > 0) {
+    return declared ?? {};
+  }
+  if (learnedReads !== undefined && learnedReads.length > 0) {
+    return { reads: [...learnedReads] };
+  }
+  return declared ?? {};
+}
+
+/**
+ * Tier-1 hit only when every key is still present (a write to any
+ * contributing resource must miss).
+ *
+ * @param get - Cache getter
+ * @param keys - Keys from {@link tier1KeysForReads}
+ */
+export function tier1Lookup<T>(
+  get: (key: string) => T | undefined,
+  keys: readonly string[],
+): T | undefined {
+  const first = keys[0];
+  if (first === undefined) return undefined;
+  const value = get(first);
+  if (value === undefined) return undefined;
+  for (let i = 1; i < keys.length; i++) {
+    const key = keys[i];
+    if (key === undefined || get(key) === undefined) return undefined;
+  }
+  return value;
+}
+
+/**
+ * Whether a flow should use automatic tier-1 cache.
+ *
+ * Read-only flows (inferred, declared, or ledgered `reads`, no `writes`)
+ * cache by default. Opt out with `cache: false`. Mutations, AI asks,
+ * durable flows, and empty effect sets stay uncached — no `cache: "30s"`
+ * or hand-declared `effects` required on the flow.
+ *
+ * @param options - Flow cache flag, durability, and effect set
+ */
+export function autoCacheEligible(options: {
+  readonly cache?: boolean | string;
+  readonly durable?: boolean;
+  readonly effects?: Effects;
+}): boolean {
+  if (options.cache === false) return false;
+  if (options.durable === true) return false;
+  const effects = options.effects ?? {};
+  if ((effects.asks?.length ?? 0) > 0) return false;
+  if ((effects.writes?.length ?? 0) > 0) return false;
+  const reads = (effects.reads ?? []).filter((r) => r !== "runs");
+  return reads.length > 0;
+}
+
+/**
+ * Dimension suffixes for a flow-scoped tier-1 key.
+ *
+ * Format after {@link computedCacheKey}: `computed:{resource}/{flow}/{input}[/{userId}]`.
+ * Invalidation still keys off the resource segment.
+ *
+ * @param flowName - Flow id
+ * @param input - Validated flow input
+ * @param userId - Caller id when present (per-user lists)
+ */
+export function tier1FlowDims(
+  flowName: string,
+  input: unknown,
+  userId?: string | null,
+): readonly string[] {
+  const dims = [flowName, fingerprintInput(input)];
+  if (userId) dims.push(userId);
+  return dims;
+}
+
+/**
+ * Per-resource dim map for {@link tier1KeysForReads} / `putTier1`.
+ *
+ * @param effects - Read effects
+ * @param flowName - Flow id
+ * @param input - Validated flow input
+ * @param userId - Caller id when present
+ */
+export function tier1DimsByResource(
+  effects: Effects,
+  flowName: string,
+  input: unknown,
+  userId?: string | null,
+): Readonly<Record<string, readonly string[]>> {
+  const dims = tier1FlowDims(flowName, input, userId);
+  const out: Record<string, readonly string[]> = {};
+  for (const resource of effects.reads ?? []) {
+    if (resource === "runs") continue;
+    out[resource] = dims;
+  }
+  return out;
+}
+
+/**
+ * Stable-enough input fingerprint for a cache dim.
+ *
+ * @param input - Validated flow input
+ */
+function fingerprintInput(input: unknown): string {
+  if (input === undefined || input === null) return "-";
+  const t = typeof input;
+  if (t === "string" || t === "number" || t === "boolean") return String(input);
+  try {
+    return JSON.stringify(input);
+  } catch {
+    return "-";
+  }
+}
+
 /**
  * Parse a short TTL string (`"5m"`, `"1h"`, `"30s"`) to milliseconds.
  *

package/src/elements/store/classify.ts

@@ -24,6 +24,10 @@ export function classify(classification: ColumnClassification): ColumnClassifica
 /**
  * Build a classification map from a nested table → column → tags structure.
  *
+ * Each classified column is registered under both JS and SQL spellings
+ * (`ownerEmail` / `owner_email`) so raw `SELECT *` and query-builder rows
+ * hit the same tags.
+ *
  * @param tables - Per-table column classifications
  */
 export function buildClassificationMap(
@@ -32,15 +36,80 @@ export function buildClassificationMap(
   const map = new Map<string, ColumnClassification>();
   for (const [table, cols] of Object.entries(tables)) {
     for (const [column, tags] of Object.entries(cols)) {
-      map.set(classificationKey(table, column), tags);
+      for (const name of piiNameAliases(column)) {
+        map.set(classificationKey(table, name), tags);
+      }
     }
   }
   return map;
 }
 
+/**
+ * JS and SQL spellings of a classified field (`ownerEmail` ↔ `owner_email`).
+ *
+ * Store rows remap to camelCase; Manifest extract / seed tables and raw
+ * `SELECT *` often keep snake_case. Masking and Console PII flags must hit both.
+ *
+ * @param col - Column / field name
+ */
+export function piiNameAliases(col: string): readonly string[] {
+  const names = [col];
+  const snake = camelToSnake(col);
+  const camel = snakeToCamel(col);
+  if (snake !== col) names.push(snake);
+  if (camel !== col) names.push(camel);
+  return names;
+}
+
+/**
+ * Register a classified field under both JS and SQL spellings.
+ *
+ * @param names - Accumulator
+ * @param col - Column / field name from the Manifest or schema
+ */
+export function addPiiFieldName(names: Set<string>, col: string): void {
+  for (const name of piiNameAliases(col)) names.add(name);
+}
+
+/**
+ * Expand classified names so a Set lookup matches either spelling.
+ *
+ * @param cols - Manifest / list PII column names
+ */
+export function expandPiiNames(cols: Iterable<string>): Set<string> {
+  const names = new Set<string>();
+  for (const col of cols) addPiiFieldName(names, col);
+  return names;
+}
+
+/**
+ * Count distinct classified columns, treating JS / SQL spellings as one.
+ *
+ * @param cols - Manifest / list PII column names
+ */
+export function piiLogicalCount(cols: Iterable<string>): number {
+  const seen = new Set<string>();
+  for (const col of cols) seen.add(camelToSnake(col));
+  return seen.size;
+}
+
+function camelToSnake(key: string): string {
+  return key
+    .replace(/([a-z0-9])([A-Z])/g, "$1_$2")
+    .replace(/([A-Z])([A-Z][a-z])/g, "$1_$2")
+    .toLowerCase();
+}
+
+function snakeToCamel(key: string): string {
+  return key.replace(/_([a-z])/g, (_, ch: string) => ch.toUpperCase());
+}
+
 /**
  * Whether a column is classified as PII.
  *
+ * Matches the given name and its camelCase / snake_case alias so a classify
+ * map keyed `ownerEmail` still masks raw SQL `owner_email` (and the reverse).
+ *
  * @param map - Classification map
  * @param table - Table name (may be unknown for opaque queries)
  * @param column - Column name
@@ -50,13 +119,19 @@ export function isPiiColumn(
   table: string | undefined,
   column: string,
 ): boolean {
+  const names = piiNameAliases(column);
   if (table) {
-    const direct = map.get(classificationKey(table, column));
-    if (direct?.pii) return true;
+    for (const name of names) {
+      const direct = map.get(classificationKey(table, name));
+      if (direct?.pii) return true;
+    }
   }
   // Survive raw SQL when only the column name is known: match any table.
   for (const [key, tags] of map) {
-    if (tags.pii && key.endsWith(`.${column}`)) return true;
+    if (!tags.pii) continue;
+    for (const name of names) {
+      if (key.endsWith(`.${name}`)) return true;
+    }
   }
   return false;
 }

package/src/elements/store/declare.ts

@@ -74,6 +74,11 @@ export interface SqlStoreDecl extends StoreDeclBase {
 export interface KvStoreOptions {
   /** Optional description. */
   readonly description?: string;
+  /**
+   * Persist this namespace in SQL (`oke_kv` JSONB on `DATABASE_URL`),
+   * not the cache Redis (`REDIS_URL`). Distinct from Flow `durable`.
+   */
+  readonly durable?: boolean;
 }
 
 /** KV store declaration. */
@@ -81,6 +86,7 @@ export interface KvStoreDecl extends StoreDeclBase {
   readonly facet: "kv";
   readonly ref: `kv:${string}`;
   readonly description?: string;
+  readonly durable?: boolean;
 }
 
 /** Options for {@link store.files}. */
@@ -179,6 +185,7 @@ export function kv(name: string, options: KvStoreOptions = {}): KvStoreDecl {
     name,
     ref: `kv:${name}`,
     ...(options.description !== undefined ? { description: options.description } : {}),
+    ...(options.durable === true ? { durable: true } : {}),
   };
 }
 

package/src/elements/store/emit-drizzle.ts

@@ -15,6 +15,7 @@ import {
   isSchemaTableDecl,
   relationsFromExports,
   schemaTable,
+  schemaTableSqlName,
   tablesFromExports,
   type RelationColumnRef,
   type SchemaColumnDecl,
@@ -207,7 +208,8 @@ export function emitDrizzleSource(
     const lines = Object.values(table.columns).map(
       (col) => `  ${col.key}: ${emitColumnSource(col, dialect)},`,
     );
-    return `export const ${exportNameForTable(table.name)} = pgTable(${JSON.stringify(table.name)}, {\n${lines.join("\n")}\n});`;
+    const sqlName = schemaTableSqlName(table) ?? table.name;
+    return `export const ${exportNameForTable(sqlName)} = pgTable(${JSON.stringify(sqlName)}, {\n${lines.join("\n")}\n});`;
   });
 
   const relationsBlock = emitRelationsSource(relations);

package/src/elements/store/files-policy.test.ts

@@ -1,5 +1,15 @@
 import { describe, expect, test } from "bun:test";
-import { contentAddressedKey, fileKeyWarnings, projectFileKeys } from "./files-policy.ts";
+import {
+  coerceSafeFileObjectKey,
+  contentAddressedKey,
+  fileKeyWarnings,
+  inferFileContentType,
+  isAsciiObjectKey,
+  projectFileKeys,
+  safeFileObjectKey,
+  safeFilePrefix,
+  slugFileSegment,
+} from "./files-policy.ts";
 
 describe("files-policy", () => {
   test("flags non-ASCII keys for signed-URL risk", () => {
@@ -24,4 +34,29 @@ describe("files-policy", () => {
     expect(rows[0]?.warnings).toEqual([]);
     expect(rows[1]?.warnings[0]?.code).toBe("non_ascii_key");
   });
+
+  test("safeFileObjectKey is ASCII and keeps a safe extension", () => {
+    expect(isAsciiObjectKey("attachments/DES-200/note.txt")).toBe(true);
+    expect(isAsciiObjectKey("вложения/shot.png")).toBe(false);
+    expect(slugFileSegment("screenshot")).toBe("screenshot");
+    expect(slugFileSegment("вложения")).toBe("file");
+    expect(safeFilePrefix("вложения/SUP-12/")).toBe("folder/SUP-12/");
+    expect(safeFilePrefix("attachments/DES-200/")).toBe("attachments/DES-200/");
+    const key = safeFileObjectKey("صبحي احمد موسي يحيي - رخصة عمل.pdf", "attachments/DES-200/");
+    expect(isAsciiObjectKey(key)).toBe(true);
+    expect(key.startsWith("attachments/DES-200/file-")).toBe(true);
+    expect(key.endsWith(".pdf")).toBe(true);
+    expect(coerceSafeFileObjectKey("docs/readme.txt")).toBe("docs/readme.txt");
+    const coerced = coerceSafeFileObjectKey("вложения/shot.png");
+    expect(isAsciiObjectKey(coerced)).toBe(true);
+    expect(coerceSafeFileObjectKey("вложения/shot.png")).toBe(coerced);
+  });
+
+  test("inferFileContentType maps suffixes", () => {
+    expect(inferFileContentType("a/b/shot.png")).toBe("image/png");
+    expect(inferFileContentType("note.txt")).toBe("text/plain");
+    expect(inferFileContentType("clip.mp4")).toBe("video/mp4");
+    expect(inferFileContentType("take.mp3")).toBe("audio/mpeg");
+    expect(inferFileContentType("blob")).toBe("application/octet-stream");
+  });
 });

package/src/elements/store/files-policy.ts

@@ -35,6 +35,107 @@ export function fileKeyWarnings(key: string): readonly FileKeyWarning[] {
   return warnings;
 }
 
+/**
+ * True when every character is printable ASCII and the key has no `..`.
+ *
+ * @param key - Object key
+ */
+export function isAsciiObjectKey(key: string): boolean {
+  if (key.length === 0 || key.startsWith("/") || key.includes("..")) return false;
+  // eslint-disable-next-line no-control-regex -- printable ASCII only
+  return /^[\x20-\x7E]+$/.test(key);
+}
+
+/**
+ * Slug one path segment to `[a-z0-9._-]`. Empty after strip → `fallback`.
+ *
+ * @param name - File or folder name
+ * @param fallback - Used when nothing ASCII remains
+ */
+export function slugFileSegment(name: string, fallback = "file"): string {
+  const slug = name
+    .normalize("NFKD")
+    .replace(/\p{M}/gu, "")
+    .toLowerCase()
+    .replace(/[^a-z0-9._-]+/g, "-")
+    .replace(/-+/g, "-")
+    .replace(/^[-.]+|[-.]+$/g, "");
+  return slug.length > 0 ? slug : fallback;
+}
+
+/**
+ * ASCII prefix (`вложения/SUP-12/` → `folder/sup-12/`).
+ *
+ * @param prefix - Folder prefix
+ */
+export function safeFilePrefix(prefix: string): string {
+  const parts = prefix
+    .replace(/^\/+|\/+$/g, "")
+    .split("/")
+    .filter((p) => p.length > 0);
+  if (parts.length === 0) return "";
+  return `${parts.map((part) => (isAsciiObjectKey(part) ? part : slugFileSegment(part, "folder"))).join("/")}/`;
+}
+
+function asciiExtension(name: string): string {
+  const base = name.replace(/\/+$/, "");
+  const slash = base.lastIndexOf("/");
+  const file = slash === -1 ? base : base.slice(slash + 1);
+  const dot = file.lastIndexOf(".");
+  if (dot <= 0 || dot === file.length - 1) return "";
+  const ext = file.slice(dot + 1).toLowerCase();
+  return /^[a-z0-9]+$/.test(ext) ? `.${ext}` : "";
+}
+
+function shortId(): string {
+  const bytes = new Uint8Array(4);
+  crypto.getRandomValues(bytes);
+  return [...bytes].map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+function idFromKey(requested: string): string {
+  let hash = 2166136261;
+  for (let i = 0; i < requested.length; i++) {
+    hash ^= requested.charCodeAt(i);
+    hash = Math.imul(hash, 16777619);
+  }
+  return (hash >>> 0).toString(16).padStart(8, "0");
+}
+
+/**
+ * URL-safe object key from a display name. Non-ASCII folders and names
+ * become slugs; a short id avoids collisions (`file-a1b2c3d4.pdf`).
+ *
+ * @param originalName - Operator-facing file name (Unicode OK)
+ * @param prefix - Folder prefix (ASCII segments kept as-is)
+ * @param id - Optional 8-hex suffix; generated when omitted
+ */
+export function safeFileObjectKey(
+  originalName: string,
+  prefix: string = "",
+  id: string = shortId(),
+): string {
+  const ext = asciiExtension(originalName);
+  const trimmed = originalName.replace(/\/+$/, "");
+  const slash = trimmed.lastIndexOf("/");
+  const file = slash === -1 ? trimmed : trimmed.slice(slash + 1);
+  const stem = ext.length > 0 ? file.slice(0, -ext.length) : file;
+  return `${safeFilePrefix(prefix)}${slugFileSegment(stem)}-${id}${ext}`;
+}
+
+/**
+ * Keep an already-safe key; otherwise build one from the last segment.
+ *
+ * @param requestedKey - Key the operator typed or the browser sent
+ */
+export function coerceSafeFileObjectKey(requestedKey: string): string {
+  if (isAsciiObjectKey(requestedKey)) return requestedKey;
+  const slash = requestedKey.lastIndexOf("/");
+  const prefix = slash === -1 ? "" : requestedKey.slice(0, slash + 1);
+  const name = slash === -1 ? requestedKey : requestedKey.slice(slash + 1);
+  return safeFileObjectKey(name, prefix, idFromKey(requestedKey));
+}
+
 /**
  * Content-addressed object key (sha256 hex of bytes).
  *
@@ -60,3 +161,53 @@ export function projectFileKeys(keys: readonly string[]): ReadonlyArray<{
     warnings: fileKeyWarnings(key),
   }));
 }
+
+const FILE_MIME: Record<string, string> = {
+  png: "image/png",
+  jpg: "image/jpeg",
+  jpeg: "image/jpeg",
+  gif: "image/gif",
+  webp: "image/webp",
+  svg: "image/svg+xml",
+  avif: "image/avif",
+  txt: "text/plain",
+  md: "text/markdown",
+  csv: "text/csv",
+  json: "application/json",
+  xml: "application/xml",
+  html: "text/html",
+  css: "text/css",
+  js: "text/javascript",
+  ts: "text/plain",
+  pdf: "application/pdf",
+  patch: "text/x-patch",
+  diff: "text/x-patch",
+  mp4: "video/mp4",
+  webm: "video/webm",
+  mov: "video/quicktime",
+  m4v: "video/mp4",
+  ogv: "video/ogg",
+  mp3: "audio/mpeg",
+  wav: "audio/wav",
+  ogg: "audio/ogg",
+  oga: "audio/ogg",
+  m4a: "audio/mp4",
+  aac: "audio/aac",
+  flac: "audio/flac",
+  opus: "audio/opus",
+};
+
+/**
+ * MIME type from an object key suffix. Unknown → `application/octet-stream`.
+ *
+ * @param key - Object key
+ */
+export function inferFileContentType(key: string): string {
+  const base = key.replace(/\/+$/, "");
+  const slash = base.lastIndexOf("/");
+  const name = slash === -1 ? base : base.slice(slash + 1);
+  const dot = name.lastIndexOf(".");
+  if (dot <= 0 || dot === name.length - 1) return "application/octet-stream";
+  const ext = name.slice(dot + 1).toLowerCase();
+  return FILE_MIME[ext] ?? "application/octet-stream";
+}

package/src/elements/store/kv-sql.test.ts

@@ -0,0 +1,86 @@
+/**
+ * Durable KV on `oke_kv` JSONB — round-trip, TTL purge, cache Redis unchanged.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { pgliteDriver } from "../../drivers/pglite.ts";
+import { createRedisFakeClient, redisDriver } from "../../drivers/redis.ts";
+import { memoryDrivers } from "../../drivers/memory.ts";
+import { store } from "./declare.ts";
+import { openSqlKvNamespace } from "./kv-sql.ts";
+import { createStoreRuntime, type KvStoreFxHandle } from "./runtime.ts";
+
+const ctx = { effects: {} };
+
+describe("oke_kv JSONB durable namespace", () => {
+  test("set/get/delete round-trips a JSON object", async () => {
+    const conn = await pgliteDriver.connect({ url: "memory://" });
+    try {
+      const ns = await openSqlKvNamespace({
+        conn,
+        namespace: "ledger",
+        now: () => 1_000_000,
+      });
+      expect(ns.driverId).toBe("pglite");
+      await ns.set("idemp", { orderId: "o1", n: 2 });
+      expect(await ns.get("idemp")).toEqual({ orderId: "o1", n: 2 });
+      expect(await ns.list("id")).toEqual(["idemp"]);
+      expect(await ns.delete("idemp")).toBe(true);
+      expect(await ns.get("idemp")).toBeUndefined();
+    } finally {
+      await conn.close();
+    }
+  });
+
+  test("expired keys are invisible and purgeExpired removes the row", async () => {
+    const conn = await pgliteDriver.connect({ url: "memory://" });
+    let t = 1_000_000;
+    try {
+      const ns = await openSqlKvNamespace({
+        conn,
+        namespace: "ledger",
+        now: () => t,
+      });
+      await ns.set("fresh", { ok: true }, "5s");
+      await ns.set("stale", { ok: false }, "1s");
+      t += 1_500;
+      expect(await ns.get("stale")).toBeUndefined();
+      expect(await ns.get("fresh")).toEqual({ ok: true });
+      const removed = await ns.purgeExpired();
+      expect(removed).toBe(1);
+      const leftover = await conn.query(`SELECT key FROM oke_kv WHERE key = ?`, ["stale"]);
+      expect(leftover).toEqual([]);
+    } finally {
+      await conn.close();
+    }
+  });
+
+  test("non-durable namespace stays on redis; durable uses pglite", async () => {
+    const fake = createRedisFakeClient();
+    const sessions = store.kv("sessions");
+    const ledger = store.kv("ledger", { durable: true });
+    const runtime = createStoreRuntime({
+      drivers: {
+        sql: pgliteDriver,
+        kv: redisDriver,
+        files: memoryDrivers.files,
+        index: memoryDrivers.index,
+      },
+      sqlUrl: "memory://",
+      kv: { sessions: { url: "redis://cache", client: fake } },
+      now: () => 1_000_000,
+    });
+    runtime.register(sessions);
+    runtime.register(ledger);
+    const cache = (await runtime.open(sessions, ctx)) as KvStoreFxHandle;
+    const durable = (await runtime.open(ledger, ctx)) as KvStoreFxHandle;
+    expect(cache.driverId).toBe("redis");
+    expect(durable.driverId).toBe("pglite");
+    await cache.set("sid", { user: "a" });
+    await durable.set("idemp", { orderId: "o1" });
+    expect(await cache.get("sid")).toEqual({ user: "a" });
+    expect(await durable.get("idemp")).toEqual({ orderId: "o1" });
+    expect([...fake.data.keys()].some((k) => k.includes("idemp"))).toBe(false);
+    await runtime.close();
+  });
+});