okengine 0.12.0 → 0.13.0

package/src/docker/recipes/redis.ts

@@ -5,7 +5,7 @@
  * `redis://` URL, different binaries. Pin via `images["store.kv"]`.
  */
 
-import { credEnv } from "../helpers.ts";
+import { credEnv, kvServerCommand } from "../helpers.ts";
 import type { ImageRecipe } from "../types.ts";
 
 /** Redis Open Source — Redis protocol on 6379. */
@@ -14,11 +14,7 @@ export const redis: ImageRecipe = {
   port: 6379,
   match: (i) => /(?:^|\/)redis(?:[:@/]|$)/i.test(i) || /keydb/i.test(i),
   apply: (s) => ({
-    command: [
-      "sh",
-      "-c",
-      'exec redis-server --requirepass "$$OKE_STORE_KV_PASSWORD" --maxmemory "$${OKE_STORE_KV_MAXMEMORY:-0}" --maxmemory-policy "$${OKE_STORE_KV_MAXMEMORY_POLICY:-noeviction}"',
-    ],
+    command: ["sh", "-c", kvServerCommand(s, "redis-server")],
     healthcheck: {
       test: ["CMD", "redis-cli", "-a", credEnv(s, "PASSWORD"), "ping"],
       interval: "5s",

package/src/docker/recipes/valkey.ts

@@ -5,7 +5,7 @@
  * `valkey/valkey`.
  */
 
-import { credEnv } from "../helpers.ts";
+import { credEnv, kvServerCommand } from "../helpers.ts";
 import type { ImageRecipe } from "../types.ts";
 
 /** Valkey — Redis protocol on 6379. */
@@ -14,11 +14,7 @@ export const valkey: ImageRecipe = {
   port: 6379,
   match: (i) => /valkey/i.test(i),
   apply: (s) => ({
-    command: [
-      "sh",
-      "-c",
-      'exec valkey-server --requirepass "$$OKE_STORE_KV_PASSWORD" --maxmemory "$${OKE_STORE_KV_MAXMEMORY:-0}" --maxmemory-policy "$${OKE_STORE_KV_MAXMEMORY_POLICY:-noeviction}"',
-    ],
+    command: ["sh", "-c", kvServerCommand(s, "valkey-server")],
     healthcheck: {
       test: ["CMD", "valkey-cli", "-a", credEnv(s, "PASSWORD"), "ping"],
       interval: "5s",

package/src/drivers/types.ts

@@ -83,7 +83,7 @@ export interface SqlDriver {
 /** Key-value namespace handle. */
 export interface KvNamespace {
   /** Protocol driver id. */
-  readonly driverId: "memory" | "redis";
+  readonly driverId: "memory" | "redis" | "postgres" | "pglite";
   /**
    * Read a key.
    *

package/src/elements/gate/runtime.ts

@@ -16,8 +16,8 @@ import { takeRate } from "./strategies.ts";
 
 /** KV surface required for rate gates. */
 export interface GateKv {
-  /** Driver id when known (`memory` · `redis`). */
-  readonly driverId?: "memory" | "redis";
+  /** Driver id when known (`memory` · `redis`; SQL ids unused on rate gates). */
+  readonly driverId?: "memory" | "redis" | "postgres" | "pglite";
   eval<T = unknown>(script: string, keys: readonly string[], args?: readonly string[]): Promise<T>;
 }
 
@@ -46,7 +46,7 @@ export interface GateRuntime {
   /** Registered declarations by name. */
   readonly gates: ReadonlyMap<string, GateDecl>;
   /** KV backend used for rate limits (`memory` · `redis`, or unset). */
-  readonly kvDriverId: "memory" | "redis" | undefined;
+  readonly kvDriverId: "memory" | "redis" | "postgres" | "pglite" | undefined;
   /**
    * Evaluate a gate chain in order; stop on first denial.
    *

package/src/elements/store/declare.ts

@@ -74,6 +74,11 @@ export interface SqlStoreDecl extends StoreDeclBase {
 export interface KvStoreOptions {
   /** Optional description. */
   readonly description?: string;
+  /**
+   * Persist this namespace in SQL (`oke_kv` JSONB on `DATABASE_URL`),
+   * not the cache Redis (`REDIS_URL`). Distinct from Flow `durable`.
+   */
+  readonly durable?: boolean;
 }
 
 /** KV store declaration. */
@@ -81,6 +86,7 @@ export interface KvStoreDecl extends StoreDeclBase {
   readonly facet: "kv";
   readonly ref: `kv:${string}`;
   readonly description?: string;
+  readonly durable?: boolean;
 }
 
 /** Options for {@link store.files}. */
@@ -179,6 +185,7 @@ export function kv(name: string, options: KvStoreOptions = {}): KvStoreDecl {
     name,
     ref: `kv:${name}`,
     ...(options.description !== undefined ? { description: options.description } : {}),
+    ...(options.durable === true ? { durable: true } : {}),
   };
 }
 

package/src/elements/store/kv-sql.test.ts

@@ -0,0 +1,86 @@
+/**
+ * Durable KV on `oke_kv` JSONB — round-trip, TTL purge, cache Redis unchanged.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { pgliteDriver } from "../../drivers/pglite.ts";
+import { createRedisFakeClient, redisDriver } from "../../drivers/redis.ts";
+import { memoryDrivers } from "../../drivers/memory.ts";
+import { store } from "./declare.ts";
+import { openSqlKvNamespace } from "./kv-sql.ts";
+import { createStoreRuntime, type KvStoreFxHandle } from "./runtime.ts";
+
+const ctx = { effects: {} };
+
+describe("oke_kv JSONB durable namespace", () => {
+  test("set/get/delete round-trips a JSON object", async () => {
+    const conn = await pgliteDriver.connect({ url: "memory://" });
+    try {
+      const ns = await openSqlKvNamespace({
+        conn,
+        namespace: "ledger",
+        now: () => 1_000_000,
+      });
+      expect(ns.driverId).toBe("pglite");
+      await ns.set("idemp", { orderId: "o1", n: 2 });
+      expect(await ns.get("idemp")).toEqual({ orderId: "o1", n: 2 });
+      expect(await ns.list("id")).toEqual(["idemp"]);
+      expect(await ns.delete("idemp")).toBe(true);
+      expect(await ns.get("idemp")).toBeUndefined();
+    } finally {
+      await conn.close();
+    }
+  });
+
+  test("expired keys are invisible and purgeExpired removes the row", async () => {
+    const conn = await pgliteDriver.connect({ url: "memory://" });
+    let t = 1_000_000;
+    try {
+      const ns = await openSqlKvNamespace({
+        conn,
+        namespace: "ledger",
+        now: () => t,
+      });
+      await ns.set("fresh", { ok: true }, "5s");
+      await ns.set("stale", { ok: false }, "1s");
+      t += 1_500;
+      expect(await ns.get("stale")).toBeUndefined();
+      expect(await ns.get("fresh")).toEqual({ ok: true });
+      const removed = await ns.purgeExpired();
+      expect(removed).toBe(1);
+      const leftover = await conn.query(`SELECT key FROM oke_kv WHERE key = ?`, ["stale"]);
+      expect(leftover).toEqual([]);
+    } finally {
+      await conn.close();
+    }
+  });
+
+  test("non-durable namespace stays on redis; durable uses pglite", async () => {
+    const fake = createRedisFakeClient();
+    const sessions = store.kv("sessions");
+    const ledger = store.kv("ledger", { durable: true });
+    const runtime = createStoreRuntime({
+      drivers: {
+        sql: pgliteDriver,
+        kv: redisDriver,
+        files: memoryDrivers.files,
+        index: memoryDrivers.index,
+      },
+      sqlUrl: "memory://",
+      kv: { sessions: { url: "redis://cache", client: fake } },
+      now: () => 1_000_000,
+    });
+    runtime.register(sessions);
+    runtime.register(ledger);
+    const cache = (await runtime.open(sessions, ctx)) as KvStoreFxHandle;
+    const durable = (await runtime.open(ledger, ctx)) as KvStoreFxHandle;
+    expect(cache.driverId).toBe("redis");
+    expect(durable.driverId).toBe("pglite");
+    await cache.set("sid", { user: "a" });
+    await durable.set("idemp", { orderId: "o1" });
+    expect(await cache.get("sid")).toEqual({ user: "a" });
+    expect(await durable.get("idemp")).toEqual({ orderId: "o1" });
+    expect([...fake.data.keys()].some((k) => k.includes("idemp"))).toBe(false);
+    await runtime.close();
+  });
+});

package/src/elements/store/kv-sql.ts

@@ -0,0 +1,178 @@
+/**
+ * SQL-backed durable KV — `oke_kv` JSONB on the shared store.sql connection.
+ *
+ * Not a catalog `drivers.store.kv` id. Cache namespaces stay redis/memory.
+ */
+
+import type { KvNamespace, SqlConnection } from "../../drivers/types.ts";
+import { parseTtlMs } from "./cache.ts";
+
+/** Engine-owned durable KV table — boot DDL, excluded from `oke db push`. */
+export const OKE_KV_TABLE = "oke_kv";
+
+/** Options for {@link openSqlKvNamespace}. */
+export interface OpenSqlKvOptions {
+  /** Shared SQL connection (`sharedSqlConn`). */
+  readonly conn: SqlConnection;
+  /** `store.kv` namespace name. */
+  readonly namespace: string;
+  /** Injectable clock (epoch ms). */
+  readonly now: () => number;
+}
+
+/** SQL KV handle — {@link KvNamespace} plus {@link SqlKvNamespace.purgeExpired}. */
+export interface SqlKvNamespace extends KvNamespace {
+  readonly driverId: "postgres" | "pglite" | "memory";
+  /**
+   * Delete rows whose `expires_at` has passed.
+   *
+   * @returns Rows removed
+   */
+  purgeExpired(): Promise<number>;
+}
+
+const ensured = new WeakSet<SqlConnection>();
+
+/**
+ * Create `oke_kv` + the expires partial index (idempotent).
+ *
+ * @param conn - Shared SQL connection
+ */
+export async function ensureOkeKvSchema(conn: SqlConnection): Promise<void> {
+  if (ensured.has(conn)) return;
+  await conn.exec(`CREATE TABLE IF NOT EXISTS ${OKE_KV_TABLE} (
+    namespace TEXT NOT NULL,
+    key TEXT NOT NULL,
+    value JSONB NOT NULL,
+    expires_at BIGINT,
+    updated_at BIGINT NOT NULL,
+    PRIMARY KEY (namespace, key)
+  )`);
+  await conn.exec(
+    `CREATE INDEX IF NOT EXISTS oke_kv_expires ON ${OKE_KV_TABLE} (expires_at) WHERE expires_at IS NOT NULL`,
+  );
+  ensured.add(conn);
+}
+
+/**
+ * Open a durable KV namespace on an already-open SQL connection.
+ *
+ * @param options - Connection / namespace / clock
+ */
+export async function openSqlKvNamespace(options: OpenSqlKvOptions): Promise<SqlKvNamespace> {
+  const { conn, namespace, now } = options;
+  await ensureOkeKvSchema(conn);
+  await purgeExpiredOn(conn, now());
+
+  return {
+    driverId: conn.driverId,
+    async get(key) {
+      const rows = await conn.query(
+        `SELECT value FROM ${OKE_KV_TABLE}
+         WHERE namespace = ? AND key = ?
+           AND (expires_at IS NULL OR expires_at > ?)`,
+        [namespace, key, now()],
+      );
+      const raw = rows[0]?.value;
+      if (raw === undefined) return undefined;
+      return decodeJsonb(raw);
+    },
+    async set(key, value, ttl) {
+      await purgeExpiredOn(conn, now());
+      const t = now();
+      const expiresAt = ttlExpiresAt(ttl, t);
+      await conn.exec(
+        `INSERT INTO ${OKE_KV_TABLE} (namespace, key, value, expires_at, updated_at)
+         VALUES (?, ?, CAST(? AS JSONB), ?, ?)
+         ON CONFLICT (namespace, key) DO UPDATE SET
+           value = EXCLUDED.value,
+           expires_at = EXCLUDED.expires_at,
+           updated_at = EXCLUDED.updated_at`,
+        [namespace, key, JSON.stringify(value), expiresAt, t],
+      );
+    },
+    async delete(key) {
+      const result = await conn.exec(
+        `DELETE FROM ${OKE_KV_TABLE} WHERE namespace = ? AND key = ?`,
+        [namespace, key],
+      );
+      return result.changes > 0;
+    },
+    async list(prefix = "") {
+      const rows = await conn.query(
+        `SELECT key FROM ${OKE_KV_TABLE}
+         WHERE namespace = ?
+           AND (expires_at IS NULL OR expires_at > ?)
+           AND key LIKE ?
+         ORDER BY key`,
+        [namespace, now(), `${prefix}%`],
+      );
+      return rows.map((row) => String(row.key ?? ""));
+    },
+    async ttlMs(key) {
+      const rows = await conn.query(
+        `SELECT expires_at FROM ${OKE_KV_TABLE}
+         WHERE namespace = ? AND key = ?
+           AND (expires_at IS NULL OR expires_at > ?)`,
+        [namespace, key, now()],
+      );
+      const raw = rows[0]?.expires_at;
+      if (raw === undefined || raw === null) return null;
+      const at = Number(raw);
+      if (!Number.isFinite(at)) return null;
+      return Math.max(0, at - now());
+    },
+    async eval() {
+      throw new Error("oke store: durable store.kv does not support eval");
+    },
+    async close() {
+      /* Shared connection — StoreRuntime.close owns it. */
+    },
+    async purgeExpired() {
+      return purgeExpiredOn(conn, now());
+    },
+  };
+}
+
+/**
+ * Decode a JSONB cell — PGlite/Postgres may return an object or a string.
+ *
+ * @param raw - Driver cell
+ */
+function decodeJsonb(raw: unknown): unknown {
+  if (typeof raw === "string") {
+    try {
+      return JSON.parse(raw) as unknown;
+    } catch {
+      return raw;
+    }
+  }
+  return raw;
+}
+
+/**
+ * Absolute expiry from a TTL string, or `null` when omitted / unparsed.
+ *
+ * @param ttl - Duration like `"7d"`
+ * @param t - Now (epoch ms)
+ */
+function ttlExpiresAt(ttl: string | undefined, t: number): number | null {
+  if (ttl === undefined) return null;
+  const ms = parseTtlMs(ttl);
+  if (ms <= 0) return null;
+  return t + ms;
+}
+
+/**
+ * Delete expired rows across every namespace on this connection.
+ *
+ * @param conn - Shared SQL connection
+ * @param t - Now (epoch ms)
+ */
+async function purgeExpiredOn(conn: SqlConnection, t: number): Promise<number> {
+  const result = await conn.exec(
+    `DELETE FROM ${OKE_KV_TABLE} WHERE expires_at IS NOT NULL AND expires_at <= ?`,
+    [t],
+  );
+  return result.changes;
+}

package/src/elements/store/runtime.ts

@@ -42,6 +42,7 @@ import type {
   SqlStoreDecl,
   StoreDecl,
 } from "./declare.ts";
+import { openSqlKvNamespace } from "./kv-sql.ts";
 import { createGatedFilesStoreHandle, type GatedFilesFxBridge } from "./files-fx.ts";
 import {
   createFilesImagePipeline,
@@ -98,6 +99,11 @@ export interface CreateStoreRuntimeOptions {
    * Boot sets `off` for docker/prod and local+autoPush.
    */
   readonly domainDdl?: DomainDdlMode;
+  /**
+   * Shared SQL URL for durable KV (and apps with no `store.sql()` decl).
+   * Durable namespaces open via `sharedSqlConn`, never a second Redis.
+   */
+  readonly sqlUrl?: string;
 }
 
 /** Per-invocation context when opening a handle through the runtime. */
@@ -118,7 +124,7 @@ export type StoreHandle =
 /** KV handle on `fx.store`. */
 export interface KvStoreFxHandle {
   readonly ref: `kv:${string}`;
-  readonly driverId: "memory" | "redis";
+  readonly driverId: "memory" | "redis" | "postgres" | "pglite";
   get(key: string): Promise<unknown>;
   set(key: string, value: unknown, ttl?: string): Promise<void>;
   delete(key: string): Promise<boolean>;
@@ -383,17 +389,37 @@ export function createStoreRuntime(options: CreateStoreRuntimeOptions): StoreRun
     return sharedSqlConn(sqlDriver, "primary", { url });
   }
 
+  /**
+   * Durable KV — JSONB table on the shared SQL connection.
+   *
+   * @param name - Namespace
+   */
+  async function openDurableKv(name: string): Promise<KvNamespace> {
+    const sqlDriver = options.drivers.sql;
+    if (!sqlDriver) {
+      throw new Error("oke store: durable store.kv needs a configured sql driver");
+    }
+    const firstSql = Object.values(options.sql ?? [])[0];
+    const url = options.sqlUrl ?? firstSql?.primary.url;
+    const conn = await sharedSqlConn(sqlDriver, "primary", { url });
+    return openSqlKvNamespace({ conn, namespace: name, now });
+  }
+
   async function openKv(decl: KvStoreDecl): Promise<KvStoreFxHandle> {
-    const driver = options.drivers.kv;
-    if (!driver) throw new Error("No kv driver configured");
     let ns = kvNs.get(decl.name);
     if (!ns) {
-      const binding = options.kv?.[decl.name] ?? {};
-      ns = await driver.open({
-        name: decl.name,
-        url: binding.url,
-        client: binding.client as never,
-      });
+      if (decl.durable === true) {
+        ns = await openDurableKv(decl.name);
+      } else {
+        const driver = options.drivers.kv;
+        if (!driver) throw new Error("No kv driver configured");
+        const binding = options.kv?.[decl.name] ?? {};
+        ns = await driver.open({
+          name: decl.name,
+          url: binding.url,
+          client: binding.client as never,
+        });
+      }
       kvNs.set(decl.name, ns);
     }
     const driverId = ns.driverId;

package/src/elements/store.test.ts

@@ -63,6 +63,8 @@ describe("store facets", () => {
   test("declares sql · kv · files · index with resource refs", () => {
     expect(store.sql("notes").ref).toBe("sql:notes");
     expect(store.kv("sessions").ref).toBe("kv:sessions");
+    expect(store.kv("ledger", { durable: true }).durable).toBe(true);
+    expect(store.kv("sessions").durable).toBeUndefined();
     expect(store.files("uploads").ref).toBe("files:uploads");
     expect(store.index("kb", { dims: 8 }).ref).toBe("index:kb");
   });

package/src/kernel/boot-bind/store.test.ts

@@ -3,11 +3,12 @@
  */
 
 import { afterEach, describe, expect, test } from "bun:test";
-import { files } from "../../elements/store.ts";
+import { files, store } from "../../elements/store.ts";
 import {
   bindStore,
   indexDriverFor,
   resetFilesFsWarnForTests,
+  resetKvDurableWarnsForTests,
   resolveFilesDriverId,
   resolveIndexDriverId,
   resolveKvDriverId,
@@ -98,6 +99,154 @@ describe("bindStore driver resolution", () => {
   });
 });
 
+describe("bindStore durable KV routing", () => {
+  const prev = {
+    redis: process.env.REDIS_URL,
+    kv: process.env.OKE_STORE_KV_URL,
+    database: process.env.DATABASE_URL,
+    sql: process.env.OKE_STORE_SQL_URL,
+  };
+
+  afterEach(() => {
+    resetKvDurableWarnsForTests();
+    if (prev.redis === undefined) delete process.env.REDIS_URL;
+    else process.env.REDIS_URL = prev.redis;
+    if (prev.kv === undefined) delete process.env.OKE_STORE_KV_URL;
+    else process.env.OKE_STORE_KV_URL = prev.kv;
+    if (prev.database === undefined) delete process.env.DATABASE_URL;
+    else process.env.DATABASE_URL = prev.database;
+    if (prev.sql === undefined) delete process.env.OKE_STORE_SQL_URL;
+    else process.env.OKE_STORE_SQL_URL = prev.sql;
+  });
+
+  test("pglite + durable in test does not throw or warn", () => {
+    const warnings: string[] = [];
+    const prevWarn = console.warn;
+    console.warn = (...args: unknown[]) => {
+      warnings.push(args.map(String).join(" "));
+    };
+    try {
+      bindStore(
+        { stores: [store.kv("ledger", { durable: true })] },
+        "test",
+        () => Date.now(),
+        false,
+      );
+      expect(warnings.some((w) => w.includes("durable"))).toBe(false);
+    } finally {
+      console.warn = prevWarn;
+    }
+  });
+
+  test("postgres + durable without DATABASE_URL throws outside test", () => {
+    delete process.env.DATABASE_URL;
+    delete process.env.OKE_STORE_SQL_URL;
+    process.env.REDIS_URL = "redis://cache";
+    expect(() =>
+      bindStore(
+        {
+          stores: [store.kv("ledger", { durable: true })],
+          config: {
+            drivers: {
+              store: {
+                sql: { dev: "postgres", test: "pglite", prod: "postgres" },
+                kv: { dev: "redis", test: "memory", prod: "redis" },
+              },
+            },
+          },
+        },
+        "dev",
+        () => Date.now(),
+        false,
+      ),
+    ).toThrow(/DATABASE_URL/);
+  });
+
+  test("redis + cache-shaped KV warns once outside test", () => {
+    process.env.REDIS_URL = "redis://cache";
+    const warnings: string[] = [];
+    const prevWarn = console.warn;
+    console.warn = (...args: unknown[]) => {
+      warnings.push(args.map(String).join(" "));
+    };
+    try {
+      const options = {
+        stores: [store.kv("sessions")],
+        config: {
+          drivers: {
+            store: {
+              sql: { dev: "memory", test: "memory", prod: "memory" },
+              kv: { dev: "redis", test: "memory", prod: "redis" },
+            },
+          },
+        },
+      };
+      bindStore(options, "dev", () => Date.now(), false);
+      bindStore(options, "dev", () => Date.now(), false);
+      expect(warnings.filter((w) => w.includes("cache-shaped"))).toHaveLength(1);
+    } finally {
+      console.warn = prevWarn;
+    }
+  });
+
+  test("non-durable KV keeps REDIS_URL; durable does not", async () => {
+    process.env.REDIS_URL = "redis://cache";
+    const sessions = store.kv("sessions");
+    const ledger = store.kv("ledger", { durable: true });
+    const runtime = bindStore(
+      {
+        stores: [sessions, ledger],
+        config: {
+          drivers: {
+            store: {
+              sql: { test: "pglite" },
+              kv: { test: "memory" },
+            },
+          },
+        },
+      },
+      "test",
+      () => Date.now(),
+      false,
+    );
+    const ctx = { effects: {} };
+    const cache = await runtime.open(sessions, ctx);
+    const durable = await runtime.open(ledger, ctx);
+    expect(cache.driverId).toBe("memory");
+    expect(durable.driverId).toBe("pglite");
+    await runtime.close();
+  });
+
+  test("memory SQL + durable in dev warns that memory cannot honor durability", () => {
+    const warnings: string[] = [];
+    const prevWarn = console.warn;
+    console.warn = (...args: unknown[]) => {
+      warnings.push(args.map(String).join(" "));
+    };
+    try {
+      bindStore(
+        {
+          stores: [store.kv("ledger", { durable: true })],
+          config: {
+            drivers: {
+              store: {
+                sql: { dev: "memory", test: "memory", prod: "memory" },
+                kv: { dev: "memory", test: "memory", prod: "redis" },
+              },
+            },
+          },
+        },
+        "dev",
+        () => Date.now(),
+        false,
+      );
+      expect(warnings.some((w) => w.includes("memory") && w.includes("durable"))).toBe(true);
+    } finally {
+      console.warn = prevWarn;
+    }
+  });
+});
+
 describe("bindStore files fs multi-instance warn", () => {
   afterEach(() => {
     resetFilesFsWarnForTests();