okengine 0.3.5 → 0.3.6

package/src/kernel/boot-bind/store.test.ts

@@ -3,7 +3,13 @@
  */
 
 import { afterEach, describe, expect, test } from "bun:test";
-import { resolveFilesDriverId, resolveKvDriverId, resolveSqlDriverId } from "./store.ts";
+import {
+  indexDriverFor,
+  resolveFilesDriverId,
+  resolveIndexDriverId,
+  resolveKvDriverId,
+  resolveSqlDriverId,
+} from "./store.ts";
 
 describe("bindStore driver resolution", () => {
   const prev = {
@@ -11,6 +17,7 @@ describe("bindStore driver resolution", () => {
     sql: process.env.OKE_SQL_DRIVER,
     kv: process.env.OKE_KV_DRIVER,
     files: process.env.OKE_FILES_DRIVER,
+    index: process.env.OKE_INDEX_DRIVER,
   };
 
   afterEach(() => {
@@ -22,6 +29,8 @@ describe("bindStore driver resolution", () => {
     else process.env.OKE_KV_DRIVER = prev.kv;
     if (prev.files === undefined) delete process.env.OKE_FILES_DRIVER;
     else process.env.OKE_FILES_DRIVER = prev.files;
+    if (prev.index === undefined) delete process.env.OKE_INDEX_DRIVER;
+    else process.env.OKE_INDEX_DRIVER = prev.index;
   });
 
   test("local env keeps sqlite / memory from config", () => {
@@ -67,3 +76,52 @@ describe("bindStore driver resolution", () => {
     expect(resolveFilesDriverId({}, "docker", true)).toBe("s3");
   });
 });
+
+describe("bindStore index driver resolution", () => {
+  const prevIndex = process.env.OKE_INDEX_DRIVER;
+
+  afterEach(() => {
+    if (prevIndex === undefined) delete process.env.OKE_INDEX_DRIVER;
+    else process.env.OKE_INDEX_DRIVER = prevIndex;
+  });
+
+  test("defaults to memory when no index driver is configured", () => {
+    expect(resolveIndexDriverId({}, "local", false)).toBe("memory");
+    expect(resolveIndexDriverId({}, "docker", true)).toBe("memory");
+    expect(indexDriverFor("memory").id).toBe("memory");
+  });
+
+  test("config drivers.store.index is honoured per env", () => {
+    const options = {
+      config: {
+        drivers: {
+          store: {
+            index: { local: "memory", docker: "pgvector", prod: "pgvector" },
+          },
+        },
+      },
+    };
+    expect(resolveIndexDriverId(options, "local", false)).toBe("memory");
+    expect(resolveIndexDriverId(options, "docker", true)).toBe("pgvector");
+    expect(resolveIndexDriverId(options, "prod", false)).toBe("pgvector");
+  });
+
+  test("libsql resolves from config", () => {
+    const options = {
+      config: { drivers: { store: { index: { local: "libsql" } } } },
+    };
+    expect(resolveIndexDriverId(options, "local", false)).toBe("libsql");
+    expect(indexDriverFor("libsql").id).toBe("libsql");
+  });
+
+  test("docker mode honours OKE_INDEX_DRIVER override", () => {
+    process.env.OKE_INDEX_DRIVER = "pgvector";
+    expect(resolveIndexDriverId({}, "docker", true)).toBe("pgvector");
+    expect(resolveIndexDriverId({}, "local", false)).toBe("memory");
+  });
+
+  test("indexDriverFor returns the configured driver and throws on unknown ids", () => {
+    expect(indexDriverFor("pgvector").id).toBe("pgvector");
+    expect(() => indexDriverFor("chroma")).toThrow(/unknown index driver "chroma"/);
+  });
+});

package/src/kernel/boot-bind/store.ts

@@ -4,12 +4,15 @@
 
 import { resolveDomainDdlMode, resolveDriverId, type ConfigEnv } from "../../config/index.ts";
 import { fsDriver } from "../../drivers/fs.ts";
+import { libsqlDriver, libsqlIndexDriver } from "../../drivers/libsql.ts";
 import { memoryDrivers } from "../../drivers/memory.ts";
+import { pgliteDriver } from "../../drivers/pglite.ts";
+import { pgvectorDriver } from "../../drivers/pgvector.ts";
 import { postgresDriver } from "../../drivers/postgres.ts";
 import { redisDriver } from "../../drivers/redis.ts";
 import { s3Driver } from "../../drivers/s3.ts";
 import { sqliteDriver } from "../../drivers/sqlite.ts";
-import type { FilesDriver, KvDriver, SqlDriver } from "../../drivers/types.ts";
+import type { FilesDriver, IndexDriver, KvDriver, SqlDriver } from "../../drivers/types.ts";
 import { createStoreRuntime, type StoreRuntime } from "../../elements/store.ts";
 import type { StoreDecl } from "../../elements/store/declare.ts";
 import type { BootOptions } from "../boot.ts"; // type-only — no cycle at runtime
@@ -34,6 +37,7 @@ export function bindStore(
   const sqlId = resolveSqlDriverId(options, env, docker);
   const kvId = resolveKvDriverId(options, env, docker);
   const filesId = resolveFilesDriverId(options, env, docker);
+  const indexId = resolveIndexDriverId(options, env, docker);
   const sqlUrl = sqlUrlFor(sqlId, docker);
   const kvUrl = kvUrlFor(kvId, docker);
   const filesRoot = filesRootFor(filesId);
@@ -41,6 +45,7 @@ export function bindStore(
   const sqlBindings: Record<string, { name: string; primary: { url: string } }> = {};
   const kvBindings: Record<string, { url?: string }> = {};
   const filesBindings: Record<string, { root?: string }> = {};
+  const indexBindings: Record<string, { url?: string }> = {};
 
   for (const decl of options.stores ?? []) {
     if (isSqlDecl(decl)) {
@@ -52,6 +57,10 @@ export function bindStore(
       kvBindings[decl.name] = kvUrl !== undefined ? { url: kvUrl } : {};
     } else if (isFilesDecl(decl)) {
       filesBindings[decl.name] = filesRoot !== undefined ? { root: filesRoot } : {};
+    } else if (isIndexDecl(decl)) {
+      // SQL-backed index drivers share the sql facet's URL — the runtime opens
+      // one connection and hands it to both (never a second, redundant one).
+      indexBindings[decl.name] = indexId === "memory" ? {} : { url: sqlUrl };
     }
   }
 
@@ -63,11 +72,12 @@ export function bindStore(
       sql: sqlDriverFor(sqlId),
       kv: kvDriverFor(kvId),
       files: filesDriverFor(filesId),
-      index: memoryDrivers.index,
+      index: indexDriverFor(indexId),
     },
     sql: sqlBindings,
     kv: kvBindings,
     files: filesBindings,
+    index: indexBindings,
     now,
     domainDdl,
   });
@@ -132,12 +142,37 @@ export function resolveFilesDriverId(
   return docker ? "s3" : "memory";
 }
 
+/**
+ * Index driver — same resolution chain as sql/kv/files. Defaults to `memory`;
+ * a configured `pgvector` / `libsql` is honoured at real boot (and fails loud
+ * if its SQL engine or peer is missing — never silently back to memory).
+ *
+ * @param options - Boot options
+ * @param env - Active env
+ * @param docker - Docker mode
+ */
+export function resolveIndexDriverId(
+  options: BootOptions,
+  env: ConfigEnv,
+  docker: boolean,
+): string {
+  const fromEnv = process.env.OKE_INDEX_DRIVER?.trim();
+  if (docker && fromEnv) return fromEnv;
+  const resolved = resolveDriverId(options.config?.drivers?.store?.index, env);
+  if (resolved) return resolved;
+  return "memory";
+}
+
 function sqlDriverFor(id: string): SqlDriver {
   switch (id) {
     case "postgres":
       return postgresDriver;
     case "sqlite":
       return sqliteDriver;
+    case "libsql":
+      return libsqlDriver;
+    case "pglite":
+      return pgliteDriver;
     case "memory":
       return memoryDrivers.sql;
     default:
@@ -169,6 +204,23 @@ function filesDriverFor(id: string): FilesDriver {
   }
 }
 
+/**
+ * Single id → index driver switch — shared by `bindStore` and the Console's
+ * Manifest runtime so resolution can never drift into two maintained copies.
+ */
+export function indexDriverFor(id: string): IndexDriver {
+  switch (id) {
+    case "memory":
+      return memoryDrivers.index;
+    case "pgvector":
+      return pgvectorDriver;
+    case "libsql":
+      return libsqlIndexDriver;
+    default:
+      throw new Error(`oke boot: unknown index driver "${id}"`);
+  }
+}
+
 function sqlUrlFor(sqlId: string, docker: boolean): string {
   if (sqlId === "postgres") {
     const url = process.env.DATABASE_URL ?? process.env.OKE_STORE_SQL_URL ?? undefined;
@@ -184,6 +236,12 @@ function sqlUrlFor(sqlId: string, docker: boolean): string {
   if (sqlId === "sqlite") {
     return process.env.OKE_SQLITE_URL ?? ".oke/app.sqlite";
   }
+  if (sqlId === "libsql") {
+    return process.env.OKE_LIBSQL_URL ?? ".oke/app.libsql";
+  }
+  if (sqlId === "pglite") {
+    return process.env.OKE_PGLITE_URL ?? ".oke/pgdata";
+  }
   return ":memory:";
 }
 
@@ -216,3 +274,7 @@ function isKvDecl(decl: StoreDecl): decl is Extract<StoreDecl, { facet: "kv" }>
 function isFilesDecl(decl: StoreDecl): decl is Extract<StoreDecl, { facet: "files" }> {
   return decl.facet === "files";
 }
+
+function isIndexDecl(decl: StoreDecl): decl is Extract<StoreDecl, { facet: "index" }> {
+  return decl.facet === "index";
+}

package/src/kernel/concurrency.test.ts

@@ -0,0 +1,236 @@
+import { describe, expect, test } from "bun:test";
+
+import { currentAbortSignal, isAbortError, withAbortSignal } from "./abort-scope.ts";
+import { createMemoryJournalStore } from "./journal.ts";
+import { createFx, createFxContext } from "./fx.ts";
+import { flow } from "./flow.ts";
+import { runDurable } from "../elements/clock/durable.ts";
+import { fxAll, fxRace } from "./concurrency.ts";
+
+describe("fx.all — structured concurrency", () => {
+  test("resolves tuple results in order", async () => {
+    const fx = createFx({ flow: "t", effects: {} });
+    const out = await fx.all([async () => 1, async () => "a", () => true]);
+    expect(out).toEqual([1, "a", true]);
+  });
+
+  test("on failure aborts sibling branches via ambient signal", async () => {
+    const fx = createFx({ flow: "t", effects: {} });
+    let siblingAborted = false;
+    let loserSawAbort = false;
+
+    await expect(
+      fx.all([
+        async () => {
+          await new Promise<void>((_resolve, reject) => {
+            const signal = fx.signal;
+            if (signal.aborted) {
+              loserSawAbort = true;
+              reject(signal.reason ?? new Error("aborted"));
+              return;
+            }
+            signal.addEventListener(
+              "abort",
+              () => {
+                siblingAborted = true;
+                loserSawAbort = true;
+                reject(new DOMException("aborted", "AbortError"));
+              },
+              { once: true },
+            );
+            // Stay pending until aborted.
+          });
+        },
+        async () => {
+          await new Promise((r) => setTimeout(r, 5));
+          throw new Error("boom");
+        },
+      ]),
+    ).rejects.toThrow("boom");
+
+    expect(siblingAborted || loserSawAbort).toBe(true);
+  });
+
+  test("empty all resolves to []", async () => {
+    await expect(fxAll([])).resolves.toEqual([]);
+  });
+});
+
+describe("fx.race — structured concurrency", () => {
+  test("winner wins and loser sees abort", async () => {
+    const fx = createFx({ flow: "t", effects: {} });
+    let loserAborted = false;
+
+    const value = await fx.race([
+      async () => {
+        await new Promise<void>((_resolve, reject) => {
+          fx.signal.addEventListener(
+            "abort",
+            () => {
+              loserAborted = true;
+              reject(new DOMException("aborted", "AbortError"));
+            },
+            { once: true },
+          );
+        });
+        return "slow";
+      },
+      async () => {
+        await new Promise((r) => setTimeout(r, 5));
+        return "fast";
+      },
+    ]);
+
+    expect(value).toBe("fast");
+    // Give the abort listener a tick.
+    await new Promise((r) => setTimeout(r, 10));
+    expect(loserAborted).toBe(true);
+  });
+
+  test("empty race throws", async () => {
+    await expect(fxRace([])).rejects.toThrow(/empty/);
+  });
+});
+
+describe("fx.retry", () => {
+  test("retries then succeeds", async () => {
+    const fx = createFx({ flow: "t", effects: {} });
+    let n = 0;
+    const value = await fx.retry(
+      async () => {
+        n += 1;
+        if (n < 3) throw new Error("transient");
+        return "ok";
+      },
+      { retries: 3, delay: 0, jitter: false },
+    );
+    expect(value).toBe("ok");
+    expect(n).toBe(3);
+  });
+
+  test("does not retry AbortError", async () => {
+    const fx = createFx({ flow: "t", effects: {} });
+    let n = 0;
+    await expect(
+      fx.retry(
+        async () => {
+          n += 1;
+          throw new DOMException("aborted", "AbortError");
+        },
+        { retries: 3, delay: 0 },
+      ),
+    ).rejects.toMatchObject({ name: "AbortError" });
+    expect(n).toBe(1);
+  });
+
+  test("honors when predicate", async () => {
+    const fx = createFx({ flow: "t", effects: {} });
+    let n = 0;
+    await expect(
+      fx.retry(
+        async () => {
+          n += 1;
+          throw new Error("nope");
+        },
+        { retries: 5, delay: 0, when: () => false },
+      ),
+    ).rejects.toThrow("nope");
+    expect(n).toBe(1);
+  });
+});
+
+describe("ambient abort signal", () => {
+  test("outside scope is never aborted", () => {
+    const fx = createFx({ flow: "t", effects: {} });
+    expect(fx.signal.aborted).toBe(false);
+    expect(isAbortError(new DOMException("x", "AbortError"))).toBe(true);
+  });
+
+  test("withAbortSignal installs ambient signal", async () => {
+    const ctrl = new AbortController();
+    await withAbortSignal(ctrl.signal, async () => {
+      expect(currentAbortSignal()).toBe(ctrl.signal);
+    });
+  });
+});
+
+describe("retry + durable journal", () => {
+  test("fx.retry inside fx.step does not re-run completed step on resume", async () => {
+    const journalStore = createMemoryJournalStore();
+    let attempts = 0;
+    let crash = true;
+
+    const f = flow({
+      name: "pay.retry",
+      durable: true,
+      do: async (_input, fx) => {
+        const charged = await fx.step("charge", () =>
+          fx.retry(
+            async () => {
+              attempts += 1;
+              return { id: "ch_1" };
+            },
+            { retries: 2, delay: 0 },
+          ),
+        );
+        if (crash) throw new Error("KILLED");
+        return charged;
+      },
+    });
+
+    const first = await runDurable({ flow: f, journalStore });
+    expect(first.status).toBe("failed");
+    expect(attempts).toBe(1);
+
+    crash = false;
+    const second = await runDurable({
+      flow: f,
+      journalStore,
+      runId: first.status === "failed" ? first.runId : "",
+    });
+    expect(second.status).toBe("completed");
+    expect(attempts).toBe(1);
+  });
+
+  test("flow({ retry }) reuses journal — completed steps do not re-run", async () => {
+    const journalStore = createMemoryJournalStore();
+    const calls: string[] = [];
+    let failOnce = true;
+
+    const f = flow({
+      name: "flow.retry",
+      durable: true,
+      retry: { retries: 2, delay: 0, jitter: false },
+      do: async (_input, fx) => {
+        await fx.step("a", () => {
+          calls.push("a");
+          return 1;
+        });
+        if (failOnce) {
+          failOnce = false;
+          throw new Error("transient");
+        }
+        return fx.step("b", () => {
+          calls.push("b");
+          return 2;
+        });
+      },
+    });
+
+    const result = await runDurable({ flow: f, journalStore });
+    expect(result.status).toBe("completed");
+    // step a ran once; flow-level retry re-entered do but journal replayed a.
+    expect(calls).toEqual(["a", "b"]);
+  });
+});
+
+describe("capability + concurrency compose", () => {
+  test("fx.all records effects from successful branches", async () => {
+    const { fx, ledger } = createFxContext({
+      flow: "parallel",
+      effects: { emits: ["a", "b"] },
+    });
+    await fx.all([() => fx.emit("a"), () => fx.emit("b")]);
+    expect(ledger.entries.map((e) => e.resource)).toEqual(["a", "b"]);
+  });
+});

package/src/kernel/concurrency.ts

@@ -0,0 +1,172 @@
+/**
+ * Structured concurrency and retry — plain Promise helpers for `fx`.
+ *
+ * No generators, no monadic wrapper. Branches are thunks so an AbortController
+ * can be installed before work starts. Retry backoff uses non-journaled sleep.
+ */
+
+import { parseDurationMs } from "../elements/clock/duration.ts";
+import {
+  abortableSleep,
+  currentAbortSignal,
+  isAbortError,
+  linkAbort,
+  withAbortSignal,
+} from "./abort-scope.ts";
+import { isJournalSuspend } from "./journal.ts";
+
+/** A unit of work started under an abort scope. */
+export type FxThunk<T> = () => T | Promise<T>;
+
+/**
+ * Retry policy — same shape as the client transport, with duration strings
+ * and optional full jitter (no Schedule DSL).
+ */
+export interface FxRetryOptions {
+  /** Extra attempts after the first (default 0). */
+  readonly retries?: number;
+  /** Initial delay: ms number or duration string (default 50). */
+  readonly delay?: number | string;
+  /** Multiplier applied after each retry (default 2). */
+  readonly backoff?: number;
+  /** Full jitter on delay (default true). */
+  readonly jitter?: boolean;
+  /** Predicate; default retries thrown errors except abort / journal suspend. */
+  readonly when?: (err: unknown) => boolean;
+}
+
+/**
+ * Default retry filter — skip abort and durable-sleep park.
+ *
+ * @param err - Thrown value
+ */
+export function defaultRetryWhen(err: unknown): boolean {
+  if (isAbortError(err)) return false;
+  if (isJournalSuspend(err)) return false;
+  return true;
+}
+
+/**
+ * Resolve a delay option to milliseconds.
+ *
+ * @param delay - Number or duration string
+ * @param fallback - Default ms
+ */
+export function resolveRetryDelayMs(delay: number | string | undefined, fallback = 50): number {
+  if (delay === undefined) return fallback;
+  if (typeof delay === "number") return Math.max(0, delay);
+  const parsed = parseDurationMs(delay);
+  return parsed > 0 ? parsed : fallback;
+}
+
+/**
+ * Run thunks in parallel; on first rejection abort siblings and rethrow.
+ *
+ * @param thunks - Work units (not already-started Promises)
+ */
+export async function fxAll<const T extends readonly unknown[]>(thunks: {
+  readonly [K in keyof T]: FxThunk<T[K]>;
+}): Promise<{ -readonly [K in keyof T]: Awaited<T[K]> }> {
+  const list = thunks as readonly FxThunk<unknown>[];
+  if (list.length === 0) {
+    return [] as { -readonly [K in keyof T]: Awaited<T[K]> };
+  }
+
+  const parent = currentAbortSignal();
+  const controllers = list.map(() => new AbortController());
+  const unsubs = controllers.map((c) => linkAbort(parent, c));
+
+  let firstError: unknown;
+  const results = await Promise.allSettled(
+    list.map((thunk, i) =>
+      withAbortSignal(controllers[i]!.signal, async () => {
+        try {
+          return await thunk();
+        } catch (err) {
+          if (firstError === undefined) {
+            firstError = err;
+            for (let j = 0; j < controllers.length; j++) {
+              if (j !== i) controllers[j]!.abort(err);
+            }
+          }
+          throw err;
+        }
+      }),
+    ),
+  );
+
+  for (const unsub of unsubs) unsub();
+
+  if (firstError !== undefined) throw firstError;
+  const values: unknown[] = [];
+  for (const r of results) {
+    if (r.status === "rejected") throw r.reason;
+    values.push(r.value);
+  }
+  return values as { -readonly [K in keyof T]: Awaited<T[K]> };
+}
+
+/**
+ * Race thunks; the first settle wins and siblings are aborted.
+ *
+ * @param thunks - Work units
+ */
+export async function fxRace<T>(thunks: ReadonlyArray<FxThunk<T>>): Promise<T> {
+  if (thunks.length === 0) {
+    throw new TypeError("fx.race: empty thunk list");
+  }
+
+  const parent = currentAbortSignal();
+  const controllers = thunks.map(() => new AbortController());
+  const unsubs = controllers.map((c) => linkAbort(parent, c));
+
+  return new Promise<T>((resolve, reject) => {
+    let done = false;
+    const settle = (ok: boolean, value: unknown): void => {
+      if (done) return;
+      done = true;
+      for (const c of controllers) {
+        if (!c.signal.aborted) c.abort();
+      }
+      for (const unsub of unsubs) unsub();
+      if (ok) resolve(value as T);
+      else reject(value);
+    };
+
+    for (let i = 0; i < thunks.length; i++) {
+      const thunk = thunks[i]!;
+      const ctrl = controllers[i]!;
+      void withAbortSignal(ctrl.signal, () => Promise.resolve().then(thunk)).then(
+        (value) => settle(true, value),
+        (err: unknown) => settle(false, err),
+      );
+    }
+  });
+}
+
+/**
+ * Retry a thunk with exponential backoff and optional full jitter.
+ *
+ * @param fn - Operation
+ * @param opts - Retry policy
+ */
+export async function fxRetry<T>(fn: FxThunk<T>, opts: FxRetryOptions = {}): Promise<T> {
+  const retries = Math.max(0, opts.retries ?? 0);
+  const backoff = opts.backoff ?? 2;
+  const jitter = opts.jitter !== false;
+  const when = opts.when ?? defaultRetryWhen;
+  let delayMs = resolveRetryDelayMs(opts.delay, 50);
+  let attempt = 0;
+
+  for (;;) {
+    try {
+      return await fn();
+    } catch (err) {
+      if (attempt >= retries || !when(err)) throw err;
+      const wait = jitter ? Math.random() * delayMs : delayMs;
+      await abortableSleep(wait, currentAbortSignal());
+      delayMs *= backoff;
+      attempt += 1;
+    }
+  }
+}

package/src/kernel/flow.ts

@@ -8,6 +8,7 @@
 
 import type { Effects, FlowPlane, Slo } from "../manifest/types.ts";
 import type { InferSchemaOutput, SchemaInput } from "../validation/standard-schema.ts";
+import type { FxRetryOptions } from "./concurrency.ts";
 import type { FlowFailure } from "./errors.ts";
 import type { Fx } from "./fx.ts";
 import type { HookFn, HookStage } from "./hooks.ts";
@@ -51,6 +52,11 @@ export interface FlowOptions<I = unknown, O = unknown, E extends FlowErrorMap =
   readonly effects?: Effects;
   /** Journal every effect call. */
   readonly durable?: boolean;
+  /**
+   * Retry the whole `do` body on thrown errors (same journal session when
+   * durable). Prefer {@link Fx.retry} inside {@link Fx.step} for fine control.
+   */
+  readonly retry?: FxRetryOptions;
   /** Live-query / push result. */
   readonly live?: boolean;
   /** Cache TTL (`true` = default policy, or a duration string). */
@@ -147,6 +153,8 @@ export interface FlowDef<
   readonly effects: Effects | undefined;
   /** Durability flag. */
   readonly durable: boolean;
+  /** Whole-body retry policy (runtime). */
+  readonly retry: FxRetryOptions | undefined;
   /** Live flag. */
   readonly live: boolean;
   /** Cache option. */
@@ -222,6 +230,7 @@ export function flow<Opts extends FlowOptions<any, any, any>>(
     errors: (options.errors ?? undefined) as InferFlowErrors<Opts> | undefined,
     effects: options.effects,
     durable: options.durable ?? false,
+    retry: options.retry,
     live: options.live ?? false,
     cache: options.cache,
     slo: options.slo,

package/src/kernel/fx.test.ts

@@ -181,6 +181,18 @@ describe("fx — wholesale swap", () => {
       async step(_name, fn) {
         return fn();
       },
+      get signal() {
+        return new AbortController().signal;
+      },
+      all(thunks) {
+        return Promise.all(thunks.map((t) => t())) as never;
+      },
+      race(thunks) {
+        return Promise.race(thunks.map((t) => t()));
+      },
+      retry(fn) {
+        return Promise.resolve().then(fn);
+      },
     };
 
     // Any code that accepts `Fx` can run against a total replacement.