okengine 0.6.1 → 0.8.0

package/src/kernel/app.ts

@@ -55,6 +55,7 @@ import {
 } from "./fx.ts";
 import {
   currentDevSurface,
+  failureDetailFromResponse,
   logDevRequest,
   shouldLogDevRequests,
 } from "../runtime/dev-request-log.ts";
@@ -70,9 +71,14 @@ import {
 import {
   createJournal,
   createMemoryJournalStore,
+  hasJournalLease,
+  isJournalLeaseBusy,
   isJournalSuspend,
+  JOURNAL_DEFAULT_LEASE_MS,
   type JournalSession,
+  type JournalStore,
 } from "./journal.ts";
+import type { JournalRuntime } from "./boot-bind/journal.ts";
 import { listBindings, resetBindings, type Binding } from "./on.ts";
 import {
   applyPrincipal,
@@ -189,6 +195,8 @@ export interface OkeOptions {
   readonly startScheduler?: boolean;
   /** Scheduler tick period ms (default 1000). Forwarded to boot. */
   readonly schedulerIntervalMs?: number;
+  /** Durable-run lease duration ms (default 30_000). Forwarded to boot. */
+  readonly journalLeaseMs?: number;
 }
 
 /** Payload for a CDC invocation. */
@@ -578,13 +586,37 @@ export function oke(options: OkeOptions): OkeApp {
   } else {
     setActiveGateAuthContext(undefined);
   }
-  const journalStore = createMemoryJournalStore();
+  // Fallback journal for pre-boot / `autoBoot: false` unit tests. A booted
+  // app replaces this with the bound `drivers.journal` store (postgres etc.).
+  const fallbackJournalStore = createMemoryJournalStore();
+  const fallbackJournalInstanceId = `app-${crypto.randomUUID()}`;
   const sleepingRuns = new Map<
     string,
     { readonly flow: AnyFlowDef; readonly input: unknown; readonly wakeAt: number }
   >();
+  // Same-process mutual exclusion per runId. The lease coordinates across
+  // processes, but same-holder renew cannot distinguish two overlapping
+  // sessions inside one process (boot orphan scan vs. scheduler tick).
+  const inflightRuns = new Set<string>();
   const EMPTY_CAPABILITIES: ReadonlyMap<string, CapabilityToken> = new Map();
 
+  /** Active journal: the boot-bound store once available, else the fallback. */
+  function activeJournal(): {
+    readonly store: JournalStore;
+    readonly instanceId: string;
+    readonly leaseMs: number;
+  } {
+    const bound: JournalRuntime | undefined = bootResult?.journal;
+    if (bound) {
+      return { store: bound.store, instanceId: bound.instanceId, leaseMs: bound.leaseMs };
+    }
+    return {
+      store: fallbackJournalStore,
+      instanceId: fallbackJournalInstanceId,
+      leaseMs: JOURNAL_DEFAULT_LEASE_MS,
+    };
+  }
+
   async function handleCronFire(name: string): Promise<void> {
     await app.dispatchEvery(name);
     // Named clocks (e.g. `clock("expire-stale", { every: "1h" })`) reconcile
@@ -599,6 +631,84 @@ export function oke(options: OkeOptions): OkeApp {
     await app.dispatchSignal(name, payload);
   }
 
+  async function handleDurableResume(): Promise<void> {
+    await app.resumeDurable();
+  }
+
+  /** Element runtimes handed to durable resumes (same bag as cron dispatch). */
+  function durableResumeFx() {
+    return {
+      storeRuntime: bootResult?.store,
+      signalRuntime: bootResult?.signal,
+      vaultRuntime: bootResult?.vault,
+      channelRuntime: bootResult?.channel,
+      aiRuntime: bootResult?.ai,
+    };
+  }
+
+  /**
+   * Resume one persisted run under its lease. A lost lease race is the
+   * coordination win — another live instance owns the run, so skip quietly.
+   */
+  /** Runs whose undeclared flow was already logged (sweep ticks must not spam). */
+  const warnedOrphanRuns = new Set<string>();
+
+  async function resumeDurableRun(
+    runId: string,
+    flowName: string,
+    input: unknown,
+    now: () => number,
+  ): Promise<void> {
+    // The lease only coordinates across processes — within one process,
+    // overlapping callers (orphan scan vs. scheduler tick) would both pass
+    // same-holder lease renewal and run two sessions for one runId.
+    if (inflightRuns.has(runId)) return;
+    inflightRuns.add(runId);
+    try {
+      const flowDef = flowsByName.get(flowName);
+      if (!flowDef) {
+        if (!warnedOrphanRuns.has(runId)) {
+          warnedOrphanRuns.add(runId);
+          console.warn(`oke: durable run "${runId}": undeclared flow "${flowName}" — skipped`);
+        }
+        return;
+      }
+      const { store, instanceId, leaseMs } = activeJournal();
+      try {
+        await runDurable({
+          flow: flowDef,
+          input,
+          journalStore: store,
+          runId,
+          ...(hasJournalLease(store) ? { lease: { instanceId, leaseMs } } : {}),
+          now,
+          fx: durableResumeFx(),
+        });
+      } catch (err) {
+        if (isJournalLeaseBusy(err)) return;
+        throw err;
+      }
+    } finally {
+      inflightRuns.delete(runId);
+    }
+  }
+
+  /** Boot-time orphan scan — `running`/`sleeping` runs with no live lease. */
+  async function resumeOrphanedDurableRuns(): Promise<void> {
+    const { store } = activeJournal();
+    if (!hasJournalLease(store)) return;
+    const now = () => bootResult?.clock?.now() ?? options.fx?.now?.() ?? Date.now();
+    const orphans = await store.listOrphans(now());
+    for (const orphan of orphans) {
+      // Future sleeps are claimed by the scheduler tick when they come due —
+      // the shared store is the schedule, no in-process seeding needed.
+      if (orphan.status === "sleeping" && orphan.wakeAt !== undefined && orphan.wakeAt > now()) {
+        continue;
+      }
+      await resumeDurableRun(orphan.id, orphan.flow, orphan.input, now);
+    }
+  }
+
   async function doBoot(overrides?: Partial<BootOptions>): Promise<BootResult> {
     const { bootApplication, resolveElementNeeds } = await import("./boot.ts");
     const { assertHttpGatePosture } = await import("../elements/gate/boot.ts");
@@ -715,13 +825,23 @@ export function oke(options: OkeOptions): OkeApp {
       instanceId: overrides?.instanceId,
       startScheduler: overrides?.startScheduler ?? options.startScheduler,
       schedulerIntervalMs: overrides?.schedulerIntervalMs ?? options.schedulerIntervalMs,
+      journalLeaseMs: overrides?.journalLeaseMs ?? options.journalLeaseMs,
       bindings: adopted,
       flows: [...flowsByName.values()],
       onCronFire: overrides?.onCronFire ?? handleCronFire,
       onSignal: overrides?.onSignal ?? handleSignalFire,
+      onDurableResume: overrides?.onDurableResume ?? handleDurableResume,
     };
     const result = await bootApplication(merged);
     bootResult = result;
+    // Boot-time orphan discovery: resume/schedule any `running` / `sleeping`
+    // run left without a live lease by a crashed (or previous) instance.
+    // Fire-and-forget — boot must not block serving on a long resume.
+    if (result.journal && hasJournalLease(result.journal.store)) {
+      void resumeOrphanedDurableRuns().catch((err) => {
+        console.error("oke: durable orphan scan failed", err);
+      });
+    }
     // Prefer the booted clock for access-token expiry checks.
     if (authBinding) {
       authBinding = createAppAuthBinding({
@@ -907,8 +1027,16 @@ export function oke(options: OkeOptions): OkeApp {
       capability = booted.capabilities.get(flowDef.name);
 
       if (flowDef.durable) {
-        const journal = createJournal({ store: journalStore, now });
+        const { store, instanceId, leaseMs } = activeJournal();
+        const journal = createJournal({
+          store,
+          now,
+          // Hold the run lease for the request's lifetime — a crash mid-run
+          // leaves an expired lease another instance can reclaim and resume.
+          ...(hasJournalLease(store) ? { lease: { instanceId, leaseMs } } : {}),
+        });
         journalSession = await journal.start(flowDef.name, input);
+        inflightRuns.add(journalSession.runId);
       }
     }
 
@@ -997,25 +1125,34 @@ export function oke(options: OkeOptions): OkeApp {
       // HTTP flows serialize before `onResponse` so the last stage can see
       // and replace the final response (plugin header/middleware surfaces).
       trigger.kind === "http" ? encodeExecuteResult : undefined,
-    );
+    ).catch((err: unknown) => {
+      // Park suspensions are already absorbed above; a real pipeline failure
+      // still leaves the run lease to expire for cross-process reclaim, but
+      // must not pin the runId in this process's in-flight guard forever.
+      if (journalSession) inflightRuns.delete(journalSession.runId);
+      throw err;
+    });
 
     const endedAt = now();
 
     if (journalSession) {
+      inflightRuns.delete(journalSession.runId);
       const sleeping = ctx.state.sleeping as
         | { readonly wakeAt: number; readonly label: string; readonly runId: string }
         | undefined;
-      if (sleeping) {
+      // Lease-capable stores make the shared row the wake schedule; the
+      // in-process map is only for custom stores without the lease surface.
+      if (sleeping && !hasJournalLease(activeJournal().store)) {
         sleepingRuns.set(sleeping.runId, {
           flow: flowDef,
           input: ctx.input,
           wakeAt: sleeping.wakeAt,
         });
-      } else if (result.failure) {
+      } else if (!sleeping && result.failure) {
         await journalSession.commit("failed", {
           error: result.failure.error.code,
         });
-      } else {
+      } else if (!sleeping) {
         await journalSession.commit("completed", { output: result.output });
       }
     }
@@ -1073,6 +1210,7 @@ export function oke(options: OkeOptions): OkeApp {
         channel: bootResult.channel,
         ai: bootResult.ai,
         runs: bootResult.runs,
+        ...(bootResult.journal ? { journal: bootResult.journal } : {}),
       };
     },
     get capabilities() {
@@ -1098,22 +1236,36 @@ export function oke(options: OkeOptions): OkeApp {
     },
     async resumeDurable(now) {
       const t = now ?? bootResult?.clock?.now() ?? options.fx?.now?.() ?? Date.now();
+      const { store, instanceId, leaseMs } = activeJournal();
+      if (hasJournalLease(store)) {
+        // Shared store is the wake schedule — claim due sleeps across all
+        // instances (SKIP LOCKED; exactly one claimant wins a raced claim).
+        for (;;) {
+          const due = await store.claimDueSleep(instanceId, t, leaseMs);
+          if (!due) break;
+          sleepingRuns.delete(due.id);
+          await resumeDurableRun(due.id, due.flow, due.input, () => t);
+        }
+        // Crash sweep: the boot orphan scan is once-only, so each tick also
+        // reclaims `running` runs whose holder's lease has expired (same
+        // takeover physics as Clock: lease expiry + next tick).
+        for (const orphan of await store.listOrphans(t)) {
+          if (orphan.status !== "running") continue;
+          await resumeDurableRun(orphan.id, orphan.flow, orphan.input, () => t);
+        }
+        return;
+      }
+      // Custom store without the lease surface — in-process sleepers only.
       for (const [runId, sleeper] of [...sleepingRuns.entries()]) {
         if (t < sleeper.wakeAt) continue;
         sleepingRuns.delete(runId);
         const result = await runDurable({
           flow: sleeper.flow,
           input: sleeper.input,
-          journalStore,
+          journalStore: store,
           runId,
           now: () => t,
-          fx: {
-            storeRuntime: bootResult?.store,
-            signalRuntime: bootResult?.signal,
-            vaultRuntime: bootResult?.vault,
-            channelRuntime: bootResult?.channel,
-            aiRuntime: bootResult?.ai,
-          },
+          fx: durableResumeFx(),
         });
         if (result.status === "sleeping") {
           sleepingRuns.set(runId, {
@@ -1180,7 +1332,7 @@ export function oke(options: OkeOptions): OkeApp {
       const url = new URL(request.url);
       const method = request.method.toUpperCase();
 
-      const respond = (response: Response): Response => {
+      const respond = async (response: Response): Promise<Response> => {
         if (shouldLogDevRequests()) {
           logDevRequest({
             surface: currentDevSurface(),
@@ -1189,6 +1341,7 @@ export function oke(options: OkeOptions): OkeApp {
             flow: flowLabel,
             status: response.status,
             ms: Math.round(performance.now() - started),
+            detail: await failureDetailFromResponse(response),
           });
         }
         return response;

package/src/kernel/boot-bind/channel.test.ts

@@ -10,6 +10,7 @@ import {
   resolveSmsDriverId,
   smtpOptionsFromEnv,
   sndrOptionsFromEnv,
+  taqnyatMailOptionsFromEnv,
   taqnyatOptionsFromEnv,
   unifonicOptionsFromEnv,
 } from "./channel.ts";
@@ -24,6 +25,8 @@ const previous = {
   sndrBase: process.env.SNDR_BASE_URL,
   taqBearer: process.env.TAQNYAT_BEARER_TOKEN,
   taqSender: process.env.TAQNYAT_SENDER,
+  taqMailToken: process.env.TAQNYAT_MAIL_TOKEN,
+  taqCampaign: process.env.TAQNYAT_CAMPAIGN,
   msegatUser: process.env.MSEGAT_USERNAME,
   msegatKey: process.env.MSEGAT_API_KEY,
   msegatSender: process.env.MSEGAT_SENDER,
@@ -41,6 +44,8 @@ afterEach(() => {
   restoreEnv("SNDR_BASE_URL", previous.sndrBase);
   restoreEnv("TAQNYAT_BEARER_TOKEN", previous.taqBearer);
   restoreEnv("TAQNYAT_SENDER", previous.taqSender);
+  restoreEnv("TAQNYAT_MAIL_TOKEN", previous.taqMailToken);
+  restoreEnv("TAQNYAT_CAMPAIGN", previous.taqCampaign);
   restoreEnv("MSEGAT_USERNAME", previous.msegatUser);
   restoreEnv("MSEGAT_API_KEY", previous.msegatKey);
   restoreEnv("MSEGAT_SENDER", previous.msegatSender);
@@ -117,6 +122,17 @@ describe("bindChannel driver resolution", () => {
     process.env.UNIFONIC_SENDER = "Brand";
     expect(unifonicOptionsFromEnv()).toEqual({ appSid: "sid", sender: "Brand" });
   });
+
+  test("taqnyat-mail env helper", () => {
+    process.env.TAQNYAT_MAIL_TOKEN = "bearer-mail";
+    process.env.TAQNYAT_CAMPAIGN = "auth";
+    expect(taqnyatMailOptionsFromEnv()).toEqual({
+      bearerToken: "bearer-mail",
+      campaignName: "auth",
+    });
+    delete process.env.TAQNYAT_CAMPAIGN;
+    expect(() => taqnyatMailOptionsFromEnv()).toThrow("TAQNYAT_CAMPAIGN");
+  });
 });
 
 function restoreEnv(key: string, value: string | undefined): void {

package/src/kernel/boot-bind/channel.ts

@@ -9,6 +9,7 @@ import { openResendChannel } from "../../drivers/channel-resend.ts";
 import { openSmtpChannel } from "../../drivers/channel-smtp.ts";
 import { openSndrChannel } from "../../drivers/channel-sndr.ts";
 import { openTaqnyatChannel } from "../../drivers/channel-taqnyat.ts";
+import { openTaqnyatMailChannel } from "../../drivers/channel-taqnyat-mail.ts";
 import { openUnifonicChannel } from "../../drivers/channel-unifonic.ts";
 import type { ChannelDriver, ChannelOpenOptions } from "../../drivers/channel-types.ts";
 import { createChannelRuntime, type ChannelRuntime } from "../../elements/channel.ts";
@@ -51,6 +52,7 @@ function emailDriverFor(options: BootOptions, env: ConfigEnv, docker: boolean):
   if (id === "smtp") return openSmtpChannel(smtpOptionsFromEnv(docker));
   if (id === "resend") return openResendChannel(resendOptionsFromEnv());
   if (id === "sndr") return openSndrChannel(sndrOptionsFromEnv());
+  if (id === "taqnyat-mail") return openTaqnyatMailChannel(taqnyatMailOptionsFromEnv());
   throw new Error(`oke boot: unknown email channel driver "${id}"`);
 }
 
@@ -150,6 +152,17 @@ export function taqnyatOptionsFromEnv(): ChannelOpenOptions {
   return { bearerToken, sender };
 }
 
+/**
+ * Resolve Taqnyat Email options from env.
+ */
+export function taqnyatMailOptionsFromEnv(): ChannelOpenOptions {
+  const bearerToken = process.env.TAQNYAT_MAIL_TOKEN?.trim();
+  const campaignName = process.env.TAQNYAT_CAMPAIGN?.trim();
+  if (!bearerToken) throw new Error("oke boot: taqnyat-mail channel needs TAQNYAT_MAIL_TOKEN");
+  if (!campaignName) throw new Error("oke boot: taqnyat-mail channel needs TAQNYAT_CAMPAIGN");
+  return { bearerToken, campaignName };
+}
+
 /**
  * Resolve Msegat SMS options from env.
  */

package/src/kernel/boot-bind/clock.ts

@@ -2,13 +2,18 @@
  * Lazy clock binder — loaded only when Clock is declared.
  */
 
+import { mkdirSync } from "node:fs";
+import { dirname, resolve } from "node:path";
 import {
   clock as declareClock,
   createClockRuntime,
+  createFileCronStore,
+  createMemoryCronStore,
   createTestClockRuntime,
   type ClockDecl,
   type ClockRuntime,
 } from "../../elements/clock.ts";
+import { createPostgresCronStore } from "../../drivers/clock-postgres.ts";
 import { resolveDriverId, type ConfigEnv } from "../../config/index.ts";
 import type { BootOptions } from "../boot.ts";
 
@@ -18,9 +23,26 @@ export interface BindClockResult {
   readonly clockDecls: ReadonlyMap<string, ClockDecl>;
 }
 
+/** Default on-disk cron store path (multi-process leader election). */
+export const DEFAULT_FILE_CRON_PATH = ".oke/crons.json";
+
+/**
+ * Resolve `drivers.clock` for the active env.
+ *
+ * @param options - Boot options
+ * @param env - Active environment
+ */
+export function resolveClockDriverId(options: BootOptions, env: ConfigEnv): string {
+  const resolved = resolveDriverId(options.config?.drivers?.clock, env);
+  if (resolved) return resolved;
+  return env === "test" ? "frozen" : "memory";
+}
+
 /**
  * Construct / adopt a Clock runtime, register decls, reconcile.
  *
+ * Supported ids: `frozen` · `memory` · `file` · `postgres`.
+ *
  * @param options - Boot options
  * @param env - Active environment
  * @param now - Clock
@@ -32,13 +54,47 @@ export async function bindClock(
   now: () => number,
   prebuilt?: ClockRuntime,
 ): Promise<BindClockResult> {
-  const clockDriver =
-    resolveDriverId(options.config?.drivers?.clock, env) ?? (env === "test" ? "frozen" : "memory");
-  const clock =
-    prebuilt ??
-    (clockDriver === "frozen" || env === "test"
-      ? createTestClockRuntime(now(), { instanceId: options.instanceId })
-      : createClockRuntime({ instanceId: options.instanceId, now }));
+  const clockDriver = resolveClockDriverId(options, env);
+
+  let clock: ClockRuntime;
+  if (prebuilt) {
+    clock = prebuilt;
+  } else if (clockDriver === "frozen" || env === "test") {
+    clock = createTestClockRuntime(now(), { instanceId: options.instanceId });
+  } else if (clockDriver === "memory") {
+    clock = createClockRuntime({
+      instanceId: options.instanceId,
+      now,
+      store: createMemoryCronStore(),
+    });
+  } else if (clockDriver === "file") {
+    const path = resolve(process.cwd(), DEFAULT_FILE_CRON_PATH);
+    mkdirSync(dirname(path), { recursive: true });
+    clock = createClockRuntime({
+      instanceId: options.instanceId,
+      now,
+      store: createFileCronStore(path),
+    });
+  } else if (clockDriver === "postgres") {
+    const url = process.env.DATABASE_URL ?? process.env.OKE_STORE_SQL_URL ?? undefined;
+    if (!url) {
+      throw new Error(
+        env === "docker"
+          ? 'oke boot: clock driver "postgres" needs DATABASE_URL (did `oke dev -d` write docker/.env.docker?)'
+          : 'oke boot: clock driver "postgres" needs DATABASE_URL',
+      );
+    }
+    const store = await createPostgresCronStore({ url });
+    clock = createClockRuntime({
+      instanceId: options.instanceId,
+      now,
+      store,
+    });
+  } else {
+    throw new Error(
+      `oke boot: unknown clock driver "${clockDriver}" (expected memory · file · postgres · frozen)`,
+    );
+  }
 
   const clockDecls = new Map<string, ClockDecl>();
   for (const c of options.clocks ?? []) {

package/src/kernel/boot-bind/gate.ts

@@ -2,18 +2,86 @@
  * Lazy gate binder — loaded only when Gate is declared (or required by AI).
  */
 
+import { resolveDriverId, type ConfigEnv } from "../../config/index.ts";
 import { memoryDrivers } from "../../drivers/memory.ts";
+import { redisDriver } from "../../drivers/redis.ts";
+import type { KvClientLike } from "../../drivers/types.ts";
 import { createGateRuntime, type GateRuntime } from "../../elements/gate.ts";
 import type { BootOptions } from "../boot.ts";
 
+/** Namespace for gate rate-limit counters (never shares an app kv declaration). */
+export const GATE_KV_NAMESPACE = "oke:gates";
+
 /**
- * Construct a Gate runtime backed by a memory kv namespace.
+ * Resolve the KV backend for Gate rates — same facet as `drivers.store.kv`.
+ *
+ * @param options - Boot options
+ * @param env - Active environment
+ * @param docker - Docker mode
+ */
+export function resolveGateKvDriverId(
+  options: BootOptions,
+  env: ConfigEnv,
+  docker = false,
+): "memory" | "redis" {
+  const fromEnv = process.env.OKE_KV_DRIVER?.trim();
+  if (docker && fromEnv === "redis") return "redis";
+  if (docker && fromEnv === "memory") return "memory";
+  const resolved = resolveDriverId(options.config?.drivers?.store?.kv, env);
+  if (resolved === "redis") return "redis";
+  if (resolved === "memory") return "memory";
+  return docker ? "redis" : "memory";
+}
+
+function kvUrlFor(docker: boolean): string | undefined {
+  const url = process.env.REDIS_URL ?? process.env.OKE_STORE_KV_URL ?? undefined;
+  if (!url) {
+    if (docker) {
+      throw new Error(
+        "oke boot: gate redis kv needs REDIS_URL (did `oke dev -d` write docker/.env.docker?)",
+      );
+    }
+    return undefined;
+  }
+  return url;
+}
+
+/**
+ * Construct a Gate runtime backed by `drivers.store.kv` (namespace {@link GATE_KV_NAMESPACE}).
  *
  * @param options - Boot options
  * @param now - Clock
+ * @param env - Active environment
+ * @param docker - Docker mode
  */
-export async function bindGate(options: BootOptions, now: () => number): Promise<GateRuntime> {
-  const kvNs = await memoryDrivers.kv.open({ name: "oke:gates" });
+export async function bindGate(
+  options: BootOptions,
+  now: () => number,
+  env: ConfigEnv = "local",
+  docker = false,
+): Promise<GateRuntime> {
+  const kvId = resolveGateKvDriverId(options, env, docker);
+  const injected = options.clients?.kv as KvClientLike | undefined;
+
+  let kvNs;
+  if (kvId === "redis") {
+    const url = injected ? undefined : kvUrlFor(docker);
+    if (!injected && url === undefined) {
+      // Local without REDIS_URL — soft-fallback to memory (same as missing redis for store would fail;
+      // gate rates should not brick local apps that only declared policy gates).
+      kvNs = await memoryDrivers.kv.open({ name: GATE_KV_NAMESPACE, nowMs: now });
+    } else {
+      kvNs = await redisDriver.open({
+        name: GATE_KV_NAMESPACE,
+        ...(url !== undefined ? { url } : {}),
+        ...(injected !== undefined ? { client: injected } : {}),
+        nowMs: now,
+      });
+    }
+  } else {
+    kvNs = await memoryDrivers.kv.open({ name: GATE_KV_NAMESPACE, nowMs: now });
+  }
+
   return createGateRuntime({
     gates: options.gates ?? [],
     kv: kvNs,