okengine 0.6.1 → 0.8.0

package/src/elements/channel.test.ts

@@ -10,6 +10,7 @@
 import { describe, expect, test } from "bun:test";
 import type { MailOptions, SendResult, Transport } from "sently";
 import { createChannelInbox, openConsoleChannel, type ChannelDriver } from "../drivers/index.ts";
+import type { SmsOtpTransport, SmsTransport } from "../drivers/channel-types.ts";
 import { channel, createChannelRuntime, createConsentStore, FallbackTransport } from "./channel.ts";
 
 /** Create a sently-compatible transport that always fails. */
@@ -207,3 +208,73 @@ describe("console driver", () => {
     expect(inbox.entries[0]!.text).toBe("code 1234");
   });
 });
+
+describe("provider-managed OTP (Taqnyat Verify)", () => {
+  function taqnyatSmsDriver(): ChannelDriver {
+    const otpTransport: SmsTransport & SmsOtpTransport = {
+      provider: "taqnyat-sms",
+      async send() {
+        return { messageId: "m", to: "", status: "sent", response: "" };
+      },
+      async sendOtp(opts) {
+        return {
+          requestId: opts.requestId,
+          to: opts.to,
+          code: 5,
+          response: "sent",
+          provider: "taqnyat-sms",
+        };
+      },
+      async verifyOtp(opts) {
+        return {
+          ok: true as const,
+          code: opts.code === "0000" ? 13 : 10,
+          message: "verified",
+          response: "ok",
+          provider: "taqnyat-sms",
+        };
+      },
+    };
+    return { id: "taqnyat", smsTransport: otpTransport };
+  }
+
+  test("sendOtp / verifyOtp dispatch to the taqnyat smsTransport", async () => {
+    const runtime = createChannelRuntime({ drivers: [taqnyatSmsDriver()] });
+    const sent = await runtime.sendOtp({ to: "+966500000000", requestId: "r1", lang: "en" });
+    expect(sent.code).toBe(5);
+    expect(sent.requestId).toBe("r1");
+    const verified = await runtime.verifyOtp({
+      to: "+966500000000",
+      requestId: "r1",
+      code: "6240",
+    });
+    expect(verified.ok).toBe(true);
+    expect(verified.code).toBe(10);
+  });
+
+  test("throws loudly when no SMS driver is bound", async () => {
+    const runtime = createChannelRuntime({});
+    await expect(runtime.sendOtp({ to: "+966500000000", requestId: "r1" })).rejects.toThrow(
+      "no SMS driver bound",
+    );
+  });
+
+  test("throws loudly naming the driver when SMS driver lacks OTP support", async () => {
+    const twilioLike: ChannelDriver = {
+      id: "msegat",
+      smsTransport: {
+        provider: "msegat",
+        async send() {
+          return { messageId: "m", to: "", status: "sent", response: "" };
+        },
+      },
+    };
+    const runtime = createChannelRuntime({ drivers: [twilioLike] });
+    await expect(runtime.sendOtp({ to: "+966500000000", requestId: "r1" })).rejects.toThrow(
+      /SMS driver "msegat" does not support provider-managed OTP/,
+    );
+    await expect(
+      runtime.verifyOtp({ to: "+966500000000", requestId: "r1", code: "1234" }),
+    ).rejects.toThrow(/does not support provider-managed OTP/);
+  });
+});

package/src/elements/clock/chaos-child.ts

@@ -0,0 +1,389 @@
+/**
+ * Clock chaos child — multi-process leader election / durable resume.
+ *
+ * Args:
+ *   tick-loop <storePath> <instanceId> <fireLogPath> <leaseMs> [every]
+ *   hold-lease <storePath> <instanceId> <markerPath> <leaseMs>
+ *   tick-loop-pg <databaseUrl> <instanceId> <fireLogPath> <leaseMs> [every]
+ *   hold-lease-pg <databaseUrl> <instanceId> <markerPath> <leaseMs>
+ *   durable-mid-step <journalPath> <markerPath> <runIdPath>
+ *   journal-pg-start <databaseUrl> <instanceId> <stepLogPath> <markerPath> <leaseMs> <blockMs>
+ *   journal-pg-resume <databaseUrl> <instanceId> <stepLogPath> <donePath> <leaseMs>
+ *   journal-pg-park <databaseUrl> <instanceId> <stepLogPath> <parkedPath> <wakeMs>
+ *   journal-pg-claim <databaseUrl> <instanceId> <stepLogPath> <leaseMs>
+ *
+ * Modes:
+ *   tick-loop / tick-loop-pg   — reconcile + tick until this instance fires once
+ *   hold-lease / hold-lease-pg — acquire lease, write marker, hang (parent SIGKILLs)
+ *   durable-mid-step           — journal step 1, write marker + runId, hang
+ *   journal-pg-start           — boot oke() app, start durable run over HTTP, hang (parent SIGKILLs)
+ *   journal-pg-resume          — boot oke() app; orphan sweep resumes the dead run; exit when done
+ *   journal-pg-park            — boot oke() app, park a durable sleep over HTTP, exit
+ *   journal-pg-claim           — boot oke() app; race claimDueSleep; exit 0 on own execution
+ */
+
+import { join } from "node:path";
+
+import { createPostgresCronStore } from "../../drivers/clock-postgres.ts";
+import { oke, type OkeApp } from "../../kernel/app.ts";
+import { flow, type AnyFlowDef } from "../../kernel/flow.ts";
+import { createFileJournalStore } from "../../kernel/journal.ts";
+import { on } from "../../kernel/on.ts";
+import { http } from "../../kernel/triggers.ts";
+import { clock } from "./declare.ts";
+import { runDurable } from "./durable.ts";
+import { createFileCronStore, type CronStore } from "./reconcile.ts";
+import { createClockRuntime } from "./runtime.ts";
+
+const mode = process.argv[2];
+
+if (!mode) {
+  console.error(
+    "usage: chaos-child <tick-loop|hold-lease|tick-loop-pg|hold-lease-pg|durable-mid-step> …",
+  );
+  process.exit(2);
+}
+
+const CRON_NAME = "chaos-job";
+
+async function runTickLoop(
+  store: CronStore,
+  instanceId: string,
+  fireLogPath: string,
+  leaseMs: number,
+  every: string,
+) {
+  const rt = createClockRuntime({
+    instanceId,
+    store,
+    leaseMs: Number.isFinite(leaseMs) ? leaseMs : 100,
+  });
+  rt.register(clock(CRON_NAME, { every }));
+  await rt.reconcile();
+  rt.onCron(CRON_NAME, async () => {
+    const line = `${JSON.stringify({ instanceId, firedAt: Date.now() })}\n`;
+    const prev = (await Bun.file(fireLogPath).exists()) ? await Bun.file(fireLogPath).text() : "";
+    await Bun.write(fireLogPath, prev + line);
+  });
+
+  const deadline = Date.now() + 15_000;
+  while (Date.now() < deadline) {
+    const { ran } = await rt.tick();
+    if (ran.includes(CRON_NAME)) process.exit(0);
+    await Bun.sleep(5);
+  }
+  console.error("tick-loop: timed out without firing");
+  process.exit(3);
+}
+
+async function runHoldLease(
+  store: CronStore,
+  instanceId: string,
+  markerPath: string,
+  leaseMs: number,
+) {
+  const rt = createClockRuntime({
+    instanceId,
+    store,
+    leaseMs: Number.isFinite(leaseMs) ? leaseMs : 100,
+  });
+  rt.register(clock(CRON_NAME, { every: "50ms" }));
+  await rt.reconcile();
+
+  rt.onCron(CRON_NAME, () => {});
+  const ok = await rt.runNow(CRON_NAME);
+  if (!ok) {
+    console.error("hold-lease: failed to acquire/fire");
+    process.exit(3);
+  }
+  const row = await store.get(CRON_NAME);
+  await Bun.write(
+    markerPath,
+    JSON.stringify({
+      instanceId,
+      heldAt: Date.now(),
+      leaderLeaseUntil: row?.leaderLeaseUntil,
+    }),
+  );
+  await Bun.sleep(60_000);
+  process.exit(0);
+}
+
+if (mode === "tick-loop") {
+  const storePath = process.argv[3];
+  const instanceId = process.argv[4];
+  const fireLogPath = process.argv[5];
+  const leaseMs = Number(process.argv[6] ?? 100);
+  const every = process.argv[7] ?? "1h";
+  if (!storePath || !instanceId || !fireLogPath) {
+    console.error(
+      "usage: chaos-child tick-loop <storePath> <instanceId> <fireLogPath> <leaseMs> [every]",
+    );
+    process.exit(2);
+  }
+  await runTickLoop(createFileCronStore(join(storePath)), instanceId, fireLogPath, leaseMs, every);
+} else if (mode === "hold-lease") {
+  const storePath = process.argv[3];
+  const instanceId = process.argv[4];
+  const markerPath = process.argv[5];
+  const leaseMs = Number(process.argv[6] ?? 100);
+  if (!storePath || !instanceId || !markerPath) {
+    console.error("usage: chaos-child hold-lease <storePath> <instanceId> <markerPath> <leaseMs>");
+    process.exit(2);
+  }
+  await runHoldLease(createFileCronStore(join(storePath)), instanceId, markerPath, leaseMs);
+} else if (mode === "tick-loop-pg") {
+  const databaseUrl = process.argv[3];
+  const instanceId = process.argv[4];
+  const fireLogPath = process.argv[5];
+  const leaseMs = Number(process.argv[6] ?? 100);
+  const every = process.argv[7] ?? "1h";
+  if (!databaseUrl || !instanceId || !fireLogPath) {
+    console.error(
+      "usage: chaos-child tick-loop-pg <databaseUrl> <instanceId> <fireLogPath> <leaseMs> [every]",
+    );
+    process.exit(2);
+  }
+  const store = await createPostgresCronStore({ url: databaseUrl });
+  await runTickLoop(store, instanceId, fireLogPath, leaseMs, every);
+} else if (mode === "hold-lease-pg") {
+  const databaseUrl = process.argv[3];
+  const instanceId = process.argv[4];
+  const markerPath = process.argv[5];
+  const leaseMs = Number(process.argv[6] ?? 100);
+  if (!databaseUrl || !instanceId || !markerPath) {
+    console.error(
+      "usage: chaos-child hold-lease-pg <databaseUrl> <instanceId> <markerPath> <leaseMs>",
+    );
+    process.exit(2);
+  }
+  const store = await createPostgresCronStore({ url: databaseUrl });
+  await runHoldLease(store, instanceId, markerPath, leaseMs);
+} else if (mode === "durable-mid-step") {
+  const journalPath = process.argv[3];
+  const markerPath = process.argv[4];
+  const runIdPath = process.argv[5];
+  if (!journalPath || !markerPath || !runIdPath) {
+    console.error("usage: chaos-child durable-mid-step <journalPath> <markerPath> <runIdPath>");
+    process.exit(2);
+  }
+
+  const journalStore = createFileJournalStore(join(journalPath));
+  const charge = flow({
+    name: "chaos.charge",
+    durable: true,
+    do: async (_input, fx) => {
+      const intent = await fx.step("create-intent", async () => {
+        const runs = await journalStore.list();
+        const run = runs[0];
+        if (run) await Bun.write(runIdPath, run.id);
+        await Bun.write(markerPath, "step1");
+        return { id: "pi_chaos" };
+      });
+      await Bun.sleep(60_000);
+      return fx.step("confirm", () => intent.id === "pi_chaos");
+    },
+  });
+
+  await runDurable({
+    flow: charge,
+    input: { orderId: "o-chaos" },
+    journalStore,
+  });
+  process.exit(0);
+} else if (mode === "journal-pg-start") {
+  const [databaseUrl, instanceId, stepLogPath, markerPath] = process.argv.slice(3);
+  const leaseMs = Number(process.argv[7] ?? 300);
+  const blockMs = Number(process.argv[8] ?? 60_000);
+  if (!databaseUrl || !instanceId || !stepLogPath || !markerPath) {
+    console.error(
+      "usage: chaos-child journal-pg-start <databaseUrl> <instanceId> <stepLogPath> <markerPath> <leaseMs> <blockMs>",
+    );
+    process.exit(2);
+  }
+  const app = await bootJournalPgApp({
+    url: databaseUrl!,
+    instanceId: instanceId!,
+    leaseMs,
+    flow: chargeFlow(instanceId!, stepLogPath!, blockMs),
+  });
+  void app.fetch(new Request("http://localhost/charge", { method: "POST", body: "{}" }));
+  // Marker once step 1 is journaled — then hang; the parent SIGKILLs mid-run.
+  const deadline = Date.now() + 15_000;
+  for (;;) {
+    const row = (await app.bootResult!.journal!.store.list())[0];
+    if (row && row.entries.length >= 1) {
+      await Bun.write(markerPath!, JSON.stringify({ runId: row.id, instanceId }));
+      break;
+    }
+    if (Date.now() > deadline) {
+      console.error("journal-pg-start: step 1 never journaled");
+      process.exit(3);
+    }
+    await Bun.sleep(10);
+  }
+  await Bun.sleep(3_600_000);
+  process.exit(0);
+} else if (mode === "journal-pg-resume") {
+  const [databaseUrl, instanceId, stepLogPath, donePath] = process.argv.slice(3);
+  const leaseMs = Number(process.argv[7] ?? 300);
+  if (!databaseUrl || !instanceId || !stepLogPath || !donePath) {
+    console.error(
+      "usage: chaos-child journal-pg-resume <databaseUrl> <instanceId> <stepLogPath> <donePath> <leaseMs>",
+    );
+    process.exit(2);
+  }
+  // No fetch — the boot orphan scan / resume sweep must discover the dead run.
+  const app = await bootJournalPgApp({
+    url: databaseUrl!,
+    instanceId: instanceId!,
+    leaseMs,
+    flow: chargeFlow(instanceId!, stepLogPath!, 0),
+  });
+  const deadline = Date.now() + 20_000;
+  for (;;) {
+    await app.resumeDurable();
+    const row = (await app.bootResult!.journal!.store.list())[0];
+    if (row?.status === "completed") {
+      await Bun.write(donePath!, JSON.stringify({ runId: row.id, completedBy: instanceId }));
+      process.exit(0);
+    }
+    if (Date.now() > deadline) {
+      console.error("journal-pg-resume: orphan never resumed");
+      process.exit(3);
+    }
+    await Bun.sleep(25);
+  }
+} else if (mode === "journal-pg-park") {
+  const [databaseUrl, instanceId, stepLogPath, parkedPath] = process.argv.slice(3);
+  const wakeMs = Number(process.argv[7] ?? 400);
+  if (!databaseUrl || !instanceId || !stepLogPath || !parkedPath) {
+    console.error(
+      "usage: chaos-child journal-pg-park <databaseUrl> <instanceId> <stepLogPath> <parkedPath> <wakeMs>",
+    );
+    process.exit(2);
+  }
+  const app = await bootJournalPgApp({
+    url: databaseUrl!,
+    instanceId: instanceId!,
+    leaseMs: 30_000,
+    flow: sleeperFlow(instanceId!, stepLogPath!, wakeMs),
+  });
+  const res = await app.fetch(
+    new Request("http://localhost/sleep", { method: "POST", body: "{}" }),
+  );
+  if (res.status !== 204) {
+    console.error(`journal-pg-park: expected 204 park, got ${res.status}`);
+    process.exit(3);
+  }
+  const row = (await app.bootResult!.journal!.store.list())[0];
+  await Bun.write(parkedPath!, JSON.stringify({ runId: row?.id, instanceId }));
+  process.exit(0);
+} else if (mode === "journal-pg-claim") {
+  const [databaseUrl, instanceId, stepLogPath] = process.argv.slice(3);
+  const leaseMs = Number(process.argv[6] ?? 300);
+  if (!databaseUrl || !instanceId || !stepLogPath) {
+    console.error(
+      "usage: chaos-child journal-pg-claim <databaseUrl> <instanceId> <stepLogPath> <leaseMs>",
+    );
+    process.exit(2);
+  }
+  const app = await bootJournalPgApp({
+    url: databaseUrl!,
+    instanceId: instanceId!,
+    leaseMs,
+    flow: sleeperFlow(instanceId!, stepLogPath!, 400),
+  });
+  const deadline = Date.now() + 15_000;
+  for (;;) {
+    await app.resumeDurable();
+    if (await logHasStep(stepLogPath!, "confirm", instanceId!)) process.exit(0);
+    const row = (await app.bootResult!.journal!.store.list())[0];
+    if (row?.status === "completed") process.exit(3); // someone else claimed it
+    if (Date.now() > deadline) process.exit(3);
+    await Bun.sleep(25);
+  }
+} else {
+  console.error(`unknown mode: ${mode}`);
+  process.exit(2);
+}
+
+/** Append one JSON line to the shared step log. */
+async function appendStepLog(
+  path: string,
+  entry: { instanceId: string; step: string; at: number },
+): Promise<void> {
+  const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
+  await Bun.write(path, `${prev}${JSON.stringify(entry)}\n`);
+}
+
+/** True when the step log holds `step` executed by `instanceId`. */
+async function logHasStep(path: string, step: string, instanceId: string): Promise<boolean> {
+  if (!(await Bun.file(path).exists())) return false;
+  const lines = (await Bun.file(path).text()).trim().split("\n").filter(Boolean);
+  return lines.some((l) => {
+    const e = JSON.parse(l) as { instanceId: string; step: string };
+    return e.step === step && e.instanceId === instanceId;
+  });
+}
+
+/** Durable charge flow: step 1 → block → step 2 (both logged). */
+function chargeFlow(instanceId: string, stepLogPath: string, blockMs: number): AnyFlowDef {
+  return flow({
+    name: "chaos.journal.charge",
+    durable: true,
+    do: async (_input, fx) => {
+      const intent = await fx.step("create-intent", async () => {
+        await appendStepLog(stepLogPath, { instanceId, step: "create-intent", at: Date.now() });
+        return { id: "pi_chaos" };
+      });
+      if (blockMs > 0) await Bun.sleep(blockMs);
+      await fx.step("mid-flight", async () => {
+        await appendStepLog(stepLogPath, { instanceId, step: "mid-flight", at: Date.now() });
+        return intent.id === "pi_chaos";
+      });
+      return { ok: true };
+    },
+  });
+}
+
+/** Durable sleeper flow: step 1 → sleep → confirm (steps logged). */
+function sleeperFlow(instanceId: string, stepLogPath: string, wakeMs: number): AnyFlowDef {
+  return flow({
+    name: "chaos.journal.sleeper",
+    durable: true,
+    do: async (_input, fx) => {
+      await fx.step("create-intent", async () => {
+        await appendStepLog(stepLogPath, { instanceId, step: "create-intent", at: Date.now() });
+        return { id: "pi_chaos" };
+      });
+      await fx.clock.sleep("verify-window", `${wakeMs}ms`);
+      await fx.step("confirm", async () => {
+        await appendStepLog(stepLogPath, { instanceId, step: "confirm", at: Date.now() });
+        return true;
+      });
+      return { ok: true };
+    },
+  });
+}
+
+/** Boot a real `oke()` app against live postgres (drivers.journal). */
+async function bootJournalPgApp(options: {
+  readonly url: string;
+  readonly instanceId: string;
+  readonly leaseMs: number;
+  readonly flow: AnyFlowDef;
+}): Promise<OkeApp> {
+  process.env.DATABASE_URL = options.url;
+  on(http.post("/charge"), options.flow);
+  on(http.post("/sleep"), options.flow);
+  const app = oke({
+    name: `chaos-journal-${options.instanceId}`,
+    env: "test",
+    startScheduler: false,
+    gate: { unguardedHttp: "allow" },
+    config: { drivers: { journal: { test: "postgres" } } },
+  });
+  await app.boot({ instanceId: options.instanceId, journalLeaseMs: options.leaseMs });
+  return app;
+}

package/src/elements/clock/chaos.test.ts

@@ -0,0 +1,204 @@
+/**
+ * Clock chaos — multi-process leader election + SIGKILL durable resume.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { flow } from "../../kernel/flow.ts";
+import { createFileJournalStore } from "../../kernel/journal.ts";
+import { runDurable } from "./durable.ts";
+
+const childPath = join(import.meta.dir, "chaos-child.ts");
+
+async function waitForFile(path: string, timeoutMs = 10_000): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (await Bun.file(path).exists()) return true;
+    await Bun.sleep(10);
+  }
+  return false;
+}
+
+describe("chaos — multi-process leader election", () => {
+  test("two OS processes sharing a file CronStore fire exactly once", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "oke-clock-leader-"));
+    const storePath = join(dir, "crons.json");
+    const fireLogPath = join(dir, "fires.jsonl");
+    const leaseMs = 200;
+    try {
+      const a = Bun.spawn({
+        cmd: [
+          "bun",
+          childPath,
+          "tick-loop",
+          storePath,
+          "inst-a",
+          fireLogPath,
+          String(leaseMs),
+          "1h",
+        ],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      const b = Bun.spawn({
+        cmd: [
+          "bun",
+          childPath,
+          "tick-loop",
+          storePath,
+          "inst-b",
+          fireLogPath,
+          String(leaseMs),
+          "1h",
+        ],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+
+      // Stop as soon as exactly one fire is observed; kill stragglers.
+      const deadline = Date.now() + 10_000;
+      while (Date.now() < deadline) {
+        if (await Bun.file(fireLogPath).exists()) {
+          const text = await Bun.file(fireLogPath).text();
+          if (text.trim().split("\n").filter(Boolean).length >= 1) break;
+        }
+        await Bun.sleep(10);
+      }
+      a.kill(9);
+      b.kill(9);
+      await Promise.all([a.exited, b.exited]);
+
+      expect(await Bun.file(fireLogPath).exists()).toBe(true);
+      const lines = (await Bun.file(fireLogPath).text()).trim().split("\n").filter(Boolean);
+      expect(lines).toHaveLength(1);
+      const fire = JSON.parse(lines[0]!) as { instanceId: string };
+      expect(["inst-a", "inst-b"]).toContain(fire.instanceId);
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+
+  test("SIGKILL leader mid-lease: survivor takes over; report real latency", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "oke-clock-takeover-"));
+    const storePath = join(dir, "crons.json");
+    const markerPath = join(dir, "held.json");
+    const fireLogPath = join(dir, "fires.jsonl");
+    const leaseMs = 120;
+    const tickSlackMs = 250;
+    try {
+      const leader = Bun.spawn({
+        cmd: ["bun", childPath, "hold-lease", storePath, "leader", markerPath, String(leaseMs)],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+
+      expect(await waitForFile(markerPath)).toBe(true);
+      const held = (await Bun.file(markerPath).json()) as {
+        leaderLeaseUntil?: number;
+        heldAt: number;
+      };
+
+      const killAt = Date.now();
+      leader.kill(9);
+      await leader.exited;
+
+      // every=50ms so the schedule is due again once the dead leader's lease expires.
+      const survivor = Bun.spawn({
+        cmd: [
+          "bun",
+          childPath,
+          "tick-loop",
+          storePath,
+          "survivor",
+          fireLogPath,
+          String(leaseMs),
+          "50ms",
+        ],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      expect(await survivor.exited).toBe(0);
+
+      expect(await Bun.file(fireLogPath).exists()).toBe(true);
+      const lines = (await Bun.file(fireLogPath).text()).trim().split("\n").filter(Boolean);
+      expect(lines).toHaveLength(1);
+      const fire = JSON.parse(lines[0]!) as { instanceId: string; firedAt: number };
+      expect(fire.instanceId).toBe("survivor");
+
+      const takeoverMs = fire.firedAt - killAt;
+      // Physics: cannot take over before the leader's lease expires.
+      const leaseRemainingAtKill = Math.max(
+        0,
+        (held.leaderLeaseUntil ?? killAt + leaseMs) - killAt,
+      );
+      expect(takeoverMs).toBeGreaterThanOrEqual(Math.max(0, leaseRemainingAtKill - 20));
+      // Upper bound: lease TTL + tick/poll slack (not a product SLA).
+      expect(takeoverMs).toBeLessThanOrEqual(leaseMs + tickSlackMs);
+
+      // Evidence for docs / later SLA choice — real measured latency.
+      console.log(
+        `[clock takeover] measured=${takeoverMs}ms leaseMs=${leaseMs} leaseRemainingAtKill=${leaseRemainingAtKill}ms`,
+      );
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});
+
+describe("chaos — SIGKILL durable mid-step", () => {
+  test("SIGKILL after step 1: resume skips completed step", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "oke-clock-durable-kill-"));
+    const journalPath = join(dir, "journal.json");
+    const markerPath = join(dir, "step1");
+    const runIdPath = join(dir, "runId");
+    try {
+      const child = Bun.spawn({
+        cmd: ["bun", childPath, "durable-mid-step", journalPath, markerPath, runIdPath],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+
+      expect(await waitForFile(markerPath)).toBe(true);
+      expect(await waitForFile(runIdPath)).toBe(true);
+      // Give the journal a moment to flush step 1 after the marker write.
+      await Bun.sleep(50);
+      child.kill(9);
+      await child.exited;
+
+      const runId = (await Bun.file(runIdPath).text()).trim();
+      expect(runId.length).toBeGreaterThan(0);
+
+      const calls: string[] = [];
+      const journalStore = createFileJournalStore(journalPath);
+      const charge = flow({
+        name: "chaos.charge",
+        durable: true,
+        do: async (_input, fx) => {
+          const intent = await fx.step("create-intent", () => {
+            calls.push("create-intent");
+            return { id: "pi_chaos" };
+          });
+          return fx.step("confirm", () => {
+            calls.push("confirm");
+            return intent.id === "pi_chaos";
+          });
+        },
+      });
+
+      const resumed = await runDurable({
+        flow: charge,
+        input: { orderId: "o-chaos" },
+        journalStore,
+        runId,
+      });
+      expect(resumed.status).toBe("completed");
+      // create-intent must not re-run; confirm runs once.
+      expect(calls).toEqual(["confirm"]);
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});

package/src/elements/clock/durable.ts

@@ -13,6 +13,7 @@ import {
   createJournal,
   isJournalSuspend,
   type Journal,
+  type JournalLeaseOptions,
   type JournalSession,
   type JournalStore,
 } from "../../kernel/journal.ts";
@@ -45,6 +46,11 @@ export interface RunDurableOptions {
   readonly journalStore: JournalStore;
   /** Resume an existing run id (crash recovery). */
   readonly runId?: string;
+  /**
+   * Run-level lease (SKIP LOCKED + lazy reclaim when the store supports it).
+   * Resume throws `JournalLeaseBusy` when another live instance holds the run.
+   */
+  readonly lease?: JournalLeaseOptions;
   /** Injectable clock. */
   readonly now?: () => number;
   /** Extra fx options (secrets, store runtime, …). */
@@ -63,6 +69,7 @@ export async function runDurable<O = unknown>(
   const journal: Journal = createJournal({
     store: options.journalStore,
     now,
+    ...(options.lease ? { lease: options.lease } : {}),
   });
 
   if (options.runId) {