okengine 0.6.0 → 0.7.0

package/src/elements/channel/runtime.ts

@@ -246,10 +246,104 @@ export function createChannelRuntime(options: CreateChannelRuntimeOptions = {}):
     }
   }
 
+  async function sendViaSmsFallback(
+    chain: ChannelDriver[],
+    message: ChannelMessage,
+  ): Promise<{ result: ChannelSendResult; attempts: ChannelAttempt[] } | undefined> {
+    const sms = chain
+      .map((d) => (d.smsTransport ? { driver: d, transport: d.smsTransport } : undefined))
+      .filter(
+        (
+          x,
+        ): x is { driver: ChannelDriver; transport: NonNullable<ChannelDriver["smsTransport"]> } =>
+          !!x,
+      );
+    if (sms.length === 0) return undefined;
+
+    const attempts: ChannelAttempt[] = [];
+    const transports = sms.map(({ transport }) =>
+      options.retry ? new RetryTransport(transport) : transport,
+    );
+    const fallback = new FallbackTransport(transports, {
+      onFallback(failedIndex, error) {
+        const provider =
+          transports[failedIndex]?.provider ?? sms[failedIndex]?.driver.id ?? `sms-${failedIndex}`;
+        attempts.push({
+          driverId: provider,
+          ok: false,
+          error: error instanceof Error ? error.message : String(error),
+          at: now(),
+        });
+      },
+    });
+
+    const body = {
+      to: message.to,
+      body: message.text ?? String(message.data?.code ?? ""),
+      ...(message.from ? { from: message.from } : {}),
+    };
+
+    try {
+      const sendResult = await fallback.send(body);
+      const driverId =
+        sendResult.provider ??
+        transports[sendResult.providerIndex ?? 0]?.provider ??
+        sms[0]?.driver.id ??
+        "sms";
+      attempts.push({
+        driverId,
+        ok: true,
+        at: now(),
+        messageId: sendResult.messageId,
+      });
+      return {
+        result: {
+          ok: true,
+          messageId: sendResult.messageId,
+          driverId,
+          attempts,
+        },
+        attempts,
+      };
+    } catch (err) {
+      const fbAttempts =
+        err &&
+        typeof err === "object" &&
+        "attempts" in err &&
+        Array.isArray((err as { attempts: FallbackAttempt[] }).attempts)
+          ? (err as { attempts: FallbackAttempt[] }).attempts
+          : [];
+      for (const a of fbAttempts) {
+        if (!attempts.some((x) => x.driverId === a.provider && !x.ok)) {
+          attempts.push({
+            driverId: a.provider,
+            ok: false,
+            error: a.error instanceof Error ? a.error.message : String(a.error),
+            at: now(),
+          });
+        }
+      }
+      return {
+        result: {
+          ok: false,
+          messageId: crypto.randomUUID(),
+          driverId: "fallback",
+          attempts,
+        },
+        attempts,
+      };
+    }
+  }
+
   async function sendViaChannelChain(
     chain: ChannelDriver[],
     message: ChannelMessage,
   ): Promise<{ result: ChannelSendResult; attempts: ChannelAttempt[] }> {
+    if (message.medium === "sms") {
+      const viaSms = await sendViaSmsFallback(chain, message);
+      if (viaSms) return viaSms;
+    }
+
     const attempts: ChannelAttempt[] = [];
     for (const d of chain) {
       if (!d.channel) continue;

package/src/elements/channel/sndr-webhooks.test.ts

@@ -0,0 +1,26 @@
+/**
+ * SNDR webhook helpers re-exported from sently.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { parseSndrWebhook } from "./mime.ts";
+
+describe("parseSndrWebhook", () => {
+  test("normalizes a delivery event payload", () => {
+    const events = parseSndrWebhook({
+      type: "email.delivered",
+      data: {
+        email_id: "em_test",
+        to: ["user@example.com"],
+      },
+    });
+    expect(events).toEqual([
+      expect.objectContaining({
+        provider: "sndr",
+        type: "delivered",
+        messageId: "em_test",
+        recipient: "user@example.com",
+      }),
+    ]);
+  });
+});

package/src/elements/channel.ts

@@ -2,7 +2,7 @@
  * Channel element — reaching humans.
  *
  * Physics: email · SMS · WhatsApp · push.
- * Drivers: `console` · `smtp` · `resend` · `unifonic` · `wa-cloud` · `fcm` · `webpush`.
+ * Drivers: `console` · `smtp` · `resend` · `sndr` · `taqnyat` · `msegat` · `unifonic` · `wa-cloud` · `fcm` · `webpush`.
  *
  * Transport interface is identical to sently's so its transports run unchanged.
  * MIME, attachments, retry, and the unified error hierarchy come from sently.
@@ -91,9 +91,18 @@ export {
   RetryTransport,
   FallbackTransport,
   FallbackError,
+  toChannelSendResult,
+  toDeliveryEvent,
+  parseSndrWebhook,
+  verifySndrSignature,
+  parseUnifonicWebhook,
   type Attachment,
   type MailOptions,
   type SendResult,
   type Transport,
   type FallbackAttempt,
+  type EmailEvent,
+  type DeliveryEvent,
+  type AnySendResult,
+  type SentlyChannelSendResult,
 } from "./channel/mime.ts";

package/src/elements/clock/chaos-child.ts

@@ -0,0 +1,150 @@
+/**
+ * Clock chaos child — multi-process leader election / durable resume.
+ *
+ * Args:
+ *   tick-loop <storePath> <instanceId> <fireLogPath> <leaseMs> [every]
+ *   hold-lease <storePath> <instanceId> <markerPath> <leaseMs>
+ *   durable-mid-step <journalPath> <markerPath> <runIdPath>
+ *
+ * Modes:
+ *   tick-loop         — reconcile + tick until this instance fires once, append fire log
+ *   hold-lease        — acquire lease, write marker, hang (parent SIGKILLs)
+ *   durable-mid-step  — journal step 1, write marker + runId, hang (parent SIGKILLs)
+ */
+
+import { join } from "node:path";
+
+import { flow } from "../../kernel/flow.ts";
+import { createFileJournalStore } from "../../kernel/journal.ts";
+import { clock } from "./declare.ts";
+import { runDurable } from "./durable.ts";
+import { createFileCronStore } from "./reconcile.ts";
+import { createClockRuntime } from "./runtime.ts";
+
+const mode = process.argv[2];
+
+if (!mode) {
+  console.error("usage: chaos-child <tick-loop|hold-lease|durable-mid-step> …");
+  process.exit(2);
+}
+
+const CRON_NAME = "chaos-job";
+
+if (mode === "tick-loop") {
+  const storePath = process.argv[3];
+  const instanceId = process.argv[4];
+  const fireLogPath = process.argv[5];
+  const leaseMs = Number(process.argv[6] ?? 100);
+  // Default 1h so a second process cannot become due again within the test window
+  // after the winner sets lastRunAt (proves one fire per tick, not one per lease).
+  const every = process.argv[7] ?? "1h";
+  if (!storePath || !instanceId || !fireLogPath) {
+    console.error(
+      "usage: chaos-child tick-loop <storePath> <instanceId> <fireLogPath> <leaseMs> [every]",
+    );
+    process.exit(2);
+  }
+
+  const store = createFileCronStore(join(storePath));
+  const rt = createClockRuntime({
+    instanceId,
+    store,
+    leaseMs: Number.isFinite(leaseMs) ? leaseMs : 100,
+  });
+  rt.register(clock(CRON_NAME, { every }));
+  await rt.reconcile();
+  rt.onCron(CRON_NAME, async () => {
+    const line = `${JSON.stringify({ instanceId, firedAt: Date.now() })}\n`;
+    const prev = (await Bun.file(fireLogPath).exists()) ? await Bun.file(fireLogPath).text() : "";
+    await Bun.write(fireLogPath, prev + line);
+  });
+
+  const deadline = Date.now() + 15_000;
+  while (Date.now() < deadline) {
+    const { ran } = await rt.tick();
+    if (ran.includes(CRON_NAME)) process.exit(0);
+    await Bun.sleep(5);
+  }
+  console.error("tick-loop: timed out without firing");
+  process.exit(3);
+}
+
+if (mode === "hold-lease") {
+  const storePath = process.argv[3];
+  const instanceId = process.argv[4];
+  const markerPath = process.argv[5];
+  const leaseMs = Number(process.argv[6] ?? 100);
+  if (!storePath || !instanceId || !markerPath) {
+    console.error("usage: chaos-child hold-lease <storePath> <instanceId> <markerPath> <leaseMs>");
+    process.exit(2);
+  }
+
+  const store = createFileCronStore(join(storePath));
+  const rt = createClockRuntime({
+    instanceId,
+    store,
+    leaseMs: Number.isFinite(leaseMs) ? leaseMs : 100,
+  });
+  rt.register(clock(CRON_NAME, { every: "50ms" }));
+  await rt.reconcile();
+
+  // Fire once so lastRunAt + lease are set (this instance is leader).
+  rt.onCron(CRON_NAME, () => {});
+  const ok = await rt.runNow(CRON_NAME);
+  if (!ok) {
+    console.error("hold-lease: failed to acquire/fire");
+    process.exit(3);
+  }
+  const row = await store.get(CRON_NAME);
+  await Bun.write(
+    markerPath,
+    JSON.stringify({
+      instanceId,
+      heldAt: Date.now(),
+      leaderLeaseUntil: row?.leaderLeaseUntil,
+    }),
+  );
+  // Hang while holding the lease — parent will SIGKILL.
+  await Bun.sleep(60_000);
+  process.exit(0);
+}
+
+if (mode === "durable-mid-step") {
+  const journalPath = process.argv[3];
+  const markerPath = process.argv[4];
+  const runIdPath = process.argv[5];
+  if (!journalPath || !markerPath || !runIdPath) {
+    console.error("usage: chaos-child durable-mid-step <journalPath> <markerPath> <runIdPath>");
+    process.exit(2);
+  }
+
+  const journalStore = createFileJournalStore(join(journalPath));
+  const charge = flow({
+    name: "chaos.charge",
+    durable: true,
+    do: async (_input, fx) => {
+      const intent = await fx.step("create-intent", async () => {
+        // Persist run id as soon as the step body runs (session already started).
+        const runs = await journalStore.list();
+        const run = runs[0];
+        if (run) await Bun.write(runIdPath, run.id);
+        await Bun.write(markerPath, "step1");
+        return { id: "pi_chaos" };
+      });
+      // Hang after journaling step 1 — parent SIGKILLs before confirm.
+      await Bun.sleep(60_000);
+      return fx.step("confirm", () => intent.id === "pi_chaos");
+    },
+  });
+
+  await runDurable({
+    flow: charge,
+    input: { orderId: "o-chaos" },
+    journalStore,
+  });
+  // Unreachable under SIGKILL.
+  process.exit(0);
+}
+
+console.error(`unknown mode: ${mode}`);
+process.exit(2);

package/src/elements/clock/chaos.test.ts

@@ -0,0 +1,204 @@
+/**
+ * Clock chaos — multi-process leader election + SIGKILL durable resume.
+ */
+
+import { describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { flow } from "../../kernel/flow.ts";
+import { createFileJournalStore } from "../../kernel/journal.ts";
+import { runDurable } from "./durable.ts";
+
+const childPath = join(import.meta.dir, "chaos-child.ts");
+
+async function waitForFile(path: string, timeoutMs = 10_000): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (await Bun.file(path).exists()) return true;
+    await Bun.sleep(10);
+  }
+  return false;
+}
+
+describe("chaos — multi-process leader election", () => {
+  test("two OS processes sharing a file CronStore fire exactly once", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "oke-clock-leader-"));
+    const storePath = join(dir, "crons.json");
+    const fireLogPath = join(dir, "fires.jsonl");
+    const leaseMs = 200;
+    try {
+      const a = Bun.spawn({
+        cmd: [
+          "bun",
+          childPath,
+          "tick-loop",
+          storePath,
+          "inst-a",
+          fireLogPath,
+          String(leaseMs),
+          "1h",
+        ],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      const b = Bun.spawn({
+        cmd: [
+          "bun",
+          childPath,
+          "tick-loop",
+          storePath,
+          "inst-b",
+          fireLogPath,
+          String(leaseMs),
+          "1h",
+        ],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+
+      // Stop as soon as exactly one fire is observed; kill stragglers.
+      const deadline = Date.now() + 10_000;
+      while (Date.now() < deadline) {
+        if (await Bun.file(fireLogPath).exists()) {
+          const text = await Bun.file(fireLogPath).text();
+          if (text.trim().split("\n").filter(Boolean).length >= 1) break;
+        }
+        await Bun.sleep(10);
+      }
+      a.kill(9);
+      b.kill(9);
+      await Promise.all([a.exited, b.exited]);
+
+      expect(await Bun.file(fireLogPath).exists()).toBe(true);
+      const lines = (await Bun.file(fireLogPath).text()).trim().split("\n").filter(Boolean);
+      expect(lines).toHaveLength(1);
+      const fire = JSON.parse(lines[0]!) as { instanceId: string };
+      expect(["inst-a", "inst-b"]).toContain(fire.instanceId);
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+
+  test("SIGKILL leader mid-lease: survivor takes over; report real latency", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "oke-clock-takeover-"));
+    const storePath = join(dir, "crons.json");
+    const markerPath = join(dir, "held.json");
+    const fireLogPath = join(dir, "fires.jsonl");
+    const leaseMs = 120;
+    const tickSlackMs = 250;
+    try {
+      const leader = Bun.spawn({
+        cmd: ["bun", childPath, "hold-lease", storePath, "leader", markerPath, String(leaseMs)],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+
+      expect(await waitForFile(markerPath)).toBe(true);
+      const held = (await Bun.file(markerPath).json()) as {
+        leaderLeaseUntil?: number;
+        heldAt: number;
+      };
+
+      const killAt = Date.now();
+      leader.kill(9);
+      await leader.exited;
+
+      // every=50ms so the schedule is due again once the dead leader's lease expires.
+      const survivor = Bun.spawn({
+        cmd: [
+          "bun",
+          childPath,
+          "tick-loop",
+          storePath,
+          "survivor",
+          fireLogPath,
+          String(leaseMs),
+          "50ms",
+        ],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      expect(await survivor.exited).toBe(0);
+
+      expect(await Bun.file(fireLogPath).exists()).toBe(true);
+      const lines = (await Bun.file(fireLogPath).text()).trim().split("\n").filter(Boolean);
+      expect(lines).toHaveLength(1);
+      const fire = JSON.parse(lines[0]!) as { instanceId: string; firedAt: number };
+      expect(fire.instanceId).toBe("survivor");
+
+      const takeoverMs = fire.firedAt - killAt;
+      // Physics: cannot take over before the leader's lease expires.
+      const leaseRemainingAtKill = Math.max(
+        0,
+        (held.leaderLeaseUntil ?? killAt + leaseMs) - killAt,
+      );
+      expect(takeoverMs).toBeGreaterThanOrEqual(Math.max(0, leaseRemainingAtKill - 20));
+      // Upper bound: lease TTL + tick/poll slack (not a product SLA).
+      expect(takeoverMs).toBeLessThanOrEqual(leaseMs + tickSlackMs);
+
+      // Evidence for docs / later SLA choice — real measured latency.
+      console.log(
+        `[clock takeover] measured=${takeoverMs}ms leaseMs=${leaseMs} leaseRemainingAtKill=${leaseRemainingAtKill}ms`,
+      );
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});
+
+describe("chaos — SIGKILL durable mid-step", () => {
+  test("SIGKILL after step 1: resume skips completed step", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "oke-clock-durable-kill-"));
+    const journalPath = join(dir, "journal.json");
+    const markerPath = join(dir, "step1");
+    const runIdPath = join(dir, "runId");
+    try {
+      const child = Bun.spawn({
+        cmd: ["bun", childPath, "durable-mid-step", journalPath, markerPath, runIdPath],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+
+      expect(await waitForFile(markerPath)).toBe(true);
+      expect(await waitForFile(runIdPath)).toBe(true);
+      // Give the journal a moment to flush step 1 after the marker write.
+      await Bun.sleep(50);
+      child.kill(9);
+      await child.exited;
+
+      const runId = (await Bun.file(runIdPath).text()).trim();
+      expect(runId.length).toBeGreaterThan(0);
+
+      const calls: string[] = [];
+      const journalStore = createFileJournalStore(journalPath);
+      const charge = flow({
+        name: "chaos.charge",
+        durable: true,
+        do: async (_input, fx) => {
+          const intent = await fx.step("create-intent", () => {
+            calls.push("create-intent");
+            return { id: "pi_chaos" };
+          });
+          return fx.step("confirm", () => {
+            calls.push("confirm");
+            return intent.id === "pi_chaos";
+          });
+        },
+      });
+
+      const resumed = await runDurable({
+        flow: charge,
+        input: { orderId: "o-chaos" },
+        journalStore,
+        runId,
+      });
+      expect(resumed.status).toBe("completed");
+      // create-intent must not re-run; confirm runs once.
+      expect(calls).toEqual(["confirm"]);
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});

package/src/elements/clock/reconcile.ts

@@ -11,6 +11,9 @@
  * The scheduler reads the effective state from the Store, not the code.
  */
 
+import { mkdir, rename, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
 import type { ClockDecl } from "./declare.ts";
 
 /** Cron lifecycle status in `oke_crons`. */
@@ -49,6 +52,8 @@ export interface CronRow {
 
 /** Store surface for reconciled crons. */
 export interface CronStore {
+  /** Backing kind when known (`memory` · `file`). */
+  readonly kind?: "memory" | "file";
   /**
    * @param name - Cron name
    */
@@ -198,6 +203,7 @@ export function createMemoryCronStore(seed?: readonly CronRow[]): CronStore {
   }
 
   return {
+    kind: "memory",
     async get(name) {
       const r = rows.get(name);
       return r ? structuredClone(r) : undefined;
@@ -228,3 +234,107 @@ export function createMemoryCronStore(seed?: readonly CronRow[]): CronStore {
     },
   };
 }
+
+/** On-disk shape for {@link createFileCronStore}. */
+interface FileCronSnapshot {
+  readonly rows: CronRow[];
+}
+
+/**
+ * File-backed cron store for multi-process leader election / chaos tests.
+ *
+ * Every mutation runs under an exclusive lock directory next to `path` so
+ * two OS processes sharing the same file cannot both win a lease race.
+ * Does not cache rows across calls (each op re-reads disk).
+ *
+ * @param path - JSON snapshot path (`{ rows: CronRow[] }`)
+ */
+export function createFileCronStore(path: string): CronStore {
+  const lockDir = `${path}.lock`;
+
+  async function withFileLock<T>(fn: () => Promise<T>): Promise<T> {
+    for (let attempt = 0; attempt < 2_000; attempt++) {
+      try {
+        await mkdir(lockDir);
+        break;
+      } catch (err) {
+        const code = (err as NodeJS.ErrnoException).code;
+        if (code !== "EEXIST") throw err;
+        await Bun.sleep(2);
+        if (attempt === 1_999) {
+          throw new Error(`createFileCronStore: lock timeout on ${lockDir}`);
+        }
+      }
+    }
+    try {
+      return await fn();
+    } finally {
+      await rm(lockDir, { recursive: true, force: true });
+    }
+  }
+
+  async function loadUnlocked(): Promise<Map<string, CronRow>> {
+    const map = new Map<string, CronRow>();
+    const file = Bun.file(path);
+    if (!(await file.exists())) return map;
+    const text = await file.text();
+    if (!text.trim()) return map;
+    const snap = JSON.parse(text) as FileCronSnapshot;
+    for (const r of snap.rows ?? []) {
+      map.set(r.name, structuredClone(r));
+    }
+    return map;
+  }
+
+  async function flushUnlocked(map: Map<string, CronRow>): Promise<void> {
+    await mkdir(dirname(path), { recursive: true });
+    const tmp = join(dirname(path), `.${Bun.hash(path).toString(16)}.tmp`);
+    const body = JSON.stringify({ rows: [...map.values()] } satisfies FileCronSnapshot, null, 2);
+    await writeFile(tmp, body, "utf8");
+    await rename(tmp, path);
+  }
+
+  return {
+    kind: "file",
+    async get(name) {
+      return withFileLock(async () => {
+        const map = await loadUnlocked();
+        const r = map.get(name);
+        return r ? structuredClone(r) : undefined;
+      });
+    },
+    async put(row) {
+      await withFileLock(async () => {
+        const map = await loadUnlocked();
+        map.set(row.name, structuredClone(row));
+        await flushUnlocked(map);
+      });
+    },
+    async list() {
+      return withFileLock(async () => {
+        const map = await loadUnlocked();
+        return [...map.values()].map((r) => structuredClone(r));
+      });
+    },
+    async acquireLease(name, instanceId, now, leaseMs) {
+      return withFileLock(async () => {
+        const map = await loadUnlocked();
+        const row = map.get(name);
+        if (!row) return false;
+        const held =
+          row.leaderLeaseUntil !== undefined &&
+          row.leaderLeaseUntil > now &&
+          row.leaderInstanceId !== undefined &&
+          row.leaderInstanceId !== instanceId;
+        if (held) return false;
+        map.set(name, {
+          ...row,
+          leaderInstanceId: instanceId,
+          leaderLeaseUntil: now + leaseMs,
+        });
+        await flushUnlocked(map);
+        return true;
+      });
+    },
+  };
+}

package/src/elements/clock/runtime.ts

@@ -42,6 +42,11 @@ export interface CreateClockRuntimeOptions {
 
 /** Clock runtime. */
 export interface ClockRuntime {
+  /**
+   * Effective driver id selected at construction
+   * (`memory` · `file` · `frozen`).
+   */
+  readonly driverId: "memory" | "file" | "frozen";
   /** Instance id used for leases. */
   readonly instanceId: string;
   /** Cron store the scheduler reads. */
@@ -107,6 +112,11 @@ export function createClockRuntime(options: CreateClockRuntimeOptions = {}): Clo
   const leaseMs = options.leaseMs ?? 30_000;
   const declarations = new Map<string, ClockDecl>();
   const handlers = new Map<string, CronHandler>();
+  const driverId: ClockRuntime["driverId"] = timeTravel
+    ? "frozen"
+    : store.kind === "file"
+      ? "file"
+      : "memory";
 
   async function fire(name: string): Promise<boolean> {
     const row = await store.get(name);
@@ -161,6 +171,7 @@ export function createClockRuntime(options: CreateClockRuntimeOptions = {}): Clo
   }
 
   const runtime: ClockRuntime = {
+    driverId,
     instanceId,
     store,
     declarations,