okengine 0.23.0 → 0.23.2

package/src/kernel/horizontal-child.ts

@@ -20,6 +20,8 @@ import { dirname, join } from "node:path";
 import { createPostgresCronStore } from "../drivers/clock-postgres.ts";
 import { createPostgresJournalStore } from "../drivers/journal-postgres.ts";
 import { memorySignalDriver } from "../drivers/signal-memory.ts";
+import { createBunSignalRedisClient, redisSignalDriver } from "../drivers/signal-redis.ts";
+import { bunRedisCacheClient } from "../elements/store/cache-bus.ts";
 import { postgresDriver } from "../drivers/postgres.ts";
 import { redisDriver } from "../drivers/redis.ts";
 import { clock } from "../elements/clock/declare.ts";
@@ -99,14 +101,40 @@ clockRt.onCron(CRON, async () => {
   await Bun.write(path, prev + line);
 });
 
+const redisSignals = process.env.OKE_HORIZONTAL_REDIS_SIGNALS === "1";
 const signalRt = createSignalRuntime({
-  driver: memorySignalDriver,
+  driver: redisSignals ? redisSignalDriver : memorySignalDriver,
+  ...(redisSignals
+    ? { redis: createBunSignalRedisClient(redisUrl), compete: true, consumerId: instanceId }
+    : {}),
   durablePath: signalPath,
   leaseMs: Number.isFinite(leaseMs) ? leaseMs : 300,
 });
 const job = signal.once(SIG, { retries: 3, deadLetter: true, optional: true });
 signalRt.register(job);
+const news = signal.broadcast("horizontal-news", { optional: true });
+const feed = signal.live("horizontal-feed", { optional: true });
+if (redisSignals) {
+  signalRt.register(news);
+  signalRt.register(feed);
+}
 const bus = await signalRt.start();
+if (redisSignals) {
+  await bus.subscribe("horizontal-news", `news-${instanceId}`, async (msg) => {
+    const line = `${JSON.stringify({ instanceId, kind: "broadcast", id: msg.id, at: Date.now() })}\n`;
+    const path = join(workDir, "broadcast.jsonl");
+    const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
+    await Bun.write(path, prev + line);
+  });
+  void (async () => {
+    for await (const event of bus.live("horizontal-feed")) {
+      const line = `${JSON.stringify({ instanceId, kind: "live", id: event.id, at: Date.now() })}\n`;
+      const path = join(workDir, "live.jsonl");
+      const prev = (await Bun.file(path).exists()) ? await Bun.file(path).text() : "";
+      await Bun.write(path, prev + line);
+    }
+  })();
+}
 await bus.subscribe(SIG, `consumer-${instanceId}`, async (msg) => {
   const line = `${JSON.stringify({ instanceId, kind: "signal", id: msg.id, at: Date.now() })}\n`;
   const path = join(workDir, "signal.jsonl");
@@ -129,6 +157,7 @@ const storeRt = createStoreRuntime({
   drivers: { sql: postgresDriver },
   sql: { db: { name: "db", primary: { url: pgUrl } } },
   now: () => Date.now(),
+  cacheBus: { kind: "redis", redis: bunRedisCacheClient(redisUrl), origin: instanceId },
 });
 storeRt.register(db);
 
@@ -138,7 +167,16 @@ const ping = flow("horizontal.ping", {
 const rate = flow("horizontal.rate", {
   do: () => ({ ok: true as const, instanceId }),
 });
+let cachedReads = 0;
+const cached = flow("horizontal.cached", {
+  effects: { reads: ["sql:db"] },
+  do: () => {
+    cachedReads += 1;
+    return { reads: cachedReads, instanceId };
+  },
+});
 const write = flow("horizontal.write", {
+  effects: { writes: ["sql:db"] },
   do: async (_input, fx) => {
     const id = okid();
     await fx
@@ -159,6 +197,18 @@ const emit = flow("horizontal.emit", {
     return { emitted: true as const };
   },
 });
+const publishNews = flow("horizontal.news", {
+  do: async (_input, fx) => {
+    await fx.emit("horizontal-news", { from: instanceId, at: Date.now() });
+    return { emitted: true as const };
+  },
+});
+const publishFeed = flow("horizontal.feed", {
+  do: async (_input, fx) => {
+    await fx.emit("horizontal-feed", { from: instanceId, at: Date.now() });
+    return { emitted: true as const };
+  },
+});
 const charge = flow("horizontal.charge", {
   durable: true,
   do: async (_input, fx) => {
@@ -193,8 +243,11 @@ const charge = flow("horizontal.charge", {
 const bindings: Binding[] = [
   { trigger: http.get("/ping").public(), flow: ping as AnyFlowDef },
   { trigger: http.get("/rate").gate(rateGate), flow: rate as AnyFlowDef },
+  { trigger: http.get("/cached").public(), flow: cached as AnyFlowDef },
   { trigger: http.post("/write").public(), flow: write as AnyFlowDef },
   { trigger: http.post("/emit").public(), flow: emit as AnyFlowDef },
+  { trigger: http.post("/news").public(), flow: publishNews as AnyFlowDef },
+  { trigger: http.post("/feed").public(), flow: publishFeed as AnyFlowDef },
   { trigger: http.post("/charge").public(), flow: charge as AnyFlowDef },
 ];
 

package/src/kernel/horizontal.integration.test.ts

@@ -124,10 +124,26 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
       const traffic = async () => {
         for (let i = 0; i < 8; i++) {
           await Promise.all([
-            fetch(`${urlA}/write`, { method: "POST", body: "{}" }),
-            fetch(`${urlB}/write`, { method: "POST", body: "{}" }),
-            fetch(`${urlA}/emit`, { method: "POST", body: "{}" }),
-            fetch(`${urlB}/emit`, { method: "POST", body: "{}" }),
+            fetch(`${urlA}/write`, {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              body: "{}",
+            }),
+            fetch(`${urlB}/write`, {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              body: "{}",
+            }),
+            fetch(`${urlA}/emit`, {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              body: "{}",
+            }),
+            fetch(`${urlB}/emit`, {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              body: "{}",
+            }),
             fetch(`${urlA}/ping`),
             fetch(`${urlB}/ping`),
           ]);
@@ -150,7 +166,11 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
       expect(limited).toBeGreaterThanOrEqual(1);
 
       // (2) Start durable on A; hang mid-step.
-      void fetch(`${urlA}/charge`, { method: "POST", body: "{}" });
+      void fetch(`${urlA}/charge`, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: "{}",
+      });
       expect(await waitForFile(join(dir, "hang-inst-a.json"), 15_000)).toBe(true);
 
       // (5) Kill A mid combined scenario — B keeps serving.
@@ -177,8 +197,24 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
       // Keep traffic going on survivor (no deadlock).
       await trafficPromise.catch(() => {});
       for (let i = 0; i < 4; i++) {
-        expect((await fetch(`${urlB}/write`, { method: "POST", body: "{}" })).status).toBe(200);
-        expect((await fetch(`${urlB}/emit`, { method: "POST", body: "{}" })).status).toBe(200);
+        expect(
+          (
+            await fetch(`${urlB}/write`, {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              body: "{}",
+            })
+          ).status,
+        ).toBe(200);
+        expect(
+          (
+            await fetch(`${urlB}/emit`, {
+              method: "POST",
+              headers: { "content-type": "application/json" },
+              body: "{}",
+            })
+          ).status,
+        ).toBe(200);
       }
 
       // (1) Cron fired exactly once across both instances.
@@ -220,6 +256,94 @@ describe.skipIf(!LIVE)("horizontal — two OS processes, Postgres + Redis", () =
       await cleanupC.close();
     }
   }, 90_000);
+
+  test("broadcast, live, and cache invalidation reach the other instance", async () => {
+    const pg = LIVE!.pg;
+    const redis = LIVE!.redis;
+    const dir = await mkdtemp(join(tmpdir(), "oke-horizontal-bus-"));
+    const signalPath = join(dir, "signal.json");
+    const post = (url: string) =>
+      fetch(url, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: "{}",
+      });
+    const spawn = (instanceId: string, port: number) =>
+      Bun.spawn({
+        cmd: [
+          "bun",
+          childPath,
+          "serve",
+          instanceId,
+          String(port),
+          pg,
+          redis,
+          signalPath,
+          dir,
+          "5000",
+        ],
+        stdout: "pipe",
+        stderr: "pipe",
+        env: {
+          ...process.env,
+          DATABASE_URL: pg,
+          REDIS_URL: redis,
+          OKE_HORIZONTAL_REDIS_SIGNALS: "1",
+        },
+      });
+    const a = spawn("bus-a", 0);
+    const b = spawn("bus-b", 0);
+    try {
+      expect(await waitForFile(join(dir, "ready-bus-a.json"))).toBe(true);
+      expect(await waitForFile(join(dir, "ready-bus-b.json"))).toBe(true);
+      const readyA = (await Bun.file(join(dir, "ready-bus-a.json")).json()) as { port: number };
+      const readyB = (await Bun.file(join(dir, "ready-bus-b.json")).json()) as { port: number };
+      const urlA = `http://127.0.0.1:${readyA.port}`;
+      const urlB = `http://127.0.0.1:${readyB.port}`;
+      const first = (await (await fetch(`${urlA}/cached`)).json()) as { data: { reads: number } };
+      const second = (await (await fetch(`${urlA}/cached`)).json()) as { data: { reads: number } };
+      expect(first.data.reads).toBe(1);
+      expect(second.data.reads).toBe(1);
+      expect((await post(`${urlB}/write`)).status).toBe(200);
+      expect(
+        await waitFor(async () => {
+          const body = (await (await fetch(`${urlA}/cached`)).json()) as {
+            data: { reads: number };
+          };
+          return body.data.reads === 2;
+        }),
+      ).toBe(true);
+      expect((await post(`${urlA}/news`)).status).toBe(200);
+      expect((await post(`${urlA}/feed`)).status).toBe(200);
+      expect(
+        await waitFor(async () => {
+          const broadcast = await loadJsonl(join(dir, "broadcast.jsonl"));
+          const live = await loadJsonl(join(dir, "live.jsonl"));
+          const ids = (rows: Array<Record<string, unknown>>) =>
+            new Set(rows.map((row) => row.instanceId));
+          return (
+            ids(broadcast).has("bus-a") &&
+            ids(broadcast).has("bus-b") &&
+            ids(live).has("bus-a") &&
+            ids(live).has("bus-b")
+          );
+        }),
+      ).toBe(true);
+    } finally {
+      try {
+        a.kill(9);
+      } catch {
+        /* already dead */
+      }
+      try {
+        b.kill(9);
+      } catch {
+        /* ignore */
+      }
+      await Promise.allSettled([a.exited, b.exited]);
+      await rm(dir, { recursive: true, force: true });
+    }
+  }, 60_000);
 });
 
 if (!LIVE) {

package/src/kernel/idempotency.test.ts

@@ -349,12 +349,12 @@ describe("HTTP idempotency", () => {
     expect(await durableStore.idempotency!.forfeit(held!, held!.claimToken)).toBe(true);
     await Bun.sleep(120);
     const resumed = durable.fetch(post({ n: 1 }));
-    expect(await waitFor(() => blockers.length === 2)).toBe(true);
-    expect(step1).toEqual(["step1"]);
+    // The holder's heartbeat kept the lease, so the same process does not
+    // start a second session for the run still inside mid-flight.
+    expect((await resumed).status).toBe(409);
+    expect(blockers).toHaveLength(1);
     blockers[0]!();
-    blockers[1]!();
     expect((await hung).status).toBe(200);
-    expect((await resumed).status).toBe(200);
     expect(step1).toEqual(["step1"]);
   }, 30_000);
 

package/src/kernel/journal-boot.test.ts

@@ -22,6 +22,7 @@ import {
   type PostgresJournalStore,
 } from "../drivers/journal-postgres.ts";
 import { oke, type OkeApp } from "./app.ts";
+import { clearJournalHeartbeats } from "./journal.ts";
 import type { JournalRuntime } from "./boot-bind/journal.ts";
 import { flow, resetFlowSeq, type AnyFlowDef } from "./flow.ts";
 import type { Binding } from "./on.ts";
@@ -47,6 +48,7 @@ async function bootDurableApp(options: {
   readonly name: string;
   readonly journal: JournalRuntime;
   readonly bindings: readonly Binding[];
+  readonly now?: () => number;
 }): Promise<OkeApp> {
   const app = oke({
     name: options.name,
@@ -57,6 +59,7 @@ async function bootDurableApp(options: {
     // oke() in the process — sibling apps need their own copies.
     bindings: [...options.bindings],
     elements: { journal: options.journal },
+    ...(options.now ? { fx: { now: options.now } } : {}),
   });
   await app.boot();
   return app;
@@ -127,11 +130,18 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
       bindings,
     });
     // Start the run through the real HTTP path; it hangs inside step 2.
-    void appA.fetch(new Request("http://localhost/charge", { method: "POST", body: "{}" }));
+    void appA.fetch(
+      new Request("http://localhost/charge", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: "{}",
+      }),
+    );
     expect(await waitFor(async () => blockers.length === 1)).toBe(true);
     const runId = (await store.list())[0]!.id;
-    // "Crash": no commit, no release — the lease (120ms) is left to expire.
-    // appA is deliberately never stopped; its promise stays dangling.
+    // "Crash": the process dies, so its heartbeat timers die with it.
+    // The in-flight promise stays dangling and the lease (120ms) expires.
+    clearJournalHeartbeats();
 
     await Bun.sleep(300);
     const appB = await bootDurableApp({
@@ -165,7 +175,11 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
       bindings,
     });
     const parked = await appA.fetch(
-      new Request("http://localhost/sleep", { method: "POST", body: "{}" }),
+      new Request("http://localhost/sleep", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: "{}",
+      }),
     );
     expect(parked.status).toBe(204); // durable park — lease released, row sleeping
     const runId = (await store.list())[0]!.id;
@@ -198,7 +212,13 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
       journal: journalRuntime(store, "A", 5_000),
       bindings,
     });
-    await appA.fetch(new Request("http://localhost/sleep", { method: "POST", body: "{}" }));
+    await appA.fetch(
+      new Request("http://localhost/sleep", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: "{}",
+      }),
+    );
     await appA.stop();
 
     // Both survivors boot while the sleep is future (orphan scan skips it).
@@ -217,10 +237,61 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
     await Promise.all([appB.resumeDurable(), appC.resumeDurable()]);
 
     expect(confirms).toHaveLength(1);
+
     await appB.stop();
     await appC.stop();
   });
 
+  test("resume uses a live clock: a lease written after the tick sample is not already expired", async () => {
+    let clock = 10_000_000;
+    const store = await createPostgresJournalStore({ sql: createPostgresJournalFake() });
+    const confirms: string[] = [];
+    const bindings = [sleeperBinding("120ms", confirms)];
+    const acquiredAt: number[] = [];
+    const acquire = store.acquireLease.bind(store);
+    store.acquireLease = async (runId, instanceId, at, leaseMs) => {
+      acquiredAt.push(at);
+      return acquire(runId, instanceId, at, leaseMs);
+    };
+    const claim = store.claimDueSleep.bind(store);
+    store.claimDueSleep = async (instanceId, at, leaseMs) => {
+      const due = await claim(instanceId, at, leaseMs);
+      // The tick already sampled `at`. A live run clock must see this jump.
+      if (due) clock += 2_700;
+      return due;
+    };
+
+    const appA = await bootDurableApp({
+      name: "clock-a",
+      journal: journalRuntime(store, "A", 5_000),
+      bindings,
+      now: () => clock,
+    });
+    await appA.fetch(
+      new Request("http://localhost/sleep", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: "{}",
+      }),
+    );
+    await appA.stop();
+
+    const appB = await bootDurableApp({
+      name: "clock-b",
+      journal: journalRuntime(store, "B", 5_000),
+      bindings,
+      now: () => clock,
+    });
+    const before = clock;
+    clock = before + 500;
+    acquiredAt.length = 0;
+    await appB.resumeDurable();
+
+    const stamped = acquiredAt.find((at) => at >= before + 2_700);
+    expect(stamped).toBeDefined();
+    await appB.stop();
+  });
+
   test("boot orphan scan skips runs held under a live lease", async () => {
     const store = await createPostgresJournalStore({ sql: createPostgresJournalFake() });
     const step1Calls: string[] = [];
@@ -233,7 +304,13 @@ describe("journal boot — shared store, real boot paths (fake SQL)", () => {
       journal: journalRuntime(store, "A", 30_000),
       bindings,
     });
-    void appA.fetch(new Request("http://localhost/charge", { method: "POST", body: "{}" }));
+    void appA.fetch(
+      new Request("http://localhost/charge", {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: "{}",
+      }),
+    );
     expect(await waitFor(async () => blockers.length === 1)).toBe(true);
 
     // B boots while A's lease is live — must not steal the run.

package/src/kernel/journal.test.ts

@@ -5,7 +5,10 @@
 
 import { describe, expect, test } from "bun:test";
 
+import { runDurable } from "../elements/clock/durable.ts";
+import { flow } from "./flow.ts";
 import {
+  clearJournalHeartbeats,
   createJournal,
   createMemoryJournalStore,
   hasJournalLease,
@@ -13,6 +16,7 @@ import {
   isJournalSuspend,
   JOURNAL_DEFAULT_LEASE_MS,
 } from "./journal.ts";
+import { OkeError } from "./errors.ts";
 
 describe("journal lease (memory store)", () => {
   test("built-in stores expose the lease surface", () => {
@@ -139,4 +143,158 @@ describe("journal lease (memory store)", () => {
     const again = await journal.resume(session.runId);
     expect(await again.step("s", () => 2)).toBe(1);
   });
+
+  test("a stale lease holder cannot append", async () => {
+    let t = 1_000;
+    const store = createMemoryJournalStore();
+    const a = createJournal({
+      store,
+      now: () => t,
+      lease: { instanceId: "a", leaseMs: 100 },
+      codeVersion: "v1",
+    });
+    const session = await a.start("charge");
+    await session.step("create-intent", () => ({ id: "pi_1" }));
+    t = 1_200;
+    const b = createJournal({
+      store,
+      now: () => t,
+      lease: { instanceId: "b", leaseMs: 100 },
+      codeVersion: "v1",
+    });
+    await b.resume(session.runId);
+    try {
+      await session.step("capture", () => ({ ok: true }));
+      expect.unreachable("stale holder must be rejected");
+    } catch (err) {
+      expect(err).toBeInstanceOf(OkeError);
+      expect((err as OkeError).code).toBe(1074);
+    }
+  });
+
+  test("heartbeat extends the lease, and a lost lease does not persist the step", async () => {
+    let t = 1_000;
+    const store = createMemoryJournalStore();
+    const a = createJournal({
+      store,
+      now: () => t,
+      lease: { instanceId: "a", leaseMs: 60 },
+      codeVersion: "v1",
+    });
+    const session = await a.start("charge");
+    t = 1_030;
+    await Bun.sleep(40);
+    expect((await store.get(session.runId))?.leaseExpiresAt).toBe(1_090);
+    t = 2_000;
+    const b = createJournal({
+      store,
+      now: () => t,
+      lease: { instanceId: "b", leaseMs: 60 },
+      codeVersion: "v1",
+    });
+    await b.resume(session.runId);
+    await Bun.sleep(40);
+    await expect(session.step("capture", () => 1)).rejects.toBeInstanceOf(OkeError);
+    expect((await store.get(session.runId))?.entries ?? []).toHaveLength(0);
+    expect((await store.get(session.runId))?.lockedBy).toBe("b");
+    clearJournalHeartbeats();
+  });
+
+  test("replay throws when the next call does not match the journal", async () => {
+    const store = createMemoryJournalStore();
+    const journal = createJournal({ store, now: () => 1, codeVersion: "v1" });
+    const session = await journal.start("charge");
+    await session.step("a", () => 1);
+    const resumed = await journal.resume(session.runId);
+    try {
+      await resumed.step("b", () => 2);
+      expect.unreachable("divergent replay must throw");
+    } catch (err) {
+      expect(err).toBeInstanceOf(OkeError);
+      expect((err as OkeError).code).toBe(1075);
+    }
+  });
+
+  test("a non-JSON journal value is OKE1077", async () => {
+    const store = createMemoryJournalStore();
+    const session = await createJournal({ store, now: () => 1 }).start("charge");
+    try {
+      await session.effect("call", "charge", () => 1n);
+      expect.unreachable("bigint must not journal");
+    } catch (err) {
+      expect(err).toBeInstanceOf(OkeError);
+      expect((err as OkeError).code).toBe(1077);
+    }
+  });
+
+  test("a 0.23.1 stamp resumes and is restamped when codeVersion is set", async () => {
+    const store = createMemoryJournalStore();
+    const started = await createJournal({ store, now: () => 1, codeVersion: "v1" }).start("charge");
+    const row = await store.get(started.runId);
+    if (row) {
+      row.codeVersion = "0.23.1";
+      await store.put(row);
+    }
+    const resumed = await createJournal({ store, now: () => 2, codeVersion: "v9" }).resume(
+      started.runId,
+    );
+    expect(resumed.run.codeVersion).toBe("app:v9");
+    expect((await store.get(started.runId))?.status).not.toBe("failed");
+  });
+
+  test("resume rejects a run stamped with another code version", async () => {
+    const store = createMemoryJournalStore();
+    const started = await createJournal({ store, now: () => 1, codeVersion: "v1" }).start("charge");
+    try {
+      await createJournal({ store, now: () => 2, codeVersion: "v2" }).resume(started.runId);
+      expect.unreachable("version skew must throw");
+    } catch (err) {
+      expect(err).toBeInstanceOf(OkeError);
+      expect((err as OkeError).code).toBe(1076);
+    }
+  });
+});
+
+describe("journal fetch snapshot", () => {
+  test("a durable flow replays fx.fetch without calling the network again", async () => {
+    const store = createMemoryJournalStore();
+    let calls = 0;
+    const original = globalThis.fetch;
+    globalThis.fetch = (async () => {
+      calls += 1;
+      return new Response("hello", { status: 201 });
+    }) as unknown as typeof fetch;
+    const load = flow("load", {
+      durable: true,
+      do: async (_input, fx) => {
+        const res = await fx.step("load", () => fx.fetch("https://example.com/data"));
+        return { status: res.status, text: await res.text() };
+      },
+    });
+    try {
+      const first = await runDurable({
+        flow: load,
+        input: {},
+        journalStore: store,
+        now: () => 1,
+      });
+      expect(first).toMatchObject({ status: "completed", output: { status: 201, text: "hello" } });
+      expect(calls).toBe(1);
+      const row = await store.get(first.runId);
+      expect(row?.entries.some((entry) => entry.kind === "step" && entry.name === "load")).toBe(
+        true,
+      );
+      const replay = await createJournal({ store, now: () => 2 }).resume(first.runId);
+      const again = await replay.step("load", () => {
+        calls += 1;
+        return new Response("nope", { status: 500 });
+      });
+      expect(again).toBeInstanceOf(Response);
+      expect(again.status).toBe(201);
+      expect(await again.text()).toBe("hello");
+      expect(calls).toBe(1);
+    } finally {
+      globalThis.fetch = original;
+    }
+  });
 });