okengine 0.6.1 → 0.7.0

package/src/elements/signal/orphan-messages.test.ts

@@ -0,0 +1,152 @@
+/**
+ * Orphan reconciliation vs in-flight / DLQ messages.
+ *
+ * Config rows become orphaned when absent from declarations; durable messages
+ * remain. Drain/replay require an active declaration — recovery is re-declare.
+ */
+
+import { afterEach, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+import { memorySignalDriver } from "../../drivers/index.ts";
+import { signal } from "./declare.ts";
+import { createMemorySignalConfigStore, reconcileSignals } from "./reconcile.ts";
+import { createSignalRuntime } from "./runtime.ts";
+
+const openRuntimes: Array<{ close(): Promise<void> }> = [];
+
+afterEach(async () => {
+  while (openRuntimes.length) {
+    await openRuntimes.pop()!.close();
+  }
+});
+
+describe("orphan signal messages", () => {
+  test("orphaned config keeps durable DLQ; replay needs re-declare", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "oke-signal-orphan-"));
+    const durablePath = join(dir, "bus.json");
+    try {
+      const legacy = signal("legacy-shipped", {
+        delivery: "once",
+        retries: 0,
+        deadLetter: true,
+      });
+      const runtime = createSignalRuntime({
+        driver: memorySignalDriver,
+        durablePath,
+      });
+      openRuntimes.push(runtime);
+      runtime.register(legacy);
+      const bus = await runtime.start();
+
+      await bus.subscribe("legacy-shipped", "c1", async () => {
+        throw new Error("gone");
+      });
+      await bus.emit("legacy-shipped", { shipmentId: "shp_9" });
+      await bus.drain();
+      expect(await bus.deadLetters("legacy-shipped")).toHaveLength(1);
+      await runtime.close();
+      openRuntimes.pop();
+
+      // Config reconciliation: signal removed from code → orphaned, not deleted.
+      const store = createMemorySignalConfigStore();
+      await reconcileSignals([legacy], store);
+      const after = await reconcileSignals([], store);
+      expect(after.orphaned).toEqual(["legacy-shipped"]);
+      expect(after.rows.find((r) => r.name === "legacy-shipped")?.status).toBe("orphaned");
+
+      // Bus without the declaration: messages still load, but ops require decl.
+      const emptyRuntime = createSignalRuntime({
+        driver: memorySignalDriver,
+        durablePath,
+      });
+      openRuntimes.push(emptyRuntime);
+      const emptyBus = await emptyRuntime.start();
+      await expect(
+        emptyBus.replay({
+          signal: "legacy-shipped",
+          ratePerSec: 10,
+          dryRun: false,
+        }),
+      ).rejects.toThrow(/Unknown signal/);
+      await emptyRuntime.close();
+      openRuntimes.pop();
+
+      // Re-declare → DLQ processable again via replay.
+      const restored = signal("legacy-shipped", {
+        delivery: "once",
+        retries: 0,
+        deadLetter: true,
+      });
+      const again = await reconcileSignals([restored], store);
+      expect(again.active).toEqual(["legacy-shipped"]);
+      expect(again.orphaned).toEqual([]);
+
+      const restoreRuntime = createSignalRuntime({
+        driver: memorySignalDriver,
+        durablePath,
+      });
+      openRuntimes.push(restoreRuntime);
+      restoreRuntime.register(restored);
+      const restoreBus = await restoreRuntime.start();
+      const seen: unknown[] = [];
+      await restoreBus.subscribe("legacy-shipped", "c2", async (m) => {
+        seen.push(m.payload);
+      });
+      const result = await restoreBus.replay({
+        signal: "legacy-shipped",
+        ratePerSec: 100,
+        dryRun: false,
+      });
+      expect(result.succeeded).toBe(1);
+      expect(seen).toEqual([{ shipmentId: "shp_9" }]);
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+
+  test("pending messages for an undeclared signal are not drained", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "oke-signal-orphan-pend-"));
+    const durablePath = join(dir, "bus.json");
+    try {
+      const once = signal("legacy-shipped", {
+        delivery: "once",
+        optional: true,
+      });
+      const runtime = createSignalRuntime({
+        driver: memorySignalDriver,
+        durablePath,
+      });
+      openRuntimes.push(runtime);
+      runtime.register(once);
+      const bus = await runtime.start();
+      await bus.emit("legacy-shipped", { id: 1 });
+      await runtime.close();
+      openRuntimes.pop();
+
+      const other = signal("order-placed", { delivery: "once" });
+      const next = createSignalRuntime({
+        driver: memorySignalDriver,
+        durablePath,
+      });
+      openRuntimes.push(next);
+      next.register(other);
+      const nextBus = await next.start();
+      const seen: unknown[] = [];
+      await nextBus.subscribe("order-placed", "c1", async (m) => {
+        seen.push(m.payload);
+      });
+      // Even a subscriber on a different name cannot claim orphaned-name rows,
+      // and legacy-shipped has no declaration so it cannot be subscribed.
+      await nextBus.drain();
+      expect(seen).toEqual([]);
+      await expect(nextBus.subscribe("legacy-shipped", "x", async () => {})).rejects.toThrow(
+        /Unknown signal/,
+      );
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
+});

package/src/elements/signal/runtime.ts

@@ -3,7 +3,7 @@
  * emit / subscribe / live / drain for `fx` and tests.
  */
 
-import type { SignalDriver, SignalBus } from "../../drivers/signal-types.ts";
+import type { SignalBus, SignalDriver, SignalEmitOptions } from "../../drivers/signal-types.ts";
 import type { SignalDecl } from "./declare.ts";
 
 /** Options for {@link createSignalRuntime}. */
@@ -12,6 +12,8 @@ export interface CreateSignalRuntimeOptions {
   readonly driver: SignalDriver;
   /** Clock. */
   readonly now?: () => number;
+  /** Visibility lease for `once` claims (ms). */
+  readonly leaseMs?: number;
   /** Durable path for chaos recovery. */
   readonly durablePath?: string;
   /** Injected clients forwarded to the driver. */
@@ -22,6 +24,8 @@ export interface CreateSignalRuntimeOptions {
 
 /** Signal runtime. */
 export interface SignalRuntime {
+  /** Bound protocol driver id (`memory` · `postgres` · `redis` · `nats`). */
+  readonly driverId: string;
   /** Underlying bus (after {@link SignalRuntime.start}). */
   readonly bus: SignalBus | null;
   /**
@@ -41,8 +45,9 @@ export interface SignalRuntime {
    *
    * @param name - Signal name
    * @param payload - Payload
+   * @param options - Optional emit options (`key` for per-key once ordering)
    */
-  emit(name: string, payload?: unknown): Promise<void>;
+  emit(name: string, payload?: unknown, options?: SignalEmitOptions): Promise<void>;
   /** Close the bus. */
   close(): Promise<void>;
 }
@@ -57,6 +62,7 @@ export function createSignalRuntime(options: CreateSignalRuntimeOptions): Signal
   let bus: SignalBus | null = null;
 
   const runtime: SignalRuntime = {
+    driverId: options.driver.id,
     get bus() {
       return bus;
     },
@@ -69,6 +75,7 @@ export function createSignalRuntime(options: CreateSignalRuntimeOptions): Signal
       bus = await options.driver.open({
         signals: declarations,
         now: options.now,
+        leaseMs: options.leaseMs,
         durablePath: options.durablePath,
         sql: options.sql,
         redis: options.redis,
@@ -76,9 +83,9 @@ export function createSignalRuntime(options: CreateSignalRuntimeOptions): Signal
       });
       return bus;
     },
-    async emit(name, payload) {
+    async emit(name, payload, options) {
       const b = await this.start();
-      await b.emit(name, payload);
+      await b.emit(name, payload, options);
     },
     async close() {
       if (bus) {

package/src/elements/signal/schema-emit.test.ts

@@ -0,0 +1,107 @@
+/**
+ * Signal `schema` enforced at emit time via Standard Schema (same as Flow `in`).
+ */
+
+import { afterEach, describe, expect, test } from "bun:test";
+import { z } from "zod";
+
+import {
+  createPostgresSignalFake,
+  memorySignalDriver,
+  postgresSignalDriver,
+  type SignalBus,
+  type SignalDriver,
+} from "../../drivers/index.ts";
+import { OkeError, OKE_ERRORS } from "../../kernel/errors.ts";
+import { signal, type SignalDecl } from "./declare.ts";
+
+const openBuses: SignalBus[] = [];
+
+afterEach(async () => {
+  while (openBuses.length) {
+    await openBuses.pop()!.close();
+  }
+});
+
+async function openBus(
+  driver: SignalDriver,
+  decls: readonly SignalDecl[],
+  extra: Record<string, unknown> = {},
+): Promise<SignalBus> {
+  const signals = new Map(decls.map((d) => [d.name, d]));
+  const bus = await driver.open({ signals, ...extra });
+  openBuses.push(bus);
+  return bus;
+}
+
+const drivers: Array<{
+  label: string;
+  driver: SignalDriver;
+  setup?: () => Record<string, unknown>;
+}> = [
+  { label: "memory", driver: memorySignalDriver },
+  {
+    label: "postgres",
+    driver: postgresSignalDriver,
+    setup: () => ({ sql: createPostgresSignalFake() }),
+  },
+];
+
+for (const { label, driver, setup } of drivers) {
+  describe(`signal schema emit · ${label}`, () => {
+    test("invalid payload rejected at emit with OKE1043; valid succeeds", async () => {
+      const orderPlaced = signal("order-placed", {
+        delivery: "once",
+        schema: z.object({
+          orderId: z.string(),
+          total: z.number(),
+        }),
+        retries: 1,
+        deadLetter: true,
+      });
+      const bus = await openBus(driver, [orderPlaced], setup?.() ?? {});
+      const got: unknown[] = [];
+      await bus.subscribe("order-placed", "c1", async (m) => {
+        got.push(m.payload);
+      });
+
+      let rejected: unknown;
+      try {
+        await bus.emit("order-placed", { orderId: "ord_1", total: "not-a-number" });
+      } catch (err) {
+        rejected = err;
+      }
+      expect(rejected).toBeInstanceOf(OkeError);
+      const oke = rejected as OkeError;
+      expect(oke.code).toBe(OKE_ERRORS.SIGNAL_SCHEMA.code);
+      expect(oke.message).toContain("OKE1043");
+      expect(oke.message).toContain("order-placed");
+
+      // Nothing staged while invalid.
+      await bus.drain();
+      expect(got).toEqual([]);
+      const statsBefore = await bus.inspect("order-placed");
+      expect(statsBefore[0]?.pending).toBe(0);
+      expect(statsBefore[0]?.delivered).toBe(0);
+
+      await bus.emit("order-placed", { orderId: "ord_1", total: 49.5 });
+      await bus.drain();
+      expect(got).toEqual([{ orderId: "ord_1", total: 49.5 }]);
+    });
+
+    test("no schema: any payload still accepted", async () => {
+      const loose = signal("hook", {
+        delivery: "once",
+        optional: true,
+      });
+      const bus = await openBus(driver, [loose], setup?.() ?? {});
+      const got: unknown[] = [];
+      await bus.subscribe("hook", "c1", async (m) => {
+        got.push(m.payload);
+      });
+      await bus.emit("hook", { anything: true });
+      await bus.drain();
+      expect(got).toEqual([{ anything: true }]);
+    });
+  });
+}

package/src/elements/signal.test.ts

@@ -3,12 +3,12 @@
  * - delivery is mandatory with no default
  * - rolled-back transaction emits nothing
  * - crash between write and emit loses nothing (transactional outbox)
- * - competing consumers under once receive each message exactly once
+ * - competing consumers under once receive each message once
  * - broadcast reaches every subscriber
  * - live is client-subscribable
  * - DLQ preserves typed failure reasons per attempt
  * - redis / nats keep an outbox relay (semantics never regress)
- * - chaos: kill process mid-transaction
+ * - chaos: kill process mid-transaction; SIGKILL after claim + lease reclaim
  */
 
 import { afterEach, describe, expect, test } from "bun:test";
@@ -148,6 +148,22 @@ for (const { label, driver, setup } of drivers) {
       expect(numbers).toEqual(["1", "2"]);
     });
 
+    test("once: a single emit is received by exactly one of two consumers", async () => {
+      const once = signal("order-placed", { delivery: "once", retries: 2 });
+      const bus = await openBus(driver, [once], setup?.() ?? {});
+      const got: string[] = [];
+      await bus.subscribe("order-placed", "worker-a", async () => {
+        got.push("a");
+      });
+      await bus.subscribe("order-placed", "worker-b", async () => {
+        got.push("b");
+      });
+      await bus.emit("order-placed", { n: 1 });
+      await bus.drain();
+      expect(got).toHaveLength(1);
+      expect(got[0] === "a" || got[0] === "b").toBe(true);
+    });
+
     test("broadcast reaches every subscriber", async () => {
       const bcast = signal("news", { delivery: "broadcast" });
       const bus = await openBus(driver, [bcast], setup?.() ?? {});
@@ -262,7 +278,7 @@ describe("postgres transactional emit (dual-write fix)", () => {
 describe("redis / nats outbox relay", () => {
   test("redis relay publishes after commit, not on rollback", async () => {
     const redis = createSignalRedisFake();
-    const once = signal("order-placed", { delivery: "once" });
+    const once = signal("order-placed", { delivery: "once", optional: true });
     const bus = await openBus(redisSignalDriver, [once], { redis });
 
     const tx = await bus.begin();
@@ -280,7 +296,7 @@ describe("redis / nats outbox relay", () => {
 
   test("nats relay publishes after commit, not on rollback", async () => {
     const nats = createSignalNatsFake();
-    const once = signal("order-placed", { delivery: "once" });
+    const once = signal("order-placed", { delivery: "once", optional: true });
     const bus = await openBus(natsSignalDriver, [once], { nats });
 
     const tx = await bus.begin();
@@ -347,7 +363,7 @@ describe("chaos — kill process mid-transaction", () => {
           import { memorySignalDriver } from ${JSON.stringify(join(import.meta.dir, "../drivers/signal-memory.ts"))};
           import { signal } from ${JSON.stringify(join(import.meta.dir, "signal/declare.ts"))};
           import { createSignalRuntime } from ${JSON.stringify(join(import.meta.dir, "signal/runtime.ts"))};
-          const orderPlaced = signal("order-placed", { delivery: "once", retries: 3, deadLetter: true });
+          const orderPlaced = signal("order-placed", { delivery: "once", retries: 3, deadLetter: true, optional: true });
           const runtime = createSignalRuntime({ driver: memorySignalDriver, durablePath: ${JSON.stringify(durablePath)} });
           runtime.register(orderPlaced);
           const bus = await runtime.start();
@@ -433,6 +449,74 @@ describe("chaos — kill process mid-transaction", () => {
       await rm(dir, { recursive: true, force: true });
     }
   });
+
+  test("SIGKILL after claim: committed message is reclaimed after lease expiry", async () => {
+    const dir = await mkdtemp(join(tmpdir(), "oke-signal-chaos-consume-"));
+    const durablePath = join(dir, "bus.json");
+    const markerPath = join(dir, "claimed");
+    const leaseMs = 80;
+    try {
+      const commit = Bun.spawn({
+        cmd: ["bun", join(import.meta.dir, "signal/chaos-child.ts"), durablePath, "commit"],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      expect(await commit.exited).toBe(0);
+
+      const consumer = Bun.spawn({
+        cmd: [
+          "bun",
+          join(import.meta.dir, "signal/chaos-child.ts"),
+          durablePath,
+          "consume-hang",
+          markerPath,
+          String(leaseMs),
+        ],
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+
+      for (let i = 0; i < 100; i++) {
+        if (await Bun.file(markerPath).exists()) break;
+        await Bun.sleep(10);
+      }
+      expect(await Bun.file(markerPath).exists()).toBe(true);
+      consumer.kill(9);
+      await consumer.exited;
+
+      // Advance past the visibility lease so reclaim-at-claim can take the row.
+      await Bun.sleep(leaseMs + 40);
+
+      let clock = Date.now() + leaseMs + 1_000;
+      const orderPlaced = signal("order-placed", {
+        delivery: "once",
+        retries: 3,
+        deadLetter: true,
+      });
+      const runtime = createSignalRuntime({
+        driver: memorySignalDriver,
+        durablePath,
+        now: () => clock,
+        leaseMs,
+      });
+      runtime.register(orderPlaced);
+      const bus = await runtime.start();
+      openBuses.push(bus);
+
+      const seen: Array<{ payload: unknown; attempts: number }> = [];
+      await bus.subscribe("order-placed", "rescue", async (m) => {
+        seen.push({ payload: m.payload, attempts: m.attempts });
+      });
+      await bus.drain();
+
+      expect(seen).toHaveLength(1);
+      expect(seen[0]!.payload).toEqual({ id: "1" });
+      // Original claim in the killed child + reclaim here.
+      expect(seen[0]!.attempts).toBeGreaterThanOrEqual(2);
+    } finally {
+      await rm(dir, { recursive: true, force: true });
+    }
+  });
 });
 
 describe("protocol naming", () => {

package/src/elements/store/files-fx.ts

@@ -0,0 +1,104 @@
+/**
+ * Capability-gated files handle for `fx.store` — keeps Bun.Image pipeline
+ * code in the store element so the kernel edge profile stays lean.
+ */
+
+import type { StoreDecl } from "./declare.ts";
+import {
+  createFilesImagePipeline,
+  putImageToBucket,
+  type FilesImageOptions,
+  type PutImageOptions,
+} from "./files-image.ts";
+import type { FilesStoreFxHandle } from "./runtime.ts";
+
+/** Gate + dry-run hooks supplied by `createFxContext`. */
+export interface GatedFilesFxBridge {
+  /**
+   * Capability-check + ledger one effect, then run `body`.
+   *
+   * @param kind - Effect kind
+   * @param resource - Resource ref
+   * @param body - Work to run under the gate
+   */
+  gate<T>(kind: "read" | "write", resource: string, body: () => Promise<T>): Promise<T>;
+  /** Throw when a driver-backed write cannot be isolated in dry-run. */
+  refuseDryRunWrite(): void;
+}
+
+/**
+ * Lazy, capability-gated files handle — `image()` stays sync for chaining;
+ * terminals and CRUD record read/write effects.
+ *
+ * @param decl - Files store declaration
+ * @param open - Opens (and caches) the runtime handle
+ * @param bridge - fx gate + dry-run hooks
+ */
+export function createGatedFilesStoreHandle(
+  decl: StoreDecl,
+  open: () => Promise<FilesStoreFxHandle>,
+  bridge: GatedFilesFxBridge,
+): FilesStoreFxHandle {
+  const ref = decl.ref as `files:${string}`;
+  let cached: FilesStoreFxHandle | undefined;
+  const ensure = async (): Promise<FilesStoreFxHandle> => {
+    if (!cached) cached = await open();
+    return cached;
+  };
+  const access = {
+    get: (key: string) =>
+      bridge.gate("read", ref, async () => {
+        const h = await ensure();
+        return h.get(key);
+      }),
+    put: (key: string, data: Uint8Array | string) =>
+      bridge.gate("write", ref, async () => {
+        bridge.refuseDryRunWrite();
+        const h = await ensure();
+        return h.put(key, data);
+      }),
+  };
+
+  return {
+    ref,
+    get driverId() {
+      return cached?.driverId ?? "memory";
+    },
+    get(key) {
+      return access.get(key);
+    },
+    list(prefix) {
+      return bridge.gate("read", ref, async () => {
+        const h = await ensure();
+        return h.list(prefix);
+      });
+    },
+    put(key, data) {
+      return access.put(key, data);
+    },
+    delete(key) {
+      return bridge.gate("write", ref, async () => {
+        bridge.refuseDryRunWrite();
+        const h = await ensure();
+        return h.delete(key);
+      });
+    },
+    image(source: string | Uint8Array, imageOpts?: FilesImageOptions) {
+      return createFilesImagePipeline(access, source, imageOpts);
+    },
+    putImage(key: string, data: Uint8Array | string, putOpts?: PutImageOptions) {
+      return bridge.gate("write", ref, async () => {
+        bridge.refuseDryRunWrite();
+        // putImage writes original + variants via bucket put; use ungated
+        // runtime puts inside the single write effect.
+        const h = await ensure();
+        return putImageToBucket(
+          { get: (k) => h.get(k), put: (k, d) => h.put(k, d) },
+          key,
+          data,
+          putOpts,
+        );
+      });
+    },
+  };
+}