@cosmicdrift/kumiko-framework 0.200.1 → 0.202.0

package/src/pipeline/__tests__/redis-pipeline.integration.test.ts

@@ -22,37 +22,44 @@ describe("idempotency guard", () => {
   const tenantA = "00000000-0000-4000-8000-00000000000a";
   const userA = "00000000-0000-4000-8000-0000000000a1";
 
-  test("returns null for new request", async () => {
+  test("returns acquired for new request", async () => {
     const guard = createIdempotencyGuard(testRedis.redis);
     const result = await guard.check(tenantA, userA, "req-new-123");
-    expect(result).toBeNull();
+    expect(result.status).toBe("acquired");
   });
 
   test("returns cached result for duplicate request", async () => {
     const guard = createIdempotencyGuard(testRedis.redis);
     const requestId = "req-dup-456";
 
-    await guard.store(tenantA, userA, requestId, { isSuccess: true, data: { id: 1 } });
+    const acquired = await guard.check(tenantA, userA, requestId);
+    if (acquired.status !== "acquired") throw new Error("expected to acquire the lock");
+    await guard.store(tenantA, userA, requestId, acquired.token, {
+      isSuccess: true,
+      data: { id: 1 },
+    });
     const cached = await guard.check(tenantA, userA, requestId);
 
-    expect(cached).not.toBeNull();
-    if (!cached) throw new Error("expected cached value");
-    expect(JSON.parse(cached)).toEqual({ isSuccess: true, data: { id: 1 } });
+    expect(cached.status).toBe("cached");
+    if (cached.status !== "cached") throw new Error("expected cached value");
+    expect(JSON.parse(cached.result)).toEqual({ isSuccess: true, data: { id: 1 } });
   });
 
   test("expires after TTL", async () => {
     const guard = createIdempotencyGuard(testRedis.redis, { ttlSeconds: 1 });
     const requestId = "req-ttl-789";
 
-    await guard.store(tenantA, userA, requestId, { done: true });
+    const acquired = await guard.check(tenantA, userA, requestId);
+    if (acquired.status !== "acquired") throw new Error("expected to acquire the lock");
+    await guard.store(tenantA, userA, requestId, acquired.token, { done: true });
 
     // Should exist immediately
-    expect(await guard.check(tenantA, userA, requestId)).not.toBeNull();
+    expect((await guard.check(tenantA, userA, requestId)).status).toBe("cached");
 
     // Wait for expiry
     await new Promise((r) => setTimeout(r, 1100));
 
-    expect(await guard.check(tenantA, userA, requestId)).toBeNull();
+    expect((await guard.check(tenantA, userA, requestId)).status).toBe("acquired");
   });
 
   test("parallel check(): second caller waits for the first's store() instead of racing", async () => {
@@ -65,7 +72,8 @@ describe("idempotency guard", () => {
 
     // Request #1 starts — claims the in-progress lock.
     const first = await guard.check(tenantA, userA, requestId);
-    expect(first).toBeNull(); // got the lock
+    expect(first.status).toBe("acquired");
+    if (first.status !== "acquired") throw new Error("expected to acquire the lock");
 
     // Request #2 runs concurrently — must block until #1 stores a result.
     const secondPromise = guard.check(tenantA, userA, requestId);
@@ -80,12 +88,17 @@ describe("idempotency guard", () => {
     expect(quickResult.done).toBe(false);
 
     // Request #1 finishes.
-    await guard.store(tenantA, userA, requestId, { isSuccess: true, data: { id: 99 } });
+    await guard.store(tenantA, userA, requestId, first.token, {
+      isSuccess: true,
+      data: { id: 99 },
+    });
 
-    // Request #2 should now see the stored result, not null — no duplicate work.
+    // Request #2 should now see the stored result, not a fresh acquisition —
+    // no duplicate work.
     const second = await secondPromise;
-    expect(second).not.toBeNull();
-    expect(JSON.parse(second as string)).toEqual({ isSuccess: true, data: { id: 99 } });
+    expect(second.status).toBe("cached");
+    if (second.status !== "cached") throw new Error("expected cached value");
+    expect(JSON.parse(second.result)).toEqual({ isSuccess: true, data: { id: 99 } });
   });
 
   test("crashed handler: pending marker expires, next caller reclaims the lock", async () => {
@@ -97,11 +110,11 @@ describe("idempotency guard", () => {
     const requestId = "req-crashed";
 
     const first = await guard.check(tenantA, userA, requestId);
-    expect(first).toBeNull(); // we acquired the lock, then "crash" — never call store()
+    expect(first.status).toBe("acquired"); // we acquired the lock, then "crash" — never call store()
 
     // After the pending-TTL lapses, a retry should be allowed to take over.
     const second = await guard.check(tenantA, userA, requestId);
-    expect(second).toBeNull(); // reclaimed
+    expect(second.status).toBe("acquired"); // reclaimed
   });
 
   test("same requestId from different tenant/user does not hit the same cache entry", async () => {
@@ -112,22 +125,103 @@ describe("idempotency guard", () => {
 
     // Tenant A / user A owns the request and stores its result.
     const firstCheck = await guard.check(tenantA, userA, requestId);
-    expect(firstCheck).toBeNull();
-    await guard.store(tenantA, userA, requestId, { isSuccess: true, data: { tenant: "A" } });
+    expect(firstCheck.status).toBe("acquired");
+    if (firstCheck.status !== "acquired") throw new Error("expected to acquire the lock");
+    await guard.store(tenantA, userA, requestId, firstCheck.token, {
+      isSuccess: true,
+      data: { tenant: "A" },
+    });
 
     // Same requestId, different tenant+user: must be treated as a fresh
     // request, not see tenant A's cached/pending state.
     const otherTenantCheck = await guard.check(tenantB, userB, requestId);
-    expect(otherTenantCheck).toBeNull();
+    expect(otherTenantCheck.status).toBe("acquired");
 
     // Different user, same tenant: also isolated.
     const otherUserCheck = await guard.check(tenantA, userB, requestId);
-    expect(otherUserCheck).toBeNull();
+    expect(otherUserCheck.status).toBe("acquired");
 
     // Tenant A's own result is still retrievable and unaffected.
     const ownResult = await guard.check(tenantA, userA, requestId);
-    expect(ownResult).not.toBeNull();
-    expect(JSON.parse(ownResult as string)).toEqual({ isSuccess: true, data: { tenant: "A" } });
+    expect(ownResult.status).toBe("cached");
+    if (ownResult.status !== "cached") throw new Error("expected cached value");
+    expect(JSON.parse(ownResult.result)).toEqual({ isSuccess: true, data: { tenant: "A" } });
+  });
+
+  test("bug 1 — wait shorter than the pending lock no longer forces a duplicate re-run", async () => {
+    // Same inverted ratio as the pre-fix defaults (waitTimeoutMs < pendingTtl),
+    // scaled to sub-second so the test stays fast. Pre-fix, the internal
+    // waitTimeoutMs was trusted as-is: the waiter gives up at 100ms and
+    // reports "acquired" even though request #1 is still legitimately
+    // running and stores its result 200ms later — the double-execute bug.
+    // Post-fix, waitTimeoutMs is clamped to stay above pendingTtl, so the
+    // waiter keeps polling and observes the real result instead.
+    const guard = createIdempotencyGuard(testRedis.redis, {
+      pendingTtlSeconds: 1,
+      waitTimeoutMs: 100,
+      pollIntervalMs: 20,
+    });
+    const requestId = "req-bug1-inverted-timeout";
+
+    const first = await guard.check(tenantA, userA, requestId);
+    expect(first.status).toBe("acquired");
+    if (first.status !== "acquired") throw new Error("expected to acquire the lock");
+
+    // Request #1 is "slow" — stores well after the old 100ms wait window,
+    // but well within pendingTtl (1s).
+    const storeAfterDelay = (async () => {
+      await new Promise((r) => setTimeout(r, 300));
+      await guard.store(tenantA, userA, requestId, first.token, {
+        isSuccess: true,
+        data: { id: "slow-handler" },
+      });
+    })();
+
+    const second = await guard.check(tenantA, userA, requestId);
+    await storeAfterDelay;
+
+    // Must observe request #1's real result — never a second "acquired".
+    expect(second.status).toBe("cached");
+    if (second.status !== "cached") throw new Error("expected cached value, not a re-run");
+    expect(JSON.parse(second.result)).toEqual({ isSuccess: true, data: { id: "slow-handler" } });
+  });
+
+  test("bug 2 (window B) — a reclaimed lock's fresh result survives the original owner's stale store()", async () => {
+    const guard = createIdempotencyGuard(testRedis.redis, {
+      pendingTtlSeconds: 1, // expire fast so we can force a reclaim quickly
+      waitTimeoutMs: 6_000,
+      pollIntervalMs: 20,
+    });
+    const requestId = "req-bug2-window-b";
+
+    // Original owner acquires, then "hangs" (never stores) past pendingTtl.
+    const original = await guard.check(tenantA, userA, requestId);
+    expect(original.status).toBe("acquired");
+    if (original.status !== "acquired") throw new Error("expected to acquire the lock");
+
+    // Let the lock expire, then a second run reclaims it and finishes fast.
+    await new Promise((r) => setTimeout(r, 1100));
+    const reclaimer = await guard.check(tenantA, userA, requestId);
+    expect(reclaimer.status).toBe("acquired");
+    if (reclaimer.status !== "acquired") throw new Error("expected to reclaim the lock");
+    await guard.store(tenantA, userA, requestId, reclaimer.token, {
+      isSuccess: true,
+      data: { owner: "reclaimer" },
+    });
+
+    // The original (now-stale) run finally "finishes" and tries to store its
+    // own, outdated result using its original token.
+    await guard.store(tenantA, userA, requestId, original.token, {
+      isSuccess: true,
+      data: { owner: "original-stale" },
+    });
+
+    // The reclaimer's fresh result must survive — the stale store() must be
+    // a no-op, not a silent overwrite.
+    const final = await guard.check(tenantA, userA, requestId);
+    expect(final.status).toBe("cached");
+    if (final.status !== "cached") throw new Error("expected cached value");
+    expect(JSON.parse(final.result)).toEqual({ isSuccess: true, data: { owner: "reclaimer" } });
   });
 });
 

package/src/pipeline/dispatch-batch.ts

@@ -29,20 +29,26 @@ export async function runBatch(
 
   // Idempotency: if the same requestId has already been processed, return the
   // cached result without re-executing. The cache holds the full BatchResult.
+  // idempotencyToken is only set when we actually acquired the lock — the
+  // corrupted-cache fallthrough below leaves it unset, so finalize() skips
+  // store() rather than writing over an entry it never owned.
+  let idempotencyToken: string | undefined;
   if (requestId && idempotency) {
-    const cached = await idempotency.check(user.tenantId, user.id, requestId);
-    if (cached) {
-      const parsed = parseJsonSafe<BatchResult | null>(cached, null);
+    const checked = await idempotency.check(user.tenantId, user.id, requestId);
+    if (checked.status === "cached") {
+      const parsed = parseJsonSafe<BatchResult | null>(checked.result, null);
       if (parsed) return parsed;
       // corrupted cache entry — treat as miss, let the request re-run
+    } else {
+      idempotencyToken = checked.token;
     }
   }
 
   // Wrap return paths: cache the final result under requestId so retries get
   // the same answer (both success and failure results are cached).
   const finalize = async (result: BatchResult): Promise<BatchResult> => {
-    if (requestId && idempotency) {
-      await idempotency.store(user.tenantId, user.id, requestId, result);
+    if (requestId && idempotency && idempotencyToken) {
+      await idempotency.store(user.tenantId, user.id, requestId, idempotencyToken, result);
     }
     return result;
   };

package/src/pipeline/dispatch-shared.ts

@@ -157,19 +157,23 @@ async function appendDomainEvent(
 
 // r.systemScope() handlers must go through ctx.systemDb's guarded methods
 // (assertTenantMatch / acknowledgeCrossTenant) instead of plain ctx.db —
-// "system" mode has no tenant filter at all, so silent ctx.db use there is a
-// cross-tenant leak. HandlerContext.db has no way to become optional per
-// handler (isSystem is a runtime-only registry lookup, not a type-level
-// discriminant), so instead of omitting the key we hand back a Proxy that
-// throws on first touch, naming the handler and pointing at ctx.systemDb.
-function createSystemScopedDbGuard(handlerType: string): TenantDb {
+// "system" mode has no tenant filter at all, so silent ctx.db or
+// ctx.dbOutsideTransaction use there is a cross-tenant leak. HandlerContext
+// has no way to make either field optional per handler (isSystem is a
+// runtime-only registry lookup, not a type-level discriminant), so instead
+// of omitting the key we hand back a Proxy that throws on first touch,
+// naming the handler and pointing at the matching ctx.systemDb escape hatch.
+function createSystemScopedDbGuard(
+  handlerType: string,
+  fieldName: "db" | "dbOutsideTransaction",
+  hint: string,
+): TenantDb {
   return new Proxy({} as TenantDb, {
     get(_target, prop) {
       throw new InternalError({
         message:
-          `Handler "${handlerType}" is r.systemScope()'d — ctx.db is unavailable ` +
-          `(it would be unfiltered across every tenant). Use ` +
-          `ctx.systemDb.assertTenantMatch(...) or ctx.systemDb.acknowledgeCrossTenant(...) ` +
+          `Handler "${handlerType}" is r.systemScope()'d — ctx.${fieldName} is unavailable ` +
+          `(it would be unfiltered across every tenant). Use ${hint} ` +
           `instead (attempted to read "${String(prop)}").`,
       });
     },
@@ -204,19 +208,41 @@ export async function buildHandlerContext(
   // the client has disconnected — handlers with many sequential queries skip
   // the rest of the chain instead of burning DB-CPU for results no one reads.
   const db = dbSource ? buildTenantScopedDb(dbSource, reqCtx?.signal) : undefined;
-  const systemDb = isSystem && db ? createUncheckedSystemDb(db) : undefined;
-  // Exposed as ctx.db below — the internal `db` above stays the real,
-  // working TenantDb for this function's own use (config/derivatives
-  // accessors, systemDb construction).
-  const exposedDb = isSystem && db ? createSystemScopedDbGuard(type) : db;
   // Unbound pool, tenant-scoped like `db` but never tx-bound — writes
   // through it survive a rollback of the handler's own transaction. No
   // AbortSignal here: a client disconnect must not abort a durability write
-  // that is meant to outlive the request.
+  // that is meant to outlive the request. Computed before `systemDb` below
+  // so its guarded escape hatch (ctx.systemDb.outsideTransaction) can wrap
+  // this same TenantDb instead of a second, independently-built one.
   const outsideTxSource = resolveDbSource(ctx, undefined);
-  const dbOutsideTransaction = outsideTxSource
+  const rawDbOutsideTransaction = outsideTxSource
     ? buildTenantScopedDb(outsideTxSource, undefined)
     : undefined;
+  const systemDb =
+    isSystem && db ? createUncheckedSystemDb(db, rawDbOutsideTransaction) : undefined;
+  // Exposed as ctx.db below — the internal `db` above stays the real,
+  // working TenantDb for this function's own use (config/derivatives
+  // accessors, systemDb construction).
+  const exposedDb =
+    isSystem && db
+      ? createSystemScopedDbGuard(
+          type,
+          "db",
+          "ctx.systemDb.assertTenantMatch(...) or ctx.systemDb.acknowledgeCrossTenant(...)",
+        )
+      : db;
+  // Same fail-closed treatment as `exposedDb` — a system handler reaching
+  // ctx.dbOutsideTransaction directly would be a second, unguarded door to
+  // the same cross-tenant access ctx.db closes off above.
+  const dbOutsideTransaction =
+    isSystem && rawDbOutsideTransaction
+      ? createSystemScopedDbGuard(
+          type,
+          "dbOutsideTransaction",
+          "ctx.systemDb.outsideTransaction.assertTenantMatch(...) or " +
+            "ctx.systemDb.outsideTransaction.acknowledgeCrossTenant(...)",
+        )
+      : rawDbOutsideTransaction;
   const log = context.log?.child({
     handler: type,
     tenantId: user.tenantId,

package/src/pipeline/idempotency.ts

@@ -1,14 +1,31 @@
 import type Redis from "ioredis";
+import { InternalError } from "../errors";
+import { generateId } from "../utils";
 import { RedisKeys } from "./redis-keys";
 
+// Discriminated so a truthy "acquired" object can never be misread as a
+// cache hit by a callsite doing `if (result)` — the caller must switch on
+// `status`.
+export type IdempotencyCheckResult =
+  | { readonly status: "cached"; readonly result: string }
+  | { readonly status: "acquired"; readonly token: string };
+
 export type IdempotencyGuard = {
-  check(tenantId: string, userId: string, requestId: string): Promise<string | null>;
-  store(tenantId: string, userId: string, requestId: string, result: unknown): Promise<void>;
+  check(tenantId: string, userId: string, requestId: string): Promise<IdempotencyCheckResult>;
+  store(
+    tenantId: string,
+    userId: string,
+    requestId: string,
+    token: string,
+    result: unknown,
+  ): Promise<void>;
 };
 
-// Sentinel stored under the key while the handler is running. A second
-// request that sees this waits for the real result instead of racing.
-const PENDING_MARKER = "__pending__";
+// Sentinel prefix stored under the key while the handler is running. Each
+// acquisition appends a unique token so store() can later prove it still
+// owns the lock it started with (see storeScript below) instead of blindly
+// overwriting whatever is currently there.
+const PENDING_PREFIX = "__pending__:";
 
 export function createIdempotencyGuard(
   redis: Redis,
@@ -25,15 +42,40 @@ export function createIdempotencyGuard(
   // handler doesn't permanently block retries, long enough to cover normal
   // batch durations.
   const pendingTtl = options.pendingTtlSeconds ?? 30;
-  const waitTimeoutMs = options.waitTimeoutMs ?? 25_000;
+  // Must stay comfortably above pendingTtl: a wait shorter than the lock's
+  // own TTL makes a retry give up and re-run the handler while the original
+  // call is still legitimately in flight — the exact case this lock exists
+  // to prevent. Clamped rather than trusted so a misconfigured explicit
+  // option can't reintroduce that inversion.
+  const waitTimeoutMs = Math.max(options.waitTimeoutMs ?? 35_000, pendingTtl * 1000 + 5_000);
   const pollIntervalMs = options.pollIntervalMs ?? 50;
   const prefix = RedisKeys.idempotency;
 
+  // Atomic compare-and-swap: only persist the result if the key still holds
+  // the exact pending token this run acquired. If a parallel retry reclaimed
+  // an expired lock in the meantime, this is a no-op — the reclaiming run
+  // owns the key now and will persist the authoritative result itself.
+  const storeScript = `
+    if redis.call("get", KEYS[1]) == ARGV[1] then
+      redis.call("set", KEYS[1], ARGV[2], "EX", ARGV[3])
+      return 1
+    else
+      return 0
+    end
+  `;
+
+  async function tryAcquire(key: string): Promise<string | null> {
+    const token = `${PENDING_PREFIX}${generateId()}`;
+    const acquired = await redis.set(key, token, "EX", pendingTtl, "NX");
+    return acquired === "OK" ? token : null;
+  }
+
   return {
     // Returns:
-    //   null   — caller owns the in-progress lock, proceed to run the handler
-    //            and then call store() with the real result.
-    //   string — serialized cached result from a concurrent or prior request.
+    //   { status: "acquired", token } — caller owns the in-progress lock,
+    //     proceed to run the handler and then call store() with this token.
+    //   { status: "cached", result }  — serialized result from a concurrent
+    //     or prior request; do not run the handler.
     //
     // The old behaviour (pure GET + SET-NX-store) let two parallel requests
     // both see a cache miss, both execute side-effects, and only one persist
@@ -42,40 +84,59 @@ export function createIdempotencyGuard(
     async check(tenantId, userId, requestId) {
       const key = `${prefix}${tenantId}:${userId}:${requestId}`;
 
-      // Try to acquire the in-progress lock.
-      const acquired = await redis.set(key, PENDING_MARKER, "EX", pendingTtl, "NX");
-      if (acquired === "OK") return null;
+      const token = await tryAcquire(key);
+      if (token) return { status: "acquired", token };
 
       // Lost the race. Poll until the lock holder stores a result, or the
       // lock expires (handler crashed) and we can try again.
       const deadline = Date.now() + waitTimeoutMs;
       while (Date.now() < deadline) {
         const value = await redis.get(key);
-        if (value === null) {
-          // Lock expired before a result was stored — try to claim it
-          // ourselves and proceed as the new owner.
-          const reclaimed = await redis.set(key, PENDING_MARKER, "EX", pendingTtl, "NX");
-          if (reclaimed === "OK") return null;
+        if (value === null || value.startsWith(PENDING_PREFIX)) {
+          if (value === null) {
+            // Lock expired before a result was stored — try to claim it
+            // ourselves and proceed as the new owner.
+            const reclaimed = await tryAcquire(key);
+            if (reclaimed) return { status: "acquired", token: reclaimed };
+          }
+          await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
           continue;
         }
-        if (value !== PENDING_MARKER) return value;
-        await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+        return { status: "cached", result: value };
       }
 
-      // Gave up waiting. Treat as a fresh request — forces the caller to
-      // re-run the handler rather than hang indefinitely.
-      return null;
+      // Gave up waiting. By construction waitTimeoutMs > pendingTtl, so the
+      // lock must have expired already — make one last claim attempt rather
+      // than silently reporting ownership we never acquired (that would
+      // reintroduce the double-execute bug). If even that loses the race,
+      // fail loudly instead of running the handler a second time.
+      const finalValue = await redis.get(key);
+      if (finalValue !== null && !finalValue.startsWith(PENDING_PREFIX)) {
+        return { status: "cached", result: finalValue };
+      }
+      const lastResort = await tryAcquire(key);
+      if (lastResort) return { status: "acquired", token: lastResort };
+      throw new InternalError({
+        message: `idempotency lock contention: gave up waiting for requestId ${requestId}`,
+      });
     },
 
-    async store(tenantId, userId, requestId, result) {
-      // Overwrite the pending marker with the real result. Plain SET (no NX)
-      // on purpose: we own the lock; writing the result is the final step.
-      await redis.set(
-        `${prefix}${tenantId}:${userId}:${requestId}`,
+    async store(tenantId, userId, requestId, token, result) {
+      const key = `${prefix}${tenantId}:${userId}:${requestId}`;
+      // @cast-boundary db-operator — Lua EVAL return type is untyped in ioredis
+      const written = (await redis.eval(
+        storeScript,
+        1,
+        key,
+        token,
         JSON.stringify(result),
-        "EX",
-        ttl,
-      );
+        String(ttl),
+      )) as number;
+      // written === 0 means another process reclaimed this key after our
+      // lock expired and is (or already did) persist its own result —
+      // silently skipping here is the fix: the old code did an unconditional
+      // SET and could stomp that fresher result with our stale one.
+      void written;
     },
   };
 }