@cosmicdrift/kumiko-framework 0.200.0 → 0.201.0

package/src/pipeline/dispatch-shared.ts

@@ -157,19 +157,23 @@ async function appendDomainEvent(
 
 // r.systemScope() handlers must go through ctx.systemDb's guarded methods
 // (assertTenantMatch / acknowledgeCrossTenant) instead of plain ctx.db —
-// "system" mode has no tenant filter at all, so silent ctx.db use there is a
-// cross-tenant leak. HandlerContext.db has no way to become optional per
-// handler (isSystem is a runtime-only registry lookup, not a type-level
-// discriminant), so instead of omitting the key we hand back a Proxy that
-// throws on first touch, naming the handler and pointing at ctx.systemDb.
-function createSystemScopedDbGuard(handlerType: string): TenantDb {
+// "system" mode has no tenant filter at all, so silent ctx.db or
+// ctx.dbOutsideTransaction use there is a cross-tenant leak. HandlerContext
+// has no way to make either field optional per handler (isSystem is a
+// runtime-only registry lookup, not a type-level discriminant), so instead
+// of omitting the key we hand back a Proxy that throws on first touch,
+// naming the handler and pointing at the matching ctx.systemDb escape hatch.
+function createSystemScopedDbGuard(
+  handlerType: string,
+  fieldName: "db" | "dbOutsideTransaction",
+  hint: string,
+): TenantDb {
   return new Proxy({} as TenantDb, {
     get(_target, prop) {
       throw new InternalError({
         message:
-          `Handler "${handlerType}" is r.systemScope()'d — ctx.db is unavailable ` +
-          `(it would be unfiltered across every tenant). Use ` +
-          `ctx.systemDb.assertTenantMatch(...) or ctx.systemDb.acknowledgeCrossTenant(...) ` +
+          `Handler "${handlerType}" is r.systemScope()'d — ctx.${fieldName} is unavailable ` +
+          `(it would be unfiltered across every tenant). Use ${hint} ` +
           `instead (attempted to read "${String(prop)}").`,
       });
     },
@@ -204,19 +208,41 @@ export async function buildHandlerContext(
   // the client has disconnected — handlers with many sequential queries skip
   // the rest of the chain instead of burning DB-CPU for results no one reads.
   const db = dbSource ? buildTenantScopedDb(dbSource, reqCtx?.signal) : undefined;
-  const systemDb = isSystem && db ? createUncheckedSystemDb(db) : undefined;
-  // Exposed as ctx.db below — the internal `db` above stays the real,
-  // working TenantDb for this function's own use (config/derivatives
-  // accessors, systemDb construction).
-  const exposedDb = isSystem && db ? createSystemScopedDbGuard(type) : db;
   // Unbound pool, tenant-scoped like `db` but never tx-bound — writes
   // through it survive a rollback of the handler's own transaction. No
   // AbortSignal here: a client disconnect must not abort a durability write
-  // that is meant to outlive the request.
+  // that is meant to outlive the request. Computed before `systemDb` below
+  // so its guarded escape hatch (ctx.systemDb.outsideTransaction) can wrap
+  // this same TenantDb instead of a second, independently-built one.
   const outsideTxSource = resolveDbSource(ctx, undefined);
-  const dbOutsideTransaction = outsideTxSource
+  const rawDbOutsideTransaction = outsideTxSource
     ? buildTenantScopedDb(outsideTxSource, undefined)
     : undefined;
+  const systemDb =
+    isSystem && db ? createUncheckedSystemDb(db, rawDbOutsideTransaction) : undefined;
+  // Exposed as ctx.db below — the internal `db` above stays the real,
+  // working TenantDb for this function's own use (config/derivatives
+  // accessors, systemDb construction).
+  const exposedDb =
+    isSystem && db
+      ? createSystemScopedDbGuard(
+          type,
+          "db",
+          "ctx.systemDb.assertTenantMatch(...) or ctx.systemDb.acknowledgeCrossTenant(...)",
+        )
+      : db;
+  // Same fail-closed treatment as `exposedDb` — a system handler reaching
+  // ctx.dbOutsideTransaction directly would be a second, unguarded door to
+  // the same cross-tenant access ctx.db closes off above.
+  const dbOutsideTransaction =
+    isSystem && rawDbOutsideTransaction
+      ? createSystemScopedDbGuard(
+          type,
+          "dbOutsideTransaction",
+          "ctx.systemDb.outsideTransaction.assertTenantMatch(...) or " +
+            "ctx.systemDb.outsideTransaction.acknowledgeCrossTenant(...)",
+        )
+      : rawDbOutsideTransaction;
   const log = context.log?.child({
     handler: type,
     tenantId: user.tenantId,

package/src/pipeline/idempotency.ts

@@ -1,14 +1,31 @@
 import type Redis from "ioredis";
+import { InternalError } from "../errors";
+import { generateId } from "../utils";
 import { RedisKeys } from "./redis-keys";
 
+// Discriminated so a truthy "acquired" object can never be misread as a
+// cache hit by a callsite doing `if (result)` — the caller must switch on
+// `status`.
+export type IdempotencyCheckResult =
+  | { readonly status: "cached"; readonly result: string }
+  | { readonly status: "acquired"; readonly token: string };
+
 export type IdempotencyGuard = {
-  check(tenantId: string, userId: string, requestId: string): Promise<string | null>;
-  store(tenantId: string, userId: string, requestId: string, result: unknown): Promise<void>;
+  check(tenantId: string, userId: string, requestId: string): Promise<IdempotencyCheckResult>;
+  store(
+    tenantId: string,
+    userId: string,
+    requestId: string,
+    token: string,
+    result: unknown,
+  ): Promise<void>;
 };
 
-// Sentinel stored under the key while the handler is running. A second
-// request that sees this waits for the real result instead of racing.
-const PENDING_MARKER = "__pending__";
+// Sentinel prefix stored under the key while the handler is running. Each
+// acquisition appends a unique token so store() can later prove it still
+// owns the lock it started with (see storeScript below) instead of blindly
+// overwriting whatever is currently there.
+const PENDING_PREFIX = "__pending__:";
 
 export function createIdempotencyGuard(
   redis: Redis,
@@ -25,15 +42,40 @@ export function createIdempotencyGuard(
   // handler doesn't permanently block retries, long enough to cover normal
   // batch durations.
   const pendingTtl = options.pendingTtlSeconds ?? 30;
-  const waitTimeoutMs = options.waitTimeoutMs ?? 25_000;
+  // Must stay comfortably above pendingTtl: a wait shorter than the lock's
+  // own TTL makes a retry give up and re-run the handler while the original
+  // call is still legitimately in flight — the exact case this lock exists
+  // to prevent. Clamped rather than trusted so a misconfigured explicit
+  // option can't reintroduce that inversion.
+  const waitTimeoutMs = Math.max(options.waitTimeoutMs ?? 35_000, pendingTtl * 1000 + 5_000);
   const pollIntervalMs = options.pollIntervalMs ?? 50;
   const prefix = RedisKeys.idempotency;
 
+  // Atomic compare-and-swap: only persist the result if the key still holds
+  // the exact pending token this run acquired. If a parallel retry reclaimed
+  // an expired lock in the meantime, this is a no-op — the reclaiming run
+  // owns the key now and will persist the authoritative result itself.
+  const storeScript = `
+    if redis.call("get", KEYS[1]) == ARGV[1] then
+      redis.call("set", KEYS[1], ARGV[2], "EX", ARGV[3])
+      return 1
+    else
+      return 0
+    end
+  `;
+
+  async function tryAcquire(key: string): Promise<string | null> {
+    const token = `${PENDING_PREFIX}${generateId()}`;
+    const acquired = await redis.set(key, token, "EX", pendingTtl, "NX");
+    return acquired === "OK" ? token : null;
+  }
+
   return {
     // Returns:
-    //   null   — caller owns the in-progress lock, proceed to run the handler
-    //            and then call store() with the real result.
-    //   string — serialized cached result from a concurrent or prior request.
+    //   { status: "acquired", token } — caller owns the in-progress lock,
+    //     proceed to run the handler and then call store() with this token.
+    //   { status: "cached", result }  — serialized result from a concurrent
+    //     or prior request; do not run the handler.
     //
     // The old behaviour (pure GET + SET-NX-store) let two parallel requests
     // both see a cache miss, both execute side-effects, and only one persist
@@ -42,40 +84,59 @@ export function createIdempotencyGuard(
     async check(tenantId, userId, requestId) {
       const key = `${prefix}${tenantId}:${userId}:${requestId}`;
 
-      // Try to acquire the in-progress lock.
-      const acquired = await redis.set(key, PENDING_MARKER, "EX", pendingTtl, "NX");
-      if (acquired === "OK") return null;
+      const token = await tryAcquire(key);
+      if (token) return { status: "acquired", token };
 
       // Lost the race. Poll until the lock holder stores a result, or the
       // lock expires (handler crashed) and we can try again.
       const deadline = Date.now() + waitTimeoutMs;
       while (Date.now() < deadline) {
         const value = await redis.get(key);
-        if (value === null) {
-          // Lock expired before a result was stored — try to claim it
-          // ourselves and proceed as the new owner.
-          const reclaimed = await redis.set(key, PENDING_MARKER, "EX", pendingTtl, "NX");
-          if (reclaimed === "OK") return null;
+        if (value === null || value.startsWith(PENDING_PREFIX)) {
+          if (value === null) {
+            // Lock expired before a result was stored — try to claim it
+            // ourselves and proceed as the new owner.
+            const reclaimed = await tryAcquire(key);
+            if (reclaimed) return { status: "acquired", token: reclaimed };
+          }
+          await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
           continue;
         }
-        if (value !== PENDING_MARKER) return value;
-        await new Promise((resolve) => setTimeout(resolve, pollIntervalMs));
+        return { status: "cached", result: value };
       }
 
-      // Gave up waiting. Treat as a fresh request — forces the caller to
-      // re-run the handler rather than hang indefinitely.
-      return null;
+      // Gave up waiting. By construction waitTimeoutMs > pendingTtl, so the
+      // lock must have expired already — make one last claim attempt rather
+      // than silently reporting ownership we never acquired (that would
+      // reintroduce the double-execute bug). If even that loses the race,
+      // fail loudly instead of running the handler a second time.
+      const finalValue = await redis.get(key);
+      if (finalValue !== null && !finalValue.startsWith(PENDING_PREFIX)) {
+        return { status: "cached", result: finalValue };
+      }
+      const lastResort = await tryAcquire(key);
+      if (lastResort) return { status: "acquired", token: lastResort };
+      throw new InternalError({
+        message: `idempotency lock contention: gave up waiting for requestId ${requestId}`,
+      });
     },
 
-    async store(tenantId, userId, requestId, result) {
-      // Overwrite the pending marker with the real result. Plain SET (no NX)
-      // on purpose: we own the lock; writing the result is the final step.
-      await redis.set(
-        `${prefix}${tenantId}:${userId}:${requestId}`,
+    async store(tenantId, userId, requestId, token, result) {
+      const key = `${prefix}${tenantId}:${userId}:${requestId}`;
+      // @cast-boundary db-operator — Lua EVAL return type is untyped in ioredis
+      const written = (await redis.eval(
+        storeScript,
+        1,
+        key,
+        token,
         JSON.stringify(result),
-        "EX",
-        ttl,
-      );
+        String(ttl),
+      )) as number;
+      // written === 0 means another process reclaimed this key after our
+      // lock expired and is (or already did) persist its own result —
+      // silently skipping here is the fix: the old code did an unconditional
+      // SET and could stomp that fresher result with our stale one.
+      void written;
     },
   };
 }