@cosmicdrift/kumiko-framework 0.305.0 → 0.306.0

package/src/pipeline/__tests__/redis-pipeline.integration.test.ts

@@ -223,6 +223,65 @@ describe("idempotency guard", () => {
     if (final.status !== "cached") throw new Error("expected cached value");
     expect(JSON.parse(final.result)).toEqual({ isSuccess: true, data: { owner: "reclaimer" } });
   });
+
+  test("release() frees the lock — a waiting check() reclaims immediately instead of polling out", async () => {
+    const guard = createIdempotencyGuard(testRedis.redis, {
+      pendingTtlSeconds: 5,
+      pollIntervalMs: 20,
+      waitTimeoutMs: 10_000,
+    });
+    const requestId = "req-release-1";
+
+    const first = await guard.check(tenantA, userA, requestId);
+    expect(first.status).toBe("acquired");
+    if (first.status !== "acquired") throw new Error("expected to acquire the lock");
+
+    // Waiter starts BEFORE release — if release is a no-op this only resolves
+    // once waitTimeoutMs elapses (10s), which the 500ms race below catches.
+    const waiterPromise = guard.check(tenantA, userA, requestId);
+
+    await guard.release(tenantA, userA, requestId, first.token);
+
+    const raced = await Promise.race([
+      waiterPromise.then((v) => ({ done: true as const, v })),
+      new Promise<{ done: false }>((r) => setTimeout(() => r({ done: false }), 500)),
+    ]);
+    expect(raced.done).toBe(true);
+    if (!raced.done) throw new Error("waiter did not reclaim after release()");
+    expect(raced.v.status).toBe("acquired");
+  });
+
+  test("release() with a stale token is a no-op — it must not clear a new owner's lock", async () => {
+    const guard = createIdempotencyGuard(testRedis.redis, {
+      pendingTtlSeconds: 1,
+      pollIntervalMs: 20,
+      waitTimeoutMs: 6_000,
+    });
+    const requestId = "req-release-stale";
+
+    const original = await guard.check(tenantA, userA, requestId);
+    expect(original.status).toBe("acquired");
+    if (original.status !== "acquired") throw new Error("expected to acquire the lock");
+
+    // Let the lock expire, then a new owner reclaims it.
+    await new Promise((r) => setTimeout(r, 1100));
+    const reclaimer = await guard.check(tenantA, userA, requestId);
+    expect(reclaimer.status).toBe("acquired");
+    if (reclaimer.status !== "acquired") throw new Error("expected to reclaim the lock");
+
+    // The original (stale) owner's release() must not touch the reclaimer's lock.
+    await guard.release(tenantA, userA, requestId, original.token);
+
+    await guard.store(tenantA, userA, requestId, reclaimer.token, {
+      isSuccess: true,
+      data: { owner: "reclaimer" },
+    });
+
+    const final = await guard.check(tenantA, userA, requestId);
+    expect(final.status).toBe("cached");
+    if (final.status !== "cached") throw new Error("expected cached value");
+    expect(JSON.parse(final.result)).toEqual({ isSuccess: true, data: { owner: "reclaimer" } });
+  });
 });
 
 // --- Event Dedup ---

package/src/pipeline/dispatch-batch.ts

@@ -1,3 +1,4 @@
+import { requestContext } from "../api/request-context";
 import type { DbConnection } from "../db/connection";
 import { transaction } from "../db/query";
 import type { DeleteContext, SaveContext, SessionUser, WriteResult } from "../engine/types";
@@ -22,6 +23,22 @@ export async function runBatch(
   commands: readonly BatchCommand[],
   user: SessionUser,
   requestId?: string,
+): Promise<BatchResult> {
+  const current = requestContext.get();
+  if (!current?.signal) {
+    return runBatchBody(ctx, commands, user, requestId);
+  }
+  // Strip the signal: a disconnect would roll back the tx, idempotency would
+  // cache a 500 for the uncommitted write and afterCommit effects would be lost.
+  const { signal: _signal, ...withoutSignal } = current;
+  return requestContext.run(withoutSignal, () => runBatchBody(ctx, commands, user, requestId));
+}
+
+async function runBatchBody(
+  ctx: DispatchContext,
+  commands: readonly BatchCommand[],
+  user: SessionUser,
+  requestId?: string,
 ): Promise<BatchResult> {
   const { idempotency, lifecycle, appContext: context } = ctx;
   if (commands.length === 0) {
@@ -45,8 +62,8 @@ export async function runBatch(
     }
   }
 
-  // Wrap return paths: cache the final result under requestId so retries get
-  // the same answer (both success and failure results are cached).
+  // Cache the result under requestId so retries get the same answer. Only a
+  // provably rolled-back 5xx releases the lock instead (releaseOrFinalize).
   const finalize = async (result: BatchResult): Promise<BatchResult> => {
     if (requestId && idempotency && idempotencyToken) {
       await idempotency.store(user.tenantId, user.id, requestId, idempotencyToken, result);
@@ -54,6 +71,19 @@ export async function runBatch(
     return result;
   };
 
+  // Never for the no-tx fallback: without a rollback, a re-run would repeat
+  // the side effects of the commands that already ran.
+  const releaseOrFinalize = async (
+    result: BatchResult,
+    isRetryableRollback: boolean,
+  ): Promise<BatchResult> => {
+    if (isRetryableRollback && requestId && idempotency && idempotencyToken) {
+      await idempotency.release(user.tenantId, user.id, requestId, idempotencyToken);
+      return result;
+    }
+    return finalize(result);
+  };
+
   const afterCommitHooks: AfterCommitHook[] = [];
   const results: WriteResult[] = [];
 
@@ -134,6 +164,7 @@ export async function runBatch(
     return finalize({ isSuccess: true, results });
   }
 
+  let transactionCallbackCompleted = false;
   try {
     await transaction(db, async (tx) => {
       for (let i = 0; i < commands.length; i++) {
@@ -153,22 +184,34 @@ export async function runBatch(
           throw new BatchRollback(i, res.error);
         }
       }
+      transactionCallbackCompleted = true;
     });
   } catch (e) {
     if (e instanceof BatchRollback) {
-      return finalize({
+      // Thrown inside the callback, so the tx rolled back. A 4xx is
+      // deterministic and stays cached; a 5xx may be transient.
+      return releaseOrFinalize(
+        {
+          isSuccess: false,
+          error: e.failureError,
+          failedIndex: e.failedIndex,
+          results,
+        },
+        e.failureError.httpStatus >= 500,
+      );
+    }
+    // A completed callback means the throw came from COMMIT (outcome unknown),
+    // so cache it; otherwise COMMIT was never sent and releasing is safe.
+    const error = toWriteErrorInfo(wrapToKumiko(e));
+    return releaseOrFinalize(
+      {
         isSuccess: false,
-        error: e.failureError,
-        failedIndex: e.failedIndex,
+        error,
+        failedIndex: results.length,
         results,
-      });
-    }
-    return finalize({
-      isSuccess: false,
-      error: toWriteErrorInfo(wrapToKumiko(e)),
-      failedIndex: results.length,
-      results,
-    });
+      },
+      !transactionCallbackCompleted && error.httpStatus >= 500,
+    );
   }
 
   // Commit succeeded — fire deferred side-effects.

package/src/pipeline/idempotency.ts

@@ -19,6 +19,7 @@ export type IdempotencyGuard = {
     token: string,
     result: unknown,
   ): Promise<void>;
+  release(tenantId: string, userId: string, requestId: string, token: string): Promise<void>;
 };
 
 // Sentinel prefix stored under the key while the handler is running. Each
@@ -64,6 +65,16 @@ export function createIdempotencyGuard(
     end
   `;
 
+  // Same CAS guard as storeScript: only clear the lock if we still own it.
+  // A stale token (lock already reclaimed by a new owner) is a no-op.
+  const releaseScript = `
+    if redis.call("get", KEYS[1]) == ARGV[1] then
+      return redis.call("del", KEYS[1])
+    else
+      return 0
+    end
+  `;
+
   async function tryAcquire(key: string): Promise<string | null> {
     const token = `${PENDING_PREFIX}${generateId()}`;
     const acquired = await redis.set(key, token, "EX", pendingTtl, "NX");
@@ -138,5 +149,10 @@ export function createIdempotencyGuard(
       // SET and could stomp that fresher result with our stale one.
       void written;
     },
+
+    async release(tenantId, userId, requestId, token) {
+      const key = `${prefix}${tenantId}:${userId}:${requestId}`;
+      await redis.eval(releaseScript, 1, key, token);
+    },
   };
 }

package/src/pipeline/system-identity-switch.ts

@@ -1,4 +1,9 @@
-import { type TenantDb, withUnsafeRawGrant } from "../db/tenant-db";
+import {
+  type TenantDb,
+  type UncheckedSystemDb,
+  withSystemDbUnsafeRawGrant,
+  withUnsafeRawGrant,
+} from "../db/tenant-db";
 import { SYSTEM_ROLE, SYSTEM_USER_ID } from "../engine/system-user";
 import type {
   ActiveMembershipResult,
@@ -214,7 +219,10 @@ function readIdentitySwitchFn<
 }
 
 // context's own keys only — never touch a property of the resolved value, which may be a Proxy that throws on any get.
-function readDbLikeValue(context: object, key: "db" | "dbOutsideTransaction"): object | undefined {
+function readDbLikeValue(
+  context: object,
+  key: "db" | "dbOutsideTransaction" | "systemDb",
+): object | undefined {
   if (!(key in context)) return undefined;
   const value = (context as Record<string, unknown>)[key];
   return typeof value === "object" && value !== null ? value : undefined;
@@ -340,6 +348,7 @@ export function withHookEscapeHatchGrant<TContext extends object>(
   const ctxQueryProjection = readIdentitySwitchFn<ProjectionReader>(context, "queryProjection");
   const ctxDb = readDbLikeValue(context, "db");
   const ctxDbOutsideTransaction = readDbLikeValue(context, "dbOutsideTransaction");
+  const ctxSystemDb = readDbLikeValue(context, "systemDb");
   if (
     !ctxQueryAs &&
     !ctxWriteAs &&
@@ -347,7 +356,8 @@ export function withHookEscapeHatchGrant<TContext extends object>(
     !ctxQueryAsMember &&
     !ctxQueryProjection &&
     !ctxDb &&
-    !ctxDbOutsideTransaction
+    !ctxDbOutsideTransaction &&
+    !ctxSystemDb
   ) {
     return context;
   }
@@ -366,10 +376,18 @@ export function withHookEscapeHatchGrant<TContext extends object>(
     ...(ctxDbOutsideTransaction && {
       dbOutsideTransaction: withUnsafeRawGrant(ctxDbOutsideTransaction as TenantDb, escapeHatch),
     }),
+    // @cast-boundary engine-bridge — withSystemDbUnsafeRawGrant passes non-UncheckedSystemDb values through unchanged.
+    ...(ctxSystemDb && {
+      systemDb: withSystemDbUnsafeRawGrant(
+        ctxSystemDb as UncheckedSystemDb,
+        escapeHatch,
+        callerLabel,
+      ),
+    }),
   };
 }
 
-// Re-gates a hook's own ctx.queryAs/ctx.writeAs/ctx.queryProjection/ctx.db/ctx.dbOutsideTransaction instead of inheriting the handler's grant.
+// Re-gates a hook's own ctx.queryAs/ctx.writeAs/ctx.queryProjection/ctx.db/ctx.dbOutsideTransaction/ctx.systemDb instead of inheriting the handler's grant.
 export function bindHookEscapeHatchGrant(
   fn: LifecycleHookFn,
   label: string,

package/src/testing/closed-connection-error.ts

@@ -0,0 +1,62 @@
+// Captures a real closed-connection error instead of a hand-built fake —
+// the production matcher checks driver-specific codes a fake can't reproduce.
+
+import postgres from "postgres";
+import { isClosedConnectionError } from "../bun-db/query";
+
+export function testDatabaseUrl(): string {
+  return (
+    process.env["TEST_DATABASE_URL"] ??
+    process.env["DATABASE_URL"] ??
+    "postgresql://kumiko:kumiko@localhost:15432/kumiko_test"
+  );
+}
+
+const MAX_WARMUP_ROUNDS = 10;
+const READS_PER_ROUND = 5;
+
+type AdminClient = { unsafe(sql: string, params?: readonly unknown[]): Promise<unknown> };
+
+// A per-round admin client hits a postgres-js reconnect timing bug (the first
+// retry after terminate hangs); a single long-lived admin client avoids it.
+export async function terminateBackendsByApplicationName(
+  admin: AdminClient,
+  applicationName: string,
+): Promise<void> {
+  await admin.unsafe(
+    "select pg_terminate_backend(pid) from pg_stat_activity where application_name = $1",
+    [applicationName],
+  );
+}
+
+// Terminates a throwaway pool's backends and races reads right after — the
+// dead-connection window is only a few ms wide, so this retries rounds until one lands.
+export async function captureClosedConnectionError(
+  url: string = testDatabaseUrl(),
+): Promise<unknown> {
+  const admin = postgres(url, { max: 1 });
+  try {
+    for (let round = 0; round < MAX_WARMUP_ROUNDS; round++) {
+      const applicationName = `kumiko-closed-conn-test-${crypto.randomUUID()}`;
+      const pool = postgres(url, { max: 1, connection: { application_name: applicationName } });
+      try {
+        await pool.unsafe("select 1");
+        await terminateBackendsByApplicationName(admin, applicationName);
+        for (let read = 0; read < READS_PER_ROUND; read++) {
+          try {
+            await pool.unsafe("select 1");
+          } catch (err) {
+            if (isClosedConnectionError(err)) return err;
+          }
+        }
+      } finally {
+        await pool.end({ timeout: 0 });
+      }
+    }
+    throw new Error(
+      `captureClosedConnectionError: no closed-connection error observed after ${MAX_WARMUP_ROUNDS} rounds.`,
+    );
+  } finally {
+    await admin.end({ timeout: 0 });
+  }
+}

package/src/testing/index.ts

@@ -15,6 +15,7 @@ export { resetEntityFieldEncryptionCacheForTests } from "../db/entity-field-encr
 export { rolesOf } from "./access-assertions";
 export { expectError, expectSuccess } from "./assertions";
 export { withBootValidatorFixture } from "./boot-validator-fixture";
+export { captureClosedConnectionError } from "./closed-connection-error";
 export { type ClearableTable, clearTables, resetTestTables } from "./db-cleanup";
 export {
   type E2EGeneratorOptions,

package/src/bun-db/__tests__/select-many-retry.test.ts

@@ -1,79 +0,0 @@
-// #1163: Bun.SQL can hand out a closed connection under load (AbortError
-// "The connection was closed."). Pure reads retry exactly once on a fresh
-// pool checkout; tx handles, non-matching errors, and genuine user aborts
-// must NOT retry.
-
-import { describe, expect, test } from "bun:test";
-import { buildEntityTable } from "../../db/table-builder";
-import { selectMany } from "../query";
-
-function closedConnectionError(): Error {
-  return Object.assign(new Error("The connection was closed."), { name: "AbortError" });
-}
-
-type FakeClient = {
-  unsafe: (sql: string, params?: readonly unknown[]) => Promise<readonly unknown[]>;
-  begin?: () => never;
-  calls: number;
-};
-
-function fakeClient(failures: Error[], opts: { tx?: boolean } = {}): FakeClient {
-  const remaining = [...failures];
-  const client: FakeClient = {
-    calls: 0,
-    unsafe: async () => {
-      client.calls++;
-      const err = remaining.shift();
-      if (err) throw err;
-      return [{ id: "r1", title: "ok", tenant_id: "t1", inserted_at: null, updated_at: null }];
-    },
-  };
-  // A top-level pool client has begin(); a transaction handle does not.
-  if (!opts.tx)
-    client.begin = () => {
-      throw new Error("not used in test");
-    };
-  return client;
-}
-
-const table = buildEntityTable("note", {
-  fields: { title: { type: "text", required: true } },
-});
-
-describe("selectMany — closed-connection retry (#1163)", () => {
-  test("retries once on AbortError 'connection was closed' and returns rows", async () => {
-    const db = fakeClient([closedConnectionError()]);
-    const rows = await selectMany(db, table);
-    expect(rows).toHaveLength(1);
-    expect(rows[0]?.title).toBe("ok");
-    expect(db.calls).toBe(2);
-  });
-
-  test("gives up after the single retry when the connection stays closed", async () => {
-    const db = fakeClient([closedConnectionError(), closedConnectionError()]);
-    await expect(selectMany(db, table)).rejects.toThrow("connection was closed");
-    expect(db.calls).toBe(2);
-  });
-
-  test("never retries on a transaction handle (no begin)", async () => {
-    const db = fakeClient([closedConnectionError()], { tx: true });
-    await expect(selectMany(db, table)).rejects.toThrow("connection was closed");
-    expect(db.calls).toBe(1);
-  });
-
-  test("does not retry a genuine user abort (different message)", async () => {
-    const userAbort = Object.assign(new Error("The operation was aborted."), {
-      name: "AbortError",
-    });
-    const db = fakeClient([userAbort]);
-    await expect(selectMany(db, table)).rejects.toThrow("operation was aborted");
-    expect(db.calls).toBe(1);
-  });
-
-  test("does not retry generic query errors", async () => {
-    const syntax = Object.assign(new Error("syntax error at or near"), { name: "PostgresError" });
-    const db = fakeClient([syntax]);
-    await expect(selectMany(db, table)).rejects.toThrow("syntax error");
-    expect(db.calls).toBe(1);
-  });
-});