@cosmicdrift/kumiko-bundled-features 0.205.0 → 0.206.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-bundled-features",
-  "version": "0.205.0",
+  "version": "0.206.0",
   "description": "Built-in features — tenant, user, auth, delivery. The stuff you'd rewrite anyway, already typed.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -126,12 +126,12 @@
     "./step-dispatcher": "./src/step-dispatcher/index.ts"
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-dispatcher-live": "0.205.0",
-    "@cosmicdrift/kumiko-framework": "0.205.0",
-    "@cosmicdrift/kumiko-headless": "0.205.0",
-    "@cosmicdrift/kumiko-renderer": "0.205.0",
-    "@cosmicdrift/kumiko-renderer-web": "0.205.0",
-    "@cosmicdrift/kumiko-types": "0.205.0",
+    "@cosmicdrift/kumiko-dispatcher-live": "0.206.0",
+    "@cosmicdrift/kumiko-framework": "0.206.0",
+    "@cosmicdrift/kumiko-headless": "0.206.0",
+    "@cosmicdrift/kumiko-renderer": "0.206.0",
+    "@cosmicdrift/kumiko-renderer-web": "0.206.0",
+    "@cosmicdrift/kumiko-types": "0.206.0",
     "@mollie/api-client": "^4.5.0",
     "@node-rs/argon2": "^2.0.2",
     "@types/mailparser": "^3.4.6",

package/src/inbound-mail-foundation/__tests__/inbound-mail-foundation.integration.test.ts

@@ -323,11 +323,11 @@ describe("scenario 2b: ingest-message — Thread-Rollup selbstkorrigierend statt
   });
 });
 
-describe("scenario 2c: ingest-message — concurrent thread-rollup race (issue #1229)", () => {
+describe("scenario 2c: ingest-message — concurrent thread-rollup race (issue #1229, closed by #2155)", () => {
   // Probabilistic — looped per house convention (mehrere Durchläufe, z.B.
-  // 20x): the version-conflict race between two step-5 thread-rollup
-  // appends only fires on genuine transaction-timing interleaving, which
-  // a single run can't force deterministically.
+  // 20x): whether a concurrent thread-rollup commit lands inside the
+  // step-5 critical section depends on real transaction-timing
+  // interleaving, which a single run can't force deterministically.
   const ITERATIONS = 20;
   const CONCURRENCY = 3;
 
@@ -352,10 +352,10 @@ describe("scenario 2c: ingest-message — concurrent thread-rollup race (issue #
         ),
       );
 
-      // No writer's transaction fails with a 500 from the thread-rollup
-      // version conflict — each of the CONCURRENCY distinct messages is
-      // its own aggregate, so step 4 never races; only step 5 (shared
-      // threadAggId) does, and the bounded retry loop must absorb it.
+      // No writer's transaction fails with a 500 — each of the CONCURRENCY
+      // distinct messages is its own aggregate, so step 4 never races; step
+      // 5 (shared threadAggId) is serialized by the pg_advisory_xact_lock,
+      // so there's nothing here for the bounded retry loop to absorb.
       for (const res of responses) {
         expect(res.status).toBe(200);
         const body = (await res.json()) as { isSuccess: boolean; data: { duplicate: boolean } };
@@ -366,8 +366,9 @@ describe("scenario 2c: ingest-message — concurrent thread-rollup race (issue #
       const threadAggId = mailThreadAggregateId(admin.tenantId, `mid:${rootMessageIdHeader}`);
       const threadRows = await selectMany(db, mailThreadsProjectionTable, { id: threadAggId });
       expect(threadRows).toHaveLength(1);
-      // Every message's contribution landed — no drift from a silently
-      // stale-read-and-succeed append (the bug the retry loop replaces).
+      // Every message's contribution landed — the advisory lock closes the
+      // TOCTOU gap between countWhere and append that used to let a stale
+      // messageCount succeed without ever surfacing as a version conflict.
       expect(threadRows[0]?.["messageCount"]).toBe(CONCURRENCY);
     }
   });

package/src/inbound-mail-foundation/handlers/ingest-message.write.ts

@@ -25,6 +25,7 @@ import {
   configuredPiiSubjectKms,
   encryptPiiFieldValues,
 } from "@cosmicdrift/kumiko-framework/crypto";
+import { acquireNamespacedAdvisoryLock } from "@cosmicdrift/kumiko-framework/db";
 import type { WriteHandlerDef } from "@cosmicdrift/kumiko-framework/engine";
 import { InternalError } from "@cosmicdrift/kumiko-framework/errors";
 import { Temporal } from "temporal-polyfill";
@@ -101,6 +102,10 @@ function buildThreadKey(p: IngestMessagePayload, effectiveMessageIdHeader: strin
 }
 
 const THREAD_ROLLUP_MAX_ATTEMPTS = 5;
+// pg_advisory_xact_lock namespace (int4) for the step-5 thread-rollup lock
+// — 'inbm' as ASCII, keeps it disjoint from the framework's other fixed
+// single-int advisory-lock keys (schema bootstrap, es-ops boot).
+const THREAD_ROLLUP_LOCK_NAMESPACE = 0x696e626d;
 export const ingestMessageHandler: WriteHandlerDef = {
   name: "ingest-message",
   schema: ingestMessageSchema,
@@ -230,35 +235,23 @@ export const ingestMessageHandler: WriteHandlerDef = {
     //    encrypted once up front (constant across retries, doesn't
     //    depend on the thread's prior state).
     //
-    //    messageCount deliberately does NOT come from a previously loaded
-    //    threadEvents snapshot (previousCount+1): two concurrent ingests
-    //    on the same thread (Watch-Push vs. Poll-Reconciliation overlap,
-    //    or two different messages of the same thread arriving at once)
-    //    can both read the same stream version before either commits.
-    //    tryAppendEvent's fresh version-read at append time makes the
-    //    loser's append fail with a VersionConflictError ONLY when the
-    //    other TX's commit lands between this reload and the append's own
-    //    version check — so we retry: reload the thread stream, recompute
-    //    the live row-count and lastMessageAt, and append again. If the
-    //    other TX instead commits in the gap between the countWhere below
-    //    and the append's version-read, the version check sees the
-    //    already-bumped stream and appends without conflict — but with the
-    //    stale row-count this countWhere captured before the concurrent
-    //    insert. That write still lands (messageCount can undercount by
-    //    one round), it is not eliminated by this retry loop; the next
-    //    ingest on the thread re-counts and self-corrects. Unlike step 4,
-    //    a lost version-conflict race here must NOT be treated as a no-op
+    //    messageCount is a live COUNT, not previousCount+1, so two
+    //    concurrent ingests on the same thread (Watch-Push vs.
+    //    Poll-Reconciliation overlap, or two different messages of the
+    //    same thread arriving at once) converge on the true count.
+    //    acquireNamespacedAdvisoryLock below (issue #2155) serializes this whole
+    //    countWhere+append section per threadAggId — transaction-scoped,
+    //    so it survives tryAppendEvent's inner SAVEPOINT and releases at
+    //    the enclosing TX's commit/rollback. Without it, a concurrent
+    //    commit landing between countWhere and tryAppendEvent's own fresh
+    //    version-read makes the append succeed with a stale count instead
+    //    of a VersionConflictError, so the retry loop below never sees it
+    //    as a conflict to retry (#1229's original gap). Unlike step 4, a
+    //    lost version-conflict race here must NOT be treated as a no-op
     //    duplicate — skipping would leave this message's contribution to
     //    messageCount/lastMessageAt out of the thread forever. Exhausting
     //    all attempts throws, rolling back the whole TX (including the
-    //    step-4 message append) for a clean re-ingest on the next poll —
-    //    never return success with a stale snapshot.
-    //
-    //    A true race-free read+append (e.g. SELECT ... FOR UPDATE or an
-    //    advisory lock on threadAggId) would close the countWhere→append
-    //    gap above and is deliberately not part of this fix — two writers
-    //    (watch + poll) make self-correcting drift an acceptable trade for
-    //    now.
+    //    step-4 message append) for a clean re-ingest.
     // ---------------------------------------------------------------
     const threadPlainPii = { tenantId, subject: payload.subject };
     const encryptedThreadFields = piiKms
@@ -274,6 +267,8 @@ export const ingestMessageHandler: WriteHandlerDef = {
         )
       : threadPlainPii;
 
+    await acquireNamespacedAdvisoryLock(ctx.db.raw, THREAD_ROLLUP_LOCK_NAMESPACE, threadAggId);
+
     let threadAppendOk = false;
     for (let attempt = 1; attempt <= THREAD_ROLLUP_MAX_ATTEMPTS && !threadAppendOk; attempt++) {
       const threadEvents = await ctx.loadAggregate(threadAggId);

package/src/user/__tests__/email-unique-blind-index.integration.test.ts

@@ -0,0 +1,140 @@
+// fw#2134 — user:create's email-uniqueness check used to be a pure
+// pre-flight fetchOne (TOCTOU: two concurrent creates can both pass the
+// pre-flight and both insert, see handlers/create.write.ts). This proves
+// the DB-level fix directly against the executor (bypassing the handler's
+// pre-flight entirely) — a unique index over email's blind-index column
+// rejects the loser of a genuine race, and the framework's F8 pg-23505
+// mapping turns that into a clean 409 unique_violation, not a 500.
+//
+// createTestDb wires no blind-index key by default (see
+// blind-index.integration.test.ts) — without configureBlindIndexKey the
+// email_bidx column stays NULL for every row, the partial bidx unique
+// index never applies, and a race "passing" here would prove nothing. So
+// this test configures one explicitly and then proves the loser was
+// actually rejected on the bidx constraint (not the plaintext-fallback
+// index) by asserting the constraint name and reading email_bidx back off
+// the surviving row.
+
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { asRawClient } from "@cosmicdrift/kumiko-framework/bun-db";
+import {
+  computeBlindIndex,
+  configureBlindIndexKey,
+  configurePiiSubjectKms,
+  decodeBlindIndexKey,
+  InMemoryKmsAdapter,
+} from "@cosmicdrift/kumiko-framework/crypto";
+import { createEventStoreExecutor, createTenantDb } from "@cosmicdrift/kumiko-framework/db";
+import { createSystemUser, SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
+import { createEventsTable } from "@cosmicdrift/kumiko-framework/event-store";
+import {
+  createTestDb,
+  type TestDb,
+  unsafeCreateEntityTable,
+} from "@cosmicdrift/kumiko-framework/stack";
+import {
+  resetBlindIndexKeyForTests,
+  resetPiiSubjectKmsForTests,
+} from "@cosmicdrift/kumiko-framework/testing";
+import { userEntity, userTable } from "../schema/user";
+
+const TEST_KEY_B64 = Buffer.alloc(32, 9).toString("base64");
+const TEST_KEY = decodeBlindIndexKey(TEST_KEY_B64);
+
+let testDb: TestDb;
+const executor = createEventStoreExecutor(userTable, userEntity, { entityName: "user" });
+
+beforeAll(async () => {
+  testDb = await createTestDb();
+  await unsafeCreateEntityTable(testDb.db, userEntity, "user");
+  await createEventsTable(testDb.db);
+});
+
+afterAll(async () => {
+  await testDb.cleanup();
+});
+
+beforeEach(async () => {
+  await asRawClient(testDb.db).unsafe(
+    `TRUNCATE kumiko_events, read_users RESTART IDENTITY CASCADE`,
+  );
+  configurePiiSubjectKms(new InMemoryKmsAdapter());
+  configureBlindIndexKey(TEST_KEY_B64);
+});
+
+afterEach(() => {
+  resetPiiSubjectKmsForTests();
+  resetBlindIndexKeyForTests();
+});
+
+describe("fw#2134 — email unique constraint on the blind-index column", () => {
+  test("two concurrent creates with the same email: exactly one wins, loser fails on the bidx constraint", async () => {
+    const email = "race@example.com";
+    const tdb = createTenantDb(testDb.db, SYSTEM_TENANT_ID, "system");
+    const systemUser = createSystemUser(SYSTEM_TENANT_ID);
+
+    const [first, second] = await Promise.all([
+      executor.create({ email, displayName: "Racer One" }, systemUser, tdb),
+      executor.create({ email, displayName: "Racer Two" }, systemUser, tdb),
+    ]);
+
+    const winner = first.isSuccess ? first : second;
+    const loser = first.isSuccess ? second : first;
+    if (loser.isSuccess || !winner.isSuccess) {
+      throw new Error("expected exactly one winner and one loser out of the two racing creates");
+    }
+
+    expect(loser.error.code).toBe("unique_violation");
+    expect(loser.error.httpStatus).toBe(409);
+    const details = loser.error.details as { constraintName?: string };
+    expect(details.constraintName).toBe("read_users_email_unique_bidx");
+
+    // Proves the race was actually decided by the blind-index constraint,
+    // not the plaintext-fallback index: the surviving row's bidx column
+    // carries the deterministic HMAC of the email.
+    const rows = (await asRawClient(testDb.db).unsafe(
+      `SELECT "email_bidx" FROM "read_users" WHERE "id" = $1::uuid`,
+      [winner.data.id],
+    )) as ReadonlyArray<{ email_bidx: string | null }>;
+    expect(rows[0]?.email_bidx).toBe(computeBlindIndex(TEST_KEY, email));
+
+    // DB-proof: only one row actually survives under that blind index.
+    const survivors = (await asRawClient(testDb.db).unsafe(
+      `SELECT count(*)::int AS n FROM "read_users" WHERE "email_bidx" = $1`,
+      [computeBlindIndex(TEST_KEY, email)],
+    )) as ReadonlyArray<{ n: number }>;
+    expect(survivors[0]?.n).toBe(1);
+  });
+
+  test("a soft-deleted user still holds its email: constraint agrees with the pre-existing pre-flight behavior", async () => {
+    // Soft-delete only flips isDeleted/deletedAt (apply-entity-event.ts) —
+    // it never touches email/email_bidx, so the row still occupies the
+    // unique slot. This isn't new behavior from fw#2134: the handler's
+    // pre-flight fetchOne is a raw, unfiltered query (see create.write.ts)
+    // and already saw soft-deleted rows before this fix. This test pins
+    // that the new DB constraint doesn't diverge from that — only the
+    // GDPR "forgotten" hard-delete (see forget-cleanup) actually frees the
+    // email slot.
+    const email = "soft-deleted@example.com";
+    const tdb = createTenantDb(testDb.db, SYSTEM_TENANT_ID, "system");
+    const systemUser = createSystemUser(SYSTEM_TENANT_ID);
+
+    const created = await executor.create(
+      { email, displayName: "Departing User" },
+      systemUser,
+      tdb,
+    );
+    if (!created.isSuccess) throw new Error("expected create to succeed");
+
+    const deleted = await executor.delete({ id: created.data.id }, systemUser, tdb);
+    if (!deleted.isSuccess) throw new Error("expected soft-delete to succeed");
+
+    const reCreated = await executor.create({ email, displayName: "Squatter" }, systemUser, tdb);
+    if (reCreated.isSuccess) {
+      throw new Error("expected re-create with the same email to be rejected while soft-deleted");
+    }
+    expect(reCreated.error.code).toBe("unique_violation");
+    const details = reCreated.error.details as { constraintName?: string };
+    expect(details.constraintName).toBe("read_users_email_unique_bidx");
+  });
+});

package/src/user/__tests__/user.integration.test.ts

@@ -107,6 +107,37 @@ describe("scenario 1: create + me", () => {
     );
     expectErrorIncludes(error, UserErrors.emailAlreadyExists);
   });
+
+  // fw#2134 — the pre-flight fetchOne alone can't decide a genuine race
+  // (both requests can see "no duplicate" before either commits); this
+  // fires two real concurrent HTTP creates and checks the loser still
+  // gets a clean 4xx, never an unhandled 500. The DB-constraint-level
+  // proof (which layer actually catches the race, and that it's the
+  // blind-index column doing it) lives in
+  // email-unique-blind-index.integration.test.ts — this test's job is
+  // narrower: no request may 500 no matter who wins.
+  test("two concurrent creates with the same email: loser gets a clean 4xx, not a 500", async () => {
+    const email = "concurrent@example.com";
+    const [resA, resB] = await Promise.all([
+      stack.http.write(UserHandlers.create, { email, displayName: "Racer A" }, systemAdmin),
+      stack.http.write(UserHandlers.create, { email, displayName: "Racer B" }, systemAdmin),
+    ]);
+
+    const bodyA = (await resA.json()) as { isSuccess: boolean };
+    const bodyB = (await resB.json()) as { isSuccess: boolean };
+    const successes = [bodyA, bodyB].filter((b) => b.isSuccess === true);
+    const failures = [
+      { status: resA.status, body: bodyA },
+      { status: resB.status, body: bodyB },
+    ].filter((r) => r.body.isSuccess === false);
+
+    expect(successes).toHaveLength(1);
+    expect(failures).toHaveLength(1);
+    const loser = failures[0];
+    if (!loser) throw new Error("expected exactly one failing response");
+    expect(loser.status).toBeGreaterThanOrEqual(400);
+    expect(loser.status).toBeLessThan(500);
+  });
 });
 
 // --- Scenario 2: field-level read access hides passwordHash ---

package/src/user/handlers/create.write.ts

@@ -1,7 +1,12 @@
 import { fetchOne } from "@cosmicdrift/kumiko-framework/bun-db";
 import { createEventStoreExecutor } from "@cosmicdrift/kumiko-framework/db";
 import { defineWriteHandler } from "@cosmicdrift/kumiko-framework/engine";
-import { ConflictError, InternalError, writeFailure } from "@cosmicdrift/kumiko-framework/errors";
+import {
+  ConflictError,
+  InternalError,
+  type WriteErrorInfo,
+  writeFailure,
+} from "@cosmicdrift/kumiko-framework/errors";
 import { isValidIanaTimeZone } from "@cosmicdrift/kumiko-framework/time";
 import { z } from "zod";
 import { UserErrors } from "../constants";
@@ -11,12 +16,16 @@ const crud = createEventStoreExecutor(userTable, userEntity, { entityName: "user
 
 // Only the Auth features (running as SYSTEM) or a SystemAdmin may create users.
 //
-// Email uniqueness is checked via a pre-flight query — the framework has no
-// `unique:` field flag yet. This check is race-prone: two concurrent requests
-// can both see "no duplicate" and both insert. Acceptable MVP behavior since
-// user creation is low-frequency and gated by privileged roles; the DB will
-// still surface a pg unique violation once we add the constraint.
-// TODO: replace with a real `unique:` field flag + DB constraint.
+// Email uniqueness has two layers (fw#2134):
+//  1. A pre-flight fetchOne — fast-fails with a friendly error on the
+//     common case, but is race-prone on its own: two concurrent requests
+//     can both see "no duplicate" and both proceed to crud.create.
+//  2. The real guard is the unique index over email's blind-index column
+//     (schema/user.ts). A losing concurrent create hits that constraint
+//     and crud.create returns a UniqueViolationError (framework's F8
+//     pg-23505 mapping) instead of throwing — remapped below to the same
+//     emailAlreadyExists shape the pre-flight path returns, so callers see
+//     one consistent error regardless of which layer caught the race.
 export const createWrite = defineWriteHandler({
   name: "user:create",
   schema: z.object({
@@ -56,6 +65,43 @@ export const createWrite = defineWriteHandler({
       );
     }
 
-    return crud.create(event.payload, event.user, db);
+    const result = await crud.create(event.payload, event.user, db);
+    if (!result.isSuccess && isEmailUniqueViolation(result.error)) {
+      return writeFailure(
+        new ConflictError({
+          message: "email already exists",
+          i18nKey: "user.errors.emailAlreadyExists",
+          details: {
+            reason: UserErrors.emailAlreadyExists,
+            field: "email",
+            constraintName: constraintNameOf(result.error),
+          },
+        }),
+      );
+    }
+    return result;
   },
 });
+
+// schema/user.ts's `read_users_email_unique` (+ its generated `_bidx`
+// pendant) is the actual race-safety net behind the pre-flight check
+// above — a losing concurrent create surfaces here as a generic
+// UniqueViolationError (framework F8 pg-23505 mapping). Only the email
+// constraint gets remapped; any other unique_violation on this entity
+// passes through unchanged.
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null;
+}
+
+function constraintNameOf(error: WriteErrorInfo): string | undefined {
+  if (!isRecord(error.details)) return undefined;
+  const constraintName = error.details["constraintName"];
+  return typeof constraintName === "string" ? constraintName : undefined;
+}
+
+function isEmailUniqueViolation(error: WriteErrorInfo): boolean {
+  return (
+    error.code === "unique_violation" &&
+    (constraintNameOf(error)?.startsWith("read_users_email_unique") ?? false)
+  );
+}

package/src/user/schema/user.ts

@@ -169,6 +169,17 @@ export const userEntity = createEntity({
       access: { write: access.privileged },
     }),
   },
+  // fw#2134 — email is `pii: true, lookupable: true`, so the column holds
+  // per-row ciphertext: a plain unique index on it can't catch duplicates.
+  // `unique: true` over a lookupable column makes buildEntityTable /
+  // deriveEntityTableMeta emit a second, partial unique index over the
+  // deterministic `emailBidx` companion column (WHERE email_bidx IS NOT
+  // NULL — erased/key-less rows stay excluded), which is what actually
+  // closes the TOCTOU race between two concurrent user:create calls.
+  // Global (not tenant-scoped): user is a tenant-agnostic identity
+  // aggregate (systemStream: true) — a composite (tenantId, email) would
+  // leave the same email resolvable twice.
+  indexes: [{ columns: ["email"], unique: true, name: "read_users_email_unique" }],
 });
 
 export const userTable = buildEntityTable("user", userEntity);