@cosmicdrift/kumiko-bundled-features 0.165.3 → 0.166.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cosmicdrift/kumiko-bundled-features",
-  "version": "0.165.3",
+  "version": "0.166.0",
   "description": "Built-in features — tenant, user, auth, delivery. The stuff you'd rewrite anyway, already typed.",
   "license": "BUSL-1.1",
   "author": "Marc Frost <marc@cosmicdriftgamestudio.com>",
@@ -119,11 +119,11 @@
     "./step-dispatcher": "./src/step-dispatcher/index.ts"
   },
   "dependencies": {
-    "@cosmicdrift/kumiko-dispatcher-live": "0.165.3",
-    "@cosmicdrift/kumiko-framework": "0.165.3",
-    "@cosmicdrift/kumiko-headless": "0.165.3",
-    "@cosmicdrift/kumiko-renderer": "0.165.3",
-    "@cosmicdrift/kumiko-renderer-web": "0.165.3",
+    "@cosmicdrift/kumiko-dispatcher-live": "0.166.0",
+    "@cosmicdrift/kumiko-framework": "0.166.0",
+    "@cosmicdrift/kumiko-headless": "0.166.0",
+    "@cosmicdrift/kumiko-renderer": "0.166.0",
+    "@cosmicdrift/kumiko-renderer-web": "0.166.0",
     "@mollie/api-client": "^4.5.0",
     "imapflow": "^1.3.3",
     "mailparser": "^3.9.8",
@@ -148,7 +148,7 @@
     "LICENSE"
   ],
   "peerDependencies": {
-    "@cosmicdrift/kumiko-types": "^0.165.3"
+    "@cosmicdrift/kumiko-types": "^0.166.0"
   },
   "devDependencies": {
     "@testing-library/user-event": "^14.6.1",

package/src/auth-mfa/__tests__/token-secrets.test.ts

@@ -0,0 +1,40 @@
+import { describe, expect, test } from "bun:test";
+import { resolveMfaTokenSecrets } from "../token-secrets";
+
+const JWT_SECRET = "app-jwt-secret-for-tests";
+
+describe("resolveMfaTokenSecrets", () => {
+  // A setup token proves "enrolling a factor", a challenge token proves
+  // "passed step one of login". One shared key would let the first be
+  // replayed as the second.
+  test("derives two different secrets", () => {
+    const { setupTokenSecret, challengeTokenSecret } = resolveMfaTokenSecrets(JWT_SECRET);
+
+    expect(setupTokenSecret).not.toBe(challengeTokenSecret);
+    expect(setupTokenSecret).not.toBe(JWT_SECRET);
+  });
+
+  // The reason this helper exists: prod entrypoint and dev server each resolve
+  // the secrets, and a purpose string drifting between them would invalidate
+  // every token minted by the other.
+  test("is stable across calls, so two boot files agree", () => {
+    expect(resolveMfaTokenSecrets(JWT_SECRET)).toEqual(resolveMfaTokenSecrets(JWT_SECRET));
+  });
+
+  test("an explicit override wins over derivation", () => {
+    const resolved = resolveMfaTokenSecrets(JWT_SECRET, { setupTokenSecret: "explicit-setup" });
+
+    expect(resolved.setupTokenSecret).toBe("explicit-setup");
+    expect(resolved.challengeTokenSecret).toBe(
+      resolveMfaTokenSecrets(JWT_SECRET).challengeTokenSecret,
+    );
+  });
+
+  test("rotating the master rotates both", () => {
+    const before = resolveMfaTokenSecrets(JWT_SECRET);
+    const after = resolveMfaTokenSecrets(`${JWT_SECRET}-rotated`);
+
+    expect(after.setupTokenSecret).not.toBe(before.setupTokenSecret);
+    expect(after.challengeTokenSecret).not.toBe(before.challengeTokenSecret);
+  });
+});

package/src/auth-mfa/index.ts

@@ -17,3 +17,8 @@ export {
 } from "./feature";
 export type { MfaStatusChecker, MfaStatusCheckResult } from "./mfa-status-checker";
 export { userMfaEntity, userMfaTable } from "./schema/user-mfa";
+export {
+  type MfaTokenSecretOverrides,
+  type ResolvedMfaTokenSecrets,
+  resolveMfaTokenSecrets,
+} from "./token-secrets";

package/src/auth-mfa/token-secrets.ts

@@ -0,0 +1,39 @@
+import { derivePurposeSecret } from "@cosmicdrift/kumiko-framework/secrets";
+
+export type MfaTokenSecretOverrides = {
+  readonly setupTokenSecret?: string;
+  readonly challengeTokenSecret?: string;
+};
+
+export type ResolvedMfaTokenSecrets = {
+  readonly setupTokenSecret: string;
+  readonly challengeTokenSecret: string;
+};
+
+// The two HKDF purposes auth-mfa needs. They live here rather than at each
+// call site because an app typically resolves them twice — once in the prod
+// entrypoint against a validated JWT_SECRET, once in the dev server against
+// its fallback — and a purpose string that drifts between those two files
+// invalidates every token issued by the other.
+//
+// Setup and challenge stay separate on purpose: a setup token proves "this
+// user is enrolling a factor", a challenge token proves "this user passed
+// step one of login". Sharing one key would let the first be replayed as the
+// second.
+const SETUP_TOKEN_PURPOSE = "mfa-setup-token-v1";
+const CHALLENGE_TOKEN_PURPOSE = "mfa-challenge-token-v1";
+
+/** Derives both MFA token secrets from the app's master secret. Pass explicit
+ *  overrides only when a deployment needs its own key for one of them —
+ *  otherwise deriving keeps a single env var authoritative. */
+export function resolveMfaTokenSecrets(
+  masterSecret: string,
+  overrides: MfaTokenSecretOverrides = {},
+): ResolvedMfaTokenSecrets {
+  return {
+    setupTokenSecret:
+      overrides.setupTokenSecret ?? derivePurposeSecret(masterSecret, SETUP_TOKEN_PURPOSE),
+    challengeTokenSecret:
+      overrides.challengeTokenSecret ?? derivePurposeSecret(masterSecret, CHALLENGE_TOKEN_PURPOSE),
+  };
+}

package/src/crypto-shredding/changes.json

@@ -1,4 +1,10 @@
 [
+  {
+    "version": "0.165.3",
+    "type": "improvement",
+    "title": "Searchable subject-PII via derived Meili index (fw#1610).",
+    "detail": "Fields may be searchable + userOwned/tenantOwned: the search consumer decrypts into Meili (ciphertext stays at rest). Forget erases the subject key and purgeSearchDocumentsForSubject removes Meili docs (ciphertext LIKE + ownership for post-anonymize). sortable + subject remains forbidden; sensitive + searchable is rejected at boot."
+  },
   {
     "version": "0.119.0",
     "type": "improvement",

package/src/document-ingest-foundation/__tests__/feature.integration.test.ts

@@ -11,7 +11,10 @@ import {
   type InMemoryFileProvider,
 } from "@cosmicdrift/kumiko-framework/files";
 import { setupTestStack, type TestStack, TestUsers } from "@cosmicdrift/kumiko-framework/stack";
+import { createComplianceProfilesFeature } from "../../compliance-profiles";
 import { createConfigFeature } from "../../config";
+import { createTenantFeature } from "../../tenant/feature";
+import { createTenantLifecycleFeature } from "../../tenant-lifecycle";
 import { documentIngestFoundationFeature } from "../feature";
 
 let stack: TestStack;
@@ -36,7 +39,13 @@ const textBytes = new TextEncoder().encode("plain text, not a supported mime typ
 beforeAll(async () => {
   provider = createInMemoryFileProvider();
   stack = await setupTestStack({
-    features: [createConfigFeature(), documentIngestFoundationFeature],
+    features: [
+      createConfigFeature(),
+      createTenantFeature(),
+      createComplianceProfilesFeature(),
+      createTenantLifecycleFeature(),
+      documentIngestFoundationFeature,
+    ],
     files: { storageProvider: provider },
   });
 });

package/src/document-ingest-foundation/__tests__/feature.test.ts

@@ -5,6 +5,7 @@
 // by feature.integration.test.ts.
 
 import { describe, expect, test } from "bun:test";
+import { EXT_TENANT_DATA } from "@cosmicdrift/kumiko-framework/engine";
 import { documentExtractEntity } from "../entity";
 import { DOCUMENT_INGEST_REQUESTED_EVENT_QN } from "../events";
 import { documentIngestFoundationFeature } from "../feature";
@@ -18,13 +19,24 @@ describe("documentIngestFoundationFeature — shape", () => {
     expect(documentIngestFoundationFeature.requires).toContain("config");
   });
 
+  test("declares tenant-lifecycle as a hard requirement — it hosts EXT_TENANT_DATA (#1621)", () => {
+    expect(documentIngestFoundationFeature.requires).toContain("tenant-lifecycle");
+  });
+
   test("registers the documentExtract entity as an implicit projection", () => {
     expect(Object.keys(documentIngestFoundationFeature.entities ?? {})).toEqual([
       "documentExtract",
     ]);
   });
 
-  test("documentExtract entity pins table name, field set, and pages encryption (#1501)", () => {
+  test("registers an entity-exact EXT_TENANT_DATA destroy hook (#1621)", () => {
+    const usage = documentIngestFoundationFeature.extensionUsages.find(
+      (u) => u.extensionName === EXT_TENANT_DATA && u.entityName === "documentExtract",
+    );
+    expect(typeof usage?.options?.["destroy"]).toBe("function");
+  });
+
+  test("documentExtract entity pins table name, field set, and pages subject-encryption (#1621)", () => {
     expect(documentExtractEntity.table).toBe("read_document_extracts");
     expect(Object.keys(documentExtractEntity.fields)).toEqual([
       "fileRefId",
@@ -32,10 +44,12 @@ describe("documentIngestFoundationFeature — shape", () => {
       "pages",
       "meta",
     ]);
+    // tenantOwned, not `encrypted: true` — the master-key path shreds nothing.
     expect(documentExtractEntity.fields.pages).toMatchObject({
       type: "longText",
-      encrypted: true,
+      tenantOwned: true,
     });
+    expect(documentExtractEntity.fields.pages).not.toHaveProperty("encrypted");
     expect(documentExtractEntity.fields.meta).toMatchObject({ type: "jsonb" });
   });
 

package/src/document-ingest-foundation/__tests__/pages-crypto-shredding.integration.test.ts

@@ -0,0 +1,138 @@
+// The point of #1621: `documentExtract.pages` carries the full extracted
+// text of invoices, IDs and contracts, and used to be `encrypted: true` —
+// master-key ciphertext with no erasure subject, so no destroy path could
+// ever make it unreadable. These tests pin the three properties the swap to
+// `tenantOwned` buys: tenant-subject ciphertext at rest, a plaintext
+// round-trip through the executor, and actual shredding when the tenant
+// subject key dies.
+
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { asRawClient } from "@cosmicdrift/kumiko-framework/bun-db";
+import {
+  configurePiiSubjectKms,
+  InMemoryKmsAdapter,
+  PII_ERASED_SENTINEL,
+  resetPiiSubjectKmsForTests,
+} from "@cosmicdrift/kumiko-framework/crypto";
+import { createEventStoreExecutor, createTenantDb } from "@cosmicdrift/kumiko-framework/db";
+import { createSystemUser } from "@cosmicdrift/kumiko-framework/engine";
+import {
+  setupTestStack,
+  type TestStack,
+  testTenantId,
+  unsafeCreateEntityTable,
+} from "@cosmicdrift/kumiko-framework/stack";
+import { createConfigFeature } from "../../config";
+import { documentExtractEntity, documentExtractsTable } from "../entity";
+import { readIngestPages, writeIngestPages } from "../pages";
+import { documentExtractTenantDestroyHook } from "../tenant-destroy-hook";
+
+let stack: TestStack;
+let kms: InMemoryKmsAdapter;
+
+const tenantId = testTenantId(1);
+const actor = createSystemUser(tenantId);
+
+const executor = createEventStoreExecutor(documentExtractsTable, documentExtractEntity, {
+  entityName: "document-extract",
+});
+
+const pages = [
+  { pageNumber: 1, text: "Rechnung Nr. 4711, Mieterin: Erika Mustermann" },
+  { pageNumber: 2, text: "IBAN DE02120300000000202051" },
+];
+
+beforeAll(async () => {
+  stack = await setupTestStack({ features: [createConfigFeature()] });
+  await unsafeCreateEntityTable(stack.db, documentExtractEntity);
+});
+
+afterAll(async () => {
+  await stack.cleanup();
+});
+
+beforeEach(async () => {
+  kms = new InMemoryKmsAdapter();
+  configurePiiSubjectKms(kms);
+  await asRawClient(stack.db).unsafe(`TRUNCATE read_document_extracts`);
+});
+
+afterEach(() => {
+  resetPiiSubjectKmsForTests();
+});
+
+async function createExtract(fileRefId: string): Promise<string> {
+  const result = await executor.create(
+    {
+      fileRefId,
+      storageKey: `s3://bucket/${fileRefId}`,
+      pages: writeIngestPages(pages),
+      meta: { provider: "test", ms: 1, needsOcr: false, pagesParsed: 2, totalPages: 2 },
+    },
+    actor,
+    createTenantDb(stack.db, tenantId),
+  );
+  if (!result.isSuccess) throw new Error(`create failed: ${result.error.message}`);
+  return String(result.data.id);
+}
+
+async function rawPages(id: string): Promise<string> {
+  const rows = (await asRawClient(stack.db).unsafe(
+    `SELECT pages FROM read_document_extracts WHERE id = $1`,
+    [id],
+  )) as ReadonlyArray<{ pages: string }>;
+  return String(rows[0]?.pages);
+}
+
+describe("documentExtract.pages — tenant-subject encryption (#1621)", () => {
+  test("stored as tenant-subject ciphertext, not master-key ciphertext", async () => {
+    const id = await createExtract("file-1");
+
+    const stored = await rawPages(id);
+    expect(stored.startsWith(`kumiko-pii:v2:tenant:${tenantId}:`)).toBe(true);
+    expect(stored).not.toContain("Erika Mustermann");
+  });
+
+  test("round-trips back to the original pages through the executor", async () => {
+    const id = await createExtract("file-2");
+
+    const row = await executor.detail({ id }, actor, createTenantDb(stack.db, tenantId));
+    expect(readIngestPages(row?.["pages"])).toEqual(pages);
+  });
+
+  test("erasing the tenant subject key makes the extracted text unreadable", async () => {
+    const id = await createExtract("file-3");
+
+    await kms.eraseKey({ kind: "tenant", tenantId });
+
+    const row = await executor.detail({ id }, actor, createTenantDb(stack.db, tenantId));
+    expect(row?.["pages"]).toBe(PII_ERASED_SENTINEL);
+    expect(readIngestPages(row?.["pages"])).toEqual([]);
+  });
+});
+
+describe("documentExtractTenantDestroyHook (#1621)", () => {
+  test("drops the tenant's extract rows and leaves other tenants alone", async () => {
+    const mine = await createExtract("file-mine");
+    const otherTenantId = testTenantId(2);
+    const otherResult = await executor.create(
+      {
+        fileRefId: "file-theirs",
+        storageKey: "s3://bucket/file-theirs",
+        pages: writeIngestPages(pages),
+        meta: {},
+      },
+      createSystemUser(otherTenantId),
+      createTenantDb(stack.db, otherTenantId),
+    );
+    if (!otherResult.isSuccess) throw new Error("setup create failed");
+
+    await documentExtractTenantDestroyHook({ db: stack.db, tenantId });
+
+    const remaining = (await asRawClient(stack.db).unsafe(
+      `SELECT id FROM read_document_extracts`,
+    )) as ReadonlyArray<{ id: string }>;
+    expect(remaining.map((row) => row.id)).not.toContain(mine);
+    expect(remaining).toHaveLength(1);
+  });
+});

package/src/document-ingest-foundation/changes.json

@@ -1 +1,9 @@
-[]
+[
+  {
+    "version": "0.166.0",
+    "type": "breaking",
+    "title": "documentExtract.pages: tenantOwned instead of encrypted, and the feature now requires tenant-lifecycle",
+    "detail": "`pages` holds the full extracted text of ingested documents (invoices, IDs, contracts) and was `encrypted: true` — app-master-key ciphertext with no erasure subject, so no Art. 17 path could ever make it unreadable. It is now `tenantOwned: true`, which binds it to the tenant subject key that tenant-destroy's eraseSubjectKeys shreds. The feature registers an EXT_TENANT_DATA destroy hook for it, and since tenant-lifecycle hosts that extension point, `document-ingest-foundation` now declares it as a hard requirement — mounting the feature without tenant-lifecycle (plus its own tenant + compliance-profiles requires) makes createRegistry throw (#1621).",
+    "migration": "Mount createTenantFeature(), createComplianceProfilesFeature() and createTenantLifecycleFeature() alongside documentIngestFoundationFeature. Rows written before this version carry master-key envelope ciphertext that the subject-decrypt path does not recognise — readIngestPages returns [] for them. There is no reencrypt job; if you have existing extracts you care about, decrypt and rewrite them before upgrading."
+  }
+]

package/src/document-ingest-foundation/entity.ts

@@ -1,3 +1,4 @@
+import { buildEntityTable } from "@cosmicdrift/kumiko-framework/db";
 import {
   createEntity,
   createJsonbField,
@@ -11,7 +12,8 @@ import {
 // jsonb has no encryption support in the engine, and this column holds the
 // full extracted text of ingested documents (invoices, IDs, contracts).
 // Writers/readers MUST use writeIngestPages / readIngestPages — do not pass
-// a raw IngestPage[] into executor.create (encrypted longText requires string).
+// a raw IngestPage[] into executor.create (the encryption hook requires a
+// string).
 export type IngestPage = {
   readonly pageNumber: number;
   readonly text: string;
@@ -35,9 +37,16 @@ export const documentExtractEntity = createEntity({
   fields: {
     fileRefId: createTextField({ required: true }),
     storageKey: createTextField({ required: true }),
-    // Encrypted — holds the full extracted document text (PII). meta is
-    // provider telemetry only and stays plaintext jsonb.
-    pages: createLongTextField({ encrypted: true }),
+    // Holds the full extracted document text (PII). `tenantOwned`, NOT
+    // `encrypted: true` (#1621): the master-key path has no erasure subject,
+    // so nothing here would ever be shreddable. Tenant-subject ciphertext
+    // dies with eraseSubjectKeys on tenant-destroy (#800 pattern). Whose
+    // subject a third party named inside a document is stays open, same as
+    // inbound-mail-foundation (#957). meta is provider telemetry only and
+    // stays plaintext jsonb.
+    pages: createLongTextField({ tenantOwned: true }),
     meta: createJsonbField(),
   },
 });
+
+export const documentExtractsTable = buildEntityTable("documentExtract", documentExtractEntity);

package/src/document-ingest-foundation/feature.ts

@@ -12,7 +12,12 @@
 // CosmicDriftGameStudio/kumiko-framework#1495 for the full phase breakdown.
 
 import { entityEventName } from "@cosmicdrift/kumiko-framework/db";
-import { access, createTenantConfig, defineFeature } from "@cosmicdrift/kumiko-framework/engine";
+import {
+  access,
+  createTenantConfig,
+  defineFeature,
+  EXT_TENANT_DATA,
+} from "@cosmicdrift/kumiko-framework/engine";
 import { z } from "zod";
 import { documentExtractEntity } from "./entity";
 import {
@@ -21,6 +26,7 @@ import {
   DOCUMENT_INGEST_REQUESTED_EVENT_SHORT,
   documentIngestRequestedPayloadSchema,
 } from "./events";
+import { documentExtractTenantDestroyHook } from "./tenant-destroy-hook";
 
 const FEATURE_NAME = "document-ingest-foundation";
 
@@ -59,9 +65,14 @@ export const documentIngestFoundationFeature = defineFeature(FEATURE_NAME, (r) =
     category: "storage",
     recommended: false,
   });
-  r.requires("config");
+  // tenant-lifecycle hosts EXT_TENANT_DATA — documentExtract.pages is
+  // tenant-subject ciphertext and needs the destroy hook below (#1621).
+  r.requires("config", "tenant-lifecycle");
 
   r.entity("documentExtract", documentExtractEntity);
+  r.useExtension(EXT_TENANT_DATA, "documentExtract", {
+    destroy: documentExtractTenantDestroyHook,
+  });
 
   const ocrLanguageConfigKey = r.config(
     "ocrLanguage",

package/src/document-ingest-foundation/index.ts

@@ -1,6 +1,11 @@
 // Public API of the document-ingest-foundation bundled-feature.
 
-export { type DocumentExtractMeta, documentExtractEntity, type IngestPage } from "./entity";
+export {
+  type DocumentExtractMeta,
+  documentExtractEntity,
+  documentExtractsTable,
+  type IngestPage,
+} from "./entity";
 export {
   DOCUMENT_INGEST_AGGREGATE_TYPE,
   DOCUMENT_INGEST_REQUESTED_EVENT_QN,

package/src/document-ingest-foundation/tenant-destroy-hook.ts

@@ -0,0 +1,49 @@
+// Tenant-destroy hook for the documentExtract entity-projection (#1621).
+//
+// Per-row forget() through the executor, not a bulk deleteMany: documentExtract
+// is an ES-managed implicit projection, so a store-table write here would be
+// eventless and a later rebuild would replay the historical create events and
+// resurrect every row this hook removed. forget() (Art. 17 hard-purge) is
+// replayed by the implicit projection itself, which keeps the erasure
+// rebuild-safe — the same reasoning as tenant-lifecycle's membership stage,
+// and the reason the feature owns an r.entity instead of an r.projection
+// (kumiko-framework#1495). The `pages` ciphertext dies separately when the
+// pipeline's later `subject-keys` stage erases the tenant subject key.
+
+import {
+  createEventStoreExecutor,
+  createTenantDb,
+  type DbRunner,
+  selectMany,
+} from "@cosmicdrift/kumiko-framework/db";
+import { createSystemUser, type TenantId } from "@cosmicdrift/kumiko-framework/engine";
+import { documentExtractEntity, documentExtractsTable } from "./entity";
+
+const executor = createEventStoreExecutor(documentExtractsTable, documentExtractEntity, {
+  entityName: "document-extract",
+});
+
+type DestroyCtx = {
+  readonly db: DbRunner;
+  readonly tenantId: TenantId;
+};
+
+export async function documentExtractTenantDestroyHook(ctx: DestroyCtx): Promise<void> {
+  const rows = await selectMany<{ id: string }>(ctx.db, documentExtractsTable, {
+    tenantId: ctx.tenantId,
+  });
+  const user = createSystemUser(ctx.tenantId);
+  const db = createTenantDb(ctx.db, ctx.tenantId, "system");
+  for (const row of rows) {
+    const result = await executor.forget({ id: row.id }, user, db);
+    // Executor writes return {isSuccess:false} instead of throwing — a
+    // discarded result would report this destroy stage "succeeded" while the
+    // extracted document text survives. Throw so the pipeline's retry/abandon
+    // handling sees it.
+    if (!result.isSuccess) {
+      throw new Error(
+        `document-ingest-foundation: failed to forget documentExtract ${row.id} for tenant ${ctx.tenantId}: ${result.error.message}`,
+      );
+    }
+  }
+}

package/src/user/changes.json

@@ -1 +1,8 @@
-[]
+[
+  {
+    "version": "0.165.3",
+    "type": "improvement",
+    "title": "user.displayName is searchable via derived Meili (fw#1610).",
+    "detail": "displayName is userOwned PII and now searchable through the decrypt-into-Meili path; forget purges the index entry with the subject key."
+  }
+]

package/src/user-data-rights/changes.json

@@ -1,4 +1,10 @@
 [
+  {
+    "version": "0.165.3",
+    "type": "improvement",
+    "title": "Forget cleanup purges derived Meili docs for searchable subject-PII (fw#1610).",
+    "detail": "run-forget-cleanup (and the forget cron path) call purgeSearchDocumentsForSubject after crypto-shred so searchable userOwned fields do not linger in Meili as plaintext after Art. 17."
+  },
   {
     "version": "0.165.0",
     "type": "breaking",