@cosmicdrift/kumiko-bundled-features 0.285.2 → 0.287.0

package/src/user-data-rights/run-export-jobs.ts

@@ -61,6 +61,7 @@ import {
   tenantExportPrefix,
   type ZipEntry,
 } from "@cosmicdrift/kumiko-framework/files";
+import type { MetricsHandle } from "@cosmicdrift/kumiko-framework/observability";
 import type { getTemporal } from "@cosmicdrift/kumiko-framework/time";
 import { resolveProfileForTenant } from "../compliance-profiles";
 import { decryptStoredPii } from "../shared";
@@ -73,6 +74,12 @@ import { generateDownloadToken } from "./token-helpers";
 
 type Instant = InstanceType<ReturnType<typeof getTemporal>["Instant"]>;
 
+// Gauge short name (feature.ts registers it via r.metric + resolves ctx.meter
+// into a MetricsHandle — jobs get no bound ctx.metrics, see feature.ts).
+// No "_seconds"/"_total" suffix: validateMetricName rejects both on a gauge.
+// Unit lives in the r.metric() `unit: "seconds"` declaration instead.
+export const EXPORT_CLEANUP_BACKLOG_AGE_METRIC = "export_cleanup_backlog_age";
+
 const crud = createEventStoreExecutor(exportJobsTable, exportJobEntity, {
   entityName: "export-job",
 });
@@ -165,6 +172,12 @@ export interface RunExportJobsArgs {
   // systemUser.id); threaded into runUserExport's per-hook TenantDb.
   readonly escapeHatchAuditSink?: EscapeHatchAuditSink;
   readonly actor?: string;
+
+  /**
+   * Reports EXPORT_CLEANUP_BACKLOG_AGE_METRIC after the storage-cleanup
+   * pass. Optional so callers without an observability provider still work.
+   */
+  readonly metrics?: MetricsHandle;
 }
 
 export interface ExportJobError {
@@ -302,6 +315,7 @@ export async function runExportJobs(args: RunExportJobsArgs): Promise<RunExportJ
     db,
     buildStorageProvider,
     now,
+    metrics: args.metrics,
   });
 
   return {
@@ -621,8 +635,9 @@ async function storageCleanupPass(args: {
   db: DbConnection;
   buildStorageProvider: (tenantId: TenantId) => Promise<FileStorageProvider>;
   now: Instant;
+  metrics?: MetricsHandle;
 }): Promise<readonly string[]> {
-  const { db, buildStorageProvider, now } = args;
+  const { db, buildStorageProvider, now, metrics } = args;
 
   // Zwei Cleanup-Pfade:
   //
@@ -659,6 +674,13 @@ async function storageCleanupPass(args: {
   );
 
   const cleaned: string[] = [];
+  // Done-status candidates past expiresAt+grace, keyed by that due-instant —
+  // used below to report how far cleanup has fallen behind if a candidate
+  // is still standing (delete threw, or the version-checked null-out lost
+  // a race) once the pass is done. Failed-status jobs have no expiresAt
+  // (immediate cleanup, no TTL) so they're out of scope for this signal —
+  // see EXPORT_CLEANUP_BACKLOG_AGE_METRIC's r.metric() description.
+  const overdueDoneCleanupAfterMs = new Map<string, number>();
   for (const c of candidates) {
     if (!c.downloadStorageKey) continue;
 
@@ -674,6 +696,7 @@ async function storageCleanupPass(args: {
         c.expiresAt.epochMilliseconds +
         profile.profile.userRights.exportStorageCleanupGraceHours * 60 * 60 * 1000;
       if (now.epochMilliseconds < cleanupAfter) continue;
+      overdueDoneCleanupAfterMs.set(c.id, cleanupAfter);
     }
     // Failed-Job-Branch: kein TTL-Check, sofort cleanup.
 
@@ -696,9 +719,54 @@ async function storageCleanupPass(args: {
     );
     if (result.isSuccess) cleaned.push(c.id);
   }
+
+  reportOldestOverdueAge(metrics, overdueDoneCleanupAfterMs, cleaned, now);
+
   return cleaned;
 }
 
+// Seconds since the longest-overdue Done-status bundle should have been
+// deleted (expiresAt+grace passed) but still isn't — 0 when the pass
+// cleaned everything that was due, or nothing was due.
+function oldestOverdueAgeSeconds(
+  overdueDoneCleanupAfterMs: ReadonlyMap<string, number>,
+  cleanedIds: readonly string[],
+  now: Instant,
+): number {
+  const cleaned = new Set(cleanedIds);
+  let oldestAgeSeconds = 0;
+  for (const [id, cleanupAfterMs] of overdueDoneCleanupAfterMs) {
+    if (cleaned.has(id)) continue;
+    const ageSeconds = (now.epochMilliseconds - cleanupAfterMs) / 1000;
+    if (ageSeconds > oldestAgeSeconds) oldestAgeSeconds = ageSeconds;
+  }
+  return oldestAgeSeconds;
+}
+
+// Best-effort like the email callbacks above: an observability hiccup
+// (metric not yet registered on this meter, provider misconfig) must not
+// fail a pass that otherwise cleaned up fine.
+function reportOldestOverdueAge(
+  metrics: MetricsHandle | undefined,
+  overdueDoneCleanupAfterMs: ReadonlyMap<string, number>,
+  cleanedIds: readonly string[],
+  now: Instant,
+): void {
+  // skip: no meter configured means there is nothing to report — the cleanup pass itself already ran to completion.
+  if (!metrics) return;
+  try {
+    metrics.set(
+      EXPORT_CLEANUP_BACKLOG_AGE_METRIC,
+      oldestOverdueAgeSeconds(overdueDoneCleanupAfterMs, cleanedIds, now),
+    );
+  } catch (err) {
+    // biome-ignore lint/suspicious/noConsole: operator-visibility for observability-emit-failure
+    console.warn(
+      `[user-data-rights:run-export-jobs] metrics.set(${EXPORT_CLEANUP_BACKLOG_AGE_METRIC}) failed err=${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+}
+
 // Wrapper: crud.update braucht TenantDb. ExportJob ist tenant-agnostisch
 // (1 Job pro userId), aber der event-store-Stream-Lookup erwartet einen
 // Tenant-Context. system-mode TenantDb bypassed den auto-tenant-filter

package/src/user-data-rights-defaults/__tests__/user-data-rights-defaults.integration.test.ts

@@ -15,7 +15,13 @@ import { authFoundationFeature } from "@cosmicdrift/kumiko-bundled-features/auth
 import { asRawClient } from "@cosmicdrift/kumiko-framework/bun-db";
 import { createTenantDb } from "@cosmicdrift/kumiko-framework/db";
 import { variantSuffix } from "@cosmicdrift/kumiko-framework/derivatives";
-import { SYSTEM_TENANT_ID } from "@cosmicdrift/kumiko-framework/engine";
+import {
+  createEntity,
+  createFileField,
+  createImageField,
+  defineFeature,
+  SYSTEM_TENANT_ID,
+} from "@cosmicdrift/kumiko-framework/engine";
 import {
   createInMemoryFileProvider,
   deriveKey,
@@ -29,7 +35,9 @@ import {
 } from "@cosmicdrift/kumiko-framework/stack";
 import { seedRow } from "@cosmicdrift/kumiko-framework/testing";
 import { createComplianceProfilesFeature } from "../../compliance-profiles";
+import { createConfigFeature } from "../../config";
 import { createDataRetentionFeature } from "../../data-retention";
+import { fileFoundationFeature } from "../../file-foundation";
 import { createFilesFeature } from "../../files";
 import { createSessionsFeature } from "../../sessions";
 import {
@@ -46,15 +54,50 @@ import { fileRefDeleteHook, fileRefExportHook, userDeleteHook, userExportHook }
 
 let stack: TestStack;
 
+// #3005 fixture: a business entity with file fields covering the three
+// field-annotation shapes fileRefDeleteHook must branch on — explicitly
+// non-personal (dealer's own business data), personal (self), and no
+// annotation at all (the conservative "no annotation" case).
+const vehicleEntity = createEntity({
+  table: "test_vehicles",
+  fields: {
+    dealerPhoto: createImageField({
+      personal: false,
+      reason: "dealer_business_data_test_fixture",
+    }),
+    driverSelfie: createImageField({ personal: "self" }),
+    unannotatedDoc: createFileField(),
+  },
+});
+
+const vehicleFeature = defineFeature("testVehicleFileFields", (r) => {
+  r.entity("vehicle", vehicleEntity);
+});
+
+// The vehicle fixture's image/file fields make server.ts's boot guard require
+// a mounted file-storage provider (registryDeclaresFileFields). The hook
+// tests below never resolve a provider through the registry — each passes
+// its own `buildStorageProvider` directly — so this one only exists to
+// satisfy that boot check.
+const bootGuardFileProvider = createInMemoryFileProvider();
+const testFileProviderFeature = defineFeature("testFileProviderBootGuard", (r) => {
+  r.requires("file-foundation");
+  r.useExtension("fileProvider", "test", { build: async () => bootGuardFileProvider });
+});
+
 const features = [
   createUserFeature(),
+  createConfigFeature(),
   createFilesFeature(),
+  fileFoundationFeature,
+  testFileProviderFeature,
   createDataRetentionFeature(),
   createComplianceProfilesFeature(),
   authFoundationFeature,
   createSessionsFeature(),
   createUserDataRightsFeature(),
   createUserDataRightsDefaultsFeature(),
+  vehicleFeature,
 ];
 
 beforeAll(async () => {
@@ -119,6 +162,35 @@ async function seedFileRef(
   );
 }
 
+// #3005: seeds a fileRef attached to an entity/field, for testing the
+// per-row PII-vs-business-data decision in fileRefDeleteHook.
+async function seedFileRefWithField(
+  id: string,
+  tenantId: string,
+  insertedById: string,
+  entityType: string | null,
+  fieldName: string | null,
+  fileName: string,
+): Promise<void> {
+  await asRawClient(stack.db).unsafe(
+    `
+    INSERT INTO file_refs (id, tenant_id, storage_key, file_name, mime_type, size, entity_type, entity_id, field_name, inserted_by_id)
+    VALUES ($1, $2, $3, $4, 'image/jpeg', 1024, $5, $6, $7, $8)
+    ON CONFLICT (id) DO NOTHING
+  `,
+    [
+      id,
+      tenantId,
+      `storage/${id}`,
+      fileName,
+      entityType,
+      entityType !== null ? "1" : null,
+      fieldName,
+      insertedById,
+    ],
+  );
+}
+
 async function fetchUser(id: string) {
   const result = await asRawClient(stack.db).unsafe(
     `
@@ -409,6 +481,131 @@ describe("S2.H2 :: fileRefDeleteHook", () => {
   });
 });
 
+describe("S2.H2 :: fileRefDeleteHook — per-row PII decision (issue #3005)", () => {
+  test('strategy="delete" — field explicitly non-personal (dealer business data): binary + row survive, insertedById=null', async () => {
+    const userId = "user-dealer-photo";
+    await seedFileRefWithField(
+      uuid(601),
+      TENANT_A,
+      userId,
+      "vehicle",
+      "dealerPhoto",
+      "vehicle-front.jpg",
+    );
+
+    await fileRefDeleteHook(
+      {
+        db: createTenantDb(stack.db, TENANT_A, "tenant"),
+        registry: stack.registry,
+        tenantId: TENANT_A,
+        userId,
+      },
+      "delete",
+    );
+
+    const ownedAfter = await fetchFileRefs(TENANT_A, userId);
+    expect(ownedAfter).toHaveLength(0);
+    const anonymized = await fetchFileRefs(TENANT_A, null);
+    const row = anonymized.find((f: { id: string }) => f.id === uuid(601));
+    expect(row).toBeDefined();
+    expect(row.inserted_by_id).toBeNull();
+  });
+
+  test('strategy="delete" — field marked personal (self): binary + row are gone', async () => {
+    const userId = "user-driver-selfie";
+    await seedFileRefWithField(
+      uuid(602),
+      TENANT_A,
+      userId,
+      "vehicle",
+      "driverSelfie",
+      "selfie.jpg",
+    );
+
+    await fileRefDeleteHook(
+      {
+        db: createTenantDb(stack.db, TENANT_A, "tenant"),
+        registry: stack.registry,
+        tenantId: TENANT_A,
+        userId,
+      },
+      "delete",
+    );
+
+    const remaining = await fetchFileRefs(TENANT_A);
+    expect(remaining.find((f: { id: string }) => f.id === uuid(602))).toBeUndefined();
+  });
+
+  test('strategy="delete" — unattached upload (entityType/fieldName both null): hard-deleted', async () => {
+    const userId = "user-unattached-hard";
+    await seedFileRefWithField(uuid(603), TENANT_A, userId, null, null, "loose-file.pdf");
+
+    await fileRefDeleteHook(
+      {
+        db: createTenantDb(stack.db, TENANT_A, "tenant"),
+        registry: stack.registry,
+        tenantId: TENANT_A,
+        userId,
+      },
+      "delete",
+    );
+
+    const remaining = await fetchFileRefs(TENANT_A);
+    expect(remaining.find((f: { id: string }) => f.id === uuid(603))).toBeUndefined();
+  });
+
+  test('strategy="delete" — field exists but carries no annotation: anonymized, not hard-deleted', async () => {
+    const userId = "user-unannotated-field";
+    await seedFileRefWithField(uuid(604), TENANT_A, userId, "vehicle", "unannotatedDoc", "doc.pdf");
+
+    await fileRefDeleteHook(
+      {
+        db: createTenantDb(stack.db, TENANT_A, "tenant"),
+        registry: stack.registry,
+        tenantId: TENANT_A,
+        userId,
+      },
+      "delete",
+    );
+
+    const ownedAfter = await fetchFileRefs(TENANT_A, userId);
+    expect(ownedAfter).toHaveLength(0);
+    const anonymized = await fetchFileRefs(TENANT_A, null);
+    const row = anonymized.find((f: { id: string }) => f.id === uuid(604));
+    expect(row).toBeDefined();
+    expect(row.inserted_by_id).toBeNull();
+  });
+
+  test('strategy="delete" — entityType not resolvable in the registry: anonymized, never a silent hard-delete', async () => {
+    const userId = "user-unresolvable-entity";
+    await seedFileRefWithField(
+      uuid(605),
+      TENANT_A,
+      userId,
+      "no-such-entity",
+      "somefield",
+      "orphaned.pdf",
+    );
+
+    await fileRefDeleteHook(
+      {
+        db: createTenantDb(stack.db, TENANT_A, "tenant"),
+        registry: stack.registry,
+        tenantId: TENANT_A,
+        userId,
+      },
+      "delete",
+    );
+
+    const ownedAfter = await fetchFileRefs(TENANT_A, userId);
+    expect(ownedAfter).toHaveLength(0);
+    const anonymized = await fetchFileRefs(TENANT_A, null);
+    const row = anonymized.find((f: { id: string }) => f.id === uuid(605));
+    expect(row).toBeDefined();
+    expect(row.inserted_by_id).toBeNull();
+  });
+});
+
 describe("S2.H2 :: fileRefDeleteHook — GDPR derivatives survive forget (issue #2461)", () => {
   async function seedFileRefWithKey(
     id: string,

package/src/user-data-rights-defaults/hooks/file-ref.userdata-hook.ts

@@ -2,6 +2,8 @@ import { createEventStoreExecutor, type TenantDb } from "@cosmicdrift/kumiko-fra
 import { derivativeListPrefix, isDerivativeKeyOf } from "@cosmicdrift/kumiko-framework/derivatives";
 import {
   createSystemUser,
+  type FieldDefinition,
+  type Registry,
   type SessionUser,
   type UserDataDeleteHook,
   type UserDataExportHook,
@@ -28,10 +30,18 @@ const crud = createEventStoreExecutor(fileRefsTable, fileRefEntity, { entityName
 // NICHT direkt — sie werden via signed-Download-URLs separat ins ZIP
 // gepackt (S2.U3 Export-Job-Pipeline orchestriert das).
 //
-// Delete-Hook entfernt FileRef-Zeile + Binary:
-//   "delete":    storageProvider.delete() pro File + Row hard-delete
-//   "anonymize": insertedById=null, Row + binary bleiben (FK-Refs
-//                koennen weiter zeigen; Personenbezug raus)
+// Delete-hook removes the fileRef row + binary:
+//   "delete":    storageProvider.delete() per file + row hard-delete — but
+//                ONLY for rows whose field is marked as PII of the person
+//                (isPersonalFileRow, #3005). All other rows (business/
+//                tenant data, no resolvable field, no annotation) go
+//                through severPersonLink instead, even when the entity
+//                strategy is "delete" — the uploader axis (insertedById) is
+//                not the subject axis.
+//   "anonymize": insertedById=null, row + binary survive (FK refs can still
+//                point at it; person-link removed) — applies to ALL rows
+//                when the entity strategy itself is already anonymize (e.g.
+//                blockDelete retention), regardless of the field.
 //
 // **Provider-Resolution:** der Provider kommt zur Lauf-Zeit aus
 // `ctx.buildStorageProvider(ctx.tenantId)` — der Forget-Orchestrator
@@ -124,6 +134,46 @@ async function resolveProvider(ctx: UserDataHookCtx): Promise<UserDataStoragePro
   }
 }
 
+// Per-row Art.17 decision (kumiko-framework#3005): whether THIS file's field
+// carries personal data of the forgotten person, or is business/tenant data
+// that merely loses its uploader-attribution. insertedById names who
+// UPLOADED the file, not whose data it is — a dealer's vehicle photo
+// uploaded by an employee is the dealer's business data, not the employee's
+// PII. The field's own personal-annotation decides:
+//   - pii / userOwned / recordOwned → the field's content IS personal data
+//     of a person (self, owner-referenced, or the record's own subject) →
+//     hard-delete path.
+//   - anything else (explicit `personal: false`, `tenantOwned`,
+//     `subjectRef`, or no annotation at all) → not personal-to-a-person
+//     content → sever the uploader link only, keep row + binary.
+function isPersonalPiiField(field: FieldDefinition | undefined): boolean {
+  if (!field) return false;
+  return (
+    ("pii" in field && field.pii === true) ||
+    ("userOwned" in field && field.userOwned !== undefined) ||
+    ("recordOwned" in field && field.recordOwned === true)
+  );
+}
+
+// Three cases the field-lookup itself can't resolve (issue #3005's rule for
+// the non-resolvable cases):
+//   1. Unattached upload (entityType/fieldName both null) → hard-delete: a
+//      file with no entity binding belongs to nobody but its uploader.
+//   2. Field exists but carries no PII annotation → anonymize (the
+//      conservative path); the boot-validator warns on a PII-typical field
+//      name so this stays visible and quiet-fixable via `personal`.
+//   3. Entity or field not resolvable (upload hardening in file-routes.ts
+//      should prevent this from ever being written; theoretical remainder
+//      only) → anonymize, never a silent hard-delete.
+function isPersonalFileRow(registry: Registry, row: Record<string, unknown>): boolean {
+  const entityType = row["entityType"]; // @cast-boundary db-row
+  const fieldName = row["fieldName"]; // @cast-boundary db-row
+  if (entityType === null && fieldName === null) return true;
+  if (typeof entityType !== "string" || typeof fieldName !== "string") return false;
+  const fieldDef = registry.getEntity(entityType)?.fields[fieldName];
+  return isPersonalPiiField(fieldDef);
+}
+
 // Derivatives (thumbnails/resized variants — see derivatives-context.ts) are
 // never tracked anywhere but the storage layer: they're rendered under a
 // deterministic key (deriveKey + variantSuffix) computed from the original's
@@ -221,14 +271,34 @@ export const fileRefDeleteHook: UserDataDeleteHook = async (ctx, strategy) => {
   if (strategy !== "delete") {
     // anonymize: insertedById=null, FileRef + binary bleiben. Use-case: shared
     // chat-Attachment im Multi-User-Channel — Author-ID raus, Datei bleibt sichtbar.
+    // Applies to ALL rows — the entity strategy comes from a retention
+    // policy (e.g. blockDelete) and overrides the per-field decision below,
+    // which only applies for strategy="delete".
     await severPersonLink(ctx.db, systemUser, rows);
     // skip: anonymize is complete — the hard-delete path below runs only for strategy "delete".
     return;
   }
 
+  // strategy="delete" decides PER ROW based on the field, not uniformly
+  // (#3005): only rows whose field is marked as PII of the person (or
+  // unattached, see isPersonalFileRow) take the hard path. Everything else
+  // only loses the uploader link.
+  const personalRows: Record<string, unknown>[] = [];
+  const businessRows: Record<string, unknown>[] = [];
+  for (const row of rows) {
+    (isPersonalFileRow(ctx.registry, row) ? personalRows : businessRows).push(row);
+  }
+
+  if (businessRows.length > 0) {
+    await severPersonLink(ctx.db, systemUser, businessRows);
+  }
+
+  // skip: nothing left to hard-delete — businessRows above already had their person link severed via severPersonLink.
+  if (personalRows.length === 0) return;
+
   const storageProvider = await resolveProvider(ctx);
   if (storageProvider) {
-    const failedKeys = await deleteBinaries(rows, storageProvider);
+    const failedKeys = await deleteBinaries(personalRows, storageProvider);
     if (failedKeys.length > 0) {
       throw new Error(
         `[user-data-rights-defaults:fileRef] ${failedKeys.length} binary delete(s) failed — aborting forget so the rows are retried next run (keys: ${failedKeys.join(", ")})`,
@@ -250,7 +320,7 @@ export const fileRefDeleteHook: UserDataDeleteHook = async (ctx, strategy) => {
   // auto-verb, the erasure replays on rebuild (created → forgotten → row gone).
   // The old hard deleteMany was resurrected on rebuild; this closes that Art.17
   // hole without a direct write.
-  for (const row of rows) {
+  for (const row of personalRows) {
     const id = row["id"]; // @cast-boundary db-row
     if (typeof id !== "string") continue;
     assertErased(await crud.forget({ id }, systemUser, ctx.db), "fileRef", id);