@cosmicdrift/kumiko-samples 0.338.0

package/packages/bundled-features/src/user-data-rights/run-user-export.ts

@@ -0,0 +1,346 @@
+// User-Data-Export-Pipeline (S2.U3) — DSGVO Art. 15 (Auskunft) +
+// Art. 20 (Datenportabilität).
+//
+// Pure Pipeline-Function: ruft alle EXT_USER_DATA-export-Hooks ueber
+// alle Tenant-Memberships eines Users + sammelt das Ergebnis als
+// strukturiertes Bundle.
+//
+// **Async ZIP + Storage (S2.U3-ext) bewusst spaeter:** Diese Foundation
+// gibt das Bundle inline zurueck (JSON-Object). Apps mit grossen
+// File-Mengen brauchen einen Job-Wrap der das Bundle nach S3 / lokalem
+// Storage schreibt + signed-URLs ergibt — kommt drauf wenn ein realer
+// User-Case >10MB Output produziert. Bis dahin reicht inline fuer
+// 99% der Apps (User-Profil + Files-Metadata + Aktivitaeten-History).
+//
+// **Cross-Tenant-Iteration:** Wie beim Forget-Pfad — User-Daten in
+// Tenant A + B kommen in dasselbe Bundle. Plan-Doc:
+// docs/plans/architecture/user-data-rights.md "Cross-Tenant-Semantik".
+//
+// **PII-Surface:** Hooks definieren selbst welche Felder ins Bundle
+// landen. user-data-rights-defaults/hooks/user.userdata-hook expose
+// expliziert KEIN passwordHash + KEINE roles (privileged columns).
+
+import { selectMany } from "@cosmicdrift/kumiko-framework/bun-db";
+import {
+  configuredPiiSubjectKms,
+  decryptPiiFieldValues,
+  isPiiCiphertext,
+} from "@cosmicdrift/kumiko-framework/crypto";
+import type { DbRunner } from "@cosmicdrift/kumiko-framework/db";
+import {
+  collectEncryptedFieldNames,
+  configuredEntityFieldEncryption,
+  createTenantDb,
+  decryptEntityFieldValues,
+} from "@cosmicdrift/kumiko-framework/db";
+import {
+  type EscapeHatchAuditSink,
+  EXT_USER_DATA,
+  extensionUsageEscapeHatchReason,
+  maskWriteOnlyFields,
+  type Registry,
+  type TenantId,
+  type UserDataExportHook,
+  type UserDataExportSnippet,
+} from "@cosmicdrift/kumiko-framework/engine";
+import {
+  createEscapeHatchReporter,
+  UNATTRIBUTED_ACTOR,
+} from "@cosmicdrift/kumiko-framework/pipeline";
+import type { getTemporal } from "@cosmicdrift/kumiko-framework/time";
+import { tenantMembershipsTable } from "../tenant/index.js";
+import { buildFileRefZipPath } from "./zip-path.js";
+
+type Instant = InstanceType<ReturnType<typeof getTemporal>["Instant"]>;
+
+export interface RunUserExportArgs {
+  readonly db: DbRunner;
+  readonly registry: Registry;
+  readonly userId: string;
+  readonly now: Instant;
+  // fw#2914 — sourced from the owning job's ctx (_escapeHatchAuditSink,
+  // systemUser.id); attributes+audits any EXT_USER_DATA usage's declared
+  // escapeHatch when its export hook calls ctx.db.unsafeRaw(reason).
+  readonly escapeHatchAuditSink?: EscapeHatchAuditSink;
+  readonly actor?: string;
+}
+
+export interface UserExportFileRef {
+  readonly fileRefId: string;
+  readonly storageKey: string;
+  readonly fileName: string;
+  /** Tenant in dem die Datei haengt — gleicher fileRefId kann nicht ueber Tenants geteilt sein. */
+  readonly tenantId: TenantId;
+  /**
+   * ZIP-internal Pfad unter dem die Datei im Export-ZIP landet. Reader-
+   * Tools (Compliance-Audit, Self-Service-Portal) verlinken bundle.json
+   * fileRefs[] auf die files/-Pfade ueber dieses Feld. Garantiert
+   * path-traversal-frei via sanitizeZipFilename.
+   */
+  readonly zipPath: string;
+}
+
+export interface UserExportTenantSection {
+  readonly tenantId: TenantId;
+  /** Pro Entity ein Snippet ({entity, rows[]}). Empty wenn Hook null returned. */
+  readonly entities: ReadonlyArray<UserDataExportSnippet>;
+}
+
+export interface UserExportBundle {
+  readonly userId: string;
+  /** ISO-8601 Generation-Timestamp — fuer Audit-Trail. */
+  readonly generatedAt: string;
+  /** Pro Tenant in dem User Mitglied ist eine Section. Orphan-User → tenants=[]. */
+  readonly tenants: ReadonlyArray<UserExportTenantSection>;
+  /**
+   * Flat-Liste aller fileRefs aus allen Tenant-Sections — der spaetere
+   * ZIP-Bau-Job iteriert hier durch + zieht Binaries aus dem Storage-
+   * Provider. Bis dahin ist das die Stueckliste fuer den Operator.
+   */
+  readonly fileRefs: ReadonlyArray<UserExportFileRef>;
+}
+
+interface HookEntry {
+  readonly entityName: string;
+  readonly exportHook: UserDataExportHook;
+  readonly escapeHatchReason: string | undefined;
+}
+
+function buildHookDb(
+  db: DbRunner,
+  tenantId: TenantId,
+  entry: HookEntry,
+  args: Pick<RunUserExportArgs, "escapeHatchAuditSink" | "actor">,
+) {
+  const report = createEscapeHatchReporter({
+    handler: `${EXT_USER_DATA}:${entry.entityName}`,
+    tenantId,
+    actor: args.actor ?? UNATTRIBUTED_ACTOR,
+    sink: args.escapeHatchAuditSink,
+  });
+  return createTenantDb(db, tenantId, "tenant", undefined, undefined, undefined, {
+    unsafeRaw:
+      entry.escapeHatchReason !== undefined ? { reason: entry.escapeHatchReason } : undefined,
+    report,
+  });
+}
+
+/**
+ * Pure function: iteriert alle EXT_USER_DATA-Hooks pro Tenant (Cross-
+ * Tenant-Memberships) + sammelt Snippets in ein UserExportBundle.
+ *
+ * **Memory-Footprint:**
+ *
+ * Der gesamte Storage-Pfad ist streaming-bound:
+ *   - File-Binaries via provider.readStream → chunk-streaming, skaliert
+ *     auf beliebige File-Sizes ohne Heap-Spike.
+ *   - ZIP-Schreiben via provider.writeStream — local nutzt fs.createWriteStream,
+ *     S3 nutzt lib-storage.Upload (multipart, ~20MB Heap-Bound bei 4
+ *     concurrent parts).
+ *
+ * **EINZIGER nicht-streaming Pfad:** das Bundle-Object selbst (bundle.json
+ * Inhalt). Es wird komplett in-memory gebaut bevor `bundleToZipEntries`
+ * es als ZIP-Entry yieldet. Hooks geben Snippets als Plain-Objects
+ * zurueck (siehe UserDataExportSnippet).
+ *
+ * **Threshold:** Web-App mit ~500 Tabellen-Rows pro User ≈ 500 KB JSON.
+ * 50k Rows ≈ 50 MB. 100k+ Rows pro User (z.B. langjaehrige Mietportal-
+ * Logs) macht Heap-Druck merkbar.
+ *
+ * **Wenn das knapp wird:** runUserExport auf AsyncIterable-Form refactoren —
+ * Hooks yielden snippets statt Object-Returns; Bundle-Schema bekommt
+ * JSON-Lines-Format. bundleToZipEntries wuerde line-by-line streamen.
+ * Eigener Sprint, nicht-trivialer Schema-Bruch.
+ *
+ * **Operator-Signal:** wenn bundle.json im ZIP > 100 MB ist, sollte
+ * Telemetry triggern + Schema-Refactor evaluieren.
+ */
+export async function runUserExport(args: RunUserExportArgs): Promise<UserExportBundle> {
+  const { db, registry, userId, now } = args;
+
+  // Memberships → Tenant-Liste fuer Hook-Iteration.
+  const memberships = await selectMany<{ tenantId: TenantId }>(db, tenantMembershipsTable, {
+    userId,
+  });
+
+  const tenantList: TenantId[] = memberships.map((m) => m.tenantId);
+
+  // EXT_USER_DATA-Usages → export-Hook-Liste.
+  const usages = registry.getExtensionUsages(EXT_USER_DATA);
+  const hookEntries: HookEntry[] = usages
+    .map((u): HookEntry | null => {
+      const opts = (u.options ?? {}) as { export?: UserDataExportHook }; // @cast-boundary engine-payload
+      return opts.export
+        ? {
+            entityName: u.entityName,
+            exportHook: opts.export,
+            escapeHatchReason: extensionUsageEscapeHatchReason(u),
+          }
+        : null;
+    })
+    .filter((x): x is HookEntry => x !== null);
+
+  const tenants: UserExportTenantSection[] = [];
+  const fileRefs: UserExportFileRef[] = [];
+
+  for (const tenantId of tenantList) {
+    const entities: UserDataExportSnippet[] = [];
+    for (const entry of hookEntries) {
+      const rawSnippet = await entry.exportHook({
+        db: buildHookDb(db, tenantId, entry, args),
+        registry,
+        tenantId,
+        userId,
+      });
+      if (rawSnippet === null) continue;
+      const snippet = await decryptSnippetFields(registry, entry.entityName, rawSnippet);
+      entities.push(snippet);
+      if (snippet.fileRefs) {
+        for (const ref of snippet.fileRefs) {
+          fileRefs.push({
+            ...ref,
+            tenantId,
+            zipPath: buildFileRefZipPath({
+              tenantId,
+              fileRefId: ref.fileRefId,
+              fileName: ref.fileName,
+            }),
+          });
+        }
+      }
+    }
+    tenants.push({ tenantId, entities });
+  }
+
+  // Edge-Case "0 Memberships": Tenant-agnostische Hooks (z.B. user-Hook)
+  // wuerden bei Cross-Tenant-Iteration mehrfach laufen. Bei orphan-User
+  // (kein Membership) wuerden sie gar nicht laufen. Loesung wie beim
+  // Forget-Runner: einen Sonder-Lauf mit einem Pseudo-Tenant ergaenzen
+  // damit die globale User-Row IM Bundle landet. Tenant-scoped Hooks
+  // sind no-op.
+  //
+  // Das Pattern matched run-forget-cleanup.ts; Memory: Cross-Tenant-
+  // Konsistenz darf nicht silent kippen wenn Memberships leer sind.
+  if (tenantList.length === 0 && hookEntries.length > 0) {
+    const orphanEntities: UserDataExportSnippet[] = [];
+    for (const entry of hookEntries) {
+      const rawSnippet = await entry.exportHook({
+        db: buildHookDb(db, SYSTEM_TENANT_ID_FOR_ORPHANS, entry, args),
+        registry,
+        tenantId: SYSTEM_TENANT_ID_FOR_ORPHANS,
+        userId,
+      });
+      if (rawSnippet === null) continue;
+      const snippet = await decryptSnippetFields(registry, entry.entityName, rawSnippet);
+      orphanEntities.push(snippet);
+      if (snippet.fileRefs) {
+        for (const ref of snippet.fileRefs) {
+          fileRefs.push({
+            ...ref,
+            tenantId: SYSTEM_TENANT_ID_FOR_ORPHANS,
+            zipPath: buildFileRefZipPath({
+              tenantId: SYSTEM_TENANT_ID_FOR_ORPHANS,
+              fileRefId: ref.fileRefId,
+              fileName: ref.fileName,
+            }),
+          });
+        }
+      }
+    }
+    if (orphanEntities.length > 0) {
+      tenants.push({ tenantId: SYSTEM_TENANT_ID_FOR_ORPHANS, entities: orphanEntities });
+    }
+  }
+
+  return {
+    userId,
+    generatedAt: now.toString(),
+    tenants,
+    fileRefs,
+  };
+}
+
+// Art. 20 verlangt die DATEN, nicht deren Ciphertext: Export-Hooks lesen
+// raw rows an der Executor-Decrypt-Schicht vorbei, darum entschluesselt
+// dieser zentrale Pass encrypted entity fields nach jedem Hook. Ohne
+// konfigurierten Cipher wird ein expliziter Marker exportiert statt den
+// base64-Blob als "Wert" auszuliefern (leak-by-confusion).
+const ENCRYPTED_UNAVAILABLE = "[encrypted:unavailable]";
+
+// Shared decrypt sweep, used for both `snippet.rows` and `snippet.fileRefs`
+// (#1955) — `encryptedFields` is keyed by field NAME, so it stays a no-op
+// for today's `fileRef` entity but covers a future generically-encrypted
+// side-channel field too.
+async function decryptRecords(
+  records: ReadonlyArray<Record<string, unknown>>,
+  encryptedFields: ReadonlySet<string>,
+  kms: ReturnType<typeof configuredPiiSubjectKms>,
+): Promise<Record<string, unknown>[]> {
+  let rows: Record<string, unknown>[] = [...records];
+
+  if (encryptedFields.size > 0) {
+    const cipher = configuredEntityFieldEncryption();
+    rows = await Promise.all(
+      rows.map(async (row) => {
+        if (cipher) return decryptEntityFieldValues(row, encryptedFields, cipher);
+        const out = { ...row };
+        for (const name of encryptedFields) {
+          if (typeof out[name] === "string") out[name] = ENCRYPTED_UNAVAILABLE;
+        }
+        return out;
+      }),
+    );
+  }
+
+  // PII-subject ciphertexts (kumiko-pii:) are self-describing — no entity
+  // context needed, covers every hook past and future. Erased subject →
+  // sentinel (honest: the value is shredded).
+  return Promise.all(
+    rows.map(async (row) => {
+      const piiKeys = Object.keys(row).filter((k) => isPiiCiphertext(row[k]));
+      if (piiKeys.length === 0) return row;
+      if (!kms) {
+        const out = { ...row };
+        for (const k of piiKeys) out[k] = ENCRYPTED_UNAVAILABLE;
+        return out;
+      }
+      return decryptPiiFieldValues(row, piiKeys, kms, {
+        requestId: "user-data-rights:export",
+      });
+    }),
+  );
+}
+
+async function decryptSnippetFields(
+  registry: Registry,
+  hookEntityName: string,
+  snippet: UserDataExportSnippet,
+): Promise<UserDataExportSnippet> {
+  const entity = registry.getEntity(snippet.entity) ?? registry.getEntity(hookEntityName);
+  const encryptedFields = entity ? collectEncryptedFieldNames(entity) : new Set<string>();
+  const kms = configuredPiiSubjectKms();
+
+  const decryptedRows = await decryptRecords(snippet.rows, encryptedFields, kms);
+  // Decrypting would otherwise hand writeOnly secrets out as plaintext.
+  const rows = entity
+    ? decryptedRows.map((row) => maskWriteOnlyFields(entity, row))
+    : decryptedRows;
+
+  if (!snippet.fileRefs || snippet.fileRefs.length === 0) {
+    return { ...snippet, rows };
+  }
+
+  const decryptedFileRefs = await decryptRecords(snippet.fileRefs, encryptedFields, kms);
+  const fileRefs = decryptedFileRefs.map((r) => ({
+    fileRefId: String(r["fileRefId"]),
+    storageKey: String(r["storageKey"]),
+    fileName: String(r["fileName"]),
+  }));
+
+  return { ...snippet, rows, fileRefs };
+}
+
+// Pseudo-Tenant fuer User ohne aktive Memberships. Identisch zum
+// Pattern in run-forget-cleanup.ts — RFC4122-Null-UUID. Tenant-scoped
+// Hooks finden hier nichts (no-op).
+const SYSTEM_TENANT_ID_FOR_ORPHANS = "00000000-0000-0000-0000-000000000000" as TenantId;

package/packages/bundled-features/src/user-data-rights/schema/download-attempt.ts

@@ -0,0 +1,57 @@
+import { buildEntityTable } from "@cosmicdrift/kumiko-framework/db";
+import {
+  createEntity,
+  createTextField,
+  createTimestampField,
+} from "@cosmicdrift/kumiko-framework/engine";
+
+// Audit-Trail invalid Download-Attempts (S2.U7).
+// Schreibt eine Row pro 4xx im download-by-{token,job}-Pfad. DPO erkennt
+// damit Brute-Force / Anomalien (gleiche IP, viele invalid-Versuche).
+// Success-Downloads landen in download-token.lastUsedAt — nicht hier.
+export const downloadAttemptEntity = createEntity({
+  table: "read_download_attempts",
+  idType: "uuid",
+  description:
+    "One rejected attempt to fetch a GDPR data-export download, recording the failure result, the path used (magic-link token or logged-in job id), the source IP, user agent and time, so brute-force probing of download links is visible.",
+  fields: {
+    // notFound | expired | failed | signedUrlNotSupported
+    result: createTextField({
+      required: true,
+      maxLength: 32,
+      personal: false,
+      reason: "technical_reference",
+    }),
+    // Welcher Pfad: "token" | "job"
+    via: createTextField({
+      required: true,
+      maxLength: 16,
+      personal: false,
+      reason: "technical_reference",
+    }),
+    // Token-Hash (token-Pfad) oder NULL (job-Pfad / unbekannter Token).
+    tokenHash: createTextField({ maxLength: 64, personal: false, reason: "technical_reference" }),
+    // Job-ID wenn der attempt einen kannte. NULL bei unbekanntem Token.
+    jobId: createTextField({ personal: false, reason: "technical_reference" }),
+    // User-ID wenn auth-Pfad (job). NULL bei anonymous (token-Pfad).
+    attemptedByUserId: createTextField({ personal: false, reason: "pseudonymous_fk" }),
+    // Same reasoning as download-token.ts's lastUsedFromIp/lastUsedUserAgent
+    // (sibling entity, same feature): DPO brute-force-audit data belongs to
+    // the tenant operator, not the (often anonymous) requester. Also avoids
+    // a real hazard on this buildEntityTable-managed entity: attemptedByUserId
+    // is null-by-design for the anonymous token path, so a subject annotation
+    // (`personal: { of: "attemptedByUserId" }`) would throw
+    // SubjectResolutionError on every such row once a KMS is configured
+    // (crypto/subject-resolver.ts resolveUserOwnedSubject).
+    ip: createTextField({ maxLength: 64, personal: false, reason: "is_business_data" }),
+    userAgent: createTextField({ maxLength: 256, personal: false, reason: "is_business_data" }),
+    attemptedAt: createTimestampField({ required: true }),
+  },
+  // 90d hardDelete: unbounded growth = disk-bomb genau gegen das System
+  // das den Brute-Force erkennen soll. Brute-Force-Patterns sind kurzfristig
+  // (Stunden bis Tage) — 90d Window deckt forensik-Reviews + DPO-quartal-
+  // Audits. Tenant kann via override verlängern (HR-Compliance).
+  retention: { keepFor: "90d", strategy: "hardDelete", reference: "attemptedAt" },
+});
+
+export const downloadAttemptsTable = buildEntityTable("downloadAttempt", downloadAttemptEntity);

package/packages/bundled-features/src/user-data-rights/schema/download-token.ts

@@ -0,0 +1,119 @@
+import { buildEntityTable } from "@cosmicdrift/kumiko-framework/db";
+import {
+  createBigIntField,
+  createEntity,
+  createTextField,
+  createTimestampField,
+} from "@cosmicdrift/kumiko-framework/engine";
+
+// Export-Download-Token (S2.U3 Atom 4a).
+//
+// Sicherheits-Token fuer den Download-Endpoint (Atom 4b). Pattern:
+//   1. Worker generiert plain-Token (32 byte random base64url) +
+//      Hash (SHA256 hex) beim Flip auf done.
+//   2. crud.create(downloadTokenEntity, ...) emittiert
+//      `exportDownloadToken.created`-Event in den event-store. DB-Row
+//      wird via Projection synchron geschrieben (Marten-Pattern, NICHT
+//      direct-INSERT). Memory `feedback_event_store_tenant_consistency`.
+//   3. Plain bleibt im Worker-Memory + wird via RunExportJobsResult an
+//      Atom 5 (Notification) weitergegeben. NIEMALS plain in DB.
+//   4. Atom 4b's Download-Endpoint: hashed incoming-Token + sucht den
+//      Hash-Eintrag. Konstanter-Zeit-Vergleich gegen timing-attacks.
+//
+// **Multi-use within TTL** (Plan-Decision 4a): Token wird beim Download
+// nicht "consumed". Mehrfach-Download bis expiresAt erlaubt — UX bei
+// Connection-Abbrueche, kein Re-Export-Zwang. Pattern matched Google-
+// Takeout (7d) + Facebook-Data-Download (4d).
+//
+// **TTL = job.expiresAt** (denormalized, gleicher Wert wie Job-Row).
+// Storage-Cleanup-Pass nullt downloadStorageKey nach
+// expiresAt + exportStorageCleanupGraceHours. Token-Row bleibt liegen
+// (Audit-Trail) — Atom 4b's Download-Endpoint check't job.downloadStorageKey
+// != null vor Streaming + returns 410 Gone wenn Storage cleared.
+//
+// **idType: "uuid"** matched ExportJob — Token-IDs reisen ueber
+// Process-Grenzen (audit-events fuer DPO, ggf. Re-Issue-Pfade).
+
+export const exportDownloadTokenEntity = createEntity({
+  table: "read_export_download_tokens",
+  idType: "uuid",
+
+  fields: {
+    // FK auf export-jobs. UNIQUE (siehe indexes) — 1 Token pro Job.
+    // Audit-Trail-Argument: separate Token-Row statt Job-Spalten weil
+    // Audit-Felder (lastUsedAt/IP/UA) semantisch zum Token, nicht zum
+    // Job gehoeren.
+    jobId: createTextField({
+      required: true,
+      personal: false,
+      reason: "technical_reference",
+    }),
+
+    // **Hash NICHT plain.** SHA256-hex (64 chars). Atom 4b verifiziert
+    // via konstanter-Zeit-Vergleich.
+    tokenHash: createTextField({
+      required: true,
+      maxLength: 64,
+      personal: false,
+      reason: "technical_reference",
+    }),
+
+    // Wann wurde das Token ausgegeben (= Job-Done-Flip-Zeit).
+    issuedAt: createTimestampField({
+      required: true,
+    }),
+
+    // Wann laeuft das Token ab. Identisch zu job.expiresAt
+    // (denormalized — Worker setzt beide gleich beim Done-Flip).
+    expiresAt: createTimestampField({
+      required: true,
+    }),
+
+    // Audit: wann wurde das Token zuletzt fuer einen Download genutzt.
+    // NULL solange noch nie heruntergeladen. Atom 4b's Download-Endpoint
+    // setzt es bei jedem successful Stream.
+    lastUsedAt: createTimestampField({}),
+
+    // Audit: Anzahl der Downloads. Atom 4b incrementiert bei jedem
+    // Stream. bigInt fuer fall einer pathologischer Re-Download-Schleife
+    // (z.B. broken Sync-Tool).
+    useCount: createBigIntField({}),
+
+    // Audit: source IP of the last download. Helps the DPO investigate
+    // unusual activity ("token was used from 5 different IPs"). Plain
+    // IPv4/IPv6, no hash — DPO needs it directly readable.
+    // `reason: "is_business_data"` because token-audit data isn't user
+    // PII (it belongs to the tenant operator).
+    lastUsedFromIp: createTextField({
+      maxLength: 45,
+      personal: false,
+      reason: "is_business_data",
+    }),
+
+    // Audit: user agent of the last download (distinguishes email
+    // client vs browser vs CLI tool). Same `reason` as above.
+    lastUsedUserAgent: createTextField({
+      maxLength: 500,
+      personal: false,
+      reason: "is_business_data",
+    }),
+  },
+
+  // 1 Token pro Job. UNIQUE auf jobId — Backstop fuer Worker-Idempotency
+  // (Atom 3b's "2× run done-Job → no-op"). Der Worker created NIE ein
+  // zweites Aggregat fuers gleiche jobId: Re-Runs rotieren das Token
+  // in place via update (fw#832) — sonst kollidiert der Projection-
+  // Rebuild an genau diesem Index.
+  indexes: [
+    {
+      unique: true,
+      columns: ["jobId"],
+      name: "read_export_download_tokens_one_per_job",
+    },
+  ],
+});
+
+export const exportDownloadTokensTable = buildEntityTable(
+  "exportDownloadToken",
+  exportDownloadTokenEntity,
+);