@crewhaus/data-retention-engine 0.1.0

package/package.json

@@ -0,0 +1,41 @@
+{
+  "name": "@crewhaus/data-retention-engine",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "GDPR-shaped data retention: retain / export / purge with cron-style sweeper + audit window override (Section 39)",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "bun test src"
+  },
+  "dependencies": {
+    "@crewhaus/errors": "0.0.0"
+  },
+  "license": "Apache-2.0",
+  "author": {
+    "name": "Max Meier",
+    "email": "max@studiomax.io",
+    "url": "https://studiomax.io"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/crewhaus/factory.git",
+    "directory": "packages/data-retention-engine"
+  },
+  "homepage": "https://github.com/crewhaus/factory/tree/main/packages/data-retention-engine#readme",
+  "bugs": {
+    "url": "https://github.com/crewhaus/factory/issues"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ]
+}

package/src/index.test.ts

@@ -0,0 +1,280 @@
+import { describe, expect, test } from "bun:test";
+import {
+  DataRetentionError,
+  InMemoryRecordStore,
+  type RetentionRecord,
+  _msPerDayForTest,
+  createDataRetentionEngine,
+} from "./index";
+
+const DAY_MS = _msPerDayForTest;
+
+function record(
+  id: string,
+  tenantId: string,
+  kind: string,
+  ageDays: number,
+  payload: unknown = {},
+  fromNow = Date.now(),
+): RetentionRecord {
+  return {
+    id,
+    tenantId,
+    kind,
+    createdAt: fromNow - ageDays * DAY_MS,
+    payload,
+  };
+}
+
+describe("retention policies (T1)", () => {
+  test("retain stores tenant+kind+duration", () => {
+    const store = new InMemoryRecordStore();
+    const eng = createDataRetentionEngine({ recordStore: store });
+    eng.retain("tenant-a", "audit", 30);
+    eng.retain("tenant-b", "metrics", 7);
+    expect(eng.listRetention()).toEqual([
+      { tenantId: "tenant-a", kind: "audit", durationDays: 30 },
+      { tenantId: "tenant-b", kind: "metrics", durationDays: 7 },
+    ]);
+  });
+
+  test("retain composition takes the LONGER duration when re-set", () => {
+    const eng = createDataRetentionEngine({ recordStore: new InMemoryRecordStore() });
+    eng.retain("tenant-a", "audit", 30);
+    eng.retain("tenant-a", "audit", 7);
+    eng.retain("tenant-a", "audit", 60);
+    const policies = eng.listRetention();
+    expect(policies).toHaveLength(1);
+    expect(policies[0]?.durationDays).toBe(60);
+  });
+
+  test("removeRetention drops the policy", () => {
+    const eng = createDataRetentionEngine({ recordStore: new InMemoryRecordStore() });
+    eng.retain("tenant-a", "audit", 30);
+    eng.removeRetention("tenant-a", "audit");
+    expect(eng.listRetention()).toEqual([]);
+  });
+
+  test("invalid retain inputs throw", () => {
+    const eng = createDataRetentionEngine({ recordStore: new InMemoryRecordStore() });
+    expect(() => eng.retain("", "k", 1)).toThrow(DataRetentionError);
+    expect(() => eng.retain("t", "", 1)).toThrow(DataRetentionError);
+    expect(() => eng.retain("t", "k", 0)).toThrow(DataRetentionError);
+    expect(() => eng.retain("t", "k", -1)).toThrow(DataRetentionError);
+  });
+});
+
+describe("purge (T1 + T8 cross-tenant isolation)", () => {
+  test("purge respects retention policy — recent records survive", async () => {
+    const now = 1_700_000_000_000;
+    const store = new InMemoryRecordStore([
+      record("a-recent", "tenant-a", "audit", 5, {}, now),
+      record("a-old", "tenant-a", "audit", 100, {}, now),
+    ]);
+    const eng = createDataRetentionEngine({ recordStore: store, now: () => now });
+    eng.retain("tenant-a", "audit", 30);
+    const result = await eng.purge("tenant-a");
+    expect(result.deleted).toBe(1);
+    expect(result.retentionDeferred).toEqual(["a-recent"]);
+    expect(store.ids()).toEqual(["a-recent"]);
+  });
+
+  test("purge with restrictKind only deletes that kind", async () => {
+    const now = 1_700_000_000_000;
+    const store = new InMemoryRecordStore([
+      record("audit-1", "tenant-a", "audit", 100, {}, now),
+      record("metrics-1", "tenant-a", "metrics", 100, {}, now),
+    ]);
+    const eng = createDataRetentionEngine({ recordStore: store, now: () => now });
+    await eng.purge("tenant-a", { kind: "audit" });
+    expect(store.ids()).toEqual(["metrics-1"]);
+  });
+
+  test("purge with `before` cutoff only deletes pre-cutoff records", async () => {
+    const now = 1_700_000_000_000;
+    const cutoff = now - 50 * DAY_MS;
+    const store = new InMemoryRecordStore([
+      record("old", "tenant-a", "audit", 100, {}, now), // pre-cutoff
+      record("recent", "tenant-a", "audit", 30, {}, now), // post-cutoff
+    ]);
+    const eng = createDataRetentionEngine({ recordStore: store, now: () => now });
+    await eng.purge("tenant-a", { before: cutoff });
+    expect(store.ids()).toEqual(["recent"]);
+  });
+
+  test("T8 — tenant-A purge does NOT touch tenant-B records", async () => {
+    const now = 1_700_000_000_000;
+    const store = new InMemoryRecordStore([
+      record("a-1", "tenant-a", "audit", 100, {}, now),
+      record("a-2", "tenant-a", "audit", 200, {}, now),
+      record("b-1", "tenant-b", "audit", 100, {}, now),
+      record("b-2", "tenant-b", "audit", 200, {}, now),
+    ]);
+    const eng = createDataRetentionEngine({ recordStore: store, now: () => now });
+    await eng.purge("tenant-a");
+    // tenant-a records gone; tenant-b records intact.
+    expect([...store.ids()].sort()).toEqual(["b-1", "b-2"]);
+  });
+
+  test("missing tenantId throws", async () => {
+    const eng = createDataRetentionEngine({ recordStore: new InMemoryRecordStore() });
+    await expect(eng.purge("")).rejects.toThrow(DataRetentionError);
+  });
+});
+
+describe("audit-window override", () => {
+  test("active audit window blocks purge", async () => {
+    const now = 1_700_000_000_000;
+    const store = new InMemoryRecordStore([record("a-1", "tenant-a", "audit", 200, {}, now)]);
+    const eng = createDataRetentionEngine({ recordStore: store, now: () => now });
+    eng.addAuditWindow({
+      frameworkId: "soc2",
+      controlId: "CC6.1",
+      expiresAt: now + 10 * DAY_MS,
+    });
+    const result = await eng.purge("tenant-a");
+    expect(result.deleted).toBe(0);
+    expect(result.deferred).toEqual(["a-1"]);
+    expect(result.auditWindowDeferred).toHaveLength(1);
+    expect(result.auditWindowDeferred[0]?.frameworkId).toBe("soc2");
+    expect(store.ids()).toEqual(["a-1"]);
+  });
+
+  test("expired audit window does not block purge", async () => {
+    const now = 1_700_000_000_000;
+    const store = new InMemoryRecordStore([record("a-1", "tenant-a", "audit", 200, {}, now)]);
+    const eng = createDataRetentionEngine({ recordStore: store, now: () => now });
+    expect(() =>
+      eng.addAuditWindow({
+        frameworkId: "soc2",
+        controlId: "CC6.1",
+        expiresAt: now - 1, // already expired
+      }),
+    ).toThrow(DataRetentionError);
+  });
+
+  test("listAuditWindows drops expired entries", () => {
+    let mutableNow = 1_700_000_000_000;
+    const eng = createDataRetentionEngine({
+      recordStore: new InMemoryRecordStore(),
+      now: () => mutableNow,
+    });
+    eng.addAuditWindow({
+      frameworkId: "soc2",
+      controlId: "CC6.1",
+      expiresAt: mutableNow + 5 * DAY_MS,
+    });
+    expect(eng.listAuditWindows()).toHaveLength(1);
+    mutableNow += 10 * DAY_MS;
+    expect(eng.listAuditWindows()).toHaveLength(0);
+  });
+});
+
+describe("export (right-to-export)", () => {
+  test("JSON format returns pretty-printed array", async () => {
+    const store = new InMemoryRecordStore([
+      record("a-1", "tenant-a", "audit", 5),
+      record("a-2", "tenant-a", "audit", 10),
+      record("b-1", "tenant-b", "audit", 5),
+    ]);
+    const eng = createDataRetentionEngine({ recordStore: store });
+    const out = await eng.export("tenant-a", { format: "json" });
+    const parsed = JSON.parse(out);
+    expect(parsed).toHaveLength(2);
+    expect(parsed[0].id).toBe("a-1");
+  });
+
+  test("NDJSON format returns one record per line", async () => {
+    const store = new InMemoryRecordStore([
+      record("a-1", "tenant-a", "audit", 5),
+      record("a-2", "tenant-a", "metrics", 5),
+    ]);
+    const eng = createDataRetentionEngine({ recordStore: store });
+    const out = await eng.export("tenant-a", { format: "ndjson" });
+    const lines = out.split("\n");
+    expect(lines).toHaveLength(2);
+    const first = lines[0];
+    expect(first).toBeDefined();
+    if (first === undefined) throw new Error("missing line");
+    expect(JSON.parse(first).id).toBe("a-1");
+  });
+
+  test("kinds filter restricts exported records", async () => {
+    const store = new InMemoryRecordStore([
+      record("a-1", "tenant-a", "audit", 5),
+      record("a-2", "tenant-a", "metrics", 5),
+    ]);
+    const eng = createDataRetentionEngine({ recordStore: store });
+    const out = await eng.export("tenant-a", { format: "ndjson", kinds: ["audit"] });
+    expect(out.split("\n")).toHaveLength(1);
+  });
+
+  test("export refuses cross-tenant fishing", async () => {
+    const store = new InMemoryRecordStore([
+      record("a-1", "tenant-a", "audit", 5),
+      record("b-1", "tenant-b", "audit", 5),
+    ]);
+    const eng = createDataRetentionEngine({ recordStore: store });
+    const out = await eng.export("tenant-a", { format: "json" });
+    expect(out.includes("tenant-b")).toBe(false);
+    expect(out.includes("b-1")).toBe(false);
+  });
+});
+
+describe("sweep (cron-style)", () => {
+  test("deletes expired records across all tenants", async () => {
+    const now = 1_700_000_000_000;
+    const store = new InMemoryRecordStore([
+      record("a-old", "tenant-a", "audit", 100, {}, now),
+      record("a-new", "tenant-a", "audit", 5, {}, now),
+      record("b-old", "tenant-b", "audit", 200, {}, now),
+    ]);
+    const eng = createDataRetentionEngine({
+      recordStore: store,
+      now: () => now,
+      defaultRetentionDays: 30,
+    });
+    const result = await eng.sweep();
+    expect(result.deletedCount).toBe(2);
+    expect(result.recordsKept).toBe(1);
+    expect(store.ids()).toEqual(["a-new"]);
+  });
+
+  test("idempotent — re-running yields zero deletes (T9)", async () => {
+    const now = 1_700_000_000_000;
+    const store = new InMemoryRecordStore([
+      record("a-old", "tenant-a", "audit", 100, {}, now),
+      record("a-new", "tenant-a", "audit", 5, {}, now),
+    ]);
+    const eng = createDataRetentionEngine({
+      recordStore: store,
+      now: () => now,
+      defaultRetentionDays: 30,
+    });
+    const r1 = await eng.sweep();
+    const r2 = await eng.sweep();
+    expect(r1.deletedCount).toBe(1);
+    expect(r2.deletedCount).toBe(0);
+  });
+
+  test("audit window blocks the entire sweep", async () => {
+    const now = 1_700_000_000_000;
+    const store = new InMemoryRecordStore([
+      record("a-old", "tenant-a", "audit", 100, {}, now),
+      record("b-old", "tenant-b", "audit", 200, {}, now),
+    ]);
+    const eng = createDataRetentionEngine({
+      recordStore: store,
+      now: () => now,
+      defaultRetentionDays: 30,
+    });
+    eng.addAuditWindow({
+      frameworkId: "iso27001",
+      controlId: "A.12.4",
+      expiresAt: now + 10 * DAY_MS,
+    });
+    const result = await eng.sweep();
+    expect(result.deletedCount).toBe(0);
+    expect(result.recordsKept).toBe(2);
+  });
+});

package/src/index.ts

@@ -0,0 +1,304 @@
+import { CrewhausError } from "@crewhaus/errors";
+
+/**
+ * Catalog R17 `data-retention-engine` — Section 39 GDPR-shaped retention.
+ *
+ * Three operations:
+ *   retain(tenantId, kind, durationDays) — pin records for at least
+ *     this long. The cron-style sweeper consults the policy table
+ *     before deleting. Multiple retain() calls compose: the longest
+ *     duration wins.
+ *   export(tenantId, format)             — right-to-export. Streams
+ *     a tenant's records out in JSON or NDJSON.
+ *   purge(tenantId, opts?)               — right-to-delete. Honors a
+ *     compliance-controls "do not purge during audit window" override
+ *     so SOC 2 evidence collection isn't disrupted.
+ *
+ * The engine is record-store-agnostic. Callers supply a `RecordStore`
+ * shape (list / get / delete / count) so the engine can operate over
+ * the audit-log JSONL files, a SQL table, an object-store prefix, etc.
+ *
+ * Audit-window override: callers register an `AuditWindow` (with
+ * `frameworkId` / `controlId` / `expiresAt`) via `addAuditWindow`. As
+ * long as any window is active, `purge()` skips records and returns
+ * the deferral reason. Cleared automatically once `expiresAt` passes.
+ *
+ * Cross-tenant isolation: every operation requires an explicit
+ * `tenantId`; the engine refuses to operate on records that don't
+ * report a matching `tenantId`. T8 isolation test asserts that a
+ * tenant-A purge does not touch tenant-B records.
+ *
+ * Layer R17. Pairs with `audit-log` (R-infra — primary record store
+ * for audit data), `tenancy` (R-infra — tenant-scoping primitives).
+ */
+
+export class DataRetentionError extends CrewhausError {
+  override readonly name = "DataRetentionError";
+  constructor(message: string, cause?: unknown) {
+    super("config", message, cause);
+  }
+}
+
+export type RetentionRecord = {
+  readonly id: string;
+  readonly tenantId: string;
+  readonly kind: string;
+  readonly createdAt: number;
+  readonly payload: unknown;
+};
+
+export interface RecordStore {
+  /** Iterate every record (retention engine filters per-tenant downstream). */
+  listAll(): AsyncIterable<RetentionRecord>;
+  /** Iterate records for a specific tenant. */
+  listByTenant(tenantId: string): AsyncIterable<RetentionRecord>;
+  /** Delete a record by id. Returns true if a record was actually deleted. */
+  delete(id: string): Promise<boolean>;
+}
+
+export type RetentionPolicy = {
+  readonly tenantId: string;
+  readonly kind: string;
+  readonly durationDays: number;
+};
+
+export type AuditWindow = {
+  readonly frameworkId: string;
+  readonly controlId: string;
+  readonly expiresAt: number;
+};
+
+export type ExportFormat = "json" | "ndjson";
+
+export type ExportOptions = {
+  readonly format: ExportFormat;
+  readonly kinds?: ReadonlyArray<string>;
+};
+
+export type PurgeOptions = {
+  readonly kind?: string;
+  readonly before?: number;
+};
+
+export type PurgeResult = {
+  readonly tenantId: string;
+  readonly deleted: number;
+  readonly deferred: ReadonlyArray<string>;
+  readonly retentionDeferred: ReadonlyArray<string>;
+  readonly auditWindowDeferred: ReadonlyArray<{
+    readonly frameworkId: string;
+    readonly controlId: string;
+    readonly expiresAt: number;
+  }>;
+};
+
+export type SweepResult = {
+  readonly deletedCount: number;
+  readonly recordsKept: number;
+};
+
+export interface DataRetentionEngine {
+  retain(tenantId: string, kind: string, durationDays: number): void;
+  removeRetention(tenantId: string, kind: string): void;
+  listRetention(): ReadonlyArray<RetentionPolicy>;
+  addAuditWindow(window: AuditWindow): void;
+  listAuditWindows(): ReadonlyArray<AuditWindow>;
+  /** Purge a single tenant's records. Honors retention + audit windows. */
+  purge(tenantId: string, opts?: PurgeOptions): Promise<PurgeResult>;
+  /** Stream a single tenant's records in the requested format. */
+  export(tenantId: string, opts: ExportOptions): Promise<string>;
+  /** Cron-style sweeper: walk all records and delete expired-by-policy ones. */
+  sweep(): Promise<SweepResult>;
+}
+
+export type CreateOptions = {
+  readonly recordStore: RecordStore;
+  readonly now?: () => number;
+  readonly defaultRetentionDays?: number;
+};
+
+const MS_PER_DAY = 24 * 60 * 60 * 1000;
+
+export function createDataRetentionEngine(opts: CreateOptions): DataRetentionEngine {
+  if (opts.recordStore === undefined) {
+    throw new DataRetentionError("recordStore is required");
+  }
+  const now = opts.now ?? ((): number => Date.now());
+  const defaultRetentionDays = opts.defaultRetentionDays ?? 90;
+  const policies = new Map<string, RetentionPolicy>();
+  const auditWindows: AuditWindow[] = [];
+
+  const policyKey = (tenantId: string, kind: string): string => `${tenantId}::${kind}`;
+
+  function activeAuditWindows(): ReadonlyArray<AuditWindow> {
+    const active = auditWindows.filter((w) => w.expiresAt > now());
+    // Mutate in place to drop expired windows so the array doesn't grow
+    // unboundedly across many sweeps.
+    auditWindows.length = 0;
+    auditWindows.push(...active);
+    return active;
+  }
+
+  function retentionEnd(record: RetentionRecord): number {
+    const key = policyKey(record.tenantId, record.kind);
+    const policy = policies.get(key);
+    const days = policy?.durationDays ?? defaultRetentionDays;
+    return record.createdAt + days * MS_PER_DAY;
+  }
+
+  return {
+    retain(tenantId, kind, durationDays): void {
+      if (typeof tenantId !== "string" || tenantId === "") {
+        throw new DataRetentionError("retain: tenantId required");
+      }
+      if (typeof kind !== "string" || kind === "") {
+        throw new DataRetentionError("retain: kind required");
+      }
+      if (!Number.isFinite(durationDays) || durationDays <= 0) {
+        throw new DataRetentionError("retain: durationDays must be > 0");
+      }
+      const key = policyKey(tenantId, kind);
+      const existing = policies.get(key);
+      if (existing === undefined || durationDays > existing.durationDays) {
+        policies.set(key, { tenantId, kind, durationDays });
+      }
+    },
+
+    removeRetention(tenantId, kind): void {
+      policies.delete(policyKey(tenantId, kind));
+    },
+
+    listRetention(): ReadonlyArray<RetentionPolicy> {
+      return [...policies.values()].sort(
+        (a, b) => a.tenantId.localeCompare(b.tenantId) || a.kind.localeCompare(b.kind),
+      );
+    },
+
+    addAuditWindow(window: AuditWindow): void {
+      if (window.expiresAt <= now()) {
+        throw new DataRetentionError(
+          `addAuditWindow: expiresAt (${window.expiresAt}) must be in the future (now=${now()})`,
+        );
+      }
+      auditWindows.push({ ...window });
+    },
+
+    listAuditWindows(): ReadonlyArray<AuditWindow> {
+      return activeAuditWindows().map((w) => ({ ...w }));
+    },
+
+    async purge(tenantId, purgeOpts = {}): Promise<PurgeResult> {
+      if (typeof tenantId !== "string" || tenantId === "") {
+        throw new DataRetentionError("purge: tenantId required");
+      }
+      const active = activeAuditWindows();
+      const result = {
+        tenantId,
+        deleted: 0,
+        deferred: [] as string[],
+        retentionDeferred: [] as string[],
+        auditWindowDeferred: [] as PurgeResult["auditWindowDeferred"][number][],
+      };
+      for (const window of active) {
+        result.auditWindowDeferred.push({
+          frameworkId: window.frameworkId,
+          controlId: window.controlId,
+          expiresAt: window.expiresAt,
+        });
+      }
+      const cutoff = purgeOpts.before;
+      const restrictKind = purgeOpts.kind;
+      for await (const record of opts.recordStore.listByTenant(tenantId)) {
+        if (record.tenantId !== tenantId) continue; // T8 cross-tenant guard
+        if (restrictKind !== undefined && record.kind !== restrictKind) continue;
+        if (cutoff !== undefined && record.createdAt >= cutoff) continue;
+        if (active.length > 0) {
+          result.deferred.push(record.id);
+          continue;
+        }
+        if (retentionEnd(record) > now()) {
+          result.retentionDeferred.push(record.id);
+          continue;
+        }
+        const deleted = await opts.recordStore.delete(record.id);
+        if (deleted) result.deleted += 1;
+      }
+      return {
+        tenantId: result.tenantId,
+        deleted: result.deleted,
+        deferred: result.deferred,
+        retentionDeferred: result.retentionDeferred,
+        auditWindowDeferred: result.auditWindowDeferred,
+      };
+    },
+
+    async export(tenantId, exportOpts): Promise<string> {
+      if (typeof tenantId !== "string" || tenantId === "") {
+        throw new DataRetentionError("export: tenantId required");
+      }
+      const records: RetentionRecord[] = [];
+      for await (const record of opts.recordStore.listByTenant(tenantId)) {
+        if (record.tenantId !== tenantId) continue; // T8 cross-tenant guard
+        if (exportOpts.kinds !== undefined && !exportOpts.kinds.includes(record.kind)) continue;
+        records.push(record);
+      }
+      if (exportOpts.format === "ndjson") {
+        return records.map((r) => JSON.stringify(r)).join("\n");
+      }
+      return JSON.stringify(records, null, 2);
+    },
+
+    async sweep(): Promise<SweepResult> {
+      const active = activeAuditWindows();
+      let deleted = 0;
+      let kept = 0;
+      for await (const record of opts.recordStore.listAll()) {
+        if (active.length > 0) {
+          kept += 1;
+          continue;
+        }
+        if (retentionEnd(record) > now()) {
+          kept += 1;
+          continue;
+        }
+        const ok = await opts.recordStore.delete(record.id);
+        if (ok) deleted += 1;
+        else kept += 1;
+      }
+      return { deletedCount: deleted, recordsKept: kept };
+    },
+  };
+}
+
+export class InMemoryRecordStore implements RecordStore {
+  private records: RetentionRecord[];
+  constructor(initial: ReadonlyArray<RetentionRecord> = []) {
+    this.records = [...initial];
+  }
+
+  async *listAll(): AsyncIterable<RetentionRecord> {
+    for (const r of [...this.records]) yield r;
+  }
+
+  async *listByTenant(tenantId: string): AsyncIterable<RetentionRecord> {
+    for (const r of [...this.records]) {
+      if (r.tenantId === tenantId) yield r;
+    }
+  }
+
+  async delete(id: string): Promise<boolean> {
+    const before = this.records.length;
+    this.records = this.records.filter((r) => r.id !== id);
+    return this.records.length < before;
+  }
+
+  size(): number {
+    return this.records.length;
+  }
+
+  ids(): ReadonlyArray<string> {
+    return this.records.map((r) => r.id);
+  }
+}
+
+export { MS_PER_DAY as _msPerDayForTest };