@dotdo/postgres 0.1.0

package/src/retention/index.ts

@@ -0,0 +1,74 @@
+/**
+ * Retention and Purge Policies Module
+ * Task: postgres-nau - Retention and purge policies
+ *
+ * This module provides comprehensive data retention and purge policies for time travel data:
+ * - Configurable retention per data type
+ * - Legal hold capability
+ * - GDPR right-to-erasure support
+ * - Audit trail for deletions
+ * - Cascade policies across related data
+ * - Regulatory compliance templates
+ *
+ * @example
+ * ```typescript
+ * import { RetentionPolicy, AutoPurger, COMPLIANCE_PROFILES } from '@dotdo/postgres/retention'
+ *
+ * // Create a retention policy with GDPR compliance
+ * const policy = new RetentionPolicy()
+ * policy.applyComplianceProfile('gdpr')
+ *
+ * // Set custom policies
+ * policy.setPolicy('snapshots', { type: 'time', days: 30 })
+ * policy.setPolicy('wal', { type: 'count', snapshots: 100 })
+ * policy.setPolicy('iceberg', { type: 'size', maxGB: 10 })
+ *
+ * // Create auto purger
+ * const purger = new AutoPurger({
+ *   policy,
+ *   fetchHandler: async (table) => {
+ *     // Fetch items from storage
+ *     return items
+ *   },
+ *   deleteHandler: async (table, items, operation) => {
+ *     // Delete items
+ *   },
+ * })
+ *
+ * // Start automatic purging
+ * purger.start()
+ *
+ * // Handle GDPR erasure request
+ * const request = policy.createErasureRequest('user-123', ['users', 'orders'])
+ * await purger.processErasureRequest(request.id)
+ * ```
+ */
+
+// Export all types from policy
+export type {
+  RetentionPolicyType,
+  DataType,
+  StorageTier,
+  PurgeOperation,
+  ComplianceFramework,
+  TimeBasedRetention,
+  CountBasedRetention,
+  SizeBasedRetention,
+  CombinedRetention,
+  RetentionPolicyConfig,
+  LegalHold,
+  ErasureRequest,
+  ComplianceProfile,
+  AuditEntry,
+  PurgeCandidate,
+  PolicyEvaluation,
+} from './policy'
+
+// Export policy classes and functions
+export { RetentionPolicy, COMPLIANCE_PROFILES, createRetentionPolicy } from './policy'
+
+// Export all types from purger
+export type { PurgeResult, PurgerConfig, PurgeStats } from './purger'
+
+// Export purger classes and functions
+export { AutoPurger, createAutoPurger } from './purger'

package/src/retention/policy.test.ts

@@ -0,0 +1,571 @@
+/**
+ * Retention Policy Tests
+ * Task: postgres-nau - Retention and purge policies
+ */
+import { describe, it, expect, beforeEach, vi } from 'vitest'
+import {
+  RetentionPolicy,
+  createRetentionPolicy,
+  COMPLIANCE_PROFILES,
+  type TimeBasedRetention,
+  type CountBasedRetention,
+  type SizeBasedRetention,
+  type CombinedRetention,
+  type PurgeCandidate,
+} from './policy'
+
+describe('RetentionPolicy', () => {
+  let policy: RetentionPolicy
+
+  beforeEach(() => {
+    policy = new RetentionPolicy()
+  })
+
+  describe('setPolicy and getPolicy', () => {
+    it('should set and retrieve a time-based policy', () => {
+      const timePolicy: TimeBasedRetention = { type: 'time', days: 30 }
+      policy.setPolicy('snapshots', timePolicy)
+
+      const retrieved = policy.getPolicy('snapshots')
+      expect(retrieved).toEqual(timePolicy)
+    })
+
+    it('should set and retrieve a count-based policy', () => {
+      const countPolicy: CountBasedRetention = { type: 'count', snapshots: 100 }
+      policy.setPolicy('wal', countPolicy)
+
+      const retrieved = policy.getPolicy('wal')
+      expect(retrieved).toEqual(countPolicy)
+    })
+
+    it('should set and retrieve a size-based policy', () => {
+      const sizePolicy: SizeBasedRetention = { type: 'size', maxGB: 10 }
+      policy.setPolicy('iceberg', sizePolicy)
+
+      const retrieved = policy.getPolicy('iceberg')
+      expect(retrieved).toEqual(sizePolicy)
+    })
+
+    it('should set and retrieve a combined policy', () => {
+      const combinedPolicy: CombinedRetention = {
+        type: 'combined',
+        time: { days: 30 },
+        size: { maxGB: 5 },
+      }
+      policy.setPolicy('custom', combinedPolicy)
+
+      const retrieved = policy.getPolicy('custom')
+      expect(retrieved).toEqual(combinedPolicy)
+    })
+  })
+
+  describe('listPolicies', () => {
+    it('should list all configured policies', () => {
+      policy.setPolicy('snapshots', { type: 'time', days: 30 })
+      policy.setPolicy('wal', { type: 'count', snapshots: 100 })
+
+      const policies = policy.listPolicies()
+      expect(policies.size).toBe(2)
+      expect(policies.has('snapshots')).toBe(true)
+      expect(policies.has('wal')).toBe(true)
+    })
+
+    it('should return empty map when no policies configured', () => {
+      const policies = policy.listPolicies()
+      expect(policies.size).toBe(0)
+    })
+  })
+
+  describe('legal hold', () => {
+    it('should set legal hold', () => {
+      policy.setLegalHold('snapshots', {
+        enabled: true,
+        reason: 'Investigation',
+        appliedBy: 'admin@example.com',
+      })
+
+      const hold = policy.getLegalHold('snapshots')
+      expect(hold?.enabled).toBe(true)
+      expect(hold?.reason).toBe('Investigation')
+      expect(hold?.appliedBy).toBe('admin@example.com')
+      expect(hold?.appliedAt).toBeDefined()
+    })
+
+    it('should check if legal hold is active', () => {
+      expect(policy.isLegalHoldActive('snapshots')).toBe(false)
+
+      policy.setLegalHold('snapshots', { enabled: true })
+      expect(policy.isLegalHoldActive('snapshots')).toBe(true)
+
+      policy.setLegalHold('snapshots', { enabled: false })
+      expect(policy.isLegalHoldActive('snapshots')).toBe(false)
+    })
+
+    it('should respect legal hold expiry', () => {
+      vi.useFakeTimers()
+      const now = Date.now()
+
+      policy.setLegalHold('snapshots', {
+        enabled: true,
+        expiresAt: now + 1000,
+      })
+
+      expect(policy.isLegalHoldActive('snapshots')).toBe(true)
+
+      vi.advanceTimersByTime(2000)
+      expect(policy.isLegalHoldActive('snapshots')).toBe(false)
+
+      vi.useRealTimers()
+    })
+  })
+
+  describe('evaluate - time-based retention', () => {
+    it('should identify items to purge based on age', () => {
+      policy.setPolicy('snapshots', { type: 'time', days: 30 })
+
+      const now = Date.now()
+      const items: PurgeCandidate[] = [
+        { id: '1', table: 'snapshots', createdAt: now - 40 * 24 * 60 * 60 * 1000, sizeBytes: 1000 }, // 40 days old
+        { id: '2', table: 'snapshots', createdAt: now - 20 * 24 * 60 * 60 * 1000, sizeBytes: 1000 }, // 20 days old
+        { id: '3', table: 'snapshots', createdAt: now - 35 * 24 * 60 * 60 * 1000, sizeBytes: 1000 }, // 35 days old
+      ]
+
+      const evaluation = policy.evaluate('snapshots', items)
+
+      expect(evaluation.toPurge.length).toBe(2)
+      expect(evaluation.toRetain.length).toBe(1)
+      expect(evaluation.toPurge.map((i) => i.id)).toEqual(['1', '3'])
+      expect(evaluation.toRetain.map((i) => i.id)).toEqual(['2'])
+    })
+
+    it('should retain all items when no policy is set', () => {
+      const items: PurgeCandidate[] = [
+        { id: '1', table: 'snapshots', createdAt: Date.now() - 100 * 24 * 60 * 60 * 1000, sizeBytes: 1000 },
+      ]
+
+      const evaluation = policy.evaluate('snapshots', items)
+
+      expect(evaluation.toPurge.length).toBe(0)
+      expect(evaluation.toRetain.length).toBe(1)
+    })
+
+    it('should prevent purge when legal hold is active', () => {
+      policy.setPolicy('snapshots', { type: 'time', days: 30 })
+      policy.setLegalHold('snapshots', { enabled: true, reason: 'Investigation' })
+
+      const items: PurgeCandidate[] = [
+        { id: '1', table: 'snapshots', createdAt: Date.now() - 100 * 24 * 60 * 60 * 1000, sizeBytes: 1000 },
+      ]
+
+      const evaluation = policy.evaluate('snapshots', items)
+
+      expect(evaluation.toPurge.length).toBe(0)
+      expect(evaluation.toRetain.length).toBe(1)
+      expect(evaluation.legalHoldActive).toBe(true)
+    })
+  })
+
+  describe('evaluate - count-based retention', () => {
+    it('should keep last N snapshots (newest)', () => {
+      policy.setPolicy('snapshots', { type: 'count', snapshots: 2 })
+
+      const now = Date.now()
+      const items: PurgeCandidate[] = [
+        { id: '1', table: 'snapshots', createdAt: now - 3000, sizeBytes: 1000 },
+        { id: '2', table: 'snapshots', createdAt: now - 2000, sizeBytes: 1000 },
+        { id: '3', table: 'snapshots', createdAt: now - 1000, sizeBytes: 1000 },
+      ]
+
+      const evaluation = policy.evaluate('snapshots', items)
+
+      expect(evaluation.toRetain.length).toBe(2)
+      expect(evaluation.toPurge.length).toBe(1)
+      expect(evaluation.toRetain.map((i) => i.id)).toEqual(['3', '2'])
+      expect(evaluation.toPurge.map((i) => i.id)).toEqual(['1'])
+    })
+
+    it('should keep last N snapshots (oldest when keepNewest=false)', () => {
+      policy.setPolicy('snapshots', { type: 'count', snapshots: 2, keepNewest: false })
+
+      const now = Date.now()
+      const items: PurgeCandidate[] = [
+        { id: '1', table: 'snapshots', createdAt: now - 3000, sizeBytes: 1000 },
+        { id: '2', table: 'snapshots', createdAt: now - 2000, sizeBytes: 1000 },
+        { id: '3', table: 'snapshots', createdAt: now - 1000, sizeBytes: 1000 },
+      ]
+
+      const evaluation = policy.evaluate('snapshots', items)
+
+      expect(evaluation.toRetain.length).toBe(2)
+      expect(evaluation.toPurge.length).toBe(1)
+      expect(evaluation.toRetain.map((i) => i.id)).toEqual(['1', '2'])
+      expect(evaluation.toPurge.map((i) => i.id)).toEqual(['3'])
+    })
+  })
+
+  describe('evaluate - size-based retention', () => {
+    it('should keep items under size limit', () => {
+      policy.setPolicy('iceberg', { type: 'size', maxGB: 0.000002 }) // 2KB in GB
+
+      const items: PurgeCandidate[] = [
+        { id: '1', table: 'iceberg', createdAt: Date.now() - 3000, sizeBytes: 1000 },
+        { id: '2', table: 'iceberg', createdAt: Date.now() - 2000, sizeBytes: 1000 },
+        { id: '3', table: 'iceberg', createdAt: Date.now() - 1000, sizeBytes: 1000 },
+      ]
+
+      const evaluation = policy.evaluate('iceberg', items)
+
+      expect(evaluation.toRetain.length).toBe(2)
+      expect(evaluation.toPurge.length).toBe(1)
+    })
+
+    it('should evict oldest first by default', () => {
+      policy.setPolicy('iceberg', { type: 'size', maxGB: 0.000002 }) // 2KB
+
+      const now = Date.now()
+      const items: PurgeCandidate[] = [
+        { id: '1', table: 'iceberg', createdAt: now - 3000, sizeBytes: 1000 }, // Oldest
+        { id: '2', table: 'iceberg', createdAt: now - 2000, sizeBytes: 1000 },
+        { id: '3', table: 'iceberg', createdAt: now - 1000, sizeBytes: 1000 }, // Newest
+      ]
+
+      const evaluation = policy.evaluate('iceberg', items)
+
+      expect(evaluation.toPurge.map((i) => i.id)).toEqual(['3'])
+    })
+  })
+
+  describe('evaluate - combined retention', () => {
+    it('should apply multiple retention rules', () => {
+      policy.setPolicy('snapshots', {
+        type: 'combined',
+        time: { days: 30 },
+        count: { snapshots: 2 },
+      })
+
+      const now = Date.now()
+      const items: PurgeCandidate[] = [
+        { id: '1', table: 'snapshots', createdAt: now - 40 * 24 * 60 * 60 * 1000, sizeBytes: 1000 }, // Too old
+        { id: '2', table: 'snapshots', createdAt: now - 20 * 24 * 60 * 60 * 1000, sizeBytes: 1000 }, // OK
+        { id: '3', table: 'snapshots', createdAt: now - 10 * 24 * 60 * 60 * 1000, sizeBytes: 1000 }, // OK
+        { id: '4', table: 'snapshots', createdAt: now - 5 * 24 * 60 * 60 * 1000, sizeBytes: 1000 }, // Exceeds count
+      ]
+
+      const evaluation = policy.evaluate('snapshots', items)
+
+      // Item 1 is too old, and with count=2, only 2 items can be retained
+      expect(evaluation.toPurge.length).toBeGreaterThan(0)
+      expect(evaluation.toRetain.length).toBeLessThanOrEqual(2)
+    })
+  })
+
+  describe('compliance profiles', () => {
+    it('should apply GDPR compliance profile', () => {
+      policy.applyComplianceProfile('gdpr')
+
+      const profile = policy.getComplianceProfile()
+      expect(profile?.framework).toBe('gdpr')
+      expect(profile?.name).toBe('GDPR Compliance')
+
+      // Check that default policies are set
+      expect(policy.getPolicy('wal')).toBeDefined()
+      expect(policy.getPolicy('iceberg')).toBeDefined()
+    })
+
+    it('should apply SOX compliance profile', () => {
+      policy.applyComplianceProfile('sox')
+
+      const profile = policy.getComplianceProfile()
+      expect(profile?.framework).toBe('sox')
+      expect(profile?.requireAuditTrail).toBe(true)
+      expect(profile?.requireLegalHold).toBe(true)
+    })
+
+    it('should apply HIPAA compliance profile', () => {
+      policy.applyComplianceProfile('hipaa')
+
+      const profile = policy.getComplianceProfile()
+      expect(profile?.framework).toBe('hipaa')
+      expect(profile?.requireAuditTrail).toBe(true)
+    })
+
+    it('should throw error for unknown compliance profile', () => {
+      expect(() => policy.applyComplianceProfile('unknown')).toThrow('Unknown compliance profile')
+    })
+
+    it('should validate policies against compliance requirements', () => {
+      policy.applyComplianceProfile('sox')
+
+      // SOX requires 7-year retention, trying to set 1-year should fail
+      expect(() => {
+        policy.setPolicy('wal', { type: 'time', days: 365 })
+      }).toThrow(/violates minimum retention/)
+    })
+
+    it('should validate maximum retention', () => {
+      policy.applyComplianceProfile('gdpr')
+
+      // GDPR has maximum retention limits
+      expect(() => {
+        policy.setPolicy('wal', { type: 'time', days: 3650 }) // 10 years, exceeds GDPR max
+      }).toThrow(/violates maximum retention/)
+    })
+  })
+
+  describe('GDPR erasure requests', () => {
+    it('should create erasure request', () => {
+      const request = policy.createErasureRequest('user-123', ['users', 'orders'], 'admin@example.com')
+
+      expect(request.id).toBeDefined()
+      expect(request.subjectId).toBe('user-123')
+      expect(request.scope).toEqual(['users', 'orders'])
+      expect(request.requestedBy).toBe('admin@example.com')
+      expect(request.status).toBe('pending')
+      expect(request.deadline).toBeGreaterThan(request.requestedAt)
+    })
+
+    it('should retrieve erasure request by ID', () => {
+      const request = policy.createErasureRequest('user-123', ['users'])
+      const retrieved = policy.getErasureRequest(request.id)
+
+      expect(retrieved).toEqual(request)
+    })
+
+    it('should list erasure requests by status', () => {
+      policy.createErasureRequest('user-1', ['users'])
+      policy.createErasureRequest('user-2', ['users'])
+
+      const pending = policy.listErasureRequests('pending')
+      expect(pending.length).toBe(2)
+
+      policy.updateErasureRequest(pending[0].id, { status: 'completed' })
+
+      const stillPending = policy.listErasureRequests('pending')
+      expect(stillPending.length).toBe(1)
+
+      const completed = policy.listErasureRequests('completed')
+      expect(completed.length).toBe(1)
+    })
+
+    it('should update erasure request status', () => {
+      const request = policy.createErasureRequest('user-123', ['users'])
+
+      policy.updateErasureRequest(request.id, {
+        status: 'in_progress',
+      })
+
+      const updated = policy.getErasureRequest(request.id)
+      expect(updated?.status).toBe('in_progress')
+    })
+
+    it('should calculate 72-hour deadline', () => {
+      const request = policy.createErasureRequest('user-123', ['users'])
+      const expectedDeadline = request.requestedAt + 72 * 60 * 60 * 1000
+
+      expect(request.deadline).toBe(expectedDeadline)
+    })
+  })
+
+  describe('audit trail', () => {
+    it('should add audit entry', () => {
+      policy.addAuditEntry({
+        operation: 'hard_delete',
+        table: 'snapshots',
+        recordCount: 10,
+        bytesFreed: 10000,
+        success: true,
+        initiatedBy: 'system',
+      })
+
+      const trail = policy.getAuditTrail()
+      expect(trail.length).toBe(1)
+      expect(trail[0].operation).toBe('hard_delete')
+      expect(trail[0].table).toBe('snapshots')
+      expect(trail[0].recordCount).toBe(10)
+    })
+
+    it('should filter audit trail by table', () => {
+      policy.addAuditEntry({
+        operation: 'soft_delete',
+        table: 'snapshots',
+        recordCount: 5,
+        bytesFreed: 5000,
+        success: true,
+      })
+
+      policy.addAuditEntry({
+        operation: 'hard_delete',
+        table: 'wal',
+        recordCount: 10,
+        bytesFreed: 10000,
+        success: true,
+      })
+
+      const snapshotTrail = policy.getAuditTrail({ table: 'snapshots' })
+      expect(snapshotTrail.length).toBe(1)
+      expect(snapshotTrail[0].table).toBe('snapshots')
+    })
+
+    it('should filter audit trail by operation', () => {
+      policy.addAuditEntry({
+        operation: 'soft_delete',
+        table: 'snapshots',
+        recordCount: 5,
+        bytesFreed: 5000,
+        success: true,
+      })
+
+      policy.addAuditEntry({
+        operation: 'hard_delete',
+        table: 'snapshots',
+        recordCount: 10,
+        bytesFreed: 10000,
+        success: true,
+      })
+
+      const softDeleteTrail = policy.getAuditTrail({ operation: 'soft_delete' })
+      expect(softDeleteTrail.length).toBe(1)
+      expect(softDeleteTrail[0].operation).toBe('soft_delete')
+    })
+
+    it('should filter audit trail by date range', () => {
+      const now = Date.now()
+
+      policy.addAuditEntry({
+        operation: 'soft_delete',
+        table: 'snapshots',
+        recordCount: 5,
+        bytesFreed: 5000,
+        success: true,
+      })
+
+      const startDate = now
+      const endDate = now + 1000
+
+      const trail = policy.getAuditTrail({ startDate, endDate })
+      expect(trail.length).toBeGreaterThanOrEqual(0)
+    })
+
+    it('should sort audit trail by timestamp (newest first)', () => {
+      policy.addAuditEntry({
+        operation: 'soft_delete',
+        table: 'snapshots',
+        recordCount: 5,
+        bytesFreed: 5000,
+        success: true,
+      })
+
+      // Small delay to ensure different timestamps
+      const delay = () => new Promise((resolve) => setTimeout(resolve, 10))
+      delay().then(() => {
+        policy.addAuditEntry({
+          operation: 'hard_delete',
+          table: 'wal',
+          recordCount: 10,
+          bytesFreed: 10000,
+          success: true,
+        })
+
+        const trail = policy.getAuditTrail()
+        expect(trail.length).toBe(2)
+        expect(trail[0].timestamp).toBeGreaterThanOrEqual(trail[1].timestamp)
+      })
+    })
+  })
+
+  describe('compliance report', () => {
+    it('should generate compliance report', () => {
+      policy.applyComplianceProfile('gdpr')
+      policy.setPolicy('custom', { type: 'time', days: 90 })
+
+      const report = policy.generateComplianceReport()
+
+      expect(report.profile?.framework).toBe('gdpr')
+      expect(report.policies.size).toBeGreaterThan(0)
+      expect(report.complianceStatus).toBe('compliant')
+      expect(report.issues.length).toBe(0)
+    })
+
+    it('should detect overdue erasure requests', () => {
+      vi.useFakeTimers()
+      const now = Date.now()
+
+      const request = policy.createErasureRequest('user-123', ['users'])
+
+      // Advance time past the 72-hour deadline
+      vi.advanceTimersByTime(73 * 60 * 60 * 1000)
+
+      const report = policy.generateComplianceReport()
+
+      expect(report.complianceStatus).toBe('violation')
+      expect(report.overdueErasureRequests.length).toBe(1)
+      expect(report.issues.some((i) => i.includes('overdue'))).toBe(true)
+
+      vi.useRealTimers()
+    })
+
+    it('should detect policy violations', () => {
+      policy.applyComplianceProfile('sox')
+
+      // Manually override a policy to violate SOX requirements
+      // This bypasses validation by directly setting the map
+      const policies = policy.listPolicies()
+      policy['policies'].set('wal', { type: 'time', days: 365 }) // Violates SOX 7-year requirement
+
+      const report = policy.generateComplianceReport()
+
+      expect(report.complianceStatus).toBe('violation')
+      expect(report.issues.length).toBeGreaterThan(0)
+    })
+
+    it('should include recent audit entries', () => {
+      policy.addAuditEntry({
+        operation: 'soft_delete',
+        table: 'snapshots',
+        recordCount: 5,
+        bytesFreed: 5000,
+        success: true,
+      })
+
+      const report = policy.generateComplianceReport()
+      expect(report.recentAuditEntries.length).toBe(1)
+    })
+  })
+})
+
+describe('createRetentionPolicy', () => {
+  it('should create policy without compliance profile', () => {
+    const policy = createRetentionPolicy()
+    expect(policy).toBeInstanceOf(RetentionPolicy)
+    expect(policy.getComplianceProfile()).toBeUndefined()
+  })
+
+  it('should create policy with compliance profile', () => {
+    const policy = createRetentionPolicy('gdpr')
+    expect(policy).toBeInstanceOf(RetentionPolicy)
+    expect(policy.getComplianceProfile()?.framework).toBe('gdpr')
+  })
+})
+
+describe('COMPLIANCE_PROFILES', () => {
+  it('should have GDPR profile', () => {
+    const gdpr = COMPLIANCE_PROFILES.gdpr
+    expect(gdpr.framework).toBe('gdpr')
+    expect(gdpr.requireAuditTrail).toBe(true)
+    expect(gdpr.defaultRetention.wal).toBeDefined()
+  })
+
+  it('should have SOX profile', () => {
+    const sox = COMPLIANCE_PROFILES.sox
+    expect(sox.framework).toBe('sox')
+    expect(sox.requireAuditTrail).toBe(true)
+    expect(sox.requireLegalHold).toBe(true)
+  })
+
+  it('should have HIPAA profile', () => {
+    const hipaa = COMPLIANCE_PROFILES.hipaa
+    expect(hipaa.framework).toBe('hipaa')
+    expect(hipaa.requireAuditTrail).toBe(true)
+    expect(hipaa.requireLegalHold).toBe(true)
+  })
+})