@dotdo/postgres 0.1.0

package/src/retention/policy.ts

@@ -0,0 +1,774 @@
+/**
+ * Retention Policy Management
+ * Task: postgres-nau - Retention and purge policies
+ *
+ * Implements comprehensive data retention and purge policies for time travel data:
+ * - Configurable retention per data type
+ * - Legal hold capability
+ * - GDPR right-to-erasure support
+ * - Audit trail for deletions
+ * - Cascade policies across related data
+ * - Regulatory compliance templates
+ */
+
+/**
+ * Supported retention policy types
+ */
+export type RetentionPolicyType = 'time' | 'count' | 'size' | 'combined'
+
+/**
+ * Data types that can have retention policies
+ */
+export type DataType = 'wal' | 'iceberg' | 'snapshots' | 'custom'
+
+/**
+ * Storage tiers for tiered retention
+ */
+export type StorageTier = 'hot' | 'warm' | 'cold' | 'archive'
+
+/**
+ * Purge operation types
+ */
+export type PurgeOperation = 'soft_delete' | 'hard_delete' | 'cryptographic_erasure' | 'cascade_delete'
+
+/**
+ * Compliance frameworks
+ */
+export type ComplianceFramework = 'gdpr' | 'sox' | 'hipaa' | 'custom'
+
+/**
+ * Time-based retention configuration
+ * Keep data for a specified duration
+ */
+export interface TimeBasedRetention {
+  /** Retention type identifier */
+  type: 'time'
+  /** Retention duration in days */
+  days: number
+  /** Storage tier configuration (optional) */
+  tiers?: {
+    tier: StorageTier
+    days: number
+  }[]
+}
+
+/**
+ * Count-based retention configuration
+ * Keep last N snapshots/records
+ */
+export interface CountBasedRetention {
+  /** Retention type identifier */
+  type: 'count'
+  /** Number of snapshots to keep */
+  snapshots: number
+  /** Whether to keep based on newest (default) or oldest */
+  keepNewest?: boolean
+}
+
+/**
+ * Size-based retention configuration
+ * Keep data under a specified size limit
+ */
+export interface SizeBasedRetention {
+  /** Retention type identifier */
+  type: 'size'
+  /** Maximum size in GB */
+  maxGB: number
+  /** Whether to evict oldest first (default: true) */
+  evictOldestFirst?: boolean
+}
+
+/**
+ * Combined retention policy
+ * Applies multiple retention rules
+ */
+export interface CombinedRetention {
+  /** Retention type identifier */
+  type: 'combined'
+  /** Time-based component (optional) */
+  time?: Omit<TimeBasedRetention, 'type'>
+  /** Count-based component (optional) */
+  count?: Omit<CountBasedRetention, 'type'>
+  /** Size-based component (optional) */
+  size?: Omit<SizeBasedRetention, 'type'>
+}
+
+/**
+ * Union type for all retention policies
+ */
+export type RetentionPolicyConfig =
+  | TimeBasedRetention
+  | CountBasedRetention
+  | SizeBasedRetention
+  | CombinedRetention
+
+/**
+ * Legal hold configuration
+ * Prevents deletion of data regardless of retention policy
+ */
+export interface LegalHold {
+  /** Whether legal hold is active */
+  enabled: boolean
+  /** Reason for legal hold (for auditing) */
+  reason?: string
+  /** When the legal hold was applied */
+  appliedAt?: number
+  /** Who applied the legal hold */
+  appliedBy?: string
+  /** Optional expiry time for the legal hold */
+  expiresAt?: number
+  /** Case/matter ID for tracking */
+  caseId?: string
+}
+
+/**
+ * GDPR erasure request
+ */
+export interface ErasureRequest {
+  /** Request ID */
+  id: string
+  /** Data subject identifier */
+  subjectId: string
+  /** Requested at timestamp */
+  requestedAt: number
+  /** Requested by */
+  requestedBy?: string
+  /** Deadline (72 hours from request by default) */
+  deadline: number
+  /** Tables/data to erase */
+  scope: string[]
+  /** Status of the request */
+  status: 'pending' | 'in_progress' | 'completed' | 'failed'
+  /** Completion timestamp */
+  completedAt?: number
+  /** Error message if failed */
+  error?: string
+}
+
+/**
+ * Compliance profile
+ */
+export interface ComplianceProfile {
+  /** Framework name */
+  framework: ComplianceFramework
+  /** Custom name for the profile */
+  name: string
+  /** Default retention periods by data type */
+  defaultRetention: Record<DataType, RetentionPolicyConfig>
+  /** Minimum retention periods (cannot delete before) */
+  minimumRetention?: Record<DataType, number>
+  /** Maximum retention periods (must delete after) */
+  maximumRetention?: Record<DataType, number>
+  /** Whether to require legal hold for certain operations */
+  requireLegalHold?: boolean
+  /** Whether to maintain audit trail */
+  requireAuditTrail?: boolean
+}
+
+/**
+ * Audit entry for deletion operations
+ */
+export interface AuditEntry {
+  /** Audit entry ID */
+  id: string
+  /** Timestamp of the operation */
+  timestamp: number
+  /** Type of operation */
+  operation: PurgeOperation
+  /** Table or data type affected */
+  table: string
+  /** Number of records affected */
+  recordCount: number
+  /** Total bytes freed */
+  bytesFreed: number
+  /** Who initiated the operation */
+  initiatedBy?: string
+  /** Reason for the operation */
+  reason?: string
+  /** Associated legal hold or erasure request */
+  relatedRequest?: string
+  /** Success status */
+  success: boolean
+  /** Error message if failed */
+  error?: string
+  /** Additional metadata */
+  metadata?: Record<string, unknown>
+}
+
+/**
+ * Item to be evaluated for purging
+ */
+export interface PurgeCandidate {
+  /** Unique identifier */
+  id: string
+  /** Table or data type */
+  table: string
+  /** Creation timestamp */
+  createdAt: number
+  /** Size in bytes */
+  sizeBytes: number
+  /** Additional metadata */
+  metadata?: Record<string, unknown>
+}
+
+/**
+ * Result of policy evaluation
+ */
+export interface PolicyEvaluation {
+  /** Items that should be purged */
+  toPurge: PurgeCandidate[]
+  /** Items that should be retained */
+  toRetain: PurgeCandidate[]
+  /** Reason for decisions */
+  reasons: Map<string, string>
+  /** Whether legal hold prevented any deletions */
+  legalHoldActive: boolean
+}
+
+/**
+ * Pre-defined compliance profiles
+ */
+export const COMPLIANCE_PROFILES: Record<string, ComplianceProfile> = {
+  gdpr: {
+    framework: 'gdpr',
+    name: 'GDPR Compliance',
+    defaultRetention: {
+      wal: { type: 'time', days: 30 },
+      iceberg: { type: 'time', days: 365 },
+      snapshots: { type: 'time', days: 30 },
+      custom: { type: 'time', days: 90 },
+    },
+    maximumRetention: {
+      wal: 90,
+      iceberg: 2555, // 7 years
+      snapshots: 90,
+      custom: 365,
+    },
+    requireAuditTrail: true,
+  },
+  sox: {
+    framework: 'sox',
+    name: 'SOX Compliance',
+    defaultRetention: {
+      wal: { type: 'time', days: 2555 }, // 7 years
+      iceberg: { type: 'time', days: 2555 },
+      snapshots: { type: 'time', days: 2555 },
+      custom: { type: 'time', days: 2555 },
+    },
+    minimumRetention: {
+      wal: 2555,
+      iceberg: 2555,
+      snapshots: 2555,
+      custom: 2555,
+    },
+    requireAuditTrail: true,
+    requireLegalHold: true,
+  },
+  hipaa: {
+    framework: 'hipaa',
+    name: 'HIPAA Compliance',
+    defaultRetention: {
+      wal: { type: 'time', days: 2190 }, // 6 years
+      iceberg: { type: 'time', days: 2190 },
+      snapshots: { type: 'time', days: 2190 },
+      custom: { type: 'time', days: 2190 },
+    },
+    minimumRetention: {
+      wal: 2190,
+      iceberg: 2190,
+      snapshots: 2190,
+      custom: 2190,
+    },
+    requireAuditTrail: true,
+    requireLegalHold: true,
+  },
+}
+
+/**
+ * Retention Policy Manager
+ *
+ * Manages retention policies for different data types with support for
+ * legal holds, GDPR erasure, and compliance frameworks.
+ *
+ * @example
+ * ```typescript
+ * const policyManager = new RetentionPolicy()
+ *
+ * // Set a time-based policy for snapshots
+ * policyManager.setPolicy('snapshots', { type: 'time', days: 30 })
+ *
+ * // Set a count-based policy for WAL
+ * policyManager.setPolicy('wal', { type: 'count', snapshots: 100 })
+ *
+ * // Set a size-based policy
+ * policyManager.setPolicy('iceberg', { type: 'size', maxGB: 10 })
+ *
+ * // Evaluate what should be purged
+ * const items = [
+ *   { id: '1', table: 'snapshots', createdAt: Date.now() - 40 * 24 * 60 * 60 * 1000, sizeBytes: 1000 },
+ *   { id: '2', table: 'snapshots', createdAt: Date.now() - 20 * 24 * 60 * 60 * 1000, sizeBytes: 1000 },
+ * ]
+ * const evaluation = policyManager.evaluate('snapshots', items)
+ * console.log(`${evaluation.toPurge.length} items to purge`)
+ *
+ * // Apply compliance profile
+ * policyManager.applyComplianceProfile('gdpr')
+ * ```
+ */
+export class RetentionPolicy {
+  private policies: Map<string, RetentionPolicyConfig> = new Map()
+  private legalHolds: Map<string, LegalHold> = new Map()
+  private erasureRequests: Map<string, ErasureRequest> = new Map()
+  private auditLog: AuditEntry[] = []
+  private complianceProfile?: ComplianceProfile
+
+  /**
+   * Set retention policy for a table/data type
+   */
+  setPolicy(table: string, policy: RetentionPolicyConfig): void {
+    // Validate against compliance profile if set
+    if (this.complianceProfile) {
+      this.validatePolicyAgainstCompliance(table, policy)
+    }
+
+    this.policies.set(table, policy)
+  }
+
+  /**
+   * Get retention policy for a table/data type
+   */
+  getPolicy(table: string): RetentionPolicyConfig | undefined {
+    return this.policies.get(table)
+  }
+
+  /**
+   * List all configured policies
+   */
+  listPolicies(): Map<string, RetentionPolicyConfig> {
+    return new Map(this.policies)
+  }
+
+  /**
+   * Set legal hold for a table
+   */
+  setLegalHold(table: string, hold: LegalHold): void {
+    this.legalHolds.set(table, {
+      ...hold,
+      appliedAt: hold.appliedAt ?? Date.now(),
+    })
+  }
+
+  /**
+   * Get legal hold for a table
+   */
+  getLegalHold(table: string): LegalHold | undefined {
+    return this.legalHolds.get(table)
+  }
+
+  /**
+   * Check if legal hold is active for a table
+   */
+  isLegalHoldActive(table: string): boolean {
+    const hold = this.legalHolds.get(table)
+    if (!hold || !hold.enabled) {
+      return false
+    }
+
+    // Check if legal hold has expired
+    if (hold.expiresAt && Date.now() > hold.expiresAt) {
+      return false
+    }
+
+    return true
+  }
+
+  /**
+   * Apply a compliance profile
+   */
+  applyComplianceProfile(profileName: string): void {
+    const profile = COMPLIANCE_PROFILES[profileName]
+    if (!profile) {
+      throw new Error(`Unknown compliance profile: ${profileName}`)
+    }
+
+    this.complianceProfile = profile
+
+    // Apply default retention policies
+    for (const [dataType, policy] of Object.entries(profile.defaultRetention)) {
+      this.policies.set(dataType, policy)
+    }
+  }
+
+  /**
+   * Get current compliance profile
+   */
+  getComplianceProfile(): ComplianceProfile | undefined {
+    return this.complianceProfile
+  }
+
+  /**
+   * Validate a policy against compliance requirements
+   */
+  private validatePolicyAgainstCompliance(table: string, policy: RetentionPolicyConfig): void {
+    if (!this.complianceProfile) {
+      return
+    }
+
+    const minRetention = this.complianceProfile.minimumRetention?.[table as DataType]
+    const maxRetention = this.complianceProfile.maximumRetention?.[table as DataType]
+
+    // Convert policy to days for comparison
+    let policyDays: number | undefined
+
+    if (policy.type === 'time') {
+      policyDays = policy.days
+    } else if (policy.type === 'combined' && policy.time) {
+      policyDays = policy.time.days
+    }
+
+    if (policyDays !== undefined) {
+      if (minRetention && policyDays < minRetention) {
+        throw new Error(
+          `Policy for ${table} violates minimum retention of ${minRetention} days (${this.complianceProfile.framework})`
+        )
+      }
+
+      if (maxRetention && policyDays > maxRetention) {
+        throw new Error(
+          `Policy for ${table} violates maximum retention of ${maxRetention} days (${this.complianceProfile.framework})`
+        )
+      }
+    }
+  }
+
+  /**
+   * Evaluate items against retention policy
+   */
+  evaluate(table: string, items: PurgeCandidate[]): PolicyEvaluation {
+    const policy = this.policies.get(table)
+    if (!policy) {
+      // No policy set - retain everything
+      return {
+        toPurge: [],
+        toRetain: items,
+        reasons: new Map(),
+        legalHoldActive: false,
+      }
+    }
+
+    // Check for legal hold
+    if (this.isLegalHoldActive(table)) {
+      const reasons = new Map(items.map((item) => [item.id, 'Legal hold active']))
+      return {
+        toPurge: [],
+        toRetain: items,
+        reasons,
+        legalHoldActive: true,
+      }
+    }
+
+    return this.evaluatePolicy(table, policy, items)
+  }
+
+  /**
+   * Evaluate items against a specific policy
+   */
+  private evaluatePolicy(
+    table: string,
+    policy: RetentionPolicyConfig,
+    items: PurgeCandidate[]
+  ): PolicyEvaluation {
+    const toPurge: PurgeCandidate[] = []
+    const toRetain: PurgeCandidate[] = []
+    const reasons = new Map<string, string>()
+    const now = Date.now()
+
+    switch (policy.type) {
+      case 'time': {
+        const retentionMs = policy.days * 24 * 60 * 60 * 1000
+        for (const item of items) {
+          const age = now - item.createdAt
+          if (age > retentionMs) {
+            toPurge.push(item)
+            reasons.set(item.id, `Older than ${policy.days} days`)
+          } else {
+            toRetain.push(item)
+            reasons.set(item.id, `Within ${policy.days} day retention`)
+          }
+        }
+        break
+      }
+
+      case 'count': {
+        // Sort by creation time
+        const sorted = [...items].sort((a, b) => {
+          return policy.keepNewest !== false ? b.createdAt - a.createdAt : a.createdAt - b.createdAt
+        })
+
+        for (let i = 0; i < sorted.length; i++) {
+          const item = sorted[i]
+          if (!item) continue
+          if (i < policy.snapshots) {
+            toRetain.push(item)
+            reasons.set(item.id, `Within top ${policy.snapshots} snapshots`)
+          } else {
+            toPurge.push(item)
+            reasons.set(item.id, `Exceeds ${policy.snapshots} snapshot limit`)
+          }
+        }
+        break
+      }
+
+      case 'size': {
+        const maxBytes = policy.maxGB * 1024 * 1024 * 1024
+        const sorted = [...items].sort((a, b) => {
+          return policy.evictOldestFirst !== false ? a.createdAt - b.createdAt : b.createdAt - a.createdAt
+        })
+
+        let cumulativeSize = 0
+        for (const item of sorted) {
+          if (cumulativeSize + item.sizeBytes <= maxBytes) {
+            toRetain.push(item)
+            cumulativeSize += item.sizeBytes
+            reasons.set(item.id, `Within ${policy.maxGB}GB size limit`)
+          } else {
+            toPurge.push(item)
+            reasons.set(item.id, `Exceeds ${policy.maxGB}GB size limit`)
+          }
+        }
+        break
+      }
+
+      case 'combined': {
+        // Apply all component policies and keep intersection
+        let currentToPurge = new Set(items.map((i) => i.id))
+        let currentToRetain = new Set<string>()
+
+        if (policy.time) {
+          const timePolicy: TimeBasedRetention = { type: 'time', ...policy.time }
+          const result = this.evaluatePolicy(table, timePolicy, items)
+          currentToRetain = new Set(result.toRetain.map((i) => i.id))
+          currentToPurge = new Set([...currentToPurge].filter((id) => !currentToRetain.has(id)))
+        }
+
+        if (policy.count) {
+          const countPolicy: CountBasedRetention = { type: 'count', ...policy.count }
+          const result = this.evaluatePolicy(table, countPolicy, items)
+          const countRetain = new Set(result.toRetain.map((i) => i.id))
+          currentToRetain = new Set([...currentToRetain].filter((id) => countRetain.has(id)))
+          currentToPurge = new Set(items.map((i) => i.id).filter((id) => !currentToRetain.has(id)))
+        }
+
+        if (policy.size) {
+          const sizePolicy: SizeBasedRetention = { type: 'size', ...policy.size }
+          const result = this.evaluatePolicy(table, sizePolicy, items)
+          const sizeRetain = new Set(result.toRetain.map((i) => i.id))
+          currentToRetain = new Set([...currentToRetain].filter((id) => sizeRetain.has(id)))
+          currentToPurge = new Set(items.map((i) => i.id).filter((id) => !currentToRetain.has(id)))
+        }
+
+        for (const item of items) {
+          if (currentToPurge.has(item.id)) {
+            toPurge.push(item)
+            reasons.set(item.id, 'Failed combined retention criteria')
+          } else {
+            toRetain.push(item)
+            reasons.set(item.id, 'Meets combined retention criteria')
+          }
+        }
+        break
+      }
+    }
+
+    return {
+      toPurge,
+      toRetain,
+      reasons,
+      legalHoldActive: false,
+    }
+  }
+
+  /**
+   * Create a GDPR erasure request
+   */
+  createErasureRequest(subjectId: string, scope: string[], requestedBy?: string): ErasureRequest {
+    const now = Date.now()
+    const request: ErasureRequest = {
+      id: `erasure-${now}-${Math.random().toString(36).substring(7)}`,
+      subjectId,
+      requestedAt: now,
+      deadline: now + 72 * 60 * 60 * 1000, // 72 hours
+      scope,
+      status: 'pending',
+    }
+
+    if (requestedBy !== undefined) {
+      request.requestedBy = requestedBy
+    }
+
+    this.erasureRequests.set(request.id, request)
+    return request
+  }
+
+  /**
+   * Get erasure request by ID
+   */
+  getErasureRequest(requestId: string): ErasureRequest | undefined {
+    return this.erasureRequests.get(requestId)
+  }
+
+  /**
+   * List all erasure requests
+   */
+  listErasureRequests(status?: ErasureRequest['status']): ErasureRequest[] {
+    const requests = Array.from(this.erasureRequests.values())
+    if (status) {
+      return requests.filter((r) => r.status === status)
+    }
+    return requests
+  }
+
+  /**
+   * Update erasure request status
+   */
+  updateErasureRequest(
+    requestId: string,
+    update: Partial<Pick<ErasureRequest, 'status' | 'completedAt' | 'error'>>
+  ): void {
+    const request = this.erasureRequests.get(requestId)
+    if (!request) {
+      throw new Error(`Erasure request not found: ${requestId}`)
+    }
+
+    Object.assign(request, update)
+  }
+
+  /**
+   * Add an audit entry
+   */
+  addAuditEntry(entry: Omit<AuditEntry, 'id' | 'timestamp'>): void {
+    const auditEntry: AuditEntry = {
+      id: `audit-${Date.now()}-${Math.random().toString(36).substring(7)}`,
+      timestamp: Date.now(),
+      ...entry,
+    }
+
+    this.auditLog.push(auditEntry)
+  }
+
+  /**
+   * Get audit trail
+   */
+  getAuditTrail(filter?: {
+    table?: string
+    operation?: PurgeOperation
+    startDate?: number
+    endDate?: number
+  }): AuditEntry[] {
+    let entries = [...this.auditLog]
+
+    if (filter?.table) {
+      entries = entries.filter((e) => e.table === filter.table)
+    }
+
+    if (filter?.operation) {
+      entries = entries.filter((e) => e.operation === filter.operation)
+    }
+
+    if (filter?.startDate) {
+      entries = entries.filter((e) => e.timestamp >= filter.startDate!)
+    }
+
+    if (filter?.endDate) {
+      entries = entries.filter((e) => e.timestamp <= filter.endDate!)
+    }
+
+    return entries.sort((a, b) => b.timestamp - a.timestamp)
+  }
+
+  /**
+   * Generate compliance report
+   */
+  generateComplianceReport(): {
+    profile?: ComplianceProfile
+    policies: Map<string, RetentionPolicyConfig>
+    legalHolds: Map<string, LegalHold>
+    activeErasureRequests: ErasureRequest[]
+    overdueErasureRequests: ErasureRequest[]
+    recentAuditEntries: AuditEntry[]
+    complianceStatus: 'compliant' | 'warning' | 'violation'
+    issues: string[]
+  } {
+    const now = Date.now()
+    const issues: string[] = []
+    let complianceStatus: 'compliant' | 'warning' | 'violation' = 'compliant'
+
+    // Check for overdue erasure requests
+    const activeRequests = this.listErasureRequests('pending').concat(
+      this.listErasureRequests('in_progress')
+    )
+    const overdueRequests = activeRequests.filter((r) => now > r.deadline)
+
+    if (overdueRequests.length > 0) {
+      complianceStatus = 'violation'
+      issues.push(`${overdueRequests.length} overdue GDPR erasure request(s)`)
+    }
+
+    // Check policies against compliance profile
+    if (this.complianceProfile) {
+      for (const [table, policy] of this.policies) {
+        try {
+          this.validatePolicyAgainstCompliance(table, policy)
+        } catch (error) {
+          complianceStatus = 'violation'
+          issues.push(error instanceof Error ? error.message : String(error))
+        }
+      }
+    }
+
+    // Recent audit entries (last 30 days)
+    const thirtyDaysAgo = now - 30 * 24 * 60 * 60 * 1000
+    const recentAuditEntries = this.getAuditTrail({ startDate: thirtyDaysAgo })
+
+    const result: {
+      profile?: ComplianceProfile
+      policies: Map<string, RetentionPolicyConfig>
+      legalHolds: Map<string, LegalHold>
+      activeErasureRequests: ErasureRequest[]
+      overdueErasureRequests: ErasureRequest[]
+      recentAuditEntries: AuditEntry[]
+      complianceStatus: 'compliant' | 'warning' | 'violation'
+      issues: string[]
+    } = {
+      policies: new Map(this.policies),
+      legalHolds: new Map(this.legalHolds),
+      activeErasureRequests: activeRequests,
+      overdueErasureRequests: overdueRequests,
+      recentAuditEntries,
+      complianceStatus,
+      issues,
+    }
+
+    if (this.complianceProfile !== undefined) {
+      result.profile = this.complianceProfile
+    }
+
+    return result
+  }
+}
+
+/**
+ * Create a retention policy with a compliance profile
+ *
+ * @example
+ * ```typescript
+ * const policy = createRetentionPolicy('gdpr')
+ * ```
+ */
+export function createRetentionPolicy(complianceProfile?: string): RetentionPolicy {
+  const policy = new RetentionPolicy()
+  if (complianceProfile) {
+    policy.applyComplianceProfile(complianceProfile)
+  }
+  return policy
+}