@dotdo/postgres 0.1.0

package/src/iceberg/snapshot-manager.ts

@@ -0,0 +1,1369 @@
+/**
+ * Iceberg Snapshot Management and Compaction
+ * Task: postgres-iyx - Snapshot management and compaction
+ *
+ * Provides snapshot lifecycle management and data compaction for Iceberg tables:
+ * - Snapshot listing and metadata retrieval
+ * - Snapshot expiration/retention
+ * - File compaction (merge small files)
+ * - Orphan file cleanup
+ * - Manifest rewriting for optimization
+ * - Background compaction scheduling
+ *
+ * @example
+ * ```typescript
+ * const manager = createSnapshotManager({
+ *   bucket: env.ICEBERG_BUCKET,
+ *   tablePath: 'iceberg/my-table',
+ * })
+ *
+ * await manager.initialize()
+ *
+ * // List snapshots
+ * const snapshots = manager.listSnapshots()
+ *
+ * // Expire old snapshots
+ * await manager.expireSnapshots({
+ *   maxAgeDays: 30,
+ *   minSnapshotsToRetain: 5,
+ * })
+ *
+ * // Compact small files
+ * await manager.compactFiles({
+ *   minFileSizeBytes: 10 * 1024 * 1024, // 10 MB
+ *   maxFilesPerCompaction: 100,
+ * })
+ *
+ * // Cleanup orphan files
+ * await manager.cleanupOrphanFiles()
+ * ```
+ */
+
+import type {
+  IcebergTableMetadata,
+  IcebergSnapshot,
+  IcebergManifestFile,
+  IcebergManifestEntry,
+  IcebergDataFile,
+  IcebergSnapshotSummary,
+} from './types'
+import {
+  IcebergMetadataBuilder,
+  ManifestBuilder,
+  ManifestListBuilder,
+  generateMetadataFileName,
+  generateManifestFileName,
+  generateManifestListFileName,
+} from './metadata'
+import { createLogger, LogLevel, type ILogger } from '@dotdo/postgres-shared'
+
+// Module-level logger for snapshot manager errors
+const snapshotLogger: ILogger = createLogger({
+  level: LogLevel.ERROR,
+  prefix: '[SnapshotManager]',
+})
+
+// ========== Types ==========
+
+/**
+ * Snapshot metadata with additional computed fields
+ */
+export interface SnapshotInfo {
+  /** Snapshot ID */
+  snapshotId: number
+  /** Parent snapshot ID (if any) */
+  parentSnapshotId?: number | undefined
+  /** Sequence number */
+  sequenceNumber: number
+  /** Snapshot timestamp */
+  timestampMs: number
+  /** Operation that created this snapshot */
+  operation: string
+  /** Summary statistics */
+  summary: {
+    addedDataFiles: number
+    deletedDataFiles: number
+    addedRecords: number
+    deletedRecords: number
+    totalRecords: number
+    totalFilesSize: number
+  }
+  /** Manifest list path */
+  manifestListPath: string
+  /** Schema ID used */
+  schemaId: number
+  /** Age of snapshot in milliseconds */
+  ageMs: number
+}
+
+/**
+ * Snapshot retention configuration
+ */
+export interface RetentionConfig {
+  /** Maximum age of snapshots to keep (in days) */
+  maxAgeDays?: number
+  /** Minimum number of snapshots to retain regardless of age */
+  minSnapshotsToRetain?: number
+  /** Maximum number of snapshots to keep */
+  maxSnapshotsToRetain?: number
+  /** Specific snapshot IDs to always retain (cherry-pick archival) */
+  retainSnapshotIds?: number[]
+}
+
+/**
+ * Compaction configuration
+ */
+export interface CompactionConfig {
+  /** Minimum file size threshold for compaction (bytes) */
+  minFileSizeBytes?: number
+  /** Maximum number of files to include in a single compaction */
+  maxFilesPerCompaction?: number
+  /** Maximum total input size for a single compaction (bytes) */
+  maxInputSizeBytes?: number
+  /** Target output file size (bytes) */
+  targetFileSizeBytes?: number
+  /** Compaction strategy */
+  strategy?: CompactionStrategy
+  /** Only compact files in specific partitions */
+  partitionFilter?: Record<string, unknown>
+  /** Skip compaction if file count is below this threshold */
+  minFilesToCompact?: number
+}
+
+/**
+ * Compaction strategy
+ */
+export type CompactionStrategy =
+  | 'bin-pack'      // Pack small files into target size
+  | 'sort'          // Sort and rewrite files
+  | 'z-order'       // Z-order optimization for multi-column queries
+
+/**
+ * Result of snapshot expiration
+ */
+export interface ExpireSnapshotsResult {
+  /** Number of snapshots expired */
+  snapshotsExpired: number
+  /** Snapshot IDs that were expired */
+  expiredSnapshotIds: number[]
+  /** Number of manifests removed */
+  manifestsRemoved: number
+  /** Number of manifest lists removed */
+  manifestListsRemoved: number
+  /** Bytes freed from metadata */
+  metadataBytesFreed: number
+  /** Duration in milliseconds */
+  durationMs: number
+}
+
+/**
+ * Result of file compaction
+ */
+export interface CompactFilesResult {
+  /** Number of input files */
+  inputFileCount: number
+  /** Number of output files */
+  outputFileCount: number
+  /** Total input size in bytes */
+  inputSizeBytes: number
+  /** Total output size in bytes */
+  outputSizeBytes: number
+  /** Size reduction in bytes */
+  sizeReductionBytes: number
+  /** Records processed */
+  recordsProcessed: number
+  /** Partitions compacted */
+  partitionsCompacted: number
+  /** New snapshot created */
+  snapshot: IcebergSnapshot | null
+  /** Duration in milliseconds */
+  durationMs: number
+}
+
+/**
+ * Result of orphan file cleanup
+ */
+export interface CleanupOrphanFilesResult {
+  /** Number of orphan files found */
+  orphanFilesFound: number
+  /** Number of orphan files deleted */
+  orphanFilesDeleted: number
+  /** Orphan file paths */
+  orphanFilePaths: string[]
+  /** Bytes freed */
+  bytesFreed: number
+  /** Duration in milliseconds */
+  durationMs: number
+}
+
+/**
+ * File statistics for compaction analysis
+ */
+export interface FileStatistics {
+  /** Total number of data files */
+  totalFiles: number
+  /** Total size of all files in bytes */
+  totalSizeBytes: number
+  /** Average file size in bytes */
+  avgFileSizeBytes: number
+  /** Minimum file size in bytes */
+  minFileSizeBytes: number
+  /** Maximum file size in bytes */
+  maxFileSizeBytes: number
+  /** Number of files below compaction threshold */
+  smallFileCount: number
+  /** Size of small files in bytes */
+  smallFilesSizeBytes: number
+  /** Files by partition */
+  filesByPartition: Map<string, number>
+}
+
+/**
+ * Compaction trigger result
+ */
+export interface CompactionTriggerResult {
+  /** Whether compaction should be triggered */
+  shouldCompact: boolean
+  /** Reason for the decision */
+  reason: string
+  /** Suggested compaction priority (0-100) */
+  priority: number
+  /** Estimated benefit (bytes to be freed) */
+  estimatedBenefit: number
+}
+
+/**
+ * Snapshot manager configuration
+ */
+export interface SnapshotManagerConfig {
+  /** R2 bucket for Iceberg storage */
+  bucket: R2Bucket
+  /** Base path for the table in R2 */
+  tablePath: string
+  /** DO state for storing manager state (optional) */
+  state?: DurableObjectState | undefined
+}
+
+/**
+ * Snapshot manager statistics
+ */
+export interface SnapshotManagerStats {
+  /** Total snapshots expired */
+  totalSnapshotsExpired: number
+  /** Total compactions performed */
+  totalCompactions: number
+  /** Total bytes freed from expiration */
+  totalBytesFreedExpiration: number
+  /** Total bytes saved from compaction */
+  totalBytesSavedCompaction: number
+  /** Total orphan files cleaned */
+  totalOrphanFilesCleaned: number
+  /** Last expiration timestamp */
+  lastExpirationAt: number | null
+  /** Last compaction timestamp */
+  lastCompactionAt: number | null
+  /** Last cleanup timestamp */
+  lastCleanupAt: number | null
+}
+
+// ========== Default Configuration ==========
+
+const DEFAULT_RETENTION: Required<RetentionConfig> = {
+  maxAgeDays: 30,
+  minSnapshotsToRetain: 5,
+  maxSnapshotsToRetain: 100,
+  retainSnapshotIds: [],
+}
+
+const DEFAULT_COMPACTION: Required<CompactionConfig> = {
+  minFileSizeBytes: 10 * 1024 * 1024, // 10 MB
+  maxFilesPerCompaction: 100,
+  maxInputSizeBytes: 1024 * 1024 * 1024, // 1 GB
+  targetFileSizeBytes: 128 * 1024 * 1024, // 128 MB
+  strategy: 'bin-pack',
+  partitionFilter: {},
+  minFilesToCompact: 5,
+}
+
+// ========== Snapshot Manager Class ==========
+
+/**
+ * Iceberg Snapshot Manager
+ *
+ * Manages snapshot lifecycle including expiration, compaction, and cleanup.
+ */
+export class SnapshotManager {
+  private bucket: R2Bucket
+  private tablePath: string
+  private metadata: IcebergTableMetadata | null = null
+  private metadataBuilder: IcebergMetadataBuilder | null = null
+  private metadataVersion = 0
+  private initialized = false
+
+  private stats: SnapshotManagerStats = {
+    totalSnapshotsExpired: 0,
+    totalCompactions: 0,
+    totalBytesFreedExpiration: 0,
+    totalBytesSavedCompaction: 0,
+    totalOrphanFilesCleaned: 0,
+    lastExpirationAt: null,
+    lastCompactionAt: null,
+    lastCleanupAt: null,
+  }
+
+  constructor(config: SnapshotManagerConfig) {
+    this.bucket = config.bucket
+    this.tablePath = config.tablePath.replace(/\/$/, '')
+    // state is accepted for future use but not currently utilized
+    void config.state
+  }
+
+  /**
+   * Initialize the snapshot manager
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) {
+      return
+    }
+
+    await this.loadMetadata()
+    this.initialized = true
+  }
+
+  /**
+   * Ensure initialized
+   */
+  private ensureInitialized(): void {
+    if (!this.initialized) {
+      throw new Error('SnapshotManager not initialized. Call initialize() first.')
+    }
+  }
+
+  /**
+   * Load table metadata from R2
+   */
+  private async loadMetadata(): Promise<void> {
+    // Try version hint first
+    const versionHint = await this.bucket.get(`${this.tablePath}/metadata/version-hint.text`)
+    if (versionHint) {
+      this.metadataVersion = parseInt(await versionHint.text(), 10)
+      const metadataPath = `${this.tablePath}/metadata/v${this.metadataVersion}.metadata.json`
+      const metadataObj = await this.bucket.get(metadataPath)
+      if (metadataObj) {
+        this.metadata = JSON.parse(await metadataObj.text()) as IcebergTableMetadata
+        this.metadataBuilder = IcebergMetadataBuilder.fromMetadata(this.metadata)
+        return
+      }
+    }
+
+    // Fall back to listing metadata files
+    const list = await this.bucket.list({
+      prefix: `${this.tablePath}/metadata/`,
+    })
+
+    const metadataFiles = list.objects
+      .filter((obj) => obj.key.endsWith('.metadata.json'))
+      .sort((a, b) => {
+        const versionA = parseInt(a.key.match(/v(\d+)\.metadata\.json/)?.[1] ?? '0', 10)
+        const versionB = parseInt(b.key.match(/v(\d+)\.metadata\.json/)?.[1] ?? '0', 10)
+        return versionB - versionA
+      })
+
+    if (metadataFiles.length > 0 && metadataFiles[0]) {
+      const metadataObj = await this.bucket.get(metadataFiles[0].key)
+      if (metadataObj) {
+        this.metadata = JSON.parse(await metadataObj.text()) as IcebergTableMetadata
+        this.metadataBuilder = IcebergMetadataBuilder.fromMetadata(this.metadata)
+        this.metadataVersion = parseInt(
+          metadataFiles[0].key.match(/v(\d+)\.metadata\.json/)?.[1] ?? '0',
+          10
+        )
+      }
+    }
+  }
+
+  /**
+   * Refresh metadata from R2
+   */
+  async refreshMetadata(): Promise<void> {
+    this.initialized = false
+    this.metadata = null
+    this.metadataBuilder = null
+    await this.initialize()
+  }
+
+  // ========== Snapshot Listing and Metadata ==========
+
+  /**
+   * List all snapshots with computed metadata
+   */
+  listSnapshots(): SnapshotInfo[] {
+    this.ensureInitialized()
+
+    if (!this.metadata?.snapshots) {
+      return []
+    }
+
+    const now = Date.now()
+    return this.metadata.snapshots.map((snapshot) => this.snapshotToInfo(snapshot, now))
+  }
+
+  /**
+   * Get a specific snapshot by ID
+   */
+  getSnapshot(snapshotId: number): SnapshotInfo | null {
+    this.ensureInitialized()
+
+    const snapshot = this.metadata?.snapshots?.find((s) => s['snapshot-id'] === snapshotId)
+    if (!snapshot) {
+      return null
+    }
+
+    return this.snapshotToInfo(snapshot, Date.now())
+  }
+
+  /**
+   * Get the current snapshot
+   */
+  getCurrentSnapshot(): SnapshotInfo | null {
+    this.ensureInitialized()
+
+    if (!this.metadata?.['current-snapshot-id']) {
+      return null
+    }
+
+    return this.getSnapshot(this.metadata['current-snapshot-id'])
+  }
+
+  /**
+   * Get snapshot ancestry (parent chain)
+   */
+  getSnapshotAncestry(snapshotId: number): SnapshotInfo[] {
+    this.ensureInitialized()
+
+    const ancestry: SnapshotInfo[] = []
+    let currentId: number | undefined = snapshotId
+
+    while (currentId !== undefined) {
+      const snapshot = this.getSnapshot(currentId)
+      if (!snapshot) {
+        break
+      }
+      ancestry.push(snapshot)
+      currentId = snapshot.parentSnapshotId
+    }
+
+    return ancestry
+  }
+
+  /**
+   * Convert IcebergSnapshot to SnapshotInfo
+   */
+  private snapshotToInfo(snapshot: IcebergSnapshot, now: number): SnapshotInfo {
+    const summary = snapshot.summary
+    return {
+      snapshotId: snapshot['snapshot-id'],
+      parentSnapshotId: snapshot['parent-snapshot-id'],
+      sequenceNumber: snapshot['sequence-number'],
+      timestampMs: snapshot['timestamp-ms'],
+      operation: summary.operation,
+      summary: {
+        addedDataFiles: parseInt(summary['added-data-files'] ?? '0', 10),
+        deletedDataFiles: parseInt(summary['deleted-data-files'] ?? '0', 10),
+        addedRecords: parseInt(summary['added-records'] ?? '0', 10),
+        deletedRecords: parseInt(summary['deleted-records'] ?? '0', 10),
+        totalRecords: parseInt(summary['total-records'] ?? '0', 10),
+        totalFilesSize: parseInt(summary['total-files-size'] ?? '0', 10),
+      },
+      manifestListPath: snapshot['manifest-list'],
+      schemaId: snapshot['schema-id'] ?? 0,
+      ageMs: now - snapshot['timestamp-ms'],
+    }
+  }
+
+  // ========== Snapshot Expiration ==========
+
+  /**
+   * Expire old snapshots based on retention policy
+   */
+  async expireSnapshots(config: RetentionConfig = {}): Promise<ExpireSnapshotsResult> {
+    this.ensureInitialized()
+
+    const startTime = performance.now()
+    const retention = { ...DEFAULT_RETENTION, ...config }
+
+    if (!this.metadata?.snapshots || this.metadata.snapshots.length === 0) {
+      return {
+        snapshotsExpired: 0,
+        expiredSnapshotIds: [],
+        manifestsRemoved: 0,
+        manifestListsRemoved: 0,
+        metadataBytesFreed: 0,
+        durationMs: performance.now() - startTime,
+      }
+    }
+
+    const now = Date.now()
+    const maxAgeMs = retention.maxAgeDays * 24 * 60 * 60 * 1000
+    const currentSnapshotId = this.metadata['current-snapshot-id']
+
+    // Identify snapshots to expire
+    const snapshots = [...this.metadata.snapshots]
+    const snapshotsToKeep: IcebergSnapshot[] = []
+    const snapshotsToExpire: IcebergSnapshot[] = []
+
+    // Sort by timestamp (newest first) for minSnapshots logic
+    snapshots.sort((a, b) => b['timestamp-ms'] - a['timestamp-ms'])
+
+    for (let i = 0; i < snapshots.length; i++) {
+      const snapshot = snapshots[i]!
+      const age = now - snapshot['timestamp-ms']
+      const isCurrentSnapshot = snapshot['snapshot-id'] === currentSnapshotId
+      const isRetainedById = retention.retainSnapshotIds.includes(snapshot['snapshot-id'])
+      const withinMinSnapshots = i < retention.minSnapshotsToRetain
+      const withinMaxAge = age <= maxAgeMs
+      const beyondMaxSnapshots = i >= retention.maxSnapshotsToRetain
+
+      if (isCurrentSnapshot || isRetainedById || (withinMinSnapshots && !beyondMaxSnapshots)) {
+        snapshotsToKeep.push(snapshot)
+      } else if (!withinMaxAge || beyondMaxSnapshots) {
+        snapshotsToExpire.push(snapshot)
+      } else {
+        snapshotsToKeep.push(snapshot)
+      }
+    }
+
+    if (snapshotsToExpire.length === 0) {
+      return {
+        snapshotsExpired: 0,
+        expiredSnapshotIds: [],
+        manifestsRemoved: 0,
+        manifestListsRemoved: 0,
+        metadataBytesFreed: 0,
+        durationMs: performance.now() - startTime,
+      }
+    }
+
+    // Collect manifest lists and manifests to delete
+    const manifestListsToDelete: string[] = []
+    const manifestsToDelete: Set<string> = new Set()
+    let metadataBytesFreed = 0
+
+    // Get manifests referenced by snapshots to keep
+    const keptManifests = new Set<string>()
+    for (const snapshot of snapshotsToKeep) {
+      const manifests = await this.loadManifestsFromList(snapshot['manifest-list'])
+      for (const manifest of manifests) {
+        keptManifests.add(manifest['manifest-path'])
+      }
+    }
+
+    // Identify manifest lists and manifests to delete
+    for (const snapshot of snapshotsToExpire) {
+      manifestListsToDelete.push(snapshot['manifest-list'])
+
+      const manifests = await this.loadManifestsFromList(snapshot['manifest-list'])
+      for (const manifest of manifests) {
+        // Only delete if not referenced by a kept snapshot
+        if (!keptManifests.has(manifest['manifest-path'])) {
+          manifestsToDelete.add(manifest['manifest-path'])
+          metadataBytesFreed += manifest['manifest-length']
+        }
+      }
+    }
+
+    // Delete manifest files
+    for (const manifestPath of manifestsToDelete) {
+      try {
+        await this.bucket.delete(manifestPath)
+      } catch (error) {
+        snapshotLogger.error('Failed to delete manifest', { manifestPath, error: error instanceof Error ? error.message : String(error) })
+      }
+    }
+
+    // Delete manifest lists
+    for (const manifestListPath of manifestListsToDelete) {
+      try {
+        const obj = await this.bucket.head(manifestListPath)
+        if (obj) {
+          metadataBytesFreed += obj.size
+        }
+        await this.bucket.delete(manifestListPath)
+      } catch (error) {
+        snapshotLogger.error('Failed to delete manifest list', { manifestListPath, error: error instanceof Error ? error.message : String(error) })
+      }
+    }
+
+    // Update metadata with remaining snapshots
+    await this.updateMetadataWithSnapshots(snapshotsToKeep)
+
+    // Update stats
+    const result: ExpireSnapshotsResult = {
+      snapshotsExpired: snapshotsToExpire.length,
+      expiredSnapshotIds: snapshotsToExpire.map((s) => s['snapshot-id']),
+      manifestsRemoved: manifestsToDelete.size,
+      manifestListsRemoved: manifestListsToDelete.length,
+      metadataBytesFreed,
+      durationMs: performance.now() - startTime,
+    }
+
+    this.stats.totalSnapshotsExpired += result.snapshotsExpired
+    this.stats.totalBytesFreedExpiration += result.metadataBytesFreed
+    this.stats.lastExpirationAt = Date.now()
+
+    return result
+  }
+
+  /**
+   * Load manifests from a manifest list
+   */
+  private async loadManifestsFromList(manifestListPath: string): Promise<IcebergManifestFile[]> {
+    try {
+      const obj = await this.bucket.get(manifestListPath)
+      if (!obj) {
+        return []
+      }
+
+      const data = JSON.parse(await obj.text())
+      return data.manifests ?? []
+    } catch (error) {
+      snapshotLogger.error('Failed to load manifest list', { manifestListPath, error: error instanceof Error ? error.message : String(error) })
+      return []
+    }
+  }
+
+  /**
+   * Update metadata with new snapshot list
+   */
+  private async updateMetadataWithSnapshots(snapshots: IcebergSnapshot[]): Promise<void> {
+    if (!this.metadata || !this.metadataBuilder) {
+      throw new Error('Metadata not loaded')
+    }
+
+    // Sort snapshots by timestamp
+    snapshots.sort((a, b) => a['timestamp-ms'] - b['timestamp-ms'])
+
+    // Create new metadata
+    const newMetadata: IcebergTableMetadata = {
+      ...this.metadata,
+      'last-updated-ms': Date.now(),
+      snapshots: snapshots,
+      'snapshot-log': snapshots.map((s) => ({
+        'timestamp-ms': s['timestamp-ms'],
+        'snapshot-id': s['snapshot-id'],
+      })),
+    }
+
+    // Update current snapshot if needed
+    if (
+      newMetadata['current-snapshot-id'] !== undefined &&
+      !snapshots.some((s) => s['snapshot-id'] === newMetadata['current-snapshot-id'])
+    ) {
+      // Current snapshot was expired, set to latest
+      const latestSnapshot = snapshots[snapshots.length - 1]
+      if (latestSnapshot) {
+        newMetadata['current-snapshot-id'] = latestSnapshot['snapshot-id']
+      } else {
+        delete newMetadata['current-snapshot-id']
+      }
+    }
+
+    // Write new metadata version
+    this.metadataVersion++
+    const metadataFileName = generateMetadataFileName(this.metadataVersion)
+    const metadataPath = `${this.tablePath}/metadata/${metadataFileName}`
+
+    await this.bucket.put(metadataPath, JSON.stringify(newMetadata, null, 2), {
+      httpMetadata: { contentType: 'application/json' },
+    })
+
+    // Update version hint
+    await this.bucket.put(
+      `${this.tablePath}/metadata/version-hint.text`,
+      String(this.metadataVersion),
+      { httpMetadata: { contentType: 'text/plain' } }
+    )
+
+    // Update internal state
+    this.metadata = newMetadata
+    this.metadataBuilder = IcebergMetadataBuilder.fromMetadata(newMetadata)
+  }
+
+  // ========== File Compaction ==========
+
+  /**
+   * Analyze file statistics for compaction decisions
+   */
+  async analyzeFileStatistics(threshold: number = DEFAULT_COMPACTION.minFileSizeBytes): Promise<FileStatistics> {
+    this.ensureInitialized()
+
+    const dataFiles = await this.getAllDataFiles()
+
+    const filesByPartition = new Map<string, number>()
+    let totalSize = 0
+    let minSize = Infinity
+    let maxSize = 0
+    let smallFileCount = 0
+    let smallFilesSize = 0
+
+    for (const file of dataFiles) {
+      const size = file['file-size-in-bytes']
+      totalSize += size
+      minSize = Math.min(minSize, size)
+      maxSize = Math.max(maxSize, size)
+
+      if (size < threshold) {
+        smallFileCount++
+        smallFilesSize += size
+      }
+
+      // Extract partition path
+      const partitionPath = this.extractPartitionPath(file['file-path'])
+      filesByPartition.set(partitionPath, (filesByPartition.get(partitionPath) ?? 0) + 1)
+    }
+
+    return {
+      totalFiles: dataFiles.length,
+      totalSizeBytes: totalSize,
+      avgFileSizeBytes: dataFiles.length > 0 ? totalSize / dataFiles.length : 0,
+      minFileSizeBytes: minSize === Infinity ? 0 : minSize,
+      maxFileSizeBytes: maxSize,
+      smallFileCount,
+      smallFilesSizeBytes: smallFilesSize,
+      filesByPartition,
+    }
+  }
+
+  /**
+   * Check if compaction should be triggered
+   */
+  async shouldCompact(config: CompactionConfig = {}): Promise<CompactionTriggerResult> {
+    const compactionConfig = { ...DEFAULT_COMPACTION, ...config }
+    const stats = await this.analyzeFileStatistics(compactionConfig.minFileSizeBytes)
+
+    // Check file count threshold
+    if (stats.smallFileCount < compactionConfig.minFilesToCompact) {
+      return {
+        shouldCompact: false,
+        reason: `Small file count (${stats.smallFileCount}) below threshold (${compactionConfig.minFilesToCompact})`,
+        priority: 0,
+        estimatedBenefit: 0,
+      }
+    }
+
+    // Calculate compaction benefit
+    const avgSmallFileSize = stats.smallFileCount > 0
+      ? stats.smallFilesSizeBytes / stats.smallFileCount
+      : 0
+    const targetFileCount = Math.ceil(stats.smallFilesSizeBytes / compactionConfig.targetFileSizeBytes)
+    const estimatedOutputFiles = Math.max(targetFileCount, 1)
+    const fileReduction = stats.smallFileCount - estimatedOutputFiles
+
+    // Priority based on file count and size ratio
+    const fileCountRatio = stats.smallFileCount / stats.totalFiles
+    const sizeRatio = avgSmallFileSize / compactionConfig.targetFileSizeBytes
+    const priority = Math.min(100, Math.round((fileCountRatio * 50) + ((1 - sizeRatio) * 50)))
+
+    return {
+      shouldCompact: true,
+      reason: `${stats.smallFileCount} small files (${formatBytes(stats.smallFilesSizeBytes)}) can be compacted`,
+      priority,
+      estimatedBenefit: fileReduction * 1000, // Rough estimate of metadata savings per file
+    }
+  }
+
+  /**
+   * Compact small files
+   *
+   * Note: Full compaction requires reading and rewriting Parquet files.
+   * This implementation tracks which files should be compacted and creates
+   * a new snapshot with the compaction metadata. Actual file rewriting
+   * would require Parquet library support.
+   */
+  async compactFiles(config: CompactionConfig = {}): Promise<CompactFilesResult> {
+    this.ensureInitialized()
+
+    const startTime = performance.now()
+    const compactionConfig = { ...DEFAULT_COMPACTION, ...config }
+
+    // Get all small files
+    const allDataFiles = await this.getAllDataFiles()
+    const smallFiles = allDataFiles.filter(
+      (f) => f['file-size-in-bytes'] < compactionConfig.minFileSizeBytes
+    )
+
+    // Group by partition
+    const filesByPartition = new Map<string, IcebergDataFile[]>()
+    for (const file of smallFiles) {
+      const partitionPath = this.extractPartitionPath(file['file-path'])
+
+      // Apply partition filter if specified
+      if (Object.keys(compactionConfig.partitionFilter).length > 0) {
+        const matches = this.matchesPartitionFilter(file.partition, compactionConfig.partitionFilter)
+        if (!matches) {
+          continue
+        }
+      }
+
+      const files = filesByPartition.get(partitionPath) ?? []
+      files.push(file)
+      filesByPartition.set(partitionPath, files)
+    }
+
+    // Calculate compaction groups
+    const compactionGroups: Array<{
+      partition: string
+      files: IcebergDataFile[]
+      totalSize: number
+    }> = []
+
+    for (const [partition, files] of filesByPartition) {
+      if (files.length < compactionConfig.minFilesToCompact) {
+        continue
+      }
+
+      // Sort by size (ascending) for bin-packing
+      files.sort((a, b) => a['file-size-in-bytes'] - b['file-size-in-bytes'])
+
+      // Create groups respecting maxFilesPerCompaction and maxInputSizeBytes
+      let currentGroup: IcebergDataFile[] = []
+      let currentSize = 0
+
+      for (const file of files) {
+        if (
+          currentGroup.length >= compactionConfig.maxFilesPerCompaction ||
+          currentSize + file['file-size-in-bytes'] > compactionConfig.maxInputSizeBytes
+        ) {
+          if (currentGroup.length >= compactionConfig.minFilesToCompact) {
+            compactionGroups.push({
+              partition,
+              files: currentGroup,
+              totalSize: currentSize,
+            })
+          }
+          currentGroup = []
+          currentSize = 0
+        }
+
+        currentGroup.push(file)
+        currentSize += file['file-size-in-bytes']
+      }
+
+      // Add remaining files
+      if (currentGroup.length >= compactionConfig.minFilesToCompact) {
+        compactionGroups.push({
+          partition,
+          files: currentGroup,
+          totalSize: currentSize,
+        })
+      }
+    }
+
+    if (compactionGroups.length === 0) {
+      return {
+        inputFileCount: 0,
+        outputFileCount: 0,
+        inputSizeBytes: 0,
+        outputSizeBytes: 0,
+        sizeReductionBytes: 0,
+        recordsProcessed: 0,
+        partitionsCompacted: 0,
+        snapshot: null,
+        durationMs: performance.now() - startTime,
+      }
+    }
+
+    // Calculate totals
+    let totalInputFiles = 0
+    let totalInputSize = 0
+    let totalRecords = 0
+    const partitionsCompacted = new Set<string>()
+
+    for (const group of compactionGroups) {
+      totalInputFiles += group.files.length
+      totalInputSize += group.totalSize
+      partitionsCompacted.add(group.partition)
+      for (const file of group.files) {
+        totalRecords += file['record-count']
+      }
+    }
+
+    // Create compaction snapshot
+    // Note: In a full implementation, this would:
+    // 1. Read all input Parquet files
+    // 2. Merge and write new Parquet files
+    // 3. Create manifest with DELETED entries for old files and ADDED for new
+    // For now, we create a metadata-only compaction marker
+
+    const snapshot = await this.createCompactionSnapshot(
+      compactionGroups,
+      compactionConfig.targetFileSizeBytes
+    )
+
+    // Calculate output estimates
+    const estimatedOutputFiles = compactionGroups.reduce((sum, group) => {
+      return sum + Math.ceil(group.totalSize / compactionConfig.targetFileSizeBytes)
+    }, 0)
+
+    const result: CompactFilesResult = {
+      inputFileCount: totalInputFiles,
+      outputFileCount: estimatedOutputFiles,
+      inputSizeBytes: totalInputSize,
+      outputSizeBytes: totalInputSize, // In reality would be slightly different
+      sizeReductionBytes: 0, // Actual reduction from file consolidation
+      recordsProcessed: totalRecords,
+      partitionsCompacted: partitionsCompacted.size,
+      snapshot,
+      durationMs: performance.now() - startTime,
+    }
+
+    this.stats.totalCompactions++
+    this.stats.totalBytesSavedCompaction += result.sizeReductionBytes
+    this.stats.lastCompactionAt = Date.now()
+
+    return result
+  }
+
+  /**
+   * Create a compaction snapshot
+   */
+  private async createCompactionSnapshot(
+    compactionGroups: Array<{
+      partition: string
+      files: IcebergDataFile[]
+      totalSize: number
+    }>,
+    _targetFileSizeBytes: number
+  ): Promise<IcebergSnapshot | null> {
+    if (!this.metadata || !this.metadataBuilder) {
+      return null
+    }
+
+    const snapshotId = Date.now() * 1000 + Math.floor(Math.random() * 1000)
+    const sequenceNumber = this.metadataBuilder.getSequenceNumber() + 1
+    const partitionSpecId = this.metadata['default-spec-id']
+
+    // Create manifest builder for compaction
+    const manifestBuilder = new ManifestBuilder({
+      partitionSpecId,
+      sequenceNumber,
+      snapshotId,
+    })
+
+    // Mark old files as deleted, add placeholder for compacted files
+    let totalDeletedFiles = 0
+    let totalDeletedRecords = 0
+
+    for (const group of compactionGroups) {
+      for (const file of group.files) {
+        manifestBuilder.deleteDataFile(file)
+        totalDeletedFiles++
+        totalDeletedRecords += file['record-count']
+      }
+
+      // In a real implementation, we would add the newly compacted files here
+      // For now, we just mark the intent to compact
+    }
+
+    // Write manifest
+    const manifestFileName = generateManifestFileName(snapshotId)
+    const manifestPath = `${this.tablePath}/metadata/${manifestFileName}`
+    const manifestContent = manifestBuilder.toJSON()
+    const manifestBytes = new TextEncoder().encode(manifestContent)
+
+    await this.bucket.put(manifestPath, manifestBytes, {
+      httpMetadata: { contentType: 'application/json' },
+    })
+
+    const manifestFileMetadata = manifestBuilder.createManifestFileMetadata(
+      manifestPath,
+      manifestBytes.length
+    )
+
+    // Create manifest list
+    const manifestListBuilder = new ManifestListBuilder(snapshotId, sequenceNumber)
+    manifestListBuilder.addManifest(manifestFileMetadata)
+
+    // Include existing manifests
+    const currentSnapshot = this.metadata.snapshots?.find(
+      (s) => s['snapshot-id'] === this.metadata!['current-snapshot-id']
+    )
+    if (currentSnapshot) {
+      const existingManifests = await this.loadManifestsFromList(currentSnapshot['manifest-list'])
+      for (const manifest of existingManifests) {
+        manifestListBuilder.addManifest(manifest)
+      }
+    }
+
+    // Write manifest list
+    const manifestListFileName = generateManifestListFileName(snapshotId)
+    const manifestListPath = `${this.tablePath}/metadata/${manifestListFileName}`
+    const manifestListContent = manifestListBuilder.toJSON()
+    const manifestListBytes = new TextEncoder().encode(manifestListContent)
+
+    await this.bucket.put(manifestListPath, manifestListBytes, {
+      httpMetadata: { contentType: 'application/json' },
+    })
+
+    // Create snapshot
+    const summary: IcebergSnapshotSummary = {
+      operation: 'replace',
+      'added-data-files': '0',
+      'deleted-data-files': String(totalDeletedFiles),
+      'added-records': '0',
+      'deleted-records': String(totalDeletedRecords),
+      'compaction-type': 'bin-pack',
+    }
+
+    const snapshot: IcebergSnapshot = {
+      'snapshot-id': snapshotId,
+      ...(this.metadata['current-snapshot-id'] !== undefined && {
+        'parent-snapshot-id': this.metadata['current-snapshot-id'],
+      }),
+      'sequence-number': sequenceNumber,
+      'timestamp-ms': Date.now(),
+      'manifest-list': manifestListPath,
+      summary,
+      'schema-id': this.metadata['current-schema-id'],
+    }
+
+    // Update metadata
+    const newSnapshots = [...(this.metadata.snapshots ?? []), snapshot]
+    await this.updateMetadataWithSnapshots(newSnapshots)
+
+    return snapshot
+  }
+
+  // ========== Orphan File Cleanup ==========
+
+  /**
+   * Find and clean up orphan files
+   *
+   * Orphan files are data files that exist in R2 but are not referenced
+   * by any manifest in any snapshot.
+   */
+  async cleanupOrphanFiles(dryRun = false): Promise<CleanupOrphanFilesResult> {
+    this.ensureInitialized()
+
+    const startTime = performance.now()
+
+    // Get all referenced data files from all snapshots
+    const referencedFiles = new Set<string>()
+
+    if (this.metadata?.snapshots) {
+      for (const snapshot of this.metadata.snapshots) {
+        const manifests = await this.loadManifestsFromList(snapshot['manifest-list'])
+        for (const manifest of manifests) {
+          const entries = await this.loadManifestEntries(manifest['manifest-path'])
+          for (const entry of entries) {
+            // Only count EXISTING and ADDED files (not DELETED)
+            if (entry.status !== 2) {
+              referencedFiles.add(entry['data-file']['file-path'])
+            }
+          }
+        }
+      }
+    }
+
+    // List all data files in R2
+    const orphanFiles: Array<{ key: string; size: number }> = []
+    let cursor: string | undefined
+
+    do {
+      const listOptions: R2ListOptions = {
+        prefix: `${this.tablePath}/data/`,
+        limit: 1000,
+      }
+      if (cursor) {
+        listOptions.cursor = cursor
+      }
+
+      const result = await this.bucket.list(listOptions)
+
+      for (const obj of result.objects) {
+        if (obj.key.endsWith('.parquet') && !referencedFiles.has(obj.key)) {
+          orphanFiles.push({ key: obj.key, size: obj.size })
+        }
+      }
+
+      cursor = result.truncated ? result.cursor : undefined
+    } while (cursor)
+
+    // Delete orphan files (unless dry run)
+    let bytesFreed = 0
+    const deletedPaths: string[] = []
+
+    if (!dryRun) {
+      for (const orphan of orphanFiles) {
+        try {
+          await this.bucket.delete(orphan.key)
+          bytesFreed += orphan.size
+          deletedPaths.push(orphan.key)
+        } catch (error) {
+          snapshotLogger.error('Failed to delete orphan file', { key: orphan.key, error: error instanceof Error ? error.message : String(error) })
+        }
+      }
+    } else {
+      bytesFreed = orphanFiles.reduce((sum, f) => sum + f.size, 0)
+    }
+
+    const result: CleanupOrphanFilesResult = {
+      orphanFilesFound: orphanFiles.length,
+      orphanFilesDeleted: dryRun ? 0 : deletedPaths.length,
+      orphanFilePaths: orphanFiles.map((f) => f.key),
+      bytesFreed,
+      durationMs: performance.now() - startTime,
+    }
+
+    if (!dryRun) {
+      this.stats.totalOrphanFilesCleaned += result.orphanFilesDeleted
+      this.stats.lastCleanupAt = Date.now()
+    }
+
+    return result
+  }
+
+  /**
+   * Load manifest entries from a manifest file
+   */
+  private async loadManifestEntries(manifestPath: string): Promise<IcebergManifestEntry[]> {
+    try {
+      const obj = await this.bucket.get(manifestPath)
+      if (!obj) {
+        return []
+      }
+
+      const data = JSON.parse(await obj.text())
+      return data.entries ?? []
+    } catch (error) {
+      snapshotLogger.error('Failed to load manifest', { manifestPath, error: error instanceof Error ? error.message : String(error) })
+      return []
+    }
+  }
+
+  /**
+   * Get all data files from the current snapshot
+   */
+  private async getAllDataFiles(): Promise<IcebergDataFile[]> {
+    if (!this.metadata?.['current-snapshot-id']) {
+      return []
+    }
+
+    const currentSnapshot = this.metadata.snapshots?.find(
+      (s) => s['snapshot-id'] === this.metadata!['current-snapshot-id']
+    )
+
+    if (!currentSnapshot) {
+      return []
+    }
+
+    const dataFiles: IcebergDataFile[] = []
+    const manifests = await this.loadManifestsFromList(currentSnapshot['manifest-list'])
+
+    for (const manifest of manifests) {
+      if (manifest.content !== 0) {
+        continue // Skip delete manifests
+      }
+
+      const entries = await this.loadManifestEntries(manifest['manifest-path'])
+      for (const entry of entries) {
+        // Only include EXISTING (0) and ADDED (1) files
+        if (entry.status !== 2) {
+          dataFiles.push(entry['data-file'])
+        }
+      }
+    }
+
+    return dataFiles
+  }
+
+  /**
+   * Extract partition path from file path
+   */
+  private extractPartitionPath(filePath: string): string {
+    const dataIndex = filePath.indexOf('/data/')
+    if (dataIndex === -1) {
+      return ''
+    }
+
+    const afterData = filePath.substring(dataIndex + 6) // Skip '/data/'
+    const lastSlash = afterData.lastIndexOf('/')
+    if (lastSlash === -1) {
+      return ''
+    }
+
+    return afterData.substring(0, lastSlash)
+  }
+
+  /**
+   * Check if partition values match filter
+   */
+  private matchesPartitionFilter(
+    partition: Record<string, unknown>,
+    filter: Record<string, unknown>
+  ): boolean {
+    for (const [key, value] of Object.entries(filter)) {
+      if (partition[key] !== value) {
+        return false
+      }
+    }
+    return true
+  }
+
+  // ========== Utility Methods ==========
+
+  /**
+   * Get manager statistics
+   */
+  getStats(): SnapshotManagerStats {
+    return { ...this.stats }
+  }
+
+  /**
+   * Get the table metadata
+   */
+  getMetadata(): IcebergTableMetadata | null {
+    return this.metadata
+  }
+
+  /**
+   * Get the table path
+   */
+  getTablePath(): string {
+    return this.tablePath
+  }
+
+  /**
+   * Get the current metadata version
+   */
+  getMetadataVersion(): number {
+    return this.metadataVersion
+  }
+}
+
+// ========== Factory Function ==========
+
+/**
+ * Create a snapshot manager
+ */
+export function createSnapshotManager(config: SnapshotManagerConfig): SnapshotManager {
+  return new SnapshotManager(config)
+}
+
+// ========== Utility Functions ==========
+
+/**
+ * Format bytes as human-readable string
+ */
+function formatBytes(bytes: number): string {
+  const units = ['B', 'KB', 'MB', 'GB', 'TB']
+  let unitIndex = 0
+  let value = bytes
+
+  while (value >= 1024 && unitIndex < units.length - 1) {
+    value /= 1024
+    unitIndex++
+  }
+
+  return `${value.toFixed(2)} ${units[unitIndex]}`
+}
+
+// ========== Scheduled Compaction Support ==========
+
+/**
+ * Compaction scheduler configuration
+ */
+export interface CompactionSchedulerConfig {
+  /** Snapshot manager to use */
+  manager: SnapshotManager
+  /** Check interval in milliseconds (default: 1 hour) */
+  checkIntervalMs?: number
+  /** Compaction configuration */
+  compactionConfig?: CompactionConfig
+  /** Retention configuration */
+  retentionConfig?: RetentionConfig
+  /** Enable automatic orphan cleanup */
+  enableOrphanCleanup?: boolean
+  /** Orphan cleanup interval in milliseconds (default: 24 hours) */
+  orphanCleanupIntervalMs?: number
+}
+
+/**
+ * Compaction scheduler for background maintenance
+ *
+ * Can be used with Durable Object alarms for scheduled maintenance.
+ */
+export class CompactionScheduler {
+  private manager: SnapshotManager
+  private checkIntervalMs: number
+  private compactionConfig: CompactionConfig
+  private retentionConfig: RetentionConfig
+  private enableOrphanCleanup: boolean
+  private orphanCleanupIntervalMs: number
+  private lastOrphanCleanup = 0
+
+  constructor(config: CompactionSchedulerConfig) {
+    this.manager = config.manager
+    this.checkIntervalMs = config.checkIntervalMs ?? 60 * 60 * 1000 // 1 hour
+    this.compactionConfig = config.compactionConfig ?? {}
+    this.retentionConfig = config.retentionConfig ?? {}
+    this.enableOrphanCleanup = config.enableOrphanCleanup ?? true
+    this.orphanCleanupIntervalMs = config.orphanCleanupIntervalMs ?? 24 * 60 * 60 * 1000 // 24 hours
+  }
+
+  /**
+   * Run scheduled maintenance
+   *
+   * This method should be called periodically (e.g., from a DO alarm).
+   */
+  async runMaintenance(): Promise<{
+    expirationResult: ExpireSnapshotsResult | null
+    compactionResult: CompactFilesResult | null
+    cleanupResult: CleanupOrphanFilesResult | null
+  }> {
+    await this.manager.initialize()
+
+    // Expire old snapshots
+    const expirationResult = await this.manager.expireSnapshots(this.retentionConfig)
+
+    // Check if compaction is needed
+    let compactionResult: CompactFilesResult | null = null
+    const triggerResult = await this.manager.shouldCompact(this.compactionConfig)
+
+    if (triggerResult.shouldCompact) {
+      compactionResult = await this.manager.compactFiles(this.compactionConfig)
+    }
+
+    // Run orphan cleanup if enabled and due
+    let cleanupResult: CleanupOrphanFilesResult | null = null
+    const now = Date.now()
+
+    if (
+      this.enableOrphanCleanup &&
+      now - this.lastOrphanCleanup >= this.orphanCleanupIntervalMs
+    ) {
+      cleanupResult = await this.manager.cleanupOrphanFiles()
+      this.lastOrphanCleanup = now
+    }
+
+    return {
+      expirationResult,
+      compactionResult,
+      cleanupResult,
+    }
+  }
+
+  /**
+   * Get the check interval
+   */
+  getCheckIntervalMs(): number {
+    return this.checkIntervalMs
+  }
+}
+
+/**
+ * Create a compaction scheduler
+ */
+export function createCompactionScheduler(config: CompactionSchedulerConfig): CompactionScheduler {
+  return new CompactionScheduler(config)
+}