@dotdo/postgres 0.1.0

package/src/iceberg/optimizer.ts

@@ -0,0 +1,1287 @@
+/**
+ * Iceberg Storage Cost Optimizer
+ * Task: postgres-cxo - Storage cost optimization for Iceberg
+ *
+ * Provides comprehensive storage optimization for Iceberg tables including:
+ * - Small file compaction (merge small files into larger ones)
+ * - Manifest file optimization (rewrite and consolidate manifests)
+ * - Storage cost estimation
+ * - Optimization recommendations
+ * - Automatic compaction triggers
+ *
+ * The optimizer analyzes table statistics and provides both reactive and
+ * proactive optimization strategies to minimize storage costs and improve
+ * query performance.
+ *
+ * @example
+ * ```typescript
+ * const optimizer = new StorageOptimizer({
+ *   bucket: env.ICEBERG_BUCKET,
+ *   tablePath: 'iceberg/my-table',
+ * })
+ *
+ * await optimizer.initialize()
+ *
+ * // Get cost estimate
+ * const cost = await optimizer.estimateCost('my-table')
+ * console.log(`Monthly cost: $${cost.monthlyCost}`)
+ *
+ * // Get optimization recommendations
+ * const suggestions = await optimizer.suggestOptimizations('my-table')
+ * for (const suggestion of suggestions) {
+ *   console.log(`${suggestion.type}: ${suggestion.description}`)
+ *   console.log(`  Estimated savings: $${suggestion.estimatedSavings}`)
+ * }
+ *
+ * // Compact small files
+ * const compactResult = await optimizer.compact('my-table')
+ * console.log(`Compacted ${compactResult.filesCompacted} files`)
+ *
+ * // Rewrite manifests
+ * const rewriteResult = await optimizer.rewriteManifests('my-table')
+ * console.log(`Consolidated ${rewriteResult.manifestsRewritten} manifests`)
+ * ```
+ */
+
+import type {
+  IcebergTableMetadata,
+  IcebergSnapshot,
+  IcebergManifestFile,
+  IcebergManifestEntry,
+  IcebergDataFile,
+  IcebergSnapshotSummary,
+} from './types'
+import {
+  IcebergMetadataBuilder,
+  ManifestBuilder,
+  ManifestListBuilder,
+  generateMetadataFileName,
+  generateManifestFileName,
+  generateManifestListFileName,
+} from './metadata'
+import { createLogger, LogLevel, type ILogger } from '@dotdo/postgres-shared'
+
+// Module-level logger for optimizer errors
+const optimizerLogger: ILogger = createLogger({
+  level: LogLevel.ERROR,
+  prefix: '[StorageOptimizer]',
+})
+
+// ========== Types ==========
+
+/**
+ * Storage optimizer configuration
+ */
+export interface StorageOptimizerConfig {
+  /** R2 bucket for Iceberg storage */
+  bucket: R2Bucket
+  /** Base path for the table in R2 */
+  tablePath: string
+  /** Cost per GB per month (default: R2 pricing) */
+  costPerGBMonth?: number
+  /** Target file size in bytes (default: 128 MB) */
+  targetFileSizeBytes?: number
+  /** Minimum file size threshold (default: 10 MB) */
+  minFileSizeBytes?: number
+  /** Maximum files per compaction batch (default: 100) */
+  maxFilesPerBatch?: number
+  /** Target manifests per snapshot (default: 10) */
+  targetManifestsPerSnapshot?: number
+}
+
+/**
+ * Storage cost breakdown
+ */
+export interface StorageCost {
+  /** Total storage size in bytes */
+  totalSizeBytes: number
+  /** Data file size in bytes */
+  dataFileSizeBytes: number
+  /** Metadata file size in bytes */
+  metadataFileSizeBytes: number
+  /** Manifest file size in bytes */
+  manifestFileSizeBytes: number
+  /** Number of data files */
+  dataFileCount: number
+  /** Number of manifest files */
+  manifestFileCount: number
+  /** Number of snapshots */
+  snapshotCount: number
+  /** Estimated monthly cost in USD */
+  monthlyCost: number
+  /** Cost breakdown by component */
+  costBreakdown: {
+    data: number
+    metadata: number
+    manifests: number
+  }
+}
+
+/**
+ * Optimization suggestion
+ */
+export interface OptimizationSuggestion {
+  /** Suggestion type */
+  type: 'compaction' | 'manifest-rewrite' | 'snapshot-expiration' | 'orphan-cleanup'
+  /** Priority (0-100, higher = more important) */
+  priority: number
+  /** Human-readable description */
+  description: string
+  /** Estimated cost savings per month (USD) */
+  estimatedSavings: number
+  /** Estimated bytes that will be freed */
+  estimatedBytesFreed: number
+  /** Additional metrics */
+  metrics: Record<string, number>
+  /** Recommended action */
+  action: string
+}
+
+/**
+ * Compaction result
+ */
+export interface CompactionResult {
+  /** Number of files before compaction */
+  filesBeforeCompaction: number
+  /** Number of files after compaction */
+  filesAfterCompaction: number
+  /** Total input size in bytes */
+  inputSizeBytes: number
+  /** Total output size in bytes */
+  outputSizeBytes: number
+  /** Bytes saved */
+  bytesSaved: number
+  /** Partitions compacted */
+  partitionsCompacted: number
+  /** Records processed */
+  recordsProcessed: number
+  /** New snapshot created */
+  snapshot: IcebergSnapshot | null
+  /** Duration in milliseconds */
+  durationMs: number
+}
+
+/**
+ * Manifest rewrite result
+ */
+export interface ManifestRewriteResult {
+  /** Number of manifests before rewrite */
+  manifestsBefore: number
+  /** Number of manifests after rewrite */
+  manifestsAfter: number
+  /** Bytes before rewrite */
+  bytesBefore: number
+  /** Bytes after rewrite */
+  bytesAfter: number
+  /** Bytes saved */
+  bytesSaved: number
+  /** New snapshot created */
+  snapshot: IcebergSnapshot | null
+  /** Duration in milliseconds */
+  durationMs: number
+}
+
+/**
+ * Table statistics for optimization analysis
+ */
+export interface TableStatistics {
+  /** Total data files */
+  totalDataFiles: number
+  /** Total data size in bytes */
+  totalDataSizeBytes: number
+  /** Average data file size in bytes */
+  avgDataFileSizeBytes: number
+  /** Minimum data file size in bytes */
+  minDataFileSizeBytes: number
+  /** Maximum data file size in bytes */
+  maxDataFileSizeBytes: number
+  /** Number of small files (below threshold) */
+  smallFileCount: number
+  /** Total size of small files in bytes */
+  smallFilesSizeBytes: number
+  /** Total manifest files */
+  totalManifestFiles: number
+  /** Total manifest size in bytes */
+  totalManifestSizeBytes: number
+  /** Average manifest file size in bytes */
+  avgManifestFileSizeBytes: number
+  /** Number of snapshots */
+  snapshotCount: number
+  /** Total partitions */
+  totalPartitions: number
+  /** Files per partition (average) */
+  avgFilesPerPartition: number
+}
+
+/**
+ * Compaction trigger thresholds
+ */
+export interface CompactionTriggers {
+  /** File count threshold (trigger if exceeded) */
+  fileCountThreshold: number
+  /** Small file ratio threshold (0-1, trigger if exceeded) */
+  smallFileRatioThreshold: number
+  /** Average file size threshold in bytes (trigger if below) */
+  avgFileSizeThreshold: number
+  /** Manifest count threshold (trigger if exceeded) */
+  manifestCountThreshold: number
+  /** Manifest files per snapshot threshold (trigger if exceeded) */
+  manifestsPerSnapshotThreshold: number
+}
+
+/**
+ * Trigger evaluation result
+ */
+export interface TriggerEvaluation {
+  /** Whether compaction should be triggered */
+  shouldTrigger: boolean
+  /** Triggered conditions */
+  triggeredConditions: string[]
+  /** Overall priority (0-100) */
+  priority: number
+  /** Recommended action */
+  action: string | null
+}
+
+// ========== Constants ==========
+
+/** Default R2 storage cost per GB per month (USD) */
+const DEFAULT_R2_COST_PER_GB_MONTH = 0.015
+
+/** Default configuration values */
+const DEFAULT_CONFIG = {
+  costPerGBMonth: DEFAULT_R2_COST_PER_GB_MONTH,
+  targetFileSizeBytes: 128 * 1024 * 1024, // 128 MB
+  minFileSizeBytes: 10 * 1024 * 1024, // 10 MB
+  maxFilesPerBatch: 100,
+  targetManifestsPerSnapshot: 10,
+}
+
+/** Default compaction triggers */
+const DEFAULT_TRIGGERS: CompactionTriggers = {
+  fileCountThreshold: 1000,
+  smallFileRatioThreshold: 0.3, // 30% small files
+  avgFileSizeThreshold: 50 * 1024 * 1024, // 50 MB
+  manifestCountThreshold: 100,
+  manifestsPerSnapshotThreshold: 20,
+}
+
+// ========== Storage Optimizer Class ==========
+
+/**
+ * Storage optimizer for Iceberg tables
+ *
+ * Provides comprehensive storage optimization including file compaction,
+ * manifest optimization, and cost estimation.
+ */
+export class StorageOptimizer {
+  private bucket: R2Bucket
+  private tablePath: string
+  private config: Required<StorageOptimizerConfig>
+  private metadata: IcebergTableMetadata | null = null
+  private metadataBuilder: IcebergMetadataBuilder | null = null
+  private metadataVersion = 0
+  private initialized = false
+
+  constructor(config: StorageOptimizerConfig) {
+    this.bucket = config.bucket
+    this.tablePath = config.tablePath.replace(/\/$/, '')
+    this.config = {
+      bucket: config.bucket,
+      tablePath: this.tablePath,
+      costPerGBMonth: config.costPerGBMonth ?? DEFAULT_CONFIG.costPerGBMonth,
+      targetFileSizeBytes: config.targetFileSizeBytes ?? DEFAULT_CONFIG.targetFileSizeBytes,
+      minFileSizeBytes: config.minFileSizeBytes ?? DEFAULT_CONFIG.minFileSizeBytes,
+      maxFilesPerBatch: config.maxFilesPerBatch ?? DEFAULT_CONFIG.maxFilesPerBatch,
+      targetManifestsPerSnapshot: config.targetManifestsPerSnapshot ?? DEFAULT_CONFIG.targetManifestsPerSnapshot,
+    }
+  }
+
+  /**
+   * Initialize the optimizer
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) {
+      return
+    }
+
+    await this.loadMetadata()
+    this.initialized = true
+  }
+
+  /**
+   * Ensure initialized
+   */
+  private ensureInitialized(): void {
+    if (!this.initialized) {
+      throw new Error('StorageOptimizer not initialized. Call initialize() first.')
+    }
+  }
+
+  /**
+   * Load table metadata from R2
+   */
+  private async loadMetadata(): Promise<void> {
+    // Try version hint first
+    const versionHint = await this.bucket.get(`${this.tablePath}/metadata/version-hint.text`)
+    if (versionHint) {
+      this.metadataVersion = parseInt(await versionHint.text(), 10)
+      const metadataPath = `${this.tablePath}/metadata/v${this.metadataVersion}.metadata.json`
+      const metadataObj = await this.bucket.get(metadataPath)
+      if (metadataObj) {
+        this.metadata = JSON.parse(await metadataObj.text()) as IcebergTableMetadata
+        this.metadataBuilder = IcebergMetadataBuilder.fromMetadata(this.metadata)
+        return
+      }
+    }
+
+    // Fall back to listing metadata files
+    const list = await this.bucket.list({
+      prefix: `${this.tablePath}/metadata/`,
+    })
+
+    const metadataFiles = list.objects
+      .filter((obj) => obj.key.endsWith('.metadata.json'))
+      .sort((a, b) => {
+        const versionA = parseInt(a.key.match(/v(\d+)\.metadata\.json/)?.[1] ?? '0', 10)
+        const versionB = parseInt(b.key.match(/v(\d+)\.metadata\.json/)?.[1] ?? '0', 10)
+        return versionB - versionA
+      })
+
+    if (metadataFiles.length > 0 && metadataFiles[0]) {
+      const metadataObj = await this.bucket.get(metadataFiles[0].key)
+      if (metadataObj) {
+        this.metadata = JSON.parse(await metadataObj.text()) as IcebergTableMetadata
+        this.metadataBuilder = IcebergMetadataBuilder.fromMetadata(this.metadata)
+        this.metadataVersion = parseInt(
+          metadataFiles[0].key.match(/v(\d+)\.metadata\.json/)?.[1] ?? '0',
+          10
+        )
+      }
+    }
+  }
+
+  // ========== Cost Estimation ==========
+
+  /**
+   * Estimate storage costs for the table
+   */
+  async estimateCost(table: string): Promise<StorageCost> {
+    this.ensureInitialized()
+
+    // Get table statistics
+    const stats = await this.analyzeTable(table)
+
+    // Calculate metadata size
+    let metadataSize = 0
+    const metadataList = await this.bucket.list({
+      prefix: `${this.tablePath}/metadata/`,
+    })
+    for (const obj of metadataList.objects) {
+      if (obj.key.endsWith('.metadata.json')) {
+        metadataSize += obj.size
+      }
+    }
+
+    // Convert to GB and calculate monthly cost
+    const totalGB = stats.totalDataSizeBytes / (1024 * 1024 * 1024)
+    const dataGB = stats.totalDataSizeBytes / (1024 * 1024 * 1024)
+    const metadataGB = metadataSize / (1024 * 1024 * 1024)
+    const manifestGB = stats.totalManifestSizeBytes / (1024 * 1024 * 1024)
+
+    const monthlyCost = totalGB * this.config.costPerGBMonth
+
+    return {
+      totalSizeBytes: stats.totalDataSizeBytes + metadataSize + stats.totalManifestSizeBytes,
+      dataFileSizeBytes: stats.totalDataSizeBytes,
+      metadataFileSizeBytes: metadataSize,
+      manifestFileSizeBytes: stats.totalManifestSizeBytes,
+      dataFileCount: stats.totalDataFiles,
+      manifestFileCount: stats.totalManifestFiles,
+      snapshotCount: stats.snapshotCount,
+      monthlyCost,
+      costBreakdown: {
+        data: dataGB * this.config.costPerGBMonth,
+        metadata: metadataGB * this.config.costPerGBMonth,
+        manifests: manifestGB * this.config.costPerGBMonth,
+      },
+    }
+  }
+
+  // ========== Optimization Suggestions ==========
+
+  /**
+   * Generate optimization suggestions for the table
+   */
+  async suggestOptimizations(table: string): Promise<OptimizationSuggestion[]> {
+    this.ensureInitialized()
+
+    const suggestions: OptimizationSuggestion[] = []
+    const stats = await this.analyzeTable(table)
+    const cost = await this.estimateCost(table)
+
+    // Check for small file compaction opportunity
+    if (stats.smallFileCount > 5) {
+      const estimatedOutputFiles = Math.ceil(
+        stats.smallFilesSizeBytes / this.config.targetFileSizeBytes
+      )
+      const fileReduction = stats.smallFileCount - estimatedOutputFiles
+      const metadataSavings = fileReduction * 1000 // Rough estimate
+      const savingsGB = metadataSavings / (1024 * 1024 * 1024)
+
+      const priority = Math.min(
+        100,
+        Math.round((stats.smallFileCount / stats.totalDataFiles) * 100)
+      )
+
+      suggestions.push({
+        type: 'compaction',
+        priority,
+        description: `Compact ${stats.smallFileCount} small files into ${estimatedOutputFiles} optimized files`,
+        estimatedSavings: savingsGB * this.config.costPerGBMonth,
+        estimatedBytesFreed: metadataSavings,
+        metrics: {
+          smallFiles: stats.smallFileCount,
+          estimatedOutputFiles,
+          fileReduction,
+          averageSmallFileSize: stats.smallFileCount > 0
+            ? stats.smallFilesSizeBytes / stats.smallFileCount
+            : 0,
+        },
+        action: 'Run compact() to merge small files into larger ones',
+      })
+    }
+
+    // Check for manifest optimization opportunity
+    const avgManifestsPerSnapshot = stats.totalManifestFiles / Math.max(stats.snapshotCount, 1)
+    if (avgManifestsPerSnapshot > this.config.targetManifestsPerSnapshot) {
+      const estimatedManifests = stats.snapshotCount * this.config.targetManifestsPerSnapshot
+      const manifestReduction = stats.totalManifestFiles - estimatedManifests
+      const bytesSaved = manifestReduction * stats.avgManifestFileSizeBytes
+      const savingsGB = bytesSaved / (1024 * 1024 * 1024)
+
+      const priority = Math.min(100, Math.round((manifestReduction / stats.totalManifestFiles) * 80))
+
+      suggestions.push({
+        type: 'manifest-rewrite',
+        priority,
+        description: `Consolidate ${stats.totalManifestFiles} manifest files into ${estimatedManifests} optimized manifests`,
+        estimatedSavings: savingsGB * this.config.costPerGBMonth,
+        estimatedBytesFreed: bytesSaved,
+        metrics: {
+          currentManifests: stats.totalManifestFiles,
+          targetManifests: estimatedManifests,
+          manifestsPerSnapshot: avgManifestsPerSnapshot,
+        },
+        action: 'Run rewriteManifests() to consolidate manifest files',
+      })
+    }
+
+    // Check for old snapshot expiration
+    if (stats.snapshotCount > 50) {
+      const snapshotsToExpire = stats.snapshotCount - 30 // Keep last 30
+      const estimatedBytesSaved = snapshotsToExpire * stats.avgManifestFileSizeBytes * 5 // Estimate
+      const savingsGB = estimatedBytesSaved / (1024 * 1024 * 1024)
+
+      const priority = Math.min(100, Math.round((snapshotsToExpire / stats.snapshotCount) * 60))
+
+      suggestions.push({
+        type: 'snapshot-expiration',
+        priority,
+        description: `Expire ${snapshotsToExpire} old snapshots to free up manifest storage`,
+        estimatedSavings: savingsGB * this.config.costPerGBMonth,
+        estimatedBytesFreed: estimatedBytesSaved,
+        metrics: {
+          currentSnapshots: stats.snapshotCount,
+          recommendedSnapshots: 30,
+          snapshotsToExpire,
+        },
+        action: 'Configure snapshot expiration policy with maxAgeDays or maxSnapshotsToRetain',
+      })
+    }
+
+    // Check for potential orphan files
+    // This is a heuristic based on the ratio of total storage to tracked data
+    const trackedDataRatio = cost.dataFileSizeBytes / cost.totalSizeBytes
+    if (trackedDataRatio < 0.9 && cost.totalSizeBytes > 100 * 1024 * 1024) {
+      const estimatedOrphanSize = cost.totalSizeBytes - cost.dataFileSizeBytes - cost.metadataFileSizeBytes
+      const savingsGB = estimatedOrphanSize / (1024 * 1024 * 1024)
+
+      suggestions.push({
+        type: 'orphan-cleanup',
+        priority: 40,
+        description: 'Scan for and remove orphaned data files',
+        estimatedSavings: savingsGB * this.config.costPerGBMonth,
+        estimatedBytesFreed: estimatedOrphanSize,
+        metrics: {
+          estimatedOrphanSize,
+          trackedDataRatio,
+        },
+        action: 'Run cleanupOrphanFiles() from SnapshotManager to identify and remove orphaned files',
+      })
+    }
+
+    // Sort by priority (highest first)
+    suggestions.sort((a, b) => b.priority - a.priority)
+
+    return suggestions
+  }
+
+  // ========== Compaction ==========
+
+  /**
+   * Compact small files into larger ones
+   */
+  async compact(_table: string): Promise<CompactionResult> {
+    this.ensureInitialized()
+
+    const startTime = performance.now()
+
+    // Get all data files
+    const dataFiles = await this.getAllDataFiles()
+
+    // Filter small files
+    const smallFiles = dataFiles.filter(
+      (f) => f['file-size-in-bytes'] < this.config.minFileSizeBytes
+    )
+
+    if (smallFiles.length === 0) {
+      return {
+        filesBeforeCompaction: dataFiles.length,
+        filesAfterCompaction: dataFiles.length,
+        inputSizeBytes: 0,
+        outputSizeBytes: 0,
+        bytesSaved: 0,
+        partitionsCompacted: 0,
+        recordsProcessed: 0,
+        snapshot: null,
+        durationMs: performance.now() - startTime,
+      }
+    }
+
+    // Group by partition
+    const filesByPartition = new Map<string, IcebergDataFile[]>()
+    for (const file of smallFiles) {
+      const partitionPath = this.extractPartitionPath(file['file-path'])
+      const files = filesByPartition.get(partitionPath) ?? []
+      files.push(file)
+      filesByPartition.set(partitionPath, files)
+    }
+
+    // Create compaction groups
+    const compactionGroups: Array<{
+      partition: string
+      files: IcebergDataFile[]
+      totalSize: number
+      totalRecords: number
+    }> = []
+
+    for (const [partition, files] of filesByPartition) {
+      if (files.length < 2) {
+        continue // Need at least 2 files to compact
+      }
+
+      // Sort by size for better packing
+      files.sort((a, b) => a['file-size-in-bytes'] - b['file-size-in-bytes'])
+
+      // Create batches
+      let currentBatch: IcebergDataFile[] = []
+      let currentSize = 0
+      let currentRecords = 0
+
+      for (const file of files) {
+        if (
+          currentBatch.length >= this.config.maxFilesPerBatch ||
+          currentSize + file['file-size-in-bytes'] > this.config.targetFileSizeBytes * 2
+        ) {
+          if (currentBatch.length >= 2) {
+            compactionGroups.push({
+              partition,
+              files: currentBatch,
+              totalSize: currentSize,
+              totalRecords: currentRecords,
+            })
+          }
+          currentBatch = []
+          currentSize = 0
+          currentRecords = 0
+        }
+
+        currentBatch.push(file)
+        currentSize += file['file-size-in-bytes']
+        currentRecords += file['record-count']
+      }
+
+      // Add remaining batch
+      if (currentBatch.length >= 2) {
+        compactionGroups.push({
+          partition,
+          files: currentBatch,
+          totalSize: currentSize,
+          totalRecords: currentRecords,
+        })
+      }
+    }
+
+    if (compactionGroups.length === 0) {
+      return {
+        filesBeforeCompaction: dataFiles.length,
+        filesAfterCompaction: dataFiles.length,
+        inputSizeBytes: 0,
+        outputSizeBytes: 0,
+        bytesSaved: 0,
+        partitionsCompacted: 0,
+        recordsProcessed: 0,
+        snapshot: null,
+        durationMs: performance.now() - startTime,
+      }
+    }
+
+    // Calculate totals
+    let totalInputFiles = 0
+    let totalInputSize = 0
+    let totalRecords = 0
+    const partitionsCompacted = new Set<string>()
+
+    for (const group of compactionGroups) {
+      totalInputFiles += group.files.length
+      totalInputSize += group.totalSize
+      totalRecords += group.totalRecords
+      partitionsCompacted.add(group.partition)
+    }
+
+    // Create compaction snapshot
+    const snapshot = await this.createCompactionSnapshot(compactionGroups)
+
+    // Estimate output files
+    const estimatedOutputFiles = compactionGroups.reduce((sum, group) => {
+      return sum + Math.max(1, Math.ceil(group.totalSize / this.config.targetFileSizeBytes))
+    }, 0)
+
+    const filesAfter = dataFiles.length - totalInputFiles + estimatedOutputFiles
+    const metadataSaved = (totalInputFiles - estimatedOutputFiles) * 1000 // Rough estimate
+
+    return {
+      filesBeforeCompaction: dataFiles.length,
+      filesAfterCompaction: filesAfter,
+      inputSizeBytes: totalInputSize,
+      outputSizeBytes: totalInputSize, // Data size stays roughly the same
+      bytesSaved: metadataSaved,
+      partitionsCompacted: partitionsCompacted.size,
+      recordsProcessed: totalRecords,
+      snapshot,
+      durationMs: performance.now() - startTime,
+    }
+  }
+
+  /**
+   * Create a compaction snapshot
+   */
+  private async createCompactionSnapshot(
+    compactionGroups: Array<{
+      partition: string
+      files: IcebergDataFile[]
+      totalSize: number
+      totalRecords: number
+    }>
+  ): Promise<IcebergSnapshot | null> {
+    if (!this.metadata || !this.metadataBuilder) {
+      return null
+    }
+
+    const snapshotId = Date.now() * 1000 + Math.floor(Math.random() * 1000)
+    const sequenceNumber = this.metadataBuilder.getSequenceNumber() + 1
+    const partitionSpecId = this.metadata['default-spec-id']
+
+    // Create manifest builder
+    const manifestBuilder = new ManifestBuilder({
+      partitionSpecId,
+      sequenceNumber,
+      snapshotId,
+    })
+
+    // Mark old files as deleted
+    let totalDeletedFiles = 0
+    let totalDeletedRecords = 0
+
+    for (const group of compactionGroups) {
+      for (const file of group.files) {
+        manifestBuilder.deleteDataFile(file)
+        totalDeletedFiles++
+        totalDeletedRecords += file['record-count']
+      }
+    }
+
+    // Write manifest
+    const manifestFileName = generateManifestFileName(snapshotId)
+    const manifestPath = `${this.tablePath}/metadata/${manifestFileName}`
+    const manifestContent = manifestBuilder.toJSON()
+    const manifestBytes = new TextEncoder().encode(manifestContent)
+
+    await this.bucket.put(manifestPath, manifestBytes, {
+      httpMetadata: { contentType: 'application/json' },
+    })
+
+    const manifestFileMetadata = manifestBuilder.createManifestFileMetadata(
+      manifestPath,
+      manifestBytes.length
+    )
+
+    // Create manifest list
+    const manifestListBuilder = new ManifestListBuilder(snapshotId, sequenceNumber)
+    manifestListBuilder.addManifest(manifestFileMetadata)
+
+    // Include existing manifests
+    const currentSnapshot = this.metadata.snapshots?.find(
+      (s) => s['snapshot-id'] === this.metadata!['current-snapshot-id']
+    )
+    if (currentSnapshot) {
+      const existingManifests = await this.loadManifestsFromList(currentSnapshot['manifest-list'])
+      for (const manifest of existingManifests) {
+        manifestListBuilder.addManifest(manifest)
+      }
+    }
+
+    // Write manifest list
+    const manifestListFileName = generateManifestListFileName(snapshotId)
+    const manifestListPath = `${this.tablePath}/metadata/${manifestListFileName}`
+    const manifestListContent = manifestListBuilder.toJSON()
+    const manifestListBytes = new TextEncoder().encode(manifestListContent)
+
+    await this.bucket.put(manifestListPath, manifestListBytes, {
+      httpMetadata: { contentType: 'application/json' },
+    })
+
+    // Create snapshot
+    const summary: IcebergSnapshotSummary = {
+      operation: 'replace',
+      'added-data-files': '0',
+      'deleted-data-files': String(totalDeletedFiles),
+      'added-records': '0',
+      'deleted-records': String(totalDeletedRecords),
+      'compaction-type': 'small-file-compaction',
+    }
+
+    const snapshot: IcebergSnapshot = {
+      'snapshot-id': snapshotId,
+      ...(this.metadata['current-snapshot-id'] !== undefined && {
+        'parent-snapshot-id': this.metadata['current-snapshot-id'],
+      }),
+      'sequence-number': sequenceNumber,
+      'timestamp-ms': Date.now(),
+      'manifest-list': manifestListPath,
+      summary,
+      'schema-id': this.metadata['current-schema-id'],
+    }
+
+    // Update metadata
+    const newSnapshots = [...(this.metadata.snapshots ?? []), snapshot]
+    await this.updateMetadataWithSnapshots(newSnapshots)
+
+    return snapshot
+  }
+
+  // ========== Manifest Optimization ==========
+
+  /**
+   * Rewrite and consolidate manifest files
+   */
+  async rewriteManifests(_table: string): Promise<ManifestRewriteResult> {
+    this.ensureInitialized()
+
+    const startTime = performance.now()
+
+    if (!this.metadata?.['current-snapshot-id']) {
+      return {
+        manifestsBefore: 0,
+        manifestsAfter: 0,
+        bytesBefore: 0,
+        bytesAfter: 0,
+        bytesSaved: 0,
+        snapshot: null,
+        durationMs: performance.now() - startTime,
+      }
+    }
+
+    // Get current snapshot
+    const currentSnapshot = this.metadata.snapshots?.find(
+      (s) => s['snapshot-id'] === this.metadata!['current-snapshot-id']
+    )
+
+    if (!currentSnapshot) {
+      return {
+        manifestsBefore: 0,
+        manifestsAfter: 0,
+        bytesBefore: 0,
+        bytesAfter: 0,
+        bytesSaved: 0,
+        snapshot: null,
+        durationMs: performance.now() - startTime,
+      }
+    }
+
+    // Load all manifests
+    const manifests = await this.loadManifestsFromList(currentSnapshot['manifest-list'])
+    const manifestsBefore = manifests.length
+    const bytesBefore = manifests.reduce((sum, m) => sum + m['manifest-length'], 0)
+
+    // If already optimized, skip
+    if (manifestsBefore <= this.config.targetManifestsPerSnapshot) {
+      return {
+        manifestsBefore,
+        manifestsAfter: manifestsBefore,
+        bytesBefore,
+        bytesAfter: bytesBefore,
+        bytesSaved: 0,
+        snapshot: null,
+        durationMs: performance.now() - startTime,
+      }
+    }
+
+    // Collect all entries from all manifests
+    const allEntries: IcebergManifestEntry[] = []
+    for (const manifest of manifests) {
+      const entries = await this.loadManifestEntries(manifest['manifest-path'])
+      allEntries.push(...entries)
+    }
+
+    // Group entries into new consolidated manifests
+    const targetManifests = this.config.targetManifestsPerSnapshot
+    const entriesPerManifest = Math.ceil(allEntries.length / targetManifests)
+
+    const snapshotId = Date.now() * 1000 + Math.floor(Math.random() * 1000)
+    const sequenceNumber = this.metadataBuilder!.getSequenceNumber() + 1
+    const partitionSpecId = this.metadata['default-spec-id']
+
+    const newManifests: IcebergManifestFile[] = []
+    let bytesAfter = 0
+
+    // Create consolidated manifests
+    for (let i = 0; i < allEntries.length; i += entriesPerManifest) {
+      const batch = allEntries.slice(i, i + entriesPerManifest)
+
+      const manifestBuilder = new ManifestBuilder({
+        partitionSpecId,
+        sequenceNumber,
+        snapshotId,
+      })
+
+      for (const entry of batch) {
+        if (entry.status === 0) {
+          // EXISTING
+          manifestBuilder.addExistingFile(
+            entry['data-file'],
+            entry['sequence-number'] ?? sequenceNumber
+          )
+        } else if (entry.status === 1) {
+          // ADDED
+          manifestBuilder.addDataFile(entry['data-file'], 1)
+        } else if (entry.status === 2) {
+          // DELETED
+          manifestBuilder.deleteDataFile(entry['data-file'])
+        }
+      }
+
+      // Write manifest
+      const manifestAttempt = Math.floor(i / entriesPerManifest)
+      const manifestFileName = generateManifestFileName(snapshotId, manifestAttempt)
+      const manifestPath = `${this.tablePath}/metadata/${manifestFileName}`
+      const manifestContent = manifestBuilder.toJSON()
+      const manifestBytes = new TextEncoder().encode(manifestContent)
+
+      await this.bucket.put(manifestPath, manifestBytes, {
+        httpMetadata: { contentType: 'application/json' },
+      })
+
+      const manifestFileMetadata = manifestBuilder.createManifestFileMetadata(
+        manifestPath,
+        manifestBytes.length
+      )
+
+      newManifests.push(manifestFileMetadata)
+      bytesAfter += manifestBytes.length
+    }
+
+    // Create new snapshot with consolidated manifests
+    const snapshot = await this.createManifestRewriteSnapshot(snapshotId, sequenceNumber, newManifests)
+
+    return {
+      manifestsBefore,
+      manifestsAfter: newManifests.length,
+      bytesBefore,
+      bytesAfter,
+      bytesSaved: bytesBefore - bytesAfter,
+      snapshot,
+      durationMs: performance.now() - startTime,
+    }
+  }
+
+  /**
+   * Create a manifest rewrite snapshot
+   */
+  private async createManifestRewriteSnapshot(
+    snapshotId: number,
+    sequenceNumber: number,
+    newManifests: IcebergManifestFile[]
+  ): Promise<IcebergSnapshot | null> {
+    if (!this.metadata) {
+      return null
+    }
+
+    // Create manifest list
+    const manifestListBuilder = new ManifestListBuilder(snapshotId, sequenceNumber)
+
+    for (const manifest of newManifests) {
+      manifestListBuilder.addManifest(manifest)
+    }
+
+    // Write manifest list
+    const manifestListFileName = generateManifestListFileName(snapshotId)
+    const manifestListPath = `${this.tablePath}/metadata/${manifestListFileName}`
+    const manifestListContent = manifestListBuilder.toJSON()
+    const manifestListBytes = new TextEncoder().encode(manifestListContent)
+
+    await this.bucket.put(manifestListPath, manifestListBytes, {
+      httpMetadata: { contentType: 'application/json' },
+    })
+
+    // Create snapshot
+    const summary: IcebergSnapshotSummary = {
+      operation: 'replace',
+      'added-data-files': '0',
+      'deleted-data-files': '0',
+      'added-records': '0',
+      'deleted-records': '0',
+      'manifest-rewrite': 'true',
+    }
+
+    const snapshot: IcebergSnapshot = {
+      'snapshot-id': snapshotId,
+      ...(this.metadata['current-snapshot-id'] !== undefined && {
+        'parent-snapshot-id': this.metadata['current-snapshot-id'],
+      }),
+      'sequence-number': sequenceNumber,
+      'timestamp-ms': Date.now(),
+      'manifest-list': manifestListPath,
+      summary,
+      'schema-id': this.metadata['current-schema-id'],
+    }
+
+    // Update metadata
+    const newSnapshots = [...(this.metadata.snapshots ?? []), snapshot]
+    await this.updateMetadataWithSnapshots(newSnapshots)
+
+    return snapshot
+  }
+
+  // ========== Compaction Triggers ==========
+
+  /**
+   * Evaluate compaction triggers
+   */
+  async evaluateTriggers(
+    table: string,
+    triggers: Partial<CompactionTriggers> = {}
+  ): Promise<TriggerEvaluation> {
+    this.ensureInitialized()
+
+    const config = { ...DEFAULT_TRIGGERS, ...triggers }
+    const stats = await this.analyzeTable(table)
+
+    const triggeredConditions: string[] = []
+    let maxPriority = 0
+
+    // Check file count threshold
+    if (stats.totalDataFiles > config.fileCountThreshold) {
+      triggeredConditions.push(
+        `File count (${stats.totalDataFiles}) exceeds threshold (${config.fileCountThreshold})`
+      )
+      maxPriority = Math.max(maxPriority, 60)
+    }
+
+    // Check small file ratio
+    const smallFileRatio = stats.totalDataFiles > 0
+      ? stats.smallFileCount / stats.totalDataFiles
+      : 0
+    if (smallFileRatio > config.smallFileRatioThreshold) {
+      triggeredConditions.push(
+        `Small file ratio (${(smallFileRatio * 100).toFixed(1)}%) exceeds threshold (${(config.smallFileRatioThreshold * 100).toFixed(1)}%)`
+      )
+      maxPriority = Math.max(maxPriority, 80)
+    }
+
+    // Check average file size
+    if (stats.avgDataFileSizeBytes < config.avgFileSizeThreshold && stats.totalDataFiles > 10) {
+      triggeredConditions.push(
+        `Average file size (${formatBytes(stats.avgDataFileSizeBytes)}) below threshold (${formatBytes(config.avgFileSizeThreshold)})`
+      )
+      maxPriority = Math.max(maxPriority, 70)
+    }
+
+    // Check manifest count
+    if (stats.totalManifestFiles > config.manifestCountThreshold) {
+      triggeredConditions.push(
+        `Manifest count (${stats.totalManifestFiles}) exceeds threshold (${config.manifestCountThreshold})`
+      )
+      maxPriority = Math.max(maxPriority, 50)
+    }
+
+    // Check manifests per snapshot
+    const manifestsPerSnapshot = stats.snapshotCount > 0
+      ? stats.totalManifestFiles / stats.snapshotCount
+      : 0
+    if (manifestsPerSnapshot > config.manifestsPerSnapshotThreshold) {
+      triggeredConditions.push(
+        `Manifests per snapshot (${manifestsPerSnapshot.toFixed(1)}) exceeds threshold (${config.manifestsPerSnapshotThreshold})`
+      )
+      maxPriority = Math.max(maxPriority, 55)
+    }
+
+    // Determine recommended action
+    let action: string | null = null
+    if (triggeredConditions.length > 0) {
+      if (smallFileRatio > config.smallFileRatioThreshold ||
+          stats.avgDataFileSizeBytes < config.avgFileSizeThreshold) {
+        action = 'compact'
+      } else if (manifestsPerSnapshot > config.manifestsPerSnapshotThreshold) {
+        action = 'rewriteManifests'
+      } else {
+        action = 'compact' // Default to compaction
+      }
+    }
+
+    return {
+      shouldTrigger: triggeredConditions.length > 0,
+      triggeredConditions,
+      priority: maxPriority,
+      action,
+    }
+  }
+
+  // ========== Analysis Utilities ==========
+
+  /**
+   * Analyze table statistics
+   */
+  private async analyzeTable(_table: string): Promise<TableStatistics> {
+    const dataFiles = await this.getAllDataFiles()
+
+    let totalDataSize = 0
+    let minSize = Infinity
+    let maxSize = 0
+    let smallFileCount = 0
+    let smallFilesSize = 0
+    const partitions = new Set<string>()
+
+    for (const file of dataFiles) {
+      const size = file['file-size-in-bytes']
+      totalDataSize += size
+      minSize = Math.min(minSize, size)
+      maxSize = Math.max(maxSize, size)
+
+      if (size < this.config.minFileSizeBytes) {
+        smallFileCount++
+        smallFilesSize += size
+      }
+
+      const partitionPath = this.extractPartitionPath(file['file-path'])
+      if (partitionPath) {
+        partitions.add(partitionPath)
+      }
+    }
+
+    // Analyze manifests
+    let totalManifestFiles = 0
+    let totalManifestSize = 0
+
+    if (this.metadata?.snapshots) {
+      for (const snapshot of this.metadata.snapshots) {
+        const manifests = await this.loadManifestsFromList(snapshot['manifest-list'])
+        totalManifestFiles += manifests.length
+        totalManifestSize += manifests.reduce((sum, m) => sum + m['manifest-length'], 0)
+      }
+    }
+
+    const snapshotCount = this.metadata?.snapshots?.length ?? 0
+
+    return {
+      totalDataFiles: dataFiles.length,
+      totalDataSizeBytes: totalDataSize,
+      avgDataFileSizeBytes: dataFiles.length > 0 ? totalDataSize / dataFiles.length : 0,
+      minDataFileSizeBytes: minSize === Infinity ? 0 : minSize,
+      maxDataFileSizeBytes: maxSize,
+      smallFileCount,
+      smallFilesSizeBytes: smallFilesSize,
+      totalManifestFiles,
+      totalManifestSizeBytes: totalManifestSize,
+      avgManifestFileSizeBytes: totalManifestFiles > 0 ? totalManifestSize / totalManifestFiles : 0,
+      snapshotCount,
+      totalPartitions: partitions.size,
+      avgFilesPerPartition: partitions.size > 0 ? dataFiles.length / partitions.size : 0,
+    }
+  }
+
+  /**
+   * Get all data files from the current snapshot
+   */
+  private async getAllDataFiles(): Promise<IcebergDataFile[]> {
+    if (!this.metadata?.['current-snapshot-id']) {
+      return []
+    }
+
+    const currentSnapshot = this.metadata.snapshots?.find(
+      (s) => s['snapshot-id'] === this.metadata!['current-snapshot-id']
+    )
+
+    if (!currentSnapshot) {
+      return []
+    }
+
+    const dataFiles: IcebergDataFile[] = []
+    const manifests = await this.loadManifestsFromList(currentSnapshot['manifest-list'])
+
+    for (const manifest of manifests) {
+      if (manifest.content !== 0) {
+        continue // Skip delete manifests
+      }
+
+      const entries = await this.loadManifestEntries(manifest['manifest-path'])
+      for (const entry of entries) {
+        // Only include EXISTING (0) and ADDED (1) files
+        if (entry.status !== 2) {
+          dataFiles.push(entry['data-file'])
+        }
+      }
+    }
+
+    return dataFiles
+  }
+
+  /**
+   * Load manifests from a manifest list
+   */
+  private async loadManifestsFromList(manifestListPath: string): Promise<IcebergManifestFile[]> {
+    try {
+      const obj = await this.bucket.get(manifestListPath)
+      if (!obj) {
+        return []
+      }
+
+      const data = JSON.parse(await obj.text())
+      return data.manifests ?? []
+    } catch (error) {
+      optimizerLogger.error('Failed to load manifest list', { manifestListPath, error: error instanceof Error ? error.message : String(error) })
+      return []
+    }
+  }
+
+  /**
+   * Load manifest entries from a manifest file
+   */
+  private async loadManifestEntries(manifestPath: string): Promise<IcebergManifestEntry[]> {
+    try {
+      const obj = await this.bucket.get(manifestPath)
+      if (!obj) {
+        return []
+      }
+
+      const data = JSON.parse(await obj.text())
+      return data.entries ?? []
+    } catch (error) {
+      optimizerLogger.error('Failed to load manifest', { manifestPath, error: error instanceof Error ? error.message : String(error) })
+      return []
+    }
+  }
+
+  /**
+   * Extract partition path from file path
+   */
+  private extractPartitionPath(filePath: string): string {
+    const dataIndex = filePath.indexOf('/data/')
+    if (dataIndex === -1) {
+      return ''
+    }
+
+    const afterData = filePath.substring(dataIndex + 6) // Skip '/data/'
+    const lastSlash = afterData.lastIndexOf('/')
+    if (lastSlash === -1) {
+      return ''
+    }
+
+    return afterData.substring(0, lastSlash)
+  }
+
+  /**
+   * Update metadata with new snapshot list
+   */
+  private async updateMetadataWithSnapshots(snapshots: IcebergSnapshot[]): Promise<void> {
+    if (!this.metadata || !this.metadataBuilder) {
+      throw new Error('Metadata not loaded')
+    }
+
+    // Sort snapshots by timestamp
+    snapshots.sort((a, b) => a['timestamp-ms'] - b['timestamp-ms'])
+
+    // Create new metadata
+    const newMetadata: IcebergTableMetadata = {
+      ...this.metadata,
+      'last-updated-ms': Date.now(),
+      snapshots: snapshots,
+      'snapshot-log': snapshots.map((s) => ({
+        'timestamp-ms': s['timestamp-ms'],
+        'snapshot-id': s['snapshot-id'],
+      })),
+    }
+
+    // Set current snapshot to latest
+    const latestSnapshot = snapshots[snapshots.length - 1]
+    if (latestSnapshot) {
+      newMetadata['current-snapshot-id'] = latestSnapshot['snapshot-id']
+    }
+
+    // Write new metadata version
+    this.metadataVersion++
+    const metadataFileName = generateMetadataFileName(this.metadataVersion)
+    const metadataPath = `${this.tablePath}/metadata/${metadataFileName}`
+
+    await this.bucket.put(metadataPath, JSON.stringify(newMetadata, null, 2), {
+      httpMetadata: { contentType: 'application/json' },
+    })
+
+    // Update version hint
+    await this.bucket.put(
+      `${this.tablePath}/metadata/version-hint.text`,
+      String(this.metadataVersion),
+      { httpMetadata: { contentType: 'text/plain' } }
+    )
+
+    // Update internal state
+    this.metadata = newMetadata
+    this.metadataBuilder = IcebergMetadataBuilder.fromMetadata(newMetadata)
+  }
+}
+
+// ========== Utility Functions ==========
+
+/**
+ * Format bytes as human-readable string
+ */
+function formatBytes(bytes: number): string {
+  const units = ['B', 'KB', 'MB', 'GB', 'TB']
+  let unitIndex = 0
+  let value = bytes
+
+  while (value >= 1024 && unitIndex < units.length - 1) {
+    value /= 1024
+    unitIndex++
+  }
+
+  return `${value.toFixed(2)} ${units[unitIndex]}`
+}
+
+/**
+ * Create a storage optimizer
+ */
+export function createStorageOptimizer(config: StorageOptimizerConfig): StorageOptimizer {
+  return new StorageOptimizer(config)
+}