@dotdo/postgres 0.1.0

package/src/iceberg/snapshot-manager.test.ts

@@ -0,0 +1,919 @@
+/**
+ * Iceberg Snapshot Manager Tests
+ * Task: postgres-iyx - Snapshot management and compaction
+ */
+
+import { describe, it, expect, beforeEach, vi } from 'vitest'
+import {
+  SnapshotManager,
+  createSnapshotManager,
+  CompactionScheduler,
+  createCompactionScheduler,
+  type RetentionConfig,
+  type CompactionConfig,
+} from './snapshot-manager'
+import type {
+  IcebergTableMetadata,
+  IcebergSnapshot,
+  IcebergManifestFile,
+  IcebergManifestEntry,
+  IcebergDataFile,
+} from './types'
+
+/**
+ * Create a mock R2Bucket
+ */
+function createMockBucket(): R2Bucket & { _storage: Map<string, { data: string | ArrayBuffer; size: number }> } {
+  const storage = new Map<string, { data: string | ArrayBuffer; size: number; uploaded: Date }>()
+
+  return {
+    _storage: storage as Map<string, { data: string | ArrayBuffer; size: number }>,
+    put: vi.fn(async (key: string, data: string | ArrayBuffer | Uint8Array | ReadableStream) => {
+      let body: string | ArrayBuffer
+      let size: number
+
+      if (typeof data === 'string') {
+        body = data
+        size = data.length
+      } else if (data instanceof Uint8Array) {
+        body = data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength)
+        size = data.length
+      } else if (data instanceof ArrayBuffer) {
+        body = data
+        size = data.byteLength
+      } else {
+        body = ''
+        size = 0
+      }
+
+      storage.set(key, { data: body, size, uploaded: new Date() })
+      return {
+        key,
+        version: 'test-version',
+        size,
+        etag: 'test-etag',
+        httpEtag: '"test-etag"',
+        uploaded: new Date(),
+        httpMetadata: {},
+        customMetadata: {},
+      } as R2Object
+    }),
+    get: vi.fn(async (key: string) => {
+      const item = storage.get(key)
+      if (!item) return null
+
+      return {
+        key,
+        version: 'test-version',
+        size: item.size,
+        etag: 'test-etag',
+        httpEtag: '"test-etag"',
+        uploaded: item.uploaded,
+        httpMetadata: {},
+        customMetadata: {},
+        text: async () => typeof item.data === 'string' ? item.data : new TextDecoder().decode(item.data as ArrayBuffer),
+        json: async () => JSON.parse(typeof item.data === 'string' ? item.data : new TextDecoder().decode(item.data as ArrayBuffer)),
+        arrayBuffer: async () => typeof item.data === 'string' ? new TextEncoder().encode(item.data).buffer : item.data,
+        bytes: async () => typeof item.data === 'string' ? new TextEncoder().encode(item.data) : new Uint8Array(item.data as ArrayBuffer),
+        blob: async () => new Blob([item.data]),
+        body: null,
+        bodyUsed: false,
+        writeHttpMetadata: () => {},
+      } as R2ObjectBody
+    }),
+    head: vi.fn(async (key: string) => {
+      const item = storage.get(key)
+      if (!item) return null
+
+      return {
+        key,
+        version: 'test-version',
+        size: item.size,
+        etag: 'test-etag',
+        httpEtag: '"test-etag"',
+        uploaded: item.uploaded,
+        httpMetadata: {},
+        customMetadata: {},
+      } as R2Object
+    }),
+    delete: vi.fn(async (keys: string | string[]) => {
+      const keyList = Array.isArray(keys) ? keys : [keys]
+      for (const key of keyList) {
+        storage.delete(key)
+      }
+    }),
+    list: vi.fn(async (options?: R2ListOptions) => {
+      const prefix = options?.prefix || ''
+      const objects: R2Object[] = []
+
+      for (const [key, item] of storage) {
+        if (key.startsWith(prefix)) {
+          objects.push({
+            key,
+            version: 'test-version',
+            size: item.size,
+            etag: 'test-etag',
+            httpEtag: '"test-etag"',
+            uploaded: item.uploaded,
+            httpMetadata: {},
+            customMetadata: {},
+          } as R2Object)
+        }
+      }
+
+      return {
+        objects,
+        truncated: false,
+        cursor: undefined,
+        delimitedPrefixes: [],
+      } as R2Objects
+    }),
+    createMultipartUpload: vi.fn(),
+    resumeMultipartUpload: vi.fn(),
+  } as unknown as R2Bucket & { _storage: Map<string, { data: string | ArrayBuffer; size: number }> }
+}
+
+/**
+ * Create a mock DurableObjectState
+ */
+function createMockState(): DurableObjectState {
+  const storage = new Map<string, unknown>()
+
+  return {
+    storage: {
+      get: vi.fn(async <T>(key: string) => storage.get(key) as T | undefined),
+      put: vi.fn(async (keyOrEntries: string | Record<string, unknown>, value?: unknown) => {
+        if (typeof keyOrEntries === 'string') {
+          storage.set(keyOrEntries, value)
+        } else {
+          for (const [k, v] of Object.entries(keyOrEntries)) {
+            storage.set(k, v)
+          }
+        }
+      }),
+      delete: vi.fn(async (keys: string | string[]) => {
+        const keyList = Array.isArray(keys) ? keys : [keys]
+        let deleted = false
+        for (const key of keyList) {
+          if (storage.delete(key)) {
+            deleted = true
+          }
+        }
+        return deleted
+      }),
+      list: vi.fn(async () => storage),
+      deleteAll: vi.fn(async () => storage.clear()),
+      transaction: vi.fn(),
+      getAlarm: vi.fn(),
+      setAlarm: vi.fn(),
+      deleteAlarm: vi.fn(),
+      sync: vi.fn(),
+    },
+    id: { toString: () => 'test-do-id' } as DurableObjectId,
+    waitUntil: vi.fn(),
+    blockConcurrencyWhile: vi.fn(async (fn) => fn()),
+    acceptWebSocket: vi.fn(),
+    getWebSockets: vi.fn(() => []),
+    setWebSocketAutoResponse: vi.fn(),
+    getWebSocketAutoResponse: vi.fn(),
+    getWebSocketAutoResponseTimestamp: vi.fn(),
+    setHibernatableWebSocketEventTimeout: vi.fn(),
+    getHibernatableWebSocketEventTimeout: vi.fn(),
+    getTags: vi.fn(() => []),
+    abort: vi.fn(),
+  } as unknown as DurableObjectState
+}
+
+/**
+ * Create a sample Iceberg snapshot
+ */
+function createSampleSnapshot(
+  snapshotId: number,
+  parentSnapshotId?: number,
+  timestampMs: number = Date.now(),
+  sequenceNumber: number = 1
+): IcebergSnapshot {
+  return {
+    'snapshot-id': snapshotId,
+    ...(parentSnapshotId !== undefined && { 'parent-snapshot-id': parentSnapshotId }),
+    'sequence-number': sequenceNumber,
+    'timestamp-ms': timestampMs,
+    'manifest-list': `test-table/metadata/snap-${snapshotId}-uuid.avro`,
+    summary: {
+      operation: 'append',
+      'added-data-files': '5',
+      'deleted-data-files': '0',
+      'added-records': '1000',
+      'deleted-records': '0',
+      'total-records': '1000',
+      'total-files-size': '10485760',
+    },
+    'schema-id': 0,
+  }
+}
+
+/**
+ * Create sample table metadata
+ */
+function createSampleMetadata(snapshots: IcebergSnapshot[]): IcebergTableMetadata {
+  const currentSnapshotId = snapshots.length > 0 ? snapshots[snapshots.length - 1]!['snapshot-id'] : undefined
+
+  return {
+    'format-version': 2,
+    'table-uuid': 'test-uuid-1234',
+    location: 'test-table',
+    'last-sequence-number': snapshots.length,
+    'last-updated-ms': Date.now(),
+    'last-column-id': 10,
+    'current-schema-id': 0,
+    schemas: [
+      {
+        type: 'struct',
+        'schema-id': 0,
+        fields: [
+          { id: 1, name: 'id', required: true, type: 'long' },
+          { id: 2, name: 'data', required: false, type: 'string' },
+        ],
+      },
+    ],
+    'default-spec-id': 0,
+    'partition-specs': [
+      {
+        'spec-id': 0,
+        fields: [],
+      },
+    ],
+    'last-partition-id': 0,
+    'default-sort-order-id': 0,
+    'sort-orders': [
+      {
+        'order-id': 0,
+        fields: [],
+      },
+    ],
+    ...(currentSnapshotId !== undefined && { 'current-snapshot-id': currentSnapshotId }),
+    ...(snapshots.length > 0 && { snapshots }),
+    ...(snapshots.length > 0 && {
+      'snapshot-log': snapshots.map((s) => ({
+        'timestamp-ms': s['timestamp-ms'],
+        'snapshot-id': s['snapshot-id'],
+      })),
+    }),
+  }
+}
+
+/**
+ * Create a sample manifest list JSON
+ */
+function createSampleManifestList(manifests: IcebergManifestFile[]): string {
+  return JSON.stringify({
+    schema: { type: 'record', name: 'manifest_file', fields: [] },
+    manifests,
+  })
+}
+
+/**
+ * Create a sample manifest JSON
+ */
+function createSampleManifest(entries: IcebergManifestEntry[]): string {
+  return JSON.stringify({
+    schema: { type: 'record', name: 'manifest_entry', fields: [] },
+    entries,
+  })
+}
+
+/**
+ * Create a sample data file
+ */
+function createSampleDataFile(
+  filePath: string,
+  recordCount: number = 100,
+  fileSizeBytes: number = 1024 * 1024
+): IcebergDataFile {
+  return {
+    content: 'data',
+    'file-path': filePath,
+    'file-format': 'parquet',
+    partition: {},
+    'record-count': recordCount,
+    'file-size-in-bytes': fileSizeBytes,
+  }
+}
+
+/**
+ * Create sample manifest file metadata
+ */
+function createSampleManifestFile(
+  snapshotId: number,
+  addedFilesCount: number = 5,
+  manifestLength: number = 1024
+): IcebergManifestFile {
+  return {
+    'manifest-path': `test-table/metadata/${snapshotId}-m0-uuid.avro`,
+    'manifest-length': manifestLength,
+    'partition-spec-id': 0,
+    content: 0,
+    'sequence-number': 1,
+    'min-sequence-number': 1,
+    'added-snapshot-id': snapshotId,
+    'added-files-count': addedFilesCount,
+    'existing-files-count': 0,
+    'deleted-files-count': 0,
+    'added-rows-count': 1000,
+    'existing-rows-count': 0,
+    'deleted-rows-count': 0,
+  }
+}
+
+describe('SnapshotManager', () => {
+  let manager: SnapshotManager
+  let mockBucket: R2Bucket & { _storage: Map<string, { data: string | ArrayBuffer; size: number }> }
+  let mockState: DurableObjectState
+
+  beforeEach(async () => {
+    mockBucket = createMockBucket()
+    mockState = createMockState()
+  })
+
+  async function setupManagerWithSnapshots(snapshots: IcebergSnapshot[]): Promise<void> {
+    const metadata = createSampleMetadata(snapshots)
+
+    // Store metadata
+    await mockBucket.put(
+      'test-table/metadata/v1.metadata.json',
+      JSON.stringify(metadata)
+    )
+    await mockBucket.put('test-table/metadata/version-hint.text', '1')
+
+    // Store manifest lists and manifests for each snapshot
+    for (const snapshot of snapshots) {
+      const manifestFile = createSampleManifestFile(snapshot['snapshot-id'])
+      const manifestList = createSampleManifestList([manifestFile])
+      await mockBucket.put(snapshot['manifest-list'], manifestList)
+
+      // Create sample manifest entries
+      const entries: IcebergManifestEntry[] = [
+        {
+          status: 1, // ADDED
+          'snapshot-id': snapshot['snapshot-id'],
+          'sequence-number': snapshot['sequence-number'],
+          'file-sequence-number': snapshot['sequence-number'],
+          'data-file': createSampleDataFile(
+            `test-table/data/data-${snapshot['snapshot-id']}.parquet`
+          ),
+        },
+      ]
+      const manifest = createSampleManifest(entries)
+      await mockBucket.put(manifestFile['manifest-path'], manifest)
+    }
+
+    manager = createSnapshotManager({
+      bucket: mockBucket,
+      tablePath: 'test-table',
+      state: mockState,
+    })
+
+    await manager.initialize()
+  }
+
+  describe('Initialization', () => {
+    it('should initialize with no metadata', async () => {
+      manager = createSnapshotManager({
+        bucket: mockBucket,
+        tablePath: 'test-table',
+      })
+
+      await manager.initialize()
+
+      expect(manager.getMetadata()).toBeNull()
+      expect(manager.listSnapshots()).toEqual([])
+    })
+
+    it('should initialize with existing metadata', async () => {
+      const snapshots = [createSampleSnapshot(1001)]
+      await setupManagerWithSnapshots(snapshots)
+
+      expect(manager.getMetadata()).not.toBeNull()
+      expect(manager.listSnapshots()).toHaveLength(1)
+    })
+
+    it('should throw if not initialized', () => {
+      manager = createSnapshotManager({
+        bucket: mockBucket,
+        tablePath: 'test-table',
+      })
+
+      expect(() => manager.listSnapshots()).toThrow('SnapshotManager not initialized')
+    })
+  })
+
+  describe('Snapshot Listing', () => {
+    it('should list all snapshots with computed metadata', async () => {
+      const now = Date.now()
+      const snapshots = [
+        createSampleSnapshot(1001, undefined, now - 86400000, 1), // 1 day old
+        createSampleSnapshot(1002, 1001, now - 3600000, 2), // 1 hour old
+        createSampleSnapshot(1003, 1002, now - 60000, 3), // 1 minute old
+      ]
+      await setupManagerWithSnapshots(snapshots)
+
+      const listed = manager.listSnapshots()
+
+      expect(listed).toHaveLength(3)
+      expect(listed[0]!.snapshotId).toBe(1001)
+      expect(listed[0]!.operation).toBe('append')
+      expect(listed[0]!.summary.addedDataFiles).toBe(5)
+      expect(listed[0]!.ageMs).toBeGreaterThan(86000000)
+    })
+
+    it('should get a specific snapshot by ID', async () => {
+      const snapshots = [
+        createSampleSnapshot(1001),
+        createSampleSnapshot(1002, 1001),
+      ]
+      await setupManagerWithSnapshots(snapshots)
+
+      const snapshot = manager.getSnapshot(1002)
+
+      expect(snapshot).not.toBeNull()
+      expect(snapshot!.snapshotId).toBe(1002)
+      expect(snapshot!.parentSnapshotId).toBe(1001)
+    })
+
+    it('should return null for non-existent snapshot', async () => {
+      const snapshots = [createSampleSnapshot(1001)]
+      await setupManagerWithSnapshots(snapshots)
+
+      const snapshot = manager.getSnapshot(9999)
+
+      expect(snapshot).toBeNull()
+    })
+
+    it('should get the current snapshot', async () => {
+      const snapshots = [
+        createSampleSnapshot(1001),
+        createSampleSnapshot(1002, 1001),
+        createSampleSnapshot(1003, 1002),
+      ]
+      await setupManagerWithSnapshots(snapshots)
+
+      const current = manager.getCurrentSnapshot()
+
+      expect(current).not.toBeNull()
+      expect(current!.snapshotId).toBe(1003)
+    })
+
+    it('should get snapshot ancestry', async () => {
+      const snapshots = [
+        createSampleSnapshot(1001, undefined, Date.now() - 3000, 1),
+        createSampleSnapshot(1002, 1001, Date.now() - 2000, 2),
+        createSampleSnapshot(1003, 1002, Date.now() - 1000, 3),
+      ]
+      await setupManagerWithSnapshots(snapshots)
+
+      const ancestry = manager.getSnapshotAncestry(1003)
+
+      expect(ancestry).toHaveLength(3)
+      expect(ancestry[0]!.snapshotId).toBe(1003)
+      expect(ancestry[1]!.snapshotId).toBe(1002)
+      expect(ancestry[2]!.snapshotId).toBe(1001)
+    })
+  })
+
+  describe('Snapshot Expiration', () => {
+    it('should expire old snapshots beyond max age', async () => {
+      const now = Date.now()
+      const snapshots = [
+        createSampleSnapshot(1001, undefined, now - 40 * 86400000, 1), // 40 days old
+        createSampleSnapshot(1002, 1001, now - 35 * 86400000, 2), // 35 days old
+        createSampleSnapshot(1003, 1002, now - 10 * 86400000, 3), // 10 days old
+        createSampleSnapshot(1004, 1003, now - 5 * 86400000, 4), // 5 days old
+        createSampleSnapshot(1005, 1004, now - 1 * 86400000, 5), // 1 day old
+      ]
+      await setupManagerWithSnapshots(snapshots)
+
+      const result = await manager.expireSnapshots({
+        maxAgeDays: 30,
+        minSnapshotsToRetain: 3,
+      })
+
+      expect(result.snapshotsExpired).toBe(2)
+      expect(result.expiredSnapshotIds).toContain(1001)
+      expect(result.expiredSnapshotIds).toContain(1002)
+
+      const remaining = manager.listSnapshots()
+      expect(remaining).toHaveLength(3)
+    })
+
+    it('should respect minSnapshotsToRetain', async () => {
+      const now = Date.now()
+      const snapshots = [
+        createSampleSnapshot(1001, undefined, now - 100 * 86400000, 1), // 100 days old
+        createSampleSnapshot(1002, 1001, now - 90 * 86400000, 2), // 90 days old
+        createSampleSnapshot(1003, 1002, now - 80 * 86400000, 3), // 80 days old
+      ]
+      await setupManagerWithSnapshots(snapshots)
+
+      const result = await manager.expireSnapshots({
+        maxAgeDays: 7,
+        minSnapshotsToRetain: 3, // Keep at least 3
+      })
+
+      expect(result.snapshotsExpired).toBe(0)
+
+      const remaining = manager.listSnapshots()
+      expect(remaining).toHaveLength(3)
+    })
+
+    it('should respect maxSnapshotsToRetain', async () => {
+      const now = Date.now()
+      const snapshots = Array.from({ length: 10 }, (_, i) =>
+        createSampleSnapshot(1001 + i, i > 0 ? 1000 + i : undefined, now - (10 - i) * 86400000, i + 1)
+      )
+      await setupManagerWithSnapshots(snapshots)
+
+      const result = await manager.expireSnapshots({
+        maxAgeDays: 365, // Long retention
+        minSnapshotsToRetain: 1,
+        maxSnapshotsToRetain: 5,
+      })
+
+      expect(result.snapshotsExpired).toBe(5)
+
+      const remaining = manager.listSnapshots()
+      expect(remaining).toHaveLength(5)
+    })
+
+    it('should retain cherry-picked snapshots', async () => {
+      const now = Date.now()
+      const snapshots = [
+        createSampleSnapshot(1001, undefined, now - 100 * 86400000, 1), // Old but cherry-picked
+        createSampleSnapshot(1002, 1001, now - 50 * 86400000, 2),
+        createSampleSnapshot(1003, 1002, now - 1 * 86400000, 3),
+      ]
+      await setupManagerWithSnapshots(snapshots)
+
+      const result = await manager.expireSnapshots({
+        maxAgeDays: 7,
+        minSnapshotsToRetain: 1,
+        retainSnapshotIds: [1001], // Cherry-pick old snapshot
+      })
+
+      expect(result.expiredSnapshotIds).not.toContain(1001)
+      expect(result.expiredSnapshotIds).toContain(1002)
+
+      const remaining = manager.listSnapshots()
+      expect(remaining.some((s) => s.snapshotId === 1001)).toBe(true)
+    })
+
+    it('should never expire the current snapshot', async () => {
+      const now = Date.now()
+      const snapshots = [
+        createSampleSnapshot(1001, undefined, now - 100 * 86400000, 1), // Very old
+      ]
+      await setupManagerWithSnapshots(snapshots)
+
+      const result = await manager.expireSnapshots({
+        maxAgeDays: 1,
+        minSnapshotsToRetain: 0,
+      })
+
+      // Current snapshot should not be expired
+      expect(result.snapshotsExpired).toBe(0)
+    })
+
+    it('should return empty result when no snapshots to expire', async () => {
+      const now = Date.now()
+      const snapshots = [
+        createSampleSnapshot(1001, undefined, now - 1 * 86400000, 1), // 1 day old
+      ]
+      await setupManagerWithSnapshots(snapshots)
+
+      const result = await manager.expireSnapshots({
+        maxAgeDays: 30,
+        minSnapshotsToRetain: 5,
+      })
+
+      expect(result.snapshotsExpired).toBe(0)
+      expect(result.expiredSnapshotIds).toEqual([])
+    })
+  })
+
+  describe('File Statistics', () => {
+    it('should analyze file statistics', async () => {
+      const snapshots = [createSampleSnapshot(1001)]
+      await setupManagerWithSnapshots(snapshots)
+
+      const stats = await manager.analyzeFileStatistics()
+
+      expect(stats.totalFiles).toBeGreaterThanOrEqual(0)
+      expect(stats.avgFileSizeBytes).toBeGreaterThanOrEqual(0)
+    })
+
+    it('should identify small files', async () => {
+      // Create snapshots with small files
+      const metadata = createSampleMetadata([createSampleSnapshot(1001)])
+      await mockBucket.put('test-table/metadata/v1.metadata.json', JSON.stringify(metadata))
+      await mockBucket.put('test-table/metadata/version-hint.text', '1')
+
+      // Create manifest with small files
+      const manifestFile = createSampleManifestFile(1001)
+      const entries: IcebergManifestEntry[] = [
+        {
+          status: 1,
+          'snapshot-id': 1001,
+          'sequence-number': 1,
+          'file-sequence-number': 1,
+          'data-file': createSampleDataFile('test-table/data/small1.parquet', 10, 1024), // 1 KB
+        },
+        {
+          status: 1,
+          'snapshot-id': 1001,
+          'sequence-number': 1,
+          'file-sequence-number': 1,
+          'data-file': createSampleDataFile('test-table/data/small2.parquet', 10, 2048), // 2 KB
+        },
+        {
+          status: 1,
+          'snapshot-id': 1001,
+          'sequence-number': 1,
+          'file-sequence-number': 1,
+          'data-file': createSampleDataFile('test-table/data/large1.parquet', 1000, 100 * 1024 * 1024), // 100 MB
+        },
+      ]
+      const manifest = createSampleManifest(entries)
+      await mockBucket.put(manifestFile['manifest-path'], manifest)
+
+      const manifestList = createSampleManifestList([manifestFile])
+      await mockBucket.put('test-table/metadata/snap-1001-uuid.avro', manifestList)
+
+      manager = createSnapshotManager({ bucket: mockBucket, tablePath: 'test-table' })
+      await manager.initialize()
+
+      const stats = await manager.analyzeFileStatistics(10 * 1024 * 1024) // 10 MB threshold
+
+      expect(stats.totalFiles).toBe(3)
+      expect(stats.smallFileCount).toBe(2)
+      expect(stats.smallFilesSizeBytes).toBe(3072) // 1 KB + 2 KB
+    })
+  })
+
+  describe('Compaction Trigger', () => {
+    it('should trigger compaction when small file count exceeds threshold', async () => {
+      const metadata = createSampleMetadata([createSampleSnapshot(1001)])
+      await mockBucket.put('test-table/metadata/v1.metadata.json', JSON.stringify(metadata))
+      await mockBucket.put('test-table/metadata/version-hint.text', '1')
+
+      // Create many small files
+      const manifestFile = createSampleManifestFile(1001)
+      const entries: IcebergManifestEntry[] = Array.from({ length: 20 }, (_, i) => ({
+        status: 1 as const,
+        'snapshot-id': 1001,
+        'sequence-number': 1,
+        'file-sequence-number': 1,
+        'data-file': createSampleDataFile(`test-table/data/small${i}.parquet`, 10, 1024 * 1024), // 1 MB each
+      }))
+      await mockBucket.put(manifestFile['manifest-path'], createSampleManifest(entries))
+      await mockBucket.put('test-table/metadata/snap-1001-uuid.avro', createSampleManifestList([manifestFile]))
+
+      manager = createSnapshotManager({ bucket: mockBucket, tablePath: 'test-table' })
+      await manager.initialize()
+
+      const result = await manager.shouldCompact({
+        minFileSizeBytes: 10 * 1024 * 1024, // 10 MB threshold
+        minFilesToCompact: 5,
+      })
+
+      expect(result.shouldCompact).toBe(true)
+      expect(result.priority).toBeGreaterThan(0)
+    })
+
+    it('should not trigger compaction when file count is below threshold', async () => {
+      const metadata = createSampleMetadata([createSampleSnapshot(1001)])
+      await mockBucket.put('test-table/metadata/v1.metadata.json', JSON.stringify(metadata))
+      await mockBucket.put('test-table/metadata/version-hint.text', '1')
+
+      // Create only 2 small files (below minFilesToCompact)
+      const manifestFile = createSampleManifestFile(1001)
+      const entries: IcebergManifestEntry[] = [
+        {
+          status: 1,
+          'snapshot-id': 1001,
+          'sequence-number': 1,
+          'file-sequence-number': 1,
+          'data-file': createSampleDataFile('test-table/data/small1.parquet', 10, 1024),
+        },
+        {
+          status: 1,
+          'snapshot-id': 1001,
+          'sequence-number': 1,
+          'file-sequence-number': 1,
+          'data-file': createSampleDataFile('test-table/data/small2.parquet', 10, 1024),
+        },
+      ]
+      await mockBucket.put(manifestFile['manifest-path'], createSampleManifest(entries))
+      await mockBucket.put('test-table/metadata/snap-1001-uuid.avro', createSampleManifestList([manifestFile]))
+
+      manager = createSnapshotManager({ bucket: mockBucket, tablePath: 'test-table' })
+      await manager.initialize()
+
+      const result = await manager.shouldCompact({
+        minFilesToCompact: 5,
+      })
+
+      expect(result.shouldCompact).toBe(false)
+    })
+  })
+
+  describe('Compaction', () => {
+    it('should return empty result when no files to compact', async () => {
+      const snapshots = [createSampleSnapshot(1001)]
+      await setupManagerWithSnapshots(snapshots)
+
+      const result = await manager.compactFiles({
+        minFileSizeBytes: 1, // Very low threshold
+        minFilesToCompact: 100, // High minimum count
+      })
+
+      expect(result.inputFileCount).toBe(0)
+      expect(result.outputFileCount).toBe(0)
+      expect(result.snapshot).toBeNull()
+    })
+
+    it('should create compaction snapshot when files meet criteria', async () => {
+      const metadata = createSampleMetadata([createSampleSnapshot(1001)])
+      await mockBucket.put('test-table/metadata/v1.metadata.json', JSON.stringify(metadata))
+      await mockBucket.put('test-table/metadata/version-hint.text', '1')
+
+      // Create many small files
+      const manifestFile = createSampleManifestFile(1001, 10)
+      const entries: IcebergManifestEntry[] = Array.from({ length: 10 }, (_, i) => ({
+        status: 1 as const,
+        'snapshot-id': 1001,
+        'sequence-number': 1,
+        'file-sequence-number': 1,
+        'data-file': createSampleDataFile(
+          `test-table/data/partition=1/small${i}.parquet`,
+          100,
+          1024 * 1024 // 1 MB each
+        ),
+      }))
+      await mockBucket.put(manifestFile['manifest-path'], createSampleManifest(entries))
+      await mockBucket.put('test-table/metadata/snap-1001-uuid.avro', createSampleManifestList([manifestFile]))
+
+      manager = createSnapshotManager({ bucket: mockBucket, tablePath: 'test-table' })
+      await manager.initialize()
+
+      const result = await manager.compactFiles({
+        minFileSizeBytes: 10 * 1024 * 1024, // 10 MB threshold
+        minFilesToCompact: 5,
+        targetFileSizeBytes: 128 * 1024 * 1024,
+      })
+
+      expect(result.inputFileCount).toBe(10)
+      expect(result.partitionsCompacted).toBe(1)
+      expect(result.snapshot).not.toBeNull()
+    })
+  })
+
+  describe('Orphan File Cleanup', () => {
+    it('should identify orphan files not in any manifest', async () => {
+      const snapshots = [createSampleSnapshot(1001)]
+      await setupManagerWithSnapshots(snapshots)
+
+      // Add orphan files that are not referenced in any manifest
+      await mockBucket.put('test-table/data/orphan1.parquet', 'orphan data 1')
+      await mockBucket.put('test-table/data/orphan2.parquet', 'orphan data 2')
+
+      const result = await manager.cleanupOrphanFiles(true) // Dry run
+
+      expect(result.orphanFilesFound).toBe(2)
+      expect(result.orphanFilesDeleted).toBe(0) // Dry run
+      expect(result.orphanFilePaths).toContain('test-table/data/orphan1.parquet')
+      expect(result.orphanFilePaths).toContain('test-table/data/orphan2.parquet')
+    })
+
+    it('should delete orphan files when not in dry run mode', async () => {
+      const snapshots = [createSampleSnapshot(1001)]
+      await setupManagerWithSnapshots(snapshots)
+
+      // Add orphan files
+      await mockBucket.put('test-table/data/orphan1.parquet', 'orphan data 1')
+      await mockBucket.put('test-table/data/orphan2.parquet', 'orphan data 2')
+
+      const result = await manager.cleanupOrphanFiles(false) // Not dry run
+
+      expect(result.orphanFilesFound).toBe(2)
+      expect(result.orphanFilesDeleted).toBe(2)
+
+      // Verify files were deleted
+      const orphan1 = await mockBucket.get('test-table/data/orphan1.parquet')
+      const orphan2 = await mockBucket.get('test-table/data/orphan2.parquet')
+      expect(orphan1).toBeNull()
+      expect(orphan2).toBeNull()
+    })
+
+    it('should not delete referenced files', async () => {
+      const snapshots = [createSampleSnapshot(1001)]
+      await setupManagerWithSnapshots(snapshots)
+
+      // The data file referenced in the manifest should not be deleted
+      await mockBucket.put('test-table/data/data-1001.parquet', 'referenced data')
+
+      const result = await manager.cleanupOrphanFiles(false)
+
+      // The referenced file should still exist
+      const referenced = await mockBucket.get('test-table/data/data-1001.parquet')
+      expect(referenced).not.toBeNull()
+
+      // Only truly orphan files should be identified
+      expect(result.orphanFilePaths).not.toContain('test-table/data/data-1001.parquet')
+    })
+  })
+
+  describe('Statistics', () => {
+    it('should track manager statistics', async () => {
+      const now = Date.now()
+      const snapshots = [
+        createSampleSnapshot(1001, undefined, now - 100 * 86400000, 1),
+        createSampleSnapshot(1002, 1001, now - 1 * 86400000, 2),
+      ]
+      await setupManagerWithSnapshots(snapshots)
+
+      // Add orphan file
+      await mockBucket.put('test-table/data/orphan.parquet', 'data')
+
+      // Run expiration
+      await manager.expireSnapshots({ maxAgeDays: 7, minSnapshotsToRetain: 1 })
+
+      // Run cleanup
+      await manager.cleanupOrphanFiles(false)
+
+      const stats = manager.getStats()
+
+      expect(stats.totalSnapshotsExpired).toBe(1)
+      expect(stats.totalOrphanFilesCleaned).toBe(1)
+      expect(stats.lastExpirationAt).not.toBeNull()
+      expect(stats.lastCleanupAt).not.toBeNull()
+    })
+  })
+})
+
+describe('CompactionScheduler', () => {
+  let scheduler: CompactionScheduler
+  let manager: SnapshotManager
+  let mockBucket: R2Bucket & { _storage: Map<string, { data: string | ArrayBuffer; size: number }> }
+
+  beforeEach(async () => {
+    mockBucket = createMockBucket()
+
+    // Setup basic metadata
+    const metadata = createSampleMetadata([createSampleSnapshot(1001)])
+    await mockBucket.put('test-table/metadata/v1.metadata.json', JSON.stringify(metadata))
+    await mockBucket.put('test-table/metadata/version-hint.text', '1')
+
+    const manifestFile = createSampleManifestFile(1001)
+    await mockBucket.put(manifestFile['manifest-path'], createSampleManifest([]))
+    await mockBucket.put('test-table/metadata/snap-1001-uuid.avro', createSampleManifestList([manifestFile]))
+
+    manager = createSnapshotManager({
+      bucket: mockBucket,
+      tablePath: 'test-table',
+    })
+  })
+
+  it('should create a scheduler with default config', () => {
+    scheduler = createCompactionScheduler({ manager })
+
+    expect(scheduler.getCheckIntervalMs()).toBe(60 * 60 * 1000) // 1 hour
+  })
+
+  it('should create a scheduler with custom config', () => {
+    scheduler = createCompactionScheduler({
+      manager,
+      checkIntervalMs: 30 * 60 * 1000, // 30 minutes
+      compactionConfig: {
+        minFileSizeBytes: 5 * 1024 * 1024,
+      },
+      retentionConfig: {
+        maxAgeDays: 14,
+      },
+    })
+
+    expect(scheduler.getCheckIntervalMs()).toBe(30 * 60 * 1000)
+  })
+
+  it('should run maintenance and return results', async () => {
+    scheduler = createCompactionScheduler({
+      manager,
+      enableOrphanCleanup: true,
+    })
+
+    const results = await scheduler.runMaintenance()
+
+    expect(results.expirationResult).not.toBeNull()
+    expect(results.expirationResult!.snapshotsExpired).toBeGreaterThanOrEqual(0)
+    expect(results.compactionResult).toBeNull() // No files to compact
+    expect(results.cleanupResult).not.toBeNull()
+  })
+})