@dotdo/postgres 0.1.0

package/dist/iceberg/snapshot-manager.js

@@ -0,0 +1,934 @@
+/**
+ * Iceberg Snapshot Management and Compaction
+ * Task: postgres-iyx - Snapshot management and compaction
+ *
+ * Provides snapshot lifecycle management and data compaction for Iceberg tables:
+ * - Snapshot listing and metadata retrieval
+ * - Snapshot expiration/retention
+ * - File compaction (merge small files)
+ * - Orphan file cleanup
+ * - Manifest rewriting for optimization
+ * - Background compaction scheduling
+ *
+ * @example
+ * ```typescript
+ * const manager = createSnapshotManager({
+ *   bucket: env.ICEBERG_BUCKET,
+ *   tablePath: 'iceberg/my-table',
+ * })
+ *
+ * await manager.initialize()
+ *
+ * // List snapshots
+ * const snapshots = manager.listSnapshots()
+ *
+ * // Expire old snapshots
+ * await manager.expireSnapshots({
+ *   maxAgeDays: 30,
+ *   minSnapshotsToRetain: 5,
+ * })
+ *
+ * // Compact small files
+ * await manager.compactFiles({
+ *   minFileSizeBytes: 10 * 1024 * 1024, // 10 MB
+ *   maxFilesPerCompaction: 100,
+ * })
+ *
+ * // Cleanup orphan files
+ * await manager.cleanupOrphanFiles()
+ * ```
+ */
+import { IcebergMetadataBuilder, ManifestBuilder, ManifestListBuilder, generateMetadataFileName, generateManifestFileName, generateManifestListFileName, } from './metadata';
+import { createLogger, LogLevel } from '@dotdo/postgres-shared';
+// Module-level logger for snapshot manager errors
+const snapshotLogger = createLogger({
+    level: LogLevel.ERROR,
+    prefix: '[SnapshotManager]',
+});
+// ========== Default Configuration ==========
+const DEFAULT_RETENTION = {
+    maxAgeDays: 30,
+    minSnapshotsToRetain: 5,
+    maxSnapshotsToRetain: 100,
+    retainSnapshotIds: [],
+};
+const DEFAULT_COMPACTION = {
+    minFileSizeBytes: 10 * 1024 * 1024, // 10 MB
+    maxFilesPerCompaction: 100,
+    maxInputSizeBytes: 1024 * 1024 * 1024, // 1 GB
+    targetFileSizeBytes: 128 * 1024 * 1024, // 128 MB
+    strategy: 'bin-pack',
+    partitionFilter: {},
+    minFilesToCompact: 5,
+};
+// ========== Snapshot Manager Class ==========
+/**
+ * Iceberg Snapshot Manager
+ *
+ * Manages snapshot lifecycle including expiration, compaction, and cleanup.
+ */
+export class SnapshotManager {
+    bucket;
+    tablePath;
+    metadata = null;
+    metadataBuilder = null;
+    metadataVersion = 0;
+    initialized = false;
+    stats = {
+        totalSnapshotsExpired: 0,
+        totalCompactions: 0,
+        totalBytesFreedExpiration: 0,
+        totalBytesSavedCompaction: 0,
+        totalOrphanFilesCleaned: 0,
+        lastExpirationAt: null,
+        lastCompactionAt: null,
+        lastCleanupAt: null,
+    };
+    constructor(config) {
+        this.bucket = config.bucket;
+        this.tablePath = config.tablePath.replace(/\/$/, '');
+        // state is accepted for future use but not currently utilized
+        void config.state;
+    }
+    /**
+     * Initialize the snapshot manager
+     */
+    async initialize() {
+        if (this.initialized) {
+            return;
+        }
+        await this.loadMetadata();
+        this.initialized = true;
+    }
+    /**
+     * Ensure initialized
+     */
+    ensureInitialized() {
+        if (!this.initialized) {
+            throw new Error('SnapshotManager not initialized. Call initialize() first.');
+        }
+    }
+    /**
+     * Load table metadata from R2
+     */
+    async loadMetadata() {
+        // Try version hint first
+        const versionHint = await this.bucket.get(`${this.tablePath}/metadata/version-hint.text`);
+        if (versionHint) {
+            this.metadataVersion = parseInt(await versionHint.text(), 10);
+            const metadataPath = `${this.tablePath}/metadata/v${this.metadataVersion}.metadata.json`;
+            const metadataObj = await this.bucket.get(metadataPath);
+            if (metadataObj) {
+                this.metadata = JSON.parse(await metadataObj.text());
+                this.metadataBuilder = IcebergMetadataBuilder.fromMetadata(this.metadata);
+                return;
+            }
+        }
+        // Fall back to listing metadata files
+        const list = await this.bucket.list({
+            prefix: `${this.tablePath}/metadata/`,
+        });
+        const metadataFiles = list.objects
+            .filter((obj) => obj.key.endsWith('.metadata.json'))
+            .sort((a, b) => {
+            const versionA = parseInt(a.key.match(/v(\d+)\.metadata\.json/)?.[1] ?? '0', 10);
+            const versionB = parseInt(b.key.match(/v(\d+)\.metadata\.json/)?.[1] ?? '0', 10);
+            return versionB - versionA;
+        });
+        if (metadataFiles.length > 0 && metadataFiles[0]) {
+            const metadataObj = await this.bucket.get(metadataFiles[0].key);
+            if (metadataObj) {
+                this.metadata = JSON.parse(await metadataObj.text());
+                this.metadataBuilder = IcebergMetadataBuilder.fromMetadata(this.metadata);
+                this.metadataVersion = parseInt(metadataFiles[0].key.match(/v(\d+)\.metadata\.json/)?.[1] ?? '0', 10);
+            }
+        }
+    }
+    /**
+     * Refresh metadata from R2
+     */
+    async refreshMetadata() {
+        this.initialized = false;
+        this.metadata = null;
+        this.metadataBuilder = null;
+        await this.initialize();
+    }
+    // ========== Snapshot Listing and Metadata ==========
+    /**
+     * List all snapshots with computed metadata
+     */
+    listSnapshots() {
+        this.ensureInitialized();
+        if (!this.metadata?.snapshots) {
+            return [];
+        }
+        const now = Date.now();
+        return this.metadata.snapshots.map((snapshot) => this.snapshotToInfo(snapshot, now));
+    }
+    /**
+     * Get a specific snapshot by ID
+     */
+    getSnapshot(snapshotId) {
+        this.ensureInitialized();
+        const snapshot = this.metadata?.snapshots?.find((s) => s['snapshot-id'] === snapshotId);
+        if (!snapshot) {
+            return null;
+        }
+        return this.snapshotToInfo(snapshot, Date.now());
+    }
+    /**
+     * Get the current snapshot
+     */
+    getCurrentSnapshot() {
+        this.ensureInitialized();
+        if (!this.metadata?.['current-snapshot-id']) {
+            return null;
+        }
+        return this.getSnapshot(this.metadata['current-snapshot-id']);
+    }
+    /**
+     * Get snapshot ancestry (parent chain)
+     */
+    getSnapshotAncestry(snapshotId) {
+        this.ensureInitialized();
+        const ancestry = [];
+        let currentId = snapshotId;
+        while (currentId !== undefined) {
+            const snapshot = this.getSnapshot(currentId);
+            if (!snapshot) {
+                break;
+            }
+            ancestry.push(snapshot);
+            currentId = snapshot.parentSnapshotId;
+        }
+        return ancestry;
+    }
+    /**
+     * Convert IcebergSnapshot to SnapshotInfo
+     */
+    snapshotToInfo(snapshot, now) {
+        const summary = snapshot.summary;
+        return {
+            snapshotId: snapshot['snapshot-id'],
+            parentSnapshotId: snapshot['parent-snapshot-id'],
+            sequenceNumber: snapshot['sequence-number'],
+            timestampMs: snapshot['timestamp-ms'],
+            operation: summary.operation,
+            summary: {
+                addedDataFiles: parseInt(summary['added-data-files'] ?? '0', 10),
+                deletedDataFiles: parseInt(summary['deleted-data-files'] ?? '0', 10),
+                addedRecords: parseInt(summary['added-records'] ?? '0', 10),
+                deletedRecords: parseInt(summary['deleted-records'] ?? '0', 10),
+                totalRecords: parseInt(summary['total-records'] ?? '0', 10),
+                totalFilesSize: parseInt(summary['total-files-size'] ?? '0', 10),
+            },
+            manifestListPath: snapshot['manifest-list'],
+            schemaId: snapshot['schema-id'] ?? 0,
+            ageMs: now - snapshot['timestamp-ms'],
+        };
+    }
+    // ========== Snapshot Expiration ==========
+    /**
+     * Expire old snapshots based on retention policy
+     */
+    async expireSnapshots(config = {}) {
+        this.ensureInitialized();
+        const startTime = performance.now();
+        const retention = { ...DEFAULT_RETENTION, ...config };
+        if (!this.metadata?.snapshots || this.metadata.snapshots.length === 0) {
+            return {
+                snapshotsExpired: 0,
+                expiredSnapshotIds: [],
+                manifestsRemoved: 0,
+                manifestListsRemoved: 0,
+                metadataBytesFreed: 0,
+                durationMs: performance.now() - startTime,
+            };
+        }
+        const now = Date.now();
+        const maxAgeMs = retention.maxAgeDays * 24 * 60 * 60 * 1000;
+        const currentSnapshotId = this.metadata['current-snapshot-id'];
+        // Identify snapshots to expire
+        const snapshots = [...this.metadata.snapshots];
+        const snapshotsToKeep = [];
+        const snapshotsToExpire = [];
+        // Sort by timestamp (newest first) for minSnapshots logic
+        snapshots.sort((a, b) => b['timestamp-ms'] - a['timestamp-ms']);
+        for (let i = 0; i < snapshots.length; i++) {
+            const snapshot = snapshots[i];
+            const age = now - snapshot['timestamp-ms'];
+            const isCurrentSnapshot = snapshot['snapshot-id'] === currentSnapshotId;
+            const isRetainedById = retention.retainSnapshotIds.includes(snapshot['snapshot-id']);
+            const withinMinSnapshots = i < retention.minSnapshotsToRetain;
+            const withinMaxAge = age <= maxAgeMs;
+            const beyondMaxSnapshots = i >= retention.maxSnapshotsToRetain;
+            if (isCurrentSnapshot || isRetainedById || (withinMinSnapshots && !beyondMaxSnapshots)) {
+                snapshotsToKeep.push(snapshot);
+            }
+            else if (!withinMaxAge || beyondMaxSnapshots) {
+                snapshotsToExpire.push(snapshot);
+            }
+            else {
+                snapshotsToKeep.push(snapshot);
+            }
+        }
+        if (snapshotsToExpire.length === 0) {
+            return {
+                snapshotsExpired: 0,
+                expiredSnapshotIds: [],
+                manifestsRemoved: 0,
+                manifestListsRemoved: 0,
+                metadataBytesFreed: 0,
+                durationMs: performance.now() - startTime,
+            };
+        }
+        // Collect manifest lists and manifests to delete
+        const manifestListsToDelete = [];
+        const manifestsToDelete = new Set();
+        let metadataBytesFreed = 0;
+        // Get manifests referenced by snapshots to keep
+        const keptManifests = new Set();
+        for (const snapshot of snapshotsToKeep) {
+            const manifests = await this.loadManifestsFromList(snapshot['manifest-list']);
+            for (const manifest of manifests) {
+                keptManifests.add(manifest['manifest-path']);
+            }
+        }
+        // Identify manifest lists and manifests to delete
+        for (const snapshot of snapshotsToExpire) {
+            manifestListsToDelete.push(snapshot['manifest-list']);
+            const manifests = await this.loadManifestsFromList(snapshot['manifest-list']);
+            for (const manifest of manifests) {
+                // Only delete if not referenced by a kept snapshot
+                if (!keptManifests.has(manifest['manifest-path'])) {
+                    manifestsToDelete.add(manifest['manifest-path']);
+                    metadataBytesFreed += manifest['manifest-length'];
+                }
+            }
+        }
+        // Delete manifest files
+        for (const manifestPath of manifestsToDelete) {
+            try {
+                await this.bucket.delete(manifestPath);
+            }
+            catch (error) {
+                snapshotLogger.error('Failed to delete manifest', { manifestPath, error: error instanceof Error ? error.message : String(error) });
+            }
+        }
+        // Delete manifest lists
+        for (const manifestListPath of manifestListsToDelete) {
+            try {
+                const obj = await this.bucket.head(manifestListPath);
+                if (obj) {
+                    metadataBytesFreed += obj.size;
+                }
+                await this.bucket.delete(manifestListPath);
+            }
+            catch (error) {
+                snapshotLogger.error('Failed to delete manifest list', { manifestListPath, error: error instanceof Error ? error.message : String(error) });
+            }
+        }
+        // Update metadata with remaining snapshots
+        await this.updateMetadataWithSnapshots(snapshotsToKeep);
+        // Update stats
+        const result = {
+            snapshotsExpired: snapshotsToExpire.length,
+            expiredSnapshotIds: snapshotsToExpire.map((s) => s['snapshot-id']),
+            manifestsRemoved: manifestsToDelete.size,
+            manifestListsRemoved: manifestListsToDelete.length,
+            metadataBytesFreed,
+            durationMs: performance.now() - startTime,
+        };
+        this.stats.totalSnapshotsExpired += result.snapshotsExpired;
+        this.stats.totalBytesFreedExpiration += result.metadataBytesFreed;
+        this.stats.lastExpirationAt = Date.now();
+        return result;
+    }
+    /**
+     * Load manifests from a manifest list
+     */
+    async loadManifestsFromList(manifestListPath) {
+        try {
+            const obj = await this.bucket.get(manifestListPath);
+            if (!obj) {
+                return [];
+            }
+            const data = JSON.parse(await obj.text());
+            return data.manifests ?? [];
+        }
+        catch (error) {
+            snapshotLogger.error('Failed to load manifest list', { manifestListPath, error: error instanceof Error ? error.message : String(error) });
+            return [];
+        }
+    }
+    /**
+     * Update metadata with new snapshot list
+     */
+    async updateMetadataWithSnapshots(snapshots) {
+        if (!this.metadata || !this.metadataBuilder) {
+            throw new Error('Metadata not loaded');
+        }
+        // Sort snapshots by timestamp
+        snapshots.sort((a, b) => a['timestamp-ms'] - b['timestamp-ms']);
+        // Create new metadata
+        const newMetadata = {
+            ...this.metadata,
+            'last-updated-ms': Date.now(),
+            snapshots: snapshots,
+            'snapshot-log': snapshots.map((s) => ({
+                'timestamp-ms': s['timestamp-ms'],
+                'snapshot-id': s['snapshot-id'],
+            })),
+        };
+        // Update current snapshot if needed
+        if (newMetadata['current-snapshot-id'] !== undefined &&
+            !snapshots.some((s) => s['snapshot-id'] === newMetadata['current-snapshot-id'])) {
+            // Current snapshot was expired, set to latest
+            const latestSnapshot = snapshots[snapshots.length - 1];
+            if (latestSnapshot) {
+                newMetadata['current-snapshot-id'] = latestSnapshot['snapshot-id'];
+            }
+            else {
+                delete newMetadata['current-snapshot-id'];
+            }
+        }
+        // Write new metadata version
+        this.metadataVersion++;
+        const metadataFileName = generateMetadataFileName(this.metadataVersion);
+        const metadataPath = `${this.tablePath}/metadata/${metadataFileName}`;
+        await this.bucket.put(metadataPath, JSON.stringify(newMetadata, null, 2), {
+            httpMetadata: { contentType: 'application/json' },
+        });
+        // Update version hint
+        await this.bucket.put(`${this.tablePath}/metadata/version-hint.text`, String(this.metadataVersion), { httpMetadata: { contentType: 'text/plain' } });
+        // Update internal state
+        this.metadata = newMetadata;
+        this.metadataBuilder = IcebergMetadataBuilder.fromMetadata(newMetadata);
+    }
+    // ========== File Compaction ==========
+    /**
+     * Analyze file statistics for compaction decisions
+     */
+    async analyzeFileStatistics(threshold = DEFAULT_COMPACTION.minFileSizeBytes) {
+        this.ensureInitialized();
+        const dataFiles = await this.getAllDataFiles();
+        const filesByPartition = new Map();
+        let totalSize = 0;
+        let minSize = Infinity;
+        let maxSize = 0;
+        let smallFileCount = 0;
+        let smallFilesSize = 0;
+        for (const file of dataFiles) {
+            const size = file['file-size-in-bytes'];
+            totalSize += size;
+            minSize = Math.min(minSize, size);
+            maxSize = Math.max(maxSize, size);
+            if (size < threshold) {
+                smallFileCount++;
+                smallFilesSize += size;
+            }
+            // Extract partition path
+            const partitionPath = this.extractPartitionPath(file['file-path']);
+            filesByPartition.set(partitionPath, (filesByPartition.get(partitionPath) ?? 0) + 1);
+        }
+        return {
+            totalFiles: dataFiles.length,
+            totalSizeBytes: totalSize,
+            avgFileSizeBytes: dataFiles.length > 0 ? totalSize / dataFiles.length : 0,
+            minFileSizeBytes: minSize === Infinity ? 0 : minSize,
+            maxFileSizeBytes: maxSize,
+            smallFileCount,
+            smallFilesSizeBytes: smallFilesSize,
+            filesByPartition,
+        };
+    }
+    /**
+     * Check if compaction should be triggered
+     */
+    async shouldCompact(config = {}) {
+        const compactionConfig = { ...DEFAULT_COMPACTION, ...config };
+        const stats = await this.analyzeFileStatistics(compactionConfig.minFileSizeBytes);
+        // Check file count threshold
+        if (stats.smallFileCount < compactionConfig.minFilesToCompact) {
+            return {
+                shouldCompact: false,
+                reason: `Small file count (${stats.smallFileCount}) below threshold (${compactionConfig.minFilesToCompact})`,
+                priority: 0,
+                estimatedBenefit: 0,
+            };
+        }
+        // Calculate compaction benefit
+        const avgSmallFileSize = stats.smallFileCount > 0
+            ? stats.smallFilesSizeBytes / stats.smallFileCount
+            : 0;
+        const targetFileCount = Math.ceil(stats.smallFilesSizeBytes / compactionConfig.targetFileSizeBytes);
+        const estimatedOutputFiles = Math.max(targetFileCount, 1);
+        const fileReduction = stats.smallFileCount - estimatedOutputFiles;
+        // Priority based on file count and size ratio
+        const fileCountRatio = stats.smallFileCount / stats.totalFiles;
+        const sizeRatio = avgSmallFileSize / compactionConfig.targetFileSizeBytes;
+        const priority = Math.min(100, Math.round((fileCountRatio * 50) + ((1 - sizeRatio) * 50)));
+        return {
+            shouldCompact: true,
+            reason: `${stats.smallFileCount} small files (${formatBytes(stats.smallFilesSizeBytes)}) can be compacted`,
+            priority,
+            estimatedBenefit: fileReduction * 1000, // Rough estimate of metadata savings per file
+        };
+    }
+    /**
+     * Compact small files
+     *
+     * Note: Full compaction requires reading and rewriting Parquet files.
+     * This implementation tracks which files should be compacted and creates
+     * a new snapshot with the compaction metadata. Actual file rewriting
+     * would require Parquet library support.
+     */
+    async compactFiles(config = {}) {
+        this.ensureInitialized();
+        const startTime = performance.now();
+        const compactionConfig = { ...DEFAULT_COMPACTION, ...config };
+        // Get all small files
+        const allDataFiles = await this.getAllDataFiles();
+        const smallFiles = allDataFiles.filter((f) => f['file-size-in-bytes'] < compactionConfig.minFileSizeBytes);
+        // Group by partition
+        const filesByPartition = new Map();
+        for (const file of smallFiles) {
+            const partitionPath = this.extractPartitionPath(file['file-path']);
+            // Apply partition filter if specified
+            if (Object.keys(compactionConfig.partitionFilter).length > 0) {
+                const matches = this.matchesPartitionFilter(file.partition, compactionConfig.partitionFilter);
+                if (!matches) {
+                    continue;
+                }
+            }
+            const files = filesByPartition.get(partitionPath) ?? [];
+            files.push(file);
+            filesByPartition.set(partitionPath, files);
+        }
+        // Calculate compaction groups
+        const compactionGroups = [];
+        for (const [partition, files] of filesByPartition) {
+            if (files.length < compactionConfig.minFilesToCompact) {
+                continue;
+            }
+            // Sort by size (ascending) for bin-packing
+            files.sort((a, b) => a['file-size-in-bytes'] - b['file-size-in-bytes']);
+            // Create groups respecting maxFilesPerCompaction and maxInputSizeBytes
+            let currentGroup = [];
+            let currentSize = 0;
+            for (const file of files) {
+                if (currentGroup.length >= compactionConfig.maxFilesPerCompaction ||
+                    currentSize + file['file-size-in-bytes'] > compactionConfig.maxInputSizeBytes) {
+                    if (currentGroup.length >= compactionConfig.minFilesToCompact) {
+                        compactionGroups.push({
+                            partition,
+                            files: currentGroup,
+                            totalSize: currentSize,
+                        });
+                    }
+                    currentGroup = [];
+                    currentSize = 0;
+                }
+                currentGroup.push(file);
+                currentSize += file['file-size-in-bytes'];
+            }
+            // Add remaining files
+            if (currentGroup.length >= compactionConfig.minFilesToCompact) {
+                compactionGroups.push({
+                    partition,
+                    files: currentGroup,
+                    totalSize: currentSize,
+                });
+            }
+        }
+        if (compactionGroups.length === 0) {
+            return {
+                inputFileCount: 0,
+                outputFileCount: 0,
+                inputSizeBytes: 0,
+                outputSizeBytes: 0,
+                sizeReductionBytes: 0,
+                recordsProcessed: 0,
+                partitionsCompacted: 0,
+                snapshot: null,
+                durationMs: performance.now() - startTime,
+            };
+        }
+        // Calculate totals
+        let totalInputFiles = 0;
+        let totalInputSize = 0;
+        let totalRecords = 0;
+        const partitionsCompacted = new Set();
+        for (const group of compactionGroups) {
+            totalInputFiles += group.files.length;
+            totalInputSize += group.totalSize;
+            partitionsCompacted.add(group.partition);
+            for (const file of group.files) {
+                totalRecords += file['record-count'];
+            }
+        }
+        // Create compaction snapshot
+        // Note: In a full implementation, this would:
+        // 1. Read all input Parquet files
+        // 2. Merge and write new Parquet files
+        // 3. Create manifest with DELETED entries for old files and ADDED for new
+        // For now, we create a metadata-only compaction marker
+        const snapshot = await this.createCompactionSnapshot(compactionGroups, compactionConfig.targetFileSizeBytes);
+        // Calculate output estimates
+        const estimatedOutputFiles = compactionGroups.reduce((sum, group) => {
+            return sum + Math.ceil(group.totalSize / compactionConfig.targetFileSizeBytes);
+        }, 0);
+        const result = {
+            inputFileCount: totalInputFiles,
+            outputFileCount: estimatedOutputFiles,
+            inputSizeBytes: totalInputSize,
+            outputSizeBytes: totalInputSize, // In reality would be slightly different
+            sizeReductionBytes: 0, // Actual reduction from file consolidation
+            recordsProcessed: totalRecords,
+            partitionsCompacted: partitionsCompacted.size,
+            snapshot,
+            durationMs: performance.now() - startTime,
+        };
+        this.stats.totalCompactions++;
+        this.stats.totalBytesSavedCompaction += result.sizeReductionBytes;
+        this.stats.lastCompactionAt = Date.now();
+        return result;
+    }
+    /**
+     * Create a compaction snapshot
+     */
+    async createCompactionSnapshot(compactionGroups, _targetFileSizeBytes) {
+        if (!this.metadata || !this.metadataBuilder) {
+            return null;
+        }
+        const snapshotId = Date.now() * 1000 + Math.floor(Math.random() * 1000);
+        const sequenceNumber = this.metadataBuilder.getSequenceNumber() + 1;
+        const partitionSpecId = this.metadata['default-spec-id'];
+        // Create manifest builder for compaction
+        const manifestBuilder = new ManifestBuilder({
+            partitionSpecId,
+            sequenceNumber,
+            snapshotId,
+        });
+        // Mark old files as deleted, add placeholder for compacted files
+        let totalDeletedFiles = 0;
+        let totalDeletedRecords = 0;
+        for (const group of compactionGroups) {
+            for (const file of group.files) {
+                manifestBuilder.deleteDataFile(file);
+                totalDeletedFiles++;
+                totalDeletedRecords += file['record-count'];
+            }
+            // In a real implementation, we would add the newly compacted files here
+            // For now, we just mark the intent to compact
+        }
+        // Write manifest
+        const manifestFileName = generateManifestFileName(snapshotId);
+        const manifestPath = `${this.tablePath}/metadata/${manifestFileName}`;
+        const manifestContent = manifestBuilder.toJSON();
+        const manifestBytes = new TextEncoder().encode(manifestContent);
+        await this.bucket.put(manifestPath, manifestBytes, {
+            httpMetadata: { contentType: 'application/json' },
+        });
+        const manifestFileMetadata = manifestBuilder.createManifestFileMetadata(manifestPath, manifestBytes.length);
+        // Create manifest list
+        const manifestListBuilder = new ManifestListBuilder(snapshotId, sequenceNumber);
+        manifestListBuilder.addManifest(manifestFileMetadata);
+        // Include existing manifests
+        const currentSnapshot = this.metadata.snapshots?.find((s) => s['snapshot-id'] === this.metadata['current-snapshot-id']);
+        if (currentSnapshot) {
+            const existingManifests = await this.loadManifestsFromList(currentSnapshot['manifest-list']);
+            for (const manifest of existingManifests) {
+                manifestListBuilder.addManifest(manifest);
+            }
+        }
+        // Write manifest list
+        const manifestListFileName = generateManifestListFileName(snapshotId);
+        const manifestListPath = `${this.tablePath}/metadata/${manifestListFileName}`;
+        const manifestListContent = manifestListBuilder.toJSON();
+        const manifestListBytes = new TextEncoder().encode(manifestListContent);
+        await this.bucket.put(manifestListPath, manifestListBytes, {
+            httpMetadata: { contentType: 'application/json' },
+        });
+        // Create snapshot
+        const summary = {
+            operation: 'replace',
+            'added-data-files': '0',
+            'deleted-data-files': String(totalDeletedFiles),
+            'added-records': '0',
+            'deleted-records': String(totalDeletedRecords),
+            'compaction-type': 'bin-pack',
+        };
+        const snapshot = {
+            'snapshot-id': snapshotId,
+            ...(this.metadata['current-snapshot-id'] !== undefined && {
+                'parent-snapshot-id': this.metadata['current-snapshot-id'],
+            }),
+            'sequence-number': sequenceNumber,
+            'timestamp-ms': Date.now(),
+            'manifest-list': manifestListPath,
+            summary,
+            'schema-id': this.metadata['current-schema-id'],
+        };
+        // Update metadata
+        const newSnapshots = [...(this.metadata.snapshots ?? []), snapshot];
+        await this.updateMetadataWithSnapshots(newSnapshots);
+        return snapshot;
+    }
+    // ========== Orphan File Cleanup ==========
+    /**
+     * Find and clean up orphan files
+     *
+     * Orphan files are data files that exist in R2 but are not referenced
+     * by any manifest in any snapshot.
+     */
+    async cleanupOrphanFiles(dryRun = false) {
+        this.ensureInitialized();
+        const startTime = performance.now();
+        // Get all referenced data files from all snapshots
+        const referencedFiles = new Set();
+        if (this.metadata?.snapshots) {
+            for (const snapshot of this.metadata.snapshots) {
+                const manifests = await this.loadManifestsFromList(snapshot['manifest-list']);
+                for (const manifest of manifests) {
+                    const entries = await this.loadManifestEntries(manifest['manifest-path']);
+                    for (const entry of entries) {
+                        // Only count EXISTING and ADDED files (not DELETED)
+                        if (entry.status !== 2) {
+                            referencedFiles.add(entry['data-file']['file-path']);
+                        }
+                    }
+                }
+            }
+        }
+        // List all data files in R2
+        const orphanFiles = [];
+        let cursor;
+        do {
+            const listOptions = {
+                prefix: `${this.tablePath}/data/`,
+                limit: 1000,
+            };
+            if (cursor) {
+                listOptions.cursor = cursor;
+            }
+            const result = await this.bucket.list(listOptions);
+            for (const obj of result.objects) {
+                if (obj.key.endsWith('.parquet') && !referencedFiles.has(obj.key)) {
+                    orphanFiles.push({ key: obj.key, size: obj.size });
+                }
+            }
+            cursor = result.truncated ? result.cursor : undefined;
+        } while (cursor);
+        // Delete orphan files (unless dry run)
+        let bytesFreed = 0;
+        const deletedPaths = [];
+        if (!dryRun) {
+            for (const orphan of orphanFiles) {
+                try {
+                    await this.bucket.delete(orphan.key);
+                    bytesFreed += orphan.size;
+                    deletedPaths.push(orphan.key);
+                }
+                catch (error) {
+                    snapshotLogger.error('Failed to delete orphan file', { key: orphan.key, error: error instanceof Error ? error.message : String(error) });
+                }
+            }
+        }
+        else {
+            bytesFreed = orphanFiles.reduce((sum, f) => sum + f.size, 0);
+        }
+        const result = {
+            orphanFilesFound: orphanFiles.length,
+            orphanFilesDeleted: dryRun ? 0 : deletedPaths.length,
+            orphanFilePaths: orphanFiles.map((f) => f.key),
+            bytesFreed,
+            durationMs: performance.now() - startTime,
+        };
+        if (!dryRun) {
+            this.stats.totalOrphanFilesCleaned += result.orphanFilesDeleted;
+            this.stats.lastCleanupAt = Date.now();
+        }
+        return result;
+    }
+    /**
+     * Load manifest entries from a manifest file
+     */
+    async loadManifestEntries(manifestPath) {
+        try {
+            const obj = await this.bucket.get(manifestPath);
+            if (!obj) {
+                return [];
+            }
+            const data = JSON.parse(await obj.text());
+            return data.entries ?? [];
+        }
+        catch (error) {
+            snapshotLogger.error('Failed to load manifest', { manifestPath, error: error instanceof Error ? error.message : String(error) });
+            return [];
+        }
+    }
+    /**
+     * Get all data files from the current snapshot
+     */
+    async getAllDataFiles() {
+        if (!this.metadata?.['current-snapshot-id']) {
+            return [];
+        }
+        const currentSnapshot = this.metadata.snapshots?.find((s) => s['snapshot-id'] === this.metadata['current-snapshot-id']);
+        if (!currentSnapshot) {
+            return [];
+        }
+        const dataFiles = [];
+        const manifests = await this.loadManifestsFromList(currentSnapshot['manifest-list']);
+        for (const manifest of manifests) {
+            if (manifest.content !== 0) {
+                continue; // Skip delete manifests
+            }
+            const entries = await this.loadManifestEntries(manifest['manifest-path']);
+            for (const entry of entries) {
+                // Only include EXISTING (0) and ADDED (1) files
+                if (entry.status !== 2) {
+                    dataFiles.push(entry['data-file']);
+                }
+            }
+        }
+        return dataFiles;
+    }
+    /**
+     * Extract partition path from file path
+     */
+    extractPartitionPath(filePath) {
+        const dataIndex = filePath.indexOf('/data/');
+        if (dataIndex === -1) {
+            return '';
+        }
+        const afterData = filePath.substring(dataIndex + 6); // Skip '/data/'
+        const lastSlash = afterData.lastIndexOf('/');
+        if (lastSlash === -1) {
+            return '';
+        }
+        return afterData.substring(0, lastSlash);
+    }
+    /**
+     * Check if partition values match filter
+     */
+    matchesPartitionFilter(partition, filter) {
+        for (const [key, value] of Object.entries(filter)) {
+            if (partition[key] !== value) {
+                return false;
+            }
+        }
+        return true;
+    }
+    // ========== Utility Methods ==========
+    /**
+     * Get manager statistics
+     */
+    getStats() {
+        return { ...this.stats };
+    }
+    /**
+     * Get the table metadata
+     */
+    getMetadata() {
+        return this.metadata;
+    }
+    /**
+     * Get the table path
+     */
+    getTablePath() {
+        return this.tablePath;
+    }
+    /**
+     * Get the current metadata version
+     */
+    getMetadataVersion() {
+        return this.metadataVersion;
+    }
+}
+// ========== Factory Function ==========
+/**
+ * Create a snapshot manager
+ */
+export function createSnapshotManager(config) {
+    return new SnapshotManager(config);
+}
+// ========== Utility Functions ==========
+/**
+ * Format bytes as human-readable string
+ */
+function formatBytes(bytes) {
+    const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+    let unitIndex = 0;
+    let value = bytes;
+    while (value >= 1024 && unitIndex < units.length - 1) {
+        value /= 1024;
+        unitIndex++;
+    }
+    return `${value.toFixed(2)} ${units[unitIndex]}`;
+}
+/**
+ * Compaction scheduler for background maintenance
+ *
+ * Can be used with Durable Object alarms for scheduled maintenance.
+ */
+export class CompactionScheduler {
+    manager;
+    checkIntervalMs;
+    compactionConfig;
+    retentionConfig;
+    enableOrphanCleanup;
+    orphanCleanupIntervalMs;
+    lastOrphanCleanup = 0;
+    constructor(config) {
+        this.manager = config.manager;
+        this.checkIntervalMs = config.checkIntervalMs ?? 60 * 60 * 1000; // 1 hour
+        this.compactionConfig = config.compactionConfig ?? {};
+        this.retentionConfig = config.retentionConfig ?? {};
+        this.enableOrphanCleanup = config.enableOrphanCleanup ?? true;
+        this.orphanCleanupIntervalMs = config.orphanCleanupIntervalMs ?? 24 * 60 * 60 * 1000; // 24 hours
+    }
+    /**
+     * Run scheduled maintenance
+     *
+     * This method should be called periodically (e.g., from a DO alarm).
+     */
+    async runMaintenance() {
+        await this.manager.initialize();
+        // Expire old snapshots
+        const expirationResult = await this.manager.expireSnapshots(this.retentionConfig);
+        // Check if compaction is needed
+        let compactionResult = null;
+        const triggerResult = await this.manager.shouldCompact(this.compactionConfig);
+        if (triggerResult.shouldCompact) {
+            compactionResult = await this.manager.compactFiles(this.compactionConfig);
+        }
+        // Run orphan cleanup if enabled and due
+        let cleanupResult = null;
+        const now = Date.now();
+        if (this.enableOrphanCleanup &&
+            now - this.lastOrphanCleanup >= this.orphanCleanupIntervalMs) {
+            cleanupResult = await this.manager.cleanupOrphanFiles();
+            this.lastOrphanCleanup = now;
+        }
+        return {
+            expirationResult,
+            compactionResult,
+            cleanupResult,
+        };
+    }
+    /**
+     * Get the check interval
+     */
+    getCheckIntervalMs() {
+        return this.checkIntervalMs;
+    }
+}
+/**
+ * Create a compaction scheduler
+ */
+export function createCompactionScheduler(config) {
+    return new CompactionScheduler(config);
+}
+//# sourceMappingURL=snapshot-manager.js.map