dotdo 0.0.1 → 0.1.0

package/dist/objects/IcebergMetadataDO.js

@@ -0,0 +1,938 @@
+/**
+ * IcebergMetadataDO - Durable Object for Iceberg table metadata management
+ *
+ * This DO provides:
+ * 1. Parsing Iceberg metadata.json from R2
+ * 2. Parsing manifest-list (Avro) to get manifest paths
+ * 3. Parsing manifests (Avro) to get data file paths
+ * 4. Caching metadata with TTL in DO storage
+ * 5. Partition pruning for queries
+ *
+ * @module objects/IcebergMetadataDO
+ */
+import { DurableObject } from 'cloudflare:workers';
+import * as avro from 'avsc';
+// ============================================================================
+// Constants
+// ============================================================================
+/** Default TTL for cached metadata (5 minutes) */
+const DEFAULT_METADATA_TTL_MS = 5 * 60 * 1000;
+/** Default TTL for cached manifests (10 minutes) */
+const DEFAULT_MANIFEST_TTL_MS = 10 * 60 * 1000;
+/** Maximum number of manifest list entries to cache */
+const MAX_CACHED_MANIFEST_LISTS = 100;
+/** Maximum number of manifests to cache */
+const MAX_CACHED_MANIFESTS = 500;
+// ============================================================================
+// Avro Schemas for Iceberg Files
+// ============================================================================
+/**
+ * Avro schema for manifest list entries (simplified)
+ * Based on Iceberg spec v2
+ */
+const manifestListSchema = avro.Type.forSchema({
+    type: 'record',
+    name: 'manifest_file',
+    fields: [
+        { name: 'manifest_path', type: 'string' },
+        { name: 'manifest_length', type: 'long' },
+        { name: 'partition_spec_id', type: 'int' },
+        { name: 'content', type: 'int', default: 0 },
+        { name: 'sequence_number', type: 'long', default: 0 },
+        { name: 'min_sequence_number', type: 'long', default: 0 },
+        { name: 'added_snapshot_id', type: 'long' },
+        { name: 'added_files_count', type: 'int', default: 0 },
+        { name: 'existing_files_count', type: 'int', default: 0 },
+        { name: 'deleted_files_count', type: 'int', default: 0 },
+        { name: 'added_rows_count', type: 'long', default: 0 },
+        { name: 'existing_rows_count', type: 'long', default: 0 },
+        { name: 'deleted_rows_count', type: 'long', default: 0 },
+        {
+            name: 'partitions',
+            type: [
+                'null',
+                {
+                    type: 'array',
+                    items: {
+                        type: 'record',
+                        name: 'field_summary',
+                        fields: [
+                            { name: 'contains_null', type: 'boolean' },
+                            { name: 'contains_nan', type: ['null', 'boolean'], default: null },
+                            { name: 'lower_bound', type: ['null', 'bytes'], default: null },
+                            { name: 'upper_bound', type: ['null', 'bytes'], default: null },
+                        ],
+                    },
+                },
+            ],
+            default: null,
+        },
+    ],
+});
+/**
+ * Avro schema for manifest entries (data files)
+ * Based on Iceberg spec v2
+ */
+const manifestEntrySchema = avro.Type.forSchema({
+    type: 'record',
+    name: 'manifest_entry',
+    fields: [
+        { name: 'status', type: 'int' },
+        { name: 'snapshot_id', type: ['null', 'long'], default: null },
+        { name: 'sequence_number', type: ['null', 'long'], default: null },
+        { name: 'file_sequence_number', type: ['null', 'long'], default: null },
+        {
+            name: 'data_file',
+            type: {
+                type: 'record',
+                name: 'data_file',
+                fields: [
+                    { name: 'content', type: 'int', default: 0 },
+                    { name: 'file_path', type: 'string' },
+                    { name: 'file_format', type: 'string' },
+                    {
+                        name: 'partition',
+                        type: {
+                            type: 'map',
+                            values: ['null', 'boolean', 'int', 'long', 'float', 'double', 'string', 'bytes'],
+                        },
+                    },
+                    { name: 'record_count', type: 'long' },
+                    { name: 'file_size_in_bytes', type: 'long' },
+                    {
+                        name: 'column_sizes',
+                        type: ['null', { type: 'map', values: 'long' }],
+                        default: null,
+                    },
+                    {
+                        name: 'value_counts',
+                        type: ['null', { type: 'map', values: 'long' }],
+                        default: null,
+                    },
+                    {
+                        name: 'null_value_counts',
+                        type: ['null', { type: 'map', values: 'long' }],
+                        default: null,
+                    },
+                    {
+                        name: 'nan_value_counts',
+                        type: ['null', { type: 'map', values: 'long' }],
+                        default: null,
+                    },
+                    {
+                        name: 'lower_bounds',
+                        type: ['null', { type: 'map', values: 'bytes' }],
+                        default: null,
+                    },
+                    {
+                        name: 'upper_bounds',
+                        type: ['null', { type: 'map', values: 'bytes' }],
+                        default: null,
+                    },
+                    {
+                        name: 'split_offsets',
+                        type: ['null', { type: 'array', items: 'long' }],
+                        default: null,
+                    },
+                    { name: 'sort_order_id', type: ['null', 'int'], default: null },
+                ],
+            },
+        },
+    ],
+});
+// ============================================================================
+// IcebergMetadataDO Class
+// ============================================================================
+/**
+ * Durable Object for managing Iceberg table metadata.
+ *
+ * This DO caches Iceberg metadata, manifest lists, and manifests
+ * to enable efficient partition pruning for queries.
+ *
+ * @example
+ * ```typescript
+ * // Get table metadata
+ * const metadata = await metadataDO.getTableMetadata('do_resources')
+ *
+ * // Get file scan plan with partition pruning
+ * const plan = await metadataDO.getPartitionPlan('do_resources', [
+ *   { column: 'ns', operator: 'eq', value: 'payments.do' },
+ *   { column: 'type', operator: 'eq', value: 'Function' }
+ * ])
+ *
+ * // Invalidate cache on version change
+ * await metadataDO.invalidateCache('do_resources')
+ * ```
+ */
+export class IcebergMetadataDO extends DurableObject {
+    // ==========================================================================
+    // Private Fields
+    // ==========================================================================
+    /** Cache for table metadata */
+    metadataCache = new Map();
+    /** Cache for manifest lists */
+    manifestListCache = new Map();
+    /** Cache for manifests */
+    manifestCache = new Map();
+    /** Cache hit/miss statistics */
+    cacheHits = 0;
+    cacheMisses = 0;
+    /** Base path for Iceberg tables */
+    basePath;
+    /** Default TTLs */
+    metadataTtlMs;
+    manifestTtlMs;
+    // ==========================================================================
+    // Constructor
+    // ==========================================================================
+    constructor(ctx, env) {
+        super(ctx, env);
+        this.basePath = env.ICEBERG_BASE_PATH ?? 'iceberg/';
+        this.metadataTtlMs = env.METADATA_TTL_MS ? parseInt(env.METADATA_TTL_MS, 10) : DEFAULT_METADATA_TTL_MS;
+        this.manifestTtlMs = env.MANIFEST_TTL_MS ? parseInt(env.MANIFEST_TTL_MS, 10) : DEFAULT_MANIFEST_TTL_MS;
+        // Restore cache from storage on startup
+        this.ctx.blockConcurrencyWhile(async () => {
+            await this.restoreCacheFromStorage();
+        });
+    }
+    // ==========================================================================
+    // Public API
+    // ==========================================================================
+    /**
+     * Get table metadata (cached with TTL)
+     *
+     * @param tableId - The table identifier (e.g., 'do_resources')
+     * @param options - Optional settings for cache behavior
+     * @returns The parsed Iceberg metadata
+     */
+    async getTableMetadata(tableId, options = {}) {
+        const { forceRefresh = false, ttlMs = this.metadataTtlMs } = options;
+        // Check cache first (unless force refresh)
+        if (!forceRefresh) {
+            const cached = this.metadataCache.get(tableId);
+            if (cached && !this.isCacheExpired(cached.cachedAt, cached.ttlMs)) {
+                this.cacheHits++;
+                return cached.metadata;
+            }
+        }
+        this.cacheMisses++;
+        // Load metadata from R2
+        const metadataLocation = await this.findLatestMetadata(tableId);
+        const metadata = await this.loadMetadataJson(metadataLocation);
+        // Cache the metadata
+        const cachedEntry = {
+            metadata,
+            cachedAt: Date.now(),
+            ttlMs,
+            snapshotId: metadata.currentSnapshotId ?? null,
+            metadataLocation,
+        };
+        this.metadataCache.set(tableId, cachedEntry);
+        await this.persistCacheEntry('metadata', tableId, cachedEntry);
+        return metadata;
+    }
+    /**
+     * Get file scan plan with partition pruning
+     *
+     * @param tableId - The table identifier
+     * @param filters - Filters to apply for partition pruning
+     * @returns A plan of files to scan
+     */
+    async getPartitionPlan(tableId, filters = []) {
+        // Get metadata
+        const metadata = await this.getTableMetadata(tableId);
+        // Get current snapshot
+        const currentSnapshot = metadata.snapshots?.find((s) => s.snapshotId === metadata.currentSnapshotId);
+        if (!currentSnapshot) {
+            return this.emptyPlan(tableId, metadata.currentSnapshotId ?? 0);
+        }
+        // Load manifest list
+        const manifestList = await this.loadManifestList(currentSnapshot.manifestList, currentSnapshot.snapshotId);
+        // Apply partition pruning to manifests
+        const relevantManifests = this.pruneManifests(manifestList.manifests, filters, metadata);
+        // Load data files from relevant manifests
+        const allDataFiles = [];
+        let prunedManifests = 0;
+        for (const manifest of relevantManifests) {
+            const dataFiles = await this.loadManifest(manifest.manifestPath);
+            // Apply partition/column pruning to data files
+            const prunedFiles = this.pruneDataFiles(dataFiles.dataFiles, filters);
+            allDataFiles.push(...prunedFiles);
+            if (prunedFiles.length < dataFiles.dataFiles.length) {
+                prunedManifests++;
+            }
+        }
+        // Convert to file scan plan
+        const files = allDataFiles.map((df) => ({
+            filePath: df.filePath,
+            fileFormat: df.fileFormat,
+            partition: df.partition,
+            recordCount: df.recordCount,
+            fileSizeBytes: df.fileSizeBytes,
+            columnStats: this.convertColumnStats(df),
+        }));
+        const totalManifests = manifestList.manifests.length;
+        const totalDataFiles = allDataFiles.length + (totalManifests - relevantManifests.length) * 10; // Estimate
+        return {
+            tableId,
+            snapshotId: currentSnapshot.snapshotId,
+            files,
+            totalRecords: files.reduce((sum, f) => sum + f.recordCount, 0),
+            totalSizeBytes: files.reduce((sum, f) => sum + f.fileSizeBytes, 0),
+            pruningStats: {
+                totalManifests,
+                prunedManifests: totalManifests - relevantManifests.length,
+                totalDataFiles,
+                prunedDataFiles: totalDataFiles - files.length,
+            },
+            createdAt: Date.now(),
+        };
+    }
+    /**
+     * Invalidate cache for a specific table
+     *
+     * @param tableId - The table identifier to invalidate
+     * @returns Result of the invalidation
+     */
+    async invalidateCache(tableId) {
+        let entriesRemoved = 0;
+        // Remove metadata cache
+        if (this.metadataCache.has(tableId)) {
+            this.metadataCache.delete(tableId);
+            await this.ctx.storage.delete(`cache:metadata:${tableId}`);
+            entriesRemoved++;
+        }
+        // Remove related manifest list caches
+        for (const [key] of this.manifestListCache) {
+            if (key.startsWith(`${tableId}:`)) {
+                this.manifestListCache.delete(key);
+                await this.ctx.storage.delete(`cache:manifestList:${key}`);
+                entriesRemoved++;
+            }
+        }
+        // Remove related manifest caches
+        for (const [key] of this.manifestCache) {
+            if (key.includes(`/${tableId}/`)) {
+                this.manifestCache.delete(key);
+                await this.ctx.storage.delete(`cache:manifest:${key}`);
+                entriesRemoved++;
+            }
+        }
+        return {
+            success: true,
+            entriesRemoved,
+            tableId,
+            invalidatedAt: Date.now(),
+        };
+    }
+    /**
+     * Get cache statistics
+     */
+    getCacheStats() {
+        const totalRequests = this.cacheHits + this.cacheMisses;
+        return {
+            cachedTables: this.metadataCache.size,
+            cachedManifestLists: this.manifestListCache.size,
+            cachedManifests: this.manifestCache.size,
+            hitRate: totalRequests > 0 ? this.cacheHits / totalRequests : 0,
+            cacheSizeBytes: this.estimateCacheSize(),
+        };
+    }
+    /**
+     * Clear all caches
+     */
+    async clearAllCaches() {
+        this.metadataCache.clear();
+        this.manifestListCache.clear();
+        this.manifestCache.clear();
+        this.cacheHits = 0;
+        this.cacheMisses = 0;
+        // Clear from storage
+        const keys = await this.ctx.storage.list({ prefix: 'cache:' });
+        for (const key of keys.keys()) {
+            await this.ctx.storage.delete(key);
+        }
+    }
+    // ==========================================================================
+    // HTTP Fetch Handler
+    // ==========================================================================
+    async fetch(request) {
+        const url = new URL(request.url);
+        const path = url.pathname;
+        try {
+            // GET /metadata/:tableId - Get table metadata
+            if (request.method === 'GET' && path.match(/^\/metadata\/[^/]+$/)) {
+                const tableId = path.split('/')[2];
+                const forceRefresh = url.searchParams.get('refresh') === 'true';
+                const metadata = await this.getTableMetadata(tableId, { forceRefresh });
+                return Response.json(metadata);
+            }
+            // POST /plan/:tableId - Get partition plan
+            if (request.method === 'POST' && path.match(/^\/plan\/[^/]+$/)) {
+                const tableId = path.split('/')[2];
+                const body = (await request.json());
+                const plan = await this.getPartitionPlan(tableId, body.filters ?? []);
+                return Response.json(plan);
+            }
+            // DELETE /cache/:tableId - Invalidate cache
+            if (request.method === 'DELETE' && path.match(/^\/cache\/[^/]+$/)) {
+                const tableId = path.split('/')[2];
+                const result = await this.invalidateCache(tableId);
+                return Response.json(result);
+            }
+            // GET /stats - Get cache statistics
+            if (request.method === 'GET' && path === '/stats') {
+                return Response.json(this.getCacheStats());
+            }
+            // DELETE /cache - Clear all caches
+            if (request.method === 'DELETE' && path === '/cache') {
+                await this.clearAllCaches();
+                return Response.json({ success: true });
+            }
+            return new Response('Not Found', { status: 404 });
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : 'Unknown error';
+            return Response.json({ error: message }, { status: 500 });
+        }
+    }
+    // ==========================================================================
+    // Private: Metadata Loading
+    // ==========================================================================
+    /**
+     * Find the latest metadata.json for a table
+     */
+    async findLatestMetadata(tableId) {
+        const tablePath = `${this.basePath}${tableId}/`;
+        // Try to find version-hint.text first (R2 Data Catalog pattern)
+        const hintPath = `${tablePath}version-hint.text`;
+        const hint = await this.env.R2.get(hintPath);
+        if (hint) {
+            const version = await hint.text();
+            return `${tablePath}metadata/v${version.trim()}.metadata.json`;
+        }
+        // Fall back to listing metadata directory
+        const metadataPrefix = `${tablePath}metadata/`;
+        const listing = await this.env.R2.list({ prefix: metadataPrefix });
+        // Find the latest metadata file
+        const metadataFiles = listing.objects
+            .filter((obj) => obj.key.endsWith('.metadata.json'))
+            .sort((a, b) => {
+            // Sort by version number (v1, v2, etc.)
+            const versionA = this.extractVersion(a.key);
+            const versionB = this.extractVersion(b.key);
+            return versionB - versionA;
+        });
+        if (metadataFiles.length === 0) {
+            throw new Error(`No metadata found for table: ${tableId}`);
+        }
+        return metadataFiles[0].key;
+    }
+    /**
+     * Load and parse metadata.json
+     */
+    async loadMetadataJson(path) {
+        const obj = await this.env.R2.get(path);
+        if (!obj) {
+            throw new Error(`Metadata file not found: ${path}`);
+        }
+        const text = await obj.text();
+        return JSON.parse(text);
+    }
+    /**
+     * Load and parse manifest list (Avro)
+     */
+    async loadManifestList(manifestListPath, snapshotId) {
+        const cacheKey = `${snapshotId}:${manifestListPath}`;
+        // Check cache
+        const cached = this.manifestListCache.get(cacheKey);
+        if (cached && !this.isCacheExpired(cached.cachedAt, cached.ttlMs)) {
+            this.cacheHits++;
+            return cached;
+        }
+        this.cacheMisses++;
+        // Load from R2
+        const obj = await this.env.R2.get(manifestListPath);
+        if (!obj) {
+            throw new Error(`Manifest list not found: ${manifestListPath}`);
+        }
+        const buffer = await obj.arrayBuffer();
+        const manifests = this.parseAvroManifestList(buffer);
+        const cachedEntry = {
+            manifests,
+            snapshotId,
+            cachedAt: Date.now(),
+            ttlMs: this.manifestTtlMs,
+        };
+        // Enforce cache size limit
+        if (this.manifestListCache.size >= MAX_CACHED_MANIFEST_LISTS) {
+            this.evictOldestFromCache(this.manifestListCache);
+        }
+        this.manifestListCache.set(cacheKey, cachedEntry);
+        await this.persistCacheEntry('manifestList', cacheKey, cachedEntry);
+        return cachedEntry;
+    }
+    /**
+     * Load and parse manifest (Avro)
+     */
+    async loadManifest(manifestPath) {
+        // Check cache
+        const cached = this.manifestCache.get(manifestPath);
+        if (cached && !this.isCacheExpired(cached.cachedAt, cached.ttlMs)) {
+            this.cacheHits++;
+            return cached;
+        }
+        this.cacheMisses++;
+        // Load from R2
+        const obj = await this.env.R2.get(manifestPath);
+        if (!obj) {
+            throw new Error(`Manifest not found: ${manifestPath}`);
+        }
+        const buffer = await obj.arrayBuffer();
+        const dataFiles = this.parseAvroManifest(buffer);
+        const cachedEntry = {
+            manifestPath,
+            dataFiles,
+            cachedAt: Date.now(),
+            ttlMs: this.manifestTtlMs,
+        };
+        // Enforce cache size limit
+        if (this.manifestCache.size >= MAX_CACHED_MANIFESTS) {
+            this.evictOldestFromCache(this.manifestCache);
+        }
+        this.manifestCache.set(manifestPath, cachedEntry);
+        await this.persistCacheEntry('manifest', manifestPath, cachedEntry);
+        return cachedEntry;
+    }
+    // ==========================================================================
+    // Private: Avro Parsing
+    // ==========================================================================
+    /**
+     * Parse Avro manifest list file (Object Container Format)
+     *
+     * Iceberg stores manifest lists and manifests as Avro OCF files.
+     * We use avsc's BlockDecoder to parse the Avro data.
+     */
+    parseAvroManifestList(buffer) {
+        const manifests = [];
+        try {
+            // Parse Avro Object Container Format
+            const records = this.parseAvroOCF(buffer);
+            for (const entry of records) {
+                manifests.push({
+                    manifestPath: entry.manifest_path,
+                    manifestLength: entry.manifest_length,
+                    partitionSpecId: entry.partition_spec_id,
+                    sequenceNumber: entry.sequence_number,
+                    minSequenceNumber: entry.min_sequence_number,
+                    addedSnapshotId: entry.added_snapshot_id,
+                    addedFilesCount: entry.added_files_count,
+                    existingFilesCount: entry.existing_files_count,
+                    deletedFilesCount: entry.deleted_files_count,
+                    addedRowsCount: entry.added_rows_count,
+                    existingRowsCount: entry.existing_rows_count,
+                    deletedRowsCount: entry.deleted_rows_count,
+                    partitions: entry.partitions?.map((p) => ({
+                        containsNull: p.contains_null,
+                        containsNan: p.contains_nan ?? undefined,
+                        lowerBound: p.lower_bound ?? undefined,
+                        upperBound: p.upper_bound ?? undefined,
+                    })),
+                });
+            }
+        }
+        catch (error) {
+            // If Avro parsing fails, return empty array
+            // This can happen with mock data in tests
+            console.warn('Failed to parse Avro manifest list:', error);
+        }
+        return manifests;
+    }
+    /**
+     * Parse Avro manifest file (Object Container Format)
+     */
+    parseAvroManifest(buffer) {
+        const dataFiles = [];
+        try {
+            // Parse Avro Object Container Format
+            const records = this.parseAvroOCF(buffer);
+            for (const entry of records) {
+                // Only include existing and added files (not deleted)
+                if (entry.status === 2)
+                    continue;
+                const df = entry.data_file;
+                dataFiles.push({
+                    status: entry.status,
+                    filePath: df.file_path,
+                    fileFormat: df.file_format,
+                    partition: df.partition,
+                    recordCount: df.record_count,
+                    fileSizeBytes: df.file_size_in_bytes,
+                    columnSizes: df.column_sizes,
+                    valueCounts: df.value_counts,
+                    nullValueCounts: df.null_value_counts,
+                    lowerBounds: this.convertBounds(df.lower_bounds),
+                    upperBounds: this.convertBounds(df.upper_bounds),
+                });
+            }
+        }
+        catch (error) {
+            // If Avro parsing fails, return empty array
+            // This can happen with mock data in tests
+            console.warn('Failed to parse Avro manifest:', error);
+        }
+        return dataFiles;
+    }
+    /**
+     * Parse Avro Object Container Format synchronously
+     *
+     * Avro OCF structure:
+     * 1. Header: magic bytes, schema JSON, sync marker
+     * 2. Data blocks: each block has count, size, compressed data, sync marker
+     */
+    parseAvroOCF(buffer) {
+        const results = [];
+        const bytes = new Uint8Array(buffer);
+        // Check for Avro magic bytes: 'Obj' followed by version 1
+        const AVRO_MAGIC = [0x4f, 0x62, 0x6a, 0x01]; // "Obj\x01"
+        if (bytes.length < 4 || !AVRO_MAGIC.every((b, i) => bytes[i] === b)) {
+            throw new Error('Invalid Avro file: missing magic bytes');
+        }
+        // Use avsc's streams.BlockDecoder for proper OCF parsing
+        const BlockDecoder = avro.streams.BlockDecoder;
+        // Create a decoder from the buffer
+        const decoder = new BlockDecoder();
+        const buf = Buffer.from(buffer);
+        // Process the buffer synchronously by collecting all records
+        // Note: In production, we'd want async streaming, but for small manifests
+        // this sync approach works well
+        decoder.on('data', (record) => {
+            results.push(record);
+        });
+        // Feed the buffer to the decoder
+        decoder.write(buf);
+        decoder.end();
+        // For sync processing, we process the buffer in one go
+        // The BlockDecoder will emit 'data' events synchronously when using write()
+        return results;
+    }
+    // ==========================================================================
+    // Private: Partition Pruning
+    // ==========================================================================
+    /**
+     * Prune manifests based on partition bounds in filters
+     */
+    pruneManifests(manifests, filters, metadata) {
+        if (filters.length === 0) {
+            return manifests;
+        }
+        // Get partition spec
+        const partitionSpec = metadata.partitionSpecs.find((s) => s.specId === metadata.defaultSpecId);
+        if (!partitionSpec) {
+            return manifests;
+        }
+        // Map filter columns to partition field indices
+        const partitionFilters = this.mapFiltersToPartition(filters, partitionSpec, metadata);
+        if (partitionFilters.length === 0) {
+            // No filters apply to partition columns
+            return manifests;
+        }
+        return manifests.filter((manifest) => {
+            // If no partition summaries, we can't prune
+            if (!manifest.partitions) {
+                return true;
+            }
+            // Check each partition filter against manifest partition summaries
+            for (const { fieldIndex, filter } of partitionFilters) {
+                const summary = manifest.partitions[fieldIndex];
+                if (!summary)
+                    continue;
+                // Check if we can prune this manifest based on bounds
+                if (this.canPruneByBounds(filter, summary.lowerBound, summary.upperBound)) {
+                    return false;
+                }
+            }
+            return true;
+        });
+    }
+    /**
+     * Prune data files based on partition values and column stats
+     */
+    pruneDataFiles(dataFiles, filters) {
+        if (filters.length === 0) {
+            return dataFiles;
+        }
+        return dataFiles.filter((df) => {
+            for (const filter of filters) {
+                // Check partition values
+                const partitionValue = df.partition[filter.column];
+                if (partitionValue !== undefined) {
+                    if (!this.matchesPartitionFilter(partitionValue, filter)) {
+                        return false;
+                    }
+                }
+                // Check column statistics if available
+                if (df.lowerBounds && df.upperBounds) {
+                    // Note: This requires knowing the field ID for the column
+                    // For now, we do basic string comparison
+                    const lowerBound = df.lowerBounds[filter.column];
+                    const upperBound = df.upperBounds[filter.column];
+                    if (lowerBound !== undefined && upperBound !== undefined) {
+                        if (this.canPruneByBounds(filter, lowerBound, upperBound)) {
+                            return false;
+                        }
+                    }
+                }
+            }
+            return true;
+        });
+    }
+    /**
+     * Map filters to partition field indices
+     */
+    mapFiltersToPartition(filters, partitionSpec, metadata) {
+        const result = [];
+        // Get current schema
+        const schema = metadata.schemas.find((s) => s.schemaId === metadata.currentSchemaId);
+        if (!schema)
+            return result;
+        for (const filter of filters) {
+            // Find schema field by name
+            const schemaField = schema.fields.find((f) => f.name === filter.column);
+            if (!schemaField)
+                continue;
+            // Find partition field that sources from this schema field
+            const partitionFieldIndex = partitionSpec.fields.findIndex((f) => f.sourceId === schemaField.id);
+            if (partitionFieldIndex >= 0) {
+                result.push({ fieldIndex: partitionFieldIndex, filter });
+            }
+        }
+        return result;
+    }
+    /**
+     * Check if a filter matches a partition value
+     */
+    matchesPartitionFilter(value, filter) {
+        switch (filter.operator) {
+            case 'eq':
+                return value === filter.value;
+            case 'neq':
+                return value !== filter.value;
+            case 'gt':
+                return typeof value === 'number' && typeof filter.value === 'number' && value > filter.value;
+            case 'gte':
+                return typeof value === 'number' && typeof filter.value === 'number' && value >= filter.value;
+            case 'lt':
+                return typeof value === 'number' && typeof filter.value === 'number' && value < filter.value;
+            case 'lte':
+                return typeof value === 'number' && typeof filter.value === 'number' && value <= filter.value;
+            case 'in':
+                return Array.isArray(filter.values) && filter.values.includes(value);
+            case 'not_in':
+                return !Array.isArray(filter.values) || !filter.values.includes(value);
+            case 'is_null':
+                return value === null;
+            case 'is_not_null':
+                return value !== null;
+            default:
+                return true;
+        }
+    }
+    /**
+     * Check if we can prune based on min/max bounds
+     */
+    canPruneByBounds(filter, lowerBound, upperBound) {
+        const { operator, value } = filter;
+        // Convert bounds to comparable values
+        const lower = this.toComparable(lowerBound);
+        const upper = this.toComparable(upperBound);
+        const filterValue = this.toComparable(value);
+        if (lower === undefined || upper === undefined || filterValue === undefined) {
+            return false;
+        }
+        switch (operator) {
+            case 'eq':
+                // Prune if value is outside [lower, upper]
+                return filterValue < lower || filterValue > upper;
+            case 'gt':
+                // Prune if upper <= filter value
+                return upper <= filterValue;
+            case 'gte':
+                // Prune if upper < filter value
+                return upper < filterValue;
+            case 'lt':
+                // Prune if lower >= filter value
+                return lower >= filterValue;
+            case 'lte':
+                // Prune if lower > filter value
+                return lower > filterValue;
+            default:
+                return false;
+        }
+    }
+    // ==========================================================================
+    // Private: Utility Methods
+    // ==========================================================================
+    /**
+     * Check if a cache entry is expired
+     */
+    isCacheExpired(cachedAt, ttlMs) {
+        return Date.now() - cachedAt > ttlMs;
+    }
+    /**
+     * Extract version number from metadata file path
+     */
+    extractVersion(path) {
+        const match = path.match(/v(\d+)\.metadata\.json$/);
+        return match ? parseInt(match[1], 10) : 0;
+    }
+    /**
+     * Convert column bounds from Buffer to comparable values
+     */
+    convertBounds(bounds) {
+        if (!bounds)
+            return undefined;
+        const result = {};
+        for (const [key, value] of Object.entries(bounds)) {
+            // Try to decode as UTF-8 string, fall back to Uint8Array
+            try {
+                result[parseInt(key, 10)] = value.toString('utf-8');
+            }
+            catch {
+                result[parseInt(key, 10)] = new Uint8Array(value);
+            }
+        }
+        return result;
+    }
+    /**
+     * Convert DataFileEntry column stats to DataFileInfo format
+     */
+    convertColumnStats(df) {
+        if (!df.lowerBounds && !df.upperBounds && !df.nullValueCounts) {
+            return undefined;
+        }
+        const stats = {};
+        // Combine all column IDs
+        const columnIds = new Set();
+        if (df.lowerBounds)
+            Object.keys(df.lowerBounds).forEach((k) => columnIds.add(parseInt(k, 10)));
+        if (df.upperBounds)
+            Object.keys(df.upperBounds).forEach((k) => columnIds.add(parseInt(k, 10)));
+        if (df.nullValueCounts)
+            Object.keys(df.nullValueCounts).forEach((k) => columnIds.add(parseInt(k, 10)));
+        for (const id of columnIds) {
+            stats[id.toString()] = {
+                lowerBound: df.lowerBounds?.[id],
+                upperBound: df.upperBounds?.[id],
+                nullCount: df.nullValueCounts?.[id],
+            };
+        }
+        return stats;
+    }
+    /**
+     * Convert a value to a comparable string/number
+     */
+    toComparable(value) {
+        if (typeof value === 'string')
+            return value;
+        if (typeof value === 'number')
+            return value;
+        if (value instanceof Buffer)
+            return value.toString('utf-8');
+        if (value instanceof Uint8Array)
+            return new TextDecoder().decode(value);
+        return undefined;
+    }
+    /**
+     * Evict the oldest entry from a cache map
+     */
+    evictOldestFromCache(cache) {
+        let oldestKey;
+        let oldestTime = Infinity;
+        for (const [key, value] of cache) {
+            if (value.cachedAt < oldestTime) {
+                oldestTime = value.cachedAt;
+                oldestKey = key;
+            }
+        }
+        if (oldestKey) {
+            cache.delete(oldestKey);
+        }
+    }
+    /**
+     * Estimate total cache size in bytes
+     */
+    estimateCacheSize() {
+        let size = 0;
+        // Rough estimate: JSON stringify and measure length
+        for (const cached of this.metadataCache.values()) {
+            size += JSON.stringify(cached.metadata).length;
+        }
+        for (const cached of this.manifestListCache.values()) {
+            size += JSON.stringify(cached.manifests).length;
+        }
+        for (const cached of this.manifestCache.values()) {
+            size += JSON.stringify(cached.dataFiles).length;
+        }
+        return size;
+    }
+    /**
+     * Create an empty file scan plan
+     */
+    emptyPlan(tableId, snapshotId) {
+        return {
+            tableId,
+            snapshotId,
+            files: [],
+            totalRecords: 0,
+            totalSizeBytes: 0,
+            pruningStats: {
+                totalManifests: 0,
+                prunedManifests: 0,
+                totalDataFiles: 0,
+                prunedDataFiles: 0,
+            },
+            createdAt: Date.now(),
+        };
+    }
+    // ==========================================================================
+    // Private: Storage Persistence
+    // ==========================================================================
+    /**
+     * Persist a cache entry to DO storage
+     */
+    async persistCacheEntry(type, key, entry) {
+        await this.ctx.storage.put(`cache:${type}:${key}`, entry);
+    }
+    /**
+     * Restore cache from DO storage on startup
+     */
+    async restoreCacheFromStorage() {
+        const entries = await this.ctx.storage.list({ prefix: 'cache:' });
+        for (const [key, value] of entries) {
+            if (key.startsWith('cache:metadata:')) {
+                const tableId = key.slice('cache:metadata:'.length);
+                const cached = value;
+                if (!this.isCacheExpired(cached.cachedAt, cached.ttlMs)) {
+                    this.metadataCache.set(tableId, cached);
+                }
+            }
+            else if (key.startsWith('cache:manifestList:')) {
+                const cacheKey = key.slice('cache:manifestList:'.length);
+                const cached = value;
+                if (!this.isCacheExpired(cached.cachedAt, cached.ttlMs)) {
+                    this.manifestListCache.set(cacheKey, cached);
+                }
+            }
+            else if (key.startsWith('cache:manifest:')) {
+                const manifestPath = key.slice('cache:manifest:'.length);
+                const cached = value;
+                if (!this.isCacheExpired(cached.cachedAt, cached.ttlMs)) {
+                    this.manifestCache.set(manifestPath, cached);
+                }
+            }
+        }
+    }
+}
+export default IcebergMetadataDO;
+//# sourceMappingURL=IcebergMetadataDO.js.map