@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3

package/dist/parquet-source-loader.js

@@ -1,17 +1,22 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
-import { hydrateArrowTable } from '@loaders.gl/arrow';
-import { BlobFile, DataSource, isBrowser } from '@loaders.gl/loader-utils';
+import { dehydrateArrowTable, hydrateArrowTable, IndexedArrowTable } from '@loaders.gl/arrow';
+import { BlobFile, DataSource, createScanQueryMetadata, emitScanExecutionTelemetry, explainTableQuery, isBrowser, makeTableScanBatch, validateTableQueryLimit } from '@loaders.gl/loader-utils';
 import { convertTable } from '@loaders.gl/schema-utils';
 import { getSchemaFromParquetReader } from "./lib/parsers/get-parquet-schema.js";
+import { canGeoParquetRowGroupMatch, combineParquetPredicates, createGeoParquetBoundingBoxPredicate } from "./lib/geo/geoparquet-covering.js";
+import { decodeParquetSplitBlockBloomFilter, encodeParquetBloomFilterValue, hashParquetBloomFilterValue, checkParquetSplitBlockBloomFilter } from "./lib/parquet-bloom-filter.js";
+import { getParquetBloomFilterProbes } from "./lib/parquet-bloom-filter-planner.js";
 import { canParquetRowGroupMatch, copyParquetPredicate, filterParquetRowIndices, gatherParquetColumns, getParquetPredicateColumns, validateParquetPredicate } from "./lib/parquet-predicate.js";
 import { createParquetPagePruningPlan, getParquetPageReadRanges } from "./lib/parquet-page-index.js";
 import { canDecodeParquetSourceOnWorker, decodeParquetSourceRowGroupOnWorker } from "./lib/parquet-source-worker-client.js";
+import { sliceParquetBatch } from "./lib/slice-parquet-batch.js";
 import { ParquetRangeFile } from "./lib/sources/parquet-range-file.js";
-import { PARQUET_SOURCE_CAPABILITIES } from "./parquet-source-capabilities.js";
+import { toPrimitive } from "./parquetjs/schema/types.js";
+import { PARQUET_TABLE_QUERY_CAPABILITIES, PARQUET_SOURCE_CAPABILITIES } from "./parquet-source-capabilities.js";
 import { ParquetSourceLoader as ParquetSourceLoaderMetadata } from "./parquet-source-loader-types.js";
-export { PARQUET_SOURCE_CAPABILITIES } from "./parquet-source-capabilities.js";
+export { PARQUET_TABLE_QUERY_CAPABILITIES, PARQUET_SOURCE_CAPABILITIES } from "./parquet-source-capabilities.js";
 import { preloadCompressions } from "./parquetjs/compression.js";
 import { CompressionCodec, Encoding } from "./parquetjs/parquet-thrift/index.js";
 import { ParquetReader } from "./parquetjs/parser/parquet-reader.js";
@@ -28,6 +33,8 @@ export const ParquetSourceLoaderWithParser = {
 export { ParquetSourceLoaderWithParser as ParquetSourceLoader };
 /** Reusable Parquet source that caches footer/schema state and selectively reads byte ranges. */
 export class ParquetSource extends DataSource {
+    /** Common projection, predicate, limit, streaming, and cancellation capabilities. */
+    tableQueryCapabilities = PARQUET_TABLE_QUERY_CAPABILITIES;
     /** Immutable feature support for the current range-backed source implementation. */
     capabilities = PARQUET_SOURCE_CAPABILITIES;
     /** Shared initialization for this source instance. */
@@ -66,39 +73,118 @@ export class ParquetSource extends DataSource {
             formatSpecificMetadata: initialization.fileMetadata
         };
     }
+    /** Discovers schema, capabilities, and footer statistics without decoding Parquet data pages. */
+    async getQueryMetadata(options = {}) {
+        const [schema, metadata] = await Promise.all([
+            this.getSchema({ signal: options.signal }),
+            this.getMetadata({ signal: options.signal })
+        ]);
+        return createScanQueryMetadata({
+            sourceType: 'parquet',
+            queryType: 'table',
+            execution: { status: 'supported', method: 'read' },
+            name: metadata.name,
+            schema,
+            capabilities: { table: this.tableQueryCapabilities },
+            columnRoles: getParquetColumnRoles(schema.fields.map(field => field.name)),
+            statistics: { rowCount: metadata.rowCount, byteLength: metadata.fileByteLength }
+        });
+    }
+    /** Plans a portable query using Parquet metadata without decoding data pages. */
+    async getScanPlan(options = {}) {
+        const readOptions = this.getReadOptions(options);
+        const initialization = await this.getInitialization(readOptions.signal);
+        const sourceColumnNames = initialization.schema.fields.map(field => field.name);
+        const signal = readOptions.signal ?? new AbortController().signal;
+        const physicalPlan = await this.planRowGroups(initialization, readOptions, signal);
+        const pagePlans = await this.planPages(initialization, readOptions, physicalPlan, signal);
+        const explanation = explainTableQuery(sourceColumnNames, {
+            columns: readOptions.columns,
+            predicate: physicalPlan.predicate,
+            limit: readOptions.limit
+        }, PARQUET_TABLE_QUERY_CAPABILITIES);
+        return Object.freeze({
+            ...explanation,
+            source: 'parquet',
+            rowGroups: Object.freeze({
+                indices: Object.freeze([...physicalPlan.rowGroupIndices]),
+                requested: physicalPlan.requested,
+                selected: physicalPlan.rowGroupIndices.length,
+                prunedByCallback: physicalPlan.prunedByCallback,
+                prunedBySpatial: physicalPlan.prunedBySpatial,
+                prunedByStatistics: physicalPlan.prunedByStatistics,
+                prunedByBloomFilter: physicalPlan.prunedByBloomFilter
+            }),
+            bloomFilters: Object.freeze({
+                read: physicalPlan.bloomFiltersRead,
+                bytesRead: physicalPlan.bloomFilterBytesRead
+            }),
+            pages: Object.freeze({
+                rowGroupsPlanned: new Set(pagePlans.map(plan => plan.rowGroupIndex)).size,
+                indexesRead: pagePlans.reduce((sum, plan) => sum + plan.indexesRead, 0),
+                total: pagePlans.reduce((sum, plan) => sum + plan.totalPages, 0),
+                selected: pagePlans.reduce((sum, plan) => sum + plan.selectedPages, 0),
+                rowsPruned: pagePlans.reduce((sum, plan) => sum + plan.rowsPruned, 0),
+                plans: Object.freeze(pagePlans)
+            })
+        });
+    }
+    /** Explains the common logical query and Parquet-specific physical scan plan. */
+    async explain(options = {}) {
+        return await this.getScanPlan(options);
+    }
+    /** Executes a previously computed physical plan while preserving its row-group decisions. */
+    async *executeScanPlan(plan, options = {}) {
+        if (plan.source !== 'parquet') {
+            throw new Error('ParquetSource can only execute Parquet scan plans');
+        }
+        const predicateStep = plan.plan.find(step => step.kind === 'filter');
+        const limitStep = plan.plan.find(step => step.kind === 'limit');
+        yield* this.read({
+            ...options,
+            columns: options.columns ?? plan.outputColumns,
+            predicate: options.predicate ??
+                (predicateStep?.kind === 'filter' ? predicateStep.predicate : undefined),
+            limit: options.limit ?? (limitStep?.kind === 'limit' ? limitStep.limit : undefined),
+            rowGroups: plan.rowGroups.indices
+        });
+    }
+    /** Common scan-architecture alias for selective Parquet reads. */
+    scan(options = {}) {
+        return this.read(options);
+    }
     /** Returns a copy of cumulative transport, decode, conversion, and pruning telemetry. */
     getTelemetry() {
         return Object.freeze({ ...this.telemetry });
     }
     /** Selectively fetches row groups and columns as ordered Arrow batches with source provenance. */
     async *read(options = {}) {
+        const startedAt = Date.now();
+        const telemetryBefore = this.getTelemetry();
         const readOptions = this.getReadOptions(options);
+        if (readOptions.limit === 0) {
+            emitScanExecutionTelemetry(options.onTelemetry, createParquetScanExecutionTelemetry(telemetryBefore, this.getTelemetry(), startedAt, 'early-terminated', 'limit'));
+            return;
+        }
         const readContext = createReadAbortContext(readOptions.signal);
         const inFlightReads = new Set();
         let completed = false;
+        let limitReached = false;
         let readError;
         this.activeReadControllers.add(readContext.abortController);
         try {
             const initialization = await this.getInitialization(readContext.abortController.signal);
             await this.getCompressionInitialization();
             throwIfAborted(readContext.abortController.signal);
-            const candidateRowGroupIndices = normalizeRowGroupIndices(readOptions.rowGroups, initialization.fileMetadata.row_groups.length);
-            const callbackFilteredRowGroupIndices = readOptions.rowGroupFilter
-                ? candidateRowGroupIndices.filter(rowGroupIndex => readOptions.rowGroupFilter(initialization.metadata.rowGroups[rowGroupIndex]))
-                : candidateRowGroupIndices;
-            const predicate = readOptions.predicate;
-            const availableColumns = new Set(initialization.schema.fields.map(field => field.name));
-            if (predicate) {
-                validateParquetPredicate(predicate, availableColumns);
-            }
-            const rowGroupIndices = predicate
-                ? callbackFilteredRowGroupIndices.filter(rowGroupIndex => canParquetRowGroupMatch(predicate, initialization.metadata.rowGroups[rowGroupIndex]))
-                : callbackFilteredRowGroupIndices;
-            const rowGroupsPrunedByStatistics = callbackFilteredRowGroupIndices.length - rowGroupIndices.length;
+            const physicalPlan = await this.planRowGroups(initialization, readOptions, readContext.abortController.signal);
+            const { predicate, rowGroupIndices } = physicalPlan;
             this.recordTelemetry('row-group-prune', {
-                rowGroupsRequested: candidateRowGroupIndices.length,
-                rowGroupsPruned: candidateRowGroupIndices.length - rowGroupIndices.length,
-                rowGroupsPrunedByStatistics
+                rowGroupsRequested: physicalPlan.requested,
+                rowGroupsPruned: physicalPlan.requested - rowGroupIndices.length,
+                rowGroupsPrunedByStatistics: physicalPlan.prunedByStatistics,
+                rowGroupsPrunedByBloomFilter: physicalPlan.prunedByBloomFilter,
+                bloomFiltersRead: physicalPlan.bloomFiltersRead,
+                bloomFilterBytesRead: physicalPlan.bloomFilterBytesRead
             }, {});
             const columns = normalizeColumns(readOptions.columns, initialization.schema);
             const predicateColumns = predicate ? getParquetPredicateColumns(predicate) : [];
@@ -109,9 +195,11 @@ export class ParquetSource extends DataSource {
             const projectedSchema = projectSchema(initialization.schema, columns);
             const projectedColumnNames = columns.length ? new Set(columns) : undefined;
             const workerOptions = this.getWorkerOptions(concurrency, readContext.abortController.signal);
-            const decodeOnWorker = canDecodeParquetSourceOnWorker(workerOptions);
+            const decodeOnWorker = canDecodeParquetSourceOnWorker(workerOptions) &&
+                (!initialization.reader.encrypted || Boolean(this.options.parquet?.keyRetriever));
             const scheduledReads = new Map();
             let nextPositionToSchedule = 0;
+            let remainingRows = readOptions.limit ?? Number.POSITIVE_INFINITY;
             const scheduleReads = () => {
                 while (nextPositionToSchedule < rowGroupIndices.length &&
                     scheduledReads.size < concurrency) {
@@ -139,9 +227,15 @@ export class ParquetSource extends DataSource {
                         const arrowTable = hydrateArrowTable(workerBatch.arrowTable);
                         const deserializationDurationMs = getCurrentTime() - deserializationStartTime;
                         this.recordTelemetry('arrow-conversion', { arrowConversionDurationMs: deserializationDurationMs }, { rowGroupIndex, durationMs: deserializationDurationMs });
-                        const batch = createParquetBatchFromArrow(initialization.metadata, projectedSchema, rowGroupIndex, workerBatch.rowGroupRowOffset, arrowTable, workerBatch.rowCount, workerBatch.rowGroupRowIndices);
-                        this.recordTelemetry('batch', { batchesEmitted: 1, rowsEmitted: workerBatch.rowCount }, { rowGroupIndex, rowCount: workerBatch.rowCount });
+                        const batch = sliceParquetBatch(createParquetBatchFromArrow(initialization.metadata, projectedSchema, rowGroupIndex, workerBatch.rowGroupRowOffset, arrowTable, workerBatch.rowCount, workerBatch.rowGroupRowIndices), Math.min(workerBatch.rowCount, remainingRows));
+                        this.recordTelemetry('batch', { batchesEmitted: 1, rowsEmitted: batch.length }, { rowGroupIndex, rowCount: batch.length });
                         yield batch;
+                        remainingRows -= batch.length;
+                        if (remainingRows === 0) {
+                            limitReached = true;
+                            completed = true;
+                            return;
+                        }
                     }
                     continue;
                 }
@@ -149,7 +243,7 @@ export class ParquetSource extends DataSource {
                 const outputBatchSize = batchSize || Math.max(outputRowCount, 1);
                 for (let outputRowOffset = 0; outputRowOffset < outputRowCount; outputRowOffset += outputBatchSize) {
                     throwIfAborted(readContext.abortController.signal);
-                    const batchRowCount = Math.min(outputBatchSize, outputRowCount - outputRowOffset);
+                    const batchRowCount = Math.min(outputBatchSize, outputRowCount - outputRowOffset, remainingRows);
                     const batchRowIndices = rowIndices
                         ? rowIndices.slice(outputRowOffset, outputRowOffset + batchRowCount)
                         : undefined;
@@ -161,6 +255,12 @@ export class ParquetSource extends DataSource {
                     this.recordTelemetry('arrow-conversion', { arrowConversionDurationMs: conversionDurationMs }, { rowGroupIndex, durationMs: conversionDurationMs });
                     this.recordTelemetry('batch', { batchesEmitted: 1, rowsEmitted: batchRowCount }, { rowGroupIndex, rowCount: batchRowCount });
                     yield batch;
+                    remainingRows -= batch.length;
+                    if (remainingRows === 0) {
+                        limitReached = true;
+                        completed = true;
+                        return;
+                    }
                 }
             }
             completed = true;
@@ -179,12 +279,125 @@ export class ParquetSource extends DataSource {
             if (!completed && readError === undefined) {
                 this.recordTelemetry('cancel', { cancellationCount: 1 }, {});
             }
+            const cancelled = readContext.abortController.signal.aborted || readOptions.signal?.aborted;
             readContext.abortController.abort();
             readContext.removeSignalListener();
             this.activeReadControllers.delete(readContext.abortController);
             await Promise.allSettled([...inFlightReads]);
+            const status = readError
+                ? cancelled
+                    ? 'cancelled'
+                    : 'failed'
+                : limitReached || !completed
+                    ? 'early-terminated'
+                    : 'completed';
+            emitScanExecutionTelemetry(options.onTelemetry, createParquetScanExecutionTelemetry(telemetryBefore, this.getTelemetry(), startedAt, status, limitReached
+                ? 'limit'
+                : !completed && readError === undefined
+                    ? 'consumer-return'
+                    : undefined, readError));
         }
     }
+    /** Produces the physical row-group plan shared by explain and execution. */
+    async planRowGroups(initialization, readOptions, signal) {
+        throwIfAborted(signal);
+        const candidateRowGroupIndices = normalizeRowGroupIndices(readOptions.rowGroups, initialization.fileMetadata.row_groups.length);
+        const callbackFilteredRowGroupIndices = readOptions.rowGroupFilter
+            ? candidateRowGroupIndices.filter(rowGroupIndex => readOptions.rowGroupFilter(initialization.metadata.rowGroups[rowGroupIndex]))
+            : candidateRowGroupIndices;
+        const spatiallyFilteredRowGroupIndices = readOptions.bbox
+            ? callbackFilteredRowGroupIndices.filter(rowGroupIndex => canGeoParquetRowGroupMatch(initialization.metadata, initialization.metadata.rowGroups[rowGroupIndex], readOptions.bbox, readOptions.geometryColumn))
+            : callbackFilteredRowGroupIndices;
+        const spatialPredicate = readOptions.bbox
+            ? createGeoParquetBoundingBoxPredicate(initialization.metadata, readOptions.bbox, readOptions.geometryColumn)
+            : undefined;
+        const predicate = combineParquetPredicates(readOptions.predicate, spatialPredicate);
+        if (predicate) {
+            validateParquetPredicate(predicate, new Set(initialization.schema.fields.map(field => field.name)));
+        }
+        const statisticsRowGroupIndices = predicate
+            ? spatiallyFilteredRowGroupIndices.filter(rowGroupIndex => canParquetRowGroupMatch(predicate, initialization.metadata.rowGroups[rowGroupIndex]))
+            : spatiallyFilteredRowGroupIndices;
+        const bloomFilterResult = predicate
+            ? await filterParquetRowGroupsWithBloomFilters(initialization, statisticsRowGroupIndices, predicate, signal)
+            : { rowGroupIndices: statisticsRowGroupIndices, filtersRead: 0, bytesRead: 0 };
+        return {
+            predicate,
+            rowGroupIndices: bloomFilterResult.rowGroupIndices,
+            requested: candidateRowGroupIndices.length,
+            prunedByCallback: candidateRowGroupIndices.length - callbackFilteredRowGroupIndices.length,
+            prunedBySpatial: callbackFilteredRowGroupIndices.length - spatiallyFilteredRowGroupIndices.length,
+            prunedByStatistics: spatiallyFilteredRowGroupIndices.length - statisticsRowGroupIndices.length,
+            prunedByBloomFilter: statisticsRowGroupIndices.length - bloomFilterResult.rowGroupIndices.length,
+            bloomFiltersRead: bloomFilterResult.filtersRead,
+            bloomFilterBytesRead: bloomFilterResult.bytesRead
+        };
+    }
+    /** Produces explainable page-index and byte-range plans for retained row groups. */
+    async planPages(initialization, readOptions, rowGroupPlan, signal) {
+        if (!rowGroupPlan.predicate)
+            return [];
+        const columns = normalizeColumns(readOptions.columns, initialization.schema);
+        const predicateColumns = getParquetPredicateColumns(rowGroupPlan.predicate);
+        const decodedColumns = columns.length === 0 ? [] : [...new Set([...columns, ...predicateColumns])];
+        const hasHiddenPredicateColumns = columns.length > 0 && predicateColumns.some(column => !columns.includes(column));
+        const phases = hasHiddenPredicateColumns
+            ? [
+                {
+                    phase: 'predicate',
+                    columns: predicateColumns.map(column => [column]),
+                    predicate: rowGroupPlan.predicate
+                },
+                {
+                    phase: 'projection',
+                    columns: columns.map(column => [column]),
+                    predicate: undefined
+                }
+            ]
+            : [
+                {
+                    phase: 'combined',
+                    columns: decodedColumns.map(column => [column]),
+                    predicate: rowGroupPlan.predicate
+                }
+            ];
+        const plans = [];
+        for (const rowGroupIndex of rowGroupPlan.rowGroupIndices) {
+            const rowGroup = initialization.fileMetadata.row_groups[rowGroupIndex];
+            let predicateProvedEmpty = false;
+            for (const phase of phases) {
+                throwIfAborted(signal);
+                if (phase.phase === 'projection' && predicateProvedEmpty) {
+                    continue;
+                }
+                await initialization.reader.resolveColumnMetadata(rowGroup, rowGroupIndex, phase.columns);
+                const pagePlan = phase.predicate
+                    ? await createParquetPagePruningPlan(initialization.file, rowGroup, initialization.parquetSchema, phase.columns, phase.predicate, signal, {
+                        rowGroupOrdinal: rowGroupIndex,
+                        int96AsTimestamp: Boolean(this.options.parquet?.int96AsTimestamp),
+                        decryptModule: (bytes, module, rowGroupOrdinal, columnOrdinal, columnChunk) => initialization.reader.decryptIndexModule(bytes, module, rowGroupOrdinal, columnOrdinal, columnChunk)
+                    })
+                    : undefined;
+                plans.push(pagePlan
+                    ? Object.freeze({
+                        rowGroupIndex,
+                        phase: phase.phase,
+                        columns: Object.freeze(phase.columns.map(column => Object.freeze([...column]))),
+                        rowRanges: Object.freeze(pagePlan.rowRanges.map(rowRange => Object.freeze({ ...rowRange }))),
+                        indexesRead: pagePlan.indexCount,
+                        totalPages: pagePlan.totalPageCount,
+                        selectedPages: pagePlan.selectedPageCount,
+                        rowsPruned: pagePlan.prunedRowCount,
+                        ranges: Object.freeze(getParquetPageReadRanges(rowGroup, phase.columns, pagePlan, initialization.parquetSchema).map(range => Object.freeze({ ...range })))
+                    })
+                    : createFullColumnScanPlan(rowGroup, rowGroupIndex, phase.phase, phase.columns));
+                if (phase.phase === 'predicate' && pagePlan?.rowRanges.length === 0) {
+                    predicateProvedEmpty = true;
+                }
+            }
+        }
+        return plans;
+    }
     /** Closes the underlying readable file and aborts active remote requests. */
     async close() {
         if (this.closed) {
@@ -196,6 +409,7 @@ export class ParquetSource extends DataSource {
         }
         await this.readableFile?.close();
         const initialization = await this.initializationPromise?.catch(() => null);
+        initialization?.reader.close();
         if (initialization?.file !== this.readableFile) {
             await initialization?.file.close();
         }
@@ -230,11 +444,17 @@ export class ParquetSource extends DataSource {
         const rowGroups = options.rowGroups ?? this.options.parquet?.rowGroups;
         const columns = options.columns ?? this.options.parquet?.columns;
         const predicate = options.predicate ?? this.options.parquet?.predicate;
+        const bbox = options.bbox ?? this.options.parquet?.bbox;
+        const limit = options.limit ?? this.options.parquet?.limit;
+        validateTableQueryLimit(limit);
         return {
             rowGroups: rowGroups && [...rowGroups],
             columns: columns && [...columns],
             rowGroupFilter: options.rowGroupFilter ?? this.options.parquet?.rowGroupFilter,
             predicate: predicate ? copyParquetPredicate(predicate) : undefined,
+            limit,
+            bbox: bbox ? [...bbox] : undefined,
+            geometryColumn: options.geometryColumn ?? this.options.parquet?.geometryColumn,
             batchSize: options.batchSize ?? this.options.parquet?.batchSize,
             concurrency: options.concurrency ?? this.options.parquet?.concurrency,
             signal: options.signal
@@ -260,9 +480,19 @@ export class ParquetSource extends DataSource {
     }
     /** Fetches and decodes one selected row group using the cached footer and schema. */
     async readRowGroup(initialization, rowGroupIndex, columnList, projectedSchema, batchSize, predicate, projectedColumnNames, workerOptions, signal) {
+        if (predicate &&
+            projectedColumnNames &&
+            getParquetPredicateColumns(predicate).some(column => !projectedColumnNames.has(column))) {
+            return await this.readRowGroupWithLateMaterialization(initialization, rowGroupIndex, columnList, projectedSchema, batchSize, predicate, projectedColumnNames, workerOptions, signal);
+        }
         const rowGroup = initialization.fileMetadata.row_groups[rowGroupIndex];
+        await initialization.reader.resolveColumnMetadata(rowGroup, rowGroupIndex, columnList);
         const pagePlan = predicate
-            ? await createParquetPagePruningPlan(initialization.file, rowGroup, initialization.parquetSchema, columnList, predicate, signal)
+            ? await createParquetPagePruningPlan(initialization.file, rowGroup, initialization.parquetSchema, columnList, predicate, signal, {
+                rowGroupOrdinal: rowGroupIndex,
+                int96AsTimestamp: Boolean(this.options.parquet?.int96AsTimestamp),
+                decryptModule: (bytes, module, rowGroupOrdinal, columnOrdinal, columnChunk) => initialization.reader.decryptIndexModule(bytes, module, rowGroupOrdinal, columnOrdinal, columnChunk)
+            })
             : undefined;
         if (pagePlan) {
             this.recordPagePruningTelemetry(rowGroupIndex, pagePlan);
@@ -275,9 +505,9 @@ export class ParquetSource extends DataSource {
         }
         const decodeStartTime = getCurrentTime();
         const decodedRowGroups = pagePlan
-            ? await Promise.all(pagePlan.rowRanges.map(rowRange => initialization.reader.readRowGroupRange(initialization.parquetSchema, rowGroup, columnList, rowRange, pagePlan.pageLocations, signal)))
+            ? await Promise.all(pagePlan.rowRanges.map(rowRange => initialization.reader.readRowGroupRange(initialization.parquetSchema, rowGroup, columnList, rowRange, pagePlan.pageLocations, signal, rowGroupIndex)))
             : [
-                await initialization.reader.readRowGroup(initialization.parquetSchema, rowGroup, columnList, signal)
+                await initialization.reader.readRowGroup(initialization.parquetSchema, rowGroup, columnList, signal, rowGroupIndex)
             ];
         throwIfAborted(signal);
         const columns = concatenateMaterializedColumns(decodedRowGroups.map(decodedRowGroup => initialization.parquetSchema.materializeColumns(decodedRowGroup)));
@@ -315,12 +545,25 @@ export class ParquetSource extends DataSource {
             return Boolean(path && (columnList.length === 0 || fieldIndexOf(columnList, path) >= 0));
         });
         const rangeDescriptors = pagePlan
-            ? getParquetPageReadRanges(rowGroup, columnList, pagePlan)
+            ? getParquetPageReadRanges(rowGroup, columnList, pagePlan, initialization.parquetSchema)
             : selectedColumnChunks.map(getColumnChunkRange);
         const ranges = await Promise.all(rangeDescriptors.map(async ({ offset, length }) => {
             return { offset, data: await initialization.file.read(offset, length, signal) };
         }));
         throwIfAborted(signal);
+        const workerColumnChunks = await Promise.all(selectedColumnChunks.map(async (columnChunk) => {
+            const descriptor = createParquetSourceWorkerColumnChunk(columnChunk);
+            descriptor.columnOrdinal = rowGroup.columns.indexOf(columnChunk);
+            if (columnChunk.crypto_metadata) {
+                const columnOrdinal = rowGroup.columns.indexOf(columnChunk);
+                const key = await initialization.reader.getColumnKeyForWorker(columnChunk, rowGroupIndex, columnOrdinal);
+                descriptor.encrypted = true;
+                descriptor.keyMetadata = key.keyMetadata?.slice().buffer;
+                descriptor.keyMaterial = key.keyMaterial.slice().buffer;
+            }
+            return descriptor;
+        }));
+        const encryptionContext = initialization.reader.getEncryptionContextForWorker();
         let workerResult;
         try {
             workerResult = await decodeParquetSourceRowGroupOnWorker({
@@ -329,12 +572,21 @@ export class ParquetSource extends DataSource {
                 uncompressedByteLength: Number(rowGroup.total_byte_size),
                 schemaDefinition: initialization.parquetSchema.schema,
                 projectedSchema,
-                columnChunks: selectedColumnChunks.map(createParquetSourceWorkerColumnChunk),
+                columnChunks: workerColumnChunks,
                 ranges,
                 batchSize: batchSize || Math.max(Number(rowGroup.num_rows), 1),
                 predicate: predicate ? copyParquetPredicate(predicate) : undefined,
                 pagePlan,
-                preserveBinary: Boolean(this.options.parquet?.preserveBinary)
+                encryption: encryptionContext
+                    ? {
+                        algorithm: encryptionContext.algorithm,
+                        aadPrefix: encryptionContext.aadPrefix?.slice().buffer,
+                        fileUnique: encryptionContext.fileUnique.slice().buffer
+                    }
+                    : undefined,
+                preserveBinary: Boolean(this.options.parquet?.preserveBinary),
+                int96AsTimestamp: Boolean(this.options.parquet?.int96AsTimestamp),
+                verifyPageChecksums: Boolean(this.options.parquet?.verifyPageChecksums)
             }, workerOptions);
         }
         catch (error) {
@@ -352,6 +604,35 @@ export class ParquetSource extends DataSource {
         }
         return { rowGroupIndex, rowCount: workerResult.rowCount, workerResult };
     }
+    /** Decodes predicate columns first, avoiding projected-column decoding for empty matches. */
+    async readRowGroupWithLateMaterialization(initialization, rowGroupIndex, columnList, projectedSchema, batchSize, predicate, projectedColumnNames, workerOptions, signal) {
+        const predicateColumns = getParquetPredicateColumns(predicate);
+        const predicateColumnList = predicateColumns.map(column => [column]);
+        const filterResult = await this.readRowGroup(initialization, rowGroupIndex, predicateColumnList, projectSchema(initialization.schema, predicateColumns), batchSize, predicate, new Set(predicateColumns), workerOptions, signal);
+        const rowIndices = filterResult.rowIndices;
+        const selectedRowIndices = rowIndices || getWorkerRowIndices(filterResult.workerResult);
+        if (!selectedRowIndices || selectedRowIndices.length === 0) {
+            return { rowGroupIndex, rowCount: 0, columns: {}, rowIndices: [] };
+        }
+        const projectedColumnList = columnList.filter(columnPath => projectedColumnNames.has(columnPath[0]));
+        const projectedResult = await this.readRowGroup(initialization, rowGroupIndex, projectedColumnList, projectedSchema, batchSize, undefined, projectedColumnNames, workerOptions, signal);
+        if (workerOptions && projectedResult.workerResult) {
+            return {
+                rowGroupIndex,
+                rowCount: selectedRowIndices.length,
+                rowIndices: selectedRowIndices,
+                workerResult: gatherWorkerProjectedRows(projectedResult.workerResult, selectedRowIndices, batchSize)
+            };
+        }
+        return {
+            rowGroupIndex,
+            rowCount: selectedRowIndices.length,
+            rowIndices: selectedRowIndices,
+            columns: projectedResult.columns
+                ? gatherParquetColumns(projectedResult.columns, selectedRowIndices, projectedColumnNames)
+                : {}
+        };
+    }
     /** Records the rows and pages avoided by one conservative page-index plan. */
     recordPagePruningTelemetry(rowGroupIndex, plan) {
         this.recordTelemetry('page-index-prune', {
@@ -369,6 +650,11 @@ export class ParquetSource extends DataSource {
         try {
             const reader = new ParquetReader(file, {
                 preserveBinary: this.options.parquet?.preserveBinary,
+                int96AsTimestamp: this.options.parquet?.int96AsTimestamp,
+                verifyPageChecksums: this.options.parquet?.verifyPageChecksums,
+                verifyFooterSignature: this.options.parquet?.verifyFooterSignature,
+                keyRetriever: this.options.parquet?.keyRetriever,
+                aadPrefix: this.options.parquet?.aadPrefix,
                 signal
             });
             const fileMetadata = await reader.getFileMetadata();
@@ -421,6 +707,26 @@ export class ParquetSource extends DataSource {
         }
     }
 }
+function getParquetColumnRoles(columnNames) {
+    const roles = {};
+    for (const columnName of columnNames) {
+        const normalizedName = columnName.toLowerCase();
+        if (normalizedName === 'geometry' || normalizedName === 'geom' || normalizedName === 'shape') {
+            roles[columnName] = 'geometry';
+        }
+        else if (normalizedName === 'longitude' ||
+            normalizedName === 'lon' ||
+            normalizedName === 'x') {
+            roles[columnName] = 'longitude';
+        }
+        else if (normalizedName === 'latitude' ||
+            normalizedName === 'lat' ||
+            normalizedName === 'y') {
+            roles[columnName] = 'latitude';
+        }
+    }
+    return roles;
+}
 /** Normalizes a decoded Parquet footer into source metadata. */
 function createParquetSourceMetadata(data, url, file, fileMetadata, schema, parquetSchema, objectVersion) {
     let rowOffset = 0;
@@ -428,7 +734,7 @@ function createParquetSourceMetadata(data, url, file, fileMetadata, schema, parq
         const columns = rowGroup.columns
             .filter(columnChunk => Boolean(columnChunk.meta_data))
             .map(columnChunk => createColumnChunkMetadata(columnChunk, parquetSchema));
-        const normalizedRowGroup = createRowGroupMetadata(index, rowOffset, Number(rowGroup.num_rows), Number(rowGroup.total_byte_size), columns);
+        const normalizedRowGroup = createRowGroupMetadata(index, rowOffset, Number(rowGroup.num_rows), Number(rowGroup.total_byte_size), columns, rowGroup.sorting_columns);
         rowOffset += normalizedRowGroup.rowCount;
         return normalizedRowGroup;
     });
@@ -455,6 +761,8 @@ function createColumnChunkMetadata(columnChunk, parquetSchema) {
         ? undefined
         : Number(columnMetadata.dictionary_page_offset);
     const statistics = createColumnChunkStatistics(columnMetadata.statistics, parquetSchema.findField(columnMetadata.path_in_schema));
+    const geospatialStatistics = createGeospatialStatistics(columnMetadata.geospatial_statistics);
+    const sizeStatistics = createSizeStatistics(columnMetadata.size_statistics);
     const compressedByteLength = Number(columnMetadata.total_compressed_size);
     const uncompressedByteLength = Number(columnMetadata.total_uncompressed_size);
     return Object.freeze({
@@ -478,7 +786,75 @@ function createColumnChunkMetadata(columnChunk, parquetSchema) {
             ? undefined
             : Number(columnChunk.offset_index_offset),
         offsetIndexByteLength: columnChunk.offset_index_length,
-        statistics
+        bloomFilterOffset: columnMetadata.bloom_filter_offset === undefined
+            ? undefined
+            : Number(columnMetadata.bloom_filter_offset),
+        bloomFilterByteLength: columnMetadata.bloom_filter_length,
+        statistics,
+        sizeStatistics,
+        geospatialStatistics
+    });
+}
+/** Normalizes optional Parquet size statistics without inventing missing counts. */
+function createSizeStatistics(statistics) {
+    if (!statistics)
+        return undefined;
+    const result = {
+        unencodedByteArrayDataBytes: statistics.unencoded_byte_array_data_bytes === undefined
+            ? undefined
+            : toSafeNumber(statistics.unencoded_byte_array_data_bytes),
+        repetitionLevelHistogram: normalizeSizeStatisticsList(statistics.repetition_level_histogram),
+        definitionLevelHistogram: normalizeSizeStatisticsList(statistics.definition_level_histogram)
+    };
+    if (result.unencodedByteArrayDataBytes === undefined &&
+        result.repetitionLevelHistogram === undefined &&
+        result.definitionLevelHistogram === undefined) {
+        return undefined;
+    }
+    return Object.freeze({
+        ...result,
+        repetitionLevelHistogram: result.repetitionLevelHistogram
+            ? Object.freeze(result.repetitionLevelHistogram)
+            : undefined,
+        definitionLevelHistogram: result.definitionLevelHistogram
+            ? Object.freeze(result.definitionLevelHistogram)
+            : undefined
+    });
+}
+function toSafeNumber(value) {
+    const numberValue = typeof value === 'number' ? value : value.toNumber?.();
+    return numberValue !== undefined && Number.isSafeInteger(numberValue) && numberValue >= 0
+        ? numberValue
+        : undefined;
+}
+/** Converts a Thrift int64 histogram only when every entry is a safe non-negative integer. */
+function normalizeSizeStatisticsList(values) {
+    if (!values)
+        return undefined;
+    const normalized = values.map(toSafeNumber);
+    return normalized.every((value) => value !== undefined) ? normalized : undefined;
+}
+/** Normalizes native Parquet geospatial statistics from the decoded footer. */
+function createGeospatialStatistics(statistics) {
+    if (!statistics)
+        return undefined;
+    const bbox = statistics.bbox
+        ? Object.freeze({
+            xmin: statistics.bbox.xmin,
+            xmax: statistics.bbox.xmax,
+            ymin: statistics.bbox.ymin,
+            ymax: statistics.bbox.ymax,
+            zmin: statistics.bbox.zmin,
+            zmax: statistics.bbox.zmax,
+            mmin: statistics.bbox.mmin,
+            mmax: statistics.bbox.mmax
+        })
+        : undefined;
+    return Object.freeze({
+        bbox,
+        geometryTypes: statistics.geospatial_types
+            ? Object.freeze([...statistics.geospatial_types])
+            : undefined
     });
 }
 /** Decodes optional footer statistics into the column's logical value representation. */
@@ -547,7 +923,7 @@ function decodeStatisticsValueSafely(bytes, field) {
     }
 }
 /** Normalizes one decoded row group and derives its compressed byte length. */
-function createRowGroupMetadata(index, rowOffset, rowCount, uncompressedByteLength, columns) {
+function createRowGroupMetadata(index, rowOffset, rowCount, uncompressedByteLength, columns, sortingColumns) {
     const compressedByteLength = columns.reduce((sum, column) => sum + column.compressedByteLength, 0);
     return Object.freeze({
         index,
@@ -557,9 +933,37 @@ function createRowGroupMetadata(index, rowOffset, rowCount, uncompressedByteLeng
         uncompressedSize: uncompressedByteLength,
         compressedByteLength,
         compressedSize: compressedByteLength,
-        columns: Object.freeze(columns)
+        columns: Object.freeze(columns),
+        sortingColumns: Object.freeze((sortingColumns || []).map(sortingColumn => Object.freeze({
+            columnIndex: sortingColumn.column_idx,
+            descending: sortingColumn.descending,
+            nullsFirst: sortingColumn.nulls_first
+        })))
+    });
+}
+/** Describes a phase that reads complete column chunks because no page index is available. */
+function createFullColumnScanPlan(rowGroup, rowGroupIndex, phase, columns) {
+    const selectedColumnChunks = rowGroup.columns.filter(columnChunk => {
+        const path = columnChunk.meta_data?.path_in_schema;
+        return Boolean(path && (columns.length === 0 || columns.some(column => pathMatchesSelection(column, path))));
+    });
+    const rowCount = Number(rowGroup.num_rows);
+    return Object.freeze({
+        rowGroupIndex,
+        phase,
+        columns: Object.freeze(columns.map(column => Object.freeze([...column]))),
+        rowRanges: Object.freeze([Object.freeze({ start: 0, end: rowCount })]),
+        indexesRead: 0,
+        totalPages: 0,
+        selectedPages: 0,
+        rowsPruned: 0,
+        ranges: Object.freeze(selectedColumnChunks.map(columnChunk => Object.freeze(getColumnChunkRange(columnChunk))))
     });
 }
+/** Returns whether a requested path selects a footer leaf or one of its nested children. */
+function pathMatchesSelection(selection, path) {
+    return selection.length <= path.length && selection.every((part, index) => part === path[index]);
+}
 /** Returns the complete contiguous byte range occupied by one Parquet column chunk. */
 function getColumnChunkRange(columnChunk) {
     if (columnChunk.file_path) {
@@ -623,11 +1027,8 @@ function createParquetBatchFromArrow(metadata, schema, rowGroupIndex, rowGroupRo
             : undefined
     });
     return {
-        batchType: 'data',
-        shape: 'arrow-table',
+        ...makeTableScanBatch({ shape: 'arrow-table', schema, data }),
         schemaType: 'explicit',
-        schema,
-        data,
         length: rowCount,
         metadata: provenance,
         ...provenance
@@ -731,6 +1132,9 @@ function createParquetTelemetry() {
         rowGroupsRequested: 0,
         rowGroupsPruned: 0,
         rowGroupsPrunedByStatistics: 0,
+        rowGroupsPrunedByBloomFilter: 0,
+        bloomFiltersRead: 0,
+        bloomFilterBytesRead: 0,
         rowGroupsPrunedByPageIndex: 0,
         pageIndexesRead: 0,
         pagesRead: 0,
@@ -745,6 +1149,34 @@ function createParquetTelemetry() {
         failedReadCount: 0
     };
 }
+/** Converts cumulative Parquet counters into one portable read-scoped execution snapshot. */
+function createParquetScanExecutionTelemetry(before, after, startedAt, status, earlyTerminationReason, error) {
+    const delta = Object.fromEntries(Object.keys(after).map(key => {
+        const telemetryKey = key;
+        return [key, after[telemetryKey] - before[telemetryKey]];
+    }));
+    const rowsRead = Math.max(delta.predicateRowsTested, delta.rowsEmitted);
+    return Object.freeze({
+        status,
+        sourcesPlanned: 1,
+        sourcesRead: status === 'early-terminated' && earlyTerminationReason === 'limit' && rowsRead === 0 ? 0 : 1,
+        batchesRead: delta.batchesEmitted,
+        batchesDecoded: delta.rowGroupsDecoded,
+        rowsRead,
+        rowsTested: delta.predicateRowsTested || undefined,
+        rowsRetained: delta.predicateRowsMatched || undefined,
+        rowsReturned: delta.rowsEmitted,
+        bytesRead: delta.downloadedBytes,
+        bytesFetched: delta.downloadedBytes,
+        filesOpened: rowsRead || delta.downloadedBytes ? 1 : 0,
+        tasksOpened: delta.rowGroupsDecoded,
+        rowsPruned: delta.rowsPrunedByPageIndex,
+        durationMilliseconds: Date.now() - startedAt,
+        earlyTerminationReason,
+        details: Object.freeze({ ...delta }),
+        ...(error === undefined ? {} : { error })
+    });
+}
 /** Returns a monotonic timestamp when available and falls back to wall-clock time. */
 function getCurrentTime() {
     return globalThis.performance?.now() ?? Date.now();
@@ -805,6 +1237,121 @@ function getSourceName(data, url) {
             : 'parquet';
     return sourceName.replace(/\.parquet$/i, '') || 'parquet';
 }
+/** Reconstructs exact row-group indexes from worker predicate batches. */
+function getWorkerRowIndices(workerResult) {
+    if (!workerResult)
+        return [];
+    return workerResult.batches.flatMap(batch => batch.rowGroupRowIndices
+        ? [...batch.rowGroupRowIndices]
+        : Array.from({ length: batch.rowCount }, (_, index) => batch.rowGroupRowOffset + index));
+}
+/** Gathers projected worker Arrow batches through the shared indexed Arrow table utility. */
+function gatherWorkerProjectedRows(workerResult, rowIndices, batchSize) {
+    const tables = workerResult.batches.map(batch => hydrateArrowTable(batch.arrowTable));
+    if (tables.length === 0) {
+        return { ...workerResult, rowCount: 0, batches: [] };
+    }
+    const table = tables.length === 1 ? tables[0] : tables[0].concat(...tables.slice(1));
+    const indexedTable = new IndexedArrowTable(table, rowIndices);
+    const outputBatchSize = batchSize || Math.max(rowIndices.length, 1);
+    const batches = [];
+    for (let offset = 0; offset < rowIndices.length; offset += outputBatchSize) {
+        const end = Math.min(offset + outputBatchSize, rowIndices.length);
+        const batchIndexes = rowIndices.slice(offset, end);
+        const batchTable = indexedTable.slice(offset, end).materializeArrowTable();
+        batches.push({
+            rowGroupRowOffset: batchIndexes[0],
+            rowCount: batchIndexes.length,
+            rowGroupRowIndices: [...batchIndexes],
+            arrowTable: dehydrateArrowTable(batchTable)
+        });
+    }
+    return {
+        sourceRowCount: workerResult.sourceRowCount,
+        rowCount: rowIndices.length,
+        batches,
+        decodeDurationMs: workerResult.decodeDurationMs,
+        arrowConversionDurationMs: workerResult.arrowConversionDurationMs
+    };
+}
+/** Applies only proven-negative Bloom-filter checks to candidate row groups. */
+async function filterParquetRowGroupsWithBloomFilters(initialization, rowGroupIndices, predicate, signal) {
+    const rowGroups = [];
+    let filtersRead = 0;
+    let bytesRead = 0;
+    for (const rowGroupIndex of rowGroupIndices) {
+        const rowGroup = initialization.metadata.rowGroups[rowGroupIndex];
+        const probes = getParquetBloomFilterProbes(predicate, rowGroup);
+        let keepRowGroup = true;
+        for (const probe of probes) {
+            if (probe.column.bloomFilterOffset === undefined ||
+                probe.column.bloomFilterByteLength === undefined) {
+                continue;
+            }
+            let field;
+            try {
+                field = initialization.parquetSchema.findField([...probe.column.path]);
+            }
+            catch {
+                throwIfAborted(signal);
+                continue;
+            }
+            if (!field.primitiveType)
+                continue;
+            let filter;
+            try {
+                const rawColumnOrdinal = initialization.fileMetadata.row_groups[rowGroupIndex].columns.findIndex(columnChunk => JSON.stringify(columnChunk.meta_data?.path_in_schema) ===
+                    JSON.stringify(probe.column.path));
+                const rawColumnChunk = rawColumnOrdinal >= 0
+                    ? initialization.fileMetadata.row_groups[rowGroupIndex].columns[rawColumnOrdinal]
+                    : undefined;
+                const data = await initialization.file.read(probe.column.bloomFilterOffset, probe.column.bloomFilterByteLength, signal);
+                const decodedBloomFilter = rawColumnChunk
+                    ? await initialization.reader.decryptBloomFilter(toUint8Array(data), rowGroupIndex, rawColumnOrdinal, rawColumnChunk)
+                    : toUint8Array(data);
+                filter = decodeParquetSplitBlockBloomFilter(decodedBloomFilter);
+                filtersRead++;
+                bytesRead += data.byteLength;
+            }
+            catch {
+                throwIfAborted(signal);
+                continue;
+            }
+            if (filter.algorithm !== 'BLOCK' ||
+                filter.hash !== 'XXHASH' ||
+                filter.compression !== 'UNCOMPRESSED') {
+                continue;
+            }
+            const mayContainValue = probe.values.some(value => {
+                try {
+                    if (value instanceof Date)
+                        return true;
+                    const physicalValue = getParquetBloomFilterPhysicalValue(value, field);
+                    const encoded = encodeParquetBloomFilterValue(physicalValue, field.primitiveType, field.typeLength);
+                    return checkParquetSplitBlockBloomFilter(filter.bitset, hashParquetBloomFilterValue(encoded));
+                }
+                catch {
+                    throwIfAborted(signal);
+                    return true;
+                }
+            });
+            if (!mayContainValue) {
+                keepRowGroup = false;
+                break;
+            }
+        }
+        if (keepRowGroup)
+            rowGroups.push(rowGroupIndex);
+    }
+    return { rowGroupIndices: rowGroups, filtersRead, bytesRead };
+}
+/** Converts logical predicate values into the physical values stored in Bloom filters. */
+function getParquetBloomFilterPhysicalValue(value, field) {
+    if (field.originalType?.startsWith('DECIMAL_')) {
+        return toPrimitive(field.originalType, value, field);
+    }
+    return value;
+}
 /** Returns true when at least one HTTP object validator was captured. */
 function hasObjectVersion(version) {
     return Boolean(version?.etag || version?.lastModified);