@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4

package/src/parquet-dataset-source.ts

@@ -2,24 +2,39 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {CoreAPI} from '@loaders.gl/loader-utils';
+import {
+  executeScanTasks,
+  emitScanExecutionTelemetry,
+  explainTableQuery,
+  validateTableQueryLimit,
+  type CoreAPI,
+  type ScanFragment,
+  type ScanFragmentProvider,
+  type ScanTask,
+  type ScanExecutionTelemetry
+} from '@loaders.gl/loader-utils';
 import type {Schema} from '@loaders.gl/schema';
 
-import {SingleBatchQueue} from './lib/sources/single-batch-queue';
 import {ParquetSource} from './parquet-source-loader';
+import {sliceParquetBatch} from './lib/slice-parquet-batch';
+import {PARQUET_TABLE_QUERY_CAPABILITIES} from './parquet-source-capabilities';
 import type {
   ParquetDatasetBatch,
   ParquetDatasetBatchProvenance,
   ParquetDatasetBoundingBox,
   ParquetDatasetFile,
+  ParquetDatasetFileScanPlan,
   ParquetDatasetFileCollection,
   ParquetDatasetFileQuery,
   ParquetDatasetFiles,
   ParquetDatasetPartitionValue,
   ParquetDatasetReadOptions,
+  ParquetDatasetExplain,
   ParquetDatasetSourceOptions,
   ParquetDatasetTelemetry,
   ParquetSourceBatch,
+  ParquetPredicate,
+  ParquetSourceExplain,
   ParquetTelemetry
 } from './parquet-source-types';
 
@@ -27,8 +42,10 @@ export type {
   ParquetDatasetBatch,
   ParquetDatasetBatchProvenance,
   ParquetDatasetBoundingBox,
+  ParquetDatasetExplain,
   ParquetDatasetFile,
   ParquetDatasetFileCollection,
+  ParquetDatasetFileScanPlan,
   ParquetDatasetFileProvider,
   ParquetDatasetFileQuery,
   ParquetDatasetFiles,
@@ -47,11 +64,15 @@ type IndexedParquetDatasetFile = {
   index: number;
 };
 
-type ScheduledFileRead = {
-  /** Bounded batch queue for this file. */
-  queue: SingleBatchQueue<ParquetDatasetBatch>;
-  /** File task used for deterministic cleanup. */
-  task: Promise<void>;
+type ParquetDatasetDiscoverySummary = {
+  /** Descriptors returned by the provider. */
+  discovered: number;
+  /** Descriptors retained after local pruning. */
+  selected: number;
+  /** Descriptors rejected by bounding-box intersection. */
+  prunedByBoundingBox: number;
+  /** Descriptors rejected by partition constraints. */
+  prunedByPartitions: number;
 };
 
 /**
@@ -60,7 +81,9 @@ type ScheduledFileRead = {
  * The source accepts static descriptors or a lazy provider, allowing STAC and other catalogs to be
  * injected without coupling `@loaders.gl/parquet` to a catalog protocol.
  */
-export class ParquetDatasetSource {
+export class ParquetDatasetSource implements ScanFragmentProvider<ParquetPredicate> {
+  /** Common projection, predicate, limit, streaming, and cancellation capabilities. */
+  readonly tableQueryCapabilities = PARQUET_TABLE_QUERY_CAPABILITIES;
   /** Static descriptors or lazy catalog-backed file provider. */
   readonly files: ParquetDatasetFiles;
   /** Child-source and dataset orchestration options. */
@@ -121,6 +144,137 @@ export class ParquetDatasetSource {
     });
   }
 
+  /** Discovers catalog-independent fragments after applying descriptor-level pruning. */
+  async getScanFragments(
+    options: ParquetDatasetReadOptions = {}
+  ): Promise<readonly ScanFragment[]> {
+    this.assertOpen();
+    const fragments: ScanFragment[] = [];
+    const selectedFiles = this.getSelectedFiles(options);
+    for await (const indexedFile of selectedFiles) {
+      const file = indexedFile.file;
+      if (typeof file.data !== 'string' && !file.id) {
+        throw new Error(
+          'Parquet dataset Blob fragments require an explicit stable id for fragment discovery'
+        );
+      }
+      fragments.push(
+        Object.freeze({
+          id: getDatasetFileId(file, indexedFile.index),
+          uri: typeof file.data === 'string' ? file.data : undefined,
+          partitionValues: file.partitions,
+          metadata: file.metadata
+        })
+      );
+    }
+    return Object.freeze(fragments);
+  }
+
+  /** Plans descriptor discovery and every retained file's Parquet physical scan. */
+  async getScanPlan(options: ParquetDatasetReadOptions = {}): Promise<ParquetDatasetExplain> {
+    this.assertOpen();
+    validateTableQueryLimit(options.limit);
+    const readContext = createDatasetAbortContext(options.signal);
+    this.activeReadControllers.add(readContext.abortController);
+    const discovery = createParquetDatasetDiscoverySummary();
+    const filePlans: ParquetDatasetFileScanPlan[] = [];
+    const tasks = this.getPlanTasks(options, readContext.abortController.signal, discovery);
+    const fileConcurrency = normalizeFileConcurrency(
+      options.fileConcurrency ?? this.options.parquetDataset?.fileConcurrency
+    );
+    try {
+      for await (const filePlan of executeScanTasks(tasks, {
+        concurrency: fileConcurrency,
+        signal: readContext.abortController.signal
+      })) {
+        filePlans.push(filePlan);
+      }
+      const firstFilePlan = filePlans[0];
+      if (!firstFilePlan) {
+        throw new Error('ParquetDatasetSource query selected no files');
+      }
+      const effectivePredicate = firstFilePlan.parquet.plan.find(step => step.kind === 'filter');
+      const effectiveLimit = options.limit ?? this.options.parquet?.limit;
+      const explanation = explainTableQuery(
+        firstFilePlan.parquet.sourceColumns,
+        {
+          columns: firstFilePlan.parquet.outputColumns,
+          predicate:
+            effectivePredicate?.kind === 'filter' ? effectivePredicate.predicate : undefined,
+          limit: effectiveLimit
+        },
+        PARQUET_TABLE_QUERY_CAPABILITIES
+      );
+      return Object.freeze({
+        ...explanation,
+        source: 'parquet-dataset' as const,
+        files: Object.freeze({...discovery}),
+        filePlans: Object.freeze(filePlans)
+      });
+    } finally {
+      readContext.abortController.abort();
+      readContext.removeSignalListener();
+      this.activeReadControllers.delete(readContext.abortController);
+    }
+  }
+
+  /** Explains the common logical query and physical multi-file Parquet plan. */
+  async explain(options: ParquetDatasetReadOptions = {}): Promise<ParquetDatasetExplain> {
+    return await this.getScanPlan(options);
+  }
+
+  /** Executes a previously computed dataset plan while retaining each file's row-group choices. */
+  async *executeScanPlan(
+    plan: ParquetDatasetExplain,
+    options: ParquetDatasetReadOptions = {}
+  ): AsyncIterable<ParquetDatasetBatch> {
+    if (plan.source !== 'parquet-dataset') {
+      throw new Error('ParquetDatasetSource can only execute dataset scan plans');
+    }
+    validateTableQueryLimit(options.limit);
+    let remainingRows = options.limit ?? this.options.parquet?.limit ?? Number.POSITIVE_INFINITY;
+    if (remainingRows === 0) return;
+    const limitStep = plan.plan.find(step => step.kind === 'limit');
+    const readOptions: ParquetDatasetReadOptions = {
+      ...options,
+      columns: options.columns ?? plan.outputColumns,
+      predicate: options.predicate,
+      limit: options.limit ?? (limitStep?.kind === 'limit' ? limitStep.limit : undefined)
+    };
+    remainingRows = readOptions.limit ?? Number.POSITIVE_INFINITY;
+    const readContext = createDatasetAbortContext(readOptions.signal);
+    this.activeReadControllers.add(readContext.abortController);
+    const filePlans = new Map(
+      plan.filePlans.map(filePlan => [filePlan.fileIndex, filePlan.parquet])
+    );
+    const tasks = this.getReadTasks(readOptions, readContext.abortController.signal, filePlans);
+    const fileConcurrency = normalizeFileConcurrency(
+      readOptions.fileConcurrency ?? this.options.parquetDataset?.fileConcurrency
+    );
+    try {
+      for await (const batch of executeScanTasks(tasks, {
+        concurrency: fileConcurrency,
+        signal: readContext.abortController.signal
+      })) {
+        const outputBatch = sliceParquetBatch(batch, Math.min(batch.length, remainingRows));
+        this.telemetry.batchesEmitted++;
+        this.telemetry.rowsEmitted += outputBatch.length;
+        yield outputBatch;
+        remainingRows -= outputBatch.length;
+        if (remainingRows === 0) return;
+      }
+    } finally {
+      readContext.abortController.abort();
+      readContext.removeSignalListener();
+      this.activeReadControllers.delete(readContext.abortController);
+    }
+  }
+
+  /** Common scan-architecture alias for ordered multi-file reads. */
+  scan(options: ParquetDatasetReadOptions = {}): AsyncIterable<ParquetDatasetBatch> {
+    return this.read(options);
+  }
+
   /**
    * Reads selected files as deterministic, file-ordered Arrow batches.
    *
@@ -129,81 +283,79 @@ export class ParquetDatasetSource {
    */
   async *read(options: ParquetDatasetReadOptions = {}): AsyncIterable<ParquetDatasetBatch> {
     this.assertOpen();
+    validateTableQueryLimit(options.limit);
+    const startedAt = Date.now();
+    const telemetryBefore = this.getTelemetry();
+    let remainingRows = options.limit ?? this.options.parquet?.limit ?? Number.POSITIVE_INFINITY;
+    if (remainingRows === 0) {
+      emitScanExecutionTelemetry(
+        options.onTelemetry,
+        createParquetDatasetScanExecutionTelemetry(
+          telemetryBefore,
+          this.getTelemetry(),
+          startedAt,
+          'early-terminated',
+          'limit'
+        )
+      );
+      return;
+    }
     const readContext = createDatasetAbortContext(options.signal);
     this.activeReadControllers.add(readContext.abortController);
-    const selectedFiles = this.getSelectedFiles({
-      ...options,
-      signal: readContext.abortController.signal
-    });
-    const iterator = selectedFiles[Symbol.asyncIterator]();
-    const scheduledReads = new Map<number, ScheduledFileRead>();
-    let nextSchedulePosition = 0;
-    let nextYieldPosition = 0;
-    let providerComplete = false;
-    let discoveryError: unknown;
+    const tasks = this.getReadTasks(options, readContext.abortController.signal);
     const fileConcurrency = normalizeFileConcurrency(
       options.fileConcurrency ?? this.options.parquetDataset?.fileConcurrency
     );
-
-    const scheduleNext = async (): Promise<void> => {
-      if (providerComplete) {
-        return;
-      }
-      const next = await iterator.next();
-      if (next.done) {
-        providerComplete = true;
-        return;
-      }
-      const position = nextSchedulePosition++;
-      const queue = new SingleBatchQueue<ParquetDatasetBatch>();
-      const task = this.readFile(next.value, options, queue, readContext.abortController.signal);
-      scheduledReads.set(position, {queue, task});
-    };
-
-    const fillAvailableSlots = async (): Promise<void> => {
-      try {
-        while (scheduledReads.size < fileConcurrency && !providerComplete) {
-          await scheduleNext();
-        }
-      } catch (error) {
-        discoveryError = error;
-        providerComplete = true;
-      }
-    };
+    let completed = false;
+    let limitReached = false;
+    let readError: unknown;
 
     try {
-      await scheduleNext();
-      let discoveryPromise = fillAvailableSlots();
-
-      while (scheduledReads.size > 0) {
-        const scheduledRead = scheduledReads.get(nextYieldPosition);
-        if (!scheduledRead) {
-          throw new Error('ParquetDatasetSource internal file ordering error');
-        }
-        for await (const batch of scheduledRead.queue) {
-          throwIfAborted(readContext.abortController.signal);
-          this.telemetry.batchesEmitted++;
-          this.telemetry.rowsEmitted += batch.length;
-          yield batch;
-        }
-        await scheduledRead.task;
-        scheduledReads.delete(nextYieldPosition++);
-        await discoveryPromise;
-        if (discoveryError !== undefined) {
-          throw discoveryError;
+      for await (const batch of executeScanTasks(tasks, {
+        concurrency: fileConcurrency,
+        signal: readContext.abortController.signal
+      })) {
+        const outputBatch = sliceParquetBatch(batch, Math.min(batch.length, remainingRows));
+        this.telemetry.batchesEmitted++;
+        this.telemetry.rowsEmitted += outputBatch.length;
+        yield outputBatch;
+        remainingRows -= outputBatch.length;
+        if (remainingRows === 0) {
+          limitReached = true;
+          return;
         }
-        discoveryPromise = fillAvailableSlots();
-      }
-      await discoveryPromise;
-      if (discoveryError !== undefined) {
-        throw discoveryError;
       }
+      completed = true;
+    } catch (error) {
+      readError = error;
+      throw error;
     } finally {
+      const cancelled = readContext.abortController.signal.aborted || options.signal?.aborted;
       readContext.abortController.abort();
       readContext.removeSignalListener();
       this.activeReadControllers.delete(readContext.abortController);
-      await iterator.return?.();
-      await Promise.allSettled([...scheduledReads.values()].map(read => read.task));
+      const status: ScanExecutionTelemetry['status'] = readError
+        ? cancelled
+          ? 'cancelled'
+          : 'failed'
+        : limitReached || !completed
+          ? 'early-terminated'
+          : 'completed';
+      emitScanExecutionTelemetry(
+        options.onTelemetry,
+        createParquetDatasetScanExecutionTelemetry(
+          telemetryBefore,
+          this.getTelemetry(),
+          startedAt,
+          status,
+          limitReached
+            ? 'limit'
+            : !completed && readError === undefined
+              ? 'consumer-return'
+              : undefined,
+          readError
+        )
+      );
     }
   }
 
@@ -219,12 +371,75 @@ export class ParquetDatasetSource {
   }
 
   /** Reads one child source into its bounded file-order queue. */
-  private async readFile(
+  private async *getReadTasks(
+    options: ParquetDatasetReadOptions,
+    signal: AbortSignal,
+    filePlans?: ReadonlyMap<number, ParquetSourceExplain>
+  ): AsyncIterable<ScanTask<ParquetDatasetBatch>> {
+    const selectedFiles = this.getSelectedFiles({...options, signal});
+    for await (const indexedFile of selectedFiles) {
+      if (filePlans && !filePlans.has(indexedFile.index)) {
+        continue;
+      }
+      yield {
+        run: taskSignal =>
+          this.readFile(indexedFile, options, taskSignal, filePlans?.get(indexedFile.index))
+      };
+    }
+  }
+
+  /** Plans retained files through bounded, ordered common scan tasks. */
+  private async *getPlanTasks(
+    options: ParquetDatasetReadOptions,
+    signal: AbortSignal,
+    discovery: ParquetDatasetDiscoverySummary
+  ): AsyncIterable<ScanTask<ParquetDatasetFileScanPlan>> {
+    const selectedFiles = this.getSelectedFiles({...options, signal}, discovery);
+    for await (const indexedFile of selectedFiles) {
+      yield {run: taskSignal => this.planFile(indexedFile, options, taskSignal)};
+    }
+  }
+
+  /** Produces one child physical plan and closes its temporary source. */
+  private async *planFile(
     indexedFile: IndexedParquetDatasetFile,
     options: ParquetDatasetReadOptions,
-    queue: SingleBatchQueue<ParquetDatasetBatch>,
     signal: AbortSignal
-  ): Promise<void> {
+  ): AsyncIterable<ParquetDatasetFileScanPlan> {
+    const source = this.createSource(indexedFile.file);
+    this.telemetry.filesOpened++;
+    try {
+      if (this.options.parquetDataset?.validateSchema !== false) {
+        await this.validateFileSchema(source, indexedFile, signal);
+      }
+      const parquet = await source.getScanPlan({
+        columns: options.columns,
+        predicate: options.predicate,
+        bbox: options.bbox,
+        geometryColumn: options.geometryColumn,
+        rowGroupFilter: options.rowGroupFilter,
+        limit: undefined,
+        signal
+      });
+      yield Object.freeze({
+        fileIndex: indexedFile.index,
+        fileId: getDatasetFileId(indexedFile.file, indexedFile.index),
+        partitions: indexedFile.file.partitions,
+        parquet
+      });
+    } finally {
+      await source.close();
+      this.addParquetTelemetry(source.getTelemetry());
+    }
+  }
+
+  /** Reads one child source as an asynchronous batch iterable. */
+  private async *readFile(
+    indexedFile: IndexedParquetDatasetFile,
+    options: ParquetDatasetReadOptions,
+    signal: AbortSignal,
+    scanPlan?: ParquetSourceExplain
+  ): AsyncIterable<ParquetDatasetBatch> {
     const source = this.createSource(indexedFile.file);
     this.telemetry.filesOpened++;
     try {
@@ -235,15 +450,18 @@ export class ParquetDatasetSource {
         columns: options.columns,
         batchSize: options.batchSize,
         concurrency: options.concurrency,
+        rowGroups: scanPlan?.rowGroups.indices,
         rowGroupFilter: options.rowGroupFilter,
-        predicate: options.predicate,
+        predicate: options.predicate ?? (!options.bbox ? getPlanPredicate(scanPlan) : undefined),
+        bbox: options.bbox,
+        geometryColumn: options.geometryColumn,
+        limit: undefined,
         signal
       })) {
-        await queue.push(createDatasetBatch(batch, indexedFile), signal);
+        yield createDatasetBatch(batch, indexedFile);
       }
-      queue.finish();
     } catch (error) {
-      queue.fail(error);
+      throw error;
     } finally {
       await source.close();
       this.addParquetTelemetry(source.getTelemetry());
@@ -275,12 +493,16 @@ export class ParquetDatasetSource {
 
   /** Creates a child source while preserving caller fetch, worker, range, and decode options. */
   private createSource(file: ParquetDatasetFile): ParquetSource {
-    return new ParquetSource(file.data, this.options, this.coreApi);
+    const parquetOptions = this.options.parquet
+      ? {...this.options.parquet, limit: undefined}
+      : undefined;
+    return new ParquetSource(file.data, {...this.options, parquet: parquetOptions}, this.coreApi);
   }
 
   /** Lazily discovers descriptors and applies conservative local pruning. */
   private async *getSelectedFiles(
-    query: ParquetDatasetFileQuery
+    query: ParquetDatasetFileQuery,
+    discovery?: ParquetDatasetDiscoverySummary
   ): AsyncIterable<IndexedParquetDatasetFile> {
     throwIfAborted(query.signal);
     const collection = await getFileCollection(this.files, query);
@@ -289,17 +511,21 @@ export class ParquetDatasetSource {
       throwIfAborted(query.signal);
       validateDatasetFile(file, index);
       this.telemetry.filesDiscovered++;
+      if (discovery) discovery.discovered++;
       if (query.bbox && file.bbox && !doBoundingBoxesIntersect(query.bbox, file.bbox)) {
         this.telemetry.filesPrunedByBoundingBox++;
+        if (discovery) discovery.prunedByBoundingBox++;
         index++;
         continue;
       }
       if (query.partitions && !matchesPartitions(file.partitions, query.partitions)) {
         this.telemetry.filesPrunedByPartitions++;
+        if (discovery) discovery.prunedByPartitions++;
         index++;
         continue;
       }
       this.telemetry.filesSelected++;
+      if (discovery) discovery.selected++;
       yield {file, index: index++};
     }
   }
@@ -319,6 +545,11 @@ export class ParquetDatasetSource {
   }
 }
 
+/** Creates operation-local descriptor discovery counters. */
+function createParquetDatasetDiscoverySummary(): ParquetDatasetDiscoverySummary {
+  return {discovered: 0, selected: 0, prunedByBoundingBox: 0, prunedByPartitions: 0};
+}
+
 /** Normalizes static descriptors and lazy providers into one file collection. */
 async function getFileCollection(
   files: ParquetDatasetFiles,
@@ -353,6 +584,12 @@ function createDatasetBatch(
   return {...batch, metadata: provenance, ...datasetProvenance};
 }
 
+/** Returns the filter predicate recorded in one child source plan. */
+function getPlanPredicate(plan: ParquetSourceExplain | undefined): ParquetPredicate | undefined {
+  const filter = plan?.plan.find(step => step.kind === 'filter');
+  return filter?.kind === 'filter' ? filter.predicate : undefined;
+}
+
 /** Returns a stable descriptor identity for batch provenance and diagnostics. */
 function getDatasetFileId(file: ParquetDatasetFile, index: number): string {
   if (file.id) {
@@ -378,18 +615,43 @@ function doBoundingBoxesIntersect(
   left: ParquetDatasetBoundingBox,
   right: ParquetDatasetBoundingBox
 ): boolean {
-  const leftMaximumX = left.length === 6 ? left[3] : left[2];
-  const leftMaximumY = left.length === 6 ? left[4] : left[3];
-  const rightMaximumX = right.length === 6 ? right[3] : right[2];
-  const rightMaximumY = right.length === 6 ? right[4] : right[3];
+  const leftMaximumX = left.length === 8 ? left[4] : left.length === 6 ? left[3] : left[2];
+  const leftMaximumY = left.length === 8 ? left[5] : left.length === 6 ? left[4] : left[3];
+  const rightMaximumX = right.length === 8 ? right[4] : right.length === 6 ? right[3] : right[2];
+  const rightMaximumY = right.length === 8 ? right[5] : right.length === 6 ? right[4] : right[3];
   return (
-    left[0] <= rightMaximumX &&
-    leftMaximumX >= right[0] &&
+    doLongitudeIntervalsIntersect(left[0], leftMaximumX, right[0], rightMaximumX) &&
     left[1] <= rightMaximumY &&
     leftMaximumY >= right[1]
   );
 }
 
+/** Tests longitude intervals that may use RFC 7946 antimeridian wrapping. */
+function doLongitudeIntervalsIntersect(
+  firstWest: number,
+  firstEast: number,
+  secondWest: number,
+  secondEast: number
+): boolean {
+  const firstIntervals =
+    firstWest <= firstEast
+      ? [[firstWest, firstEast]]
+      : [
+          [firstWest, 180],
+          [-180, firstEast]
+        ];
+  const secondIntervals =
+    secondWest <= secondEast
+      ? [[secondWest, secondEast]]
+      : [
+          [secondWest, 180],
+          [-180, secondEast]
+        ];
+  return firstIntervals.some(first =>
+    secondIntervals.some(second => first[0] <= second[1] && first[1] >= second[0])
+  );
+}
+
 /** Tests exact partition constraints without rejecting files that omit a requested key. */
 function matchesPartitions(
   partitions: Readonly<Record<string, ParquetDatasetPartitionValue>> | undefined,
@@ -471,6 +733,9 @@ function createParquetDatasetTelemetry(): ParquetDatasetTelemetry {
       rowGroupsRequested: 0,
       rowGroupsPruned: 0,
       rowGroupsPrunedByStatistics: 0,
+      rowGroupsPrunedByBloomFilter: 0,
+      bloomFiltersRead: 0,
+      bloomFilterBytesRead: 0,
       rowGroupsPrunedByPageIndex: 0,
       pageIndexesRead: 0,
       pagesRead: 0,
@@ -486,3 +751,50 @@ function createParquetDatasetTelemetry(): ParquetDatasetTelemetry {
     }
   };
 }
+
+/** Converts cumulative dataset and child counters into one portable read-scoped snapshot. */
+function createParquetDatasetScanExecutionTelemetry(
+  before: ParquetDatasetTelemetry,
+  after: ParquetDatasetTelemetry,
+  startedAt: number,
+  status: ScanExecutionTelemetry['status'],
+  earlyTerminationReason?: ScanExecutionTelemetry['earlyTerminationReason'],
+  error?: unknown
+): ScanExecutionTelemetry {
+  const parquet = Object.fromEntries(
+    Object.keys(after.parquet).map(key => {
+      const telemetryKey = key as keyof ParquetTelemetry;
+      return [key, after.parquet[telemetryKey] - before.parquet[telemetryKey]];
+    })
+  ) as ParquetTelemetry;
+  const dataset = Object.freeze({
+    filesDiscovered: after.filesDiscovered - before.filesDiscovered,
+    filesSelected: after.filesSelected - before.filesSelected,
+    filesPrunedByBoundingBox: after.filesPrunedByBoundingBox - before.filesPrunedByBoundingBox,
+    filesPrunedByPartitions: after.filesPrunedByPartitions - before.filesPrunedByPartitions,
+    filesOpened: after.filesOpened - before.filesOpened,
+    batchesEmitted: after.batchesEmitted - before.batchesEmitted,
+    rowsEmitted: after.rowsEmitted - before.rowsEmitted
+  });
+  const rowsRead = Math.max(parquet.predicateRowsTested, parquet.rowsEmitted, dataset.rowsEmitted);
+  return Object.freeze({
+    status,
+    sourcesPlanned: Math.max(dataset.filesSelected, dataset.filesOpened),
+    sourcesRead: dataset.filesOpened,
+    batchesRead: dataset.batchesEmitted,
+    batchesDecoded: parquet.rowGroupsDecoded,
+    rowsRead,
+    rowsTested: parquet.predicateRowsTested || undefined,
+    rowsRetained: parquet.predicateRowsMatched || undefined,
+    rowsReturned: dataset.rowsEmitted,
+    bytesRead: parquet.downloadedBytes,
+    bytesFetched: parquet.downloadedBytes,
+    filesOpened: dataset.filesOpened,
+    tasksOpened: parquet.rowGroupsDecoded,
+    rowsPruned: parquet.rowsPrunedByPageIndex,
+    durationMilliseconds: Date.now() - startedAt,
+    earlyTerminationReason,
+    details: Object.freeze({dataset, parquet: Object.freeze({...parquet})}),
+    ...(error === undefined ? {} : {error})
+  });
+}