@loaders.gl/scan 5.0.0-alpha.3

package/src/hudi.ts

@@ -0,0 +1,200 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  CoreAPI,
+  ScanFragment,
+  ScanFragmentProvider,
+  ScanQueryMetadata,
+  ScanQueryMetadataOptions,
+  TableScanSource
+} from '@loaders.gl/loader-utils';
+import {createScanQueryMetadata, DataSource} from '@loaders.gl/loader-utils';
+import {ParquetDatasetSource} from '@loaders.gl/parquet/parquet-dataset-source';
+import {PARQUET_TABLE_QUERY_CAPABILITIES} from '@loaders.gl/parquet/parquet-source-capabilities';
+import type {
+  ParquetDatasetBatch,
+  ParquetDatasetExplain,
+  ParquetDatasetReadOptions,
+  ParquetPredicate,
+  ParquetDatasetSourceOptions
+} from '@loaders.gl/parquet/parquet-source-types';
+
+/** A physical Parquet base file recorded by a Hudi snapshot descriptor. */
+export type HudiBaseFile = Readonly<{
+  path: string;
+  size?: number;
+  numRecords?: number;
+  partitionValues?: Readonly<Record<string, unknown>>;
+}>;
+
+/** Minimal, portable descriptor used by the Hudi proof of concept. */
+export type HudiSnapshotDescriptor = Readonly<{
+  tableType?: 'COPY_ON_WRITE' | 'MERGE_ON_READ';
+  basePath?: string;
+  completedInstant?: string;
+  files: readonly HudiBaseFile[];
+}>;
+
+/** Options for reading a Hudi snapshot descriptor and its Parquet base files. */
+export type HudiSourceOptions = ParquetDatasetSourceOptions & {
+  hudi?: {
+    baseUrl?: string;
+    headers?: HeadersInit;
+  };
+};
+
+/** Query options accepted by a Hudi snapshot scan. */
+export type HudiScanOptions = ParquetDatasetReadOptions;
+
+/**
+ * Minimal read-only Hudi Copy-on-Write source.
+ *
+ * The POC consumes a small JSON snapshot descriptor containing the selected base files. It does
+ * not claim to decode Hudi log files, merge-on-read records, or incremental timeline queries.
+ */
+export class HudiTableSource
+  extends DataSource<string | Blob, HudiSourceOptions>
+  implements
+    TableScanSource<ParquetDatasetBatch, ParquetPredicate>,
+    ScanFragmentProvider<ParquetPredicate>
+{
+  private descriptorPromise: Promise<HudiSnapshotDescriptor> | null = null;
+
+  /** Creates a source from a Hudi snapshot descriptor URL or Blob. */
+  constructor(data: string | Blob, options: HudiSourceOptions = {}, coreApi?: CoreAPI) {
+    super(data, options, undefined, coreApi);
+  }
+
+  /** Loads and validates the POC snapshot descriptor. */
+  async getDescriptor(signal?: AbortSignal): Promise<HudiSnapshotDescriptor> {
+    if (!this.descriptorPromise) {
+      this.descriptorPromise = this.loadDescriptor(signal).catch(error => {
+        this.descriptorPromise = null;
+        throw error;
+      });
+    }
+    return this.descriptorPromise;
+  }
+
+  /** Discovers the Copy-on-Write Parquet base files in the selected snapshot. */
+  async getScanFragments(options: HudiScanOptions = {}): Promise<readonly ScanFragment[]> {
+    const descriptor = await this.getDescriptor(options.signal);
+    if (descriptor.tableType === 'MERGE_ON_READ') {
+      throw new Error('Hudi Merge-on-Read snapshots are not supported by this POC');
+    }
+    return Object.freeze(
+      descriptor.files.map(file =>
+        Object.freeze({
+          id: file.path,
+          uri: this.resolveDataFile(file.path, descriptor.basePath),
+          partitionValues: file.partitionValues,
+          byteLength: file.size,
+          rowCount: file.numRecords,
+          metadata: {completedInstant: descriptor.completedInstant}
+        })
+      )
+    );
+  }
+
+  /** Discovers schema and snapshot statistics without decoding every row. */
+  async getQueryMetadata(options: ScanQueryMetadataOptions = {}): Promise<ScanQueryMetadata> {
+    const fragments = await this.getScanFragments(options);
+    if (!fragments.length) throw new Error('Hudi snapshot contains no Parquet base files');
+    const dataset = this.createDataset(fragments);
+    const schema = await dataset.getSchema({signal: options.signal});
+    return createScanQueryMetadata({
+      sourceType: 'hudi',
+      queryType: 'table',
+      execution: {status: 'supported', method: 'read'},
+      name: this.data instanceof Blob ? undefined : this.data,
+      schema,
+      capabilities: {table: PARQUET_TABLE_QUERY_CAPABILITIES},
+      statistics: {
+        rowCount: fragments.reduce((total, fragment) => total + Number(fragment.rowCount || 0), 0),
+        byteLength: fragments.reduce(
+          (total, fragment) => total + Number(fragment.byteLength || 0),
+          0
+        )
+      }
+    });
+  }
+
+  /** Explains the Hudi snapshot and delegated Parquet physical plan. */
+  async getScanPlan(options: HudiScanOptions = {}): Promise<ParquetDatasetExplain> {
+    return this.createDataset(await this.getScanFragments(options)).getScanPlan(options);
+  }
+
+  /** Reads the selected Hudi snapshot through the shared Parquet executor. */
+  read(options: HudiScanOptions = {}): AsyncIterable<ParquetDatasetBatch> {
+    return this.createDatasetFromOptions(options).read(options);
+  }
+
+  /** Alias for `read()` used by scan-aware consumers. */
+  scan(options: HudiScanOptions = {}): AsyncIterable<ParquetDatasetBatch> {
+    return this.read(options);
+  }
+
+  private async loadDescriptor(signal?: AbortSignal): Promise<HudiSnapshotDescriptor> {
+    const text =
+      typeof this.data === 'string'
+        ? await (await this.fetch(this.data, {headers: this.options.hudi?.headers, signal})).text()
+        : await this.data.text();
+    let descriptor: unknown;
+    try {
+      descriptor = JSON.parse(text);
+    } catch {
+      throw new Error('Hudi snapshot descriptor must be valid JSON');
+    }
+    const files =
+      descriptor && typeof descriptor === 'object'
+        ? (descriptor as {files?: unknown}).files
+        : undefined;
+    if (!Array.isArray(files)) {
+      throw new Error('Hudi snapshot descriptor must contain a files array');
+    }
+    const value = descriptor as HudiSnapshotDescriptor;
+    if (
+      value.tableType &&
+      value.tableType !== 'COPY_ON_WRITE' &&
+      value.tableType !== 'MERGE_ON_READ'
+    ) {
+      throw new Error(`Unsupported Hudi table type: ${String(value.tableType)}`);
+    }
+    return value;
+  }
+
+  private createDataset(fragments: readonly ScanFragment[]): ParquetDatasetSource {
+    return new ParquetDatasetSource(
+      fragments.map(fragment => ({
+        data: fragment.uri!,
+        id: fragment.id,
+        partitions: fragment.partitionValues as never,
+        metadata: fragment.metadata
+      })),
+      this.options
+    );
+  }
+
+  private createDatasetFromOptions(options: HudiScanOptions): ParquetDatasetSource {
+    return new ParquetDatasetSource(
+      () =>
+        this.getScanFragments(options).then(fragments =>
+          fragments.map(fragment => ({
+            data: fragment.uri!,
+            id: fragment.id,
+            partitions: fragment.partitionValues as never,
+            metadata: fragment.metadata
+          }))
+        ),
+      this.options
+    );
+  }
+
+  private resolveDataFile(path: string, basePath?: string): string {
+    const baseUrl = this.options.hudi?.baseUrl || basePath;
+    if (baseUrl) return new URL(path, baseUrl).toString();
+    return path;
+  }
+}

package/src/iceberg.ts

@@ -0,0 +1,36 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * Incubating Iceberg table-format adapters.
+ *
+ * This entry point is intentionally separate from the scan package root. The adapter remains
+ * backed by the Parquet implementation while its public API and planning semantics mature.
+ */
+export {IcebergTableSource} from '@loaders.gl/parquet/iceberg-table-source';
+export type {
+  IcebergScanOptions,
+  IcebergSourceOptions
+} from '@loaders.gl/parquet/iceberg-table-source';
+export {IcebergRestCatalog} from '@loaders.gl/parquet/iceberg-rest-catalog';
+export type {
+  IcebergRestCatalogOptions,
+  IcebergRestTable,
+  IcebergRestTableIdentifier
+} from '@loaders.gl/parquet/iceberg-rest-catalog';
+export type {
+  IcebergBoundingBox,
+  IcebergDataFile,
+  IcebergDeleteFile,
+  IcebergManifestFile,
+  IcebergParquetFile,
+  IcebergPartitionSpec,
+  IcebergScanPlan,
+  IcebergSchema,
+  IcebergSnapshot,
+  IcebergSnapshotReference,
+  IcebergSpatialFilter,
+  IcebergTableMetadata,
+  IcebergTableSourceOptions
+} from '@loaders.gl/parquet/iceberg-types';

package/src/index.ts

@@ -0,0 +1,97 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export {createScanEngine, registerScanBackend} from './scan-engine';
+// The low-level executor lives in loader-utils so format packages can reuse it without importing
+// this optional module. It is re-exported here as the scan package's common batch adapter.
+export {
+  executeTableScanBatches,
+  filterTableBatch,
+  makeTableScanBatch,
+  projectTableBatch,
+  projectTableSchema,
+  truncateTableBatch
+} from '@loaders.gl/loader-utils';
+export type {TableScanBatchReader, TableScanBatchOperators} from '@loaders.gl/loader-utils';
+export type {
+  ScanBackend,
+  ScanBackendLoader,
+  ScanBackendName,
+  ScanEngine,
+  ScanEngineOptions
+} from './scan-engine';
+
+// Re-export the query vocabulary from one optional entry point for applications. Format adapters
+// should continue importing the lightweight contracts from @loaders.gl/loader-utils.
+export {parseSQLPredicate} from '@loaders.gl/sql';
+export type {ArrowQueryOptions, SQLPredicate} from '@loaders.gl/sql';
+export type {
+  RelationalAggregate,
+  RelationalExpression,
+  RelationalOrderKey,
+  TableQueryOptions
+} from '@loaders.gl/loader-utils';
+
+export {
+  createScanQueryMetadata,
+  intersectPointCloudBounds,
+  selectPointCloudScanTiles,
+  validatePointCloudQueryOptions
+} from '@loaders.gl/loader-utils';
+export type {
+  CreateScanQueryMetadataOptions,
+  PointCloudQueryBounds,
+  PointCloudQueryCapabilities,
+  PointCloudQueryOptions,
+  PointCloudScanChildrenLoader,
+  PointCloudScanReadOptions,
+  PointCloudScanSource,
+  PointCloudScanTile,
+  RasterQueryCapabilities,
+  RasterQueryOptions,
+  ScanBounds,
+  ScanColumnMetadata,
+  ScanColumnRole,
+  ScanExecutionMethod,
+  ScanExecutionTelemetry,
+  ScanExecutionTelemetryCallback,
+  ScanExecutionTelemetryStatus,
+  ScanSourceExecutionTelemetry,
+  ScanExecutionSupport,
+  ScanQueryCapabilities,
+  ScanQueryMetadata,
+  ScanQueryMetadataOptions,
+  ScanQueryMetadataProvider,
+  ScanRasterLevel,
+  ScanSourceStatistics,
+  ScanSpatialMetadata,
+  TableScanReadOptions,
+  TableScanSource
+} from '@loaders.gl/loader-utils';
+export type {ScanQuery} from './scan-query';
+
+export {
+  FEDERATED_TABLE_QUERY_CAPABILITIES,
+  FederatedTableScanSource
+} from './federated-table-scan-source';
+export type {
+  FederatedTableBatch,
+  FederatedTableBatchProvenance,
+  FederatedTableScanExplain,
+  FederatedTableScanSourceOptions,
+  FederatedTableSchemaPolicy,
+  FederatedTableSourceEntry,
+  FederatedTableSourceExplain
+} from './federated-table-scan-source';
+
+export {
+  AddressedVectorTableScanSource,
+  VectorFeatureTableScanSource,
+  VectorTileTableScanSource
+} from './vector-table-scan-source';
+export type {
+  VectorFeatureTableScanSourceOptions,
+  VectorTableScanSourceOptions,
+  VectorTileTableScanSourceOptions
+} from './vector-table-scan-source';

package/src/scan-engine.ts

@@ -0,0 +1,96 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ArrowTable} from '@loaders.gl/schema';
+import {
+  explainArrowTableQuery,
+  queryArrowTable,
+  type ArrowQueryOptions,
+  type SQLPredicate
+} from '@loaders.gl/sql';
+import type {TableQueryExplain} from '@loaders.gl/loader-utils';
+
+/** Names accepted by the optional scan engine factory. */
+export type ScanBackendName = 'arrow' | (string & {});
+
+/** A backend implementation for the Arrow table query surface. */
+export type ScanBackend = Readonly<{
+  /** Stable backend name reported by the engine. */
+  name: ScanBackendName;
+  /** Executes a query over one in-memory Arrow table. */
+  query(sourceTable: ArrowTable, options?: ArrowQueryOptions): ArrowTable;
+  /** Optional asynchronous execution hook for backends that compile or schedule work. */
+  queryAsync?: (sourceTable: ArrowTable, options?: ArrowQueryOptions) => Promise<ArrowTable>;
+  /** Explains a query without evaluating table rows. */
+  explain(sourceTable: ArrowTable, options?: ArrowQueryOptions): TableQueryExplain<SQLPredicate>;
+}>;
+
+/** Lazy loader used to register an optional scan backend. */
+export type ScanBackendLoader = () => ScanBackend | Promise<ScanBackend>;
+
+/** Options for creating a scan engine. */
+export type ScanEngineOptions = Readonly<{
+  /** Backend to use. Arrow is the built-in reference backend. */
+  backend?: ScanBackendName;
+}>;
+
+/** Stable application-facing scan engine returned by {@link createScanEngine}. */
+export type ScanEngine = ScanBackend & {
+  /** Executes a query asynchronously, allowing backends to schedule remote or compiled work. */
+  queryAsync: (sourceTable: ArrowTable, options?: ArrowQueryOptions) => Promise<ArrowTable>;
+};
+
+const scanBackendLoaders = new Map<ScanBackendName, ScanBackendLoader>();
+
+const arrowScanBackend: ScanBackend = Object.freeze({
+  name: 'arrow',
+  query: queryArrowTable,
+  queryAsync: async (sourceTable, options) => queryArrowTable(sourceTable, options),
+  explain: explainArrowTableQuery
+});
+
+scanBackendLoaders.set('arrow', () => arrowScanBackend);
+
+/**
+ * Registers a lazy scan backend without adding it to the default bundle.
+ *
+ * This is the extension point for future DuckDB and GPU implementations. The backend package can
+ * register a dynamic loader from application code while the public import remains
+ * `@loaders.gl/scan`.
+ */
+export function registerScanBackend(name: ScanBackendName, loader: ScanBackendLoader): void {
+  if (!name || !loader) {
+    throw new Error('A scan backend name and loader are required');
+  }
+  scanBackendLoaders.set(name, loader);
+}
+
+/**
+ * Creates the configured scan engine.
+ *
+ * The factory is asynchronous even for Arrow so optional backends can be loaded lazily later
+ * without changing the application-facing API.
+ */
+export async function createScanEngine(options: ScanEngineOptions = {}): Promise<ScanEngine> {
+  const backendName = options.backend || 'arrow';
+  const loader = scanBackendLoaders.get(backendName);
+  if (!loader) {
+    throw new Error(
+      `Scan backend "${backendName}" is not registered. The built-in backend is "arrow".`
+    );
+  }
+  const backend = await loader();
+  if (backend.name !== backendName) {
+    throw new Error(
+      `Scan backend loader returned "${backend.name}" while "${backendName}" was requested.`
+    );
+  }
+  const query = backend.query.bind(backend);
+  const explain = backend.explain.bind(backend);
+  const queryAsync = backend.queryAsync
+    ? backend.queryAsync.bind(backend)
+    : async (sourceTable: ArrowTable, queryOptions?: ArrowQueryOptions) =>
+        backend.query.call(backend, sourceTable, queryOptions);
+  return Object.freeze({...backend, query, explain, queryAsync});
+}

package/src/scan-query.ts

@@ -0,0 +1,27 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * Source-neutral values emitted by a scan query editor.
+ *
+ * This type is deliberately smaller than any one format's executor options. Applications can
+ * pass it between metadata-driven controls and adapters, which may normalize or ignore values
+ * that are not supported by a particular source family.
+ */
+export type ScanQuery = Readonly<{
+  /** Output columns; an empty selection means all columns. */
+  columns?: readonly string[];
+  /** Maximum number of rows, features, or points to return. */
+  limit?: number;
+  /** Optional source-coordinate bounding box in minX, minY, maxX, maxY order. */
+  boundingBox?: readonly [number, number, number, number];
+  /** Optional raster overview or point-cloud hierarchy level. */
+  level?: number;
+  /** Optional point-cloud minimum hierarchy level. */
+  minimumLevel?: number;
+  /** Optional point-cloud maximum hierarchy level. */
+  maximumLevel?: number;
+  /** Optional point-cloud target spacing. */
+  targetSpacing?: number;
+}>;