@loaders.gl/scan 5.0.0-alpha.3 → 5.0.0-alpha.5

package/src/hudi.ts

@@ -23,9 +23,13 @@ import type {
 
 /** A physical Parquet base file recorded by a Hudi snapshot descriptor. */
 export type HudiBaseFile = Readonly<{
+  /** Path or URL of the Parquet base file. */
   path: string;
+  /** Compressed file size in bytes, when recorded by the descriptor. */
   size?: number;
+  /** Number of records in the base file, when recorded by the descriptor. */
   numRecords?: number;
+  /** Partition column values associated with the base file. */
   partitionValues?: Readonly<Record<string, unknown>>;
 }>;
 
@@ -139,7 +143,7 @@ export class HudiTableSource
   private async loadDescriptor(signal?: AbortSignal): Promise<HudiSnapshotDescriptor> {
     const text =
       typeof this.data === 'string'
-        ? await (await this.fetch(this.data, {headers: this.options.hudi?.headers, signal})).text()
+        ? await (await this.fetch(this.url, {headers: this.options.hudi?.headers, signal})).text()
         : await this.data.text();
     let descriptor: unknown;
     try {
@@ -154,6 +158,37 @@ export class HudiTableSource
     if (!Array.isArray(files)) {
       throw new Error('Hudi snapshot descriptor must contain a files array');
     }
+    for (const [index, file] of files.entries()) {
+      if (!file || typeof file !== 'object' || Array.isArray(file)) {
+        throw new Error(`Hudi snapshot file at index ${index} must be an object`);
+      }
+      const value = file as Record<string, unknown>;
+      if (typeof value.path !== 'string' || !value.path) {
+        throw new Error(
+          `Hudi snapshot file at index ${index} must contain a non-empty string path`
+        );
+      }
+      if (
+        value.size !== undefined &&
+        (typeof value.size !== 'number' || !Number.isFinite(value.size))
+      ) {
+        throw new Error(`Hudi snapshot file at index ${index} size must be a finite number`);
+      }
+      if (
+        value.numRecords !== undefined &&
+        (typeof value.numRecords !== 'number' || !Number.isFinite(value.numRecords))
+      ) {
+        throw new Error(`Hudi snapshot file at index ${index} numRecords must be a finite number`);
+      }
+      if (
+        value.partitionValues !== undefined &&
+        (!value.partitionValues ||
+          typeof value.partitionValues !== 'object' ||
+          Array.isArray(value.partitionValues))
+      ) {
+        throw new Error(`Hudi snapshot file at index ${index} partitionValues must be an object`);
+      }
+    }
     const value = descriptor as HudiSnapshotDescriptor;
     if (
       value.tableType &&

package/src/scan-engine.ts

@@ -41,16 +41,19 @@ export type ScanEngine = ScanBackend & {
   queryAsync: (sourceTable: ArrowTable, options?: ArrowQueryOptions) => Promise<ArrowTable>;
 };
 
-const scanBackendLoaders = new Map<ScanBackendName, ScanBackendLoader>();
+const scanBackends = new Map<ScanBackendName, ScanBackend | ScanBackendLoader>();
 
 const arrowScanBackend: ScanBackend = Object.freeze({
   name: 'arrow',
   query: queryArrowTable,
-  queryAsync: async (sourceTable, options) => queryArrowTable(sourceTable, options),
+  queryAsync: (sourceTable, options) =>
+    Promise.resolve().then(queryArrowTable.bind(null, sourceTable, options)),
   explain: explainArrowTableQuery
 });
 
-scanBackendLoaders.set('arrow', () => arrowScanBackend);
+// Store the built-in backend directly. Apart from avoiding needless indirection, this keeps
+// coverage stable when browser shards transform this shared module independently.
+scanBackends.set('arrow', arrowScanBackend);
 
 /**
  * Registers a lazy scan backend without adding it to the default bundle.
@@ -63,7 +66,7 @@ export function registerScanBackend(name: ScanBackendName, loader: ScanBackendLo
   if (!name || !loader) {
     throw new Error('A scan backend name and loader are required');
   }
-  scanBackendLoaders.set(name, loader);
+  scanBackends.set(name, loader);
 }
 
 /**
@@ -74,13 +77,14 @@ export function registerScanBackend(name: ScanBackendName, loader: ScanBackendLo
  */
 export async function createScanEngine(options: ScanEngineOptions = {}): Promise<ScanEngine> {
   const backendName = options.backend || 'arrow';
-  const loader = scanBackendLoaders.get(backendName);
-  if (!loader) {
+  const registeredBackend = scanBackends.get(backendName);
+  if (!registeredBackend) {
     throw new Error(
       `Scan backend "${backendName}" is not registered. The built-in backend is "arrow".`
     );
   }
-  const backend = await loader();
+  const backend =
+    typeof registeredBackend === 'function' ? await registeredBackend() : registeredBackend;
   if (backend.name !== backendName) {
     throw new Error(
       `Scan backend loader returned "${backend.name}" while "${backendName}" was requested.`