@loaders.gl/parquet 4.2.1 → 4.3.0-alpha.2

package/dist/index.cjs

@@ -1745,6 +1745,9 @@ var Buffer3 = installBufferPolyfill();
 // dist/parquet-loader.js
 var import_loader_utils2 = require("@loaders.gl/loader-utils");
 
+// dist/lib/parsers/parse-parquet.js
+var import_log = __toESM(require("@probe.gl/log"), 1);
+
 // dist/parquetjs/codecs/plain.js
 var import_int53 = __toESM(require("int53"), 1);
 function encodeValues(type, values, opts) {
@@ -6320,8 +6323,8 @@ var FileMetaData = class {
 };
 
 // dist/lib/constants.js
-var VERSION = true ? "4.2.0" : "latest";
-var PARQUET_WASM_URL = "https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm";
+var VERSION = true ? "4.3.0-alpha.1" : "latest";
+var PARQUET_WASM_URL = "https://unpkg.com/parquet-wasm@0.6.1/esm/parquet_wasm_bg.wasm";
 var PARQUET_MAGIC = "PAR1";
 var PARQUET_MAGIC_ENCRYPTED = "PARE";
 var PARQUET_RDLVL_TYPE = "INT32";
@@ -6961,9 +6964,18 @@ async function parseParquetFile(file, options) {
   const rows = [];
   const rowBatches = reader.rowBatchIterator(options == null ? void 0 : options.parquet);
   for await (const rowBatch of rowBatches) {
+    let limitHasReached = false;
     for (const row of rowBatch) {
+      if ((options == null ? void 0 : options.limit) && rows.length >= (options == null ? void 0 : options.limit)) {
+        limitHasReached = true;
+        break;
+      }
       rows.push(row);
     }
+    if (limitHasReached) {
+      import_log.default.warn(`Rows number limit has been reached. Only first ${options == null ? void 0 : options.limit} are loaded`)();
+      break;
+    }
   }
   const objectRowTable = {
     shape: "object-row-table",
@@ -7101,7 +7113,7 @@ function convertRowGroupToTableBatch(rowGroup, parquetSchema, schema) {
 }
 
 // dist/parquet-loader.js
-var VERSION2 = true ? "4.2.0" : "latest";
+var VERSION2 = true ? "4.3.0-alpha.1" : "latest";
 var ParquetWorkerLoader = {
   dataType: null,
   batchType: null,
@@ -7190,7 +7202,7 @@ var ParquetColumnarLoader = {
 };
 
 // dist/parquet-writer.js
-var VERSION3 = true ? "4.2.0" : "latest";
+var VERSION3 = true ? "4.3.0-alpha.1" : "latest";
 var ParquetWriter = {
   name: "Apache Parquet",
   id: "parquet",
@@ -7207,10 +7219,14 @@ function encodeSync(data, options) {
   return new ArrayBuffer(0);
 }
 
-// dist/lib/wasm/parse-parquet-wasm.js
+// dist/parquet-wasm-loader.js
+var import_loader_utils4 = require("@loaders.gl/loader-utils");
+
+// dist/lib/parsers/parse-parquet-wasm.js
 var import_arrow = require("@loaders.gl/arrow");
+var arrow = __toESM(require("apache-arrow"), 1);
 
-// dist/lib/wasm/load-wasm.js
+// dist/lib/utils/load-wasm.js
 var import_parquet_wasm = __toESM(require("parquet-wasm"), 1);
 var parquetWasm = __toESM(require("parquet-wasm"), 1);
 var initializePromise;
@@ -7225,23 +7241,75 @@ async function loadWasm(wasmUrl = PARQUET_WASM_URL) {
   return parquetWasm;
 }
 
-// dist/lib/wasm/parse-parquet-wasm.js
-var arrow = __toESM(require("apache-arrow"), 1);
-async function parseParquetWasm(arrayBuffer, options) {
-  var _a;
-  const arr = new Uint8Array(arrayBuffer);
-  const wasmUrl = (_a = options == null ? void 0 : options.parquet) == null ? void 0 : _a.wasmUrl;
-  const wasm = await loadWasm(wasmUrl);
-  const wasmTable = wasm.readParquet(arr);
+// dist/lib/utils/make-stream-iterator.js
+var import_loader_utils3 = require("@loaders.gl/loader-utils");
+function makeStreamIterator(stream, options) {
+  return import_loader_utils3.isBrowser ? makeBrowserStreamIterator(stream, options) : makeNodeStreamIterator(stream, options);
+}
+async function* makeBrowserStreamIterator(stream, options) {
+  const reader = stream.getReader();
+  let nextBatchPromise;
   try {
-    const ipcStream = wasmTable.intoIPCStream();
-    const arrowTable = arrow.tableFromIPC(ipcStream);
-    return {
+    while (true) {
+      const currentBatchPromise = nextBatchPromise || reader.read();
+      if (options == null ? void 0 : options._streamReadAhead) {
+        nextBatchPromise = reader.read();
+      }
+      const { done, value } = await currentBatchPromise;
+      if (done) {
+        return;
+      }
+      if (value) {
+        yield value;
+      }
+    }
+  } catch (error) {
+    reader.releaseLock();
+  }
+}
+async function* makeNodeStreamIterator(stream, options) {
+  yield* stream;
+}
+
+// dist/lib/parsers/parse-parquet-wasm.js
+async function parseParquetFileWasm(file, options) {
+  const wasmUrl = options == null ? void 0 : options.wasmUrl;
+  const wasm = await loadWasm(wasmUrl);
+  let parquetFile;
+  if (file.handle instanceof Blob) {
+    parquetFile = await wasm.ParquetFile.fromFile(file.handle);
+  } else {
+    parquetFile = await wasm.ParquetFile.fromUrl(file.url);
+  }
+  const wasmTable = await parquetFile.read(options);
+  const ipcStream = wasmTable.intoIPCStream();
+  const arrowTable = arrow.tableFromIPC(ipcStream);
+  return {
+    shape: "arrow-table",
+    schema: (0, import_arrow.serializeArrowSchema)(arrowTable.schema),
+    data: arrowTable
+  };
+}
+async function* parseParquetFileInBatchesWasm(file, options) {
+  const wasmUrl = options == null ? void 0 : options.wasmUrl;
+  const wasm = await loadWasm(wasmUrl);
+  let parquetFile;
+  if (file.handle instanceof Blob) {
+    parquetFile = await wasm.ParquetFile.fromFile(file.handle);
+  } else {
+    parquetFile = await wasm.ParquetFile.fromUrl(file.url);
+  }
+  const stream = await parquetFile.stream(options);
+  let schema;
+  for await (const table of makeStreamIterator(stream)) {
+    schema ||= (0, import_arrow.serializeArrowSchema)(table.schema);
+    yield {
+      batchType: "data",
       shape: "arrow-table",
-      schema: (0, import_arrow.serializeArrowSchema)(arrowTable.schema),
-      data: arrowTable
+      schema,
+      data: table.batches[0],
+      length: table.numRows
     };
-  } finally {
   }
 }
 
@@ -7261,7 +7329,19 @@ var ParquetWasmWorkerLoader = {
   tests: ["PAR1", "PARE"],
   options: {
     parquet: {
-      type: "arrow-table",
+      shape: "arrow-table",
+      limit: void 0,
+      // Provide a limit to the number of rows to be read.
+      offset: 0,
+      // Provide an offset to skip over the given number of rows.
+      batchSize: void 0,
+      // The number of rows in each batch. If not provided, the upstream parquet default is 1024.
+      columns: void 0,
+      // The column names from the file to read.
+      rowGroups: void 0,
+      // Only read data from the provided row group indexes.
+      concurrency: void 0,
+      // The number of concurrent requests to make
       wasmUrl: PARQUET_WASM_URL
     }
   }
@@ -7269,12 +7349,20 @@ var ParquetWasmWorkerLoader = {
 var ParquetWasmLoader = {
   ...ParquetWasmWorkerLoader,
   parse(arrayBuffer, options) {
-    options = { parquet: { ...ParquetWasmLoader.options.parquet, ...options == null ? void 0 : options.parquet }, ...options };
-    return parseParquetWasm(arrayBuffer, options);
+    const wasmOptions = { ...ParquetWasmLoader.options.parquet, ...options == null ? void 0 : options.parquet };
+    return parseParquetFileWasm(new import_loader_utils4.BlobFile(arrayBuffer), wasmOptions);
+  },
+  parseFile(file, options) {
+    const wasmOptions = { ...ParquetWasmLoader.options.parquet, ...options == null ? void 0 : options.parquet };
+    return parseParquetFileWasm(file, wasmOptions);
+  },
+  parseFileInBatches(file, options) {
+    const wasmOptions = { ...ParquetWasmLoader.options.parquet, ...options == null ? void 0 : options.parquet };
+    return parseParquetFileInBatchesWasm(file, wasmOptions);
   }
 };
 
-// dist/lib/wasm/encode-parquet-wasm.js
+// dist/lib/encoders/encode-parquet-wasm.js
 var arrow2 = __toESM(require("apache-arrow"), 1);
 async function encode(table, options) {
   var _a;