@loaders.gl/parquet 4.2.0-alpha.6 → 4.2.0-beta.2

package/dist/index.cjs

@@ -1743,7 +1743,7 @@ globalThis.process.env = globalThis.process.env || {};
 var Buffer3 = installBufferPolyfill();
 
 // dist/parquet-loader.js
-var import_loader_utils = require("@loaders.gl/loader-utils");
+var import_loader_utils2 = require("@loaders.gl/loader-utils");
 
 // dist/parquetjs/codecs/plain.js
 var import_int53 = __toESM(require("int53"), 1);
@@ -2005,12 +2005,16 @@ function decodeValues2(type, cursor, count, opts) {
   while (values.length < count) {
     const header = import_varint.default.decode(cursor.buffer, cursor.offset);
     cursor.offset += import_varint.default.encodingLength(header);
+    let decodedValues;
     if (header & 1) {
       const count2 = (header >> 1) * 8;
-      values.push(...decodeRunBitpacked(cursor, count2, opts));
+      decodedValues = decodeRunBitpacked(cursor, count2, opts);
     } else {
       const count2 = header >> 1;
-      values.push(...decodeRunRepeated(cursor, count2, opts));
+      decodedValues = decodeRunRepeated(cursor, count2, opts);
+    }
+    for (const value of decodedValues) {
+      values.push(value);
     }
   }
   values = values.slice(0, count);
@@ -2102,6 +2106,7 @@ var PARQUET_CODECS = {
 
 // dist/parquetjs/compression.js
 var import_compression = require("@loaders.gl/compression");
+var import_loader_utils = require("@loaders.gl/loader-utils");
 var import_lz4js = __toESM(require("lz4js"), 1);
 function toBuffer(arrayBuffer) {
   return Buffer.from(arrayBuffer);
@@ -2138,8 +2143,9 @@ var PARQUET_COMPRESSION_METHODS = {
   ZSTD: new import_compression.ZstdCompression({ modules })
 };
 async function preloadCompressions(options) {
+  (0, import_loader_utils.registerJSModules)(options == null ? void 0 : options.modules);
   const compressions = Object.values(PARQUET_COMPRESSION_METHODS);
-  return await Promise.all(compressions.map((compression) => compression.preload()));
+  return await Promise.all(compressions.map((compression) => compression.preload(options == null ? void 0 : options.modules)));
 }
 async function deflate(method, value) {
   const compression = PARQUET_COMPRESSION_METHODS[method];
@@ -6314,7 +6320,7 @@ var FileMetaData = class {
 };
 
 // dist/lib/constants.js
-var VERSION = true ? "4.2.0-alpha.5" : "latest";
+var VERSION = true ? "4.2.0-beta.1" : "latest";
 var PARQUET_WASM_URL = "https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm";
 var PARQUET_MAGIC = "PAR1";
 var PARQUET_MAGIC_ENCRYPTED = "PARE";
@@ -6393,6 +6399,7 @@ function fieldIndexOf(arr, elem) {
 
 // dist/parquetjs/parser/decoders.js
 async function decodeDataPages(buffer, context) {
+  var _a;
   const cursor = {
     buffer,
     offset: 0,
@@ -6415,7 +6422,8 @@ async function decodeDataPages(buffer, context) {
       dictionary = page.dictionary;
       continue;
     }
-    if (dictionary.length) {
+    const valueEncoding = getThriftEnum(Encoding, (_a = page.pageHeader.data_page_header) == null ? void 0 : _a.encoding);
+    if (dictionary.length && valueEncoding !== "PLAIN") {
       page.values = page.values.map((value) => dictionary[value]);
     }
     for (let index = 0; index < page.rlevels.length; index++) {
@@ -6945,6 +6953,7 @@ async function getSchemaFromParquetReader(reader) {
 async function parseParquetFile(file, options) {
   var _a, _b;
   installBufferPolyfill();
+  await preloadCompressions(options);
   const reader = new ParquetReader(file, {
     preserveBinary: (_a = options == null ? void 0 : options.parquet) == null ? void 0 : _a.preserveBinary
   });
@@ -6966,6 +6975,8 @@ async function parseParquetFile(file, options) {
 }
 async function* parseParquetFileInBatches(file, options) {
   var _a, _b;
+  installBufferPolyfill();
+  await preloadCompressions(options);
   const reader = new ParquetReader(file, {
     preserveBinary: (_a = options == null ? void 0 : options.parquet) == null ? void 0 : _a.preserveBinary
   });
@@ -7057,6 +7068,7 @@ function convertBatch(objectRowBatch, shape) {
 // dist/lib/parsers/parse-parquet-to-columns.js
 async function parseParquetFileInColumns(file, options) {
   installBufferPolyfill();
+  await preloadCompressions(options);
   for await (const batch of parseParquetFileInColumnarBatches(file, options)) {
     return {
       shape: "columnar-table",
@@ -7067,6 +7079,8 @@ async function parseParquetFileInColumns(file, options) {
   throw new Error("empty table");
 }
 async function* parseParquetFileInColumnarBatches(file, options) {
+  installBufferPolyfill();
+  await preloadCompressions(options);
   const reader = new ParquetReader(file);
   const schema = await getSchemaFromParquetReader(reader);
   const parquetSchema = await reader.getSchema();
@@ -7087,8 +7101,10 @@ function convertRowGroupToTableBatch(rowGroup, parquetSchema, schema) {
 }
 
 // dist/parquet-loader.js
-var VERSION2 = true ? "4.2.0-alpha.5" : "latest";
+var VERSION2 = true ? "4.2.0-beta.1" : "latest";
 var ParquetWorkerLoader = {
+  dataType: null,
+  batchType: null,
   name: "Apache Parquet",
   id: "parquet",
   module: "parquet",
@@ -7111,12 +7127,16 @@ var ParquetWorkerLoader = {
 };
 var ParquetLoader = {
   ...ParquetWorkerLoader,
-  parse: (arrayBuffer, options) => parseParquetFile(new import_loader_utils.BlobFile(arrayBuffer), options),
+  dataType: null,
+  batchType: null,
+  parse: (arrayBuffer, options) => parseParquetFile(new import_loader_utils2.BlobFile(arrayBuffer), options),
   parseFile: parseParquetFile,
   parseFileInBatches: parseParquetFileInBatches
 };
 ParquetLoader.Buffer = Buffer;
 var GeoParquetWorkerLoader = {
+  dataType: null,
+  batchType: null,
   name: "Apache Parquet",
   id: "parquet",
   module: "parquet",
@@ -7140,12 +7160,14 @@ var GeoParquetWorkerLoader = {
 var GeoParquetLoader = {
   ...GeoParquetWorkerLoader,
   parse(arrayBuffer, options) {
-    return parseGeoParquetFile(new import_loader_utils.BlobFile(arrayBuffer), options);
+    return parseGeoParquetFile(new import_loader_utils2.BlobFile(arrayBuffer), options);
   },
   parseFile: parseGeoParquetFile,
   parseFileInBatches: parseGeoParquetFileInBatches
 };
 var ParquetColumnarWorkerLoader = {
+  dataType: null,
+  batchType: null,
   name: "Apache Parquet",
   id: "parquet",
   module: "parquet",
@@ -7161,14 +7183,14 @@ var ParquetColumnarWorkerLoader = {
 var ParquetColumnarLoader = {
   ...ParquetColumnarWorkerLoader,
   parse(arrayBuffer, options) {
-    return parseParquetFileInColumns(new import_loader_utils.BlobFile(arrayBuffer), options);
+    return parseParquetFileInColumns(new import_loader_utils2.BlobFile(arrayBuffer), options);
   },
   parseFile: parseParquetFileInColumns,
   parseFileInBatches: parseParquetFileInColumnarBatches
 };
 
 // dist/parquet-writer.js
-var VERSION3 = true ? "4.2.0-alpha.5" : "latest";
+var VERSION3 = true ? "4.2.0-beta.1" : "latest";
 var ParquetWriter = {
   name: "Apache Parquet",
   id: "parquet",
@@ -7225,6 +7247,8 @@ async function parseParquetWasm(arrayBuffer, options) {
 
 // dist/parquet-wasm-loader.js
 var ParquetWasmWorkerLoader = {
+  dataType: null,
+  batchType: null,
   name: "Apache Parquet",
   id: "parquet-wasm",
   module: "parquet",