@loaders.gl/parquet 4.2.0-beta.1 → 4.2.0

package/dist/lib/constants.js

@@ -3,7 +3,7 @@
 // Copyright (c) vis.gl contributors
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-export const VERSION = typeof "4.2.0-alpha.6" !== 'undefined' ? "4.2.0-alpha.6" : 'latest';
+export const VERSION = typeof "4.2.0-beta.2" !== 'undefined' ? "4.2.0-beta.2" : 'latest';
 export const PARQUET_WASM_URL = 'https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm';
 /**
  * Parquet File Magic String

package/dist/parquet-loader.js

@@ -8,7 +8,7 @@ import { parseParquetFileInColumns, parseParquetFileInColumnarBatches } from "./
 export { Buffer } from "./polyfills/buffer/install-buffer-polyfill.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.2.0-alpha.6" !== 'undefined' ? "4.2.0-alpha.6" : 'latest';
+const VERSION = typeof "4.2.0-beta.2" !== 'undefined' ? "4.2.0-beta.2" : 'latest';
 /**
  * ParquetJS table loader
  */

package/dist/parquet-writer.js

@@ -3,7 +3,7 @@
 // Copyright (c) vis.gl contributors
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.2.0-alpha.6" !== 'undefined' ? "4.2.0-alpha.6" : 'latest';
+const VERSION = typeof "4.2.0-beta.2" !== 'undefined' ? "4.2.0-beta.2" : 'latest';
 export const ParquetWriter = {
     name: 'Apache Parquet',
     id: 'parquet',

package/dist/parquetjs/codecs/rle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rle.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/rle.ts"],"names":[],"mappings":";AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,6BAA0B;AACrD,OAAO,KAAK,EAAC,YAAY,EAAE,mBAAmB,EAAC,qBAAkB;AAIjE,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,GAAG,EAAE,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,CA4DR;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,EAAE,CA4BV"}
+{"version":3,"file":"rle.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/rle.ts"],"names":[],"mappings":";AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,6BAA0B;AACrD,OAAO,KAAK,EAAC,YAAY,EAAE,mBAAmB,EAAC,qBAAkB;AAIjE,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,GAAG,EAAE,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,CA4DR;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,EAAE,CAgCV"}

package/dist/parquetjs/codecs/rle.js

@@ -67,13 +67,17 @@ export function decodeValues(type, cursor, count, opts) {
     while (values.length < count) {
         const header = varint.decode(cursor.buffer, cursor.offset);
         cursor.offset += varint.encodingLength(header);
+        let decodedValues;
         if (header & 1) {
             const count = (header >> 1) * 8;
-            values.push(...decodeRunBitpacked(cursor, count, opts));
+            decodedValues = decodeRunBitpacked(cursor, count, opts);
         }
         else {
             const count = header >> 1;
-            values.push(...decodeRunRepeated(cursor, count, opts));
+            decodedValues = decodeRunRepeated(cursor, count, opts);
+        }
+        for (const value of decodedValues) {
+            values.push(value);
         }
     }
     values = values.slice(0, count);

package/dist/parquetjs/parser/decoders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":";AACA,OAAO,EAEL,kBAAkB,EAClB,oBAAoB,EACpB,eAAe,EAGf,gBAAgB,EACjB,6BAA0B;AAC3B,OAAO,EAAC,YAAY,EAAsC,2BAAwB;AAClF,OAAO,EAML,aAAa,EAEd,mCAAgC;AAKjC;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,kBAAkB,CAAC,CAmD7B;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,eAAe,CAAC,CA0B1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,aAAa,EAAE,EAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CA4DA"}
+{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":";AACA,OAAO,EAEL,kBAAkB,EAClB,oBAAoB,EACpB,eAAe,EAGf,gBAAgB,EACjB,6BAA0B;AAC3B,OAAO,EAAC,YAAY,EAAsC,2BAAwB;AAClF,OAAO,EAML,aAAa,EAEd,mCAAgC;AAKjC;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,kBAAkB,CAAC,CAyD7B;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,eAAe,CAAC,CA0B1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,aAAa,EAAE,EAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CA4DA"}

package/dist/parquetjs/parser/decoders.js

@@ -35,7 +35,10 @@ export async function decodeDataPages(buffer, context) {
             // eslint-disable-next-line no-continue
             continue;
         }
-        if (dictionary.length) {
+        const valueEncoding = getThriftEnum(Encoding, page.pageHeader.data_page_header?.encoding);
+        // Pages might be in different encodings. We don't need to decode in case
+        // of 'PLAIN' encoding because all values are already in place
+        if (dictionary.length && valueEncoding !== 'PLAIN') {
             // eslint-disable-next-line no-loop-func
             page.values = page.values.map((value) => dictionary[value]);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "4.2.0-beta.1",
+  "version": "4.2.0",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "type": "module",
@@ -60,13 +60,13 @@
     "base64-js and ieee754 are used by buffer polyfill"
   ],
   "dependencies": {
-    "@loaders.gl/arrow": "4.2.0-beta.1",
-    "@loaders.gl/bson": "4.2.0-beta.1",
-    "@loaders.gl/compression": "4.2.0-beta.1",
-    "@loaders.gl/gis": "4.2.0-beta.1",
-    "@loaders.gl/loader-utils": "4.2.0-beta.1",
-    "@loaders.gl/schema": "4.2.0-beta.1",
-    "@loaders.gl/wkt": "4.2.0-beta.1",
+    "@loaders.gl/arrow": "4.2.0",
+    "@loaders.gl/bson": "4.2.0",
+    "@loaders.gl/compression": "4.2.0",
+    "@loaders.gl/gis": "4.2.0",
+    "@loaders.gl/loader-utils": "4.2.0",
+    "@loaders.gl/schema": "4.2.0",
+    "@loaders.gl/wkt": "4.2.0",
     "async-mutex": "^0.2.2",
     "base64-js": "^1.3.1",
     "brotli": "^1.3.2",
@@ -93,5 +93,5 @@
     "@loaders.gl/core": "^4.0.0",
     "apache-arrow": ">= 15.0.0"
   },
-  "gitHead": "c386a9196516fe3ff24847b40e6c77be039cf905"
+  "gitHead": "0d9f54fca98dd1fae8998ef5829004af01feaaff"
 }

package/src/parquetjs/codecs/rle.ts

@@ -89,12 +89,16 @@ export function decodeValues(
   while (values.length < count) {
     const header = varint.decode(cursor.buffer, cursor.offset);
     cursor.offset += varint.encodingLength(header);
+    let decodedValues: number[];
     if (header & 1) {
       const count = (header >> 1) * 8;
-      values.push(...decodeRunBitpacked(cursor, count, opts));
+      decodedValues = decodeRunBitpacked(cursor, count, opts);
     } else {
       const count = header >> 1;
-      values.push(...decodeRunRepeated(cursor, count, opts));
+      decodedValues = decodeRunRepeated(cursor, count, opts);
+    }
+    for (const value of decodedValues) {
+      values.push(value);
     }
   }
   values = values.slice(0, count);

package/src/parquetjs/parser/decoders.ts

@@ -63,7 +63,13 @@ export async function decodeDataPages(
       continue;
     }
 
-    if (dictionary.length) {
+    const valueEncoding = getThriftEnum(
+      Encoding,
+      page.pageHeader.data_page_header?.encoding!
+    ) as ParquetCodec;
+    // Pages might be in different encodings. We don't need to decode in case
+    // of 'PLAIN' encoding because all values are already in place
+    if (dictionary.length && valueEncoding !== 'PLAIN') {
       // eslint-disable-next-line no-loop-func
       page.values = page.values.map((value) => dictionary[value]);
     }