@loaders.gl/parquet 4.4.0-alpha.2 → 4.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "4.4.0-alpha.2",
+  "version": "4.4.0",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "type": "module",
@@ -30,6 +30,9 @@
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js",
       "require": "./dist/index.cjs"
+    },
+    "./parquet-worker.js": {
+      "import": "./dist/parquet-worker.js"
     }
   },
   "sideEffects": false,
@@ -60,20 +63,21 @@
     "base64-js and ieee754 are used by buffer polyfill"
   ],
   "dependencies": {
-    "@loaders.gl/arrow": "4.4.0-alpha.2",
-    "@loaders.gl/bson": "4.4.0-alpha.2",
-    "@loaders.gl/compression": "4.4.0-alpha.2",
-    "@loaders.gl/geoarrow": "4.4.0-alpha.2",
-    "@loaders.gl/gis": "4.4.0-alpha.2",
-    "@loaders.gl/loader-utils": "4.4.0-alpha.2",
-    "@loaders.gl/schema": "4.4.0-alpha.2",
-    "@loaders.gl/wkt": "4.4.0-alpha.2",
-    "@probe.gl/log": "^4.0.9",
+    "@loaders.gl/arrow": "4.4.0",
+    "@loaders.gl/bson": "4.4.0",
+    "@loaders.gl/compression": "4.4.0",
+    "@loaders.gl/geoarrow": "4.4.0",
+    "@loaders.gl/gis": "4.4.0",
+    "@loaders.gl/loader-utils": "4.4.0",
+    "@loaders.gl/schema": "4.4.0",
+    "@loaders.gl/wkt": "4.4.0",
+    "@probe.gl/log": "^4.1.1",
     "async-mutex": "^0.2.2",
     "base64-js": "^1.3.1",
     "brotli": "^1.3.2",
     "ieee754": "^1.2.1",
     "int53": "^0.2.4",
+    "isomorphic-ws": "^5.0.0",
     "lz4js": "^0.2.0",
     "node-int64": "^0.4.0",
     "object-stream": "0.0.1",
@@ -85,14 +89,13 @@
     "zstd-codec": "^0.1"
   },
   "devDependencies": {
-    "@types/node": "^10.14.15",
     "@types/node-int64": "^0.4.29",
     "@types/thrift": "^0.10.8",
     "@types/varint": "^5.0.0"
   },
   "peerDependencies": {
     "@loaders.gl/core": "4.4.0-alpha.1",
-    "apache-arrow": ">= 16.1.0"
+    "apache-arrow": ">= 17.0.0"
   },
-  "gitHead": "3d9fed050eabdc0812ddf2f4d5fb9914a34ee0c2"
+  "gitHead": "b8a23bc05946bde7089936ff3ea1651ee9a45536"
 }

package/src/lib/encoders/encode-arrow-to-parquet.ts

@@ -29,10 +29,12 @@ export async function encodeArrowToParquet(
   try {
     const parquetBytes = wasm.writeParquet(wasmTable, wasmProperties);
     // const parquetBytes = wasm.writeParquet(wasmTable, wasmProperties);
-    return parquetBytes.buffer.slice(
+    const bytes = new Uint8Array(
+      parquetBytes.buffer,
       parquetBytes.byteOffset,
-      parquetBytes.byteLength + parquetBytes.byteOffset
+      parquetBytes.byteLength
     );
+    return bytes.slice().buffer;
   } finally {
     // wasmTable.free();
     // wasmProperties.free();

package/src/lib/parsers/parse-parquet-to-json.ts

@@ -39,14 +39,16 @@ export async function parseParquetFile(
     let limitHasReached = false;
     // we have only one input batch so return
     for (const row of rowBatch) {
-      if (options?.limit && rows.length >= options?.limit) {
+      if (options?.limit && rows.length >= options?.core?.limit!) {
         limitHasReached = true;
         break;
       }
       rows.push(row);
     }
     if (limitHasReached) {
-      log.warn(`Rows number limit has been reached. Only first ${options?.limit} are loaded`)();
+      log.warn(
+        `Rows number limit has been reached. Only first ${options?.core?.limit} are loaded`
+      )();
       break;
     }
   }

package/src/parquetjs/codecs/plain.ts

@@ -69,7 +69,7 @@ function encodeValues_BOOLEAN(values: boolean[]): Buffer {
   buf.fill(0);
   for (let i = 0; i < values.length; i++) {
     if (values[i]) {
-      buf[Math.floor(i / 8)] |= 1 << i % 8;
+      buf[Math.floor(i / 8)] |= 1 << (i % 8);
     }
   }
   return buf;
@@ -79,7 +79,7 @@ function decodeValues_BOOLEAN(cursor: CursorBuffer, count: number): boolean[] {
   const values: boolean[] = [];
   for (let i = 0; i < count; i++) {
     const b = cursor.buffer[cursor.offset + Math.floor(i / 8)];
-    values.push((b & (1 << i % 8)) > 0);
+    values.push((b & (1 << (i % 8))) > 0);
   }
   cursor.offset += Math.ceil(count / 8);
   return values;
@@ -194,7 +194,7 @@ function encodeValues_BYTE_ARRAY(values: Buffer[]): Buffer {
   let buf_pos = 0;
   for (let i = 0; i < values.length; i++) {
     buf.writeUInt32LE(values[i].length, buf_pos);
-    values[i].copy(buf, buf_pos + 4);
+    values[i].copy(buf as Uint8Array, buf_pos + 4);
     buf_pos += 4 + values[i].length;
   }
   return buf;
@@ -222,7 +222,7 @@ function encodeValues_FIXED_LEN_BYTE_ARRAY(values: Buffer[], opts: ParquetCodecO
       throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);
     }
   }
-  return Buffer.concat(values);
+  return Buffer.concat(values as Uint8Array[]);
 }
 
 function decodeValues_FIXED_LEN_BYTE_ARRAY(

package/src/parquetjs/codecs/rle.ts

@@ -40,7 +40,7 @@ export function encodeValues(
     if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
       // If we have any data in runs we need to encode them
       if (run.length) {
-        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
+        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)] as Uint8Array[]);
         run = [];
       }
       repeats = 1;
@@ -49,7 +49,7 @@ export function encodeValues(
     } else {
       // If values changes we need to post any previous repeated values
       if (repeats) {
-        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)]);
+        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)] as Uint8Array[]);
         repeats = 0;
       }
       run.push(values[i]);
@@ -57,9 +57,12 @@ export function encodeValues(
   }
 
   if (repeats) {
-    buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
+    buf = Buffer.concat([
+      buf,
+      encodeRunRepeated(values[values.length - 1], repeats, opts)
+    ] as Uint8Array[]);
   } else if (run.length) {
-    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
+    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)] as Uint8Array[]);
   }
 
   if (opts.disableEnvelope) {
@@ -70,7 +73,7 @@ export function encodeValues(
 
   // @ts-ignore buffer polyfill
   envelope.writeUInt32LE(buf.length, undefined);
-  buf.copy(envelope, 4);
+  buf.copy(envelope as Uint8Array, 4);
 
   return envelope;
 }
@@ -131,8 +134,8 @@ function decodeRunBitpacked(
   // tslint:disable-next-line:prefer-array-literal
   const values = new Array(count).fill(0);
   for (let b = 0; b < bitWidth * count; b++) {
-    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << b % 8)) {
-      values[Math.floor(b / bitWidth)] |= 1 << b % bitWidth;
+    if (cursor.buffer[cursor.offset + Math.floor(b / 8)] & (1 << (b % 8))) {
+      values[Math.floor(b / bitWidth)] |= 1 << (b % bitWidth);
     }
   }
 
@@ -170,12 +173,15 @@ function encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer
 
   const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
   for (let b = 0; b < bitWidth * values.length; b++) {
-    if ((values[Math.floor(b / bitWidth)] & (1 << b % bitWidth)) > 0) {
-      buf[Math.floor(b / 8)] |= 1 << b % 8;
+    if ((values[Math.floor(b / bitWidth)] & (1 << (b % bitWidth))) > 0) {
+      buf[Math.floor(b / 8)] |= 1 << (b % 8);
     }
   }
 
-  return Buffer.concat([Buffer.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);
+  return Buffer.concat([
+    Buffer.from(varint.encode(((values.length / 8) << 1) | 1)),
+    buf
+  ] as Uint8Array[]);
 }
 
 function encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {
@@ -190,5 +196,5 @@ function encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptio
     value >> 8; //  TODO - this looks wrong
   }
 
-  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf]);
+  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf] as Uint8Array[]);
 }

package/src/parquetjs/encoder/parquet-encoder.ts

@@ -376,7 +376,7 @@ async function encodeDataPage(
   page: Buffer;
 }> {
   /* encode repetition and definition levels */
-  let rLevelsBuf = Buffer.alloc(0);
+  let rLevelsBuf: Buffer = Buffer.alloc(0);
   if (column.rLevelMax > 0) {
     rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
       bitWidth: getBitWidth(column.rLevelMax)
@@ -384,7 +384,7 @@ async function encodeDataPage(
     });
   }
 
-  let dLevelsBuf = Buffer.alloc(0);
+  let dLevelsBuf: Buffer = Buffer.alloc(0);
   if (column.dLevelMax > 0) {
     dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
       bitWidth: getBitWidth(column.dLevelMax)
@@ -398,7 +398,7 @@ async function encodeDataPage(
     bitWidth: column.typeLength
   });
 
-  const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);
+  const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf] as Uint8Array[]);
 
   // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
   const compressedBuf = await Compression.deflate(column.compression!, dataBuf);
@@ -418,7 +418,7 @@ async function encodeDataPage(
 
   /* concat page header, repetition and definition levels and values */
   const headerBuf = serializeThrift(header);
-  const page = Buffer.concat([headerBuf, compressedBuf]);
+  const page = Buffer.concat([headerBuf, compressedBuf] as Uint8Array[]);
 
   return {header, headerSize: headerBuf.length, page};
 }
@@ -445,7 +445,7 @@ async function encodeDataPageV2(
   const compressedBuf = await Compression.deflate(column.compression!, valuesBuf);
 
   /* encode repetition and definition levels */
-  let rLevelsBuf = Buffer.alloc(0);
+  let rLevelsBuf: Buffer = Buffer.alloc(0);
   if (column.rLevelMax > 0) {
     rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
       bitWidth: getBitWidth(column.rLevelMax),
@@ -453,7 +453,7 @@ async function encodeDataPageV2(
     });
   }
 
-  let dLevelsBuf = Buffer.alloc(0);
+  let dLevelsBuf: Buffer = Buffer.alloc(0);
   if (column.dLevelMax > 0) {
     dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
       bitWidth: getBitWidth(column.dLevelMax),
@@ -479,7 +479,7 @@ async function encodeDataPageV2(
 
   /* concat page header, repetition and definition levels and values */
   const headerBuf = serializeThrift(header);
-  const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);
+  const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf] as Uint8Array[]);
   return {header, headerSize: headerBuf.length, page};
 }
 
@@ -536,7 +536,7 @@ async function encodeColumnChunk(
 
   /* concat metadata header and data pages */
   const metadataOffset = baseOffset + pageBuf.length;
-  const body = Buffer.concat([pageBuf, serializeThrift(metadata)]);
+  const body = Buffer.concat([pageBuf, serializeThrift(metadata)] as Uint8Array[]);
   return {body, metadata, metadataOffset};
 }
 
@@ -573,7 +573,7 @@ async function encodeRowGroup(
     metadata.columns.push(cchunk);
     metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);
 
-    body = Buffer.concat([body, cchunkData.body]);
+    body = Buffer.concat([body, cchunkData.body] as Uint8Array[]);
   }
 
   return {body, metadata};
@@ -638,7 +638,7 @@ function encodeFooter(
   const metadataEncoded = serializeThrift(metadata);
   const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
 
-  metadataEncoded.copy(footerEncoded);
+  metadataEncoded.copy(footerEncoded as Uint8Array);
   footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);
   footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);
   return footerEncoded;

package/src/parquetjs/parser/decoders.ts

@@ -424,7 +424,7 @@ async function decodeDictionaryPage(
 ): Promise<(string | ArrayBuffer)[]> {
   const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
 
-  let dictCursor = {
+  let dictCursor: CursorBuffer = {
     offset: 0,
     buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
     size: cursorEnd - cursor.offset
@@ -475,7 +475,7 @@ function preserveBinary(d: any): ArrayBuffer | ArrayBufferView | string {
   }
   // Convert to ArrayBuffer
   if (Buffer.isBuffer(d)) {
-    return d.buffer.slice(d.byteOffset, d.byteLength);
+    return new Uint8Array(d.buffer, d.byteOffset, d.byteLength).slice().buffer;
   }
   return d.toString();
 }

package/src/parquetjs/utils/read-utils.ts

@@ -30,7 +30,7 @@ export function serializeThrift(obj: any): Buffer {
   obj.write(protocol);
   transport.flush();
 
-  return Buffer.concat(output);
+  return Buffer.concat(output as Uint8Array[]);
 }
 
 export function decodeThrift(obj: any, buf: Buffer, offset?: number) {