@loaders.gl/parquet 4.4.2 → 5.0.0-alpha.1

package/src/parquetjs/codecs/plain.ts

@@ -7,13 +7,28 @@
 /* eslint-disable camelcase */
 import type {PrimitiveType} from '../schema/declare';
 import type {CursorBuffer, ParquetCodecOptions} from './declare';
-import INT53 from 'int53';
+import {
+  concatUint8Arrays,
+  copyUint8Array,
+  encodeUtf8,
+  readDoubleLE,
+  readFloatLE,
+  readInt32LE,
+  readInt64LE,
+  readUInt32LE,
+  toUint8Array,
+  writeDoubleLE,
+  writeFloatLE,
+  writeInt32LE,
+  writeInt64LE,
+  writeUInt32LE
+} from '../utils/binary-utils';
 
 export function encodeValues(
   type: PrimitiveType,
   values: any[],
   opts: ParquetCodecOptions
-): Buffer {
+): Uint8Array {
   switch (type) {
     case 'BOOLEAN':
       return encodeValues_BOOLEAN(values);
@@ -64,9 +79,8 @@ export function decodeValues(
   }
 }
 
-function encodeValues_BOOLEAN(values: boolean[]): Buffer {
-  const buf = Buffer.alloc(Math.ceil(values.length / 8));
-  buf.fill(0);
+function encodeValues_BOOLEAN(values: boolean[]): Uint8Array {
+  const buf = new Uint8Array(Math.ceil(values.length / 8));
   for (let i = 0; i < values.length; i++) {
     if (values[i]) {
       buf[Math.floor(i / 8)] |= 1 << (i % 8);
@@ -85,10 +99,10 @@ function decodeValues_BOOLEAN(cursor: CursorBuffer, count: number): boolean[] {
   return values;
 }
 
-function encodeValues_INT32(values: number[]): Buffer {
-  const buf = Buffer.alloc(4 * values.length);
+function encodeValues_INT32(values: number[]): Uint8Array {
+  const buf = new Uint8Array(4 * values.length);
   for (let i = 0; i < values.length; i++) {
-    buf.writeInt32LE(values[i], i * 4);
+    writeInt32LE(buf, values[i], i * 4);
   }
   return buf;
 }
@@ -96,16 +110,16 @@ function encodeValues_INT32(values: number[]): Buffer {
 function decodeValues_INT32(cursor: CursorBuffer, count: number): number[] {
   const values: number[] = [];
   for (let i = 0; i < count; i++) {
-    values.push(cursor.buffer.readInt32LE(cursor.offset));
+    values.push(readInt32LE(cursor.buffer, cursor.offset));
     cursor.offset += 4;
   }
   return values;
 }
 
-function encodeValues_INT64(values: number[]): Buffer {
-  const buf = Buffer.alloc(8 * values.length);
+function encodeValues_INT64(values: number[]): Uint8Array {
+  const buf = new Uint8Array(8 * values.length);
   for (let i = 0; i < values.length; i++) {
-    INT53.writeInt64LE(values[i], buf, i * 8);
+    writeInt64LE(buf, values[i], i * 8);
   }
   return buf;
 }
@@ -113,22 +127,17 @@ function encodeValues_INT64(values: number[]): Buffer {
 function decodeValues_INT64(cursor: CursorBuffer, count: number): number[] {
   const values: number[] = [];
   for (let i = 0; i < count; i++) {
-    values.push(INT53.readInt64LE(cursor.buffer, cursor.offset));
+    values.push(readInt64LE(cursor.buffer, cursor.offset));
     cursor.offset += 8;
   }
   return values;
 }
 
-function encodeValues_INT96(values: number[]): Buffer {
-  const buf = Buffer.alloc(12 * values.length);
+function encodeValues_INT96(values: number[]): Uint8Array {
+  const buf = new Uint8Array(12 * values.length);
   for (let i = 0; i < values.length; i++) {
-    if (values[i] >= 0) {
-      INT53.writeInt64LE(values[i], buf, i * 12);
-      buf.writeUInt32LE(0, i * 12 + 8); // truncate to 64 actual precision
-    } else {
-      INT53.writeInt64LE(~-values[i] + 1, buf, i * 12);
-      buf.writeUInt32LE(0xffffffff, i * 12 + 8); // truncate to 64 actual precision
-    }
+    writeInt64LE(buf, values[i], i * 12);
+    writeUInt32LE(buf, values[i] >= 0 ? 0 : 0xffffffff, i * 12 + 8);
   }
   return buf;
 }
@@ -136,8 +145,8 @@ function encodeValues_INT96(values: number[]): Buffer {
 function decodeValues_INT96(cursor: CursorBuffer, count: number): number[] {
   const values: number[] = [];
   for (let i = 0; i < count; i++) {
-    const low = INT53.readInt64LE(cursor.buffer, cursor.offset);
-    const high = cursor.buffer.readUInt32LE(cursor.offset + 8);
+    const low = readInt64LE(cursor.buffer, cursor.offset);
+    const high = readUInt32LE(cursor.buffer, cursor.offset + 8);
     if (high === 0xffffffff) {
       values.push(~-low + 1); // truncate to 64 actual precision
     } else {
@@ -148,10 +157,10 @@ function decodeValues_INT96(cursor: CursorBuffer, count: number): number[] {
   return values;
 }
 
-function encodeValues_FLOAT(values: number[]): Buffer {
-  const buf = Buffer.alloc(4 * values.length);
+function encodeValues_FLOAT(values: number[]): Uint8Array {
+  const buf = new Uint8Array(4 * values.length);
   for (let i = 0; i < values.length; i++) {
-    buf.writeFloatLE(values[i], i * 4);
+    writeFloatLE(buf, values[i], i * 4);
   }
   return buf;
 }
@@ -159,16 +168,16 @@ function encodeValues_FLOAT(values: number[]): Buffer {
 function decodeValues_FLOAT(cursor: CursorBuffer, count: number): number[] {
   const values: number[] = [];
   for (let i = 0; i < count; i++) {
-    values.push(cursor.buffer.readFloatLE(cursor.offset));
+    values.push(readFloatLE(cursor.buffer, cursor.offset));
     cursor.offset += 4;
   }
   return values;
 }
 
-function encodeValues_DOUBLE(values: number[]): Buffer {
-  const buf = Buffer.alloc(8 * values.length);
+function encodeValues_DOUBLE(values: number[]): Uint8Array {
+  const buf = new Uint8Array(8 * values.length);
   for (let i = 0; i < values.length; i++) {
-    buf.writeDoubleLE(values[i], i * 8);
+    writeDoubleLE(buf, values[i], i * 8);
   }
   return buf;
 }
@@ -176,67 +185,72 @@ function encodeValues_DOUBLE(values: number[]): Buffer {
 function decodeValues_DOUBLE(cursor: CursorBuffer, count: number): number[] {
   const values: number[] = [];
   for (let i = 0; i < count; i++) {
-    values.push(cursor.buffer.readDoubleLE(cursor.offset));
+    values.push(readDoubleLE(cursor.buffer, cursor.offset));
     cursor.offset += 8;
   }
   return values;
 }
 
-function encodeValues_BYTE_ARRAY(values: Buffer[]): Buffer {
+function encodeValues_BYTE_ARRAY(values: any[]): Uint8Array {
+  const byteValues = values.map(toPrimitiveBytes);
   // tslint:disable-next-line:variable-name
   let buf_len = 0;
-  for (let i = 0; i < values.length; i++) {
-    values[i] = Buffer.from(values[i]);
-    buf_len += 4 + values[i].length;
+  for (let i = 0; i < byteValues.length; i++) {
+    buf_len += 4 + byteValues[i].length;
   }
-  const buf = Buffer.alloc(buf_len);
+  const buf = new Uint8Array(buf_len);
   // tslint:disable-next-line:variable-name
   let buf_pos = 0;
-  for (let i = 0; i < values.length; i++) {
-    buf.writeUInt32LE(values[i].length, buf_pos);
-    values[i].copy(buf as Uint8Array, buf_pos + 4);
-    buf_pos += 4 + values[i].length;
+  for (let i = 0; i < byteValues.length; i++) {
+    writeUInt32LE(buf, byteValues[i].length, buf_pos);
+    buf.set(byteValues[i], buf_pos + 4);
+    buf_pos += 4 + byteValues[i].length;
   }
   return buf;
 }
 
-function decodeValues_BYTE_ARRAY(cursor: CursorBuffer, count: number): Buffer[] {
-  const values: Buffer[] = [];
+function decodeValues_BYTE_ARRAY(cursor: CursorBuffer, count: number): Uint8Array[] {
+  const values: Uint8Array[] = [];
   for (let i = 0; i < count; i++) {
-    const len = cursor.buffer.readUInt32LE(cursor.offset);
+    const len = readUInt32LE(cursor.buffer, cursor.offset);
     cursor.offset += 4;
-    // values.push(cursor.buffer.buffer.slice(cursor.offset, cursor.offset + len));
-    values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
+    values.push(copyUint8Array(cursor.buffer.subarray(cursor.offset, cursor.offset + len)));
     cursor.offset += len;
   }
   return values;
 }
 
-function encodeValues_FIXED_LEN_BYTE_ARRAY(values: Buffer[], opts: ParquetCodecOptions): Buffer {
+function encodeValues_FIXED_LEN_BYTE_ARRAY(values: any[], opts: ParquetCodecOptions): Uint8Array {
   if (!opts.typeLength) {
     throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
   }
+  const byteValues = values.map(toPrimitiveBytes);
   for (let i = 0; i < values.length; i++) {
-    values[i] = Buffer.from(values[i]);
-    if (values[i].length !== opts.typeLength) {
+    if (byteValues[i].length !== opts.typeLength) {
       throw new Error(`invalid value for FIXED_LEN_BYTE_ARRAY: ${values[i]}`);
     }
   }
-  return Buffer.concat(values as Uint8Array[]);
+  return concatUint8Arrays(byteValues);
 }
 
 function decodeValues_FIXED_LEN_BYTE_ARRAY(
   cursor: CursorBuffer,
   count: number,
   opts: ParquetCodecOptions
-): Buffer[] {
-  const values: Buffer[] = [];
+): Uint8Array[] {
+  const values: Uint8Array[] = [];
   if (!opts.typeLength) {
     throw new Error('missing option: typeLength (required for FIXED_LEN_BYTE_ARRAY)');
   }
   for (let i = 0; i < count; i++) {
-    values.push(cursor.buffer.slice(cursor.offset, cursor.offset + opts.typeLength));
+    values.push(
+      copyUint8Array(cursor.buffer.subarray(cursor.offset, cursor.offset + opts.typeLength))
+    );
     cursor.offset += opts.typeLength;
   }
   return values;
 }
+
+function toPrimitiveBytes(value: any): Uint8Array {
+  return typeof value === 'string' ? encodeUtf8(value) : toUint8Array(value);
+}

package/src/parquetjs/codecs/rle.ts

@@ -6,6 +6,7 @@
 
 import type {PrimitiveType} from '../schema/declare';
 import type {CursorBuffer, ParquetCodecOptions} from './declare';
+import {concatUint8Arrays, writeUInt32LE} from '../utils/binary-utils';
 import varint from 'varint';
 
 // eslint-disable-next-line max-statements, complexity
@@ -13,7 +14,7 @@ export function encodeValues(
   type: PrimitiveType,
   values: any[],
   opts: ParquetCodecOptions
-): Buffer {
+): Uint8Array {
   if (!('bitWidth' in opts)) {
     throw new Error('bitWidth is required');
   }
@@ -23,14 +24,14 @@ export function encodeValues(
     case 'INT32':
     case 'INT64':
       // tslint:disable-next-line:no-parameter-reassignment
-      values = values.map((x) => parseInt(x, 10));
+      values = values.map(x => parseInt(x, 10));
       break;
 
     default:
       throw new Error(`unsupported type: ${type}`);
   }
 
-  let buf = Buffer.alloc(0);
+  const buffers: Uint8Array[] = [];
   let run: any[] = [];
   let repeats = 0;
 
@@ -40,7 +41,7 @@ export function encodeValues(
     if (repeats === 0 && run.length % 8 === 0 && values[i] === values[i + 1]) {
       // If we have any data in runs we need to encode them
       if (run.length) {
-        buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)] as Uint8Array[]);
+        buffers.push(encodeRunBitpacked(run, opts));
         run = [];
       }
       repeats = 1;
@@ -49,7 +50,7 @@ export function encodeValues(
     } else {
       // If values changes we need to post any previous repeated values
       if (repeats) {
-        buf = Buffer.concat([buf, encodeRunRepeated(values[i - 1], repeats, opts)] as Uint8Array[]);
+        buffers.push(encodeRunRepeated(values[i - 1], repeats, opts));
         repeats = 0;
       }
       run.push(values[i]);
@@ -57,23 +58,19 @@ export function encodeValues(
   }
 
   if (repeats) {
-    buf = Buffer.concat([
-      buf,
-      encodeRunRepeated(values[values.length - 1], repeats, opts)
-    ] as Uint8Array[]);
+    buffers.push(encodeRunRepeated(values[values.length - 1], repeats, opts));
   } else if (run.length) {
-    buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)] as Uint8Array[]);
+    buffers.push(encodeRunBitpacked(run, opts));
   }
 
+  const buf = concatUint8Arrays(buffers);
   if (opts.disableEnvelope) {
     return buf;
   }
 
-  const envelope = Buffer.alloc(buf.length + 4);
-
-  // @ts-ignore buffer polyfill
-  envelope.writeUInt32LE(buf.length, undefined);
-  buf.copy(envelope as Uint8Array, 4);
+  const envelope = new Uint8Array(buf.length + 4);
+  writeUInt32LE(envelope, buf.length, 0);
+  envelope.set(buf, 4);
 
   return envelope;
 }
@@ -94,7 +91,7 @@ export function decodeValues(
 
   let values: number[] = [];
   while (values.length < count) {
-    const header = varint.decode(cursor.buffer, cursor.offset);
+    const header = varint.decode(cursor.buffer as any, cursor.offset);
     cursor.offset += varint.encodingLength(header);
     let decodedValues: number[];
     if (header & 1) {
@@ -163,7 +160,7 @@ function decodeRunRepeated(
   return new Array(count).fill(value);
 }
 
-function encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer {
+function encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Uint8Array {
   // @ts-ignore
   const bitWidth: number = opts.bitWidth;
 
@@ -171,30 +168,27 @@ function encodeRunBitpacked(values: number[], opts: ParquetCodecOptions): Buffer
     values.push(0);
   }
 
-  const buf = Buffer.alloc(Math.ceil(bitWidth * (values.length / 8)));
+  const buf = new Uint8Array(Math.ceil(bitWidth * (values.length / 8)));
   for (let b = 0; b < bitWidth * values.length; b++) {
     if ((values[Math.floor(b / bitWidth)] & (1 << (b % bitWidth))) > 0) {
       buf[Math.floor(b / 8)] |= 1 << (b % 8);
     }
   }
 
-  return Buffer.concat([
-    Buffer.from(varint.encode(((values.length / 8) << 1) | 1)),
-    buf
-  ] as Uint8Array[]);
+  return concatUint8Arrays([Uint8Array.from(varint.encode(((values.length / 8) << 1) | 1)), buf]);
 }
 
-function encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Buffer {
+function encodeRunRepeated(value: number, count: number, opts: ParquetCodecOptions): Uint8Array {
   // @ts-ignore
   const bitWidth: number = opts.bitWidth;
 
-  const buf = Buffer.alloc(Math.ceil(bitWidth / 8));
+  const buf = new Uint8Array(Math.ceil(bitWidth / 8));
 
   for (let i = 0; i < buf.length; i++) {
-    buf.writeUInt8(value & 0xff, i);
+    buf[i] = value & 0xff;
     // eslint-disable-next-line
     value >> 8; //  TODO - this looks wrong
   }
 
-  return Buffer.concat([Buffer.from(varint.encode(count << 1)), buf] as Uint8Array[]);
+  return concatUint8Arrays([Uint8Array.from(varint.encode(count << 1)), buf]);
 }

package/src/parquetjs/compression.ts

@@ -18,20 +18,7 @@ import {
 import {registerJSModules} from '@loaders.gl/loader-utils';
 
 import {ParquetCompression} from './schema/declare';
-
-/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */
-function toBuffer(arrayBuffer: ArrayBuffer): Buffer {
-  return Buffer.from(arrayBuffer);
-}
-
-function toArrayBuffer(buffer: Buffer): ArrayBuffer {
-  // TODO - per docs we should just be able to call buffer.buffer, but there are issues
-  if (Buffer.isBuffer(buffer)) {
-    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
-    return typedArray.slice().buffer;
-  }
-  return buffer;
-}
+import {toArrayBuffer, toUint8Array} from './utils/binary-utils';
 
 // TODO switch to worker compression to avoid bundling...
 
@@ -79,22 +66,20 @@ export const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression
 export async function preloadCompressions(options?: {modules?: {[key: string]: any}}) {
   registerJSModules(options?.modules);
   const compressions = Object.values(PARQUET_COMPRESSION_METHODS);
-  return await Promise.all(
-    compressions.map((compression) => compression.preload(options?.modules))
-  );
+  return await Promise.all(compressions.map(compression => compression.preload(options?.modules)));
 }
 
 /**
  * Deflate a value using compression method `method`
  */
-export async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {
+export async function deflate(method: ParquetCompression, value: Uint8Array): Promise<Uint8Array> {
   const compression = PARQUET_COMPRESSION_METHODS[method];
   if (!compression) {
     throw new Error(`parquet: invalid compression method: ${method}`);
   }
   const inputArrayBuffer = toArrayBuffer(value);
   const compressedArrayBuffer = await compression.compress(inputArrayBuffer);
-  return toBuffer(compressedArrayBuffer);
+  return toUint8Array(compressedArrayBuffer);
 }
 
 /**
@@ -102,106 +87,14 @@ export async function deflate(method: ParquetCompression, value: Buffer): Promis
  */
 export async function decompress(
   method: ParquetCompression,
-  value: Buffer,
+  value: Uint8Array,
   size: number
-): Promise<Buffer> {
+): Promise<Uint8Array> {
   const compression = PARQUET_COMPRESSION_METHODS[method];
   if (!compression) {
     throw new Error(`parquet: invalid compression method: ${method}`);
   }
   const inputArrayBuffer = toArrayBuffer(value);
   const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);
-  return toBuffer(compressedArrayBuffer);
-}
-
-/*
- * Inflate a value using compression method `method`
- */
-export function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {
-  if (!(method in PARQUET_COMPRESSION_METHODS)) {
-    throw new Error(`invalid compression method: ${method}`);
-  }
-  // @ts-ignore
-  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);
-}
-
-/*
-function deflate_identity(value: Buffer): Buffer {
-  return value;
-}
-
-function deflate_gzip(value: Buffer): Buffer {
-  return zlib.gzipSync(value);
-}
-
-function deflate_snappy(value: Buffer): Buffer {
-  return snappyjs.compress(value);
-}
-
-function deflate_lzo(value: Buffer): Buffer {
-  lzo = lzo || Util.load('lzo');
-  return lzo.compress(value);
-}
-
-function deflate_brotli(value: Buffer): Buffer {
-  brotli = brotli || Util.load('brotli');
-  const result = brotli.compress(value, {
-    mode: 0,
-    quality: 8,
-    lgwin: 22
-  });
-  return result ? Buffer.from(result) : Buffer.alloc(0);
-}
-
-function deflate_lz4(value: Buffer): Buffer {
-  lz4js = lz4js || Util.load('lz4js');
-  try {
-    // let result = Buffer.alloc(lz4js.encodeBound(value.length));
-    // const compressedSize = lz4.encodeBlock(value, result);
-    // // remove unnecessary bytes
-    // result = result.slice(0, compressedSize);
-    // return result;
-    return Buffer.from(lz4js.compress(value));
-  } catch (err) {
-    throw err;
-  }
-}
-function inflate_identity(value: Buffer): Buffer {
-  return value;
-}
-
-function inflate_gzip(value: Buffer): Buffer {
-  return zlib.gunzipSync(value);
-}
-
-function inflate_snappy(value: Buffer): Buffer {
-  return snappyjs.uncompress(value);
-}
-
-function inflate_lzo(value: Buffer, size: number): Buffer {
-  lzo = lzo || Util.load('lzo');
-  return lzo.decompress(value, size);
-}
-
-function inflate_lz4(value: Buffer, size: number): Buffer {
-  lz4js = lz4js || Util.load('lz4js');
-  try {
-    // let result = Buffer.alloc(size);
-    // const uncompressedSize = lz4js.decodeBlock(value, result);
-    // // remove unnecessary bytes
-    // result = result.slice(0, uncompressedSize);
-    // return result;
-    return Buffer.from(lz4js.decompress(value, size));
-  } catch (err) {
-    throw err;
-  }
-}
-
-function inflate_brotli(value: Buffer): Buffer {
-  brotli = brotli || Util.load('brotli');
-  if (!value.length) {
-    return Buffer.alloc(0);
-  }
-  return Buffer.from(brotli.decompress(value));
+  return toUint8Array(compressedArrayBuffer);
 }
-*/