@loaders.gl/parquet 4.4.2 → 5.0.0-alpha.1

package/src/parquetjs/schema/types.ts

@@ -5,8 +5,16 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license
 
 /* eslint-disable camelcase */
-import {BSONLoader, BSONWriter} from '@loaders.gl/bson';
+import {BSONWriter, parseBSONSync} from '@loaders.gl/bson';
 import {OriginalType, ParquetField, ParquetType, PrimitiveType} from './declare';
+import {
+  decodeUtf8,
+  encodeUtf8,
+  readUInt32LE,
+  toArrayBuffer,
+  toUint8Array,
+  writeUInt32LE
+} from '../utils/binary-utils';
 
 export interface ParquetTypeKit {
   primitiveType: PrimitiveType;
@@ -207,7 +215,7 @@ function fromPrimitive_BOOLEAN(value: any): boolean {
 
 function toPrimitive_FLOAT(value: any): number {
   const v = parseFloat(value);
-  if (isNaN(v)) {
+  if (Number.isNaN(v)) {
     throw new Error(`invalid value for FLOAT: ${value}`);
   }
   return v;
@@ -215,7 +223,7 @@ function toPrimitive_FLOAT(value: any): number {
 
 function toPrimitive_DOUBLE(value: any): number {
   const v = parseFloat(value);
-  if (isNaN(v)) {
+  if (Number.isNaN(v)) {
     throw new Error(`invalid value for DOUBLE: ${value}`);
   }
   return v;
@@ -223,7 +231,7 @@ function toPrimitive_DOUBLE(value: any): number {
 
 function toPrimitive_INT8(value: any) {
   const v = parseInt(value, 10);
-  if (v < -0x80 || v > 0x7f || isNaN(v)) {
+  if (v < -0x80 || v > 0x7f || Number.isNaN(v)) {
     throw new Error(`invalid value for INT8: ${value}`);
   }
 
@@ -232,7 +240,7 @@ function toPrimitive_INT8(value: any) {
 
 function toPrimitive_UINT8(value: any) {
   const v = parseInt(value, 10);
-  if (v < 0 || v > 0xff || isNaN(v)) {
+  if (v < 0 || v > 0xff || Number.isNaN(v)) {
     throw new Error(`invalid value for UINT8: ${value}`);
   }
 
@@ -241,7 +249,7 @@ function toPrimitive_UINT8(value: any) {
 
 function toPrimitive_INT16(value: any) {
   const v = parseInt(value, 10);
-  if (v < -0x8000 || v > 0x7fff || isNaN(v)) {
+  if (v < -0x8000 || v > 0x7fff || Number.isNaN(v)) {
     throw new Error(`invalid value for INT16: ${value}`);
   }
 
@@ -250,7 +258,7 @@ function toPrimitive_INT16(value: any) {
 
 function toPrimitive_UINT16(value: any) {
   const v = parseInt(value, 10);
-  if (v < 0 || v > 0xffff || isNaN(v)) {
+  if (v < 0 || v > 0xffff || Number.isNaN(v)) {
     throw new Error(`invalid value for UINT16: ${value}`);
   }
 
@@ -259,7 +267,7 @@ function toPrimitive_UINT16(value: any) {
 
 function toPrimitive_INT32(value: any) {
   const v = parseInt(value, 10);
-  if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {
+  if (v < -0x80000000 || v > 0x7fffffff || Number.isNaN(v)) {
     throw new Error(`invalid value for INT32: ${value}`);
   }
 
@@ -269,7 +277,7 @@ function toPrimitive_INT32(value: any) {
 function decimalToPrimitive_INT32(value: number, field: ParquetField): number {
   const primitiveValue = value * 10 ** (field.scale || 0);
   const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);
-  if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {
+  if (v < -0x80000000 || v > 0x7fffffff || Number.isNaN(v)) {
     throw new Error(`invalid value for INT32: ${value}`);
   }
   return v;
@@ -277,7 +285,7 @@ function decimalToPrimitive_INT32(value: number, field: ParquetField): number {
 
 function toPrimitive_UINT32(value: any): number {
   const v = parseInt(value, 10);
-  if (v < 0 || v > 0xffffffffffff || isNaN(v)) {
+  if (v < 0 || v > 0xffffffffffff || Number.isNaN(v)) {
     throw new Error(`invalid value for UINT32: ${value}`);
   }
   return v;
@@ -285,7 +293,7 @@ function toPrimitive_UINT32(value: any): number {
 
 function toPrimitive_INT64(value: any): number {
   const v = parseInt(value, 10);
-  if (isNaN(v)) {
+  if (Number.isNaN(v)) {
     throw new Error(`invalid value for INT64: ${value}`);
   }
   return v;
@@ -294,7 +302,7 @@ function toPrimitive_INT64(value: any): number {
 function decimalToPrimitive_INT64(value: number, field: ParquetField) {
   const primitiveValue = value * 10 ** (field.scale || 0);
   const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);
-  if (isNaN(v)) {
+  if (Number.isNaN(v)) {
     throw new Error(`invalid value for INT64: ${value}`);
   }
 
@@ -303,7 +311,7 @@ function decimalToPrimitive_INT64(value: number, field: ParquetField) {
 
 function toPrimitive_UINT64(value: any) {
   const v = parseInt(value, 10);
-  if (v < 0 || isNaN(v)) {
+  if (v < 0 || Number.isNaN(v)) {
     throw new Error(`invalid value for UINT64: ${value}`);
   }
 
@@ -312,52 +320,52 @@ function toPrimitive_UINT64(value: any) {
 
 function toPrimitive_INT96(value: any) {
   const v = parseInt(value, 10);
-  if (isNaN(v)) {
+  if (Number.isNaN(v)) {
     throw new Error(`invalid value for INT96: ${value}`);
   }
 
   return v;
 }
 
-function toPrimitive_BYTE_ARRAY(value: any): Buffer {
-  return Buffer.from(value);
+function toPrimitive_BYTE_ARRAY(value: any): Uint8Array {
+  return typeof value === 'string' ? encodeUtf8(value) : toUint8Array(value);
 }
 
-function decimalToPrimitive_BYTE_ARRAY(value: any): Buffer {
+function decimalToPrimitive_BYTE_ARRAY(value: any): Uint8Array {
   // TBD
-  return Buffer.from(value);
+  return typeof value === 'string' ? encodeUtf8(value) : toUint8Array(value);
 }
 
-function toPrimitive_UTF8(value: any): Buffer {
-  return Buffer.from(value, 'utf8');
+function toPrimitive_UTF8(value: any): Uint8Array {
+  return encodeUtf8(String(value));
 }
 
 function fromPrimitive_UTF8(value: any): string {
-  return value.toString();
+  return decodeUtf8(toUint8Array(value));
 }
 
-function toPrimitive_JSON(value: any): Buffer {
-  return Buffer.from(JSON.stringify(value));
+function toPrimitive_JSON(value: any): Uint8Array {
+  return encodeUtf8(JSON.stringify(value));
 }
 
 function fromPrimitive_JSON(value: any): unknown {
-  return JSON.parse(value);
+  return JSON.parse(decodeUtf8(toUint8Array(value)));
 }
 
-function toPrimitive_BSON(value: any): Buffer {
+function toPrimitive_BSON(value: any): Uint8Array {
   // @ts-ignore
   const arrayBuffer: ArrayBuffer = BSONWriter.encodeSync?.(value);
-  return Buffer.from(arrayBuffer);
+  return toUint8Array(arrayBuffer);
 }
 
 function fromPrimitive_BSON(value: any) {
-  return BSONLoader.parseSync?.(value);
+  return parseBSONSync(toArrayBuffer(value));
 }
 
 function toPrimitive_TIME_MILLIS(value: any) {
   const v = parseInt(value, 10);
   // eslint-disable-next-line @typescript-eslint/no-loss-of-precision
-  if (v < 0 || v > 0xffffffffffffffff || isNaN(v)) {
+  if (v < 0 || v > 0xffffffffffffffff || Number.isNaN(v)) {
     throw new Error(`invalid value for TIME_MILLIS: ${value}`);
   }
 
@@ -366,7 +374,7 @@ function toPrimitive_TIME_MILLIS(value: any) {
 
 function toPrimitive_TIME_MICROS(value: any): number {
   const v = parseInt(value, 10);
-  if (v < 0 || isNaN(v)) {
+  if (v < 0 || Number.isNaN(v)) {
     throw new Error(`invalid value for TIME_MICROS: ${value}`);
   }
   return v;
@@ -383,7 +391,7 @@ function toPrimitive_DATE(value: any): number {
   /* convert from integer */
   {
     const v = parseInt(value, 10);
-    if (v < 0 || isNaN(v)) {
+    if (v < 0 || Number.isNaN(v)) {
       throw new Error(`invalid value for DATE: ${value}`);
     }
 
@@ -404,7 +412,7 @@ function toPrimitive_TIMESTAMP_MILLIS(value: any): number {
   /* convert from integer */
   {
     const v = parseInt(value, 10);
-    if (v < 0 || isNaN(v)) {
+    if (v < 0 || Number.isNaN(v)) {
       throw new Error(`invalid value for TIMESTAMP_MILLIS: ${value}`);
     }
 
@@ -425,7 +433,7 @@ function toPrimitive_TIMESTAMP_MICROS(value: any) {
   /* convert from integer */
   {
     const v = parseInt(value, 10);
-    if (v < 0 || isNaN(v)) {
+    if (v < 0 || Number.isNaN(v)) {
       throw new Error(`invalid value for TIMESTAMP_MICROS: ${value}`);
     }
 
@@ -444,19 +452,19 @@ function toPrimitive_INTERVAL(value: any) {
     );
   }
 
-  const buf = Buffer.alloc(12);
+  const buf = new Uint8Array(12);
 
-  buf.writeUInt32LE(value.months, 0);
-  buf.writeUInt32LE(value.days, 4);
-  buf.writeUInt32LE(value.milliseconds, 8);
+  writeUInt32LE(buf, value.months, 0);
+  writeUInt32LE(buf, value.days, 4);
+  writeUInt32LE(buf, value.milliseconds, 8);
   return buf;
 }
 
 function fromPrimitive_INTERVAL(value: any) {
-  const buf = Buffer.from(value);
-  const months = buf.readUInt32LE(0);
-  const days = buf.readUInt32LE(4);
-  const millis = buf.readUInt32LE(8);
+  const bytes = toUint8Array(value);
+  const months = readUInt32LE(bytes, 0);
+  const days = readUInt32LE(bytes, 4);
+  const millis = readUInt32LE(bytes, 8);
 
   return {months, days, milliseconds: millis};
 }

package/src/parquetjs/utils/binary-utils.ts

@@ -0,0 +1,127 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+const TEXT_DECODER = new TextDecoder();
+const TEXT_ENCODER = new TextEncoder();
+
+/** Returns bytes as an exact Uint8Array view. */
+export function toUint8Array(binary: ArrayBuffer | ArrayBufferView): Uint8Array {
+  if (binary instanceof Uint8Array) {
+    return binary;
+  }
+  if (ArrayBuffer.isView(binary)) {
+    return new Uint8Array(binary.buffer, binary.byteOffset, binary.byteLength);
+  }
+  return new Uint8Array(binary);
+}
+
+/** Returns bytes as a copied ArrayBuffer with no extra prefix/suffix bytes. */
+export function toArrayBuffer(binary: ArrayBuffer | ArrayBufferView): ArrayBuffer {
+  const bytes = toUint8Array(binary);
+  return bytes.slice().buffer;
+}
+
+/** Returns a copied Uint8Array. */
+export function copyUint8Array(binary: Uint8Array): Uint8Array {
+  return binary.slice();
+}
+
+/** Decode UTF-8 bytes. */
+export function decodeUtf8(binary: Uint8Array): string {
+  return TEXT_DECODER.decode(binary);
+}
+
+/** Encode text as UTF-8 bytes. */
+export function encodeUtf8(text: string): Uint8Array {
+  return TEXT_ENCODER.encode(text);
+}
+
+/** Decode an ASCII/UTF-8 range from bytes. */
+export function decodeString(
+  binary: Uint8Array,
+  start: number = 0,
+  end: number = binary.length
+): string {
+  return decodeUtf8(binary.subarray(start, end));
+}
+
+function getDataView(binary: Uint8Array): DataView {
+  return new DataView(binary.buffer, binary.byteOffset, binary.byteLength);
+}
+
+/** Read signed little-endian 32-bit integer. */
+export function readInt32LE(binary: Uint8Array, offset: number): number {
+  return getDataView(binary).getInt32(offset, true);
+}
+
+/** Read unsigned little-endian 32-bit integer. */
+export function readUInt32LE(binary: Uint8Array, offset: number): number {
+  return getDataView(binary).getUint32(offset, true);
+}
+
+/** Read signed little-endian 64-bit integer as number. */
+export function readInt64LE(binary: Uint8Array, offset: number): number {
+  return Number(getDataView(binary).getBigInt64(offset, true));
+}
+
+/** Read little-endian 32-bit float. */
+export function readFloatLE(binary: Uint8Array, offset: number): number {
+  return getDataView(binary).getFloat32(offset, true);
+}
+
+/** Read little-endian 64-bit float. */
+export function readDoubleLE(binary: Uint8Array, offset: number): number {
+  return getDataView(binary).getFloat64(offset, true);
+}
+
+/** Concatenate bytes into one exact Uint8Array. */
+export function concatUint8Arrays(chunks: Uint8Array[]): Uint8Array {
+  let byteLength = 0;
+  for (const chunk of chunks) {
+    byteLength += chunk.length;
+  }
+
+  const result = new Uint8Array(byteLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return result;
+}
+
+/** Write signed little-endian 16-bit integer. */
+export function writeInt16LE(binary: Uint8Array, value: number, offset: number): void {
+  getDataView(binary).setInt16(offset, value, true);
+}
+
+/** Write unsigned little-endian 16-bit integer. */
+export function writeUInt16LE(binary: Uint8Array, value: number, offset: number): void {
+  getDataView(binary).setUint16(offset, value, true);
+}
+
+/** Write signed little-endian 32-bit integer. */
+export function writeInt32LE(binary: Uint8Array, value: number, offset: number): void {
+  getDataView(binary).setInt32(offset, value, true);
+}
+
+/** Write unsigned little-endian 32-bit integer. */
+export function writeUInt32LE(binary: Uint8Array, value: number, offset: number): void {
+  getDataView(binary).setUint32(offset, value, true);
+}
+
+/** Write signed little-endian 64-bit integer. */
+export function writeInt64LE(binary: Uint8Array, value: number | bigint, offset: number): void {
+  getDataView(binary).setBigInt64(offset, BigInt(value), true);
+}
+
+/** Write little-endian 32-bit float. */
+export function writeFloatLE(binary: Uint8Array, value: number, offset: number): void {
+  getDataView(binary).setFloat32(offset, value, true);
+}
+
+/** Write little-endian 64-bit float. */
+export function writeDoubleLE(binary: Uint8Array, value: number, offset: number): void {
+  getDataView(binary).setFloat64(offset, value, true);
+}

package/src/parquetjs/utils/file-utils.ts

@@ -19,9 +19,9 @@ export interface WriteStreamOptions {
   start?: number;
 }
 
-export function oswrite(os: stream.Writable, buf: Buffer): Promise<void> {
+export function oswrite(os: stream.Writable, buf: Uint8Array): Promise<void> {
   return new Promise((resolve, reject) => {
-    os.write(buf, (err) => {
+    os.write(buf, err => {
       if (err) {
         reject(err);
       } else {
@@ -46,7 +46,7 @@ export function osclose(os: stream.Writable): Promise<void> {
 export function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {
   return new Promise((resolve, reject) => {
     const outputStream = fs.createWriteStream(path, opts as any);
-    outputStream.once('open', (fd) => resolve(outputStream));
-    outputStream.once('error', (err) => reject(err));
+    outputStream.once('open', _fd => resolve(outputStream));
+    outputStream.once('error', err => reject(err));
   });
 }

package/src/parquetjs/utils/read-utils.ts

@@ -4,44 +4,29 @@
 // Copyright (c) 2017 ironSource Ltd.
 // Forked from https://github.com/kbajalc/parquets under MIT license
 
-import {
-  TBufferedTransport,
-  TCompactProtocol,
-  TFramedTransport,
-  FileMetaData,
-  PageHeader
-} from '../parquet-thrift/index';
-
-class UFramedTransport extends TFramedTransport {
-  public readPos: number = 0;
-}
+import {FileMetaData, PageHeader} from '../parquet-thrift/index';
+import {Uint8ArrayCompactProtocol} from './uint8-array-compact-protocol';
+import {Uint8ArrayCompactProtocolWriter} from './uint8-array-compact-protocol-writer';
+import {Uint8ArrayTransport} from './uint8-array-transport';
 
 /**
  * Helper function that serializes a thrift object into a buffer
  */
-export function serializeThrift(obj: any): Buffer {
-  const output: Buffer[] = [];
-
-  const transport = new TBufferedTransport(undefined, (buf) => {
-    output.push(buf as unknown as Buffer);
-  });
-
-  const protocol = new TCompactProtocol(transport);
-  obj.write(protocol);
-  transport.flush();
-
-  return Buffer.concat(output as Uint8Array[]);
+export function serializeThrift(obj: any): Uint8Array {
+  const protocol = new Uint8ArrayCompactProtocolWriter();
+  obj.write(protocol as any);
+  return protocol.getBytes();
 }
 
-export function decodeThrift(obj: any, buf: Buffer, offset?: number) {
+export function decodeThrift(obj: any, buf: Uint8Array, offset?: number) {
   if (!offset) {
     // tslint:disable-next-line:no-parameter-reassignment
     offset = 0;
   }
 
-  const transport = new UFramedTransport(buf);
+  const transport = new Uint8ArrayTransport(buf);
   transport.readPos = offset;
-  const protocol = new TCompactProtocol(transport);
+  const protocol = new Uint8ArrayCompactProtocol(transport);
   obj.read(protocol);
   return transport.readPos - offset;
 }
@@ -58,29 +43,29 @@ export function getThriftEnum(klass: any, value: number | string): string {
   throw new Error('Invalid ENUM value');
 }
 
-export function decodeFileMetadata(buf: Buffer, offset?: number) {
+export function decodeFileMetadata(buf: Uint8Array, offset?: number) {
   if (!offset) {
     // tslint:disable-next-line:no-parameter-reassignment
     offset = 0;
   }
 
-  const transport = new UFramedTransport(buf);
+  const transport = new Uint8ArrayTransport(buf);
   transport.readPos = offset;
-  const protocol = new TCompactProtocol(transport);
-  const metadata = FileMetaData.read(protocol);
+  const protocol = new Uint8ArrayCompactProtocol(transport);
+  const metadata = FileMetaData.read(protocol as any);
   return {length: transport.readPos - offset, metadata};
 }
 
-export function decodePageHeader(buf: Buffer, offset?: number) {
+export function decodePageHeader(buf: Uint8Array, offset?: number) {
   if (!offset) {
     // tslint:disable-next-line:no-parameter-reassignment
     offset = 0;
   }
 
-  const transport = new UFramedTransport(buf);
+  const transport = new Uint8ArrayTransport(buf);
   transport.readPos = offset;
-  const protocol = new TCompactProtocol(transport);
-  const pageHeader = PageHeader.read(protocol);
+  const protocol = new Uint8ArrayCompactProtocol(transport);
+  const pageHeader = PageHeader.read(protocol as any);
   return {length: transport.readPos - offset, pageHeader};
 }