@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/src/parquetjs/utils/file-utils.ts

@@ -19,9 +19,9 @@ export interface WriteStreamOptions {
   start?: number;
 }
 
-export function oswrite(os: stream.Writable, buf: Buffer): Promise<void> {
+export function oswrite(os: stream.Writable, buf: Uint8Array): Promise<void> {
   return new Promise((resolve, reject) => {
-    os.write(buf, (err) => {
+    os.write(buf, err => {
       if (err) {
         reject(err);
       } else {
@@ -46,7 +46,7 @@ export function osclose(os: stream.Writable): Promise<void> {
 export function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {
   return new Promise((resolve, reject) => {
     const outputStream = fs.createWriteStream(path, opts as any);
-    outputStream.once('open', (fd) => resolve(outputStream));
-    outputStream.once('error', (err) => reject(err));
+    outputStream.once('open', _fd => resolve(outputStream));
+    outputStream.once('error', err => reject(err));
   });
 }

package/src/parquetjs/utils/read-utils.ts

@@ -4,83 +4,79 @@
 // Copyright (c) 2017 ironSource Ltd.
 // Forked from https://github.com/kbajalc/parquets under MIT license
 
-import {
-  TBufferedTransport,
-  TCompactProtocol,
-  TFramedTransport,
-  FileMetaData,
-  PageHeader
-} from '../parquet-thrift/index';
-
-class UFramedTransport extends TFramedTransport {
-  public readPos: number = 0;
-}
+import {FileMetaData, PageHeader} from '../parquet-thrift/index';
+import {Uint8ArrayCompactProtocol} from './uint8-array-compact-protocol';
+import {Uint8ArrayCompactProtocolWriter} from './uint8-array-compact-protocol-writer';
+import {Uint8ArrayTransport} from './uint8-array-transport';
 
 /**
  * Helper function that serializes a thrift object into a buffer
  */
-export function serializeThrift(obj: any): Buffer {
-  const output: Buffer[] = [];
-
-  const transport = new TBufferedTransport(undefined, (buf) => {
-    output.push(buf as unknown as Buffer);
-  });
-
-  const protocol = new TCompactProtocol(transport);
-  obj.write(protocol);
-  transport.flush();
-
-  return Buffer.concat(output as Uint8Array[]);
+export function serializeThrift(obj: any): Uint8Array {
+  const protocol = new Uint8ArrayCompactProtocolWriter();
+  obj.write(protocol as any);
+  return protocol.getBytes();
 }
 
-export function decodeThrift(obj: any, buf: Buffer, offset?: number) {
+export function decodeThrift(obj: any, buf: Uint8Array, offset?: number) {
   if (!offset) {
     // tslint:disable-next-line:no-parameter-reassignment
     offset = 0;
   }
 
-  const transport = new UFramedTransport(buf);
+  const transport = new Uint8ArrayTransport(buf);
   transport.readPos = offset;
-  const protocol = new TCompactProtocol(transport);
+  const protocol = new Uint8ArrayCompactProtocol(transport);
   obj.read(protocol);
   return transport.readPos - offset;
 }
 
 /**
- * FIXME not ideal that this is linear
+ * Returns the generated TypeScript enum member for a Thrift value.
+ * @param thriftEnum generated numeric enum object
+ * @param value serialized enum value
+ * @returns enum member name
  */
-export function getThriftEnum(klass: any, value: number | string): string {
-  for (const k in klass) {
-    if (klass[k] === value) {
-      return k;
+export function getThriftEnum(thriftEnum: object, value: number | string): string {
+  const enumValues = thriftEnum as Record<string | number, number | string>;
+  if (typeof value === 'number') {
+    const enumName = enumValues[value];
+    if (typeof enumName === 'string') {
+      return enumName;
+    }
+  }
+
+  for (const enumName in enumValues) {
+    if (enumValues[enumName] === value) {
+      return enumName;
     }
   }
   throw new Error('Invalid ENUM value');
 }
 
-export function decodeFileMetadata(buf: Buffer, offset?: number) {
+export function decodeFileMetadata(buf: Uint8Array, offset?: number) {
   if (!offset) {
     // tslint:disable-next-line:no-parameter-reassignment
     offset = 0;
   }
 
-  const transport = new UFramedTransport(buf);
+  const transport = new Uint8ArrayTransport(buf);
   transport.readPos = offset;
-  const protocol = new TCompactProtocol(transport);
-  const metadata = FileMetaData.read(protocol);
+  const protocol = new Uint8ArrayCompactProtocol(transport);
+  const metadata = FileMetaData.read(protocol as any);
   return {length: transport.readPos - offset, metadata};
 }
 
-export function decodePageHeader(buf: Buffer, offset?: number) {
+export function decodePageHeader(buf: Uint8Array, offset?: number) {
   if (!offset) {
     // tslint:disable-next-line:no-parameter-reassignment
     offset = 0;
   }
 
-  const transport = new UFramedTransport(buf);
+  const transport = new Uint8ArrayTransport(buf);
   transport.readPos = offset;
-  const protocol = new TCompactProtocol(transport);
-  const pageHeader = PageHeader.read(protocol);
+  const protocol = new Uint8ArrayCompactProtocol(transport);
+  const pageHeader = PageHeader.read(protocol as any);
   return {length: transport.readPos - offset, pageHeader};
 }
 
@@ -95,26 +91,31 @@ export function getBitWidth(val: number): number {
   return Math.ceil(Math.log2(val + 1));
 }
 
-// Supports MQTT path wildcards
-// + all immediate children
-// # all descendents
+/**
+ * Finds the selected field path that contains a Parquet leaf path.
+ * A selected parent includes all descendant leaves. MQTT-style `+` and `#` path wildcards are
+ * retained for internal callers (`+` matches one segment and `#` matches all remaining segments).
+ */
 export function fieldIndexOf(arr: string[][], elem: string[]): number {
-  for (let j = 0; j < arr.length; j++) {
-    if (arr[j].length > elem.length) {
-      continue; // eslint-disable-line no-continue
-    }
-    let m = true;
-    for (let i = 0; i < elem.length; i++) {
-      if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {
-        continue; // eslint-disable-line no-continue
+  for (let fieldIndex = 0; fieldIndex < arr.length; fieldIndex++) {
+    const selectedPath = arr[fieldIndex];
+    let matches = true;
+    for (let pathIndex = 0; pathIndex < selectedPath.length; pathIndex++) {
+      const selectedSegment = selectedPath[pathIndex];
+      if (selectedSegment === '#') {
+        return fieldIndex;
       }
-      if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {
-        continue; // eslint-disable-line no-continue
+      if (
+        pathIndex >= elem.length ||
+        (selectedSegment !== '+' && selectedSegment !== elem[pathIndex])
+      ) {
+        matches = false;
+        break;
       }
-      m = false;
-      break;
     }
-    if (m) return j;
+    if (matches) {
+      return fieldIndex;
+    }
   }
   return -1;
 }

package/src/parquetjs/utils/thrift-runtime.ts

@@ -0,0 +1,109 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Minimal Thrift runtime types needed by the generated Parquet metadata classes. */
+export namespace Thrift {
+  /** Thrift wire types used by the Parquet compact protocol. */
+  export enum Type {
+    STOP = 0,
+    VOID = 1,
+    BOOL = 2,
+    BYTE = 3,
+    DOUBLE = 4,
+    I16 = 6,
+    I32 = 8,
+    I64 = 10,
+    STRING = 11,
+    STRUCT = 12,
+    MAP = 13,
+    SET = 14,
+    LIST = 15,
+    UTF8 = 16,
+    UTF16 = 17
+  }
+
+  /** Protocol failure categories retained for generated-code compatibility. */
+  export enum TProtocolExceptionType {
+    UNKNOWN = 0,
+    INVALID_DATA = 1,
+    NEGATIVE_SIZE = 2,
+    SIZE_LIMIT = 3,
+    BAD_VERSION = 4,
+    NOT_IMPLEMENTED = 5,
+    DEPTH_LIMIT = 6
+  }
+
+  /** Error raised when generated metadata cannot be encoded or decoded. */
+  export class TProtocolException extends Error {
+    /** Thrift protocol error category. */
+    readonly type: TProtocolExceptionType;
+
+    /** Creates a protocol exception without importing the Node-oriented thrift package. */
+    constructor(type: TProtocolExceptionType, message = 'Thrift protocol error') {
+      super(message);
+      this.name = 'TProtocolException';
+      this.type = type;
+    }
+  }
+}
+
+/** Thrift struct descriptor returned by compact-protocol readers. */
+export type TStruct = {fname: string};
+
+/** Thrift field descriptor returned by compact-protocol readers. */
+export type TField = {fname: string; ftype: Thrift.Type; fid: number};
+
+/** Thrift map descriptor returned by compact-protocol readers. */
+export type TMap = {ktype: Thrift.Type; vtype: Thrift.Type; size: number};
+
+/** Thrift list descriptor returned by compact-protocol readers. */
+export type TList = {etype: Thrift.Type; size: number};
+
+/** Thrift set descriptor returned by compact-protocol readers. */
+export type TSet = TList;
+
+/**
+ * Compact-protocol surface consumed by the generated Parquet metadata classes.
+ * Binary values use typed-array views and remain browser-native.
+ */
+export interface TProtocol {
+  writeStructBegin(name: string): void;
+  writeStructEnd(): void;
+  writeFieldBegin(name: string, fieldType: Thrift.Type, fieldId: number): void;
+  writeFieldEnd(): void;
+  writeFieldStop(): void;
+  writeMapBegin(keyType: Thrift.Type, valueType: Thrift.Type, size: number): void;
+  writeMapEnd(): void;
+  writeListBegin(elementType: Thrift.Type, size: number): void;
+  writeListEnd(): void;
+  writeSetBegin(elementType: Thrift.Type, size: number): void;
+  writeSetEnd(): void;
+  writeBool(value: boolean): void;
+  writeByte(value: number): void;
+  writeI16(value: number): void;
+  writeI32(value: number): void;
+  writeI64(value: unknown): void;
+  writeDouble(value: number): void;
+  writeString(value: string): void;
+  writeBinary(value: ArrayBuffer | ArrayBufferView): void;
+  readStructBegin(): TStruct;
+  readStructEnd(): void;
+  readFieldBegin(): TField;
+  readFieldEnd(): void;
+  readMapBegin(): TMap;
+  readMapEnd(): void;
+  readListBegin(): TList;
+  readListEnd(): void;
+  readSetBegin(): TSet;
+  readSetEnd(): void;
+  readBool(): boolean;
+  readByte(): number;
+  readI16(): number;
+  readI32(): number;
+  readI64(): any;
+  readDouble(): number;
+  readString(): string;
+  readBinary(): Uint8Array;
+  skip(type: Thrift.Type): void;
+}

package/src/parquetjs/utils/uint8-array-compact-protocol-writer.ts

@@ -0,0 +1,254 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import * as thrift from './thrift-runtime';
+import {concatUint8Arrays, encodeUtf8, toUint8Array, writeDoubleLE} from './binary-utils';
+
+const COMPACT_TYPE_STOP = 0x00;
+const COMPACT_TYPE_BOOLEAN_TRUE = 0x01;
+const COMPACT_TYPE_BOOLEAN_FALSE = 0x02;
+const COMPACT_TYPE_BYTE = 0x03;
+const COMPACT_TYPE_I16 = 0x04;
+const COMPACT_TYPE_I32 = 0x05;
+const COMPACT_TYPE_I64 = 0x06;
+const COMPACT_TYPE_DOUBLE = 0x07;
+const COMPACT_TYPE_BINARY = 0x08;
+const COMPACT_TYPE_LIST = 0x09;
+const COMPACT_TYPE_SET = 0x0a;
+const COMPACT_TYPE_MAP = 0x0b;
+const COMPACT_TYPE_STRUCT = 0x0c;
+
+type Int64Like = number | bigint | {toString(radix?: number): string; valueOf?: () => number};
+
+/** Write-only Thrift compact protocol that assembles exact Uint8Array output. */
+export class Uint8ArrayCompactProtocolWriter {
+  /** Serialized byte chunks accumulated by thrift write calls. */
+  private readonly chunks: Uint8Array[] = [];
+
+  /** Field id at the current nesting depth, used to emit compact field-id deltas. */
+  private lastFieldId = 0;
+
+  /** Parent struct field ids restored when nested structs end. */
+  private readonly lastFieldIds: number[] = [];
+
+  /** Boolean field header waiting for the following boolean payload. */
+  private pendingBooleanField: {fid: number} | null = null;
+
+  /** Enter a struct and reset the delta-encoded field id state. */
+  writeStructBegin(_name: string): void {
+    this.lastFieldIds.push(this.lastFieldId);
+    this.lastFieldId = 0;
+  }
+
+  /** Exit a struct and restore the parent field id state. */
+  writeStructEnd(): void {
+    this.lastFieldId = this.lastFieldIds.pop() || 0;
+  }
+
+  /** Write a field header, deferring boolean fields until their value is known. */
+  writeFieldBegin(_name: string, fieldType: thrift.Thrift.Type, fid: number): void {
+    if (fieldType === thrift.Thrift.Type.BOOL) {
+      this.pendingBooleanField = {fid};
+      return;
+    }
+
+    this.writeFieldHeader(this.getCompactType(fieldType), fid);
+  }
+
+  /** Complete the current field; compact protocol has no field-end bytes. */
+  writeFieldEnd(): void {}
+
+  /** Write the stop marker that terminates a thrift struct. */
+  writeFieldStop(): void {
+    this.writeByteDirect(COMPACT_TYPE_STOP);
+  }
+
+  /** Write a map header with compact key/value element types. */
+  writeMapBegin(keyType: thrift.Thrift.Type, valueType: thrift.Thrift.Type, size: number): void {
+    if (size === 0) {
+      this.writeByteDirect(0);
+      return;
+    }
+
+    this.writeVarint32(size);
+    this.writeByteDirect((this.getCompactType(keyType) << 4) | this.getCompactType(valueType));
+  }
+
+  /** Complete the current map; compact protocol has no map-end bytes. */
+  writeMapEnd(): void {}
+
+  /** Write a list header with compact element type and element count. */
+  writeListBegin(elementType: thrift.Thrift.Type, size: number): void {
+    this.writeCollectionBegin(elementType, size);
+  }
+
+  /** Complete the current list; compact protocol has no list-end bytes. */
+  writeListEnd(): void {}
+
+  /** Write a set header with compact element type and element count. */
+  writeSetBegin(elementType: thrift.Thrift.Type, size: number): void {
+    this.writeCollectionBegin(elementType, size);
+  }
+
+  /** Complete the current set; compact protocol has no set-end bytes. */
+  writeSetEnd(): void {}
+
+  /** Write a boolean, either inline in its field header or as a value byte. */
+  writeBool(value: boolean): void {
+    const compactType = value ? COMPACT_TYPE_BOOLEAN_TRUE : COMPACT_TYPE_BOOLEAN_FALSE;
+    if (this.pendingBooleanField) {
+      this.writeFieldHeader(compactType, this.pendingBooleanField.fid);
+      this.pendingBooleanField = null;
+      return;
+    }
+
+    this.writeByteDirect(compactType);
+  }
+
+  /** Write a signed byte. */
+  writeByte(value: number): void {
+    this.writeByteDirect(value);
+  }
+
+  /** Write a signed 16-bit integer as zig-zag varint. */
+  writeI16(value: number): void {
+    this.writeI32(value);
+  }
+
+  /** Write a signed 32-bit integer as zig-zag varint. */
+  writeI32(value: number): void {
+    this.writeVarint32(this.encodeZigZag32(value));
+  }
+
+  /** Write a signed 64-bit-like value as zig-zag varint. */
+  writeI64(value: Int64Like): void {
+    this.writeVarint64(this.encodeZigZag64(this.toBigInt(value)));
+  }
+
+  /** Write a little-endian 64-bit float. */
+  writeDouble(value: number): void {
+    const bytes = new Uint8Array(8);
+    writeDoubleLE(bytes, value, 0);
+    this.writeBytes(bytes);
+  }
+
+  /** Write a length-prefixed UTF-8 string. */
+  writeString(value: string): void {
+    this.writeBinary(encodeUtf8(value));
+  }
+
+  /** Write a length-prefixed binary payload from any typed-array view. */
+  writeBinary(value: ArrayBuffer | ArrayBufferView): void {
+    const bytes = toUint8Array(value);
+    this.writeVarint32(bytes.length);
+    this.writeBytes(bytes);
+  }
+
+  /** Return all serialized chunks as one contiguous byte array. */
+  getBytes(): Uint8Array {
+    return concatUint8Arrays(this.chunks);
+  }
+
+  /** Write a compact collection header for list or set values. */
+  private writeCollectionBegin(elementType: thrift.Thrift.Type, size: number): void {
+    const compactType = this.getCompactType(elementType);
+    if (size <= 14) {
+      this.writeByteDirect((size << 4) | compactType);
+      return;
+    }
+
+    this.writeByteDirect(0xf0 | compactType);
+    this.writeVarint32(size);
+  }
+
+  /** Write a compact field header using a field-id delta when possible. */
+  private writeFieldHeader(compactType: number, fid: number): void {
+    const fieldIdDelta = fid - this.lastFieldId;
+    if (fieldIdDelta > 0 && fieldIdDelta <= 15) {
+      this.writeByteDirect((fieldIdDelta << 4) | compactType);
+    } else {
+      this.writeByteDirect(compactType);
+      this.writeI16(fid);
+    }
+    this.lastFieldId = fid;
+  }
+
+  /** Append an already serialized byte range without copying. */
+  private writeBytes(bytes: Uint8Array): void {
+    this.chunks.push(bytes);
+  }
+
+  /** Append one byte to the serialized output. */
+  private writeByteDirect(value: number): void {
+    this.chunks.push(new Uint8Array([value & 0xff]));
+  }
+
+  /** Write an unsigned 32-bit value as a compact varint. */
+  private writeVarint32(value: number): void {
+    this.writeVarint64(BigInt(value >>> 0));
+  }
+
+  /** Write an unsigned 64-bit value as a compact varint. */
+  private writeVarint64(value: bigint): void {
+    let nextValue = value;
+    while (true) {
+      if ((nextValue & ~0x7fn) === 0n) {
+        this.writeByteDirect(Number(nextValue));
+        return;
+      }
+
+      this.writeByteDirect(Number((nextValue & 0x7fn) | 0x80n));
+      nextValue >>= 7n;
+    }
+  }
+
+  /** Encode a signed 32-bit integer as an unsigned zig-zag value. */
+  private encodeZigZag32(value: number): number {
+    return ((value << 1) ^ (value >> 31)) >>> 0;
+  }
+
+  /** Encode a signed 64-bit integer as an unsigned zig-zag value. */
+  private encodeZigZag64(value: bigint): bigint {
+    return (value << 1n) ^ (value >> 63n);
+  }
+
+  /** Normalize thrift int64-like values to bigint before zig-zag encoding. */
+  private toBigInt(value: Int64Like): bigint {
+    return typeof value === 'bigint' || typeof value === 'number'
+      ? BigInt(value)
+      : BigInt(value.toString());
+  }
+
+  /** Convert a generated-thrift type to a compact protocol wire type. */
+  private getCompactType(thriftType: thrift.Thrift.Type): number {
+    switch (thriftType) {
+      case thrift.Thrift.Type.STOP:
+        return COMPACT_TYPE_STOP;
+      case thrift.Thrift.Type.BOOL:
+        return COMPACT_TYPE_BOOLEAN_TRUE;
+      case thrift.Thrift.Type.BYTE:
+        return COMPACT_TYPE_BYTE;
+      case thrift.Thrift.Type.I16:
+        return COMPACT_TYPE_I16;
+      case thrift.Thrift.Type.I32:
+        return COMPACT_TYPE_I32;
+      case thrift.Thrift.Type.I64:
+        return COMPACT_TYPE_I64;
+      case thrift.Thrift.Type.DOUBLE:
+        return COMPACT_TYPE_DOUBLE;
+      case thrift.Thrift.Type.STRING:
+        return COMPACT_TYPE_BINARY;
+      case thrift.Thrift.Type.LIST:
+        return COMPACT_TYPE_LIST;
+      case thrift.Thrift.Type.SET:
+        return COMPACT_TYPE_SET;
+      case thrift.Thrift.Type.MAP:
+        return COMPACT_TYPE_MAP;
+      case thrift.Thrift.Type.STRUCT:
+        return COMPACT_TYPE_STRUCT;
+      default:
+        throw new Error(`Unsupported thrift type: ${thriftType}`);
+    }
+  }
+}