@loaders.gl/parquet 3.0.12 → 3.1.0-alpha.4

package/src/parquetjs/schema/declare.ts

@@ -1,10 +1,23 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+import Int64 from 'node-int64';
+import type {PageHeader} from '../parquet-thrift';
 
-export type ParquetCodec = 'PLAIN' | 'RLE';
-export type ParquetCompression = 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI' | 'LZ4';
+export type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';
+export type ParquetCompression =
+  | 'UNCOMPRESSED'
+  | 'GZIP'
+  | 'SNAPPY'
+  | 'LZO'
+  | 'BROTLI'
+  | 'LZ4'
+  | 'LZ4_RAW'
+  | 'ZSTD';
 export type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';
 export type ParquetType = PrimitiveType | OriginalType;
 
+/**
+ * Physical type
+ */
 export type PrimitiveType =
   // Base Types
   | 'BOOLEAN' // 0
@@ -16,6 +29,9 @@ export type PrimitiveType =
   | 'BYTE_ARRAY' // 6,
   | 'FIXED_LEN_BYTE_ARRAY'; // 7
 
+/**
+ * Logical type
+ */
 export type OriginalType =
   // Converted Types
   | 'UTF8' // 0
@@ -24,6 +40,10 @@ export type OriginalType =
   // | 'LIST' // 3
   // | 'ENUM' // 4
   // | 'DECIMAL' // 5
+  | 'DECIMAL_INT32' // 5
+  | 'DECIMAL_INT64' // 5
+  | 'DECIMAL_BYTE_ARRAY' // 5
+  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5
   | 'DATE' // 6
   | 'TIME_MILLIS' // 7
   | 'TIME_MICROS' // 8
@@ -41,6 +61,8 @@ export type OriginalType =
   | 'BSON' // 20
   | 'INTERVAL'; // 21
 
+export type ParquetDictionary = string[];
+
 export interface SchemaDefinition {
   [string: string]: FieldDefinition;
 }
@@ -48,6 +70,8 @@ export interface SchemaDefinition {
 export interface FieldDefinition {
   type?: ParquetType;
   typeLength?: number;
+  presision?: number;
+  scale?: number;
   encoding?: ParquetCodec;
   compression?: ParquetCompression;
   optional?: boolean;
@@ -63,6 +87,8 @@ export interface ParquetField {
   originalType?: OriginalType;
   repetitionType: RepetitionType;
   typeLength?: number;
+  presision?: number;
+  scale?: number;
   encoding?: ParquetCodec;
   compression?: ParquetCompression;
   rLevelMax: number;
@@ -72,11 +98,31 @@ export interface ParquetField {
   fields?: Record<string, ParquetField>;
 }
 
+export interface ParquetOptions {
+  type: ParquetType;
+  rLevelMax: number;
+  dLevelMax: number;
+  compression: ParquetCompression;
+  column: ParquetField;
+  numValues?: Int64;
+  dictionary?: ParquetDictionary;
+}
+
 export interface ParquetData {
   dlevels: number[];
   rlevels: number[];
   values: any[];
   count: number;
+  pageHeaders: PageHeader[];
+}
+
+export interface ParquetPageData {
+  dlevels: number[];
+  rlevels: number[];
+  values: any[];
+  count: number;
+  dictionary?: ParquetDictionary;
+  pageHeader: PageHeader;
 }
 
 export interface ParquetRecord {

package/src/parquetjs/schema/schema.ts

@@ -173,6 +173,8 @@ function buildFields(
       encoding: opts.encoding,
       compression: opts.compression,
       typeLength: opts.typeLength || typeDef.typeLength,
+      presision: opts.presision,
+      scale: opts.scale,
       rLevelMax,
       dLevelMax
     };

package/src/parquetjs/schema/shred.ts

@@ -13,6 +13,7 @@ export function shredBuffer(schema: ParquetSchema): ParquetBuffer {
       dlevels: [],
       rlevels: [],
       values: [],
+      pageHeaders: [],
       count: 0
     };
   }
@@ -203,7 +204,8 @@ function materializeColumn(
       const value = Types.fromPrimitive(
         // @ts-ignore
         field.originalType || field.primitiveType,
-        data.values[vIndex]
+        data.values[vIndex],
+        field
       );
       vIndex++;
       if (field.repetitionType === 'REPEATED') {

package/src/parquetjs/schema/types.ts

@@ -1,7 +1,7 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 /* eslint-disable camelcase */
 import BSON from 'bson';
-import {OriginalType, ParquetType, PrimitiveType} from './declare';
+import {OriginalType, ParquetField, ParquetType, PrimitiveType} from './declare';
 
 export interface ParquetTypeKit {
   primitiveType: PrimitiveType;
@@ -137,6 +137,30 @@ export const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {
     typeLength: 12,
     toPrimitive: toPrimitive_INTERVAL,
     fromPrimitive: fromPrimitive_INTERVAL
+  },
+  DECIMAL_INT32: {
+    primitiveType: 'INT32',
+    originalType: 'DECIMAL_INT32',
+    toPrimitive: decimalToPrimitive_INT32,
+    fromPrimitive: decimalFromPrimitive_INT
+  },
+  DECIMAL_INT64: {
+    primitiveType: 'INT64',
+    originalType: 'DECIMAL_INT64',
+    toPrimitive: decimalToPrimitive_INT64,
+    fromPrimitive: decimalFromPrimitive_INT
+  },
+  DECIMAL_BYTE_ARRAY: {
+    primitiveType: 'BYTE_ARRAY',
+    originalType: 'DECIMAL_BYTE_ARRAY',
+    toPrimitive: decimalToPrimitive_BYTE_ARRAY,
+    fromPrimitive: decimalFromPrimitive_BYTE_ARRAY
+  },
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: {
+    primitiveType: 'FIXED_LEN_BYTE_ARRAY',
+    originalType: 'DECIMAL_FIXED_LEN_BYTE_ARRAY',
+    toPrimitive: decimalToPrimitive_BYTE_ARRAY,
+    fromPrimitive: decimalFromPrimitive_BYTE_ARRAY
   }
 };
 
@@ -144,25 +168,25 @@ export const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {
  * Convert a value from it's native representation to the internal/underlying
  * primitive type
  */
-export function toPrimitive(type: ParquetType, value: any) {
+export function toPrimitive(type: ParquetType, value: any, field?: ParquetField) {
   if (!(type in PARQUET_LOGICAL_TYPES)) {
     throw new Error(`invalid type: ${type}`);
   }
 
-  return PARQUET_LOGICAL_TYPES[type].toPrimitive(value);
+  return PARQUET_LOGICAL_TYPES[type].toPrimitive(value, field);
 }
 
 /**
  * Convert a value from it's internal/underlying primitive representation to
  * the native representation
  */
-export function fromPrimitive(type: ParquetType, value: any) {
+export function fromPrimitive(type: ParquetType, value: any, field?: ParquetField) {
   if (!(type in PARQUET_LOGICAL_TYPES)) {
     throw new Error(`invalid type: ${type}`);
   }
 
   if ('fromPrimitive' in PARQUET_LOGICAL_TYPES[type]) {
-    return PARQUET_LOGICAL_TYPES[type].fromPrimitive?.(value);
+    return PARQUET_LOGICAL_TYPES[type].fromPrimitive?.(value, field);
     // tslint:disable-next-line:no-else-after-return
   }
   return value;
@@ -239,6 +263,16 @@ function toPrimitive_INT32(value: any) {
   return v;
 }
 
+function decimalToPrimitive_INT32(value: number, field: ParquetField) {
+  const primitiveValue = value * 10 ** (field.scale || 0);
+  const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);
+  if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {
+    throw new Error(`invalid value for INT32: ${value}`);
+  }
+
+  return v;
+}
+
 function toPrimitive_UINT32(value: any) {
   const v = parseInt(value, 10);
   if (v < 0 || v > 0xffffffffffff || isNaN(v)) {
@@ -257,6 +291,16 @@ function toPrimitive_INT64(value: any) {
   return v;
 }
 
+function decimalToPrimitive_INT64(value: number, field: ParquetField) {
+  const primitiveValue = value * 10 ** (field.scale || 0);
+  const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);
+  if (isNaN(v)) {
+    throw new Error(`invalid value for INT64: ${value}`);
+  }
+
+  return v;
+}
+
 function toPrimitive_UINT64(value: any) {
   const v = parseInt(value, 10);
   if (v < 0 || isNaN(v)) {
@@ -279,6 +323,11 @@ function toPrimitive_BYTE_ARRAY(value: any) {
   return Buffer.from(value);
 }
 
+function decimalToPrimitive_BYTE_ARRAY(value: any) {
+  // TBD
+  return Buffer.from(value);
+}
+
 function toPrimitive_UTF8(value: any) {
   return Buffer.from(value, 'utf8');
 }
@@ -409,3 +458,31 @@ function fromPrimitive_INTERVAL(value: any) {
 
   return {months, days, milliseconds: millis};
 }
+
+function decimalFromPrimitive_INT(value: any, field: ParquetField) {
+  const presisionInt = Math.round(((value * 10 ** -field.presision!) % 1) * 10 ** field.presision!);
+  return presisionInt * 10 ** -(field.scale || 0);
+}
+
+function decimalFromPrimitive_BYTE_ARRAY(value: any, field: ParquetField) {
+  let number = 0;
+  if (value.length <= 4) {
+    // Bytewise operators faster. Use them if it is possible
+    for (let i = 0; i < value.length; i++) {
+      // `value.length - i - 1` bytes have reverse order (big-endian)
+      const component = value[i] << (8 * (value.length - i - 1));
+      number += component;
+    }
+  } else {
+    for (let i = 0; i < value.length; i++) {
+      // `value.length - i - 1` bytes have reverse order (big-endian)
+      const component = value[i] * 2 ** (8 * (value.length - 1 - i));
+      number += component;
+    }
+  }
+
+  const presisionInt = Math.round(
+    ((number * 10 ** -field.presision!) % 1) * 10 ** field.presision!
+  );
+  return presisionInt * 10 ** -(field.scale || 0);
+}

package/src/parquetjs/utils/buffer-utils.ts

@@ -0,0 +1,18 @@
+/**
+ * Convert Buffer to ArrayBuffer
+ */
+export function toArrayBuffer(buffer: Buffer): ArrayBuffer {
+  // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+  if (Buffer.isBuffer(buffer)) {
+    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
+    return typedArray.slice().buffer;
+  }
+  return buffer;
+}
+
+/**
+ * Convert (copy) ArrayBuffer to Buffer
+ */
+export function toBuffer(arrayBuffer: ArrayBuffer): Buffer {
+  return Buffer.from(arrayBuffer);
+}

package/src/parquetjs/utils/file-utils.ts

@@ -0,0 +1,96 @@
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+import fs from 'fs';
+import {Writable} from 'stream';
+
+export function load(name: string): any {
+  return (module || (global as any)).require(name);
+}
+export interface WriteStreamOptions {
+  flags?: string;
+  encoding?: string;
+  fd?: number;
+  mode?: number;
+  autoClose?: boolean;
+  start?: number;
+}
+
+export function fopen(filePath: string): Promise<number> {
+  return new Promise((resolve, reject) => {
+    fs.open(filePath, 'r', (err, fd) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(fd);
+      }
+    });
+  });
+}
+
+export function fstat(filePath: string): Promise<fs.Stats> {
+  return new Promise((resolve, reject) => {
+    fs.stat(filePath, (err, stat) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(stat);
+      }
+    });
+  });
+}
+
+export function fread(fd: number, position: number, length: number): Promise<Buffer> {
+  const buffer = Buffer.alloc(length);
+  return new Promise((resolve, reject) => {
+    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
+      if (err || bytesRead !== length) {
+        reject(err || Error('read failed'));
+      } else {
+        resolve(buf);
+      }
+    });
+  });
+}
+
+export function fclose(fd: number): Promise<void> {
+  return new Promise((resolve, reject) => {
+    fs.close(fd, (err) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+export function oswrite(os: Writable, buf: Buffer): Promise<void> {
+  return new Promise((resolve, reject) => {
+    os.write(buf, (err) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+export function osclose(os: Writable): Promise<void> {
+  return new Promise((resolve, reject) => {
+    (os as any).close((err: any) => {
+      if (err) {
+        reject(err);
+      } else {
+        resolve();
+      }
+    });
+  });
+}
+
+export function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {
+  return new Promise((resolve, reject) => {
+    const outputStream = fs.createWriteStream(path, opts as any);
+    outputStream.once('open', (fd) => resolve(outputStream));
+    outputStream.once('error', (err) => reject(err));
+  });
+}

package/src/parquetjs/{util.ts → utils/read-utils.ts}

@@ -1,17 +1,5 @@
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-import fs from 'fs';
 import {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';
-import {FileMetaData, PageHeader} from './parquet-thrift';
-import {Writable} from 'stream';
-
-export interface WriteStreamOptions {
-  flags?: string;
-  encoding?: string;
-  fd?: number;
-  mode?: number;
-  autoClose?: boolean;
-  start?: number;
-}
+import {FileMetaData, PageHeader} from '../parquet-thrift';
 
 class UFramedTransport extends TFramedTransport {
   public readPos: number = 0;
@@ -47,6 +35,18 @@ export function decodeThrift(obj: any, buf: Buffer, offset?: number) {
   return transport.readPos - offset;
 }
 
+/**
+ * FIXME not ideal that this is linear
+ */
+export function getThriftEnum(klass: any, value: number | string): string {
+  for (const k in klass) {
+    if (klass[k] === value) {
+      return k;
+    }
+  }
+  throw new Error('Invalid ENUM value');
+}
+
 export function decodeFileMetadata(buf: Buffer, offset?: number) {
   if (!offset) {
     // tslint:disable-next-line:no-parameter-reassignment
@@ -84,99 +84,6 @@ export function getBitWidth(val: number): number {
   return Math.ceil(Math.log2(val + 1));
 }
 
-/**
- * FIXME not ideal that this is linear
- */
-export function getThriftEnum(klass: any, value: number | string): string {
-  for (const k in klass) {
-    if (klass[k] === value) {
-      return k;
-    }
-  }
-  throw new Error('Invalid ENUM value');
-}
-
-export function fopen(filePath: string): Promise<number> {
-  return new Promise((resolve, reject) => {
-    fs.open(filePath, 'r', (err, fd) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(fd);
-      }
-    });
-  });
-}
-
-export function fstat(filePath: string): Promise<fs.Stats> {
-  return new Promise((resolve, reject) => {
-    fs.stat(filePath, (err, stat) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stat);
-      }
-    });
-  });
-}
-
-export function fread(fd: number, position: number, length: number): Promise<Buffer> {
-  const buffer = Buffer.alloc(length);
-  return new Promise((resolve, reject) => {
-    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-      if (err || bytesRead !== length) {
-        reject(err || Error('read failed'));
-      } else {
-        resolve(buf);
-      }
-    });
-  });
-}
-
-export function fclose(fd: number): Promise<void> {
-  return new Promise((resolve, reject) => {
-    fs.close(fd, (err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
-export function oswrite(os: Writable, buf: Buffer): Promise<void> {
-  return new Promise((resolve, reject) => {
-    os.write(buf, (err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
-export function osclose(os: Writable): Promise<void> {
-  return new Promise((resolve, reject) => {
-    (os as any).close((err: any) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
-export function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {
-  return new Promise((resolve, reject) => {
-    const outputStream = fs.createWriteStream(path, opts as any);
-    outputStream.once('open', (fd) => resolve(outputStream));
-    outputStream.once('error', (err) => reject(err));
-  });
-}
-
 // Supports MQTT path wildcards
 // + all immediate children
 // # all descendents
@@ -200,7 +107,3 @@ export function fieldIndexOf(arr: string[][], elem: string[]): number {
   }
   return -1;
 }
-
-export function load(name: string): any {
-  return (module || (global as any)).require(name);
-}