@loaders.gl/parquet 3.3.0 → 3.4.0-alpha.1

package/src/parquetjs/schema/shred.ts

@@ -146,14 +146,9 @@ function shredRecordFields(
  */
 export function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {
   const records: ParquetRecord[] = [];
-  for (let i = 0; i < buffer.rowCount; i++) {
-    records.push({});
-  }
+  for (let i = 0; i < buffer.rowCount; i++) records.push({});
   for (const key in buffer.columnData) {
-    const columnData = buffer.columnData[key];
-    if (columnData.count) {
-      materializeColumn(schema, columnData, key, records);
-    }
+    materializeColumn(schema, buffer, key, records);
   }
   return records;
 }
@@ -161,151 +156,33 @@ export function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer)
 // eslint-disable-next-line max-statements, complexity
 function materializeColumn(
   schema: ParquetSchema,
-  columnData: ParquetData,
+  buffer: ParquetBuffer,
   key: string,
   records: ParquetRecord[]
-): void {
+) {
+  const data = buffer.columnData[key];
+  if (!data.count) return;
+
   const field = schema.findField(key);
   const branch = schema.findFieldBranch(key);
 
   // tslint:disable-next-line:prefer-array-literal
   const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
   let vIndex = 0;
-  for (let i = 0; i < columnData.count; i++) {
-    const dLevel = columnData.dlevels[i];
-    const rLevel = columnData.rlevels[i];
+  for (let i = 0; i < data.count; i++) {
+    const dLevel = data.dlevels[i];
+    const rLevel = data.rlevels[i];
     rLevels[rLevel]++;
     rLevels.fill(0, rLevel + 1);
 
     let rIndex = 0;
     let record = records[rLevels[rIndex++] - 1];
 
-    // Internal nodes - Build a nested row object
+    // Internal nodes
     for (const step of branch) {
-      if (step === field || dLevel < step.dLevelMax) {
-        break;
-      }
-
-      switch (step.repetitionType) {
-        case 'REPEATED':
-          if (!(step.name in record)) {
-            // eslint-disable max-depth
-            record[step.name] = [];
-          }
-          const ix = rLevels[rIndex++];
-          while (record[step.name].length <= ix) {
-            // eslint-disable max-depth
-            record[step.name].push({});
-          }
-          record = record[step.name][ix];
-          break;
-
-        default:
-          record[step.name] = record[step.name] || {};
-          record = record[step.name];
-      }
-    }
-
-    // Leaf node - Add the value
-    if (dLevel === field.dLevelMax) {
-      const value = Types.fromPrimitive(
-        // @ts-ignore
-        field.originalType || field.primitiveType,
-        columnData.values[vIndex],
-        field
-      );
-      vIndex++;
-
-      switch (field.repetitionType) {
-        case 'REPEATED':
-          if (!(field.name in record)) {
-            // eslint-disable max-depth
-            record[field.name] = [];
-          }
-          const ix = rLevels[rIndex];
-          while (record[field.name].length <= ix) {
-            // eslint-disable max-depth
-            record[field.name].push(null);
-          }
-          record[field.name][ix] = value;
-          break;
-
-        default:
-          record[field.name] = value;
-      }
-    }
-  }
-}
-
-// Columnar export
-
-/**
- * 'Materialize' a list of <value, repetition_level, definition_level>
- * tuples back to nested records (objects/arrays) using the Google Dremel
- * Algorithm..
- *
- * The buffer argument must point to an object with the following structure (i.e.
- * the same structure that is returned by shredRecords):
- *
- *   buffer = {
- *     columnData: [
- *       'my_col': {
- *          dlevels: [d1, d2, .. dN],
- *          rlevels: [r1, r2, .. rN],
- *          values: [v1, v2, .. vN],
- *        }, ...
- *      ],
- *      rowCount: X,
- *   }
- *
-export function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {
-  const columns: ParquetRecord = {};
-  for (const key in buffer.columnData) {
-    const columnData = buffer.columnData[key];
-    if (columnData.count) {
-      extractColumn(schema, columnData, key, columns);
-    }
-  }
-  return columns;
-}
-
-// eslint-disable-next-line max-statements, complexity
-function extractColumn(
-  schema: ParquetSchema,
-  columnData: ParquetData,
-  key: string,
-  columns: Record<string, unknown> 
-) {
-  if (columnData.count <= 0) {
-    return;
-  }
-
-  const record = columns;
-
-  const field = schema.findField(key);
-  const branch = schema.findFieldBranch(key);
-
-  // tslint:disable-next-line:prefer-array-literal
-  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
-  let vIndex = 0;
-
-  let i = 0;
-  const dLevel = columnData.dlevels[i];
-  const rLevel = columnData.rlevels[i];
-  rLevels[rLevel]++;
-  rLevels.fill(0, rLevel + 1);
-
-  let rIndex = 0;
-  let record = records[rLevels[rIndex++] - 1];
-
-  // Internal nodes
-  for (const step of branch) {
-    if (step === field || dLevel < step.dLevelMax) {
-      break;
-    }
-
-    switch (step.repetitionType) {
-      case 'REPEATED':
+      if (step === field) break;
+      if (dLevel < step.dLevelMax) break;
+      if (step.repetitionType === 'REPEATED') {
         if (!(step.name in record)) {
           // eslint-disable max-depth
           record[step.name] = [];
@@ -316,26 +193,22 @@ function extractColumn(
           record[step.name].push({});
         }
         record = record[step.name][ix];
-        break;
-
-      default:
+      } else {
         record[step.name] = record[step.name] || {};
         record = record[step.name];
+      }
     }
-  }
-
-  // Leaf node
-  if (dLevel === field.dLevelMax) {
-    const value = Types.fromPrimitive(
-      // @ts-ignore
-      field.originalType || field.primitiveType,
-      columnData.values[vIndex],
-      field
-    );
-    vIndex++;
 
-    switch (field.repetitionType) {
-      case 'REPEATED':
+    // Leaf node
+    if (dLevel === field.dLevelMax) {
+      const value = Types.fromPrimitive(
+        // @ts-ignore
+        field.originalType || field.primitiveType,
+        data.values[vIndex],
+        field
+      );
+      vIndex++;
+      if (field.repetitionType === 'REPEATED') {
         if (!(field.name in record)) {
           // eslint-disable max-depth
           record[field.name] = [];
@@ -346,11 +219,9 @@ function extractColumn(
           record[field.name].push(null);
         }
         record[field.name][ix] = value;
-        break;
-
-      default:
+      } else {
         record[field.name] = value;
+      }
     }
   }
 }
-*/

package/src/parquetjs/schema/types.ts

@@ -168,7 +168,7 @@ export const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {
  * Convert a value from it's native representation to the internal/underlying
  * primitive type
  */
-export function toPrimitive(type: ParquetType, value: unknown, field?: ParquetField): unknown {
+export function toPrimitive(type: ParquetType, value: any, field?: ParquetField) {
   if (!(type in PARQUET_LOGICAL_TYPES)) {
     throw new Error(`invalid type: ${type}`);
   }
@@ -180,7 +180,7 @@ export function toPrimitive(type: ParquetType, value: unknown, field?: ParquetFi
  * Convert a value from it's internal/underlying primitive representation to
  * the native representation
  */
-export function fromPrimitive(type: ParquetType, value: unknown, field?: ParquetField) {
+export function fromPrimitive(type: ParquetType, value: any, field?: ParquetField) {
   if (!(type in PARQUET_LOGICAL_TYPES)) {
     throw new Error(`invalid type: ${type}`);
   }
@@ -192,27 +192,29 @@ export function fromPrimitive(type: ParquetType, value: unknown, field?: Parquet
   return value;
 }
 
-function toPrimitive_BOOLEAN(value: unknown): boolean {
+function toPrimitive_BOOLEAN(value: any) {
   return Boolean(value);
 }
 
-function fromPrimitive_BOOLEAN(value: any): boolean {
+function fromPrimitive_BOOLEAN(value: any) {
   return Boolean(value);
 }
 
-function toPrimitive_FLOAT(value: any): number {
+function toPrimitive_FLOAT(value: any) {
   const v = parseFloat(value);
   if (isNaN(v)) {
     throw new Error(`invalid value for FLOAT: ${value}`);
   }
+
   return v;
 }
 
-function toPrimitive_DOUBLE(value: any): number {
+function toPrimitive_DOUBLE(value: any) {
   const v = parseFloat(value);
   if (isNaN(v)) {
     throw new Error(`invalid value for DOUBLE: ${value}`);
   }
+
   return v;
 }
 
@@ -261,28 +263,31 @@ function toPrimitive_INT32(value: any) {
   return v;
 }
 
-function decimalToPrimitive_INT32(value: number, field: ParquetField): number {
+function decimalToPrimitive_INT32(value: number, field: ParquetField) {
   const primitiveValue = value * 10 ** (field.scale || 0);
   const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);
   if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {
     throw new Error(`invalid value for INT32: ${value}`);
   }
+
   return v;
 }
 
-function toPrimitive_UINT32(value: any): number {
+function toPrimitive_UINT32(value: any) {
   const v = parseInt(value, 10);
   if (v < 0 || v > 0xffffffffffff || isNaN(v)) {
     throw new Error(`invalid value for UINT32: ${value}`);
   }
+
   return v;
 }
 
-function toPrimitive_INT64(value: any): number {
+function toPrimitive_INT64(value: any) {
   const v = parseInt(value, 10);
   if (isNaN(v)) {
     throw new Error(`invalid value for INT64: ${value}`);
   }
+
   return v;
 }
 
@@ -314,32 +319,32 @@ function toPrimitive_INT96(value: any) {
   return v;
 }
 
-function toPrimitive_BYTE_ARRAY(value: any): Buffer {
+function toPrimitive_BYTE_ARRAY(value: any) {
   return Buffer.from(value);
 }
 
-function decimalToPrimitive_BYTE_ARRAY(value: any): Buffer {
+function decimalToPrimitive_BYTE_ARRAY(value: any) {
   // TBD
   return Buffer.from(value);
 }
 
-function toPrimitive_UTF8(value: any): Buffer {
+function toPrimitive_UTF8(value: any) {
   return Buffer.from(value, 'utf8');
 }
 
-function fromPrimitive_UTF8(value: any): string {
+function fromPrimitive_UTF8(value: any) {
   return value.toString();
 }
 
-function toPrimitive_JSON(value: any): Buffer {
+function toPrimitive_JSON(value: any) {
   return Buffer.from(JSON.stringify(value));
 }
 
-function fromPrimitive_JSON(value: any): unknown {
+function fromPrimitive_JSON(value: any) {
   return JSON.parse(value);
 }
 
-function toPrimitive_BSON(value: any): Buffer {
+function toPrimitive_BSON(value: any) {
   return Buffer.from(BSON.serialize(value));
 }
 
@@ -356,17 +361,18 @@ function toPrimitive_TIME_MILLIS(value: any) {
   return v;
 }
 
-function toPrimitive_TIME_MICROS(value: any): number {
+function toPrimitive_TIME_MICROS(value: any) {
   const v = parseInt(value, 10);
   if (v < 0 || isNaN(v)) {
     throw new Error(`invalid value for TIME_MICROS: ${value}`);
   }
+
   return v;
 }
 
 const kMillisPerDay = 86400000;
 
-function toPrimitive_DATE(value: any): number {
+function toPrimitive_DATE(value: any) {
   /* convert from date */
   if (value instanceof Date) {
     return value.getTime() / kMillisPerDay;
@@ -383,11 +389,11 @@ function toPrimitive_DATE(value: any): number {
   }
 }
 
-function fromPrimitive_DATE(value: any): Date {
+function fromPrimitive_DATE(value: any) {
   return new Date(value * kMillisPerDay);
 }
 
-function toPrimitive_TIMESTAMP_MILLIS(value: any): number {
+function toPrimitive_TIMESTAMP_MILLIS(value: any) {
   /* convert from date */
   if (value instanceof Date) {
     return value.getTime();
@@ -404,7 +410,7 @@ function toPrimitive_TIMESTAMP_MILLIS(value: any): number {
   }
 }
 
-function fromPrimitive_TIMESTAMP_MILLIS(value: any): Date {
+function fromPrimitive_TIMESTAMP_MILLIS(value: any) {
   return new Date(value);
 }
 

package/src/parquetjs/utils/buffer-utils.ts

@@ -0,0 +1,18 @@
+/**
+ * Convert Buffer to ArrayBuffer
+ */
+export function toArrayBuffer(buffer: Buffer): ArrayBuffer {
+  // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+  if (Buffer.isBuffer(buffer)) {
+    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
+    return typedArray.slice().buffer;
+  }
+  return buffer;
+}
+
+/**
+ * Convert (copy) ArrayBuffer to Buffer
+ */
+export function toBuffer(arrayBuffer: ArrayBuffer): Buffer {
+  return Buffer.from(arrayBuffer);
+}

package/src/parquetjs/utils/file-utils.ts

@@ -1,5 +1,6 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-import {fs, stream} from '@loaders.gl/loader-utils';
+import fs from 'fs';
+import {Writable} from 'stream';
 
 export function load(name: string): any {
   return (module || (global as any)).require(name);
@@ -13,7 +14,7 @@ export interface WriteStreamOptions {
   start?: number;
 }
 
-export function oswrite(os: stream.Writable, buf: Buffer): Promise<void> {
+export function oswrite(os: Writable, buf: Buffer): Promise<void> {
   return new Promise((resolve, reject) => {
     os.write(buf, (err) => {
       if (err) {
@@ -25,7 +26,7 @@ export function oswrite(os: stream.Writable, buf: Buffer): Promise<void> {
   });
 }
 
-export function osclose(os: stream.Writable): Promise<void> {
+export function osclose(os: Writable): Promise<void> {
   return new Promise((resolve, reject) => {
     (os as any).close((err: any) => {
       if (err) {