@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.3

package/src/parquetjs/schema/shred.ts

@@ -146,9 +146,14 @@ function shredRecordFields(
  */
 export function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {
   const records: ParquetRecord[] = [];
-  for (let i = 0; i < buffer.rowCount; i++) records.push({});
+  for (let i = 0; i < buffer.rowCount; i++) {
+    records.push({});
+  }
   for (const key in buffer.columnData) {
-    materializeColumn(schema, buffer, key, records);
+    const columnData = buffer.columnData[key];
+    if (columnData.count) {
+      materializeColumn(schema, columnData, key, records);
+    }
   }
   return records;
 }
@@ -156,33 +161,151 @@ export function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer)
 // eslint-disable-next-line max-statements, complexity
 function materializeColumn(
   schema: ParquetSchema,
-  buffer: ParquetBuffer,
+  columnData: ParquetData,
   key: string,
   records: ParquetRecord[]
-) {
-  const data = buffer.columnData[key];
-  if (!data.count) return;
-
+): void {
   const field = schema.findField(key);
   const branch = schema.findFieldBranch(key);
 
   // tslint:disable-next-line:prefer-array-literal
   const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
   let vIndex = 0;
-  for (let i = 0; i < data.count; i++) {
-    const dLevel = data.dlevels[i];
-    const rLevel = data.rlevels[i];
+  for (let i = 0; i < columnData.count; i++) {
+    const dLevel = columnData.dlevels[i];
+    const rLevel = columnData.rlevels[i];
     rLevels[rLevel]++;
     rLevels.fill(0, rLevel + 1);
 
     let rIndex = 0;
     let record = records[rLevels[rIndex++] - 1];
 
-    // Internal nodes
+    // Internal nodes - Build a nested row object
     for (const step of branch) {
-      if (step === field) break;
-      if (dLevel < step.dLevelMax) break;
-      if (step.repetitionType === 'REPEATED') {
+      if (step === field || dLevel < step.dLevelMax) {
+        break;
+      }
+
+      switch (step.repetitionType) {
+        case 'REPEATED':
+          if (!(step.name in record)) {
+            // eslint-disable max-depth
+            record[step.name] = [];
+          }
+          const ix = rLevels[rIndex++];
+          while (record[step.name].length <= ix) {
+            // eslint-disable max-depth
+            record[step.name].push({});
+          }
+          record = record[step.name][ix];
+          break;
+
+        default:
+          record[step.name] = record[step.name] || {};
+          record = record[step.name];
+      }
+    }
+
+    // Leaf node - Add the value
+    if (dLevel === field.dLevelMax) {
+      const value = Types.fromPrimitive(
+        // @ts-ignore
+        field.originalType || field.primitiveType,
+        columnData.values[vIndex],
+        field
+      );
+      vIndex++;
+
+      switch (field.repetitionType) {
+        case 'REPEATED':
+          if (!(field.name in record)) {
+            // eslint-disable max-depth
+            record[field.name] = [];
+          }
+          const ix = rLevels[rIndex];
+          while (record[field.name].length <= ix) {
+            // eslint-disable max-depth
+            record[field.name].push(null);
+          }
+          record[field.name][ix] = value;
+          break;
+
+        default:
+          record[field.name] = value;
+      }
+    }
+  }
+}
+
+// Columnar export
+
+/**
+ * 'Materialize' a list of <value, repetition_level, definition_level>
+ * tuples back to nested records (objects/arrays) using the Google Dremel
+ * Algorithm..
+ *
+ * The buffer argument must point to an object with the following structure (i.e.
+ * the same structure that is returned by shredRecords):
+ *
+ *   buffer = {
+ *     columnData: [
+ *       'my_col': {
+ *          dlevels: [d1, d2, .. dN],
+ *          rlevels: [r1, r2, .. rN],
+ *          values: [v1, v2, .. vN],
+ *        }, ...
+ *      ],
+ *      rowCount: X,
+ *   }
+ *
+export function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {
+  const columns: ParquetRecord = {};
+  for (const key in buffer.columnData) {
+    const columnData = buffer.columnData[key];
+    if (columnData.count) {
+      extractColumn(schema, columnData, key, columns);
+    }
+  }
+  return columns;
+}
+
+// eslint-disable-next-line max-statements, complexity
+function extractColumn(
+  schema: ParquetSchema,
+  columnData: ParquetData,
+  key: string,
+  columns: Record<string, unknown> 
+) {
+  if (columnData.count <= 0) {
+    return;
+  }
+
+  const record = columns;
+
+  const field = schema.findField(key);
+  const branch = schema.findFieldBranch(key);
+
+  // tslint:disable-next-line:prefer-array-literal
+  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
+  let vIndex = 0;
+
+  let i = 0;
+  const dLevel = columnData.dlevels[i];
+  const rLevel = columnData.rlevels[i];
+  rLevels[rLevel]++;
+  rLevels.fill(0, rLevel + 1);
+
+  let rIndex = 0;
+  let record = records[rLevels[rIndex++] - 1];
+
+  // Internal nodes
+  for (const step of branch) {
+    if (step === field || dLevel < step.dLevelMax) {
+      break;
+    }
+
+    switch (step.repetitionType) {
+      case 'REPEATED':
         if (!(step.name in record)) {
           // eslint-disable max-depth
           record[step.name] = [];
@@ -193,22 +316,26 @@ function materializeColumn(
           record[step.name].push({});
         }
         record = record[step.name][ix];
-      } else {
+        break;
+
+      default:
         record[step.name] = record[step.name] || {};
         record = record[step.name];
-      }
     }
+  }
 
-    // Leaf node
-    if (dLevel === field.dLevelMax) {
-      const value = Types.fromPrimitive(
-        // @ts-ignore
-        field.originalType || field.primitiveType,
-        data.values[vIndex],
-        field
-      );
-      vIndex++;
-      if (field.repetitionType === 'REPEATED') {
+  // Leaf node
+  if (dLevel === field.dLevelMax) {
+    const value = Types.fromPrimitive(
+      // @ts-ignore
+      field.originalType || field.primitiveType,
+      columnData.values[vIndex],
+      field
+    );
+    vIndex++;
+
+    switch (field.repetitionType) {
+      case 'REPEATED':
         if (!(field.name in record)) {
           // eslint-disable max-depth
           record[field.name] = [];
@@ -219,9 +346,11 @@ function materializeColumn(
           record[field.name].push(null);
         }
         record[field.name][ix] = value;
-      } else {
+        break;
+
+      default:
         record[field.name] = value;
-      }
     }
   }
 }
+*/

package/src/parquetjs/schema/types.ts

@@ -1,6 +1,6 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 /* eslint-disable camelcase */
-import BSON from 'bson';
+import {BSONLoader, BSONWriter} from '@loaders.gl/bson';
 import {OriginalType, ParquetField, ParquetType, PrimitiveType} from './declare';
 
 export interface ParquetTypeKit {
@@ -168,7 +168,7 @@ export const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {
  * Convert a value from it's native representation to the internal/underlying
  * primitive type
  */
-export function toPrimitive(type: ParquetType, value: any, field?: ParquetField) {
+export function toPrimitive(type: ParquetType, value: unknown, field?: ParquetField): unknown {
   if (!(type in PARQUET_LOGICAL_TYPES)) {
     throw new Error(`invalid type: ${type}`);
   }
@@ -180,7 +180,7 @@ export function toPrimitive(type: ParquetType, value: any, field?: ParquetField)
  * Convert a value from it's internal/underlying primitive representation to
  * the native representation
  */
-export function fromPrimitive(type: ParquetType, value: any, field?: ParquetField) {
+export function fromPrimitive(type: ParquetType, value: unknown, field?: ParquetField) {
   if (!(type in PARQUET_LOGICAL_TYPES)) {
     throw new Error(`invalid type: ${type}`);
   }
@@ -192,29 +192,27 @@ export function fromPrimitive(type: ParquetType, value: any, field?: ParquetFiel
   return value;
 }
 
-function toPrimitive_BOOLEAN(value: any) {
+function toPrimitive_BOOLEAN(value: unknown): boolean {
   return Boolean(value);
 }
 
-function fromPrimitive_BOOLEAN(value: any) {
+function fromPrimitive_BOOLEAN(value: any): boolean {
   return Boolean(value);
 }
 
-function toPrimitive_FLOAT(value: any) {
+function toPrimitive_FLOAT(value: any): number {
   const v = parseFloat(value);
   if (isNaN(v)) {
     throw new Error(`invalid value for FLOAT: ${value}`);
   }
-
   return v;
 }
 
-function toPrimitive_DOUBLE(value: any) {
+function toPrimitive_DOUBLE(value: any): number {
   const v = parseFloat(value);
   if (isNaN(v)) {
     throw new Error(`invalid value for DOUBLE: ${value}`);
   }
-
   return v;
 }
 
@@ -263,31 +261,28 @@ function toPrimitive_INT32(value: any) {
   return v;
 }
 
-function decimalToPrimitive_INT32(value: number, field: ParquetField) {
+function decimalToPrimitive_INT32(value: number, field: ParquetField): number {
   const primitiveValue = value * 10 ** (field.scale || 0);
   const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);
   if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {
     throw new Error(`invalid value for INT32: ${value}`);
   }
-
   return v;
 }
 
-function toPrimitive_UINT32(value: any) {
+function toPrimitive_UINT32(value: any): number {
   const v = parseInt(value, 10);
   if (v < 0 || v > 0xffffffffffff || isNaN(v)) {
     throw new Error(`invalid value for UINT32: ${value}`);
   }
-
   return v;
 }
 
-function toPrimitive_INT64(value: any) {
+function toPrimitive_INT64(value: any): number {
   const v = parseInt(value, 10);
   if (isNaN(v)) {
     throw new Error(`invalid value for INT64: ${value}`);
   }
-
   return v;
 }
 
@@ -319,37 +314,38 @@ function toPrimitive_INT96(value: any) {
   return v;
 }
 
-function toPrimitive_BYTE_ARRAY(value: any) {
+function toPrimitive_BYTE_ARRAY(value: any): Buffer {
   return Buffer.from(value);
 }
 
-function decimalToPrimitive_BYTE_ARRAY(value: any) {
+function decimalToPrimitive_BYTE_ARRAY(value: any): Buffer {
   // TBD
   return Buffer.from(value);
 }
 
-function toPrimitive_UTF8(value: any) {
+function toPrimitive_UTF8(value: any): Buffer {
   return Buffer.from(value, 'utf8');
 }
 
-function fromPrimitive_UTF8(value: any) {
+function fromPrimitive_UTF8(value: any): string {
   return value.toString();
 }
 
-function toPrimitive_JSON(value: any) {
+function toPrimitive_JSON(value: any): Buffer {
   return Buffer.from(JSON.stringify(value));
 }
 
-function fromPrimitive_JSON(value: any) {
+function fromPrimitive_JSON(value: any): unknown {
   return JSON.parse(value);
 }
 
-function toPrimitive_BSON(value: any) {
-  return Buffer.from(BSON.serialize(value));
+function toPrimitive_BSON(value: any): Buffer {
+  const arrayBuffer = BSONWriter.encodeSync?.(value) as ArrayBuffer;
+  return Buffer.from(arrayBuffer);
 }
 
 function fromPrimitive_BSON(value: any) {
-  return BSON.deserialize(value);
+  return BSONLoader.parseSync?.(value);
 }
 
 function toPrimitive_TIME_MILLIS(value: any) {
@@ -361,18 +357,17 @@ function toPrimitive_TIME_MILLIS(value: any) {
   return v;
 }
 
-function toPrimitive_TIME_MICROS(value: any) {
+function toPrimitive_TIME_MICROS(value: any): number {
   const v = parseInt(value, 10);
   if (v < 0 || isNaN(v)) {
     throw new Error(`invalid value for TIME_MICROS: ${value}`);
   }
-
   return v;
 }
 
 const kMillisPerDay = 86400000;
 
-function toPrimitive_DATE(value: any) {
+function toPrimitive_DATE(value: any): number {
   /* convert from date */
   if (value instanceof Date) {
     return value.getTime() / kMillisPerDay;
@@ -389,11 +384,11 @@ function toPrimitive_DATE(value: any) {
   }
 }
 
-function fromPrimitive_DATE(value: any) {
+function fromPrimitive_DATE(value: any): Date {
   return new Date(value * kMillisPerDay);
 }
 
-function toPrimitive_TIMESTAMP_MILLIS(value: any) {
+function toPrimitive_TIMESTAMP_MILLIS(value: any): number {
   /* convert from date */
   if (value instanceof Date) {
     return value.getTime();
@@ -410,7 +405,7 @@ function toPrimitive_TIMESTAMP_MILLIS(value: any) {
   }
 }
 
-function fromPrimitive_TIMESTAMP_MILLIS(value: any) {
+function fromPrimitive_TIMESTAMP_MILLIS(value: any): Date {
   return new Date(value);
 }
 

package/src/parquetjs/utils/file-utils.ts

@@ -1,6 +1,5 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-import fs from 'fs';
-import {Writable} from 'stream';
+import {fs, stream} from '@loaders.gl/loader-utils';
 
 export function load(name: string): any {
   return (module || (global as any)).require(name);
@@ -14,7 +13,7 @@ export interface WriteStreamOptions {
   start?: number;
 }
 
-export function oswrite(os: Writable, buf: Buffer): Promise<void> {
+export function oswrite(os: stream.Writable, buf: Buffer): Promise<void> {
   return new Promise((resolve, reject) => {
     os.write(buf, (err) => {
       if (err) {
@@ -26,7 +25,7 @@ export function oswrite(os: Writable, buf: Buffer): Promise<void> {
   });
 }
 
-export function osclose(os: Writable): Promise<void> {
+export function osclose(os: stream.Writable): Promise<void> {
   return new Promise((resolve, reject) => {
     (os as any).close((err: any) => {
       if (err) {

package/dist/es5/lib/convert-schema.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-schema.js","names":["PARQUET_TYPE_MAPPING","BOOLEAN","Bool","INT32","Int32","INT64","Float64","INT96","FLOAT","Float32","DOUBLE","BYTE_ARRAY","Binary","FIXED_LEN_BYTE_ARRAY","UTF8","Utf8","DATE","TIME_MILLIS","Int64","TIME_MICROS","TIMESTAMP_MILLIS","TIMESTAMP_MICROS","UINT_8","UINT_16","Uint16","UINT_32","Uint32","UINT_64","Uint64","INT_8","Int8","INT_16","Int16","INT_32","INT_64","JSON","BSON","INTERVAL","DECIMAL_INT32","DECIMAL_INT64","DECIMAL_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","convertParquetToArrowSchema","parquetSchema","fields","getFields","schema","Schema","getFieldMetadata","field","metadata","Map","key","value","stringify","set","name","childField","nestedField","Field","Struct","optional","push","FieldType","type","arrowField"],"sources":["../../../src/lib/convert-schema.ts"],"sourcesContent":["import type {ParquetSchema} from '../parquetjs/schema/schema';\nimport type {FieldDefinition, ParquetField, ParquetType} from '../parquetjs/schema/declare';\n\nimport {\n  Schema,\n  Struct,\n  Field,\n  DataType,\n  Bool,\n  Float64,\n  Int32,\n  Float32,\n  Binary,\n  Utf8,\n  Int64,\n  Uint16,\n  Uint32,\n  Uint64,\n  Int8,\n  Int16\n} from '@loaders.gl/schema';\n\nexport const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {\n  BOOLEAN: Bool,\n  INT32: Int32,\n  INT64: Float64,\n  INT96: Float64,\n  FLOAT: Float32,\n  DOUBLE: Float64,\n  BYTE_ARRAY: Binary,\n  FIXED_LEN_BYTE_ARRAY: Binary,\n  UTF8: Utf8,\n  DATE: Int32,\n  TIME_MILLIS: Int64,\n  TIME_MICROS: Int64,\n  TIMESTAMP_MILLIS: Int64,\n  TIMESTAMP_MICROS: Int64,\n  UINT_8: Int32,\n  UINT_16: Uint16,\n  UINT_32: Uint32,\n  UINT_64: Uint64,\n  INT_8: Int8,\n  INT_16: Int16,\n  INT_32: Int32,\n  INT_64: Int64,\n  JSON: Binary,\n  BSON: Binary,\n  // TODO check interal type\n  INTERVAL: Binary,\n  DECIMAL_INT32: Float32,\n  DECIMAL_INT64: Float64,\n  DECIMAL_BYTE_ARRAY: Float64,\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64\n};\n\nexport function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema {\n  const fields = getFields(parquetSchema.schema);\n\n  // TODO add metadata if needed.\n  return new Schema(fields);\n}\n\nfunction getFieldMetadata(field: ParquetField): Map<string, string> {\n  const metadata = new Map();\n\n  for (const key in field) {\n    if (key !== 'name') {\n      const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];\n      metadata.set(key, value);\n    }\n  }\n\n  return metadata;\n}\n\nfunction getFields(schema: FieldDefinition): Field[] {\n  const fields: Field[] = [];\n\n  for (const name in schema) {\n    const field = schema[name];\n\n    if (field.fields) {\n      const childField = getFields(field.fields);\n      const nestedField = new Field(name, new Struct(childField), field.optional);\n      fields.push(nestedField);\n    } else {\n      const FieldType = PARQUET_TYPE_MAPPING[field.type];\n      const metadata = getFieldMetadata(field);\n      const arrowField = new Field(name, new FieldType(), field.optional, metadata);\n      fields.push(arrowField);\n    }\n  }\n\n  return fields;\n}\n"],"mappings":";;;;;;;AAGA;AAmBO,IAAMA,oBAA8D,GAAG;EAC5EC,OAAO,EAAEC,YAAI;EACbC,KAAK,EAAEC,aAAK;EACZC,KAAK,EAAEC,eAAO;EACdC,KAAK,EAAED,eAAO;EACdE,KAAK,EAAEC,eAAO;EACdC,MAAM,EAAEJ,eAAO;EACfK,UAAU,EAAEC,cAAM;EAClBC,oBAAoB,EAAED,cAAM;EAC5BE,IAAI,EAAEC,YAAI;EACVC,IAAI,EAAEZ,aAAK;EACXa,WAAW,EAAEC,aAAK;EAClBC,WAAW,EAAED,aAAK;EAClBE,gBAAgB,EAAEF,aAAK;EACvBG,gBAAgB,EAAEH,aAAK;EACvBI,MAAM,EAAElB,aAAK;EACbmB,OAAO,EAAEC,cAAM;EACfC,OAAO,EAAEC,cAAM;EACfC,OAAO,EAAEC,cAAM;EACfC,KAAK,EAAEC,YAAI;EACXC,MAAM,EAAEC,aAAK;EACbC,MAAM,EAAE7B,aAAK;EACb8B,MAAM,EAAEhB,aAAK;EACbiB,IAAI,EAAEvB,cAAM;EACZwB,IAAI,EAAExB,cAAM;EAEZyB,QAAQ,EAAEzB,cAAM;EAChB0B,aAAa,EAAE7B,eAAO;EACtB8B,aAAa,EAAEjC,eAAO;EACtBkC,kBAAkB,EAAElC,eAAO;EAC3BmC,4BAA4B,EAAEnC;AAChC,CAAC;AAAC;AAEK,SAASoC,2BAA2B,CAACC,aAA4B,EAAU;EAChF,IAAMC,MAAM,GAAGC,SAAS,CAACF,aAAa,CAACG,MAAM,CAAC;;EAG9C,OAAO,IAAIC,cAAM,CAACH,MAAM,CAAC;AAC3B;AAEA,SAASI,gBAAgB,CAACC,KAAmB,EAAuB;EAClE,IAAMC,QAAQ,GAAG,IAAIC,GAAG,EAAE;EAE1B,KAAK,IAAMC,GAAG,IAAIH,KAAK,EAAE;IACvB,IAAIG,GAAG,KAAK,MAAM,EAAE;MAClB,IAAMC,KAAK,GAAG,OAAOJ,KAAK,CAACG,GAAG,CAAC,KAAK,QAAQ,GAAGjB,IAAI,CAACmB,SAAS,CAACL,KAAK,CAACG,GAAG,CAAC,CAAC,GAAGH,KAAK,CAACG,GAAG,CAAC;MACtFF,QAAQ,CAACK,GAAG,CAACH,GAAG,EAAEC,KAAK,CAAC;IAC1B;EACF;EAEA,OAAOH,QAAQ;AACjB;AAEA,SAASL,SAAS,CAACC,MAAuB,EAAW;EACnD,IAAMF,MAAe,GAAG,EAAE;EAE1B,KAAK,IAAMY,IAAI,IAAIV,MAAM,EAAE;IACzB,IAAMG,KAAK,GAAGH,MAAM,CAACU,IAAI,CAAC;IAE1B,IAAIP,KAAK,CAACL,MAAM,EAAE;MAChB,IAAMa,UAAU,GAAGZ,SAAS,CAACI,KAAK,CAACL,MAAM,CAAC;MAC1C,IAAMc,WAAW,GAAG,IAAIC,aAAK,CAACH,IAAI,EAAE,IAAII,cAAM,CAACH,UAAU,CAAC,EAAER,KAAK,CAACY,QAAQ,CAAC;MAC3EjB,MAAM,CAACkB,IAAI,CAACJ,WAAW,CAAC;IAC1B,CAAC,MAAM;MACL,IAAMK,SAAS,GAAG/D,oBAAoB,CAACiD,KAAK,CAACe,IAAI,CAAC;MAClD,IAAMd,QAAQ,GAAGF,gBAAgB,CAACC,KAAK,CAAC;MACxC,IAAMgB,UAAU,GAAG,IAAIN,aAAK,CAACH,IAAI,EAAE,IAAIO,SAAS,EAAE,EAAEd,KAAK,CAACY,QAAQ,EAAEX,QAAQ,CAAC;MAC7EN,MAAM,CAACkB,IAAI,CAACG,UAAU,CAAC;IACzB;EACF;EAEA,OAAOrB,MAAM;AACf"}

package/dist/es5/lib/parse-parquet.js

@@ -1,130 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.parseParquet = parseParquet;
-exports.parseParquetFileInBatches = parseParquetFileInBatches;
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
-var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
-var _parquetReader = require("../parquetjs/parser/parquet-reader");
-function _asyncIterator(iterable) { var method, async, sync, retry = 2; for ("undefined" != typeof Symbol && (async = Symbol.asyncIterator, sync = Symbol.iterator); retry--;) { if (async && null != (method = iterable[async])) return method.call(iterable); if (sync && null != (method = iterable[sync])) return new AsyncFromSyncIterator(method.call(iterable)); async = "@@asyncIterator", sync = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
-function AsyncFromSyncIterator(s) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var done = r.done; return Promise.resolve(r.value).then(function (value) { return { value: value, done: done }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(s) { this.s = s, this.n = s.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, return: function _return(value) { var ret = this.s.return; return void 0 === ret ? Promise.resolve({ value: value, done: !0 }) : AsyncFromSyncIteratorContinuation(ret.apply(this.s, arguments)); }, throw: function _throw(value) { var thr = this.s.return; return void 0 === thr ? Promise.reject(value) : AsyncFromSyncIteratorContinuation(thr.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(s); }
-function parseParquet(_x3, _x4) {
-  return _parseParquet.apply(this, arguments);
-}
-function _parseParquet() {
-  _parseParquet = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(arrayBuffer, options) {
-    var blob, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, batch;
-    return _regenerator.default.wrap(function _callee2$(_context2) {
-      while (1) {
-        switch (_context2.prev = _context2.next) {
-          case 0:
-            blob = new Blob([arrayBuffer]);
-            _iteratorAbruptCompletion = false;
-            _didIteratorError = false;
-            _context2.prev = 3;
-            _iterator = _asyncIterator(parseParquetFileInBatches(blob, options));
-          case 5:
-            _context2.next = 7;
-            return _iterator.next();
-          case 7:
-            if (!(_iteratorAbruptCompletion = !(_step = _context2.sent).done)) {
-              _context2.next = 13;
-              break;
-            }
-            batch = _step.value;
-            return _context2.abrupt("return", batch);
-          case 10:
-            _iteratorAbruptCompletion = false;
-            _context2.next = 5;
-            break;
-          case 13:
-            _context2.next = 19;
-            break;
-          case 15:
-            _context2.prev = 15;
-            _context2.t0 = _context2["catch"](3);
-            _didIteratorError = true;
-            _iteratorError = _context2.t0;
-          case 19:
-            _context2.prev = 19;
-            _context2.prev = 20;
-            if (!(_iteratorAbruptCompletion && _iterator.return != null)) {
-              _context2.next = 24;
-              break;
-            }
-            _context2.next = 24;
-            return _iterator.return();
-          case 24:
-            _context2.prev = 24;
-            if (!_didIteratorError) {
-              _context2.next = 27;
-              break;
-            }
-            throw _iteratorError;
-          case 27:
-            return _context2.finish(24);
-          case 28:
-            return _context2.finish(19);
-          case 29:
-            return _context2.abrupt("return", null);
-          case 30:
-          case "end":
-            return _context2.stop();
-        }
-      }
-    }, _callee2, null, [[3, 15, 19, 29], [20,, 24, 28]]);
-  }));
-  return _parseParquet.apply(this, arguments);
-}
-function parseParquetFileInBatches(_x, _x2) {
-  return _parseParquetFileInBatches.apply(this, arguments);
-}
-function _parseParquetFileInBatches() {
-  _parseParquetFileInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(blob, options) {
-    var reader, rows, cursor, record;
-    return _regenerator.default.wrap(function _callee$(_context) {
-      while (1) {
-        switch (_context.prev = _context.next) {
-          case 0:
-            _context.next = 2;
-            return (0, _awaitAsyncGenerator2.default)(_parquetReader.ParquetReader.openBlob(blob));
-          case 2:
-            reader = _context.sent;
-            rows = [];
-            _context.prev = 4;
-            cursor = reader.getCursor();
-          case 6:
-            _context.next = 8;
-            return (0, _awaitAsyncGenerator2.default)(cursor.next());
-          case 8:
-            if (!(record = _context.sent)) {
-              _context.next = 12;
-              break;
-            }
-            rows.push(record);
-            _context.next = 6;
-            break;
-          case 12:
-            _context.prev = 12;
-            _context.next = 15;
-            return (0, _awaitAsyncGenerator2.default)(reader.close());
-          case 15:
-            return _context.finish(12);
-          case 16:
-            _context.next = 18;
-            return rows;
-          case 18:
-          case "end":
-            return _context.stop();
-        }
-      }
-    }, _callee, null, [[4,, 12, 16]]);
-  }));
-  return _parseParquetFileInBatches.apply(this, arguments);
-}
-//# sourceMappingURL=parse-parquet.js.map

package/dist/es5/lib/parse-parquet.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-parquet.js","names":["parseParquet","arrayBuffer","options","blob","Blob","parseParquetFileInBatches","batch","ParquetReader","openBlob","reader","rows","cursor","getCursor","next","record","push","close"],"sources":["../../../src/lib/parse-parquet.ts"],"sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../parquet-loader';\n\nimport {ParquetReader} from '../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInBatches(blob, options)) {\n    return batch;\n  }\n  return null;\n}\n\nexport async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {\n  const reader = await ParquetReader.openBlob(blob);\n  const rows: any[][] = [];\n  try {\n    const cursor = reader.getCursor();\n    let record: any[] | null;\n    while ((record = await cursor.next())) {\n      rows.push(record);\n    }\n  } finally {\n    await reader.close();\n  }\n  yield rows;\n}\n"],"mappings":";;;;;;;;;;;;AAGA;AAAiE;AAAA;AAAA,SAE3CA,YAAY;EAAA;AAAA;AAAA;EAAA,0EAA3B,kBAA4BC,WAAwB,EAAEC,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YACnFC,IAAI,GAAG,IAAIC,IAAI,CAAC,CAACH,WAAW,CAAC,CAAC;YAAA;YAAA;YAAA;YAAA,2BACVI,yBAAyB,CAACF,IAAI,EAAED,OAAO,CAAC;UAAA;YAAA;YAAA;UAAA;YAAA;cAAA;cAAA;YAAA;YAAjDI,KAAK;YAAA,kCACbA,KAAK;UAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;UAAA;YAAA;UAAA;YAAA;UAAA;YAAA,kCAEP,IAAI;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACZ;EAAA;AAAA;AAAA,SAEsBD,yBAAyB;EAAA;AAAA;AAAA;EAAA,yFAAzC,iBAA0CF,IAAU,EAAED,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YAAA;YAAA,0CACpEK,4BAAa,CAACC,QAAQ,CAACL,IAAI,CAAC;UAAA;YAA3CM,MAAM;YACNC,IAAa,GAAG,EAAE;YAAA;YAEhBC,MAAM,GAAGF,MAAM,CAACG,SAAS,EAAE;UAAA;YAAA;YAAA,0CAEVD,MAAM,CAACE,IAAI,EAAE;UAAA;YAAA,MAA5BC,MAAM;cAAA;cAAA;YAAA;YACZJ,IAAI,CAACK,IAAI,CAACD,MAAM,CAAC;YAAC;YAAA;UAAA;YAAA;YAAA;YAAA,0CAGdL,MAAM,CAACO,KAAK,EAAE;UAAA;YAAA;UAAA;YAAA;YAEtB,OAAMN,IAAI;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACX;EAAA;AAAA"}

package/dist/es5/lib/read-array-buffer.js

@@ -1,43 +0,0 @@
-"use strict";
-
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-Object.defineProperty(exports, "__esModule", {
-  value: true
-});
-exports.readArrayBuffer = readArrayBuffer;
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
-function readArrayBuffer(_x, _x2, _x3) {
-  return _readArrayBuffer.apply(this, arguments);
-}
-function _readArrayBuffer() {
-  _readArrayBuffer = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(file, start, length) {
-    var slice;
-    return _regenerator.default.wrap(function _callee$(_context) {
-      while (1) {
-        switch (_context.prev = _context.next) {
-          case 0:
-            if (!(file instanceof Blob)) {
-              _context.next = 5;
-              break;
-            }
-            slice = file.slice(start, start + length);
-            _context.next = 4;
-            return slice.arrayBuffer();
-          case 4:
-            return _context.abrupt("return", _context.sent);
-          case 5:
-            _context.next = 7;
-            return file.read(start, start + length);
-          case 7:
-            return _context.abrupt("return", _context.sent);
-          case 8:
-          case "end":
-            return _context.stop();
-        }
-      }
-    }, _callee);
-  }));
-  return _readArrayBuffer.apply(this, arguments);
-}
-//# sourceMappingURL=read-array-buffer.js.map

package/dist/es5/lib/read-array-buffer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"read-array-buffer.js","names":["readArrayBuffer","file","start","length","Blob","slice","arrayBuffer","read"],"sources":["../../../src/lib/read-array-buffer.ts"],"sourcesContent":["// Random-Access read\n\nexport async function readArrayBuffer(\n  file: Blob | ArrayBuffer | any,\n  start: number,\n  length: number\n): Promise<ArrayBuffer> {\n  if (file instanceof Blob) {\n    const slice = file.slice(start, start + length);\n    return await slice.arrayBuffer();\n  }\n  return await file.read(start, start + length);\n}\n\n/**\n * Read a slice of a Blob or File, without loading the entire file into memory\n * The trick when reading File objects is to read successive \"slices\" of the File\n * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields\n * Actually reading from file happens in `readAsArrayBuffer`\n * @param blob to read\n export async function readBlob(blob: Blob): Promise<ArrayBuffer> {\n  return await new Promise((resolve, reject) => {\n    const fileReader = new FileReader();\n    fileReader.onload = (event: ProgressEvent<FileReader>) =>\n      resolve(event?.target?.result as ArrayBuffer);\n    // TODO - reject with a proper Error\n    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);\n    fileReader.readAsArrayBuffer(blob);\n  });\n}\n*/\n"],"mappings":";;;;;;;;;SAEsBA,eAAe;EAAA;AAAA;AAAA;EAAA,6EAA9B,iBACLC,IAA8B,EAC9BC,KAAa,EACbC,MAAc;IAAA;IAAA;MAAA;QAAA;UAAA;YAAA,MAEVF,IAAI,YAAYG,IAAI;cAAA;cAAA;YAAA;YAChBC,KAAK,GAAGJ,IAAI,CAACI,KAAK,CAACH,KAAK,EAAEA,KAAK,GAAGC,MAAM,CAAC;YAAA;YAAA,OAClCE,KAAK,CAACC,WAAW,EAAE;UAAA;YAAA;UAAA;YAAA;YAAA,OAErBL,IAAI,CAACM,IAAI,CAACL,KAAK,EAAEA,KAAK,GAAGC,MAAM,CAAC;UAAA;YAAA;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CAC9C;EAAA;AAAA"}