@loaders.gl/parquet 3.3.0 → 3.4.0-alpha.1

package/dist/parquetjs/schema/shred.d.ts

@@ -45,119 +45,4 @@ export declare function shredRecord(schema: ParquetSchema, record: any, buffer:
  *   }
  */
 export declare function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[];
-/**
- * 'Materialize' a list of <value, repetition_level, definition_level>
- * tuples back to nested records (objects/arrays) using the Google Dremel
- * Algorithm..
- *
- * The buffer argument must point to an object with the following structure (i.e.
- * the same structure that is returned by shredRecords):
- *
- *   buffer = {
- *     columnData: [
- *       'my_col': {
- *          dlevels: [d1, d2, .. dN],
- *          rlevels: [r1, r2, .. rN],
- *          values: [v1, v2, .. vN],
- *        }, ...
- *      ],
- *      rowCount: X,
- *   }
- *
-export function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {
-  const columns: ParquetRecord = {};
-  for (const key in buffer.columnData) {
-    const columnData = buffer.columnData[key];
-    if (columnData.count) {
-      extractColumn(schema, columnData, key, columns);
-    }
-  }
-  return columns;
-}
-
-// eslint-disable-next-line max-statements, complexity
-function extractColumn(
-  schema: ParquetSchema,
-  columnData: ParquetData,
-  key: string,
-  columns: Record<string, unknown>
-) {
-  if (columnData.count <= 0) {
-    return;
-  }
-
-  const record = columns;
-
-  const field = schema.findField(key);
-  const branch = schema.findFieldBranch(key);
-
-  // tslint:disable-next-line:prefer-array-literal
-  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
-  let vIndex = 0;
-
-  let i = 0;
-  const dLevel = columnData.dlevels[i];
-  const rLevel = columnData.rlevels[i];
-  rLevels[rLevel]++;
-  rLevels.fill(0, rLevel + 1);
-
-  let rIndex = 0;
-  let record = records[rLevels[rIndex++] - 1];
-
-  // Internal nodes
-  for (const step of branch) {
-    if (step === field || dLevel < step.dLevelMax) {
-      break;
-    }
-
-    switch (step.repetitionType) {
-      case 'REPEATED':
-        if (!(step.name in record)) {
-          // eslint-disable max-depth
-          record[step.name] = [];
-        }
-        const ix = rLevels[rIndex++];
-        while (record[step.name].length <= ix) {
-          // eslint-disable max-depth
-          record[step.name].push({});
-        }
-        record = record[step.name][ix];
-        break;
-
-      default:
-        record[step.name] = record[step.name] || {};
-        record = record[step.name];
-    }
-  }
-
-  // Leaf node
-  if (dLevel === field.dLevelMax) {
-    const value = Types.fromPrimitive(
-      // @ts-ignore
-      field.originalType || field.primitiveType,
-      columnData.values[vIndex],
-      field
-    );
-    vIndex++;
-
-    switch (field.repetitionType) {
-      case 'REPEATED':
-        if (!(field.name in record)) {
-          // eslint-disable max-depth
-          record[field.name] = [];
-        }
-        const ix = rLevels[rIndex];
-        while (record[field.name].length <= ix) {
-          // eslint-disable max-depth
-          record[field.name].push(null);
-        }
-        record[field.name][ix] = value;
-        break;
-
-      default:
-        record[field.name] = value;
-    }
-  }
-}
-*/
 //# sourceMappingURL=shred.d.ts.map

package/dist/parquetjs/schema/shred.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,aAAa,EAA6B,aAAa,EAAC,MAAM,WAAW,CAAC;AAClF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,aAAa,EAAC,CAAC;AAEvB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAYhE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI,CAmB3F;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,aAAa,EAAE,CAYhG;AAmFD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkHE"}
+{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,aAAa,EAA6B,aAAa,EAAC,MAAM,WAAW,CAAC;AAClF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,aAAa,EAAC,CAAC;AAEvB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAYhE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI,CAmB3F;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,aAAa,EAAE,CAOhG"}

package/dist/parquetjs/schema/shred.js

@@ -155,193 +155,75 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
  */
 function materializeRecords(schema, buffer) {
     const records = [];
-    for (let i = 0; i < buffer.rowCount; i++) {
+    for (let i = 0; i < buffer.rowCount; i++)
         records.push({});
-    }
     for (const key in buffer.columnData) {
-        const columnData = buffer.columnData[key];
-        if (columnData.count) {
-            materializeColumn(schema, columnData, key, records);
-        }
+        materializeColumn(schema, buffer, key, records);
     }
     return records;
 }
 exports.materializeRecords = materializeRecords;
 // eslint-disable-next-line max-statements, complexity
-function materializeColumn(schema, columnData, key, records) {
+function materializeColumn(schema, buffer, key, records) {
+    const data = buffer.columnData[key];
+    if (!data.count)
+        return;
     const field = schema.findField(key);
     const branch = schema.findFieldBranch(key);
     // tslint:disable-next-line:prefer-array-literal
     const rLevels = new Array(field.rLevelMax + 1).fill(0);
     let vIndex = 0;
-    for (let i = 0; i < columnData.count; i++) {
-        const dLevel = columnData.dlevels[i];
-        const rLevel = columnData.rlevels[i];
+    for (let i = 0; i < data.count; i++) {
+        const dLevel = data.dlevels[i];
+        const rLevel = data.rlevels[i];
         rLevels[rLevel]++;
         rLevels.fill(0, rLevel + 1);
         let rIndex = 0;
         let record = records[rLevels[rIndex++] - 1];
-        // Internal nodes - Build a nested row object
+        // Internal nodes
         for (const step of branch) {
-            if (step === field || dLevel < step.dLevelMax) {
+            if (step === field)
                 break;
+            if (dLevel < step.dLevelMax)
+                break;
+            if (step.repetitionType === 'REPEATED') {
+                if (!(step.name in record)) {
+                    // eslint-disable max-depth
+                    record[step.name] = [];
+                }
+                const ix = rLevels[rIndex++];
+                while (record[step.name].length <= ix) {
+                    // eslint-disable max-depth
+                    record[step.name].push({});
+                }
+                record = record[step.name][ix];
             }
-            switch (step.repetitionType) {
-                case 'REPEATED':
-                    if (!(step.name in record)) {
-                        // eslint-disable max-depth
-                        record[step.name] = [];
-                    }
-                    const ix = rLevels[rIndex++];
-                    while (record[step.name].length <= ix) {
-                        // eslint-disable max-depth
-                        record[step.name].push({});
-                    }
-                    record = record[step.name][ix];
-                    break;
-                default:
-                    record[step.name] = record[step.name] || {};
-                    record = record[step.name];
+            else {
+                record[step.name] = record[step.name] || {};
+                record = record[step.name];
             }
         }
-        // Leaf node - Add the value
+        // Leaf node
         if (dLevel === field.dLevelMax) {
             const value = Types.fromPrimitive(
             // @ts-ignore
-            field.originalType || field.primitiveType, columnData.values[vIndex], field);
+            field.originalType || field.primitiveType, data.values[vIndex], field);
             vIndex++;
-            switch (field.repetitionType) {
-                case 'REPEATED':
-                    if (!(field.name in record)) {
-                        // eslint-disable max-depth
-                        record[field.name] = [];
-                    }
-                    const ix = rLevels[rIndex];
-                    while (record[field.name].length <= ix) {
-                        // eslint-disable max-depth
-                        record[field.name].push(null);
-                    }
-                    record[field.name][ix] = value;
-                    break;
-                default:
-                    record[field.name] = value;
+            if (field.repetitionType === 'REPEATED') {
+                if (!(field.name in record)) {
+                    // eslint-disable max-depth
+                    record[field.name] = [];
+                }
+                const ix = rLevels[rIndex];
+                while (record[field.name].length <= ix) {
+                    // eslint-disable max-depth
+                    record[field.name].push(null);
+                }
+                record[field.name][ix] = value;
+            }
+            else {
+                record[field.name] = value;
             }
         }
     }
 }
-// Columnar export
-/**
- * 'Materialize' a list of <value, repetition_level, definition_level>
- * tuples back to nested records (objects/arrays) using the Google Dremel
- * Algorithm..
- *
- * The buffer argument must point to an object with the following structure (i.e.
- * the same structure that is returned by shredRecords):
- *
- *   buffer = {
- *     columnData: [
- *       'my_col': {
- *          dlevels: [d1, d2, .. dN],
- *          rlevels: [r1, r2, .. rN],
- *          values: [v1, v2, .. vN],
- *        }, ...
- *      ],
- *      rowCount: X,
- *   }
- *
-export function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {
-  const columns: ParquetRecord = {};
-  for (const key in buffer.columnData) {
-    const columnData = buffer.columnData[key];
-    if (columnData.count) {
-      extractColumn(schema, columnData, key, columns);
-    }
-  }
-  return columns;
-}
-
-// eslint-disable-next-line max-statements, complexity
-function extractColumn(
-  schema: ParquetSchema,
-  columnData: ParquetData,
-  key: string,
-  columns: Record<string, unknown>
-) {
-  if (columnData.count <= 0) {
-    return;
-  }
-
-  const record = columns;
-
-  const field = schema.findField(key);
-  const branch = schema.findFieldBranch(key);
-
-  // tslint:disable-next-line:prefer-array-literal
-  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
-  let vIndex = 0;
-
-  let i = 0;
-  const dLevel = columnData.dlevels[i];
-  const rLevel = columnData.rlevels[i];
-  rLevels[rLevel]++;
-  rLevels.fill(0, rLevel + 1);
-
-  let rIndex = 0;
-  let record = records[rLevels[rIndex++] - 1];
-
-  // Internal nodes
-  for (const step of branch) {
-    if (step === field || dLevel < step.dLevelMax) {
-      break;
-    }
-
-    switch (step.repetitionType) {
-      case 'REPEATED':
-        if (!(step.name in record)) {
-          // eslint-disable max-depth
-          record[step.name] = [];
-        }
-        const ix = rLevels[rIndex++];
-        while (record[step.name].length <= ix) {
-          // eslint-disable max-depth
-          record[step.name].push({});
-        }
-        record = record[step.name][ix];
-        break;
-
-      default:
-        record[step.name] = record[step.name] || {};
-        record = record[step.name];
-    }
-  }
-
-  // Leaf node
-  if (dLevel === field.dLevelMax) {
-    const value = Types.fromPrimitive(
-      // @ts-ignore
-      field.originalType || field.primitiveType,
-      columnData.values[vIndex],
-      field
-    );
-    vIndex++;
-
-    switch (field.repetitionType) {
-      case 'REPEATED':
-        if (!(field.name in record)) {
-          // eslint-disable max-depth
-          record[field.name] = [];
-        }
-        const ix = rLevels[rIndex];
-        while (record[field.name].length <= ix) {
-          // eslint-disable max-depth
-          record[field.name].push(null);
-        }
-        record[field.name][ix] = value;
-        break;
-
-      default:
-        record[field.name] = value;
-    }
-  }
-}
-*/

package/dist/parquetjs/schema/types.d.ts

@@ -11,10 +11,10 @@ export declare const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit>;
  * Convert a value from it's native representation to the internal/underlying
  * primitive type
  */
-export declare function toPrimitive(type: ParquetType, value: unknown, field?: ParquetField): unknown;
+export declare function toPrimitive(type: ParquetType, value: any, field?: ParquetField): any;
 /**
  * Convert a value from it's internal/underlying primitive representation to
  * the native representation
  */
-export declare function fromPrimitive(type: ParquetType, value: unknown, field?: ParquetField): any;
+export declare function fromPrimitive(type: ParquetType, value: any, field?: ParquetField): any;
 //# sourceMappingURL=types.d.ts.map

package/dist/parquetjs/schema/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAC,MAAM,WAAW,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,aAAa,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,QAAQ,CAAC;IACtB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AAED,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,WAAW,EAAE,cAAc,CAuJrE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,YAAY,GAAG,OAAO,CAM5F;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,YAAY,OAUpF"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAC,MAAM,WAAW,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,aAAa,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,QAAQ,CAAC;IACtB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AAED,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,WAAW,EAAE,cAAc,CAuJrE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,YAAY,OAM9E;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,YAAY,OAUhF"}

package/dist/parquetjs/utils/buffer-utils.d.ts

@@ -0,0 +1,10 @@
+/// <reference types="node" />
+/**
+ * Convert Buffer to ArrayBuffer
+ */
+export declare function toArrayBuffer(buffer: Buffer): ArrayBuffer;
+/**
+ * Convert (copy) ArrayBuffer to Buffer
+ */
+export declare function toBuffer(arrayBuffer: ArrayBuffer): Buffer;
+//# sourceMappingURL=buffer-utils.d.ts.map

package/dist/parquetjs/utils/buffer-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buffer-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/buffer-utils.ts"],"names":[],"mappings":";AAAA;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAOzD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,CAEzD"}

package/dist/parquetjs/utils/buffer-utils.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toBuffer = exports.toArrayBuffer = void 0;
+/**
+ * Convert Buffer to ArrayBuffer
+ */
+function toArrayBuffer(buffer) {
+    // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+    if (Buffer.isBuffer(buffer)) {
+        const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
+        return typedArray.slice().buffer;
+    }
+    return buffer;
+}
+exports.toArrayBuffer = toArrayBuffer;
+/**
+ * Convert (copy) ArrayBuffer to Buffer
+ */
+function toBuffer(arrayBuffer) {
+    return Buffer.from(arrayBuffer);
+}
+exports.toBuffer = toBuffer;

package/dist/parquetjs/utils/file-utils.d.ts

@@ -1,7 +1,8 @@
 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
-import { fs, stream } from '@loaders.gl/loader-utils';
+import fs from 'fs';
+import { Writable } from 'stream';
 export declare function load(name: string): any;
 export interface WriteStreamOptions {
     flags?: string;
@@ -11,7 +12,7 @@ export interface WriteStreamOptions {
     autoClose?: boolean;
     start?: number;
 }
-export declare function oswrite(os: stream.Writable, buf: Buffer): Promise<void>;
-export declare function osclose(os: stream.Writable): Promise<void>;
+export declare function oswrite(os: Writable, buf: Buffer): Promise<void>;
+export declare function osclose(os: Writable): Promise<void>;
 export declare function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream>;
 //# sourceMappingURL=file-utils.d.ts.map

package/dist/parquetjs/utils/file-utils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":[],"mappings":";;;AACA,OAAO,EAAC,EAAE,EAAE,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAEpD,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAEtC;AACD,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUvE;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAU1D;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,CAMvF"}
+{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":[],"mappings":";;;AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAEhC,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAEtC;AACD,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUhE;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAUnD;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,CAMvF"}

package/dist/parquetjs/utils/file-utils.js

@@ -1,8 +1,11 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.osopen = exports.osclose = exports.oswrite = exports.load = void 0;
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-const loader_utils_1 = require("@loaders.gl/loader-utils");
+const fs_1 = __importDefault(require("fs"));
 function load(name) {
     return (module || global).require(name);
 }
@@ -35,7 +38,7 @@ function osclose(os) {
 exports.osclose = osclose;
 function osopen(path, opts) {
     return new Promise((resolve, reject) => {
-        const outputStream = loader_utils_1.fs.createWriteStream(path, opts);
+        const outputStream = fs_1.default.createWriteStream(path, opts);
         outputStream.once('open', (fd) => resolve(outputStream));
         outputStream.once('error', (err) => reject(err));
     });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "3.3.0",
+  "version": "3.4.0-alpha.1",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -37,14 +37,12 @@
     "net": false,
     "tls": false,
     "lzo": false,
-    "stream": false,
-    "fs": false,
     "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts"
   },
   "dependencies": {
-    "@loaders.gl/compression": "3.3.0",
-    "@loaders.gl/loader-utils": "3.3.0",
-    "@loaders.gl/schema": "3.3.0",
+    "@loaders.gl/compression": "3.4.0-alpha.1",
+    "@loaders.gl/loader-utils": "3.4.0-alpha.1",
+    "@loaders.gl/schema": "3.4.0-alpha.1",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "bson": "^1.0.4",
@@ -70,5 +68,5 @@
     "@types/varint": "^5.0.0",
     "apache-arrow": "^4.0.0"
   },
-  "gitHead": "566d194f175a03e042d3e588cdd514048f428cc1"
+  "gitHead": "4085b0323050e4361614471319a1fb4729547bbf"
 }

package/src/index.ts

@@ -32,8 +32,8 @@ export {preloadCompressions} from './parquetjs/compression';
 
 export {ParquetSchema} from './parquetjs/schema/schema';
 export {ParquetReader} from './parquetjs/parser/parquet-reader';
-export {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';
-
+export {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';
+// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';
 export {convertParquetToArrowSchema} from './lib/convert-schema';
 
 // TESTS

package/src/lib/parse-parquet.ts

@@ -1,7 +1,6 @@
 // import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
-// import {ColumnarTableBatch} from '@loaders.gl/schema';
-import {makeReadableFile} from '@loaders.gl/loader-utils';
 import type {ParquetLoaderOptions} from '../parquet-loader';
+
 import {ParquetReader} from '../parquetjs/parser/parquet-reader';
 
 export async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
@@ -13,28 +12,16 @@ export async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLo
 }
 
 export async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {
-  const file = makeReadableFile(blob);
-  const reader = new ParquetReader(file);
-  const rowBatches = reader.rowBatchIterator(options?.parquet);
-  for await (const rows of rowBatches) {
-    yield rows;
+  const reader = await ParquetReader.openBlob(blob);
+  const rows: any[][] = [];
+  try {
+    const cursor = reader.getCursor();
+    let record: any[] | null;
+    while ((record = await cursor.next())) {
+      rows.push(record);
+    }
+  } finally {
+    await reader.close();
   }
+  yield rows;
 }
-
-// export async function* parseParquetFileInColumnarBatches(blob: Blob, options?: {columnList?: string[][]}): AsyncIterable<ColumnarTableBatch> {
-//   const rowGroupReader = new ParquetRowGroupReader({data: blob, columnList: options?.columnList});
-//   try {
-//     for await (const rowGroup of rowGroupReader) {
-//       yield convertRowGroupToTableBatch(rowGroup);
-//     }
-//   } finally {
-//     await rowGroupReader.close();
-//   }
-// }
-
-// function convertRowGroupToTableBatch(rowGroup): ColumnarTableBatch {
-//   // @ts-expect-error
-//   return {
-//     data: rowGroup
-//   };
-// }

package/src/parquet-loader.ts

@@ -8,15 +8,13 @@ export type ParquetLoaderOptions = LoaderOptions & {
   parquet?: {
     type?: 'object-row-table';
     url?: string;
-    columnList?: string[] | string[][];
   };
 };
 
 const DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {
   parquet: {
     type: 'object-row-table',
-    url: undefined,
-    columnList: []
+    url: undefined
   }
 };
 

package/src/parquetjs/compression.ts

@@ -14,20 +14,7 @@ import {
 } from '@loaders.gl/compression';
 
 import {ParquetCompression} from './schema/declare';
-
-/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */
-function toBuffer(arrayBuffer: ArrayBuffer): Buffer {
-  return Buffer.from(arrayBuffer);
-}
-
-function toArrayBuffer(buffer: Buffer): ArrayBuffer {
-  // TODO - per docs we should just be able to call buffer.buffer, but there are issues
-  if (Buffer.isBuffer(buffer)) {
-    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
-    return typedArray.slice().buffer;
-  }
-  return buffer;
-}
+import {toArrayBuffer, toBuffer} from './utils/buffer-utils';
 
 // TODO switch to worker compression to avoid bundling...