@loaders.gl/parquet 3.3.0 → 3.4.0-alpha.1

package/dist/parquetjs/compression.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AAuCpD,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAU/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAGlF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}
+{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AA0BpD,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAU/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAGlF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}

package/dist/parquetjs/compression.js

@@ -8,18 +8,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.inflate = exports.decompress = exports.deflate = exports.preloadCompressions = exports.PARQUET_COMPRESSION_METHODS = void 0;
 const compression_1 = require("@loaders.gl/compression");
-/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */
-function toBuffer(arrayBuffer) {
-    return Buffer.from(arrayBuffer);
-}
-function toArrayBuffer(buffer) {
-    // TODO - per docs we should just be able to call buffer.buffer, but there are issues
-    if (Buffer.isBuffer(buffer)) {
-        const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
-        return typedArray.slice().buffer;
-    }
-    return buffer;
-}
+const buffer_utils_1 = require("./utils/buffer-utils");
 // TODO switch to worker compression to avoid bundling...
 // import brotli from 'brotli'; - brotli has problems with decompress in browsers
 // import brotliDecompress from 'brotli/decompress';
@@ -68,9 +57,9 @@ async function deflate(method, value) {
     if (!compression) {
         throw new Error(`parquet: invalid compression method: ${method}`);
     }
-    const inputArrayBuffer = toArrayBuffer(value);
+    const inputArrayBuffer = (0, buffer_utils_1.toArrayBuffer)(value);
     const compressedArrayBuffer = await compression.compress(inputArrayBuffer);
-    return toBuffer(compressedArrayBuffer);
+    return (0, buffer_utils_1.toBuffer)(compressedArrayBuffer);
 }
 exports.deflate = deflate;
 /**
@@ -81,9 +70,9 @@ async function decompress(method, value, size) {
     if (!compression) {
         throw new Error(`parquet: invalid compression method: ${method}`);
     }
-    const inputArrayBuffer = toArrayBuffer(value);
+    const inputArrayBuffer = (0, buffer_utils_1.toArrayBuffer)(value);
     const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);
-    return toBuffer(compressedArrayBuffer);
+    return (0, buffer_utils_1.toBuffer)(compressedArrayBuffer);
 }
 exports.decompress = decompress;
 /*

package/dist/parquetjs/encoder/{parquet-encoder.d.ts → writer.d.ts}

@@ -1,10 +1,10 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { stream } from '@loaders.gl/loader-utils';
+import { Transform, Writable } from 'stream';
 import { ParquetBuffer } from '../schema/declare';
 import { ParquetSchema } from '../schema/schema';
 import { RowGroup } from '../parquet-thrift';
-export interface ParquetEncoderOptions {
+export interface ParquetWriterOptions {
     baseOffset?: number;
     rowGroupSize?: number;
     pageSize?: number;
@@ -17,21 +17,21 @@ export interface ParquetEncoderOptions {
     start?: number;
 }
 /**
- * Write a parquet file to an output stream. The ParquetEncoder will perform
+ * Write a parquet file to an output stream. The ParquetWriter will perform
  * buffering/batching for performance, so close() must be called after all rows
  * are written.
  */
-export declare class ParquetEncoder<T> {
+export declare class ParquetWriter<T> {
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified file
      */
-    static openFile<T>(schema: ParquetSchema, path: string, opts?: ParquetEncoderOptions): Promise<ParquetEncoder<T>>;
+    static openFile<T>(schema: ParquetSchema, path: string, opts?: ParquetWriterOptions): Promise<ParquetWriter<T>>;
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified stream
      */
-    static openStream<T>(schema: ParquetSchema, outputStream: stream.Writable, opts?: ParquetEncoderOptions): Promise<ParquetEncoder<T>>;
+    static openStream<T>(schema: ParquetSchema, outputStream: Writable, opts?: ParquetWriterOptions): Promise<ParquetWriter<T>>;
     schema: ParquetSchema;
     envelopeWriter: ParquetEnvelopeWriter;
     rowBuffer: ParquetBuffer;
@@ -41,7 +41,7 @@ export declare class ParquetEncoder<T> {
     /**
      * Create a new buffered parquet writer for a given envelope writer
      */
-    constructor(schema: ParquetSchema, envelopeWriter: ParquetEnvelopeWriter, opts: ParquetEncoderOptions);
+    constructor(schema: ParquetSchema, envelopeWriter: ParquetEnvelopeWriter, opts: ParquetWriterOptions);
     writeHeader(): Promise<void>;
     /**
      * Append a single row to the parquet file. Rows are buffered in memory until
@@ -82,7 +82,7 @@ export declare class ParquetEnvelopeWriter {
     /**
      * Create a new parquet envelope writer that writes to the specified stream
      */
-    static openStream(schema: ParquetSchema, outputStream: stream.Writable, opts: ParquetEncoderOptions): Promise<ParquetEnvelopeWriter>;
+    static openStream(schema: ParquetSchema, outputStream: Writable, opts: ParquetWriterOptions): Promise<ParquetEnvelopeWriter>;
     schema: ParquetSchema;
     write: (buf: Buffer) => Promise<void>;
     close: () => Promise<void>;
@@ -91,7 +91,7 @@ export declare class ParquetEnvelopeWriter {
     rowGroups: RowGroup[];
     pageSize: number;
     useDataPageV2: boolean;
-    constructor(schema: ParquetSchema, writeFn: (buf: Buffer) => Promise<void>, closeFn: () => Promise<void>, fileOffset: number, opts: ParquetEncoderOptions);
+    constructor(schema: ParquetSchema, writeFn: (buf: Buffer) => Promise<void>, closeFn: () => Promise<void>, fileOffset: number, opts: ParquetWriterOptions);
     writeSection(buf: Buffer): Promise<void>;
     /**
      * Encode the parquet file header
@@ -112,4 +112,13 @@ export declare class ParquetEnvelopeWriter {
      */
     setPageSize(cnt: number): void;
 }
-//# sourceMappingURL=parquet-encoder.d.ts.map
+/**
+ * Create a parquet transform stream
+ */
+export declare class ParquetTransformer<T> extends Transform {
+    writer: ParquetWriter<T>;
+    constructor(schema: ParquetSchema, opts?: ParquetWriterOptions);
+    _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void>;
+    _flush(callback: (val?: any) => void): Promise<void>;
+}
+//# sourceMappingURL=writer.d.ts.map

package/dist/parquetjs/encoder/writer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/writer.ts"],"names":[],"mappings":";;AAEA,OAAO,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAG3C,OAAO,EACL,aAAa,EAKd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAaL,QAAQ,EAGT,MAAM,mBAAmB,CAAC;AA2B3B,MAAM,WAAW,oBAAoB;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AAEH,qBAAa,aAAa,CAAC,CAAC;IAC1B;;;OAGG;WACU,QAAQ,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,oBAAoB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAK5B;;;OAGG;WACU,UAAU,CAAC,CAAC,EACvB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,QAAQ,EACtB,IAAI,CAAC,EAAE,oBAAoB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAWrB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,EAAE,qBAAqB,CAAC;IACtC,SAAS,EAAE,aAAa,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5C;;OAEG;gBAED,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,qBAAqB,EACrC,IAAI,EAAE,oBAAoB;IActB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC;;;OAGG;IACG,SAAS,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzC;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7C;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIlC;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC;;OAEG;WACU,UAAU,CACrB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,QAAQ,EACtB,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,qBAAqB,CAAC;IAM1B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EACvC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC5B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,oBAAoB;IAY5B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1D;;OAEG;IACH,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;GAEG;AACH,qBAAa,kBAAkB,CAAC,CAAC,CAAE,SAAQ,SAAS;IAC3C,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;gBAEpB,MAAM,EAAE,aAAa,EAAE,IAAI,GAAE,oBAAyB;IAiBlE,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAS9E,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,IAAI;CAG3C"}

package/dist/parquetjs/encoder/{parquet-encoder.js → writer.js}

@@ -26,7 +26,10 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetEnvelopeWriter = exports.ParquetEncoder = void 0;
+exports.ParquetTransformer = exports.ParquetEnvelopeWriter = exports.ParquetWriter = void 0;
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+/* eslint-disable camelcase */
+const stream_1 = require("stream");
 const codecs_1 = require("../codecs");
 const Compression = __importStar(require("../compression"));
 const Shred = __importStar(require("../schema/shred"));
@@ -53,27 +56,31 @@ const PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;
 const PARQUET_RDLVL_TYPE = 'INT32';
 const PARQUET_RDLVL_ENCODING = 'RLE';
 /**
- * Write a parquet file to an output stream. The ParquetEncoder will perform
+ * Write a parquet file to an output stream. The ParquetWriter will perform
  * buffering/batching for performance, so close() must be called after all rows
  * are written.
  */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-class ParquetEncoder {
+class ParquetWriter {
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified file
      */
     static async openFile(schema, path, opts) {
         const outputStream = await (0, file_utils_1.osopen)(path, opts);
-        return ParquetEncoder.openStream(schema, outputStream, opts);
+        return ParquetWriter.openStream(schema, outputStream, opts);
     }
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified stream
      */
-    static async openStream(schema, outputStream, opts = {}) {
+    static async openStream(schema, outputStream, opts) {
+        if (!opts) {
+            // tslint:disable-next-line:no-parameter-reassignment
+            opts = {};
+        }
         const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
-        return new ParquetEncoder(schema, envelopeWriter, opts);
+        return new ParquetWriter(schema, envelopeWriter, opts);
     }
     /**
      * Create a new buffered parquet writer for a given envelope writer
@@ -159,7 +166,7 @@ class ParquetEncoder {
         this.envelopeWriter.setPageSize(cnt);
     }
 }
-exports.ParquetEncoder = ParquetEncoder;
+exports.ParquetWriter = ParquetWriter;
 /**
  * Create a parquet file from a schema and a number of row groups. This class
  * performs direct, unbuffered writes to the underlying output stream and is
@@ -230,40 +237,31 @@ class ParquetEnvelopeWriter {
 exports.ParquetEnvelopeWriter = ParquetEnvelopeWriter;
 /**
  * Create a parquet transform stream
-export class ParquetTransformer<T> extends stream.Transform {
-  public writer: ParquetEncoder<T>;
-
-  constructor(schema: ParquetSchema, opts: ParquetEncoderOptions = {}) {
-    super({objectMode: true});
-
-    const writeProxy = (function (t: ParquetTransformer<any>) {
-      return async function (b: any): Promise<void> {
-        t.push(b);
-      };
-    })(this);
-
-    this.writer = new ParquetEncoder(
-      schema,
-      new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts),
-      opts
-    );
-  }
-
-  // tslint:disable-next-line:function-name
-  _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void> {
-    if (row) {
-      return this.writer.appendRow(row).then(callback);
+ */
+class ParquetTransformer extends stream_1.Transform {
+    constructor(schema, opts = {}) {
+        super({ objectMode: true });
+        const writeProxy = (function (t) {
+            return async function (b) {
+                t.push(b);
+            };
+        })(this);
+        this.writer = new ParquetWriter(schema, new ParquetEnvelopeWriter(schema, writeProxy, async () => { }, 0, opts), opts);
+    }
+    // tslint:disable-next-line:function-name
+    _transform(row, encoding, callback) {
+        if (row) {
+            return this.writer.appendRow(row).then(callback);
+        }
+        callback();
+        return Promise.resolve();
+    }
+    // tslint:disable-next-line:function-name
+    async _flush(callback) {
+        await this.writer.close(callback);
     }
-    callback();
-    return Promise.resolve();
-  }
-
-  // tslint:disable-next-line:function-name
-  async _flush(callback: (val?: any) => void) {
-    await this.writer.close(callback);
-  }
 }
- */
+exports.ParquetTransformer = ParquetTransformer;
 /**
  * Encode a consecutive array of data using one of the parquet encodings
  */

package/dist/parquetjs/file.d.ts

@@ -0,0 +1,10 @@
+/// <reference types="node" />
+import fs from 'fs';
+export declare function fopen(filePath: any): Promise<unknown>;
+export declare function fstat(filePath: any): Promise<fs.Stats>;
+export declare function fread(fd: any, position: any, length: any): Promise<unknown>;
+export declare function fclose(fd: any): Promise<unknown>;
+export declare function oswrite(os: any, buf: any): Promise<void>;
+export declare function osclose(os: any): Promise<void>;
+export declare function osopen(path: any, opts: any): Promise<unknown>;
+//# sourceMappingURL=file.d.ts.map

package/dist/parquetjs/file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../src/parquetjs/file.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,wBAAgB,KAAK,CAAC,QAAQ,KAAA,oBAU7B;AAED,wBAAgB,KAAK,CAAC,QAAQ,KAAA,qBAU7B;AAED,wBAAgB,KAAK,CAAC,EAAE,KAAA,EAAE,QAAQ,KAAA,EAAE,MAAM,KAAA,oBAYzC;AAED,wBAAgB,MAAM,CAAC,EAAE,KAAA,oBAUxB;AAED,wBAAgB,OAAO,CAAC,EAAE,KAAA,EAAE,GAAG,KAAA,GAAG,OAAO,CAAC,IAAI,CAAC,CAU9C;AAED,wBAAgB,OAAO,CAAC,EAAE,KAAA,GAAG,OAAO,CAAC,IAAI,CAAC,CAUzC;AAED,wBAAgB,MAAM,CAAC,IAAI,KAAA,EAAE,IAAI,KAAA,oBAYhC"}

package/dist/parquetjs/file.js

@@ -0,0 +1,99 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.osopen = exports.osclose = exports.oswrite = exports.fclose = exports.fread = exports.fstat = exports.fopen = void 0;
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+const fs_1 = __importDefault(require("fs"));
+function fopen(filePath) {
+    return new Promise((resolve, reject) => {
+        fs_1.default.open(filePath, 'r', (err, fd) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve(fd);
+            }
+        });
+    });
+}
+exports.fopen = fopen;
+function fstat(filePath) {
+    return new Promise((resolve, reject) => {
+        fs_1.default.stat(filePath, (err, stat) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve(stat);
+            }
+        });
+    });
+}
+exports.fstat = fstat;
+function fread(fd, position, length) {
+    const buffer = Buffer.alloc(length);
+    return new Promise((resolve, reject) => {
+        fs_1.default.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
+            if (err || bytesRead !== length) {
+                reject(err || Error('read failed'));
+            }
+            else {
+                resolve(buf);
+            }
+        });
+    });
+}
+exports.fread = fread;
+function fclose(fd) {
+    return new Promise((resolve, reject) => {
+        fs_1.default.close(fd, (err) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve(err);
+            }
+        });
+    });
+}
+exports.fclose = fclose;
+function oswrite(os, buf) {
+    return new Promise((resolve, reject) => {
+        os.write(buf, (err) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+}
+exports.oswrite = oswrite;
+function osclose(os) {
+    return new Promise((resolve, reject) => {
+        os.close((err) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+}
+exports.osclose = osclose;
+function osopen(path, opts) {
+    return new Promise((resolve, reject) => {
+        const outputStream = fs_1.default.createWriteStream(path, opts);
+        outputStream.on('open', function (fd) {
+            resolve(outputStream);
+        });
+        outputStream.on('error', function (err) {
+            reject(err);
+        });
+    });
+}
+exports.osopen = osopen;

package/dist/parquetjs/parser/parquet-cursor.d.ts

@@ -0,0 +1,36 @@
+import { FileMetaData } from '../parquet-thrift';
+import { ParquetEnvelopeReader } from './parquet-envelope-reader';
+import { ParquetSchema } from '../schema/schema';
+import { ParquetRecord } from '../schema/declare';
+/**
+ * A parquet cursor is used to retrieve rows from a parquet file in order
+ */
+export declare class ParquetCursor<T> implements AsyncIterable<T> {
+    metadata: FileMetaData;
+    envelopeReader: ParquetEnvelopeReader;
+    schema: ParquetSchema;
+    columnList: string[][];
+    rowGroup: ParquetRecord[];
+    rowGroupIndex: number;
+    /**
+     * Create a new parquet reader from the file metadata and an envelope reader.
+     * It is usually not recommended to call this constructor directly except for
+     * advanced and internal use cases. Consider using getCursor() on the
+     * ParquetReader instead
+     */
+    constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader, schema: ParquetSchema, columnList: string[][]);
+    /**
+     * Retrieve the next row from the cursor. Returns a row or NULL if the end
+     * of the file was reached
+     */
+    next<T = any>(): Promise<T>;
+    /**
+     * Rewind the cursor the the beginning of the file
+     */
+    rewind(): void;
+    /**
+     * Implement AsyncIterable
+     */
+    [Symbol.asyncIterator](): AsyncIterator<T>;
+}
+//# sourceMappingURL=parquet-cursor.d.ts.map

package/dist/parquetjs/parser/parquet-cursor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-cursor.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-cursor.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAC,qBAAqB,EAAC,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAC,aAAa,EAAC,MAAM,mBAAmB,CAAC;AAGhD;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,CAAC;IAChD,QAAQ,EAAE,YAAY,CAAC;IACvB,cAAc,EAAE,qBAAqB,CAAC;IACtC,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,EAAE,CAAC;IACvB,QAAQ,EAAE,aAAa,EAAE,CAAM;IAC/B,aAAa,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;gBAED,QAAQ,EAAE,YAAY,EACtB,cAAc,EAAE,qBAAqB,EACrC,MAAM,EAAE,aAAa,EACrB,UAAU,EAAE,MAAM,EAAE,EAAE;IASxB;;;OAGG;IACG,IAAI,CAAC,CAAC,GAAG,GAAG,KAAK,OAAO,CAAC,CAAC,CAAC;IAiBjC;;OAEG;IACH,MAAM,IAAI,IAAI;IAKd;;OAEG;IAEH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;CAuB3C"}

package/dist/parquetjs/parser/parquet-cursor.js

@@ -0,0 +1,74 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetCursor = void 0;
+const shred_1 = require("../schema/shred");
+/**
+ * A parquet cursor is used to retrieve rows from a parquet file in order
+ */
+class ParquetCursor {
+    /**
+     * Create a new parquet reader from the file metadata and an envelope reader.
+     * It is usually not recommended to call this constructor directly except for
+     * advanced and internal use cases. Consider using getCursor() on the
+     * ParquetReader instead
+     */
+    constructor(metadata, envelopeReader, schema, columnList) {
+        this.rowGroup = [];
+        this.metadata = metadata;
+        this.envelopeReader = envelopeReader;
+        this.schema = schema;
+        this.columnList = columnList;
+        this.rowGroupIndex = 0;
+    }
+    /**
+     * Retrieve the next row from the cursor. Returns a row or NULL if the end
+     * of the file was reached
+     */
+    async next() {
+        if (this.rowGroup.length === 0) {
+            if (this.rowGroupIndex >= this.metadata.row_groups.length) {
+                // @ts-ignore
+                return null;
+            }
+            const rowBuffer = await this.envelopeReader.readRowGroup(this.schema, this.metadata.row_groups[this.rowGroupIndex], this.columnList);
+            this.rowGroup = (0, shred_1.materializeRecords)(this.schema, rowBuffer);
+            this.rowGroupIndex++;
+        }
+        return this.rowGroup.shift();
+    }
+    /**
+     * Rewind the cursor the the beginning of the file
+     */
+    rewind() {
+        this.rowGroup = [];
+        this.rowGroupIndex = 0;
+    }
+    /**
+     * Implement AsyncIterable
+     */
+    // tslint:disable-next-line:function-name
+    [Symbol.asyncIterator]() {
+        let done = false;
+        return {
+            next: async () => {
+                if (done) {
+                    return { done, value: null };
+                }
+                const value = await this.next();
+                if (value === null) {
+                    return { done: true, value };
+                }
+                return { done: false, value };
+            },
+            return: async () => {
+                done = true;
+                return { done, value: null };
+            },
+            throw: async () => {
+                done = true;
+                return { done: true, value: null };
+            }
+        };
+    }
+}
+exports.ParquetCursor = ParquetCursor;

package/dist/parquetjs/parser/parquet-envelope-reader.d.ts

@@ -0,0 +1,40 @@
+/// <reference types="node" />
+import { ParquetSchema } from '../schema/schema';
+import { ColumnChunk, FileMetaData, RowGroup } from '../parquet-thrift';
+import { ParquetBuffer, ParquetData, ParquetOptions } from '../schema/declare';
+/**
+ * The parquet envelope reader allows direct, unbuffered access to the individual
+ * sections of the parquet file, namely the header, footer and the row groups.
+ * This class is intended for advanced/internal users; if you just want to retrieve
+ * rows from a parquet file use the ParquetReader instead
+ */
+export declare class ParquetEnvelopeReader {
+    read: (position: number, length: number) => Promise<Buffer>;
+    /**
+     * Close this parquet reader. You MUST call this method once you're finished
+     * reading rows
+     */
+    close: () => Promise<void>;
+    fileSize: number;
+    defaultDictionarySize: number;
+    static openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader>;
+    constructor(read: (position: number, length: number) => Promise<Buffer>, close: () => Promise<void>, fileSize: number, options?: any);
+    readHeader(): Promise<void>;
+    readRowGroup(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][]): Promise<ParquetBuffer>;
+    /**
+     * Do reading of parquet file's column chunk
+     * @param schema
+     * @param colChunk
+     */
+    readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData>;
+    /**
+     * Getting dictionary for allows to flatten values by indices.
+     * @param dictionaryPageOffset
+     * @param options
+     * @param pagesOffset
+     * @returns
+     */
+    getDictionary(dictionaryPageOffset: number, options: ParquetOptions, pagesOffset: number): Promise<string[]>;
+    readFooter(): Promise<FileMetaData>;
+}
+//# sourceMappingURL=parquet-envelope-reader.d.ts.map

package/dist/parquetjs/parser/parquet-envelope-reader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-envelope-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-envelope-reader.ts"],"names":[],"mappings":";AACA,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,aAAa,EAEb,WAAW,EAEX,cAAc,EACf,MAAM,mBAAmB,CAAC;AAM3B;;;;;GAKG;AACH,qBAAa,qBAAqB;IACzB,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACnE;;;OAGG;IACI,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,qBAAqB,EAAE,MAAM,CAAC;WAExB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;gBAQrE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,EAC3D,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,GAAG;IAQT,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAc3B,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,GACrB,OAAO,CAAC,aAAa,CAAC;IAgBzB;;;;OAIG;IACG,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAoDzF;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,EAAE,CAAC;IAwBd,UAAU,IAAI,OAAO,CAAC,YAAY,CAAC;CAqB1C"}