@loaders.gl/parquet 3.1.3 → 4.0.0-alpha.5

package/dist/parquetjs/parser/parquet-reader.js

@@ -1,134 +1,120 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetReader = void 0;
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-const parquet_envelope_reader_1 = require("./parquet-envelope-reader");
-const schema_1 = require("../schema/schema");
-const parquet_cursor_1 = require("./parquet-cursor");
-const constants_1 = require("../../constants");
-const decoders_1 = require("./decoders");
-/**
- * A parquet reader allows retrieving the rows from a parquet file in order.
- * The basic usage is to create a reader and then retrieve a cursor/iterator
- * which allows you to consume row after row until all rows have been read. It is
- * important that you call close() after you are finished reading the file to
- * avoid leaking file descriptors.
- */
-class ParquetReader {
-    /**
-     * Create a new parquet reader from the file metadata and an envelope reader.
-     * It is not recommended to call this constructor directly except for advanced
-     * and internal use cases. Consider using one of the open{File,Buffer} methods
-     * instead
-     */
-    constructor(metadata, envelopeReader) {
-        if (metadata.version !== constants_1.PARQUET_VERSION) {
-            throw new Error('invalid parquet version');
-        }
-        this.metadata = metadata;
-        this.envelopeReader = envelopeReader;
-        const root = this.metadata.schema[0];
-        const { schema } = (0, decoders_1.decodeSchema)(this.metadata.schema, 1, root.num_children);
-        this.schema = new schema_1.ParquetSchema(schema);
+import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
+
+let _Symbol$asyncIterator;
+
+import { ParquetEnvelopeReader } from './parquet-envelope-reader';
+import { ParquetSchema } from '../schema/schema';
+import { ParquetCursor } from './parquet-cursor';
+import { PARQUET_VERSION } from '../../constants';
+import { decodeSchema } from './decoders';
+_Symbol$asyncIterator = Symbol.asyncIterator;
+export class ParquetReader {
+  static async openBlob(blob) {
+    const readFn = async (start, length) => {
+      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();
+      return Buffer.from(arrayBuffer);
+    };
+
+    const closeFn = async () => {};
+
+    const size = blob.size;
+    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);
+
+    try {
+      await envelopeReader.readHeader();
+      const metadata = await envelopeReader.readFooter();
+      return new ParquetReader(metadata, envelopeReader);
+    } catch (err) {
+      await envelopeReader.close();
+      throw err;
     }
-    /**
-     * return a new parquet reader initialized with a read function
-     */
-    static async openBlob(blob) {
-        const readFn = async (start, length) => {
-            const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();
-            return Buffer.from(arrayBuffer);
-        };
-        const closeFn = async () => { };
-        const size = blob.size;
-        const envelopeReader = new parquet_envelope_reader_1.ParquetEnvelopeReader(readFn, closeFn, size);
-        try {
-            await envelopeReader.readHeader();
-            const metadata = await envelopeReader.readFooter();
-            return new ParquetReader(metadata, envelopeReader);
-        }
-        catch (err) {
-            await envelopeReader.close();
-            throw err;
-        }
+  }
+
+  static async openArrayBuffer(arrayBuffer) {
+    const readFn = async (start, length) => Buffer.from(arrayBuffer, start, length);
+
+    const closeFn = async () => {};
+
+    const size = arrayBuffer.byteLength;
+    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);
+
+    try {
+      await envelopeReader.readHeader();
+      const metadata = await envelopeReader.readFooter();
+      return new ParquetReader(metadata, envelopeReader);
+    } catch (err) {
+      await envelopeReader.close();
+      throw err;
     }
-    /**
-     * return a new parquet reader initialized with a read function
-     */
-    static async openArrayBuffer(arrayBuffer) {
-        const readFn = async (start, length) => Buffer.from(arrayBuffer, start, length);
-        const closeFn = async () => { };
-        const size = arrayBuffer.byteLength;
-        const envelopeReader = new parquet_envelope_reader_1.ParquetEnvelopeReader(readFn, closeFn, size);
-        try {
-            await envelopeReader.readHeader();
-            const metadata = await envelopeReader.readFooter();
-            return new ParquetReader(metadata, envelopeReader);
-        }
-        catch (err) {
-            await envelopeReader.close();
-            throw err;
-        }
+  }
+
+  static async openBuffer(buffer) {
+    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);
+
+    try {
+      await envelopeReader.readHeader();
+      const metadata = await envelopeReader.readFooter();
+      return new ParquetReader(metadata, envelopeReader);
+    } catch (err) {
+      await envelopeReader.close();
+      throw err;
     }
-    static async openBuffer(buffer) {
-        const envelopeReader = await parquet_envelope_reader_1.ParquetEnvelopeReader.openBuffer(buffer);
-        try {
-            await envelopeReader.readHeader();
-            const metadata = await envelopeReader.readFooter();
-            return new ParquetReader(metadata, envelopeReader);
-        }
-        catch (err) {
-            await envelopeReader.close();
-            throw err;
-        }
+  }
+
+  constructor(metadata, envelopeReader) {
+    _defineProperty(this, "metadata", void 0);
+
+    _defineProperty(this, "envelopeReader", void 0);
+
+    _defineProperty(this, "schema", void 0);
+
+    if (metadata.version !== PARQUET_VERSION) {
+      throw new Error('invalid parquet version');
     }
-    /**
-     * Close this parquet reader. You MUST call this method once you're finished
-     * reading rows
-     */
-    async close() {
-        await this.envelopeReader.close();
-        // this.envelopeReader = null;
-        // this.metadata = null;
+
+    this.metadata = metadata;
+    this.envelopeReader = envelopeReader;
+    const root = this.metadata.schema[0];
+    const {
+      schema
+    } = decodeSchema(this.metadata.schema, 1, root.num_children);
+    this.schema = new ParquetSchema(schema);
+  }
+
+  async close() {
+    await this.envelopeReader.close();
+  }
+
+  getCursor(columnList) {
+    if (!columnList) {
+      columnList = [];
     }
-    getCursor(columnList) {
-        if (!columnList) {
-            // tslint:disable-next-line:no-parameter-reassignment
-            columnList = [];
-        }
-        // tslint:disable-next-line:no-parameter-reassignment
-        columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));
-        return new parquet_cursor_1.ParquetCursor(this.metadata, this.envelopeReader, this.schema, columnList);
-    }
-    /**
-     * Return the number of rows in this file. Note that the number of rows is
-     * not neccessarily equal to the number of rows in each column.
-     */
-    getRowCount() {
-        return Number(this.metadata.num_rows);
-    }
-    /**
-     * Returns the ParquetSchema for this file
-     */
-    getSchema() {
-        return this.schema;
-    }
-    /**
-     * Returns the user (key/value) metadata for this file
-     */
-    getMetadata() {
-        const md = {};
-        for (const kv of this.metadata.key_value_metadata) {
-            md[kv.key] = kv.value;
-        }
-        return md;
-    }
-    /**
-     * Implement AsyncIterable
-     */
-    // tslint:disable-next-line:function-name
-    [Symbol.asyncIterator]() {
-        return this.getCursor()[Symbol.asyncIterator]();
+
+    columnList = columnList.map(x => Array.isArray(x) ? x : [x]);
+    return new ParquetCursor(this.metadata, this.envelopeReader, this.schema, columnList);
+  }
+
+  getRowCount() {
+    return Number(this.metadata.num_rows);
+  }
+
+  getSchema() {
+    return this.schema;
+  }
+
+  getMetadata() {
+    const md = {};
+
+    for (const kv of this.metadata.key_value_metadata) {
+      md[kv.key] = kv.value;
     }
+
+    return md;
+  }
+
+  [_Symbol$asyncIterator]() {
+    return this.getCursor()[Symbol.asyncIterator]();
+  }
+
 }
-exports.ParquetReader = ParquetReader;
+//# sourceMappingURL=parquet-reader.js.map

package/dist/parquetjs/parser/parquet-reader.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":["ParquetEnvelopeReader","ParquetSchema","ParquetCursor","PARQUET_VERSION","decodeSchema","Symbol","asyncIterator","ParquetReader","openBlob","blob","readFn","start","length","arrayBuffer","slice","Buffer","from","closeFn","size","envelopeReader","readHeader","metadata","readFooter","err","close","openArrayBuffer","byteLength","openBuffer","buffer","constructor","version","Error","root","schema","num_children","getCursor","columnList","map","x","Array","isArray","getRowCount","Number","num_rows","getSchema","getMetadata","md","kv","key_value_metadata","key","value"],"mappings":";;;;AACA,SAAQA,qBAAR,QAAoC,2BAApC;AAEA,SAAQC,aAAR,QAA4B,kBAA5B;AACA,SAAQC,aAAR,QAA4B,kBAA5B;AACA,SAAQC,eAAR,QAA8B,iBAA9B;AACA,SAAQC,YAAR,QAA2B,YAA3B;wBAyJGC,MAAM,CAACC,a;AAhJV,OAAO,MAAMC,aAAN,CAAmD;AAInC,eAARC,QAAQ,CAAIC,IAAJ,EAA2C;AAC9D,UAAMC,MAAM,GAAG,OAAOC,KAAP,EAAsBC,MAAtB,KAAyC;AACtD,YAAMC,WAAW,GAAG,MAAMJ,IAAI,CAACK,KAAL,CAAWH,KAAX,EAAkBA,KAAK,GAAGC,MAA1B,EAAkCC,WAAlC,EAA1B;AACA,aAAOE,MAAM,CAACC,IAAP,CAAYH,WAAZ,CAAP;AACD,KAHD;;AAIA,UAAMI,OAAO,GAAG,YAAY,CAAE,CAA9B;;AACA,UAAMC,IAAI,GAAGT,IAAI,CAACS,IAAlB;AACA,UAAMC,cAAc,GAAG,IAAInB,qBAAJ,CAA0BU,MAA1B,EAAkCO,OAAlC,EAA2CC,IAA3C,CAAvB;;AACA,QAAI;AACF,YAAMC,cAAc,CAACC,UAAf,EAAN;AACA,YAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAf,EAAvB;AACA,aAAO,IAAIf,aAAJ,CAAkBc,QAAlB,EAA4BF,cAA5B,CAAP;AACD,KAJD,CAIE,OAAOI,GAAP,EAAY;AACZ,YAAMJ,cAAc,CAACK,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAK2B,eAAfE,eAAe,CAAIZ,WAAJ,EAAyD;AACnF,UAAMH,MAAM,GAAG,OAAOC,KAAP,EAAsBC,MAAtB,KAAyCG,MAAM,CAACC,IAAP,CAAYH,WAAZ,EAAyBF,KAAzB,EAAgCC,MAAhC,CAAxD;;AACA,UAAMK,OAAO,GAAG,YAAY,CAAE,CAA9B;;AACA,UAAMC,IAAI,GAAGL,WAAW,CAACa,UAAzB;AACA,UAAMP,cAAc,GAAG,IAAInB,qBAAJ,CAA0BU,MAA1B,EAAkCO,OAAlC,EAA2CC,IAA3C,CAAvB;;AACA,QAAI;AACF,YAAMC,cAAc,CAACC,UAAf,EAAN;AACA,YAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAf,EAAvB;AACA,aAAO,IAAIf,aAAJ,CAAkBc,QAAlB,EAA4BF,cAA5B,CAAP;AACD,KAJD,CAIE,OAAOI,GAAP,EAAY;AACZ,YAAMJ,cAAc,CAACK,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAEsB,eAAVI,UAAU,CAAIC,MAAJ,EAA+C;AACpE,UAAMT,cAAc,GAAG,MAAMnB,qBAAqB,CAAC2B,UAAtB,CAAiCC,MAAjC,CAA7B;;AACA,QAAI;AACF,YAAMT,cAAc,CAACC,UAAf,EAAN;AACA,YAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAf,EAAvB;AACA,aAAO,IAAIf,aAAJ,CAAqBc,QAArB,EAA+BF,cAA/B,CAAP;AACD,KAJD,CAIE,OAAOI,GAAP,EAAY;AACZ,YAAMJ,cAAc,CAACK,KAAf,EAAN;AACA,YAAMD,GAAN;AACD;AACF;;AAYDM,EAAAA,WAAW,CAACR,QAAD,EAAyBF,cAAzB,EAAgE;AAAA;;AAAA;;AAAA;;AACzE,QAAIE,QAAQ,CAACS,OAAT,KAAqB3B,eAAzB,EAA0C;AACxC,YAAM,IAAI4B,KAAJ,CAAU,yBAAV,CAAN;AACD;;AAED,SAAKV,QAAL,GAAgBA,QAAhB;AACA,SAAKF,cAAL,GAAsBA,cAAtB;AACA,UAAMa,IAAI,GAAG,KAAKX,QAAL,CAAcY,MAAd,CAAqB,CAArB,CAAb;AACA,UAAM;AAACA,MAAAA;AAAD,QAAW7B,YAAY,CAAC,KAAKiB,QAAL,CAAcY,MAAf,EAAuB,CAAvB,EAA0BD,IAAI,CAACE,YAA/B,CAA7B;AACA,SAAKD,MAAL,GAAc,IAAIhC,aAAJ,CAAkBgC,MAAlB,CAAd;AACD;;AAMU,QAALT,KAAK,GAAkB;AAC3B,UAAM,KAAKL,cAAL,CAAoBK,KAApB,EAAN;AAGD;;AAeDW,EAAAA,SAAS,CAACC,UAAD,EAAgE;AACvE,QAAI,CAACA,UAAL,EAAiB;AAEfA,MAAAA,UAAU,GAAG,EAAb;AACD;;AAGDA,IAAAA,UAAU,GAAGA,UAAU,CAACC,GAAX,CAAgBC,CAAD,IAAQC,KAAK,CAACC,OAAN,CAAcF,CAAd,IAAmBA,CAAnB,GAAuB,CAACA,CAAD,CAA9C,CAAb;AAEA,WAAO,IAAIpC,aAAJ,CACL,KAAKmB,QADA,EAEL,KAAKF,cAFA,EAGL,KAAKc,MAHA,EAILG,UAJK,CAAP;AAMD;;AAMDK,EAAAA,WAAW,GAAW;AACpB,WAAOC,MAAM,CAAC,KAAKrB,QAAL,CAAcsB,QAAf,CAAb;AACD;;AAKDC,EAAAA,SAAS,GAAkB;AACzB,WAAO,KAAKX,MAAZ;AACD;;AAKDY,EAAAA,WAAW,GAA2B;AACpC,UAAMC,EAA0B,GAAG,EAAnC;;AACA,SAAK,MAAMC,EAAX,IAAiB,KAAK1B,QAAL,CAAc2B,kBAA/B,EAAoD;AAClDF,MAAAA,EAAE,CAACC,EAAE,CAACE,GAAJ,CAAF,GAAaF,EAAE,CAACG,KAAhB;AACD;;AACD,WAAOJ,EAAP;AACD;;AAMD,4BAA2C;AACzC,WAAO,KAAKX,SAAL,GAAiB9B,MAAM,CAACC,aAAxB,GAAP;AACD;;AAlJuD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetCursor} from './parquet-cursor';\nimport {PARQUET_VERSION} from '../../constants';\nimport {decodeSchema} from './decoders';\n\n/**\n * A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n */\nexport class ParquetReader<T> implements AsyncIterable<T> {\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openBlob<T>(blob: Blob): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => {\n      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();\n      return Buffer.from(arrayBuffer);\n    };\n    const closeFn = async () => {};\n    const size = blob.size;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n    const closeFn = async () => {};\n    const size = arrayBuffer.byteLength;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is not recommended to call this constructor directly except for advanced\n   * and internal use cases. Consider using one of the open{File,Buffer} methods\n   * instead\n   */\n  constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n    if (metadata.version !== PARQUET_VERSION) {\n      throw new Error('invalid parquet version');\n    }\n\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    const root = this.metadata.schema[0];\n    const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n    this.schema = new ParquetSchema(schema);\n  }\n\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  async close(): Promise<void> {\n    await this.envelopeReader.close();\n    // this.envelopeReader = null;\n    // this.metadata = null;\n  }\n\n  /**\n   * Return a cursor to the file. You may open more than one cursor and use\n   * them concurrently. All cursors become invalid once close() is called on\n   * the reader object.\n   *\n   * The required_columns parameter controls which columns are actually read\n   * from disk. An empty array or no value implies all columns. A list of column\n   * names means that only those columns should be loaded from disk.\n   */\n  getCursor(): ParquetCursor<T>;\n  // @ts-ignore\n  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;\n  getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;\n  getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {\n    if (!columnList) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      columnList = [];\n    }\n\n    // tslint:disable-next-line:no-parameter-reassignment\n    columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n    return new ParquetCursor<T>(\n      this.metadata,\n      this.envelopeReader,\n      this.schema,\n      columnList as string[][]\n    );\n  }\n\n  /**\n   * Return the number of rows in this file. Note that the number of rows is\n   * not neccessarily equal to the number of rows in each column.\n   */\n  getRowCount(): number {\n    return Number(this.metadata.num_rows);\n  }\n\n  /**\n   * Returns the ParquetSchema for this file\n   */\n  getSchema(): ParquetSchema {\n    return this.schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   */\n  getMetadata(): Record<string, string> {\n    const md: Record<string, string> = {};\n    for (const kv of this.metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    return this.getCursor()[Symbol.asyncIterator]();\n  }\n}\n"],"file":"parquet-reader.js"}

package/dist/parquetjs/schema/declare.js

@@ -1,10 +1,13 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetBuffer = void 0;
-class ParquetBuffer {
-    constructor(rowCount = 0, columnData = {}) {
-        this.rowCount = rowCount;
-        this.columnData = columnData;
-    }
+import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
+export class ParquetBuffer {
+  constructor(rowCount = 0, columnData = {}) {
+    _defineProperty(this, "rowCount", void 0);
+
+    _defineProperty(this, "columnData", void 0);
+
+    this.rowCount = rowCount;
+    this.columnData = columnData;
+  }
+
 }
-exports.ParquetBuffer = ParquetBuffer;
+//# sourceMappingURL=declare.js.map

package/dist/parquetjs/schema/declare.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../src/parquetjs/schema/declare.ts"],"names":["ParquetBuffer","constructor","rowCount","columnData"],"mappings":";AAmIA,OAAO,MAAMA,aAAN,CAAoB;AAGzBC,EAAAA,WAAW,CAACC,QAAgB,GAAG,CAApB,EAAuBC,UAAuC,GAAG,EAAjE,EAAqE;AAAA;;AAAA;;AAC9E,SAAKD,QAAL,GAAgBA,QAAhB;AACA,SAAKC,UAAL,GAAkBA,UAAlB;AACD;;AANwB","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  dictionary?: ParquetDictionary;\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\nexport class ParquetBuffer {\n  rowCount: number;\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n"],"file":"declare.js"}

package/dist/parquetjs/schema/schema.js

@@ -1,162 +1,176 @@
-"use strict";
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetSchema = void 0;
-const codecs_1 = require("../codecs");
-const compression_1 = require("../compression");
-const shred_1 = require("./shred");
-const types_1 = require("./types");
-/**
- * A parquet file schema
- */
-class ParquetSchema {
-    /**
-     * Create a new schema from a JSON schema definition
-     */
-    constructor(schema) {
-        this.schema = schema;
-        this.fields = buildFields(schema, 0, 0, []);
-        this.fieldList = listFields(this.fields);
+import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
+import { PARQUET_CODECS } from '../codecs';
+import { PARQUET_COMPRESSION_METHODS } from '../compression';
+import { materializeRecords, shredBuffer, shredRecord } from './shred';
+import { PARQUET_LOGICAL_TYPES } from './types';
+export class ParquetSchema {
+  constructor(schema) {
+    _defineProperty(this, "schema", void 0);
+
+    _defineProperty(this, "fields", void 0);
+
+    _defineProperty(this, "fieldList", void 0);
+
+    this.schema = schema;
+    this.fields = buildFields(schema, 0, 0, []);
+    this.fieldList = listFields(this.fields);
+  }
+
+  findField(path) {
+    if (typeof path === 'string') {
+      path = path.split(',');
+    } else {
+      path = path.slice(0);
     }
-    /**
-     * Retrieve a field definition
-     */
-    findField(path) {
-        if (typeof path === 'string') {
-            // tslint:disable-next-line:no-parameter-reassignment
-            path = path.split(',');
-        }
-        else {
-            // tslint:disable-next-line:no-parameter-reassignment
-            path = path.slice(0); // clone array
-        }
-        let n = this.fields;
-        for (; path.length > 1; path.shift()) {
-            n = n[path[0]].fields;
-        }
-        return n[path[0]];
+
+    let n = this.fields;
+
+    for (; path.length > 1; path.shift()) {
+      n = n[path[0]].fields;
     }
-    /**
-     * Retrieve a field definition and all the field's ancestors
-     */
-    findFieldBranch(path) {
-        if (typeof path === 'string') {
-            // tslint:disable-next-line:no-parameter-reassignment
-            path = path.split(',');
-        }
-        const branch = [];
-        let n = this.fields;
-        for (; path.length > 0; path.shift()) {
-            branch.push(n[path[0]]);
-            if (path.length > 1) {
-                n = n[path[0]].fields;
-            }
-        }
-        return branch;
+
+    return n[path[0]];
+  }
+
+  findFieldBranch(path) {
+    if (typeof path === 'string') {
+      path = path.split(',');
     }
-    shredRecord(record, buffer) {
-        (0, shred_1.shredRecord)(this, record, buffer);
-    }
-    materializeRecords(buffer) {
-        return (0, shred_1.materializeRecords)(this, buffer);
-    }
-    compress(type) {
-        setCompress(this.schema, type);
-        setCompress(this.fields, type);
-        return this;
-    }
-    buffer() {
-        return (0, shred_1.shredBuffer)(this);
+
+    const branch = [];
+    let n = this.fields;
+
+    for (; path.length > 0; path.shift()) {
+      branch.push(n[path[0]]);
+
+      if (path.length > 1) {
+        n = n[path[0]].fields;
+      }
     }
+
+    return branch;
+  }
+
+  shredRecord(record, buffer) {
+    shredRecord(this, record, buffer);
+  }
+
+  materializeRecords(buffer) {
+    return materializeRecords(this, buffer);
+  }
+
+  compress(type) {
+    setCompress(this.schema, type);
+    setCompress(this.fields, type);
+    return this;
+  }
+
+  buffer() {
+    return shredBuffer(this);
+  }
+
 }
-exports.ParquetSchema = ParquetSchema;
+
 function setCompress(schema, type) {
-    for (const name in schema) {
-        const node = schema[name];
-        if (node.fields) {
-            setCompress(node.fields, type);
-        }
-        else {
-            node.compression = type;
-        }
+  for (const name in schema) {
+    const node = schema[name];
+
+    if (node.fields) {
+      setCompress(node.fields, type);
+    } else {
+      node.compression = type;
     }
+  }
 }
-// eslint-disable-next-line max-statements, complexity
+
 function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
-    const fieldList = {};
-    for (const name in schema) {
-        const opts = schema[name];
-        /* field repetition type */
-        const required = !opts.optional;
-        const repeated = Boolean(opts.repeated);
-        let rLevelMax = rLevelParentMax;
-        let dLevelMax = dLevelParentMax;
-        let repetitionType = 'REQUIRED';
-        if (!required) {
-            repetitionType = 'OPTIONAL';
-            dLevelMax++;
-        }
-        if (repeated) {
-            repetitionType = 'REPEATED';
-            rLevelMax++;
-            if (required)
-                dLevelMax++;
-        }
-        /* nested field */
-        if (opts.fields) {
-            const cpath = path.concat([name]);
-            fieldList[name] = {
-                name,
-                path: cpath,
-                key: cpath.join(),
-                repetitionType,
-                rLevelMax,
-                dLevelMax,
-                isNested: true,
-                fieldCount: Object.keys(opts.fields).length,
-                fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
-            };
-            continue; // eslint-disable-line no-continue
-        }
-        const typeDef = types_1.PARQUET_LOGICAL_TYPES[opts.type];
-        if (!typeDef) {
-            throw new Error(`invalid parquet type: ${opts.type}`);
-        }
-        opts.encoding = opts.encoding || 'PLAIN';
-        if (!(opts.encoding in codecs_1.PARQUET_CODECS)) {
-            throw new Error(`unsupported parquet encoding: ${opts.encoding}`);
-        }
-        opts.compression = opts.compression || 'UNCOMPRESSED';
-        if (!(opts.compression in compression_1.PARQUET_COMPRESSION_METHODS)) {
-            throw new Error(`unsupported compression method: ${opts.compression}`);
-        }
-        /* add to schema */
-        const cpath = path.concat([name]);
-        fieldList[name] = {
-            name,
-            primitiveType: typeDef.primitiveType,
-            originalType: typeDef.originalType,
-            path: cpath,
-            key: cpath.join(),
-            repetitionType,
-            encoding: opts.encoding,
-            compression: opts.compression,
-            typeLength: opts.typeLength || typeDef.typeLength,
-            presision: opts.presision,
-            scale: opts.scale,
-            rLevelMax,
-            dLevelMax
-        };
+  const fieldList = {};
+
+  for (const name in schema) {
+    const opts = schema[name];
+    const required = !opts.optional;
+    const repeated = Boolean(opts.repeated);
+    let rLevelMax = rLevelParentMax;
+    let dLevelMax = dLevelParentMax;
+    let repetitionType = 'REQUIRED';
+
+    if (!required) {
+      repetitionType = 'OPTIONAL';
+      dLevelMax++;
+    }
+
+    if (repeated) {
+      repetitionType = 'REPEATED';
+      rLevelMax++;
+      if (required) dLevelMax++;
+    }
+
+    if (opts.fields) {
+      const cpath = path.concat([name]);
+      fieldList[name] = {
+        name,
+        path: cpath,
+        key: cpath.join(),
+        repetitionType,
+        rLevelMax,
+        dLevelMax,
+        isNested: true,
+        fieldCount: Object.keys(opts.fields).length,
+        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
+      };
+      continue;
+    }
+
+    const typeDef = PARQUET_LOGICAL_TYPES[opts.type];
+
+    if (!typeDef) {
+      throw new Error("invalid parquet type: ".concat(opts.type));
+    }
+
+    opts.encoding = opts.encoding || 'PLAIN';
+
+    if (!(opts.encoding in PARQUET_CODECS)) {
+      throw new Error("unsupported parquet encoding: ".concat(opts.encoding));
+    }
+
+    opts.compression = opts.compression || 'UNCOMPRESSED';
+
+    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {
+      throw new Error("unsupported compression method: ".concat(opts.compression));
     }
-    return fieldList;
+
+    const cpath = path.concat([name]);
+    fieldList[name] = {
+      name,
+      primitiveType: typeDef.primitiveType,
+      originalType: typeDef.originalType,
+      path: cpath,
+      key: cpath.join(),
+      repetitionType,
+      encoding: opts.encoding,
+      compression: opts.compression,
+      typeLength: opts.typeLength || typeDef.typeLength,
+      presision: opts.presision,
+      scale: opts.scale,
+      rLevelMax,
+      dLevelMax
+    };
+  }
+
+  return fieldList;
 }
+
 function listFields(fields) {
-    let list = [];
-    for (const k in fields) {
-        list.push(fields[k]);
-        if (fields[k].isNested) {
-            list = list.concat(listFields(fields[k].fields));
-        }
+  let list = [];
+
+  for (const k in fields) {
+    list.push(fields[k]);
+
+    if (fields[k].isNested) {
+      list = list.concat(listFields(fields[k].fields));
     }
-    return list;
+  }
+
+  return list;
 }
+//# sourceMappingURL=schema.js.map