@loaders.gl/parquet 3.1.0-alpha.5 → 3.1.0-beta.5

package/dist/parquetjs/parser/parquet-envelope-reader.js

@@ -0,0 +1,136 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetEnvelopeReader = void 0;
+const constants_1 = require("../../constants");
+const parquet_thrift_1 = require("../parquet-thrift");
+const read_utils_1 = require("../utils/read-utils");
+const decoders_1 = require("./decoders");
+const DEFAULT_DICTIONARY_SIZE = 1e6;
+/**
+ * The parquet envelope reader allows direct, unbuffered access to the individual
+ * sections of the parquet file, namely the header, footer and the row groups.
+ * This class is intended for advanced/internal users; if you just want to retrieve
+ * rows from a parquet file use the ParquetReader instead
+ */
+class ParquetEnvelopeReader {
+    constructor(read, close, fileSize, options) {
+        this.read = read;
+        this.close = close;
+        this.fileSize = fileSize;
+        this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;
+    }
+    static async openBuffer(buffer) {
+        const readFn = (position, length) => Promise.resolve(buffer.slice(position, position + length));
+        const closeFn = () => Promise.resolve();
+        return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
+    }
+    async readHeader() {
+        const buffer = await this.read(0, constants_1.PARQUET_MAGIC.length);
+        const magic = buffer.toString();
+        switch (magic) {
+            case constants_1.PARQUET_MAGIC:
+                break;
+            case constants_1.PARQUET_MAGIC_ENCRYPTED:
+                throw new Error('Encrypted parquet file not supported');
+            default:
+                throw new Error(`Invalid parquet file (magic=${magic})`);
+        }
+    }
+    async readRowGroup(schema, rowGroup, columnList) {
+        const buffer = {
+            rowCount: Number(rowGroup.num_rows),
+            columnData: {}
+        };
+        for (const colChunk of rowGroup.columns) {
+            const colMetadata = colChunk.meta_data;
+            const colKey = colMetadata?.path_in_schema;
+            if (columnList.length > 0 && (0, read_utils_1.fieldIndexOf)(columnList, colKey) < 0) {
+                continue; // eslint-disable-line no-continue
+            }
+            buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
+        }
+        return buffer;
+    }
+    /**
+     * Do reading of parquet file's column chunk
+     * @param schema
+     * @param colChunk
+     */
+    async readColumnChunk(schema, colChunk) {
+        if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
+            throw new Error('external references are not supported');
+        }
+        const field = schema.findField(colChunk.meta_data?.path_in_schema);
+        const type = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.Type, colChunk.meta_data?.type);
+        if (type !== field.primitiveType) {
+            throw new Error(`chunk type not matching schema: ${type}`);
+        }
+        const compression = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.CompressionCodec, colChunk.meta_data?.codec);
+        const pagesOffset = Number(colChunk.meta_data?.data_page_offset);
+        let pagesSize = Number(colChunk.meta_data?.total_compressed_size);
+        if (!colChunk.file_path) {
+            pagesSize = Math.min(this.fileSize - pagesOffset, Number(colChunk.meta_data?.total_compressed_size));
+        }
+        const options = {
+            type,
+            rLevelMax: field.rLevelMax,
+            dLevelMax: field.dLevelMax,
+            compression,
+            column: field,
+            numValues: colChunk.meta_data?.num_values,
+            dictionary: []
+        };
+        let dictionary;
+        const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;
+        if (dictionaryPageOffset) {
+            const dictionaryOffset = Number(dictionaryPageOffset);
+            // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.
+            dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
+        }
+        dictionary = options.dictionary?.length ? options.dictionary : dictionary;
+        const pagesBuf = await this.read(pagesOffset, pagesSize);
+        return await (0, decoders_1.decodeDataPages)(pagesBuf, { ...options, dictionary });
+    }
+    /**
+     * Getting dictionary for allows to flatten values by indices.
+     * @param dictionaryPageOffset
+     * @param options
+     * @param pagesOffset
+     * @returns
+     */
+    async getDictionary(dictionaryPageOffset, options, pagesOffset) {
+        if (dictionaryPageOffset === 0) {
+            // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);
+            // pagesBuf = await this.read(pagesOffset, dictionarySize);
+            // In this case we are working with parquet-mr files format. Problem is described below:
+            // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding
+            // We need to get dictionary page from column chunk if it exists.
+            // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.
+            return [];
+        }
+        const dictionarySize = Math.min(this.fileSize - dictionaryPageOffset, this.defaultDictionarySize);
+        const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);
+        const cursor = { buffer: pagesBuf, offset: 0, size: pagesBuf.length };
+        const decodedPage = await (0, decoders_1.decodePage)(cursor, options);
+        return decodedPage.dictionary;
+    }
+    async readFooter() {
+        const trailerLen = constants_1.PARQUET_MAGIC.length + 4;
+        const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);
+        const magic = trailerBuf.slice(4).toString();
+        if (magic !== constants_1.PARQUET_MAGIC) {
+            throw new Error(`Not a valid parquet file (magic="${magic})`);
+        }
+        const metadataSize = trailerBuf.readUInt32LE(0);
+        const metadataOffset = this.fileSize - metadataSize - trailerLen;
+        if (metadataOffset < constants_1.PARQUET_MAGIC.length) {
+            throw new Error(`Invalid metadata size ${metadataOffset}`);
+        }
+        const metadataBuf = await this.read(metadataOffset, metadataSize);
+        // let metadata = new parquet_thrift.FileMetaData();
+        // parquet_util.decodeThrift(metadata, metadataBuf);
+        const { metadata } = (0, read_utils_1.decodeFileMetadata)(metadataBuf);
+        return metadata;
+    }
+}
+exports.ParquetEnvelopeReader = ParquetEnvelopeReader;

package/dist/parquetjs/parser/parquet-reader.d.ts

@@ -19,11 +19,6 @@ export declare class ParquetReader<T> implements AsyncIterable<T> {
      * return a new parquet reader initialized with a read function
      */
     static openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>>;
-    /**
-     * Open the parquet file pointed to by the specified path and return a new
-     * parquet reader
-     */
-    static openFile<T>(filePath: string): Promise<ParquetReader<T>>;
     static openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>>;
     metadata: FileMetaData;
     envelopeReader: ParquetEnvelopeReader;
@@ -70,3 +65,4 @@ export declare class ParquetReader<T> implements AsyncIterable<T> {
      */
     [Symbol.asyncIterator](): AsyncIterator<T>;
 }
+//# sourceMappingURL=parquet-reader.d.ts.map

package/dist/parquetjs/parser/parquet-reader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":";AACA,OAAO,EAAC,qBAAqB,EAAC,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAI/C;;;;;;GAMG;AACH,qBAAa,aAAa,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,CAAC;IACvD;;OAEG;WACU,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAkB/D;;OAEG;WACU,eAAe,CAAC,CAAC,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;WAevE,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAY9D,QAAQ,EAAE,YAAY,CAAC;IACvB,cAAc,EAAE,qBAAqB,CAAC;IACtC,MAAM,EAAE,aAAa,CAAC;IAE7B;;;;;OAKG;gBACS,QAAQ,EAAE,YAAY,EAAE,cAAc,EAAE,qBAAqB;IAYzE;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B;;;;;;;;OAQG;IACH,SAAS,IAAI,aAAa,CAAC,CAAC,CAAC;IAE7B,SAAS,CAAC,CAAC,SAAS,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChF,SAAS,CAAC,UAAU,EAAE,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,EAAE,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAkBvE;;;OAGG;IACH,WAAW,IAAI,MAAM;IAIrB;;OAEG;IACH,SAAS,IAAI,aAAa;IAI1B;;OAEG;IACH,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAQrC;;OAEG;IAEH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;CAG3C"}

package/dist/parquetjs/parser/parquet-reader.js

@@ -0,0 +1,134 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetReader = void 0;
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+const parquet_envelope_reader_1 = require("./parquet-envelope-reader");
+const schema_1 = require("../schema/schema");
+const parquet_cursor_1 = require("./parquet-cursor");
+const constants_1 = require("../../constants");
+const decoders_1 = require("./decoders");
+/**
+ * A parquet reader allows retrieving the rows from a parquet file in order.
+ * The basic usage is to create a reader and then retrieve a cursor/iterator
+ * which allows you to consume row after row until all rows have been read. It is
+ * important that you call close() after you are finished reading the file to
+ * avoid leaking file descriptors.
+ */
+class ParquetReader {
+    /**
+     * Create a new parquet reader from the file metadata and an envelope reader.
+     * It is not recommended to call this constructor directly except for advanced
+     * and internal use cases. Consider using one of the open{File,Buffer} methods
+     * instead
+     */
+    constructor(metadata, envelopeReader) {
+        if (metadata.version !== constants_1.PARQUET_VERSION) {
+            throw new Error('invalid parquet version');
+        }
+        this.metadata = metadata;
+        this.envelopeReader = envelopeReader;
+        const root = this.metadata.schema[0];
+        const { schema } = (0, decoders_1.decodeSchema)(this.metadata.schema, 1, root.num_children);
+        this.schema = new schema_1.ParquetSchema(schema);
+    }
+    /**
+     * return a new parquet reader initialized with a read function
+     */
+    static async openBlob(blob) {
+        const readFn = async (start, length) => {
+            const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();
+            return Buffer.from(arrayBuffer);
+        };
+        const closeFn = async () => { };
+        const size = blob.size;
+        const envelopeReader = new parquet_envelope_reader_1.ParquetEnvelopeReader(readFn, closeFn, size);
+        try {
+            await envelopeReader.readHeader();
+            const metadata = await envelopeReader.readFooter();
+            return new ParquetReader(metadata, envelopeReader);
+        }
+        catch (err) {
+            await envelopeReader.close();
+            throw err;
+        }
+    }
+    /**
+     * return a new parquet reader initialized with a read function
+     */
+    static async openArrayBuffer(arrayBuffer) {
+        const readFn = async (start, length) => Buffer.from(arrayBuffer, start, length);
+        const closeFn = async () => { };
+        const size = arrayBuffer.byteLength;
+        const envelopeReader = new parquet_envelope_reader_1.ParquetEnvelopeReader(readFn, closeFn, size);
+        try {
+            await envelopeReader.readHeader();
+            const metadata = await envelopeReader.readFooter();
+            return new ParquetReader(metadata, envelopeReader);
+        }
+        catch (err) {
+            await envelopeReader.close();
+            throw err;
+        }
+    }
+    static async openBuffer(buffer) {
+        const envelopeReader = await parquet_envelope_reader_1.ParquetEnvelopeReader.openBuffer(buffer);
+        try {
+            await envelopeReader.readHeader();
+            const metadata = await envelopeReader.readFooter();
+            return new ParquetReader(metadata, envelopeReader);
+        }
+        catch (err) {
+            await envelopeReader.close();
+            throw err;
+        }
+    }
+    /**
+     * Close this parquet reader. You MUST call this method once you're finished
+     * reading rows
+     */
+    async close() {
+        await this.envelopeReader.close();
+        // this.envelopeReader = null;
+        // this.metadata = null;
+    }
+    getCursor(columnList) {
+        if (!columnList) {
+            // tslint:disable-next-line:no-parameter-reassignment
+            columnList = [];
+        }
+        // tslint:disable-next-line:no-parameter-reassignment
+        columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));
+        return new parquet_cursor_1.ParquetCursor(this.metadata, this.envelopeReader, this.schema, columnList);
+    }
+    /**
+     * Return the number of rows in this file. Note that the number of rows is
+     * not neccessarily equal to the number of rows in each column.
+     */
+    getRowCount() {
+        return Number(this.metadata.num_rows);
+    }
+    /**
+     * Returns the ParquetSchema for this file
+     */
+    getSchema() {
+        return this.schema;
+    }
+    /**
+     * Returns the user (key/value) metadata for this file
+     */
+    getMetadata() {
+        const md = {};
+        for (const kv of this.metadata.key_value_metadata) {
+            md[kv.key] = kv.value;
+        }
+        return md;
+    }
+    /**
+     * Implement AsyncIterable
+     */
+    // tslint:disable-next-line:function-name
+    [Symbol.asyncIterator]() {
+        return this.getCursor()[Symbol.asyncIterator]();
+    }
+}
+exports.ParquetReader = ParquetReader;

package/dist/parquetjs/schema/declare.d.ts

@@ -77,3 +77,4 @@ export declare class ParquetBuffer {
     columnData: Record<string, ParquetData>;
     constructor(rowCount?: number, columnData?: Record<string, ParquetData>);
 }
+//# sourceMappingURL=declare.d.ts.map

package/dist/parquetjs/schema/declare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,oBAAY,YAAY,GAAG,OAAO,GAAG,KAAK,GAAG,kBAAkB,CAAC;AAChE,oBAAY,kBAAkB,GAC1B,cAAc,GACd,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,KAAK,GACL,SAAS,GACT,MAAM,CAAC;AACX,oBAAY,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAClE,oBAAY,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAEvD;;GAEG;AACH,oBAAY,aAAa,GAErB,SAAS,GACT,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,sBAAsB,CAAC;AAE3B;;GAEG;AACH,oBAAY,YAAY,GAEpB,MAAM,GAMN,eAAe,GACf,eAAe,GACf,oBAAoB,GACpB,8BAA8B,GAC9B,MAAM,GACN,aAAa,GACb,aAAa,GACb,kBAAkB,GAClB,kBAAkB,GAClB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,CAAC;AAEf,oBAAY,iBAAiB,GAAG,MAAM,EAAE,CAAC;AAEzC,MAAM,WAAW,gBAAgB;IAC/B,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,EAAE,cAAc,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,qBAAa,aAAa;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBAC5B,QAAQ,GAAE,MAAU,EAAE,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAM;CAI/E"}

package/dist/parquetjs/schema/declare.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetBuffer = void 0;
+class ParquetBuffer {
+    constructor(rowCount = 0, columnData = {}) {
+        this.rowCount = rowCount;
+        this.columnData = columnData;
+    }
+}
+exports.ParquetBuffer = ParquetBuffer;

package/dist/parquetjs/schema/schema.d.ts

@@ -23,3 +23,4 @@ export declare class ParquetSchema {
     compress(type: ParquetCompression): this;
     buffer(): ParquetBuffer;
 }
+//# sourceMappingURL=schema.d.ts.map

package/dist/parquetjs/schema/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/schema.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,aAAa,EAEb,gBAAgB,EACjB,MAAM,WAAW,CAAC;AAInB;;GAEG;AACH,qBAAa,aAAa;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACrC,SAAS,EAAE,YAAY,EAAE,CAAC;IAEjC;;OAEG;gBACS,MAAM,EAAE,gBAAgB;IAMpC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,YAAY;IAiBhD;;OAEG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,YAAY,EAAE;IAgBxD,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI;IAI/D,kBAAkB,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,EAAE;IAI1D,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI;IAMxC,MAAM,IAAI,aAAa;CAGxB"}

package/dist/parquetjs/schema/schema.js

@@ -0,0 +1,162 @@
+"use strict";
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetSchema = void 0;
+const codecs_1 = require("../codecs");
+const compression_1 = require("../compression");
+const shred_1 = require("./shred");
+const types_1 = require("./types");
+/**
+ * A parquet file schema
+ */
+class ParquetSchema {
+    /**
+     * Create a new schema from a JSON schema definition
+     */
+    constructor(schema) {
+        this.schema = schema;
+        this.fields = buildFields(schema, 0, 0, []);
+        this.fieldList = listFields(this.fields);
+    }
+    /**
+     * Retrieve a field definition
+     */
+    findField(path) {
+        if (typeof path === 'string') {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.split(',');
+        }
+        else {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.slice(0); // clone array
+        }
+        let n = this.fields;
+        for (; path.length > 1; path.shift()) {
+            n = n[path[0]].fields;
+        }
+        return n[path[0]];
+    }
+    /**
+     * Retrieve a field definition and all the field's ancestors
+     */
+    findFieldBranch(path) {
+        if (typeof path === 'string') {
+            // tslint:disable-next-line:no-parameter-reassignment
+            path = path.split(',');
+        }
+        const branch = [];
+        let n = this.fields;
+        for (; path.length > 0; path.shift()) {
+            branch.push(n[path[0]]);
+            if (path.length > 1) {
+                n = n[path[0]].fields;
+            }
+        }
+        return branch;
+    }
+    shredRecord(record, buffer) {
+        (0, shred_1.shredRecord)(this, record, buffer);
+    }
+    materializeRecords(buffer) {
+        return (0, shred_1.materializeRecords)(this, buffer);
+    }
+    compress(type) {
+        setCompress(this.schema, type);
+        setCompress(this.fields, type);
+        return this;
+    }
+    buffer() {
+        return (0, shred_1.shredBuffer)(this);
+    }
+}
+exports.ParquetSchema = ParquetSchema;
+function setCompress(schema, type) {
+    for (const name in schema) {
+        const node = schema[name];
+        if (node.fields) {
+            setCompress(node.fields, type);
+        }
+        else {
+            node.compression = type;
+        }
+    }
+}
+// eslint-disable-next-line max-statements, complexity
+function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
+    const fieldList = {};
+    for (const name in schema) {
+        const opts = schema[name];
+        /* field repetition type */
+        const required = !opts.optional;
+        const repeated = Boolean(opts.repeated);
+        let rLevelMax = rLevelParentMax;
+        let dLevelMax = dLevelParentMax;
+        let repetitionType = 'REQUIRED';
+        if (!required) {
+            repetitionType = 'OPTIONAL';
+            dLevelMax++;
+        }
+        if (repeated) {
+            repetitionType = 'REPEATED';
+            rLevelMax++;
+            if (required)
+                dLevelMax++;
+        }
+        /* nested field */
+        if (opts.fields) {
+            const cpath = path.concat([name]);
+            fieldList[name] = {
+                name,
+                path: cpath,
+                key: cpath.join(),
+                repetitionType,
+                rLevelMax,
+                dLevelMax,
+                isNested: true,
+                fieldCount: Object.keys(opts.fields).length,
+                fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
+            };
+            continue; // eslint-disable-line no-continue
+        }
+        const typeDef = types_1.PARQUET_LOGICAL_TYPES[opts.type];
+        if (!typeDef) {
+            throw new Error(`invalid parquet type: ${opts.type}`);
+        }
+        opts.encoding = opts.encoding || 'PLAIN';
+        if (!(opts.encoding in codecs_1.PARQUET_CODECS)) {
+            throw new Error(`unsupported parquet encoding: ${opts.encoding}`);
+        }
+        opts.compression = opts.compression || 'UNCOMPRESSED';
+        if (!(opts.compression in compression_1.PARQUET_COMPRESSION_METHODS)) {
+            throw new Error(`unsupported compression method: ${opts.compression}`);
+        }
+        /* add to schema */
+        const cpath = path.concat([name]);
+        fieldList[name] = {
+            name,
+            primitiveType: typeDef.primitiveType,
+            originalType: typeDef.originalType,
+            path: cpath,
+            key: cpath.join(),
+            repetitionType,
+            encoding: opts.encoding,
+            compression: opts.compression,
+            typeLength: opts.typeLength || typeDef.typeLength,
+            presision: opts.presision,
+            scale: opts.scale,
+            rLevelMax,
+            dLevelMax
+        };
+    }
+    return fieldList;
+}
+function listFields(fields) {
+    let list = [];
+    for (const k in fields) {
+        list.push(fields[k]);
+        if (fields[k].isNested) {
+            list = list.concat(listFields(fields[k].fields));
+        }
+    }
+    return list;
+}

package/dist/parquetjs/schema/shred.d.ts

@@ -45,3 +45,4 @@ export declare function shredRecord(schema: ParquetSchema, record: any, buffer:
  *   }
  */
 export declare function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[];
+//# sourceMappingURL=shred.d.ts.map

package/dist/parquetjs/schema/shred.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,aAAa,EAA6B,aAAa,EAAC,MAAM,WAAW,CAAC;AAClF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,aAAa,EAAC,CAAC;AAEvB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAYhE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI,CAmB3F;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,aAAa,EAAE,CAOhG"}