@loaders.gl/parquet 3.3.0-alpha.7 → 3.3.0

package/dist/parquetjs/parser/parquet-reader.js

@@ -1,134 +1,200 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ParquetReader = void 0;
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-const parquet_envelope_reader_1 = require("./parquet-envelope-reader");
 const schema_1 = require("../schema/schema");
-const parquet_cursor_1 = require("./parquet-cursor");
-const constants_1 = require("../../constants");
 const decoders_1 = require("./decoders");
+const shred_1 = require("../schema/shred");
+const constants_1 = require("../../constants");
+const parquet_thrift_1 = require("../parquet-thrift");
+const read_utils_1 = require("../utils/read-utils");
+const decoders_2 = require("./decoders");
+const DEFAULT_PROPS = {
+    defaultDictionarySize: 1e6
+};
 /**
- * A parquet reader allows retrieving the rows from a parquet file in order.
- * The basic usage is to create a reader and then retrieve a cursor/iterator
- * which allows you to consume row after row until all rows have been read. It is
- * important that you call close() after you are finished reading the file to
- * avoid leaking file descriptors.
+ * The parquet envelope reader allows direct, unbuffered access to the individual
+ * sections of the parquet file, namely the header, footer and the row groups.
+ * This class is intended for advanced/internal users; if you just want to retrieve
+ * rows from a parquet file use the ParquetReader instead
  */
 class ParquetReader {
-    /**
-     * Create a new parquet reader from the file metadata and an envelope reader.
-     * It is not recommended to call this constructor directly except for advanced
-     * and internal use cases. Consider using one of the open{File,Buffer} methods
-     * instead
-     */
-    constructor(metadata, envelopeReader) {
-        if (metadata.version !== constants_1.PARQUET_VERSION) {
-            throw new Error('invalid parquet version');
+    constructor(file, props) {
+        this.metadata = null;
+        this.file = file;
+        this.props = { ...DEFAULT_PROPS, ...props };
+    }
+    close() {
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
+        this.file.close();
+    }
+    // HIGH LEVEL METHODS
+    /** Yield one row at a time */
+    async *rowIterator(props) {
+        for await (const rows of this.rowBatchIterator(props)) {
+            // yield *rows
+            for (const row of rows) {
+                yield row;
+            }
         }
-        this.metadata = metadata;
-        this.envelopeReader = envelopeReader;
-        const root = this.metadata.schema[0];
-        const { schema } = (0, decoders_1.decodeSchema)(this.metadata.schema, 1, root.num_children);
-        this.schema = new schema_1.ParquetSchema(schema);
     }
-    /**
-     * return a new parquet reader initialized with a read function
-     */
-    static async openBlob(blob) {
-        const readFn = async (start, length) => {
-            const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();
-            return Buffer.from(arrayBuffer);
-        };
-        const closeFn = async () => { };
-        const size = blob.size;
-        const envelopeReader = new parquet_envelope_reader_1.ParquetEnvelopeReader(readFn, closeFn, size);
-        try {
-            await envelopeReader.readHeader();
-            const metadata = await envelopeReader.readFooter();
-            return new ParquetReader(metadata, envelopeReader);
+    /** Yield one batch of rows at a time */
+    async *rowBatchIterator(props) {
+        const schema = await this.getSchema();
+        for await (const rowGroup of this.rowGroupIterator(props)) {
+            yield (0, shred_1.materializeRecords)(schema, rowGroup);
         }
-        catch (err) {
-            await envelopeReader.close();
-            throw err;
+    }
+    /** Iterate over the raw row groups */
+    async *rowGroupIterator(props) {
+        // Ensure strings are nested in arrays
+        const columnList = (props?.columnList || []).map((x) => Array.isArray(x) ? x : [x]);
+        const metadata = await this.getFileMetadata();
+        const schema = await this.getSchema();
+        const rowGroupCount = metadata?.row_groups.length || 0;
+        for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {
+            const rowGroup = await this.readRowGroup(schema, metadata.row_groups[rowGroupIndex], columnList);
+            yield rowGroup;
         }
     }
+    async getRowCount() {
+        const metadata = await this.getFileMetadata();
+        return Number(metadata.num_rows);
+    }
+    async getSchema() {
+        const metadata = await this.getFileMetadata();
+        const root = metadata.schema[0];
+        const { schema: schemaDefinition } = (0, decoders_1.decodeSchema)(metadata.schema, 1, root.num_children);
+        const schema = new schema_1.ParquetSchema(schemaDefinition);
+        return schema;
+    }
     /**
-     * return a new parquet reader initialized with a read function
+     * Returns the user (key/value) metadata for this file
+     * In parquet this is not stored on the schema like it is in arrow
      */
-    static async openArrayBuffer(arrayBuffer) {
-        const readFn = async (start, length) => Buffer.from(arrayBuffer, start, length);
-        const closeFn = async () => { };
-        const size = arrayBuffer.byteLength;
-        const envelopeReader = new parquet_envelope_reader_1.ParquetEnvelopeReader(readFn, closeFn, size);
-        try {
-            await envelopeReader.readHeader();
-            const metadata = await envelopeReader.readFooter();
-            return new ParquetReader(metadata, envelopeReader);
-        }
-        catch (err) {
-            await envelopeReader.close();
-            throw err;
+    async getSchemaMetadata() {
+        const metadata = await this.getFileMetadata();
+        const md = {};
+        for (const kv of metadata.key_value_metadata) {
+            md[kv.key] = kv.value;
         }
+        return md;
     }
-    static async openBuffer(buffer) {
-        const envelopeReader = await parquet_envelope_reader_1.ParquetEnvelopeReader.openBuffer(buffer);
-        try {
-            await envelopeReader.readHeader();
-            const metadata = await envelopeReader.readFooter();
-            return new ParquetReader(metadata, envelopeReader);
+    async getFileMetadata() {
+        if (!this.metadata) {
+            await this.readHeader();
+            this.metadata = this.readFooter();
         }
-        catch (err) {
-            await envelopeReader.close();
-            throw err;
-        }
-    }
-    /**
-     * Close this parquet reader. You MUST call this method once you're finished
-     * reading rows
-     */
-    async close() {
-        await this.envelopeReader.close();
-        // this.envelopeReader = null;
-        // this.metadata = null;
+        return this.metadata;
     }
-    getCursor(columnList) {
-        if (!columnList) {
-            // tslint:disable-next-line:no-parameter-reassignment
-            columnList = [];
+    // LOW LEVEL METHODS
+    /** Metadata is stored in the footer */
+    async readHeader() {
+        const buffer = await this.file.read(0, constants_1.PARQUET_MAGIC.length);
+        const magic = buffer.toString();
+        switch (magic) {
+            case constants_1.PARQUET_MAGIC:
+                break;
+            case constants_1.PARQUET_MAGIC_ENCRYPTED:
+                throw new Error('Encrypted parquet file not supported');
+            default:
+                throw new Error(`Invalid parquet file (magic=${magic})`);
         }
-        // tslint:disable-next-line:no-parameter-reassignment
-        columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));
-        return new parquet_cursor_1.ParquetCursor(this.metadata, this.envelopeReader, this.schema, columnList);
     }
-    /**
-     * Return the number of rows in this file. Note that the number of rows is
-     * not neccessarily equal to the number of rows in each column.
-     */
-    getRowCount() {
-        return Number(this.metadata.num_rows);
+    /** Metadata is stored in the footer */
+    async readFooter() {
+        const trailerLen = constants_1.PARQUET_MAGIC.length + 4;
+        const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);
+        const magic = trailerBuf.slice(4).toString();
+        if (magic !== constants_1.PARQUET_MAGIC) {
+            throw new Error(`Not a valid parquet file (magic="${magic})`);
+        }
+        const metadataSize = trailerBuf.readUInt32LE(0);
+        const metadataOffset = this.file.size - metadataSize - trailerLen;
+        if (metadataOffset < constants_1.PARQUET_MAGIC.length) {
+            throw new Error(`Invalid metadata size ${metadataOffset}`);
+        }
+        const metadataBuf = await this.file.read(metadataOffset, metadataSize);
+        // let metadata = new parquet_thrift.FileMetaData();
+        // parquet_util.decodeThrift(metadata, metadataBuf);
+        const { metadata } = (0, read_utils_1.decodeFileMetadata)(metadataBuf);
+        return metadata;
     }
-    /**
-     * Returns the ParquetSchema for this file
-     */
-    getSchema() {
-        return this.schema;
+    /** Data is stored in row groups (similar to Apache Arrow record batches) */
+    async readRowGroup(schema, rowGroup, columnList) {
+        const buffer = {
+            rowCount: Number(rowGroup.num_rows),
+            columnData: {}
+        };
+        for (const colChunk of rowGroup.columns) {
+            const colMetadata = colChunk.meta_data;
+            const colKey = colMetadata?.path_in_schema;
+            if (columnList.length > 0 && (0, read_utils_1.fieldIndexOf)(columnList, colKey) < 0) {
+                continue; // eslint-disable-line no-continue
+            }
+            buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
+        }
+        return buffer;
     }
     /**
-     * Returns the user (key/value) metadata for this file
+     * Each row group contains column chunks for all the columns.
      */
-    getMetadata() {
-        const md = {};
-        for (const kv of this.metadata.key_value_metadata) {
-            md[kv.key] = kv.value;
+    async readColumnChunk(schema, colChunk) {
+        if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
+            throw new Error('external references are not supported');
         }
-        return md;
+        const field = schema.findField(colChunk.meta_data?.path_in_schema);
+        const type = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.Type, colChunk.meta_data?.type);
+        if (type !== field.primitiveType) {
+            throw new Error(`chunk type not matching schema: ${type}`);
+        }
+        const compression = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.CompressionCodec, colChunk.meta_data?.codec);
+        const pagesOffset = Number(colChunk.meta_data?.data_page_offset);
+        let pagesSize = Number(colChunk.meta_data?.total_compressed_size);
+        if (!colChunk.file_path) {
+            pagesSize = Math.min(this.file.size - pagesOffset, Number(colChunk.meta_data?.total_compressed_size));
+        }
+        const options = {
+            type,
+            rLevelMax: field.rLevelMax,
+            dLevelMax: field.dLevelMax,
+            compression,
+            column: field,
+            numValues: colChunk.meta_data?.num_values,
+            dictionary: []
+        };
+        let dictionary;
+        const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;
+        if (dictionaryPageOffset) {
+            const dictionaryOffset = Number(dictionaryPageOffset);
+            // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.
+            dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
+        }
+        dictionary = options.dictionary?.length ? options.dictionary : dictionary;
+        const pagesBuf = await this.file.read(pagesOffset, pagesSize);
+        return await (0, decoders_2.decodeDataPages)(pagesBuf, { ...options, dictionary });
     }
     /**
-     * Implement AsyncIterable
+     * Getting dictionary for allows to flatten values by indices.
+     * @param dictionaryPageOffset
+     * @param options
+     * @param pagesOffset
+     * @returns
      */
-    // tslint:disable-next-line:function-name
-    [Symbol.asyncIterator]() {
-        return this.getCursor()[Symbol.asyncIterator]();
+    async getDictionary(dictionaryPageOffset, options, pagesOffset) {
+        if (dictionaryPageOffset === 0) {
+            // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);
+            // pagesBuf = await this.read(pagesOffset, dictionarySize);
+            // In this case we are working with parquet-mr files format. Problem is described below:
+            // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding
+            // We need to get dictionary page from column chunk if it exists.
+            // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.
+            return [];
+        }
+        const dictionarySize = Math.min(this.file.size - dictionaryPageOffset, this.props.defaultDictionarySize);
+        const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);
+        const cursor = { buffer: pagesBuf, offset: 0, size: pagesBuf.length };
+        const decodedPage = await (0, decoders_2.decodePage)(cursor, options);
+        return decodedPage.dictionary;
     }
 }
 exports.ParquetReader = ParquetReader;

package/dist/parquetjs/schema/declare.d.ts

@@ -1,18 +1,18 @@
 import Int64 from 'node-int64';
 import type { PageHeader } from '../parquet-thrift';
-export declare type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';
-export declare type ParquetCompression = 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI' | 'LZ4' | 'LZ4_RAW' | 'ZSTD';
-export declare type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';
-export declare type ParquetType = PrimitiveType | OriginalType;
+export type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';
+export type ParquetCompression = 'UNCOMPRESSED' | 'GZIP' | 'SNAPPY' | 'LZO' | 'BROTLI' | 'LZ4' | 'LZ4_RAW' | 'ZSTD';
+export type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';
+export type ParquetType = PrimitiveType | OriginalType;
 /**
  * Physical type
  */
-export declare type PrimitiveType = 'BOOLEAN' | 'INT32' | 'INT64' | 'INT96' | 'FLOAT' | 'DOUBLE' | 'BYTE_ARRAY' | 'FIXED_LEN_BYTE_ARRAY';
+export type PrimitiveType = 'BOOLEAN' | 'INT32' | 'INT64' | 'INT96' | 'FLOAT' | 'DOUBLE' | 'BYTE_ARRAY' | 'FIXED_LEN_BYTE_ARRAY';
 /**
  * Logical type
  */
-export declare type OriginalType = 'UTF8' | 'DECIMAL_INT32' | 'DECIMAL_INT64' | 'DECIMAL_BYTE_ARRAY' | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' | 'DATE' | 'TIME_MILLIS' | 'TIME_MICROS' | 'TIMESTAMP_MILLIS' | 'TIMESTAMP_MICROS' | 'UINT_8' | 'UINT_16' | 'UINT_32' | 'UINT_64' | 'INT_8' | 'INT_16' | 'INT_32' | 'INT_64' | 'JSON' | 'BSON' | 'INTERVAL';
-export declare type ParquetDictionary = string[];
+export type OriginalType = 'UTF8' | 'DECIMAL_INT32' | 'DECIMAL_INT64' | 'DECIMAL_BYTE_ARRAY' | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' | 'DATE' | 'TIME_MILLIS' | 'TIME_MICROS' | 'TIMESTAMP_MILLIS' | 'TIMESTAMP_MICROS' | 'UINT_8' | 'UINT_16' | 'UINT_32' | 'UINT_64' | 'INT_8' | 'INT_16' | 'INT_32' | 'INT_64' | 'JSON' | 'BSON' | 'INTERVAL';
+export type ParquetDictionary = string[];
 export interface SchemaDefinition {
     [string: string]: FieldDefinition;
 }
@@ -45,6 +45,7 @@ export interface ParquetField {
     fieldCount?: number;
     fields?: Record<string, ParquetField>;
 }
+/** @todo better name, this is an internal type? */
 export interface ParquetOptions {
     type: ParquetType;
     rLevelMax: number;
@@ -54,27 +55,33 @@ export interface ParquetOptions {
     numValues?: Int64;
     dictionary?: ParquetDictionary;
 }
-export interface ParquetData {
-    dlevels: number[];
-    rlevels: number[];
-    values: any[];
-    count: number;
-    pageHeaders: PageHeader[];
-}
 export interface ParquetPageData {
     dlevels: number[];
     rlevels: number[];
+    /** Actual column chunks */
     values: any[];
     count: number;
     dictionary?: ParquetDictionary;
+    /** The "raw" page header from the file */
     pageHeader: PageHeader;
 }
 export interface ParquetRecord {
     [key: string]: any;
 }
+/** @
+ * Holds data for one row group (column chunks) */
 export declare class ParquetBuffer {
+    /** Number of rows in this page */
     rowCount: number;
     columnData: Record<string, ParquetData>;
     constructor(rowCount?: number, columnData?: Record<string, ParquetData>);
 }
+/** Holds the data for one column chunk */
+export interface ParquetData {
+    dlevels: number[];
+    rlevels: number[];
+    values: any[];
+    count: number;
+    pageHeaders: PageHeader[];
+}
 //# sourceMappingURL=declare.d.ts.map

package/dist/parquetjs/schema/declare.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,oBAAY,YAAY,GAAG,OAAO,GAAG,KAAK,GAAG,kBAAkB,CAAC;AAChE,oBAAY,kBAAkB,GAC1B,cAAc,GACd,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,KAAK,GACL,SAAS,GACT,MAAM,CAAC;AACX,oBAAY,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAClE,oBAAY,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAEvD;;GAEG;AACH,oBAAY,aAAa,GAErB,SAAS,GACT,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,sBAAsB,CAAC;AAE3B;;GAEG;AACH,oBAAY,YAAY,GAEpB,MAAM,GAMN,eAAe,GACf,eAAe,GACf,oBAAoB,GACpB,8BAA8B,GAC9B,MAAM,GACN,aAAa,GACb,aAAa,GACb,kBAAkB,GAClB,kBAAkB,GAClB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,CAAC;AAEf,oBAAY,iBAAiB,GAAG,MAAM,EAAE,CAAC;AAEzC,MAAM,WAAW,gBAAgB;IAC/B,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,EAAE,cAAc,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED,qBAAa,aAAa;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBAC5B,QAAQ,GAAE,MAAU,EAAE,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAM;CAI/E"}
+{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,KAAK,GAAG,kBAAkB,CAAC;AAChE,MAAM,MAAM,kBAAkB,GAC1B,cAAc,GACd,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,KAAK,GACL,SAAS,GACT,MAAM,CAAC;AACX,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAEvD;;GAEG;AACH,MAAM,MAAM,aAAa,GAErB,SAAS,GACT,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,sBAAsB,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,YAAY,GAEpB,MAAM,GAMN,eAAe,GACf,eAAe,GACf,oBAAoB,GACpB,8BAA8B,GAC9B,MAAM,GACN,aAAa,GACb,aAAa,GACb,kBAAkB,GAClB,kBAAkB,GAClB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,CAAC;AAEf,MAAM,MAAM,iBAAiB,GAAG,MAAM,EAAE,CAAC;AAEzC,MAAM,WAAW,gBAAgB;IAC/B,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,EAAE,cAAc,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,mDAAmD;AACnD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,2BAA2B;IAC3B,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,0CAA0C;IAC1C,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED;kDACkD;AAClD,qBAAa,aAAa;IACxB,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IAEjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBAC5B,QAAQ,GAAE,MAAU,EAAE,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAM;CAI/E;AAED,0CAA0C;AAC1C,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B"}

package/dist/parquetjs/schema/declare.js

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ParquetBuffer = void 0;
+/** @
+ * Holds data for one row group (column chunks) */
 class ParquetBuffer {
     constructor(rowCount = 0, columnData = {}) {
         this.rowCount = rowCount;

package/dist/parquetjs/schema/shred.d.ts

@@ -45,4 +45,119 @@ export declare function shredRecord(schema: ParquetSchema, record: any, buffer:
  *   }
  */
 export declare function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[];
+/**
+ * 'Materialize' a list of <value, repetition_level, definition_level>
+ * tuples back to nested records (objects/arrays) using the Google Dremel
+ * Algorithm..
+ *
+ * The buffer argument must point to an object with the following structure (i.e.
+ * the same structure that is returned by shredRecords):
+ *
+ *   buffer = {
+ *     columnData: [
+ *       'my_col': {
+ *          dlevels: [d1, d2, .. dN],
+ *          rlevels: [r1, r2, .. rN],
+ *          values: [v1, v2, .. vN],
+ *        }, ...
+ *      ],
+ *      rowCount: X,
+ *   }
+ *
+export function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {
+  const columns: ParquetRecord = {};
+  for (const key in buffer.columnData) {
+    const columnData = buffer.columnData[key];
+    if (columnData.count) {
+      extractColumn(schema, columnData, key, columns);
+    }
+  }
+  return columns;
+}
+
+// eslint-disable-next-line max-statements, complexity
+function extractColumn(
+  schema: ParquetSchema,
+  columnData: ParquetData,
+  key: string,
+  columns: Record<string, unknown>
+) {
+  if (columnData.count <= 0) {
+    return;
+  }
+
+  const record = columns;
+
+  const field = schema.findField(key);
+  const branch = schema.findFieldBranch(key);
+
+  // tslint:disable-next-line:prefer-array-literal
+  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
+  let vIndex = 0;
+
+  let i = 0;
+  const dLevel = columnData.dlevels[i];
+  const rLevel = columnData.rlevels[i];
+  rLevels[rLevel]++;
+  rLevels.fill(0, rLevel + 1);
+
+  let rIndex = 0;
+  let record = records[rLevels[rIndex++] - 1];
+
+  // Internal nodes
+  for (const step of branch) {
+    if (step === field || dLevel < step.dLevelMax) {
+      break;
+    }
+
+    switch (step.repetitionType) {
+      case 'REPEATED':
+        if (!(step.name in record)) {
+          // eslint-disable max-depth
+          record[step.name] = [];
+        }
+        const ix = rLevels[rIndex++];
+        while (record[step.name].length <= ix) {
+          // eslint-disable max-depth
+          record[step.name].push({});
+        }
+        record = record[step.name][ix];
+        break;
+
+      default:
+        record[step.name] = record[step.name] || {};
+        record = record[step.name];
+    }
+  }
+
+  // Leaf node
+  if (dLevel === field.dLevelMax) {
+    const value = Types.fromPrimitive(
+      // @ts-ignore
+      field.originalType || field.primitiveType,
+      columnData.values[vIndex],
+      field
+    );
+    vIndex++;
+
+    switch (field.repetitionType) {
+      case 'REPEATED':
+        if (!(field.name in record)) {
+          // eslint-disable max-depth
+          record[field.name] = [];
+        }
+        const ix = rLevels[rIndex];
+        while (record[field.name].length <= ix) {
+          // eslint-disable max-depth
+          record[field.name].push(null);
+        }
+        record[field.name][ix] = value;
+        break;
+
+      default:
+        record[field.name] = value;
+    }
+  }
+}
+*/
 //# sourceMappingURL=shred.d.ts.map

package/dist/parquetjs/schema/shred.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,aAAa,EAA6B,aAAa,EAAC,MAAM,WAAW,CAAC;AAClF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,aAAa,EAAC,CAAC;AAEvB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAYhE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI,CAmB3F;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,aAAa,EAAE,CAOhG"}
+{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,aAAa,EAA6B,aAAa,EAAC,MAAM,WAAW,CAAC;AAClF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,aAAa,EAAC,CAAC;AAEvB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAYhE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI,CAmB3F;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,aAAa,EAAE,CAYhG;AAmFD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkHE"}