@loaders.gl/parquet 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/parquetjs/parquet-thrift/Type.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Type = void 0;
+/* tslint:disable */
+/* eslint-disable */
+/*
+ * Autogenerated by @creditkarma/thrift-typescript v3.7.2
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ */
+var Type;
+(function (Type) {
+    Type[Type["BOOLEAN"] = 0] = "BOOLEAN";
+    Type[Type["INT32"] = 1] = "INT32";
+    Type[Type["INT64"] = 2] = "INT64";
+    Type[Type["INT96"] = 3] = "INT96";
+    Type[Type["FLOAT"] = 4] = "FLOAT";
+    Type[Type["DOUBLE"] = 5] = "DOUBLE";
+    Type[Type["BYTE_ARRAY"] = 6] = "BYTE_ARRAY";
+    Type[Type["FIXED_LEN_BYTE_ARRAY"] = 7] = "FIXED_LEN_BYTE_ARRAY";
+})(Type = exports.Type || (exports.Type = {}));

package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts

@@ -0,0 +1,9 @@
+import * as thrift from 'thrift';
+export interface ITypeDefinedOrderArgs {
+}
+export declare class TypeDefinedOrder {
+    constructor();
+    write(output: thrift.TProtocol): void;
+    static read(input: thrift.TProtocol): TypeDefinedOrder;
+}
+//# sourceMappingURL=TypeDefinedOrder.d.ts.map

package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"TypeDefinedOrder.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parquet-thrift/TypeDefinedOrder.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,MAAM,WAAW,qBAAqB;CAAG;AACzC,qBAAa,gBAAgB;;IAEpB,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,GAAG,IAAI;WAM9B,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,GAAG,gBAAgB;CAmB9D"}

package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js

@@ -0,0 +1,58 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TypeDefinedOrder = void 0;
+/* tslint:disable */
+/* eslint-disable */
+/*
+ * Autogenerated by @creditkarma/thrift-typescript v3.7.2
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ */
+const thrift = __importStar(require("thrift"));
+class TypeDefinedOrder {
+    constructor() { }
+    write(output) {
+        output.writeStructBegin('TypeDefinedOrder');
+        output.writeFieldStop();
+        output.writeStructEnd();
+        return;
+    }
+    static read(input) {
+        input.readStructBegin();
+        while (true) {
+            const ret = input.readFieldBegin();
+            const fieldType = ret.ftype;
+            const fieldId = ret.fid;
+            if (fieldType === thrift.Thrift.Type.STOP) {
+                break;
+            }
+            switch (fieldId) {
+                default: {
+                    input.skip(fieldType);
+                }
+            }
+            input.readFieldEnd();
+        }
+        input.readStructEnd();
+        return new TypeDefinedOrder();
+    }
+}
+exports.TypeDefinedOrder = TypeDefinedOrder;

package/dist/parquetjs/parquet-thrift/UUIDType.d.ts

@@ -0,0 +1,9 @@
+import * as thrift from 'thrift';
+export interface IUUIDTypeArgs {
+}
+export declare class UUIDType {
+    constructor();
+    write(output: thrift.TProtocol): void;
+    static read(input: thrift.TProtocol): UUIDType;
+}
+//# sourceMappingURL=UUIDType.d.ts.map

package/dist/parquetjs/parquet-thrift/UUIDType.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"UUIDType.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parquet-thrift/UUIDType.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,MAAM,WAAW,aAAa;CAAG;AACjC,qBAAa,QAAQ;;IAEZ,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,GAAG,IAAI;WAM9B,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,GAAG,QAAQ;CAmBtD"}

package/dist/parquetjs/parquet-thrift/UUIDType.js

@@ -0,0 +1,58 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.UUIDType = void 0;
+/* tslint:disable */
+/* eslint-disable */
+/*
+ * Autogenerated by @creditkarma/thrift-typescript v3.7.2
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ */
+const thrift = __importStar(require("thrift"));
+class UUIDType {
+    constructor() { }
+    write(output) {
+        output.writeStructBegin('UUIDType');
+        output.writeFieldStop();
+        output.writeStructEnd();
+        return;
+    }
+    static read(input) {
+        input.readStructBegin();
+        while (true) {
+            const ret = input.readFieldBegin();
+            const fieldType = ret.ftype;
+            const fieldId = ret.fid;
+            if (fieldType === thrift.Thrift.Type.STOP) {
+                break;
+            }
+            switch (fieldId) {
+                default: {
+                    input.skip(fieldType);
+                }
+            }
+            input.readFieldEnd();
+        }
+        input.readStructEnd();
+        return new UUIDType();
+    }
+}
+exports.UUIDType = UUIDType;

package/dist/parquetjs/parquet-thrift/index.d.ts

@@ -0,0 +1,44 @@
+export * from './Type';
+export * from './ConvertedType';
+export * from './FieldRepetitionType';
+export * from './Encoding';
+export * from './CompressionCodec';
+export * from './PageType';
+export * from './BoundaryOrder';
+export * from './Statistics';
+export * from './StringType';
+export * from './UUIDType';
+export * from './MapType';
+export * from './ListType';
+export * from './EnumType';
+export * from './DateType';
+export * from './NullType';
+export * from './DecimalType';
+export * from './MilliSeconds';
+export * from './MicroSeconds';
+export * from './TimestampType';
+export * from './TimeType';
+export * from './IntType';
+export * from './JsonType';
+export * from './BsonType';
+export * from './SchemaElement';
+export * from './DataPageHeader';
+export * from './IndexPageHeader';
+export * from './DictionaryPageHeader';
+export * from './DataPageHeaderV2';
+export * from './PageHeader';
+export * from './KeyValue';
+export * from './SortingColumn';
+export * from './PageEncodingStats';
+export * from './ColumnMetaData';
+export * from './ColumnChunk';
+export * from './RowGroup';
+export * from './TypeDefinedOrder';
+export * from './PageLocation';
+export * from './OffsetIndex';
+export * from './ColumnIndex';
+export * from './FileMetaData';
+export * from './TimeUnit';
+export * from './LogicalType';
+export * from './ColumnOrder';
+//# sourceMappingURL=index.d.ts.map

package/dist/parquetjs/parquet-thrift/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parquet-thrift/index.ts"],"names":[],"mappings":"AAMA,cAAc,QAAQ,CAAC;AACvB,cAAc,iBAAiB,CAAC;AAChC,cAAc,uBAAuB,CAAC;AACtC,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,mBAAmB,CAAC;AAClC,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC;AACpC,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,eAAe,CAAC"}

package/dist/parquetjs/parquet-thrift/index.js

@@ -0,0 +1,61 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+/* tslint:disable */
+/* eslint-disable */
+/*
+ * Autogenerated by @creditkarma/thrift-typescript v3.7.2
+ * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+ */
+__exportStar(require("./Type"), exports);
+__exportStar(require("./ConvertedType"), exports);
+__exportStar(require("./FieldRepetitionType"), exports);
+__exportStar(require("./Encoding"), exports);
+__exportStar(require("./CompressionCodec"), exports);
+__exportStar(require("./PageType"), exports);
+__exportStar(require("./BoundaryOrder"), exports);
+__exportStar(require("./Statistics"), exports);
+__exportStar(require("./StringType"), exports);
+__exportStar(require("./UUIDType"), exports);
+__exportStar(require("./MapType"), exports);
+__exportStar(require("./ListType"), exports);
+__exportStar(require("./EnumType"), exports);
+__exportStar(require("./DateType"), exports);
+__exportStar(require("./NullType"), exports);
+__exportStar(require("./DecimalType"), exports);
+__exportStar(require("./MilliSeconds"), exports);
+__exportStar(require("./MicroSeconds"), exports);
+__exportStar(require("./TimestampType"), exports);
+__exportStar(require("./TimeType"), exports);
+__exportStar(require("./IntType"), exports);
+__exportStar(require("./JsonType"), exports);
+__exportStar(require("./BsonType"), exports);
+__exportStar(require("./SchemaElement"), exports);
+__exportStar(require("./DataPageHeader"), exports);
+__exportStar(require("./IndexPageHeader"), exports);
+__exportStar(require("./DictionaryPageHeader"), exports);
+__exportStar(require("./DataPageHeaderV2"), exports);
+__exportStar(require("./PageHeader"), exports);
+__exportStar(require("./KeyValue"), exports);
+__exportStar(require("./SortingColumn"), exports);
+__exportStar(require("./PageEncodingStats"), exports);
+__exportStar(require("./ColumnMetaData"), exports);
+__exportStar(require("./ColumnChunk"), exports);
+__exportStar(require("./RowGroup"), exports);
+__exportStar(require("./TypeDefinedOrder"), exports);
+__exportStar(require("./PageLocation"), exports);
+__exportStar(require("./OffsetIndex"), exports);
+__exportStar(require("./ColumnIndex"), exports);
+__exportStar(require("./FileMetaData"), exports);
+__exportStar(require("./TimeUnit"), exports);
+__exportStar(require("./LogicalType"), exports);
+__exportStar(require("./ColumnOrder"), exports);

package/dist/parquetjs/parser/decoders.d.ts

@@ -0,0 +1,34 @@
+/// <reference types="node" />
+import { ParquetData, ParquetOptions, ParquetPageData, SchemaDefinition } from '../schema/declare';
+import { CursorBuffer } from '../codecs';
+import { SchemaElement } from '../parquet-thrift';
+/**
+ * Decode data pages
+ * @param buffer - input data
+ * @param column - parquet column
+ * @param compression - compression type
+ * @returns parquet data page data
+ */
+export declare function decodeDataPages(buffer: Buffer, options: ParquetOptions): Promise<ParquetData>;
+/**
+ * Decode parquet page based on page type
+ * @param cursor
+ * @param options
+ */
+export declare function decodePage(cursor: CursorBuffer, options: ParquetOptions): Promise<ParquetPageData>;
+/**
+ * Decode parquet schema
+ * @param schemaElements input schema elements data
+ * @param offset offset to read from
+ * @param len length of data
+ * @returns result.offset
+ *   result.next - offset at the end of function
+ *   result.schema - schema read from the input data
+ * @todo output offset is the same as input - possibly excess output field
+ */
+export declare function decodeSchema(schemaElements: SchemaElement[], offset: number, len: number): {
+    offset: number;
+    next: number;
+    schema: SchemaDefinition;
+};
+//# sourceMappingURL=decoders.d.ts.map

package/dist/parquetjs/parser/decoders.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":";AACA,OAAO,EAEL,WAAW,EACX,cAAc,EACd,eAAe,EAGf,gBAAgB,EACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,YAAY,EAAsC,MAAM,WAAW,CAAC;AAC5E,OAAO,EAML,aAAa,EAEd,MAAM,mBAAmB,CAAC;AAK3B;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,WAAW,CAAC,CAmDtB;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,eAAe,CAAC,CAyB1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,aAAa,EAAE,EAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CA4DA"}

package/dist/parquetjs/parser/decoders.js

@@ -0,0 +1,318 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.decodeSchema = exports.decodePage = exports.decodeDataPages = void 0;
+const codecs_1 = require("../codecs");
+const parquet_thrift_1 = require("../parquet-thrift");
+const compression_1 = require("../compression");
+const constants_1 = require("../../constants");
+const read_utils_1 = require("../utils/read-utils");
+/**
+ * Decode data pages
+ * @param buffer - input data
+ * @param column - parquet column
+ * @param compression - compression type
+ * @returns parquet data page data
+ */
+async function decodeDataPages(buffer, options) {
+    const cursor = {
+        buffer,
+        offset: 0,
+        size: buffer.length
+    };
+    const data = {
+        rlevels: [],
+        dlevels: [],
+        values: [],
+        pageHeaders: [],
+        count: 0
+    };
+    let dictionary = options.dictionary || [];
+    while (
+    // @ts-ignore size can be undefined
+    cursor.offset < cursor.size &&
+        (!options.numValues || data.dlevels.length < Number(options.numValues))) {
+        // Looks like we have to decode these in sequence due to cursor updates?
+        const page = await decodePage(cursor, options);
+        if (page.dictionary) {
+            dictionary = page.dictionary;
+            // eslint-disable-next-line no-continue
+            continue;
+        }
+        if (dictionary.length) {
+            // eslint-disable-next-line no-loop-func
+            page.values = page.values.map((value) => dictionary[value]);
+        }
+        for (let index = 0; index < page.rlevels.length; index++) {
+            data.rlevels.push(page.rlevels[index]);
+            data.dlevels.push(page.dlevels[index]);
+            const value = page.values[index];
+            if (value !== undefined) {
+                data.values.push(value);
+            }
+        }
+        data.count += page.count;
+        data.pageHeaders.push(page.pageHeader);
+    }
+    return data;
+}
+exports.decodeDataPages = decodeDataPages;
+/**
+ * Decode parquet page based on page type
+ * @param cursor
+ * @param options
+ */
+async function decodePage(cursor, options) {
+    let page;
+    const { pageHeader, length } = (0, read_utils_1.decodePageHeader)(cursor.buffer, cursor.offset);
+    cursor.offset += length;
+    const pageType = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.PageType, pageHeader.type);
+    switch (pageType) {
+        case 'DATA_PAGE':
+            page = await decodeDataPage(cursor, pageHeader, options);
+            break;
+        case 'DATA_PAGE_V2':
+            page = await decodeDataPageV2(cursor, pageHeader, options);
+            break;
+        case 'DICTIONARY_PAGE':
+            page = {
+                dictionary: await decodeDictionaryPage(cursor, pageHeader, options),
+                pageHeader
+            };
+            break;
+        default:
+            throw new Error(`invalid page type: ${pageType}`);
+    }
+    return page;
+}
+exports.decodePage = decodePage;
+/**
+ * Decode parquet schema
+ * @param schemaElements input schema elements data
+ * @param offset offset to read from
+ * @param len length of data
+ * @returns result.offset
+ *   result.next - offset at the end of function
+ *   result.schema - schema read from the input data
+ * @todo output offset is the same as input - possibly excess output field
+ */
+function decodeSchema(schemaElements, offset, len) {
+    const schema = {};
+    let next = offset;
+    for (let i = 0; i < len; i++) {
+        const schemaElement = schemaElements[next];
+        const repetitionType = next > 0 ? (0, read_utils_1.getThriftEnum)(parquet_thrift_1.FieldRepetitionType, schemaElement.repetition_type) : 'ROOT';
+        let optional = false;
+        let repeated = false;
+        switch (repetitionType) {
+            case 'REQUIRED':
+                break;
+            case 'OPTIONAL':
+                optional = true;
+                break;
+            case 'REPEATED':
+                repeated = true;
+                break;
+            default:
+                throw new Error('parquet: unknown repetition type');
+        }
+        if (schemaElement.num_children > 0) {
+            const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
+            next = res.next;
+            schema[schemaElement.name] = {
+                // type: undefined,
+                optional,
+                repeated,
+                fields: res.schema
+            };
+        }
+        else {
+            const type = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.Type, schemaElement.type);
+            let logicalType = type;
+            if (schemaElement.converted_type) {
+                logicalType = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.ConvertedType, schemaElement.converted_type);
+            }
+            switch (logicalType) {
+                case 'DECIMAL':
+                    logicalType = `${logicalType}_${type}`;
+                    break;
+                default:
+            }
+            schema[schemaElement.name] = {
+                type: logicalType,
+                typeLength: schemaElement.type_length,
+                presision: schemaElement.precision,
+                scale: schemaElement.scale,
+                optional,
+                repeated
+            };
+            next++;
+        }
+    }
+    return { schema, offset, next };
+}
+exports.decodeSchema = decodeSchema;
+/**
+ * Decode a consecutive array of data using one of the parquet encodings
+ */
+function decodeValues(type, encoding, cursor, count, opts) {
+    if (!(encoding in codecs_1.PARQUET_CODECS)) {
+        throw new Error(`invalid encoding: ${encoding}`);
+    }
+    return codecs_1.PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);
+}
+/**
+ * Do decoding of parquet dataPage from column chunk
+ * @param cursor
+ * @param header
+ * @param options
+ */
+async function decodeDataPage(cursor, header, options) {
+    const cursorEnd = cursor.offset + header.compressed_page_size;
+    const valueCount = header.data_page_header?.num_values;
+    /* uncompress page */
+    let dataCursor = cursor;
+    if (options.compression !== 'UNCOMPRESSED') {
+        const valuesBuf = await (0, compression_1.decompress)(options.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
+        dataCursor = {
+            buffer: valuesBuf,
+            offset: 0,
+            size: valuesBuf.length
+        };
+        cursor.offset = cursorEnd;
+    }
+    /* read repetition levels */
+    const rLevelEncoding = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.Encoding, header.data_page_header?.repetition_level_encoding);
+    // tslint:disable-next-line:prefer-array-literal
+    let rLevels = new Array(valueCount);
+    if (options.column.rLevelMax > 0) {
+        rLevels = decodeValues(constants_1.PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
+            bitWidth: (0, read_utils_1.getBitWidth)(options.column.rLevelMax),
+            disableEnvelope: false
+            // column: opts.column
+        });
+    }
+    else {
+        rLevels.fill(0);
+    }
+    /* read definition levels */
+    const dLevelEncoding = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.Encoding, header.data_page_header?.definition_level_encoding);
+    // tslint:disable-next-line:prefer-array-literal
+    let dLevels = new Array(valueCount);
+    if (options.column.dLevelMax > 0) {
+        dLevels = decodeValues(constants_1.PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
+            bitWidth: (0, read_utils_1.getBitWidth)(options.column.dLevelMax),
+            disableEnvelope: false
+            // column: opts.column
+        });
+    }
+    else {
+        dLevels.fill(0);
+    }
+    let valueCountNonNull = 0;
+    for (const dlvl of dLevels) {
+        if (dlvl === options.column.dLevelMax) {
+            valueCountNonNull++;
+        }
+    }
+    /* read values */
+    const valueEncoding = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.Encoding, header.data_page_header?.encoding);
+    const decodeOptions = {
+        typeLength: options.column.typeLength,
+        bitWidth: options.column.typeLength
+    };
+    const values = decodeValues(options.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
+    return {
+        dlevels: dLevels,
+        rlevels: rLevels,
+        values,
+        count: valueCount,
+        pageHeader: header
+    };
+}
+/**
+ * Do decoding of parquet dataPage in version 2 from column chunk
+ * @param cursor
+ * @param header
+ * @param opts
+ * @returns
+ */
+async function decodeDataPageV2(cursor, header, opts) {
+    const cursorEnd = cursor.offset + header.compressed_page_size;
+    const valueCount = header.data_page_header_v2?.num_values;
+    // @ts-ignore
+    const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;
+    const valueEncoding = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.Encoding, header.data_page_header_v2?.encoding);
+    /* read repetition levels */
+    // tslint:disable-next-line:prefer-array-literal
+    let rLevels = new Array(valueCount);
+    if (opts.column.rLevelMax > 0) {
+        rLevels = decodeValues(constants_1.PARQUET_RDLVL_TYPE, constants_1.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
+            bitWidth: (0, read_utils_1.getBitWidth)(opts.column.rLevelMax),
+            disableEnvelope: true
+        });
+    }
+    else {
+        rLevels.fill(0);
+    }
+    /* read definition levels */
+    // tslint:disable-next-line:prefer-array-literal
+    let dLevels = new Array(valueCount);
+    if (opts.column.dLevelMax > 0) {
+        dLevels = decodeValues(constants_1.PARQUET_RDLVL_TYPE, constants_1.PARQUET_RDLVL_ENCODING, cursor, valueCount, {
+            bitWidth: (0, read_utils_1.getBitWidth)(opts.column.dLevelMax),
+            disableEnvelope: true
+        });
+    }
+    else {
+        dLevels.fill(0);
+    }
+    /* read values */
+    let valuesBufCursor = cursor;
+    if (header.data_page_header_v2?.is_compressed) {
+        const valuesBuf = await (0, compression_1.decompress)(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
+        valuesBufCursor = {
+            buffer: valuesBuf,
+            offset: 0,
+            size: valuesBuf.length
+        };
+        cursor.offset = cursorEnd;
+    }
+    const decodeOptions = {
+        typeLength: opts.column.typeLength,
+        bitWidth: opts.column.typeLength
+    };
+    const values = decodeValues(opts.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
+    return {
+        dlevels: dLevels,
+        rlevels: rLevels,
+        values,
+        count: valueCount,
+        pageHeader: header
+    };
+}
+/**
+ * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.
+ * @param cursor
+ * @param pageHeader
+ * @param options
+ */
+async function decodeDictionaryPage(cursor, pageHeader, options) {
+    const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
+    let dictCursor = {
+        offset: 0,
+        buffer: cursor.buffer.slice(cursor.offset, cursorEnd),
+        size: cursorEnd - cursor.offset
+    };
+    cursor.offset = cursorEnd;
+    if (options.compression !== 'UNCOMPRESSED') {
+        const valuesBuf = await (0, compression_1.decompress)(options.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
+        dictCursor = {
+            buffer: valuesBuf,
+            offset: 0,
+            size: valuesBuf.length
+        };
+        cursor.offset = cursorEnd;
+    }
+    const numValues = pageHeader?.dictionary_page_header?.num_values || 0;
+    return decodeValues(options.column.primitiveType, options.column.encoding, dictCursor, numValues, options).map((d) => d.toString());
+}

package/dist/parquetjs/parser/parquet-cursor.d.ts

@@ -0,0 +1,36 @@
+import { FileMetaData } from '../parquet-thrift';
+import { ParquetEnvelopeReader } from './parquet-envelope-reader';
+import { ParquetSchema } from '../schema/schema';
+import { ParquetRecord } from '../schema/declare';
+/**
+ * A parquet cursor is used to retrieve rows from a parquet file in order
+ */
+export declare class ParquetCursor<T> implements AsyncIterable<T> {
+    metadata: FileMetaData;
+    envelopeReader: ParquetEnvelopeReader;
+    schema: ParquetSchema;
+    columnList: string[][];
+    rowGroup: ParquetRecord[];
+    rowGroupIndex: number;
+    /**
+     * Create a new parquet reader from the file metadata and an envelope reader.
+     * It is usually not recommended to call this constructor directly except for
+     * advanced and internal use cases. Consider using getCursor() on the
+     * ParquetReader instead
+     */
+    constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader, schema: ParquetSchema, columnList: string[][]);
+    /**
+     * Retrieve the next row from the cursor. Returns a row or NULL if the end
+     * of the file was reached
+     */
+    next<T = any>(): Promise<T>;
+    /**
+     * Rewind the cursor the the beginning of the file
+     */
+    rewind(): void;
+    /**
+     * Implement AsyncIterable
+     */
+    [Symbol.asyncIterator](): AsyncIterator<T>;
+}
+//# sourceMappingURL=parquet-cursor.d.ts.map

package/dist/parquetjs/parser/parquet-cursor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-cursor.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-cursor.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAC,qBAAqB,EAAC,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAC,aAAa,EAAC,MAAM,mBAAmB,CAAC;AAGhD;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,CAAC;IAChD,QAAQ,EAAE,YAAY,CAAC;IACvB,cAAc,EAAE,qBAAqB,CAAC;IACtC,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,EAAE,CAAC;IACvB,QAAQ,EAAE,aAAa,EAAE,CAAM;IAC/B,aAAa,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;gBAED,QAAQ,EAAE,YAAY,EACtB,cAAc,EAAE,qBAAqB,EACrC,MAAM,EAAE,aAAa,EACrB,UAAU,EAAE,MAAM,EAAE,EAAE;IASxB;;;OAGG;IACG,IAAI,CAAC,CAAC,GAAG,GAAG,KAAK,OAAO,CAAC,CAAC,CAAC;IAiBjC;;OAEG;IACH,MAAM,IAAI,IAAI;IAKd;;OAEG;IAEH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;CAuB3C"}