npm - @loaders.gl/parquet - Versions diffs - 3.4.0-alpha.1 → 3.4.0-alpha.3 - Mend

@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (378) hide show

package/dist/esm/parquetjs/parquet-thrift/UUIDType.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"UUIDType.js","names":["thrift","UUIDType","constructor","write","output","writeStructBegin","writeFieldStop","writeStructEnd","read","input","readStructBegin","ret","readFieldBegin","fieldType","ftype","fieldId","fid","Thrift","Type","STOP","skip","readFieldEnd","readStructEnd"],"sources":["../../../../src/parquetjs/parquet-thrift/UUIDType.ts"],"sourcesContent":["/* tslint:disable /\n/ eslint-disable /\n/\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n /\nimport as thrift from 'thrift';\nexport interface IUUIDTypeArgs {}\nexport class UUIDType {\n constructor() {}\n public write(output: thrift.TProtocol): void {\n output.writeStructBegin('UUIDType');\n output.writeFieldStop();\n output.writeStructEnd();\n return;\n }\n public static read(input: thrift.TProtocol): UUIDType {\n input.readStructBegin();\n while (true) {\n const ret: thrift.TField = input.readFieldBegin();\n const fieldType: thrift.Thrift.Type = ret.ftype;\n const fieldId: number = ret.fid;\n if (fieldType === thrift.Thrift.Type.STOP) {\n break;\n }\n switch (fieldId) {\n default: {\n input.skip(fieldType);\n }\n }\n input.readFieldEnd();\n }\n input.readStructEnd();\n return new UUIDType();\n }\n}\n"],"mappings":";AAMA,OAAO,KAAKA,MAAM,MAAM,QAAQ;AAEhC,OAAO,MAAMC,QAAQ,CAAC;EACpBC,~~WAAW~~,~~GAAG~~,CAAC;EACRC,~~KAAK~~,CAACC,MAAwB,EAAQ;IAC3CA,MAAM,CAACC,gBAAgB,CAAC,UAAU,CAAC;IACnCD,MAAM,CAACE,cAAc,EAAE;IACvBF,MAAM,CAACG,cAAc,EAAE;IACvB;EACF;EACA,OAAcC,~~IAAI~~,CAACC,KAAuB,EAAY;IACpDA,KAAK,CAACC,eAAe,EAAE;IACvB,OAAO,IAAI,EAAE;MACX,MAAMC,GAAkB,GAAGF,KAAK,CAACG,cAAc,EAAE;MACjD,MAAMC,SAA6B,GAAGF,GAAG,CAACG,KAAK;MAC/C,MAAMC,OAAe,GAAGJ,GAAG,CAACK,GAAG;MAC/B,IAAIH,SAAS,KAAKb,MAAM,CAACiB,MAAM,CAACC,IAAI,CAACC,IAAI,EAAE;QACzC;MACF;MACA,QAAQJ,OAAO;QACb;UAAS;YACPN,KAAK,CAACW,IAAI,CAACP,SAAS,CAAC;UACvB;MAAC;MAEHJ,KAAK,CAACY,YAAY,EAAE;IACtB;IACAZ,KAAK,CAACa,aAAa,EAAE;IACrB,OAAO,IAAIrB,QAAQ,EAAE;EACvB;AACF"}
1	+ {"version":3,"file":"UUIDType.js","names":["thrift","UUIDType","constructor","write","output","writeStructBegin","writeFieldStop","writeStructEnd","read","input","readStructBegin","ret","readFieldBegin","fieldType","ftype","fieldId","fid","Thrift","Type","STOP","skip","readFieldEnd","readStructEnd"],"sources":["../../../../src/parquetjs/parquet-thrift/UUIDType.ts"],"sourcesContent":["/* tslint:disable /\n/ eslint-disable /\n/\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n /\nimport as thrift from 'thrift';\nexport interface IUUIDTypeArgs {}\nexport class UUIDType {\n constructor() {}\n public write(output: thrift.TProtocol): void {\n output.writeStructBegin('UUIDType');\n output.writeFieldStop();\n output.writeStructEnd();\n return;\n }\n public static read(input: thrift.TProtocol): UUIDType {\n input.readStructBegin();\n while (true) {\n const ret: thrift.TField = input.readFieldBegin();\n const fieldType: thrift.Thrift.Type = ret.ftype;\n const fieldId: number = ret.fid;\n if (fieldType === thrift.Thrift.Type.STOP) {\n break;\n }\n switch (fieldId) {\n default: {\n input.skip(fieldType);\n }\n }\n input.readFieldEnd();\n }\n input.readStructEnd();\n return new UUIDType();\n }\n}\n"],"mappings":"AAMA,OAAO,KAAKA,MAAM,MAAM,QAAQ;AAEhC,OAAO,MAAMC,QAAQ,CAAC;EACpBC,WAAWA,CAAA,EAAG,CAAC;EACRC,KAAKA,CAACC,MAAwB,EAAQ;IAC3CA,MAAM,CAACC,gBAAgB,CAAC,UAAU,CAAC;IACnCD,MAAM,CAACE,cAAc,EAAE;IACvBF,MAAM,CAACG,cAAc,EAAE;IACvB;EACF;EACA,OAAcC,IAAIA,CAACC,KAAuB,EAAY;IACpDA,KAAK,CAACC,eAAe,EAAE;IACvB,OAAO,IAAI,EAAE;MACX,MAAMC,GAAkB,GAAGF,KAAK,CAACG,cAAc,EAAE;MACjD,MAAMC,SAA6B,GAAGF,GAAG,CAACG,KAAK;MAC/C,MAAMC,OAAe,GAAGJ,GAAG,CAACK,GAAG;MAC/B,IAAIH,SAAS,KAAKb,MAAM,CAACiB,MAAM,CAACC,IAAI,CAACC,IAAI,EAAE;QACzC;MACF;MACA,QAAQJ,OAAO;QACb;UAAS;YACPN,KAAK,CAACW,IAAI,CAACP,SAAS,CAAC;UACvB;MAAC;MAEHJ,KAAK,CAACY,YAAY,EAAE;IACtB;IACAZ,KAAK,CAACa,aAAa,EAAE;IACrB,OAAO,IAAIrB,QAAQ,EAAE;EACvB;AACF"}

package/dist/esm/parquetjs/parquet-thrift/index.js CHANGED Viewed

@@ -1,4 +1,3 @@
 export * from './Type';
 export * from './ConvertedType';
 export * from './FieldRepetitionType';

package/dist/esm/parquetjs/parquet-thrift/index.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.js","names":[],"sources":["../../../../src/parquetjs/parquet-thrift/index.ts"],"sourcesContent":["/* tslint:disable /\n/ eslint-disable /\n/\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n /\nexport from './Type';\nexport * from './ConvertedType';\nexport * from './FieldRepetitionType';\nexport * from './Encoding';\nexport * from './CompressionCodec';\nexport * from './PageType';\nexport * from './BoundaryOrder';\nexport * from './Statistics';\nexport * from './StringType';\nexport * from './UUIDType';\nexport * from './MapType';\nexport * from './ListType';\nexport * from './EnumType';\nexport * from './DateType';\nexport * from './NullType';\nexport * from './DecimalType';\nexport * from './MilliSeconds';\nexport * from './MicroSeconds';\nexport * from './TimestampType';\nexport * from './TimeType';\nexport * from './IntType';\nexport * from './JsonType';\nexport * from './BsonType';\nexport * from './SchemaElement';\nexport * from './DataPageHeader';\nexport * from './IndexPageHeader';\nexport * from './DictionaryPageHeader';\nexport * from './DataPageHeaderV2';\nexport * from './PageHeader';\nexport * from './KeyValue';\nexport * from './SortingColumn';\nexport * from './PageEncodingStats';\nexport * from './ColumnMetaData';\nexport * from './ColumnChunk';\nexport * from './RowGroup';\nexport * from './TypeDefinedOrder';\nexport * from './PageLocation';\nexport * from './OffsetIndex';\nexport * from './ColumnIndex';\nexport * from './FileMetaData';\nexport * from './TimeUnit';\nexport * from './LogicalType';\nexport * from './ColumnOrder';\n"],"mappings":";AAMA,cAAc,QAAQ;AACtB,cAAc,iBAAiB;AAC/B,cAAc,uBAAuB;AACrC,cAAc,YAAY;AAC1B,cAAc,oBAAoB;AAClC,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,cAAc;AAC5B,cAAc,cAAc;AAC5B,cAAc,YAAY;AAC1B,cAAc,WAAW;AACzB,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,eAAe;AAC7B,cAAc,gBAAgB;AAC9B,cAAc,gBAAgB;AAC9B,cAAc,iBAAiB;AAC/B,cAAc,YAAY;AAC1B,cAAc,WAAW;AACzB,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,kBAAkB;AAChC,cAAc,mBAAmB;AACjC,cAAc,wBAAwB;AACtC,cAAc,oBAAoB;AAClC,cAAc,cAAc;AAC5B,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,qBAAqB;AACnC,cAAc,kBAAkB;AAChC,cAAc,eAAe;AAC7B,cAAc,YAAY;AAC1B,cAAc,oBAAoB;AAClC,cAAc,gBAAgB;AAC9B,cAAc,eAAe;AAC7B,cAAc,eAAe;AAC7B,cAAc,gBAAgB;AAC9B,cAAc,YAAY;AAC1B,cAAc,eAAe;AAC7B,cAAc,eAAe"}
1	+ {"version":3,"file":"index.js","names":[],"sources":["../../../../src/parquetjs/parquet-thrift/index.ts"],"sourcesContent":["/* tslint:disable /\n/ eslint-disable /\n/\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n /\nexport from './Type';\nexport * from './ConvertedType';\nexport * from './FieldRepetitionType';\nexport * from './Encoding';\nexport * from './CompressionCodec';\nexport * from './PageType';\nexport * from './BoundaryOrder';\nexport * from './Statistics';\nexport * from './StringType';\nexport * from './UUIDType';\nexport * from './MapType';\nexport * from './ListType';\nexport * from './EnumType';\nexport * from './DateType';\nexport * from './NullType';\nexport * from './DecimalType';\nexport * from './MilliSeconds';\nexport * from './MicroSeconds';\nexport * from './TimestampType';\nexport * from './TimeType';\nexport * from './IntType';\nexport * from './JsonType';\nexport * from './BsonType';\nexport * from './SchemaElement';\nexport * from './DataPageHeader';\nexport * from './IndexPageHeader';\nexport * from './DictionaryPageHeader';\nexport * from './DataPageHeaderV2';\nexport * from './PageHeader';\nexport * from './KeyValue';\nexport * from './SortingColumn';\nexport * from './PageEncodingStats';\nexport * from './ColumnMetaData';\nexport * from './ColumnChunk';\nexport * from './RowGroup';\nexport * from './TypeDefinedOrder';\nexport * from './PageLocation';\nexport * from './OffsetIndex';\nexport * from './ColumnIndex';\nexport * from './FileMetaData';\nexport * from './TimeUnit';\nexport * from './LogicalType';\nexport * from './ColumnOrder';\n"],"mappings":"AAMA,cAAc,QAAQ;AACtB,cAAc,iBAAiB;AAC/B,cAAc,uBAAuB;AACrC,cAAc,YAAY;AAC1B,cAAc,oBAAoB;AAClC,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,cAAc;AAC5B,cAAc,cAAc;AAC5B,cAAc,YAAY;AAC1B,cAAc,WAAW;AACzB,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,eAAe;AAC7B,cAAc,gBAAgB;AAC9B,cAAc,gBAAgB;AAC9B,cAAc,iBAAiB;AAC/B,cAAc,YAAY;AAC1B,cAAc,WAAW;AACzB,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,kBAAkB;AAChC,cAAc,mBAAmB;AACjC,cAAc,wBAAwB;AACtC,cAAc,oBAAoB;AAClC,cAAc,cAAc;AAC5B,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,qBAAqB;AACnC,cAAc,kBAAkB;AAChC,cAAc,eAAe;AAC7B,cAAc,YAAY;AAC1B,cAAc,oBAAoB;AAClC,cAAc,gBAAgB;AAC9B,cAAc,eAAe;AAC7B,cAAc,eAAe;AAC7B,cAAc,gBAAgB;AAC9B,cAAc,YAAY;AAC1B,cAAc,eAAe;AAC7B,cAAc,eAAe"}

package/dist/esm/parquetjs/parser/decoders.js CHANGED Viewed

@@ -1,11 +1,8 @@
 import { PARQUET_CODECS } from '../codecs';
 import { ConvertedType, Encoding, FieldRepetitionType, PageType, Type } from '../parquet-thrift';
 import { decompress } from '../compression';
 import { PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING } from '../../constants';
 import { decodePageHeader, getThriftEnum, getBitWidth } from '../utils/read-utils';
 export async function decodeDataPages(buffer, options) {
   const cursor = {
     buffer,
@@ -20,8 +17,7 @@ export async function decodeDataPages(buffer, options) {
     count: 0
   };
   let dictionary = options.dictionary || [];
-  while (
-  cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues))) {
+  while (cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues))) {
     const page = await decodePage(cursor, options);
     if (page.dictionary) {
       dictionary = page.dictionary;
@@ -43,7 +39,6 @@ export async function decodeDataPages(buffer, options) {
   }
   return data;
 }
 export async function decodePage(cursor, options) {
   let page;
   const {
@@ -70,7 +65,6 @@ export async function decodePage(cursor, options) {
   }
   return page;
 }
 export function decodeSchema(schemaElements, offset, len) {
   const schema = {};
   let next = offset;
@@ -128,19 +122,16 @@ export function decodeSchema(schemaElements, offset, len) {
     next
   };
 }
 function decodeValues(type, encoding, cursor, count, opts) {
   if (!(encoding in PARQUET_CODECS)) {
     throw new Error("invalid encoding: ".concat(encoding));
   }
   return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);
 }
 async function decodeDataPage(cursor, header, options) {
   var _header$data_page_hea, _header$data_page_hea2, _header$data_page_hea3, _header$data_page_hea4;
   const cursorEnd = cursor.offset + header.compressed_page_size;
   const valueCount = (_header$data_page_hea = header.data_page_header) === null || _header$data_page_hea === void 0 ? void 0 : _header$data_page_hea.num_values;
   let dataCursor = cursor;
   if (options.compression !== 'UNCOMPRESSED') {
     const valuesBuf = await decompress(options.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
@@ -151,7 +142,6 @@ async function decodeDataPage(cursor, header, options) {
     };
     cursor.offset = cursorEnd;
   }
   const rLevelEncoding = getThriftEnum(Encoding, (_header$data_page_hea2 = header.data_page_header) === null || _header$data_page_hea2 === void 0 ? void 0 : _header$data_page_hea2.repetition_level_encoding);
   let rLevels = new Array(valueCount);
   if (options.column.rLevelMax > 0) {
@@ -162,7 +152,6 @@ async function decodeDataPage(cursor, header, options) {
   } else {
     rLevels.fill(0);
   }
   const dLevelEncoding = getThriftEnum(Encoding, (_header$data_page_hea3 = header.data_page_header) === null || _header$data_page_hea3 === void 0 ? void 0 : _header$data_page_hea3.definition_level_encoding);
   let dLevels = new Array(valueCount);
   if (options.column.dLevelMax > 0) {
@@ -179,7 +168,6 @@ async function decodeDataPage(cursor, header, options) {
       valueCountNonNull++;
     }
   }
   const valueEncoding = getThriftEnum(Encoding, (_header$data_page_hea4 = header.data_page_header) === null || _header$data_page_hea4 === void 0 ? void 0 : _header$data_page_hea4.encoding);
   const decodeOptions = {
     typeLength: options.column.typeLength,
@@ -194,14 +182,12 @@ async function decodeDataPage(cursor, header, options) {
     pageHeader: header
   };
 }
 async function decodeDataPageV2(cursor, header, opts) {
   var _header$data_page_hea5, _header$data_page_hea6, _header$data_page_hea7, _header$data_page_hea8;
   const cursorEnd = cursor.offset + header.compressed_page_size;
   const valueCount = (_header$data_page_hea5 = header.data_page_header_v2) === null || _header$data_page_hea5 === void 0 ? void 0 : _header$data_page_hea5.num_values;
   const valueCountNonNull = valueCount - ((_header$data_page_hea6 = header.data_page_header_v2) === null || _header$data_page_hea6 === void 0 ? void 0 : _header$data_page_hea6.num_nulls);
   const valueEncoding = getThriftEnum(Encoding, (_header$data_page_hea7 = header.data_page_header_v2) === null || _header$data_page_hea7 === void 0 ? void 0 : _header$data_page_hea7.encoding);
   let rLevels = new Array(valueCount);
   if (opts.column.rLevelMax > 0) {
     rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
@@ -211,7 +197,6 @@ async function decodeDataPageV2(cursor, header, opts) {
   } else {
     rLevels.fill(0);
   }
   let dLevels = new Array(valueCount);
   if (opts.column.dLevelMax > 0) {
     dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
@@ -221,7 +206,6 @@ async function decodeDataPageV2(cursor, header, opts) {
   } else {
     dLevels.fill(0);
   }
   let valuesBufCursor = cursor;
   if ((_header$data_page_hea8 = header.data_page_header_v2) !== null && _header$data_page_hea8 !== void 0 && _header$data_page_hea8.is_compressed) {
     const valuesBuf = await decompress(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
@@ -245,7 +229,6 @@ async function decodeDataPageV2(cursor, header, opts) {
     pageHeader: header
   };
 }
 async function decodeDictionaryPage(cursor, pageHeader, options) {
   var _pageHeader$dictionar;
   const cursorEnd = cursor.offset + pageHeader.compressed_page_size;

package/dist/esm/parquetjs/parser/decoders.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"decoders.js","names":["PARQUET_CODECS","ConvertedType","Encoding","FieldRepetitionType","PageType","Type","decompress","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","decodePageHeader","getThriftEnum","getBitWidth","decodeDataPages","buffer","options","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","pageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","repetition_type","optional","repeated","num_children","res","name","fields","logicalType","converted_type","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","header","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","slice","uncompressed_page_size","rLevelEncoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","data_page_header_v2","num_nulls","valuesBufCursor","is_compressed","dictCursor","dictionary_page_header","d","toString"],"sources":["../../../../src/parquetjs/parser/decoders.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n ParquetCodec,\n ParquetData,\n ParquetOptions,\n ParquetPageData,\n ParquetType,\n PrimitiveType,\n SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n ConvertedType,\n Encoding,\n FieldRepetitionType,\n PageHeader,\n PageType,\n SchemaElement,\n Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/*\n Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n /\nexport async function decodeDataPages(\n buffer: Buffer,\n options: ParquetOptions\n): Promise<ParquetData> {\n const cursor: CursorBuffer = {\n buffer,\n offset: 0,\n size: buffer.length\n };\n\n const data: ParquetData = {\n rlevels: [],\n dlevels: [],\n values: [],\n pageHeaders: [],\n count: 0\n };\n\n let dictionary = options.dictionary \|\| [];\n\n while (\n // @ts-ignore size can be undefined\n cursor.offset < cursor.size &&\n (!options.numValues \|\| data.dlevels.length < Number(options.numValues))\n ) {\n // Looks like we have to decode these in sequence due to cursor updates?\n const page = await decodePage(cursor, options);\n\n if (page.dictionary) {\n dictionary = page.dictionary;\n // eslint-disable-next-line no-continue\n continue;\n }\n\n if (dictionary.length) {\n // eslint-disable-next-line no-loop-func\n page.values = page.values.map((value) => dictionary[value]);\n }\n\n for (let index = 0; index < page.rlevels.length; index++) {\n data.rlevels.push(page.rlevels[index]);\n data.dlevels.push(page.dlevels[index]);\n const value = page.values[index];\n\n if (value !== undefined) {\n data.values.push(value);\n }\n }\n\n data.count += page.count;\n data.pageHeaders.push(page.pageHeader);\n }\n\n return data;\n}\n\n/\n Decode parquet page based on page type\n * @param cursor\n * @param options\n /\nexport async function decodePage(\n cursor: CursorBuffer,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n let page;\n const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n cursor.offset += length;\n\n const pageType = getThriftEnum(PageType, pageHeader.type);\n\n switch (pageType) {\n case 'DATA_PAGE':\n page = await decodeDataPage(cursor, pageHeader, options);\n break;\n case 'DATA_PAGE_V2':\n page = await decodeDataPageV2(cursor, pageHeader, options);\n break;\n case 'DICTIONARY_PAGE':\n page = {\n dictionary: await decodeDictionaryPage(cursor, pageHeader, options),\n pageHeader\n };\n break;\n default:\n throw new Error(`invalid page type: ${pageType}`);\n }\n\n return page;\n}\n\n/\n Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n * result.next - offset at the end of function\n * result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n /\nexport function decodeSchema(\n schemaElements: SchemaElement[],\n offset: number,\n len: number\n): {\n offset: number;\n next: number;\n schema: SchemaDefinition;\n} {\n const schema: SchemaDefinition = {};\n let next = offset;\n for (let i = 0; i < len; i++) {\n const schemaElement = schemaElements[next];\n\n const repetitionType =\n next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n let optional = false;\n let repeated = false;\n switch (repetitionType) {\n case 'REQUIRED':\n break;\n case 'OPTIONAL':\n optional = true;\n break;\n case 'REPEATED':\n repeated = true;\n break;\n default:\n throw new Error('parquet: unknown repetition type');\n }\n\n if (schemaElement.num_children! > 0) {\n const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n next = res.next;\n schema[schemaElement.name] = {\n // type: undefined,\n optional,\n repeated,\n fields: res.schema\n };\n } else {\n const type = getThriftEnum(Type, schemaElement.type!);\n let logicalType = type;\n\n if (schemaElement.converted_type) {\n logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n }\n\n switch (logicalType) {\n case 'DECIMAL':\n logicalType = `${logicalType}_${type}` as ParquetType;\n break;\n default:\n }\n\n schema[schemaElement.name] = {\n type: logicalType as ParquetType,\n typeLength: schemaElement.type_length,\n presision: schemaElement.precision,\n scale: schemaElement.scale,\n optional,\n repeated\n };\n next++;\n }\n }\n return {schema, offset, next};\n}\n\n/\n Decode a consecutive array of data using one of the parquet encodings\n /\nfunction decodeValues(\n type: PrimitiveType,\n encoding: ParquetCodec,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): any[] {\n if (!(encoding in PARQUET_CODECS)) {\n throw new Error(`invalid encoding: ${encoding}`);\n }\n return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/\n Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n /\nasync function decodeDataPage(\n cursor: CursorBuffer,\n header: PageHeader,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n const valueCount = header.data_page_header?.num_values;\n\n / uncompress page /\n let dataCursor = cursor;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n dataCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n cursor.offset = cursorEnd;\n }\n\n / read repetition levels /\n const rLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.repetition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n\n if (options.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.rLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n rLevels.fill(0);\n }\n\n / read definition levels /\n const dLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.definition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (options.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.dLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n dLevels.fill(0);\n }\n let valueCountNonNull = 0;\n for (const dlvl of dLevels) {\n if (dlvl === options.column.dLevelMax) {\n valueCountNonNull++;\n }\n }\n\n / read values /\n const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n const decodeOptions = {\n typeLength: options.column.typeLength,\n bitWidth: options.column.typeLength\n };\n\n const values = decodeValues(\n options.column.primitiveType!,\n valueEncoding,\n dataCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/\n Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n /\nasync function decodeDataPageV2(\n cursor: CursorBuffer,\n header: PageHeader,\n opts: any\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n\n const valueCount = header.data_page_header_v2?.num_values;\n // @ts-ignore\n const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n const valueEncoding = getThriftEnum(\n Encoding,\n header.data_page_header_v2?.encoding!\n ) as ParquetCodec;\n\n / read repetition levels /\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n if (opts.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.rLevelMax),\n disableEnvelope: true\n });\n } else {\n rLevels.fill(0);\n }\n\n / read definition levels /\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (opts.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.dLevelMax),\n disableEnvelope: true\n });\n } else {\n dLevels.fill(0);\n }\n\n / read values /\n let valuesBufCursor = cursor;\n\n if (header.data_page_header_v2?.is_compressed) {\n const valuesBuf = await decompress(\n opts.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n\n valuesBufCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const decodeOptions = {\n typeLength: opts.column.typeLength,\n bitWidth: opts.column.typeLength\n };\n\n const values = decodeValues(\n opts.column.primitiveType!,\n valueEncoding,\n valuesBufCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/\n Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param options\n */\nasync function decodeDictionaryPage(\n cursor: CursorBuffer,\n pageHeader: PageHeader,\n options: ParquetOptions\n): Promise<string[]> {\n const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n let dictCursor = {\n offset: 0,\n buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n size: cursorEnd - cursor.offset\n };\n\n cursor.offset = cursorEnd;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n pageHeader.uncompressed_page_size\n );\n\n dictCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const numValues = pageHeader?.dictionary_page_header?.num_values \|\| 0;\n\n return decodeValues(\n options.column.primitiveType!,\n options.column.encoding!,\n dictCursor,\n numValues,\n options as ParquetCodecOptions\n ).map((d) => d.toString());\n}\n"],"mappings":";;AAUA,SAA2CA,cAAc,QAAO,WAAW;AAC3E,SACEC,aAAa,EACbC,QAAQ,EACRC,mBAAmB,EAEnBC,QAAQ,EAERC,IAAI,QACC,mBAAmB;AAC1B,SAAQC,UAAU,QAAO,gBAAgB;AACzC,SAAQC,kBAAkB,EAAEC,sBAAsB,QAAO,iBAAiB;AAC1E,SAAQC,gBAAgB,EAAEC,aAAa,EAAEC,WAAW,QAAO,qBAAqB;;AAShF,OAAO,eAAeC,eAAe,CACnCC,MAAc,EACdC,OAAuB,EACD;EACtB,MAAMC,MAAoB,GAAG;IAC3BF,MAAM;IACNG,MAAM,EAAE,CAAC;IACTC,IAAI,EAAEJ,MAAM,CAACK;EACf,CAAC;EAED,MAAMC,IAAiB,GAAG;IACxBC,OAAO,EAAE,EAAE;IACXC,OAAO,EAAE,EAAE;IACXC,MAAM,EAAE,EAAE;IACVC,WAAW,EAAE,EAAE;IACfC,KAAK,EAAE;EACT,CAAC;EAED,IAAIC,UAAU,GAAGX,OAAO,CAACW,UAAU,IAAI,EAAE;EAEzC;EAEEV,MAAM,CAACC,MAAM,GAAGD,MAAM,CAACE,IAAI,KAC1B,CAACH,OAAO,CAACY,SAAS,IAAIP,IAAI,CAACE,OAAO,CAACH,MAAM,GAAGS,MAAM,CAACb,OAAO,CAACY,SAAS,CAAC,CAAC,EACvE;IAEA,MAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAM,EAAED,OAAO,CAAC;IAE9C,IAAIc,IAAI,CAACH,UAAU,EAAE;MACnBA,UAAU,GAAGG,IAAI,CAACH,UAAU;MAE5B;IACF;IAEA,IAAIA,UAAU,CAACP,MAAM,EAAE;MAErBU,IAAI,CAACN,MAAM,GAAGM,IAAI,CAACN,MAAM,CAACQ,GAAG,CAAEC,KAAK,IAAKN,UAAU,CAACM,KAAK,CAAC,CAAC;IAC7D;IAEA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGJ,IAAI,CAACR,OAAO,CAACF,MAAM,EAAEc,KAAK,EAAE,EAAE;MACxDb,IAAI,CAACC,OAAO,CAACa,IAAI,CAACL,IAAI,CAACR,OAAO,CAACY,KAAK,CAAC,CAAC;MACtCb,IAAI,CAACE,OAAO,CAACY,IAAI,CAACL,IAAI,CAACP,OAAO,CAACW,KAAK,CAAC,CAAC;MACtC,MAAMD,KAAK,GAAGH,IAAI,CAACN,MAAM,CAACU,KAAK,CAAC;MAEhC,IAAID,KAAK,KAAKG,SAAS,EAAE;QACvBf,IAAI,CAACG,MAAM,CAACW,IAAI,CAACF,KAAK,CAAC;MACzB;IACF;IAEAZ,IAAI,CAACK,KAAK,IAAII,IAAI,CAACJ,KAAK;IACxBL,IAAI,CAACI,WAAW,CAACU,IAAI,CAACL,IAAI,CAACO,UAAU,CAAC;EACxC;EAEA,OAAOhB,IAAI;AACb;;AAOA,OAAO,eAAeU,UAAU,CAC9Bd,MAAoB,EACpBD,OAAuB,EACG;EAC1B,IAAIc,IAAI;EACR,MAAM;IAACO,UAAU;IAAEjB;EAAM,CAAC,GAAGT,gBAAgB,CAACM,MAAM,CAACF,MAAM,EAAEE,MAAM,CAACC,MAAM,CAAC;EAC3ED,MAAM,CAACC,MAAM,IAAIE,MAAM;EAEvB,MAAMkB,QAAQ,GAAG1B,aAAa,CAACN,QAAQ,EAAE+B,UAAU,CAACE,IAAI,CAAC;EAEzD,QAAQD,QAAQ;IACd,KAAK,WAAW;MACdR,IAAI,GAAG,MAAMU,cAAc,CAACvB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MACxD;IACF,KAAK,cAAc;MACjBc,IAAI,GAAG,MAAMW,gBAAgB,CAACxB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MAC1D;IACF,KAAK,iBAAiB;MACpBc,IAAI,GAAG;QACLH,UAAU,EAAE,MAAMe,oBAAoB,CAACzB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;QACnEqB;MACF,CAAC;MACD;IACF;MACE,MAAM,IAAIM,KAAK,8BAAuBL,QAAQ,EAAG;EAAC;EAGtD,OAAOR,IAAI;AACb;;AAYA,OAAO,SAASc,YAAY,CAC1BC,cAA+B,EAC/B3B,MAAc,EACd4B,GAAW,EAKX;EACA,MAAMC,MAAwB,GAAG,CAAC,CAAC;EACnC,IAAIC,IAAI,GAAG9B,MAAM;EACjB,KAAK,IAAI+B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,GAAG,EAAEG,CAAC,EAAE,EAAE;IAC5B,MAAMC,aAAa,GAAGL,cAAc,CAACG,IAAI,CAAC;IAE1C,MAAMG,cAAc,GAClBH,IAAI,GAAG,CAAC,GAAGpC,aAAa,CAACP,mBAAmB,EAAE6C,aAAa,CAACE,eAAe,CAAE,GAAG,MAAM;IAExF,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,QAAQ,GAAG,KAAK;IACpB,QAAQH,cAAc;MACpB,KAAK,UAAU;QACb;MACF,KAAK,UAAU;QACbE,QAAQ,GAAG,IAAI;QACf;MACF,KAAK,UAAU;QACbC,QAAQ,GAAG,IAAI;QACf;MACF;QACE,MAAM,IAAIX,KAAK,CAAC,kCAAkC,CAAC;IAAC;IAGxD,IAAIO,aAAa,CAACK,YAAY,GAAI,CAAC,EAAE;MACnC,MAAMC,GAAG,GAAGZ,YAAY,CAACC,cAAc,EAAEG,IAAI,GAAG,CAAC,EAAEE,aAAa,CAACK,YAAY,CAAE;MAC/EP,IAAI,GAAGQ,GAAG,CAACR,IAAI;MACfD,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAE3BJ,QAAQ;QACRC,QAAQ;QACRI,MAAM,EAAEF,GAAG,CAACT;MACd,CAAC;IACH,CAAC,MAAM;MACL,MAAMR,IAAI,GAAG3B,aAAa,CAACL,IAAI,EAAE2C,aAAa,CAACX,IAAI,CAAE;MACrD,IAAIoB,WAAW,GAAGpB,IAAI;MAEtB,IAAIW,aAAa,CAACU,cAAc,EAAE;QAChCD,WAAW,GAAG/C,aAAa,CAACT,aAAa,EAAE+C,aAAa,CAACU,cAAc,CAAC;MAC1E;MAEA,QAAQD,WAAW;QACjB,KAAK,SAAS;UACZA,WAAW,aAAMA,WAAW,cAAIpB,IAAI,CAAiB;UACrD;QACF;MAAQ;MAGVQ,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAC3BlB,IAAI,EAAEoB,WAA0B;QAChCE,UAAU,EAAEX,aAAa,CAACY,WAAW;QACrCC,SAAS,EAAEb,aAAa,CAACc,SAAS;QAClCC,KAAK,EAAEf,aAAa,CAACe,KAAK;QAC1BZ,QAAQ;QACRC;MACF,CAAC;MACDN,IAAI,EAAE;IACR;EACF;EACA,OAAO;IAACD,MAAM;IAAE7B,MAAM;IAAE8B;EAAI,CAAC;AAC/B;;AAKA,SAASkB,YAAY,CACnB3B,IAAmB,EACnB4B,QAAsB,EACtBlD,MAAoB,EACpBS,KAAa,EACb0C,IAAyB,EAClB;EACP,IAAI,EAAED,QAAQ,IAAIjE,cAAc,CAAC,EAAE;IACjC,MAAM,IAAIyC,KAAK,6BAAsBwB,QAAQ,EAAG;EAClD;EACA,OAAOjE,cAAc,CAACiE,QAAQ,CAAC,CAACD,YAAY,CAAC3B,IAAI,EAAEtB,MAAM,EAAES,KAAK,EAAE0C,IAAI,CAAC;AACzE;;AAQA,eAAe5B,cAAc,CAC3BvB,MAAoB,EACpBoD,MAAkB,EAClBrD,OAAuB,EACG;EAAA;EAC1B,MAAMsD,SAAS,GAAGrD,MAAM,CAACC,MAAM,GAAGmD,MAAM,CAACE,oBAAoB;EAC7D,MAAMC,UAAU,4BAAGH,MAAM,CAACI,gBAAgB,0DAAvB,sBAAyBC,UAAU;;EAGtD,IAAIC,UAAU,GAAG1D,MAAM;EAEvB,IAAID,OAAO,CAAC4D,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAMrE,UAAU,CAChCQ,OAAO,CAAC4D,WAAW,EACnB3D,MAAM,CAACF,MAAM,CAAC+D,KAAK,CAAC7D,MAAM,CAACC,MAAM,EAAEoD,SAAS,CAAC,EAC7CD,MAAM,CAACU,sBAAsB,CAC9B;IACDJ,UAAU,GAAG;MACX5D,MAAM,EAAE8D,SAAS;MACjB3D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE0D,SAAS,CAACzD;IAClB,CAAC;IACDH,MAAM,CAACC,MAAM,GAAGoD,SAAS;EAC3B;;EAGA,MAAMU,cAAc,GAAGpE,aAAa,CAClCR,QAAQ,4BACRiE,MAAM,CAACI,gBAAgB,2DAAvB,uBAAyBQ,yBAAyB,CACnC;EAEjB,IAAIC,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EAEnC,IAAIxD,OAAO,CAACoE,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGhB,YAAY,CAACzD,kBAAkB,EAAEuE,cAAc,EAAEL,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAEzE,WAAW,CAACG,OAAO,CAACoE,MAAM,CAACC,SAAS,CAAC;MAC/CE,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;;EAGA,MAAMC,cAAc,GAAG7E,aAAa,CAClCR,QAAQ,4BACRiE,MAAM,CAACI,gBAAgB,2DAAvB,uBAAyBiB,yBAAyB,CACnC;EAEjB,IAAIC,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAIxD,OAAO,CAACoE,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAGzB,YAAY,CAACzD,kBAAkB,EAAEgF,cAAc,EAAEd,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAEzE,WAAW,CAACG,OAAO,CAACoE,MAAM,CAACQ,SAAS,CAAC;MAC/CL,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EACA,IAAIK,iBAAiB,GAAG,CAAC;EACzB,KAAK,MAAMC,IAAI,IAAIH,OAAO,EAAE;IAC1B,IAAIG,IAAI,KAAK9E,OAAO,CAACoE,MAAM,CAACQ,SAAS,EAAE;MACrCC,iBAAiB,EAAE;IACrB;EACF;;EAGA,MAAME,aAAa,GAAGnF,aAAa,CAACR,QAAQ,4BAAEiE,MAAM,CAACI,gBAAgB,2DAAvB,uBAAyBN,QAAQ,CAAkB;EACjG,MAAM6B,aAAa,GAAG;IACpBnC,UAAU,EAAE7C,OAAO,CAACoE,MAAM,CAACvB,UAAU;IACrCyB,QAAQ,EAAEtE,OAAO,CAACoE,MAAM,CAACvB;EAC3B,CAAC;EAED,MAAMrC,MAAM,GAAG0C,YAAY,CACzBlD,OAAO,CAACoE,MAAM,CAACa,aAAa,EAC5BF,aAAa,EACbpB,UAAU,EACVkB,iBAAiB,EACjBG,aAAa,CACd;EAED,OAAO;IACLzE,OAAO,EAAEoE,OAAO;IAChBrE,OAAO,EAAE4D,OAAO;IAChB1D,MAAM;IACNE,KAAK,EAAE8C,UAAW;IAClBnC,UAAU,EAAEgC;EACd,CAAC;AACH;;AASA,eAAe5B,gBAAgB,CAC7BxB,MAAoB,EACpBoD,MAAkB,EAClBD,IAAS,EACiB;EAAA;EAC1B,MAAME,SAAS,GAAGrD,MAAM,CAACC,MAAM,GAAGmD,MAAM,CAACE,oBAAoB;EAE7D,MAAMC,UAAU,6BAAGH,MAAM,CAAC6B,mBAAmB,2DAA1B,uBAA4BxB,UAAU;EAEzD,MAAMmB,iBAAiB,GAAGrB,UAAU,8BAAGH,MAAM,CAAC6B,mBAAmB,2DAA1B,uBAA4BC,SAAS;EAC5E,MAAMJ,aAAa,GAAGnF,aAAa,CACjCR,QAAQ,4BACRiE,MAAM,CAAC6B,mBAAmB,2DAA1B,uBAA4B/B,QAAQ,CACrB;;EAIjB,IAAIe,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EACnC,IAAIJ,IAAI,CAACgB,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAC7BH,OAAO,GAAGhB,YAAY,CAACzD,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAEuD,UAAU,EAAG;MACtFc,QAAQ,EAAEzE,WAAW,CAACuD,IAAI,CAACgB,MAAM,CAACC,SAAS,CAAC;MAC5CE,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;;EAIA,IAAIG,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAIJ,IAAI,CAACgB,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAC7BD,OAAO,GAAGzB,YAAY,CAACzD,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAEuD,UAAU,EAAG;MACtFc,QAAQ,EAAEzE,WAAW,CAACuD,IAAI,CAACgB,MAAM,CAACQ,SAAS,CAAC;MAC5CL,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;;EAGA,IAAIY,eAAe,GAAGnF,MAAM;EAE5B,8BAAIoD,MAAM,CAAC6B,mBAAmB,mDAA1B,uBAA4BG,aAAa,EAAE;IAC7C,MAAMxB,SAAS,GAAG,MAAMrE,UAAU,CAChC4D,IAAI,CAACQ,WAAW,EAChB3D,MAAM,CAACF,MAAM,CAAC+D,KAAK,CAAC7D,MAAM,CAACC,MAAM,EAAEoD,SAAS,CAAC,EAC7CD,MAAM,CAACU,sBAAsB,CAC9B;IAEDqB,eAAe,GAAG;MAChBrF,MAAM,EAAE8D,SAAS;MACjB3D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE0D,SAAS,CAACzD;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGoD,SAAS;EAC3B;EAEA,MAAM0B,aAAa,GAAG;IACpBnC,UAAU,EAAEO,IAAI,CAACgB,MAAM,CAACvB,UAAU;IAClCyB,QAAQ,EAAElB,IAAI,CAACgB,MAAM,CAACvB;EACxB,CAAC;EAED,MAAMrC,MAAM,GAAG0C,YAAY,CACzBE,IAAI,CAACgB,MAAM,CAACa,aAAa,EACzBF,aAAa,EACbK,eAAe,EACfP,iBAAiB,EACjBG,aAAa,CACd;EAED,OAAO;IACLzE,OAAO,EAAEoE,OAAO;IAChBrE,OAAO,EAAE4D,OAAO;IAChB1D,MAAM;IACNE,KAAK,EAAE8C,UAAW;IAClBnC,UAAU,EAAEgC;EACd,CAAC;AACH;;AAQA,eAAe3B,oBAAoB,CACjCzB,MAAoB,EACpBoB,UAAsB,EACtBrB,OAAuB,EACJ;EAAA;EACnB,MAAMsD,SAAS,GAAGrD,MAAM,CAACC,MAAM,GAAGmB,UAAU,CAACkC,oBAAoB;EAEjE,IAAI+B,UAAU,GAAG;IACfpF,MAAM,EAAE,CAAC;IACTH,MAAM,EAAEE,MAAM,CAACF,MAAM,CAAC+D,KAAK,CAAC7D,MAAM,CAACC,MAAM,EAAEoD,SAAS,CAAC;IACrDnD,IAAI,EAAEmD,SAAS,GAAGrD,MAAM,CAACC;EAC3B,CAAC;EAEDD,MAAM,CAACC,MAAM,GAAGoD,SAAS;EAEzB,IAAItD,OAAO,CAAC4D,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAMrE,UAAU,CAChCQ,OAAO,CAAC4D,WAAW,EACnB0B,UAAU,CAACvF,MAAM,CAAC+D,KAAK,CAACwB,UAAU,CAACpF,MAAM,EAAEoD,SAAS,CAAC,EACrDjC,UAAU,CAAC0C,sBAAsB,CAClC;IAEDuB,UAAU,GAAG;MACXvF,MAAM,EAAE8D,SAAS;MACjB3D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE0D,SAAS,CAACzD;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGoD,SAAS;EAC3B;EAEA,MAAM1C,SAAS,GAAG,CAAAS,UAAU,aAAVA,UAAU,gDAAVA,UAAU,CAAEkE,sBAAsB,0DAAlC,sBAAoC7B,UAAU,KAAI,CAAC;EAErE,OAAOR,YAAY,CACjBlD,OAAO,CAACoE,MAAM,CAACa,aAAa,EAC5BjF,OAAO,CAACoE,MAAM,CAACjB,QAAQ,EACvBmC,UAAU,EACV1E,SAAS,EACTZ,OAAO,CACR,CAACgB,GAAG,CAAEwE,CAAC,IAAKA,CAAC,CAACC,QAAQ,EAAE,CAAC;AAC5B"}
1	+ {"version":3,"file":"decoders.js","names":["PARQUET_CODECS","ConvertedType","Encoding","FieldRepetitionType","PageType","Type","decompress","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","decodePageHeader","getThriftEnum","getBitWidth","decodeDataPages","buffer","options","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","pageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","concat","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","repetition_type","optional","repeated","num_children","res","name","fields","logicalType","converted_type","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","header","_header$data_page_hea","_header$data_page_hea2","_header$data_page_hea3","_header$data_page_hea4","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","slice","uncompressed_page_size","rLevelEncoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","_header$data_page_hea5","_header$data_page_hea6","_header$data_page_hea7","_header$data_page_hea8","data_page_header_v2","num_nulls","valuesBufCursor","is_compressed","_pageHeader$dictionar","dictCursor","dictionary_page_header","d","toString"],"sources":["../../../../src/parquetjs/parser/decoders.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n ParquetCodec,\n ParquetData,\n ParquetOptions,\n ParquetPageData,\n ParquetType,\n PrimitiveType,\n SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n ConvertedType,\n Encoding,\n FieldRepetitionType,\n PageHeader,\n PageType,\n SchemaElement,\n Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/*\n Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n /\nexport async function decodeDataPages(\n buffer: Buffer,\n options: ParquetOptions\n): Promise<ParquetData> {\n const cursor: CursorBuffer = {\n buffer,\n offset: 0,\n size: buffer.length\n };\n\n const data: ParquetData = {\n rlevels: [],\n dlevels: [],\n values: [],\n pageHeaders: [],\n count: 0\n };\n\n let dictionary = options.dictionary \|\| [];\n\n while (\n // @ts-ignore size can be undefined\n cursor.offset < cursor.size &&\n (!options.numValues \|\| data.dlevels.length < Number(options.numValues))\n ) {\n // Looks like we have to decode these in sequence due to cursor updates?\n const page = await decodePage(cursor, options);\n\n if (page.dictionary) {\n dictionary = page.dictionary;\n // eslint-disable-next-line no-continue\n continue;\n }\n\n if (dictionary.length) {\n // eslint-disable-next-line no-loop-func\n page.values = page.values.map((value) => dictionary[value]);\n }\n\n for (let index = 0; index < page.rlevels.length; index++) {\n data.rlevels.push(page.rlevels[index]);\n data.dlevels.push(page.dlevels[index]);\n const value = page.values[index];\n\n if (value !== undefined) {\n data.values.push(value);\n }\n }\n\n data.count += page.count;\n data.pageHeaders.push(page.pageHeader);\n }\n\n return data;\n}\n\n/\n Decode parquet page based on page type\n * @param cursor\n * @param options\n /\nexport async function decodePage(\n cursor: CursorBuffer,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n let page;\n const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n cursor.offset += length;\n\n const pageType = getThriftEnum(PageType, pageHeader.type);\n\n switch (pageType) {\n case 'DATA_PAGE':\n page = await decodeDataPage(cursor, pageHeader, options);\n break;\n case 'DATA_PAGE_V2':\n page = await decodeDataPageV2(cursor, pageHeader, options);\n break;\n case 'DICTIONARY_PAGE':\n page = {\n dictionary: await decodeDictionaryPage(cursor, pageHeader, options),\n pageHeader\n };\n break;\n default:\n throw new Error(`invalid page type: ${pageType}`);\n }\n\n return page;\n}\n\n/\n Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n * result.next - offset at the end of function\n * result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n /\nexport function decodeSchema(\n schemaElements: SchemaElement[],\n offset: number,\n len: number\n): {\n offset: number;\n next: number;\n schema: SchemaDefinition;\n} {\n const schema: SchemaDefinition = {};\n let next = offset;\n for (let i = 0; i < len; i++) {\n const schemaElement = schemaElements[next];\n\n const repetitionType =\n next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n let optional = false;\n let repeated = false;\n switch (repetitionType) {\n case 'REQUIRED':\n break;\n case 'OPTIONAL':\n optional = true;\n break;\n case 'REPEATED':\n repeated = true;\n break;\n default:\n throw new Error('parquet: unknown repetition type');\n }\n\n if (schemaElement.num_children! > 0) {\n const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n next = res.next;\n schema[schemaElement.name] = {\n // type: undefined,\n optional,\n repeated,\n fields: res.schema\n };\n } else {\n const type = getThriftEnum(Type, schemaElement.type!);\n let logicalType = type;\n\n if (schemaElement.converted_type) {\n logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n }\n\n switch (logicalType) {\n case 'DECIMAL':\n logicalType = `${logicalType}_${type}` as ParquetType;\n break;\n default:\n }\n\n schema[schemaElement.name] = {\n type: logicalType as ParquetType,\n typeLength: schemaElement.type_length,\n presision: schemaElement.precision,\n scale: schemaElement.scale,\n optional,\n repeated\n };\n next++;\n }\n }\n return {schema, offset, next};\n}\n\n/\n Decode a consecutive array of data using one of the parquet encodings\n /\nfunction decodeValues(\n type: PrimitiveType,\n encoding: ParquetCodec,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): any[] {\n if (!(encoding in PARQUET_CODECS)) {\n throw new Error(`invalid encoding: ${encoding}`);\n }\n return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/\n Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n /\nasync function decodeDataPage(\n cursor: CursorBuffer,\n header: PageHeader,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n const valueCount = header.data_page_header?.num_values;\n\n / uncompress page /\n let dataCursor = cursor;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n dataCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n cursor.offset = cursorEnd;\n }\n\n / read repetition levels /\n const rLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.repetition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n\n if (options.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.rLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n rLevels.fill(0);\n }\n\n / read definition levels /\n const dLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.definition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (options.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.dLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n dLevels.fill(0);\n }\n let valueCountNonNull = 0;\n for (const dlvl of dLevels) {\n if (dlvl === options.column.dLevelMax) {\n valueCountNonNull++;\n }\n }\n\n / read values /\n const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n const decodeOptions = {\n typeLength: options.column.typeLength,\n bitWidth: options.column.typeLength\n };\n\n const values = decodeValues(\n options.column.primitiveType!,\n valueEncoding,\n dataCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/\n Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n /\nasync function decodeDataPageV2(\n cursor: CursorBuffer,\n header: PageHeader,\n opts: any\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n\n const valueCount = header.data_page_header_v2?.num_values;\n // @ts-ignore\n const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n const valueEncoding = getThriftEnum(\n Encoding,\n header.data_page_header_v2?.encoding!\n ) as ParquetCodec;\n\n / read repetition levels /\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n if (opts.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.rLevelMax),\n disableEnvelope: true\n });\n } else {\n rLevels.fill(0);\n }\n\n / read definition levels /\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (opts.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.dLevelMax),\n disableEnvelope: true\n });\n } else {\n dLevels.fill(0);\n }\n\n / read values /\n let valuesBufCursor = cursor;\n\n if (header.data_page_header_v2?.is_compressed) {\n const valuesBuf = await decompress(\n opts.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n\n valuesBufCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const decodeOptions = {\n typeLength: opts.column.typeLength,\n bitWidth: opts.column.typeLength\n };\n\n const values = decodeValues(\n opts.column.primitiveType!,\n valueEncoding,\n valuesBufCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/\n Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param options\n */\nasync function decodeDictionaryPage(\n cursor: CursorBuffer,\n pageHeader: PageHeader,\n options: ParquetOptions\n): Promise<string[]> {\n const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n let dictCursor = {\n offset: 0,\n buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n size: cursorEnd - cursor.offset\n };\n\n cursor.offset = cursorEnd;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n pageHeader.uncompressed_page_size\n );\n\n dictCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const numValues = pageHeader?.dictionary_page_header?.num_values \|\| 0;\n\n return decodeValues(\n options.column.primitiveType!,\n options.column.encoding!,\n dictCursor,\n numValues,\n options as ParquetCodecOptions\n ).map((d) => d.toString());\n}\n"],"mappings":"AAUA,SAA2CA,cAAc,QAAO,WAAW;AAC3E,SACEC,aAAa,EACbC,QAAQ,EACRC,mBAAmB,EAEnBC,QAAQ,EAERC,IAAI,QACC,mBAAmB;AAC1B,SAAQC,UAAU,QAAO,gBAAgB;AACzC,SAAQC,kBAAkB,EAAEC,sBAAsB,QAAO,iBAAiB;AAC1E,SAAQC,gBAAgB,EAAEC,aAAa,EAAEC,WAAW,QAAO,qBAAqB;AAShF,OAAO,eAAeC,eAAeA,CACnCC,MAAc,EACdC,OAAuB,EACD;EACtB,MAAMC,MAAoB,GAAG;IAC3BF,MAAM;IACNG,MAAM,EAAE,CAAC;IACTC,IAAI,EAAEJ,MAAM,CAACK;EACf,CAAC;EAED,MAAMC,IAAiB,GAAG;IACxBC,OAAO,EAAE,EAAE;IACXC,OAAO,EAAE,EAAE;IACXC,MAAM,EAAE,EAAE;IACVC,WAAW,EAAE,EAAE;IACfC,KAAK,EAAE;EACT,CAAC;EAED,IAAIC,UAAU,GAAGX,OAAO,CAACW,UAAU,IAAI,EAAE;EAEzC,OAEEV,MAAM,CAACC,MAAM,GAAGD,MAAM,CAACE,IAAI,KAC1B,CAACH,OAAO,CAACY,SAAS,IAAIP,IAAI,CAACE,OAAO,CAACH,MAAM,GAAGS,MAAM,CAACb,OAAO,CAACY,SAAS,CAAC,CAAC,EACvE;IAEA,MAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAM,EAAED,OAAO,CAAC;IAE9C,IAAIc,IAAI,CAACH,UAAU,EAAE;MACnBA,UAAU,GAAGG,IAAI,CAACH,UAAU;MAE5B;IACF;IAEA,IAAIA,UAAU,CAACP,MAAM,EAAE;MAErBU,IAAI,CAACN,MAAM,GAAGM,IAAI,CAACN,MAAM,CAACQ,GAAG,CAAEC,KAAK,IAAKN,UAAU,CAACM,KAAK,CAAC,CAAC;IAC7D;IAEA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGJ,IAAI,CAACR,OAAO,CAACF,MAAM,EAAEc,KAAK,EAAE,EAAE;MACxDb,IAAI,CAACC,OAAO,CAACa,IAAI,CAACL,IAAI,CAACR,OAAO,CAACY,KAAK,CAAC,CAAC;MACtCb,IAAI,CAACE,OAAO,CAACY,IAAI,CAACL,IAAI,CAACP,OAAO,CAACW,KAAK,CAAC,CAAC;MACtC,MAAMD,KAAK,GAAGH,IAAI,CAACN,MAAM,CAACU,KAAK,CAAC;MAEhC,IAAID,KAAK,KAAKG,SAAS,EAAE;QACvBf,IAAI,CAACG,MAAM,CAACW,IAAI,CAACF,KAAK,CAAC;MACzB;IACF;IAEAZ,IAAI,CAACK,KAAK,IAAII,IAAI,CAACJ,KAAK;IACxBL,IAAI,CAACI,WAAW,CAACU,IAAI,CAACL,IAAI,CAACO,UAAU,CAAC;EACxC;EAEA,OAAOhB,IAAI;AACb;AAOA,OAAO,eAAeU,UAAUA,CAC9Bd,MAAoB,EACpBD,OAAuB,EACG;EAC1B,IAAIc,IAAI;EACR,MAAM;IAACO,UAAU;IAAEjB;EAAM,CAAC,GAAGT,gBAAgB,CAACM,MAAM,CAACF,MAAM,EAAEE,MAAM,CAACC,MAAM,CAAC;EAC3ED,MAAM,CAACC,MAAM,IAAIE,MAAM;EAEvB,MAAMkB,QAAQ,GAAG1B,aAAa,CAACN,QAAQ,EAAE+B,UAAU,CAACE,IAAI,CAAC;EAEzD,QAAQD,QAAQ;IACd,KAAK,WAAW;MACdR,IAAI,GAAG,MAAMU,cAAc,CAACvB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MACxD;IACF,KAAK,cAAc;MACjBc,IAAI,GAAG,MAAMW,gBAAgB,CAACxB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MAC1D;IACF,KAAK,iBAAiB;MACpBc,IAAI,GAAG;QACLH,UAAU,EAAE,MAAMe,oBAAoB,CAACzB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;QACnEqB;MACF,CAAC;MACD;IACF;MACE,MAAM,IAAIM,KAAK,uBAAAC,MAAA,CAAuBN,QAAQ,EAAG;EAAC;EAGtD,OAAOR,IAAI;AACb;AAYA,OAAO,SAASe,YAAYA,CAC1BC,cAA+B,EAC/B5B,MAAc,EACd6B,GAAW,EAKX;EACA,MAAMC,MAAwB,GAAG,CAAC,CAAC;EACnC,IAAIC,IAAI,GAAG/B,MAAM;EACjB,KAAK,IAAIgC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,GAAG,EAAEG,CAAC,EAAE,EAAE;IAC5B,MAAMC,aAAa,GAAGL,cAAc,CAACG,IAAI,CAAC;IAE1C,MAAMG,cAAc,GAClBH,IAAI,GAAG,CAAC,GAAGrC,aAAa,CAACP,mBAAmB,EAAE8C,aAAa,CAACE,eAAe,CAAE,GAAG,MAAM;IAExF,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,QAAQ,GAAG,KAAK;IACpB,QAAQH,cAAc;MACpB,KAAK,UAAU;QACb;MACF,KAAK,UAAU;QACbE,QAAQ,GAAG,IAAI;QACf;MACF,KAAK,UAAU;QACbC,QAAQ,GAAG,IAAI;QACf;MACF;QACE,MAAM,IAAIZ,KAAK,CAAC,kCAAkC,CAAC;IAAC;IAGxD,IAAIQ,aAAa,CAACK,YAAY,GAAI,CAAC,EAAE;MACnC,MAAMC,GAAG,GAAGZ,YAAY,CAACC,cAAc,EAAEG,IAAI,GAAG,CAAC,EAAEE,aAAa,CAACK,YAAY,CAAE;MAC/EP,IAAI,GAAGQ,GAAG,CAACR,IAAI;MACfD,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAE3BJ,QAAQ;QACRC,QAAQ;QACRI,MAAM,EAAEF,GAAG,CAACT;MACd,CAAC;IACH,CAAC,MAAM;MACL,MAAMT,IAAI,GAAG3B,aAAa,CAACL,IAAI,EAAE4C,aAAa,CAACZ,IAAI,CAAE;MACrD,IAAIqB,WAAW,GAAGrB,IAAI;MAEtB,IAAIY,aAAa,CAACU,cAAc,EAAE;QAChCD,WAAW,GAAGhD,aAAa,CAACT,aAAa,EAAEgD,aAAa,CAACU,cAAc,CAAC;MAC1E;MAEA,QAAQD,WAAW;QACjB,KAAK,SAAS;UACZA,WAAW,MAAAhB,MAAA,CAAMgB,WAAW,OAAAhB,MAAA,CAAIL,IAAI,CAAiB;UACrD;QACF;MAAQ;MAGVS,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAC3BnB,IAAI,EAAEqB,WAA0B;QAChCE,UAAU,EAAEX,aAAa,CAACY,WAAW;QACrCC,SAAS,EAAEb,aAAa,CAACc,SAAS;QAClCC,KAAK,EAAEf,aAAa,CAACe,KAAK;QAC1BZ,QAAQ;QACRC;MACF,CAAC;MACDN,IAAI,EAAE;IACR;EACF;EACA,OAAO;IAACD,MAAM;IAAE9B,MAAM;IAAE+B;EAAI,CAAC;AAC/B;AAKA,SAASkB,YAAYA,CACnB5B,IAAmB,EACnB6B,QAAsB,EACtBnD,MAAoB,EACpBS,KAAa,EACb2C,IAAyB,EAClB;EACP,IAAI,EAAED,QAAQ,IAAIlE,cAAc,CAAC,EAAE;IACjC,MAAM,IAAIyC,KAAK,sBAAAC,MAAA,CAAsBwB,QAAQ,EAAG;EAClD;EACA,OAAOlE,cAAc,CAACkE,QAAQ,CAAC,CAACD,YAAY,CAAC5B,IAAI,EAAEtB,MAAM,EAAES,KAAK,EAAE2C,IAAI,CAAC;AACzE;AAQA,eAAe7B,cAAcA,CAC3BvB,MAAoB,EACpBqD,MAAkB,EAClBtD,OAAuB,EACG;EAAA,IAAAuD,qBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAMC,SAAS,GAAG1D,MAAM,CAACC,MAAM,GAAGoD,MAAM,CAACM,oBAAoB;EAC7D,MAAMC,UAAU,IAAAN,qBAAA,GAAGD,MAAM,CAACQ,gBAAgB,cAAAP,qBAAA,uBAAvBA,qBAAA,CAAyBQ,UAAU;EAGtD,IAAIC,UAAU,GAAG/D,MAAM;EAEvB,IAAID,OAAO,CAACiE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAM1E,UAAU,CAChCQ,OAAO,CAACiE,WAAW,EACnBhE,MAAM,CAACF,MAAM,CAACoE,KAAK,CAAClE,MAAM,CAACC,MAAM,EAAEyD,SAAS,CAAC,EAC7CL,MAAM,CAACc,sBAAsB,CAC9B;IACDJ,UAAU,GAAG;MACXjE,MAAM,EAAEmE,SAAS;MACjBhE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE+D,SAAS,CAAC9D;IAClB,CAAC;IACDH,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAC3B;EAGA,MAAMU,cAAc,GAAGzE,aAAa,CAClCR,QAAQ,GAAAoE,sBAAA,GACRF,MAAM,CAACQ,gBAAgB,cAAAN,sBAAA,uBAAvBA,sBAAA,CAAyBc,yBAAyB,CACnC;EAEjB,IAAIC,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EAEnC,IAAI7D,OAAO,CAACyE,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGpB,YAAY,CAAC1D,kBAAkB,EAAE4E,cAAc,EAAEL,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAE9E,WAAW,CAACG,OAAO,CAACyE,MAAM,CAACC,SAAS,CAAC;MAC/CE,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,MAAMC,cAAc,GAAGlF,aAAa,CAClCR,QAAQ,GAAAqE,sBAAA,GACRH,MAAM,CAACQ,gBAAgB,cAAAL,sBAAA,uBAAvBA,sBAAA,CAAyBsB,yBAAyB,CACnC;EAEjB,IAAIC,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI7D,OAAO,CAACyE,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAG7B,YAAY,CAAC1D,kBAAkB,EAAEqF,cAAc,EAAEd,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAE9E,WAAW,CAACG,OAAO,CAACyE,MAAM,CAACQ,SAAS,CAAC;MAC/CL,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EACA,IAAIK,iBAAiB,GAAG,CAAC;EACzB,KAAK,MAAMC,IAAI,IAAIH,OAAO,EAAE;IAC1B,IAAIG,IAAI,KAAKnF,OAAO,CAACyE,MAAM,CAACQ,SAAS,EAAE;MACrCC,iBAAiB,EAAE;IACrB;EACF;EAGA,MAAME,aAAa,GAAGxF,aAAa,CAACR,QAAQ,GAAAsE,sBAAA,GAAEJ,MAAM,CAACQ,gBAAgB,cAAAJ,sBAAA,uBAAvBA,sBAAA,CAAyBN,QAAQ,CAAkB;EACjG,MAAMiC,aAAa,GAAG;IACpBvC,UAAU,EAAE9C,OAAO,CAACyE,MAAM,CAAC3B,UAAU;IACrC6B,QAAQ,EAAE3E,OAAO,CAACyE,MAAM,CAAC3B;EAC3B,CAAC;EAED,MAAMtC,MAAM,GAAG2C,YAAY,CACzBnD,OAAO,CAACyE,MAAM,CAACa,aAAa,EAC5BF,aAAa,EACbpB,UAAU,EACVkB,iBAAiB,EACjBG,aAAa,CACd;EAED,OAAO;IACL9E,OAAO,EAAEyE,OAAO;IAChB1E,OAAO,EAAEiE,OAAO;IAChB/D,MAAM;IACNE,KAAK,EAAEmD,UAAW;IAClBxC,UAAU,EAAEiC;EACd,CAAC;AACH;AASA,eAAe7B,gBAAgBA,CAC7BxB,MAAoB,EACpBqD,MAAkB,EAClBD,IAAS,EACiB;EAAA,IAAAkC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAM/B,SAAS,GAAG1D,MAAM,CAACC,MAAM,GAAGoD,MAAM,CAACM,oBAAoB;EAE7D,MAAMC,UAAU,IAAA0B,sBAAA,GAAGjC,MAAM,CAACqC,mBAAmB,cAAAJ,sBAAA,uBAA1BA,sBAAA,CAA4BxB,UAAU;EAEzD,MAAMmB,iBAAiB,GAAGrB,UAAU,KAAA2B,sBAAA,GAAGlC,MAAM,CAACqC,mBAAmB,cAAAH,sBAAA,uBAA1BA,sBAAA,CAA4BI,SAAS;EAC5E,MAAMR,aAAa,GAAGxF,aAAa,CACjCR,QAAQ,GAAAqG,sBAAA,GACRnC,MAAM,CAACqC,mBAAmB,cAAAF,sBAAA,uBAA1BA,sBAAA,CAA4BrC,QAAQ,CACrB;EAIjB,IAAImB,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EACnC,IAAIR,IAAI,CAACoB,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAC7BH,OAAO,GAAGpB,YAAY,CAAC1D,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAE4D,UAAU,EAAG;MACtFc,QAAQ,EAAE9E,WAAW,CAACwD,IAAI,CAACoB,MAAM,CAACC,SAAS,CAAC;MAC5CE,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;EAIA,IAAIG,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAIR,IAAI,CAACoB,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAC7BD,OAAO,GAAG7B,YAAY,CAAC1D,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAE4D,UAAU,EAAG;MACtFc,QAAQ,EAAE9E,WAAW,CAACwD,IAAI,CAACoB,MAAM,CAACQ,SAAS,CAAC;MAC5CL,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,IAAIgB,eAAe,GAAG5F,MAAM;EAE5B,KAAAyF,sBAAA,GAAIpC,MAAM,CAACqC,mBAAmB,cAAAD,sBAAA,eAA1BA,sBAAA,CAA4BI,aAAa,EAAE;IAC7C,MAAM5B,SAAS,GAAG,MAAM1E,UAAU,CAChC6D,IAAI,CAACY,WAAW,EAChBhE,MAAM,CAACF,MAAM,CAACoE,KAAK,CAAClE,MAAM,CAACC,MAAM,EAAEyD,SAAS,CAAC,EAC7CL,MAAM,CAACc,sBAAsB,CAC9B;IAEDyB,eAAe,GAAG;MAChB9F,MAAM,EAAEmE,SAAS;MACjBhE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE+D,SAAS,CAAC9D;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAC3B;EAEA,MAAM0B,aAAa,GAAG;IACpBvC,UAAU,EAAEO,IAAI,CAACoB,MAAM,CAAC3B,UAAU;IAClC6B,QAAQ,EAAEtB,IAAI,CAACoB,MAAM,CAAC3B;EACxB,CAAC;EAED,MAAMtC,MAAM,GAAG2C,YAAY,CACzBE,IAAI,CAACoB,MAAM,CAACa,aAAa,EACzBF,aAAa,EACbS,eAAe,EACfX,iBAAiB,EACjBG,aAAa,CACd;EAED,OAAO;IACL9E,OAAO,EAAEyE,OAAO;IAChB1E,OAAO,EAAEiE,OAAO;IAChB/D,MAAM;IACNE,KAAK,EAAEmD,UAAW;IAClBxC,UAAU,EAAEiC;EACd,CAAC;AACH;AAQA,eAAe5B,oBAAoBA,CACjCzB,MAAoB,EACpBoB,UAAsB,EACtBrB,OAAuB,EACJ;EAAA,IAAA+F,qBAAA;EACnB,MAAMpC,SAAS,GAAG1D,MAAM,CAACC,MAAM,GAAGmB,UAAU,CAACuC,oBAAoB;EAEjE,IAAIoC,UAAU,GAAG;IACf9F,MAAM,EAAE,CAAC;IACTH,MAAM,EAAEE,MAAM,CAACF,MAAM,CAACoE,KAAK,CAAClE,MAAM,CAACC,MAAM,EAAEyD,SAAS,CAAC;IACrDxD,IAAI,EAAEwD,SAAS,GAAG1D,MAAM,CAACC;EAC3B,CAAC;EAEDD,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAEzB,IAAI3D,OAAO,CAACiE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAM1E,UAAU,CAChCQ,OAAO,CAACiE,WAAW,EACnB+B,UAAU,CAACjG,MAAM,CAACoE,KAAK,CAAC6B,UAAU,CAAC9F,MAAM,EAAEyD,SAAS,CAAC,EACrDtC,UAAU,CAAC+C,sBAAsB,CAClC;IAED4B,UAAU,GAAG;MACXjG,MAAM,EAAEmE,SAAS;MACjBhE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE+D,SAAS,CAAC9D;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAC3B;EAEA,MAAM/C,SAAS,GAAG,CAAAS,UAAU,aAAVA,UAAU,wBAAA0E,qBAAA,GAAV1E,UAAU,CAAE4E,sBAAsB,cAAAF,qBAAA,uBAAlCA,qBAAA,CAAoChC,UAAU,KAAI,CAAC;EAErE,OAAOZ,YAAY,CACjBnD,OAAO,CAACyE,MAAM,CAACa,aAAa,EAC5BtF,OAAO,CAACyE,MAAM,CAACrB,QAAQ,EACvB4C,UAAU,EACVpF,SAAS,EACTZ,OAAO,CACR,CAACgB,GAAG,CAAEkF,CAAC,IAAKA,CAAC,CAACC,QAAQ,EAAE,CAAC;AAC5B"}

package/dist/esm/parquetjs/parser/parquet-reader.js CHANGED Viewed

@@ -1,103 +1,176 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-let _Symbol$asyncIterator;
-import { ParquetEnvelopeReader } from './parquet-envelope-reader';
 import { ParquetSchema } from '../schema/schema';
-import { ParquetCursor } from './parquet-cursor';
-import { PARQUET_VERSION } from '../../constants';
 import { decodeSchema } from './decoders';
-_Symbol$asyncIterator = Symbol.asyncIterator;
+import { materializeRecords } from '../schema/shred';
+import { PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED } from '../../constants';
+import { CompressionCodec, Type } from '../parquet-thrift';
+import { decodeFileMetadata, getThriftEnum, fieldIndexOf } from '../utils/read-utils';
+import { decodeDataPages, decodePage } from './decoders';
+const DEFAULT_PROPS = {
+  defaultDictionarySize: 1e6
+};
 export class ParquetReader {
-  static async openBlob(blob) {
-    const readFn = async (start, length) => {
-      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();
-      return Buffer.from(arrayBuffer);
+  constructor(file, props) {
+    _defineProperty(this, "props", void 0);
+    _defineProperty(this, "file", void 0);
+    _defineProperty(this, "metadata", null);
+    this.file = file;
+    this.props = {
+      ...DEFAULT_PROPS,
+      ...props
     };
-    const closeFn = async () => {};
-    const size = blob.size;
-    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);
-    try {
-      await envelopeReader.readHeader();
-      const metadata = await envelopeReader.readFooter();
-      return new ParquetReader(metadata, envelopeReader);
-    } catch (err) {
-      await envelopeReader.close();
-      throw err;
-    }
   }
-  static async openArrayBuffer(arrayBuffer) {
-    const readFn = async (start, length) => Buffer.from(arrayBuffer, start, length);
-    const closeFn = async () => {};
-    const size = arrayBuffer.byteLength;
-    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);
-    try {
-      await envelopeReader.readHeader();
-      const metadata = await envelopeReader.readFooter();
-      return new ParquetReader(metadata, envelopeReader);
-    } catch (err) {
-      await envelopeReader.close();
-      throw err;
-    }
+  close() {
+    this.file.close();
   }
-  static async openBuffer(buffer) {
-    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);
-    try {
-      await envelopeReader.readHeader();
-      const metadata = await envelopeReader.readFooter();
-      return new ParquetReader(metadata, envelopeReader);
-    } catch (err) {
-      await envelopeReader.close();
-      throw err;
+  async *rowIterator(props) {
+    for await (const rows of this.rowBatchIterator(props)) {
+      for (const row of rows) {
+        yield row;
+      }
     }
   }
-  constructor(metadata, envelopeReader) {
-    _defineProperty(this, "metadata", void 0);
-    _defineProperty(this, "envelopeReader", void 0);
-    _defineProperty(this, "schema", void 0);
-    if (metadata.version !== PARQUET_VERSION) {
-      throw new Error('invalid parquet version');
+  async *rowBatchIterator(props) {
+    const schema = await this.getSchema();
+    for await (const rowGroup of this.rowGroupIterator(props)) {
+      yield materializeRecords(schema, rowGroup);
     }
-    this.metadata = metadata;
-    this.envelopeReader = envelopeReader;
-    const root = this.metadata.schema[0];
-    const {
-      schema
-    } = decodeSchema(this.metadata.schema, 1, root.num_children);
-    this.schema = new ParquetSchema(schema);
   }
-  async close() {
-    await this.envelopeReader.close();
-  }
-  getCursor(columnList) {
-    if (!columnList) {
-      columnList = [];
+  async *rowGroupIterator(props) {
+    const columnList = ((props === null || props === void 0 ? void 0 : props.columnList) || []).map(x => Array.isArray(x) ? x : [x]);
+    const metadata = await this.getFileMetadata();
+    const schema = await this.getSchema();
+    const rowGroupCount = (metadata === null || metadata === void 0 ? void 0 : metadata.row_groups.length) || 0;
+    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {
+      const rowGroup = await this.readRowGroup(schema, metadata.row_groups[rowGroupIndex], columnList);
+      yield rowGroup;
     }
-    columnList = columnList.map(x => Array.isArray(x) ? x : [x]);
-    return new ParquetCursor(this.metadata, this.envelopeReader, this.schema, columnList);
   }
-  getRowCount() {
-    return Number(this.metadata.num_rows);
+  async getRowCount() {
+    const metadata = await this.getFileMetadata();
+    return Number(metadata.num_rows);
   }
-  getSchema() {
-    return this.schema;
+  async getSchema() {
+    const metadata = await this.getFileMetadata();
+    const root = metadata.schema[0];
+    const {
+      schema: schemaDefinition
+    } = decodeSchema(metadata.schema, 1, root.num_children);
+    const schema = new ParquetSchema(schemaDefinition);
+    return schema;
   }
-  getMetadata() {
+  async getSchemaMetadata() {
+    const metadata = await this.getFileMetadata();
     const md = {};
-    for (const kv of this.metadata.key_value_metadata) {
+    for (const kv of metadata.key_value_metadata) {
       md[kv.key] = kv.value;
     }
     return md;
   }
-  [_Symbol$asyncIterator]() {
-    return this.getCursor()[Symbol.asyncIterator]();
+  async getFileMetadata() {
+    if (!this.metadata) {
+      await this.readHeader();
+      this.metadata = this.readFooter();
+    }
+    return this.metadata;
+  }
+  async readHeader() {
+    const buffer = await this.file.read(0, PARQUET_MAGIC.length);
+    const magic = buffer.toString();
+    switch (magic) {
+      case PARQUET_MAGIC:
+        break;
+      case PARQUET_MAGIC_ENCRYPTED:
+        throw new Error('Encrypted parquet file not supported');
+      default:
+        throw new Error("Invalid parquet file (magic=".concat(magic, ")"));
+    }
+  }
+  async readFooter() {
+    const trailerLen = PARQUET_MAGIC.length + 4;
+    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);
+    const magic = trailerBuf.slice(4).toString();
+    if (magic !== PARQUET_MAGIC) {
+      throw new Error("Not a valid parquet file (magic=\"".concat(magic, ")"));
+    }
+    const metadataSize = trailerBuf.readUInt32LE(0);
+    const metadataOffset = this.file.size - metadataSize - trailerLen;
+    if (metadataOffset < PARQUET_MAGIC.length) {
+      throw new Error("Invalid metadata size ".concat(metadataOffset));
+    }
+    const metadataBuf = await this.file.read(metadataOffset, metadataSize);
+    const {
+      metadata
+    } = decodeFileMetadata(metadataBuf);
+    return metadata;
+  }
+  async readRowGroup(schema, rowGroup, columnList) {
+    const buffer = {
+      rowCount: Number(rowGroup.num_rows),
+      columnData: {}
+    };
+    for (const colChunk of rowGroup.columns) {
+      const colMetadata = colChunk.meta_data;
+      const colKey = colMetadata === null || colMetadata === void 0 ? void 0 : colMetadata.path_in_schema;
+      if (columnList.length > 0 && fieldIndexOf(columnList, colKey) < 0) {
+        continue;
+      }
+      buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
+    }
+    return buffer;
+  }
+  async readColumnChunk(schema, colChunk) {
+    var _colChunk$meta_data, _colChunk$meta_data2, _colChunk$meta_data3, _colChunk$meta_data4, _colChunk$meta_data5, _colChunk$meta_data7, _colChunk$meta_data8, _options$dictionary;
+    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
+      throw new Error('external references are not supported');
+    }
+    const field = schema.findField((_colChunk$meta_data = colChunk.meta_data) === null || _colChunk$meta_data === void 0 ? void 0 : _colChunk$meta_data.path_in_schema);
+    const type = getThriftEnum(Type, (_colChunk$meta_data2 = colChunk.meta_data) === null || _colChunk$meta_data2 === void 0 ? void 0 : _colChunk$meta_data2.type);
+    if (type !== field.primitiveType) {
+      throw new Error("chunk type not matching schema: ".concat(type));
+    }
+    const compression = getThriftEnum(CompressionCodec, (_colChunk$meta_data3 = colChunk.meta_data) === null || _colChunk$meta_data3 === void 0 ? void 0 : _colChunk$meta_data3.codec);
+    const pagesOffset = Number((_colChunk$meta_data4 = colChunk.meta_data) === null || _colChunk$meta_data4 === void 0 ? void 0 : _colChunk$meta_data4.data_page_offset);
+    let pagesSize = Number((_colChunk$meta_data5 = colChunk.meta_data) === null || _colChunk$meta_data5 === void 0 ? void 0 : _colChunk$meta_data5.total_compressed_size);
+    if (!colChunk.file_path) {
+      var _colChunk$meta_data6;
+      pagesSize = Math.min(this.file.size - pagesOffset, Number((_colChunk$meta_data6 = colChunk.meta_data) === null || _colChunk$meta_data6 === void 0 ? void 0 : _colChunk$meta_data6.total_compressed_size));
+    }
+    const options = {
+      type,
+      rLevelMax: field.rLevelMax,
+      dLevelMax: field.dLevelMax,
+      compression,
+      column: field,
+      numValues: (_colChunk$meta_data7 = colChunk.meta_data) === null || _colChunk$meta_data7 === void 0 ? void 0 : _colChunk$meta_data7.num_values,
+      dictionary: []
+    };
+    let dictionary;
+    const dictionaryPageOffset = colChunk === null || colChunk === void 0 ? void 0 : (_colChunk$meta_data8 = colChunk.meta_data) === null || _colChunk$meta_data8 === void 0 ? void 0 : _colChunk$meta_data8.dictionary_page_offset;
+    if (dictionaryPageOffset) {
+      const dictionaryOffset = Number(dictionaryPageOffset);
+      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
+    }
+    dictionary = (_options$dictionary = options.dictionary) !== null && _options$dictionary !== void 0 && _options$dictionary.length ? options.dictionary : dictionary;
+    const pagesBuf = await this.file.read(pagesOffset, pagesSize);
+    return await decodeDataPages(pagesBuf, {
+      ...options,
+      dictionary
+    });
+  }
+  async getDictionary(dictionaryPageOffset, options, pagesOffset) {
+    if (dictionaryPageOffset === 0) {
+      return [];
+    }
+    const dictionarySize = Math.min(this.file.size - dictionaryPageOffset, this.props.defaultDictionarySize);
+    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);
+    const cursor = {
+      buffer: pagesBuf,
+      offset: 0,
+      size: pagesBuf.length
+    };
+    const decodedPage = await decodePage(cursor, options);
+    return decodedPage.dictionary;
   }
 }
 //# sourceMappingURL=parquet-reader.js.map

package/dist/esm/parquetjs/parser/parquet-reader.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"parquet-reader.js","names":["ParquetEnvelopeReader","ParquetSchema","ParquetCursor","PARQUET_VERSION","decodeSchema","Symbol","asyncIterator","ParquetReader","openBlob","blob","readFn","start","length","arrayBuffer","slice","Buffer","from","closeFn","size","envelopeReader","readHeader","metadata","readFooter","err","close","openArrayBuffer","byteLength","openBuffer","buffer","constructor","version","Error","root","schema","num_children","getCursor","columnList","map","x","Array","isArray","getRowCount","Number","num_rows","getSchema","getMetadata","md","kv","key_value_metadata","key","value"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetCursor} from './parquet-cursor';\nimport {PARQUET_VERSION} from '../../constants';\nimport {decodeSchema} from './decoders';\n\n/*\n A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n /\nexport class ParquetReader<T> implements AsyncIterable<T> {\n /\n return a new parquet reader initialized with a read function\n /\n static async openBlob<T>(blob: Blob): Promise<ParquetReader<T>> {\n const readFn = async (start: number, length: number) => {\n const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();\n return Buffer.from(arrayBuffer);\n };\n const closeFn = async () => {};\n const size = blob.size;\n const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n try {\n await envelopeReader.readHeader();\n const metadata = await envelopeReader.readFooter();\n return new ParquetReader(metadata, envelopeReader);\n } catch (err) {\n await envelopeReader.close();\n throw err;\n }\n }\n\n /\n return a new parquet reader initialized with a read function\n /\n static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n const closeFn = async () => {};\n const size = arrayBuffer.byteLength;\n const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n try {\n await envelopeReader.readHeader();\n const metadata = await envelopeReader.readFooter();\n return new ParquetReader(metadata, envelopeReader);\n } catch (err) {\n await envelopeReader.close();\n throw err;\n }\n }\n\n static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n try {\n await envelopeReader.readHeader();\n const metadata = await envelopeReader.readFooter();\n return new ParquetReader<T>(metadata, envelopeReader);\n } catch (err) {\n await envelopeReader.close();\n throw err;\n }\n }\n\n public metadata: FileMetaData;\n public envelopeReader: ParquetEnvelopeReader;\n public schema: ParquetSchema;\n\n /\n Create a new parquet reader from the file metadata and an envelope reader.\n * It is not recommended to call this constructor directly except for advanced\n * and internal use cases. Consider using one of the open{File,Buffer} methods\n * instead\n /\n constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n if (metadata.version !== PARQUET_VERSION) {\n throw new Error('invalid parquet version');\n }\n\n this.metadata = metadata;\n this.envelopeReader = envelopeReader;\n const root = this.metadata.schema[0];\n const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n this.schema = new ParquetSchema(schema);\n }\n\n /\n Close this parquet reader. You MUST call this method once you're finished\n * reading rows\n /\n async close(): Promise<void> {\n await this.envelopeReader.close();\n // this.envelopeReader = null;\n // this.metadata = null;\n }\n\n /\n Return a cursor to the file. You may open more than one cursor and use\n * them concurrently. All cursors become invalid once close() is called on\n * the reader object.\n \n The required_columns parameter controls which columns are actually read\n * from disk. An empty array or no value implies all columns. A list of column\n * names means that only those columns should be loaded from disk.\n /\n getCursor(): ParquetCursor<T>;\n // @ts-ignore\n getCursor<K extends keyof T>(columnList: (K \| K[])[]): ParquetCursor<Pick<T, K>>;\n getCursor(columnList: (string \| string[])[]): ParquetCursor<Partial<T>>;\n getCursor(columnList?: (string \| string[])[]): ParquetCursor<Partial<T>> {\n if (!columnList) {\n // tslint:disable-next-line:no-parameter-reassignment\n columnList = [];\n }\n\n // tslint:disable-next-line:no-parameter-reassignment\n columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n return new ParquetCursor<T>(\n this.metadata,\n this.envelopeReader,\n this.schema,\n columnList as string[][]\n );\n }\n\n /\n Return the number of rows in this file. Note that the number of rows is\n * not neccessarily equal to the number of rows in each column.\n /\n getRowCount(): number {\n return Number(this.metadata.num_rows);\n }\n\n /\n Returns the ParquetSchema for this file\n /\n getSchema(): ParquetSchema {\n return this.schema;\n }\n\n /\n Returns the user (key/value) metadata for this file\n /\n getMetadata(): Record<string, string> {\n const md: Record<string, string> = {};\n for (const kv of this.metadata.key_value_metadata!) {\n md[kv.key] = kv.value!;\n }\n return md;\n }\n\n /\n Implement AsyncIterable\n */\n // tslint:disable-next-line:function-name\n [Symbol.asyncIterator](): AsyncIterator<T> {\n return this.getCursor()[Symbol.asyncIterator]();\n }\n}\n"],"mappings":";;AACA,SAAQA,qBAAqB,QAAO,2BAA2B;AAE/D,SAAQC,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,eAAe,QAAO,iBAAiB;AAC/C,SAAQC,YAAY,QAAO,YAAY;;AAAC,wBAyJrCC,MAAM,CAACC,aAAa;AAhJvB,OAAO,MAAMC,aAAa,CAAgC;EAIxD,aAAaC,QAAQ,CAAIC,IAAU,EAA6B;IAC9D,MAAMC,MAAM,GAAG,OAAOC,KAAa,EAAEC,MAAc,KAAK;MACtD,MAAMC,WAAW,GAAG,MAAMJ,IAAI,CAACK,KAAK,CAACH,KAAK,EAAEA,KAAK,GAAGC,MAAM,CAAC,CAACC,WAAW,EAAE;MACzE,OAAOE,MAAM,CAACC,IAAI,CAACH,WAAW,CAAC;IACjC,CAAC;IACD,MAAMI,OAAO,GAAG,YAAY,CAAC,CAAC;IAC9B,MAAMC,IAAI,GAAGT,IAAI,CAACS,IAAI;IACtB,MAAMC,cAAc,GAAG,IAAInB,qBAAqB,CAACU,MAAM,EAAEO,OAAO,EAAEC,IAAI,CAAC;IACvE,IAAI;MACF,MAAMC,cAAc,CAACC,UAAU,EAAE;MACjC,MAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAU,EAAE;MAClD,OAAO,IAAIf,aAAa,CAACc,QAAQ,EAAEF,cAAc,CAAC;IACpD,CAAC,CAAC,OAAOI,GAAG,EAAE;MACZ,MAAMJ,cAAc,CAACK,KAAK,EAAE;MAC5B,MAAMD,GAAG;IACX;EACF;;EAKA,aAAaE,eAAe,CAAIZ,WAAwB,EAA6B;IACnF,MAAMH,MAAM,GAAG,OAAOC,KAAa,EAAEC,MAAc,KAAKG,MAAM,CAACC,IAAI,CAACH,WAAW,EAAEF,KAAK,EAAEC,MAAM,CAAC;IAC/F,MAAMK,OAAO,GAAG,YAAY,CAAC,CAAC;IAC9B,MAAMC,IAAI,GAAGL,WAAW,CAACa,UAAU;IACnC,MAAMP,cAAc,GAAG,IAAInB,qBAAqB,CAACU,MAAM,EAAEO,OAAO,EAAEC,IAAI,CAAC;IACvE,IAAI;MACF,MAAMC,cAAc,CAACC,UAAU,EAAE;MACjC,MAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAU,EAAE;MAClD,OAAO,IAAIf,aAAa,CAACc,QAAQ,EAAEF,cAAc,CAAC;IACpD,CAAC,CAAC,OAAOI,GAAG,EAAE;MACZ,MAAMJ,cAAc,CAACK,KAAK,EAAE;MAC5B,MAAMD,GAAG;IACX;EACF;EAEA,aAAaI,UAAU,CAAIC,MAAc,EAA6B;IACpE,MAAMT,cAAc,GAAG,MAAMnB,qBAAqB,CAAC2B,UAAU,CAACC,MAAM,CAAC;IACrE,IAAI;MACF,MAAMT,cAAc,CAACC,UAAU,EAAE;MACjC,MAAMC,QAAQ,GAAG,MAAMF,cAAc,CAACG,UAAU,EAAE;MAClD,OAAO,IAAIf,aAAa,CAAIc,QAAQ,EAAEF,cAAc,CAAC;IACvD,CAAC,CAAC,OAAOI,GAAG,EAAE;MACZ,MAAMJ,cAAc,CAACK,KAAK,EAAE;MAC5B,MAAMD,GAAG;IACX;EACF;EAYAM,WAAW,CAACR,QAAsB,EAAEF,cAAqC,EAAE;IAAA;IAAA;IAAA;IACzE,IAAIE,QAAQ,CAACS,OAAO,KAAK3B,eAAe,EAAE;MACxC,MAAM,IAAI4B,KAAK,CAAC,yBAAyB,CAAC;IAC5C;IAEA,IAAI,CAACV,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACF,cAAc,GAAGA,cAAc;IACpC,MAAMa,IAAI,GAAG,IAAI,CAACX,QAAQ,CAACY,MAAM,CAAC,CAAC,CAAC;IACpC,MAAM;MAACA;IAAM,CAAC,GAAG7B,YAAY,CAAC,IAAI,CAACiB,QAAQ,CAACY,MAAM,EAAE,CAAC,EAAED,IAAI,CAACE,YAAY,CAAE;IAC1E,IAAI,CAACD,MAAM,GAAG,IAAIhC,aAAa,CAACgC,MAAM,CAAC;EACzC;;EAMA,MAAMT,KAAK,GAAkB;IAC3B,MAAM,IAAI,CAACL,cAAc,CAACK,KAAK,EAAE;EAGnC;;EAeAW,SAAS,CAACC,UAAkC,EAA6B;IACvE,IAAI,CAACA,UAAU,EAAE;MAEfA,UAAU,GAAG,EAAE;IACjB;;IAGAA,UAAU,GAAGA,UAAU,CAACC,GAAG,CAAEC,CAAC,IAAMC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAE,CAAC;IAEhE,OAAO,IAAIpC,aAAa,CACtB,IAAI,CAACmB,QAAQ,EACb,IAAI,CAACF,cAAc,EACnB,IAAI,CAACc,MAAM,EACXG,UAAU,CACX;EACH;;EAMAK,WAAW,GAAW;IACpB,OAAOC,MAAM,CAAC,IAAI,CAACrB,QAAQ,CAACsB,QAAQ,CAAC;EACvC;;EAKAC,SAAS,GAAkB;IACzB,OAAO,IAAI,CAACX,MAAM;EACpB;;EAKAY,WAAW,GAA2B;IACpC,MAAMC,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAI,IAAI,CAAC1B,QAAQ,CAAC2B,kBAAkB,EAAG;MAClDF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;;EAMA,0BAA2C;IACzC,OAAO,IAAI,CAACX,SAAS,EAAE,CAAC9B,MAAM,CAACC,aAAa,CAAC,EAAE;EACjD;AACF"}
1	+ {"version":3,"file":"parquet-reader.js","names":["ParquetSchema","decodeSchema","materializeRecords","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","DEFAULT_PROPS","defaultDictionarySize","ParquetReader","constructor","file","props","_defineProperty","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","columnList","map","x","Array","isArray","metadata","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","num_children","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","buffer","read","magic","toString","Error","concat","trailerLen","trailerBuf","size","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_options$dictionary","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","_colChunk$meta_data6","Math","min","options","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","decodedPage"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRecords} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n ParquetBuffer,\n ParquetCompression,\n ParquetData,\n PrimitiveType,\n ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader /\nexport type ParquetIterationProps = {\n /* Filter allowing some columns to be dropped /\n columnList?: string[] \| string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n defaultDictionarySize: 1e6\n};\n\n/\n The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n /\nexport class ParquetReader {\n props: Required<ParquetReaderProps>;\n file: ReadableFile;\n metadata: Promise<FileMetaData> \| null = null;\n\n constructor(file: ReadableFile, props?: ParquetReaderProps) {\n this.file = file;\n this.props = {...DEFAULT_PROPS, ...props};\n }\n\n close(): void {\n // eslint-disable-next-line @typescript-eslint/no-floating-promises\n this.file.close();\n }\n\n // HIGH LEVEL METHODS\n\n /* Yield one row at a time /\n async rowIterator(props?: ParquetIterationProps) {\n for await (const rows of this.rowBatchIterator(props)) {\n // yield rows\n for (const row of rows) {\n yield row;\n }\n }\n }\n\n /* Yield one batch of rows at a time /\n async rowBatchIterator(props?: ParquetIterationProps) {\n const schema = await this.getSchema();\n for await (const rowGroup of this.rowGroupIterator(props)) {\n yield materializeRecords(schema, rowGroup);\n }\n }\n\n /** Iterate over the raw row groups /\n async rowGroupIterator(props?: ParquetIterationProps) {\n // Ensure strings are nested in arrays\n const columnList: string[][] = (props?.columnList \|\| []).map((x) =>\n Array.isArray(x) ? x : [x]\n );\n\n const metadata = await this.getFileMetadata();\n const schema = await this.getSchema();\n\n const rowGroupCount = metadata?.row_groups.length \|\| 0;\n\n for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n const rowGroup = await this.readRowGroup(\n schema,\n metadata.row_groups[rowGroupIndex],\n columnList\n );\n yield rowGroup;\n }\n }\n\n async getRowCount(): Promise<number> {\n const metadata = await this.getFileMetadata();\n return Number(metadata.num_rows);\n }\n\n async getSchema(): Promise<ParquetSchema> {\n const metadata = await this.getFileMetadata();\n const root = metadata.schema[0];\n const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n const schema = new ParquetSchema(schemaDefinition);\n return schema;\n }\n\n /*\n Returns the user (key/value) metadata for this file\n * In parquet this is not stored on the schema like it is in arrow\n /\n async getSchemaMetadata(): Promise<Record<string, string>> {\n const metadata = await this.getFileMetadata();\n const md: Record<string, string> = {};\n for (const kv of metadata.key_value_metadata!) {\n md[kv.key] = kv.value!;\n }\n return md;\n }\n\n async getFileMetadata(): Promise<FileMetaData> {\n if (!this.metadata) {\n await this.readHeader();\n this.metadata = this.readFooter();\n }\n return this.metadata;\n }\n\n // LOW LEVEL METHODS\n\n /* Metadata is stored in the footer /\n async readHeader(): Promise<void> {\n const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n const magic = buffer.toString();\n switch (magic) {\n case PARQUET_MAGIC:\n break;\n case PARQUET_MAGIC_ENCRYPTED:\n throw new Error('Encrypted parquet file not supported');\n default:\n throw new Error(`Invalid parquet file (magic=${magic})`);\n }\n }\n\n /* Metadata is stored in the footer /\n async readFooter(): Promise<FileMetaData> {\n const trailerLen = PARQUET_MAGIC.length + 4;\n const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n const magic = trailerBuf.slice(4).toString();\n if (magic !== PARQUET_MAGIC) {\n throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n }\n\n const metadataSize = trailerBuf.readUInt32LE(0);\n const metadataOffset = this.file.size - metadataSize - trailerLen;\n if (metadataOffset < PARQUET_MAGIC.length) {\n throw new Error(`Invalid metadata size ${metadataOffset}`);\n }\n\n const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n // let metadata = new parquet_thrift.FileMetaData();\n // parquet_util.decodeThrift(metadata, metadataBuf);\n const {metadata} = decodeFileMetadata(metadataBuf);\n return metadata;\n }\n\n /* Data is stored in row groups (similar to Apache Arrow record batches) /\n async readRowGroup(\n schema: ParquetSchema,\n rowGroup: RowGroup,\n columnList: string[][]\n ): Promise<ParquetBuffer> {\n const buffer: ParquetBuffer = {\n rowCount: Number(rowGroup.num_rows),\n columnData: {}\n };\n for (const colChunk of rowGroup.columns) {\n const colMetadata = colChunk.meta_data;\n const colKey = colMetadata?.path_in_schema;\n if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n continue; // eslint-disable-line no-continue\n }\n buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n }\n return buffer;\n }\n\n /\n Each row group contains column chunks for all the columns.\n /\n async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n throw new Error('external references are not supported');\n }\n\n const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n if (type !== field.primitiveType) {\n throw new Error(`chunk type not matching schema: ${type}`);\n }\n\n const compression: ParquetCompression = getThriftEnum(\n CompressionCodec,\n colChunk.meta_data?.codec!\n ) as any;\n\n const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n if (!colChunk.file_path) {\n pagesSize = Math.min(\n this.file.size - pagesOffset,\n Number(colChunk.meta_data?.total_compressed_size)\n );\n }\n\n const options: ParquetOptions = {\n type,\n rLevelMax: field.rLevelMax,\n dLevelMax: field.dLevelMax,\n compression,\n column: field,\n numValues: colChunk.meta_data?.num_values,\n dictionary: []\n };\n\n let dictionary;\n\n const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n if (dictionaryPageOffset) {\n const dictionaryOffset = Number(dictionaryPageOffset);\n // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n }\n\n dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n return await decodeDataPages(pagesBuf, {...options, dictionary});\n }\n\n /\n Getting dictionary for allows to flatten values by indices.\n * @param dictionaryPageOffset\n * @param options\n * @param pagesOffset\n * @returns\n */\n async getDictionary(\n dictionaryPageOffset: number,\n options: ParquetOptions,\n pagesOffset: number\n ): Promise<string[]> {\n if (dictionaryPageOffset === 0) {\n // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n // In this case we are working with parquet-mr files format. Problem is described below:\n // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n // We need to get dictionary page from column chunk if it exists.\n // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n return [];\n }\n\n const dictionarySize = Math.min(\n this.file.size - dictionaryPageOffset,\n this.props.defaultDictionarySize\n );\n const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n const decodedPage = await decodePage(cursor, options);\n\n return decodedPage.dictionary!;\n }\n}\n"],"mappings":";AAGA,SAAQA,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,YAAY,QAAO,YAAY;AACvC,SAAQC,kBAAkB,QAAO,iBAAiB;AAElD,SAAQC,aAAa,EAAEC,uBAAuB,QAAO,iBAAiB;AACtE,SAAqBC,gBAAgB,EAA0BC,IAAI,QAAO,mBAAmB;AAQ7F,SAAQC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY,QAAO,qBAAqB;AACnF,SAAQC,eAAe,EAAEC,UAAU,QAAO,YAAY;AAYtD,MAAMC,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;AAQD,OAAO,MAAMC,aAAa,CAAC;EAKzBC,WAAWA,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAAC,eAAA;IAAAA,eAAA;IAAAA,eAAA,mBAFnB,IAAI;IAG3C,IAAI,CAACF,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGL,aAAa;MAAE,GAAGK;IAAK,CAAC;EAC3C;EAEAE,KAAKA,CAAA,EAAS;IAEZ,IAAI,CAACH,IAAI,CAACG,KAAK,EAAE;EACnB;EAKA,OAAOC,WAAWA,CAACH,KAA6B,EAAE;IAChD,WAAW,MAAMI,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACL,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMM,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;EAGA,OAAOD,gBAAgBA,CAACL,KAA6B,EAAE;IACrD,MAAMO,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,EAAE;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACV,KAAK,CAAC,EAAE;MACzD,MAAMf,kBAAkB,CAACsB,MAAM,EAAEE,QAAQ,CAAC;IAC5C;EACF;EAGA,OAAOC,gBAAgBA,CAACV,KAA6B,EAAE;IAErD,MAAMW,UAAsB,GAAG,CAAC,CAAAX,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEW,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC,CAC3B;IAED,MAAMG,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMV,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,EAAE;IAErC,MAAMU,aAAa,GAAG,CAAAF,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEG,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMZ,QAAQ,GAAG,MAAM,IAAI,CAACa,YAAY,CACtCf,MAAM,EACNS,QAAQ,CAACG,UAAU,CAACE,aAAa,CAAC,EAClCV,UAAU,CACX;MACD,MAAMF,QAAQ;IAChB;EACF;EAEA,MAAMc,WAAWA,CAAA,EAAoB;IACnC,MAAMP,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,OAAOO,MAAM,CAACR,QAAQ,CAACS,QAAQ,CAAC;EAClC;EAEA,MAAMjB,SAASA,CAAA,EAA2B;IACxC,MAAMQ,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMS,IAAI,GAAGV,QAAQ,CAACT,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEoB;IAAgB,CAAC,GAAG3C,YAAY,CAACgC,QAAQ,CAACT,MAAM,EAAE,CAAC,EAAEmB,IAAI,CAACE,YAAY,CAAE;IACvF,MAAMrB,MAAM,GAAG,IAAIxB,aAAa,CAAC4C,gBAAgB,CAAC;IAClD,OAAOpB,MAAM;EACf;EAMA,MAAMsB,iBAAiBA,CAAA,EAAoC;IACzD,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMa,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAIf,QAAQ,CAACgB,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMb,eAAeA,CAAA,EAA0B;IAC7C,IAAI,CAAC,IAAI,CAACD,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACmB,UAAU,EAAE;MACvB,IAAI,CAACnB,QAAQ,GAAG,IAAI,CAACoB,UAAU,EAAE;IACnC;IACA,OAAO,IAAI,CAACpB,QAAQ;EACtB;EAKA,MAAMmB,UAAUA,CAAA,EAAkB;IAChC,MAAME,MAAM,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAAC,CAAC,EAAEpD,aAAa,CAACkC,MAAM,CAAC;IAC5D,MAAMmB,KAAK,GAAGF,MAAM,CAACG,QAAQ,EAAE;IAC/B,QAAQD,KAAK;MACX,KAAKrD,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAIsD,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,gCAAAC,MAAA,CAAgCH,KAAK,OAAI;IAAC;EAE/D;EAGA,MAAMH,UAAUA,CAAA,EAA0B;IACxC,MAAMO,UAAU,GAAGzD,aAAa,CAACkC,MAAM,GAAG,CAAC;IAC3C,MAAMwB,UAAU,GAAG,MAAM,IAAI,CAAC7C,IAAI,CAACuC,IAAI,CAAC,IAAI,CAACvC,IAAI,CAAC8C,IAAI,GAAGF,UAAU,EAAEA,UAAU,CAAC;IAEhF,MAAMJ,KAAK,GAAGK,UAAU,CAACE,KAAK,CAAC,CAAC,CAAC,CAACN,QAAQ,EAAE;IAC5C,IAAID,KAAK,KAAKrD,aAAa,EAAE;MAC3B,MAAM,IAAIuD,KAAK,sCAAAC,MAAA,CAAqCH,KAAK,OAAI;IAC/D;IAEA,MAAMQ,YAAY,GAAGH,UAAU,CAACI,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAAClD,IAAI,CAAC8C,IAAI,GAAGE,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAG/D,aAAa,CAACkC,MAAM,EAAE;MACzC,MAAM,IAAIqB,KAAK,0BAAAC,MAAA,CAA0BO,cAAc,EAAG;IAC5D;IAEA,MAAMC,WAAW,GAAG,MAAM,IAAI,CAACnD,IAAI,CAACuC,IAAI,CAACW,cAAc,EAAEF,YAAY,CAAC;IAGtE,MAAM;MAAC/B;IAAQ,CAAC,GAAG1B,kBAAkB,CAAC4D,WAAW,CAAC;IAClD,OAAOlC,QAAQ;EACjB;EAGA,MAAMM,YAAYA,CAChBf,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB,EACE;IACxB,MAAM0B,MAAqB,GAAG;MAC5Bc,QAAQ,EAAE3B,MAAM,CAACf,QAAQ,CAACgB,QAAQ,CAAC;MACnC2B,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAI5C,QAAQ,CAAC6C,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAI/C,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI5B,YAAY,CAACmB,UAAU,EAAE8C,MAAM,CAAE,GAAG,CAAC,EAAE;QAClE;MACF;MACApB,MAAM,CAACe,UAAU,CAACK,MAAM,CAAEE,IAAI,EAAE,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACrD,MAAM,EAAE8C,QAAQ,CAAC;IAClF;IACA,OAAOhB,MAAM;EACf;EAKA,MAAMuB,eAAeA,CAACrD,MAAqB,EAAE8C,QAAqB,EAAwB;IAAA,IAAAQ,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;IACxF,IAAIf,QAAQ,CAACgB,SAAS,KAAKC,SAAS,IAAIjB,QAAQ,CAACgB,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAI5B,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAM8B,KAAK,GAAGhE,MAAM,CAACiE,SAAS,EAAAX,mBAAA,GAACR,QAAQ,CAACG,SAAS,cAAAK,mBAAA,uBAAlBA,mBAAA,CAAoBH,cAAc,CAAE;IACnE,MAAMe,IAAmB,GAAGlF,aAAa,CAACF,IAAI,GAAAyE,oBAAA,GAAET,QAAQ,CAACG,SAAS,cAAAM,oBAAA,uBAAlBA,oBAAA,CAAoBW,IAAI,CAAS;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAIjC,KAAK,oCAAAC,MAAA,CAAoC+B,IAAI,EAAG;IAC5D;IAEA,MAAME,WAA+B,GAAGpF,aAAa,CACnDH,gBAAgB,GAAA2E,oBAAA,GAChBV,QAAQ,CAACG,SAAS,cAAAO,oBAAA,uBAAlBA,oBAAA,CAAoBa,KAAK,CACnB;IAER,MAAMC,WAAW,GAAGrD,MAAM,EAAAwC,oBAAA,GAACX,QAAQ,CAACG,SAAS,cAAAQ,oBAAA,uBAAlBA,oBAAA,CAAoBc,gBAAgB,CAAE;IACjE,IAAIC,SAAS,GAAGvD,MAAM,EAAAyC,oBAAA,GAACZ,QAAQ,CAACG,SAAS,cAAAS,oBAAA,uBAAlBA,oBAAA,CAAoBe,qBAAqB,CAAE;IAElE,IAAI,CAAC3B,QAAQ,CAACgB,SAAS,EAAE;MAAA,IAAAY,oBAAA;MACvBF,SAAS,GAAGG,IAAI,CAACC,GAAG,CAClB,IAAI,CAACpF,IAAI,CAAC8C,IAAI,GAAGgC,WAAW,EAC5BrD,MAAM,EAAAyD,oBAAA,GAAC5B,QAAQ,CAACG,SAAS,cAAAyB,oBAAA,uBAAlBA,oBAAA,CAAoBD,qBAAqB,CAAC,CAClD;IACH;IAEA,MAAMI,OAAuB,GAAG;MAC9BX,IAAI;MACJY,SAAS,EAAEd,KAAK,CAACc,SAAS;MAC1BC,SAAS,EAAEf,KAAK,CAACe,SAAS;MAC1BX,WAAW;MACXY,MAAM,EAAEhB,KAAK;MACbiB,SAAS,GAAAtB,oBAAA,GAAEb,QAAQ,CAACG,SAAS,cAAAU,oBAAA,uBAAlBA,oBAAA,CAAoBuB,UAAU;MACzCC,UAAU,EAAE;IACd,CAAC;IAED,IAAIA,UAAU;IAEd,MAAMC,oBAAoB,GAAGtC,QAAQ,aAARA,QAAQ,wBAAAc,oBAAA,GAARd,QAAQ,CAAEG,SAAS,cAAAW,oBAAA,uBAAnBA,oBAAA,CAAqByB,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAGrE,MAAM,CAACmE,oBAAoB,CAAC;MAErDD,UAAU,GAAG,MAAM,IAAI,CAACI,aAAa,CAACD,gBAAgB,EAAET,OAAO,EAAEP,WAAW,CAAC;IAC/E;IAEAa,UAAU,GAAG,CAAAtB,mBAAA,GAAAgB,OAAO,CAACM,UAAU,cAAAtB,mBAAA,eAAlBA,mBAAA,CAAoBhD,MAAM,GAAGgE,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMK,QAAQ,GAAG,MAAM,IAAI,CAAChG,IAAI,CAACuC,IAAI,CAACuC,WAAW,EAAEE,SAAS,CAAC;IAC7D,OAAO,MAAMtF,eAAe,CAACsG,QAAQ,EAAE;MAAC,GAAGX,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;EASA,MAAMI,aAAaA,CACjBH,oBAA4B,EAC5BP,OAAuB,EACvBP,WAAmB,EACA;IACnB,IAAIc,oBAAoB,KAAK,CAAC,EAAE;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGd,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACpF,IAAI,CAAC8C,IAAI,GAAG8C,oBAAoB,EACrC,IAAI,CAAC3F,KAAK,CAACJ,qBAAqB,CACjC;IACD,MAAMmG,QAAQ,GAAG,MAAM,IAAI,CAAChG,IAAI,CAACuC,IAAI,CAACqD,oBAAoB,EAAEK,cAAc,CAAC;IAE3E,MAAMC,MAAM,GAAG;MAAC5D,MAAM,EAAE0D,QAAQ;MAAEG,MAAM,EAAE,CAAC;MAAErD,IAAI,EAAEkD,QAAQ,CAAC3E;IAAM,CAAC;IACnE,MAAM+E,WAAW,GAAG,MAAMzG,UAAU,CAACuG,MAAM,EAAEb,OAAO,CAAC;IAErD,OAAOe,WAAW,CAACT,UAAU;EAC/B;AACF"}

package/dist/esm/parquetjs/schema/declare.js CHANGED Viewed

@@ -1,5 +1,4 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 export class ParquetBuffer {
   constructor() {
     let rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;

package/dist/esm/parquetjs/schema/declare.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"declare.js","names":["ParquetBuffer","constructor","rowCount","columnData"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' \| 'RLE' \| 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n \| 'UNCOMPRESSED'\n \| 'GZIP'\n \| 'SNAPPY'\n \| 'LZO'\n \| 'BROTLI'\n \| 'LZ4'\n \| 'LZ4_RAW'\n \| 'ZSTD';\nexport type RepetitionType = 'REQUIRED' \| 'OPTIONAL' \| 'REPEATED';\nexport type ParquetType = PrimitiveType \| OriginalType;\n\n/*\n Physical type\n /\nexport type PrimitiveType =\n // Base Types\n \| 'BOOLEAN' // 0\n \| 'INT32' // 1\n \| 'INT64' // 2\n \| 'INT96' // 3\n \| 'FLOAT' // 4\n \| 'DOUBLE' // 5\n \| 'BYTE_ARRAY' // 6,\n \| 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/\n Logical type\n */\nexport type OriginalType =\n // Converted Types\n \| 'UTF8' // 0\n // \| 'MAP' // 1\n // \| 'MAP_KEY_VALUE' // 2\n // \| 'LIST' // 3\n // \| 'ENUM' // 4\n // \| 'DECIMAL' // 5\n \| 'DECIMAL_INT32' // 5\n \| 'DECIMAL_INT64' // 5\n \| 'DECIMAL_BYTE_ARRAY' // 5\n \| 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n \| 'DATE' // 6\n \| 'TIME_MILLIS' // 7\n \| 'TIME_MICROS' // 8\n \| 'TIMESTAMP_MILLIS' // 9\n \| 'TIMESTAMP_MICROS' // 10\n \| 'UINT_8' // 11\n \| 'UINT_16' // 12\n \| 'UINT_32' // 13\n \| 'UINT_64' // 14\n \| 'INT_8' // 15\n \| 'INT_16' // 16\n \| 'INT_32' // 17\n \| 'INT_64' // 18\n \| 'JSON' // 19\n \| 'BSON' // 20\n \| 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n type?: ParquetType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n optional?: boolean;\n repeated?: boolean;\n fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n name: string;\n path: string[];\n key: string;\n primitiveType?: PrimitiveType;\n originalType?: OriginalType;\n repetitionType: RepetitionType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n rLevelMax: number;\n dLevelMax: number;\n isNested?: boolean;\n fieldCount?: number;\n fields?: Record<string, ParquetField>;\n}\n\nexport interface ParquetOptions {\n type: ParquetType;\n rLevelMax: number;\n dLevelMax: number;\n compression: ParquetCompression;\n column: ParquetField;\n numValues?: Int64;\n dictionary?: ParquetDictionary;\n}\n\nexport interface ~~ParquetData~~ {\n dlevels: number[];\n rlevels: number[];\n ~~values:~~ ~~any[];\n count:~~ ~~number;\n pageHeaders:~~ ~~PageHeader[];\n}\n\nexport~~ ~~interface ParquetPageData {\~~n ~~dlevels~~: ~~number~~[]~~;\n rlevels:~~ ~~number[];\n values:~~ any~~[];\~~n count: number;\n dictionary?: ParquetDictionary;\n pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n [key: string]: any;\n}\n\nexport class ParquetBuffer {\n rowCount: number;\n columnData: Record<string, ParquetData>;\n constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n this.rowCount = rowCount;\n this.columnData = columnData;\n }\n}\n"],"mappings":"~~;;AAmIA~~,OAAO,MAAMA,aAAa,CAAC;~~EAGzBC~~,~~WAAW~~,~~GAAqE~~;IAAA,IAApEC,QAAgB,~~uEAAG~~,CAAC;IAAA,~~IAAEC~~,UAAuC,~~uEAAG~~,CAAC,CAAC;~~IAAA~~;~~IAAA~~;IAC5E,IAAI,~~CAACD~~,QAAQ,GAAGA,QAAQ;IACxB,IAAI,~~CAACC~~,UAAU,GAAGA,UAAU;EAC9B;AACF"}
1	+ {"version":3,"file":"declare.js","names":["ParquetBuffer","constructor","rowCount","arguments","length","undefined","columnData","_defineProperty"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' \| 'RLE' \| 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n \| 'UNCOMPRESSED'\n \| 'GZIP'\n \| 'SNAPPY'\n \| 'LZO'\n \| 'BROTLI'\n \| 'LZ4'\n \| 'LZ4_RAW'\n \| 'ZSTD';\nexport type RepetitionType = 'REQUIRED' \| 'OPTIONAL' \| 'REPEATED';\nexport type ParquetType = PrimitiveType \| OriginalType;\n\n/*\n Physical type\n /\nexport type PrimitiveType =\n // Base Types\n \| 'BOOLEAN' // 0\n \| 'INT32' // 1\n \| 'INT64' // 2\n \| 'INT96' // 3\n \| 'FLOAT' // 4\n \| 'DOUBLE' // 5\n \| 'BYTE_ARRAY' // 6,\n \| 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/\n Logical type\n /\nexport type OriginalType =\n // Converted Types\n \| 'UTF8' // 0\n // \| 'MAP' // 1\n // \| 'MAP_KEY_VALUE' // 2\n // \| 'LIST' // 3\n // \| 'ENUM' // 4\n // \| 'DECIMAL' // 5\n \| 'DECIMAL_INT32' // 5\n \| 'DECIMAL_INT64' // 5\n \| 'DECIMAL_BYTE_ARRAY' // 5\n \| 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n \| 'DATE' // 6\n \| 'TIME_MILLIS' // 7\n \| 'TIME_MICROS' // 8\n \| 'TIMESTAMP_MILLIS' // 9\n \| 'TIMESTAMP_MICROS' // 10\n \| 'UINT_8' // 11\n \| 'UINT_16' // 12\n \| 'UINT_32' // 13\n \| 'UINT_64' // 14\n \| 'INT_8' // 15\n \| 'INT_16' // 16\n \| 'INT_32' // 17\n \| 'INT_64' // 18\n \| 'JSON' // 19\n \| 'BSON' // 20\n \| 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n type?: ParquetType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n optional?: boolean;\n repeated?: boolean;\n fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n name: string;\n path: string[];\n key: string;\n primitiveType?: PrimitiveType;\n originalType?: OriginalType;\n repetitionType: RepetitionType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n rLevelMax: number;\n dLevelMax: number;\n isNested?: boolean;\n fieldCount?: number;\n fields?: Record<string, ParquetField>;\n}\n\n/* @todo better name, this is an internal type? /\nexport interface ParquetOptions {\n type: ParquetType;\n rLevelMax: number;\n dLevelMax: number;\n compression: ParquetCompression;\n column: ParquetField;\n numValues?: Int64;\n dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n dlevels: number[];\n rlevels: number[];\n /* Actual column chunks /\n values: any[]; // ArrayLike<any>;\n count: number;\n dictionary?: ParquetDictionary;\n /* The \"raw\" page header from the file /\n pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n [key: string]: any;\n}\n\n/* @\n * Holds data for one row group (column chunks) /\nexport class ParquetBuffer {\n /* Number of rows in this page /\n rowCount: number;\n\n columnData: Record<string, ParquetData>;\n constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n this.rowCount = rowCount;\n this.columnData = columnData;\n }\n}\n\n/* Holds the data for one column chunk */\nexport interface ParquetData {\n dlevels: number[];\n rlevels: number[];\n values: any[];\n count: number;\n pageHeaders: PageHeader[];\n}\n"],"mappings":";AAgIA,OAAO,MAAMA,aAAa,CAAC;EAKzBC,WAAWA,CAAA,EAAqE;IAAA,IAApEC,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;IAAA,IAAEG,UAAuC,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;IAAAI,eAAA;IAAAA,eAAA;IAC5E,IAAI,CAACL,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;EAC9B;AACF"}