npm - @loaders.gl/parquet - Versions diffs - 3.4.0-alpha.2 → 3.4.0-alpha.3 - Mend

@loaders.gl/parquet 3.4.0-alpha.2 → 3.4.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (247) hide show

package/dist/esm/parquetjs/parquet-thrift/UUIDType.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"UUIDType.js","names":["thrift","UUIDType","constructor","write","output","writeStructBegin","writeFieldStop","writeStructEnd","read","input","readStructBegin","ret","readFieldBegin","fieldType","ftype","fieldId","fid","Thrift","Type","STOP","skip","readFieldEnd","readStructEnd"],"sources":["../../../../src/parquetjs/parquet-thrift/UUIDType.ts"],"sourcesContent":["/* tslint:disable /\n/ eslint-disable /\n/\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n /\nimport as thrift from 'thrift';\nexport interface IUUIDTypeArgs {}\nexport class UUIDType {\n constructor() {}\n public write(output: thrift.TProtocol): void {\n output.writeStructBegin('UUIDType');\n output.writeFieldStop();\n output.writeStructEnd();\n return;\n }\n public static read(input: thrift.TProtocol): UUIDType {\n input.readStructBegin();\n while (true) {\n const ret: thrift.TField = input.readFieldBegin();\n const fieldType: thrift.Thrift.Type = ret.ftype;\n const fieldId: number = ret.fid;\n if (fieldType === thrift.Thrift.Type.STOP) {\n break;\n }\n switch (fieldId) {\n default: {\n input.skip(fieldType);\n }\n }\n input.readFieldEnd();\n }\n input.readStructEnd();\n return new UUIDType();\n }\n}\n"],"mappings":";AAMA,OAAO,KAAKA,MAAM,MAAM,QAAQ;AAEhC,OAAO,MAAMC,QAAQ,CAAC;EACpBC,~~WAAW~~,~~GAAG~~,CAAC;EACRC,~~KAAK~~,CAACC,MAAwB,EAAQ;IAC3CA,MAAM,CAACC,gBAAgB,CAAC,UAAU,CAAC;IACnCD,MAAM,CAACE,cAAc,EAAE;IACvBF,MAAM,CAACG,cAAc,EAAE;IACvB;EACF;EACA,OAAcC,~~IAAI~~,CAACC,KAAuB,EAAY;IACpDA,KAAK,CAACC,eAAe,EAAE;IACvB,OAAO,IAAI,EAAE;MACX,MAAMC,GAAkB,GAAGF,KAAK,CAACG,cAAc,EAAE;MACjD,MAAMC,SAA6B,GAAGF,GAAG,CAACG,KAAK;MAC/C,MAAMC,OAAe,GAAGJ,GAAG,CAACK,GAAG;MAC/B,IAAIH,SAAS,KAAKb,MAAM,CAACiB,MAAM,CAACC,IAAI,CAACC,IAAI,EAAE;QACzC;MACF;MACA,QAAQJ,OAAO;QACb;UAAS;YACPN,KAAK,CAACW,IAAI,CAACP,SAAS,CAAC;UACvB;MAAC;MAEHJ,KAAK,CAACY,YAAY,EAAE;IACtB;IACAZ,KAAK,CAACa,aAAa,EAAE;IACrB,OAAO,IAAIrB,QAAQ,EAAE;EACvB;AACF"}
1	+ {"version":3,"file":"UUIDType.js","names":["thrift","UUIDType","constructor","write","output","writeStructBegin","writeFieldStop","writeStructEnd","read","input","readStructBegin","ret","readFieldBegin","fieldType","ftype","fieldId","fid","Thrift","Type","STOP","skip","readFieldEnd","readStructEnd"],"sources":["../../../../src/parquetjs/parquet-thrift/UUIDType.ts"],"sourcesContent":["/* tslint:disable /\n/ eslint-disable /\n/\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n /\nimport as thrift from 'thrift';\nexport interface IUUIDTypeArgs {}\nexport class UUIDType {\n constructor() {}\n public write(output: thrift.TProtocol): void {\n output.writeStructBegin('UUIDType');\n output.writeFieldStop();\n output.writeStructEnd();\n return;\n }\n public static read(input: thrift.TProtocol): UUIDType {\n input.readStructBegin();\n while (true) {\n const ret: thrift.TField = input.readFieldBegin();\n const fieldType: thrift.Thrift.Type = ret.ftype;\n const fieldId: number = ret.fid;\n if (fieldType === thrift.Thrift.Type.STOP) {\n break;\n }\n switch (fieldId) {\n default: {\n input.skip(fieldType);\n }\n }\n input.readFieldEnd();\n }\n input.readStructEnd();\n return new UUIDType();\n }\n}\n"],"mappings":"AAMA,OAAO,KAAKA,MAAM,MAAM,QAAQ;AAEhC,OAAO,MAAMC,QAAQ,CAAC;EACpBC,WAAWA,CAAA,EAAG,CAAC;EACRC,KAAKA,CAACC,MAAwB,EAAQ;IAC3CA,MAAM,CAACC,gBAAgB,CAAC,UAAU,CAAC;IACnCD,MAAM,CAACE,cAAc,EAAE;IACvBF,MAAM,CAACG,cAAc,EAAE;IACvB;EACF;EACA,OAAcC,IAAIA,CAACC,KAAuB,EAAY;IACpDA,KAAK,CAACC,eAAe,EAAE;IACvB,OAAO,IAAI,EAAE;MACX,MAAMC,GAAkB,GAAGF,KAAK,CAACG,cAAc,EAAE;MACjD,MAAMC,SAA6B,GAAGF,GAAG,CAACG,KAAK;MAC/C,MAAMC,OAAe,GAAGJ,GAAG,CAACK,GAAG;MAC/B,IAAIH,SAAS,KAAKb,MAAM,CAACiB,MAAM,CAACC,IAAI,CAACC,IAAI,EAAE;QACzC;MACF;MACA,QAAQJ,OAAO;QACb;UAAS;YACPN,KAAK,CAACW,IAAI,CAACP,SAAS,CAAC;UACvB;MAAC;MAEHJ,KAAK,CAACY,YAAY,EAAE;IACtB;IACAZ,KAAK,CAACa,aAAa,EAAE;IACrB,OAAO,IAAIrB,QAAQ,EAAE;EACvB;AACF"}

package/dist/esm/parquetjs/parquet-thrift/index.js CHANGED Viewed

@@ -1,4 +1,3 @@
 export * from './Type';
 export * from './ConvertedType';
 export * from './FieldRepetitionType';

package/dist/esm/parquetjs/parquet-thrift/index.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.js","names":[],"sources":["../../../../src/parquetjs/parquet-thrift/index.ts"],"sourcesContent":["/* tslint:disable /\n/ eslint-disable /\n/\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n /\nexport from './Type';\nexport * from './ConvertedType';\nexport * from './FieldRepetitionType';\nexport * from './Encoding';\nexport * from './CompressionCodec';\nexport * from './PageType';\nexport * from './BoundaryOrder';\nexport * from './Statistics';\nexport * from './StringType';\nexport * from './UUIDType';\nexport * from './MapType';\nexport * from './ListType';\nexport * from './EnumType';\nexport * from './DateType';\nexport * from './NullType';\nexport * from './DecimalType';\nexport * from './MilliSeconds';\nexport * from './MicroSeconds';\nexport * from './TimestampType';\nexport * from './TimeType';\nexport * from './IntType';\nexport * from './JsonType';\nexport * from './BsonType';\nexport * from './SchemaElement';\nexport * from './DataPageHeader';\nexport * from './IndexPageHeader';\nexport * from './DictionaryPageHeader';\nexport * from './DataPageHeaderV2';\nexport * from './PageHeader';\nexport * from './KeyValue';\nexport * from './SortingColumn';\nexport * from './PageEncodingStats';\nexport * from './ColumnMetaData';\nexport * from './ColumnChunk';\nexport * from './RowGroup';\nexport * from './TypeDefinedOrder';\nexport * from './PageLocation';\nexport * from './OffsetIndex';\nexport * from './ColumnIndex';\nexport * from './FileMetaData';\nexport * from './TimeUnit';\nexport * from './LogicalType';\nexport * from './ColumnOrder';\n"],"mappings":";AAMA,cAAc,QAAQ;AACtB,cAAc,iBAAiB;AAC/B,cAAc,uBAAuB;AACrC,cAAc,YAAY;AAC1B,cAAc,oBAAoB;AAClC,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,cAAc;AAC5B,cAAc,cAAc;AAC5B,cAAc,YAAY;AAC1B,cAAc,WAAW;AACzB,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,eAAe;AAC7B,cAAc,gBAAgB;AAC9B,cAAc,gBAAgB;AAC9B,cAAc,iBAAiB;AAC/B,cAAc,YAAY;AAC1B,cAAc,WAAW;AACzB,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,kBAAkB;AAChC,cAAc,mBAAmB;AACjC,cAAc,wBAAwB;AACtC,cAAc,oBAAoB;AAClC,cAAc,cAAc;AAC5B,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,qBAAqB;AACnC,cAAc,kBAAkB;AAChC,cAAc,eAAe;AAC7B,cAAc,YAAY;AAC1B,cAAc,oBAAoB;AAClC,cAAc,gBAAgB;AAC9B,cAAc,eAAe;AAC7B,cAAc,eAAe;AAC7B,cAAc,gBAAgB;AAC9B,cAAc,YAAY;AAC1B,cAAc,eAAe;AAC7B,cAAc,eAAe"}
1	+ {"version":3,"file":"index.js","names":[],"sources":["../../../../src/parquetjs/parquet-thrift/index.ts"],"sourcesContent":["/* tslint:disable /\n/ eslint-disable /\n/\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n /\nexport from './Type';\nexport * from './ConvertedType';\nexport * from './FieldRepetitionType';\nexport * from './Encoding';\nexport * from './CompressionCodec';\nexport * from './PageType';\nexport * from './BoundaryOrder';\nexport * from './Statistics';\nexport * from './StringType';\nexport * from './UUIDType';\nexport * from './MapType';\nexport * from './ListType';\nexport * from './EnumType';\nexport * from './DateType';\nexport * from './NullType';\nexport * from './DecimalType';\nexport * from './MilliSeconds';\nexport * from './MicroSeconds';\nexport * from './TimestampType';\nexport * from './TimeType';\nexport * from './IntType';\nexport * from './JsonType';\nexport * from './BsonType';\nexport * from './SchemaElement';\nexport * from './DataPageHeader';\nexport * from './IndexPageHeader';\nexport * from './DictionaryPageHeader';\nexport * from './DataPageHeaderV2';\nexport * from './PageHeader';\nexport * from './KeyValue';\nexport * from './SortingColumn';\nexport * from './PageEncodingStats';\nexport * from './ColumnMetaData';\nexport * from './ColumnChunk';\nexport * from './RowGroup';\nexport * from './TypeDefinedOrder';\nexport * from './PageLocation';\nexport * from './OffsetIndex';\nexport * from './ColumnIndex';\nexport * from './FileMetaData';\nexport * from './TimeUnit';\nexport * from './LogicalType';\nexport * from './ColumnOrder';\n"],"mappings":"AAMA,cAAc,QAAQ;AACtB,cAAc,iBAAiB;AAC/B,cAAc,uBAAuB;AACrC,cAAc,YAAY;AAC1B,cAAc,oBAAoB;AAClC,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,cAAc;AAC5B,cAAc,cAAc;AAC5B,cAAc,YAAY;AAC1B,cAAc,WAAW;AACzB,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,eAAe;AAC7B,cAAc,gBAAgB;AAC9B,cAAc,gBAAgB;AAC9B,cAAc,iBAAiB;AAC/B,cAAc,YAAY;AAC1B,cAAc,WAAW;AACzB,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,kBAAkB;AAChC,cAAc,mBAAmB;AACjC,cAAc,wBAAwB;AACtC,cAAc,oBAAoB;AAClC,cAAc,cAAc;AAC5B,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,qBAAqB;AACnC,cAAc,kBAAkB;AAChC,cAAc,eAAe;AAC7B,cAAc,YAAY;AAC1B,cAAc,oBAAoB;AAClC,cAAc,gBAAgB;AAC9B,cAAc,eAAe;AAC7B,cAAc,eAAe;AAC7B,cAAc,gBAAgB;AAC9B,cAAc,YAAY;AAC1B,cAAc,eAAe;AAC7B,cAAc,eAAe"}

package/dist/esm/parquetjs/parser/decoders.js CHANGED Viewed

@@ -1,11 +1,8 @@
 import { PARQUET_CODECS } from '../codecs';
 import { ConvertedType, Encoding, FieldRepetitionType, PageType, Type } from '../parquet-thrift';
 import { decompress } from '../compression';
 import { PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING } from '../../constants';
 import { decodePageHeader, getThriftEnum, getBitWidth } from '../utils/read-utils';
 export async function decodeDataPages(buffer, options) {
   const cursor = {
     buffer,
@@ -20,8 +17,7 @@ export async function decodeDataPages(buffer, options) {
     count: 0
   };
   let dictionary = options.dictionary || [];
-  while (
-  cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues))) {
+  while (cursor.offset < cursor.size && (!options.numValues || data.dlevels.length < Number(options.numValues))) {
     const page = await decodePage(cursor, options);
     if (page.dictionary) {
       dictionary = page.dictionary;
@@ -43,7 +39,6 @@ export async function decodeDataPages(buffer, options) {
   }
   return data;
 }
 export async function decodePage(cursor, options) {
   let page;
   const {
@@ -70,7 +65,6 @@ export async function decodePage(cursor, options) {
   }
   return page;
 }
 export function decodeSchema(schemaElements, offset, len) {
   const schema = {};
   let next = offset;
@@ -128,19 +122,16 @@ export function decodeSchema(schemaElements, offset, len) {
     next
   };
 }
 function decodeValues(type, encoding, cursor, count, opts) {
   if (!(encoding in PARQUET_CODECS)) {
     throw new Error("invalid encoding: ".concat(encoding));
   }
   return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);
 }
 async function decodeDataPage(cursor, header, options) {
   var _header$data_page_hea, _header$data_page_hea2, _header$data_page_hea3, _header$data_page_hea4;
   const cursorEnd = cursor.offset + header.compressed_page_size;
   const valueCount = (_header$data_page_hea = header.data_page_header) === null || _header$data_page_hea === void 0 ? void 0 : _header$data_page_hea.num_values;
   let dataCursor = cursor;
   if (options.compression !== 'UNCOMPRESSED') {
     const valuesBuf = await decompress(options.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
@@ -151,7 +142,6 @@ async function decodeDataPage(cursor, header, options) {
     };
     cursor.offset = cursorEnd;
   }
   const rLevelEncoding = getThriftEnum(Encoding, (_header$data_page_hea2 = header.data_page_header) === null || _header$data_page_hea2 === void 0 ? void 0 : _header$data_page_hea2.repetition_level_encoding);
   let rLevels = new Array(valueCount);
   if (options.column.rLevelMax > 0) {
@@ -162,7 +152,6 @@ async function decodeDataPage(cursor, header, options) {
   } else {
     rLevels.fill(0);
   }
   const dLevelEncoding = getThriftEnum(Encoding, (_header$data_page_hea3 = header.data_page_header) === null || _header$data_page_hea3 === void 0 ? void 0 : _header$data_page_hea3.definition_level_encoding);
   let dLevels = new Array(valueCount);
   if (options.column.dLevelMax > 0) {
@@ -179,7 +168,6 @@ async function decodeDataPage(cursor, header, options) {
       valueCountNonNull++;
     }
   }
   const valueEncoding = getThriftEnum(Encoding, (_header$data_page_hea4 = header.data_page_header) === null || _header$data_page_hea4 === void 0 ? void 0 : _header$data_page_hea4.encoding);
   const decodeOptions = {
     typeLength: options.column.typeLength,
@@ -194,14 +182,12 @@ async function decodeDataPage(cursor, header, options) {
     pageHeader: header
   };
 }
 async function decodeDataPageV2(cursor, header, opts) {
   var _header$data_page_hea5, _header$data_page_hea6, _header$data_page_hea7, _header$data_page_hea8;
   const cursorEnd = cursor.offset + header.compressed_page_size;
   const valueCount = (_header$data_page_hea5 = header.data_page_header_v2) === null || _header$data_page_hea5 === void 0 ? void 0 : _header$data_page_hea5.num_values;
   const valueCountNonNull = valueCount - ((_header$data_page_hea6 = header.data_page_header_v2) === null || _header$data_page_hea6 === void 0 ? void 0 : _header$data_page_hea6.num_nulls);
   const valueEncoding = getThriftEnum(Encoding, (_header$data_page_hea7 = header.data_page_header_v2) === null || _header$data_page_hea7 === void 0 ? void 0 : _header$data_page_hea7.encoding);
   let rLevels = new Array(valueCount);
   if (opts.column.rLevelMax > 0) {
     rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
@@ -211,7 +197,6 @@ async function decodeDataPageV2(cursor, header, opts) {
   } else {
     rLevels.fill(0);
   }
   let dLevels = new Array(valueCount);
   if (opts.column.dLevelMax > 0) {
     dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
@@ -221,7 +206,6 @@ async function decodeDataPageV2(cursor, header, opts) {
   } else {
     dLevels.fill(0);
   }
   let valuesBufCursor = cursor;
   if ((_header$data_page_hea8 = header.data_page_header_v2) !== null && _header$data_page_hea8 !== void 0 && _header$data_page_hea8.is_compressed) {
     const valuesBuf = await decompress(opts.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
@@ -245,7 +229,6 @@ async function decodeDataPageV2(cursor, header, opts) {
     pageHeader: header
   };
 }
 async function decodeDictionaryPage(cursor, pageHeader, options) {
   var _pageHeader$dictionar;
   const cursorEnd = cursor.offset + pageHeader.compressed_page_size;

package/dist/esm/parquetjs/parser/decoders.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"decoders.js","names":["PARQUET_CODECS","ConvertedType","Encoding","FieldRepetitionType","PageType","Type","decompress","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","decodePageHeader","getThriftEnum","getBitWidth","decodeDataPages","buffer","options","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","pageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","repetition_type","optional","repeated","num_children","res","name","fields","logicalType","converted_type","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","header","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","slice","uncompressed_page_size","rLevelEncoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","data_page_header_v2","num_nulls","valuesBufCursor","is_compressed","dictCursor","dictionary_page_header","d","toString"],"sources":["../../../../src/parquetjs/parser/decoders.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n ParquetCodec,\n ParquetData,\n ParquetOptions,\n ParquetPageData,\n ParquetType,\n PrimitiveType,\n SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n ConvertedType,\n Encoding,\n FieldRepetitionType,\n PageHeader,\n PageType,\n SchemaElement,\n Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/*\n Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n /\nexport async function decodeDataPages(\n buffer: Buffer,\n options: ParquetOptions\n): Promise<ParquetData> {\n const cursor: CursorBuffer = {\n buffer,\n offset: 0,\n size: buffer.length\n };\n\n const data: ParquetData = {\n rlevels: [],\n dlevels: [],\n values: [],\n pageHeaders: [],\n count: 0\n };\n\n let dictionary = options.dictionary \|\| [];\n\n while (\n // @ts-ignore size can be undefined\n cursor.offset < cursor.size &&\n (!options.numValues \|\| data.dlevels.length < Number(options.numValues))\n ) {\n // Looks like we have to decode these in sequence due to cursor updates?\n const page = await decodePage(cursor, options);\n\n if (page.dictionary) {\n dictionary = page.dictionary;\n // eslint-disable-next-line no-continue\n continue;\n }\n\n if (dictionary.length) {\n // eslint-disable-next-line no-loop-func\n page.values = page.values.map((value) => dictionary[value]);\n }\n\n for (let index = 0; index < page.rlevels.length; index++) {\n data.rlevels.push(page.rlevels[index]);\n data.dlevels.push(page.dlevels[index]);\n const value = page.values[index];\n\n if (value !== undefined) {\n data.values.push(value);\n }\n }\n\n data.count += page.count;\n data.pageHeaders.push(page.pageHeader);\n }\n\n return data;\n}\n\n/\n Decode parquet page based on page type\n * @param cursor\n * @param options\n /\nexport async function decodePage(\n cursor: CursorBuffer,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n let page;\n const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n cursor.offset += length;\n\n const pageType = getThriftEnum(PageType, pageHeader.type);\n\n switch (pageType) {\n case 'DATA_PAGE':\n page = await decodeDataPage(cursor, pageHeader, options);\n break;\n case 'DATA_PAGE_V2':\n page = await decodeDataPageV2(cursor, pageHeader, options);\n break;\n case 'DICTIONARY_PAGE':\n page = {\n dictionary: await decodeDictionaryPage(cursor, pageHeader, options),\n pageHeader\n };\n break;\n default:\n throw new Error(`invalid page type: ${pageType}`);\n }\n\n return page;\n}\n\n/\n Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n * result.next - offset at the end of function\n * result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n /\nexport function decodeSchema(\n schemaElements: SchemaElement[],\n offset: number,\n len: number\n): {\n offset: number;\n next: number;\n schema: SchemaDefinition;\n} {\n const schema: SchemaDefinition = {};\n let next = offset;\n for (let i = 0; i < len; i++) {\n const schemaElement = schemaElements[next];\n\n const repetitionType =\n next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n let optional = false;\n let repeated = false;\n switch (repetitionType) {\n case 'REQUIRED':\n break;\n case 'OPTIONAL':\n optional = true;\n break;\n case 'REPEATED':\n repeated = true;\n break;\n default:\n throw new Error('parquet: unknown repetition type');\n }\n\n if (schemaElement.num_children! > 0) {\n const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n next = res.next;\n schema[schemaElement.name] = {\n // type: undefined,\n optional,\n repeated,\n fields: res.schema\n };\n } else {\n const type = getThriftEnum(Type, schemaElement.type!);\n let logicalType = type;\n\n if (schemaElement.converted_type) {\n logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n }\n\n switch (logicalType) {\n case 'DECIMAL':\n logicalType = `${logicalType}_${type}` as ParquetType;\n break;\n default:\n }\n\n schema[schemaElement.name] = {\n type: logicalType as ParquetType,\n typeLength: schemaElement.type_length,\n presision: schemaElement.precision,\n scale: schemaElement.scale,\n optional,\n repeated\n };\n next++;\n }\n }\n return {schema, offset, next};\n}\n\n/\n Decode a consecutive array of data using one of the parquet encodings\n /\nfunction decodeValues(\n type: PrimitiveType,\n encoding: ParquetCodec,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): any[] {\n if (!(encoding in PARQUET_CODECS)) {\n throw new Error(`invalid encoding: ${encoding}`);\n }\n return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/\n Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n /\nasync function decodeDataPage(\n cursor: CursorBuffer,\n header: PageHeader,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n const valueCount = header.data_page_header?.num_values;\n\n / uncompress page /\n let dataCursor = cursor;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n dataCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n cursor.offset = cursorEnd;\n }\n\n / read repetition levels /\n const rLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.repetition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n\n if (options.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.rLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n rLevels.fill(0);\n }\n\n / read definition levels /\n const dLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.definition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (options.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.dLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n dLevels.fill(0);\n }\n let valueCountNonNull = 0;\n for (const dlvl of dLevels) {\n if (dlvl === options.column.dLevelMax) {\n valueCountNonNull++;\n }\n }\n\n / read values /\n const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n const decodeOptions = {\n typeLength: options.column.typeLength,\n bitWidth: options.column.typeLength\n };\n\n const values = decodeValues(\n options.column.primitiveType!,\n valueEncoding,\n dataCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/\n Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n /\nasync function decodeDataPageV2(\n cursor: CursorBuffer,\n header: PageHeader,\n opts: any\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n\n const valueCount = header.data_page_header_v2?.num_values;\n // @ts-ignore\n const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n const valueEncoding = getThriftEnum(\n Encoding,\n header.data_page_header_v2?.encoding!\n ) as ParquetCodec;\n\n / read repetition levels /\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n if (opts.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.rLevelMax),\n disableEnvelope: true\n });\n } else {\n rLevels.fill(0);\n }\n\n / read definition levels /\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (opts.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.dLevelMax),\n disableEnvelope: true\n });\n } else {\n dLevels.fill(0);\n }\n\n / read values /\n let valuesBufCursor = cursor;\n\n if (header.data_page_header_v2?.is_compressed) {\n const valuesBuf = await decompress(\n opts.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n\n valuesBufCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const decodeOptions = {\n typeLength: opts.column.typeLength,\n bitWidth: opts.column.typeLength\n };\n\n const values = decodeValues(\n opts.column.primitiveType!,\n valueEncoding,\n valuesBufCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/\n Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param options\n */\nasync function decodeDictionaryPage(\n cursor: CursorBuffer,\n pageHeader: PageHeader,\n options: ParquetOptions\n): Promise<string[]> {\n const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n let dictCursor = {\n offset: 0,\n buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n size: cursorEnd - cursor.offset\n };\n\n cursor.offset = cursorEnd;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n pageHeader.uncompressed_page_size\n );\n\n dictCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const numValues = pageHeader?.dictionary_page_header?.num_values \|\| 0;\n\n return decodeValues(\n options.column.primitiveType!,\n options.column.encoding!,\n dictCursor,\n numValues,\n options as ParquetCodecOptions\n ).map((d) => d.toString());\n}\n"],"mappings":";;AAUA,SAA2CA,cAAc,QAAO,WAAW;AAC3E,SACEC,aAAa,EACbC,QAAQ,EACRC,mBAAmB,EAEnBC,QAAQ,EAERC,IAAI,QACC,mBAAmB;AAC1B,SAAQC,UAAU,QAAO,gBAAgB;AACzC,SAAQC,kBAAkB,EAAEC,sBAAsB,QAAO,iBAAiB;AAC1E,SAAQC,gBAAgB,EAAEC,aAAa,EAAEC,WAAW,QAAO,qBAAqB;;AAShF,OAAO,eAAeC,eAAe,CACnCC,MAAc,EACdC,OAAuB,EACD;EACtB,MAAMC,MAAoB,GAAG;IAC3BF,MAAM;IACNG,MAAM,EAAE,CAAC;IACTC,IAAI,EAAEJ,MAAM,CAACK;EACf,CAAC;EAED,MAAMC,IAAiB,GAAG;IACxBC,OAAO,EAAE,EAAE;IACXC,OAAO,EAAE,EAAE;IACXC,MAAM,EAAE,EAAE;IACVC,WAAW,EAAE,EAAE;IACfC,KAAK,EAAE;EACT,CAAC;EAED,IAAIC,UAAU,GAAGX,OAAO,CAACW,UAAU,IAAI,EAAE;EAEzC;EAEEV,MAAM,CAACC,MAAM,GAAGD,MAAM,CAACE,IAAI,KAC1B,CAACH,OAAO,CAACY,SAAS,IAAIP,IAAI,CAACE,OAAO,CAACH,MAAM,GAAGS,MAAM,CAACb,OAAO,CAACY,SAAS,CAAC,CAAC,EACvE;IAEA,MAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAM,EAAED,OAAO,CAAC;IAE9C,IAAIc,IAAI,CAACH,UAAU,EAAE;MACnBA,UAAU,GAAGG,IAAI,CAACH,UAAU;MAE5B;IACF;IAEA,IAAIA,UAAU,CAACP,MAAM,EAAE;MAErBU,IAAI,CAACN,MAAM,GAAGM,IAAI,CAACN,MAAM,CAACQ,GAAG,CAAEC,KAAK,IAAKN,UAAU,CAACM,KAAK,CAAC,CAAC;IAC7D;IAEA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGJ,IAAI,CAACR,OAAO,CAACF,MAAM,EAAEc,KAAK,EAAE,EAAE;MACxDb,IAAI,CAACC,OAAO,CAACa,IAAI,CAACL,IAAI,CAACR,OAAO,CAACY,KAAK,CAAC,CAAC;MACtCb,IAAI,CAACE,OAAO,CAACY,IAAI,CAACL,IAAI,CAACP,OAAO,CAACW,KAAK,CAAC,CAAC;MACtC,MAAMD,KAAK,GAAGH,IAAI,CAACN,MAAM,CAACU,KAAK,CAAC;MAEhC,IAAID,KAAK,KAAKG,SAAS,EAAE;QACvBf,IAAI,CAACG,MAAM,CAACW,IAAI,CAACF,KAAK,CAAC;MACzB;IACF;IAEAZ,IAAI,CAACK,KAAK,IAAII,IAAI,CAACJ,KAAK;IACxBL,IAAI,CAACI,WAAW,CAACU,IAAI,CAACL,IAAI,CAACO,UAAU,CAAC;EACxC;EAEA,OAAOhB,IAAI;AACb;;AAOA,OAAO,eAAeU,UAAU,CAC9Bd,MAAoB,EACpBD,OAAuB,EACG;EAC1B,IAAIc,IAAI;EACR,MAAM;IAACO,UAAU;IAAEjB;EAAM,CAAC,GAAGT,gBAAgB,CAACM,MAAM,CAACF,MAAM,EAAEE,MAAM,CAACC,MAAM,CAAC;EAC3ED,MAAM,CAACC,MAAM,IAAIE,MAAM;EAEvB,MAAMkB,QAAQ,GAAG1B,aAAa,CAACN,QAAQ,EAAE+B,UAAU,CAACE,IAAI,CAAC;EAEzD,QAAQD,QAAQ;IACd,KAAK,WAAW;MACdR,IAAI,GAAG,MAAMU,cAAc,CAACvB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MACxD;IACF,KAAK,cAAc;MACjBc,IAAI,GAAG,MAAMW,gBAAgB,CAACxB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MAC1D;IACF,KAAK,iBAAiB;MACpBc,IAAI,GAAG;QACLH,UAAU,EAAE,MAAMe,oBAAoB,CAACzB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;QACnEqB;MACF,CAAC;MACD;IACF;MACE,MAAM,IAAIM,KAAK,8BAAuBL,QAAQ,EAAG;EAAC;EAGtD,OAAOR,IAAI;AACb;;AAYA,OAAO,SAASc,YAAY,CAC1BC,cAA+B,EAC/B3B,MAAc,EACd4B,GAAW,EAKX;EACA,MAAMC,MAAwB,GAAG,CAAC,CAAC;EACnC,IAAIC,IAAI,GAAG9B,MAAM;EACjB,KAAK,IAAI+B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,GAAG,EAAEG,CAAC,EAAE,EAAE;IAC5B,MAAMC,aAAa,GAAGL,cAAc,CAACG,IAAI,CAAC;IAE1C,MAAMG,cAAc,GAClBH,IAAI,GAAG,CAAC,GAAGpC,aAAa,CAACP,mBAAmB,EAAE6C,aAAa,CAACE,eAAe,CAAE,GAAG,MAAM;IAExF,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,QAAQ,GAAG,KAAK;IACpB,QAAQH,cAAc;MACpB,KAAK,UAAU;QACb;MACF,KAAK,UAAU;QACbE,QAAQ,GAAG,IAAI;QACf;MACF,KAAK,UAAU;QACbC,QAAQ,GAAG,IAAI;QACf;MACF;QACE,MAAM,IAAIX,KAAK,CAAC,kCAAkC,CAAC;IAAC;IAGxD,IAAIO,aAAa,CAACK,YAAY,GAAI,CAAC,EAAE;MACnC,MAAMC,GAAG,GAAGZ,YAAY,CAACC,cAAc,EAAEG,IAAI,GAAG,CAAC,EAAEE,aAAa,CAACK,YAAY,CAAE;MAC/EP,IAAI,GAAGQ,GAAG,CAACR,IAAI;MACfD,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAE3BJ,QAAQ;QACRC,QAAQ;QACRI,MAAM,EAAEF,GAAG,CAACT;MACd,CAAC;IACH,CAAC,MAAM;MACL,MAAMR,IAAI,GAAG3B,aAAa,CAACL,IAAI,EAAE2C,aAAa,CAACX,IAAI,CAAE;MACrD,IAAIoB,WAAW,GAAGpB,IAAI;MAEtB,IAAIW,aAAa,CAACU,cAAc,EAAE;QAChCD,WAAW,GAAG/C,aAAa,CAACT,aAAa,EAAE+C,aAAa,CAACU,cAAc,CAAC;MAC1E;MAEA,QAAQD,WAAW;QACjB,KAAK,SAAS;UACZA,WAAW,aAAMA,WAAW,cAAIpB,IAAI,CAAiB;UACrD;QACF;MAAQ;MAGVQ,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAC3BlB,IAAI,EAAEoB,WAA0B;QAChCE,UAAU,EAAEX,aAAa,CAACY,WAAW;QACrCC,SAAS,EAAEb,aAAa,CAACc,SAAS;QAClCC,KAAK,EAAEf,aAAa,CAACe,KAAK;QAC1BZ,QAAQ;QACRC;MACF,CAAC;MACDN,IAAI,EAAE;IACR;EACF;EACA,OAAO;IAACD,MAAM;IAAE7B,MAAM;IAAE8B;EAAI,CAAC;AAC/B;;AAKA,SAASkB,YAAY,CACnB3B,IAAmB,EACnB4B,QAAsB,EACtBlD,MAAoB,EACpBS,KAAa,EACb0C,IAAyB,EAClB;EACP,IAAI,EAAED,QAAQ,IAAIjE,cAAc,CAAC,EAAE;IACjC,MAAM,IAAIyC,KAAK,6BAAsBwB,QAAQ,EAAG;EAClD;EACA,OAAOjE,cAAc,CAACiE,QAAQ,CAAC,CAACD,YAAY,CAAC3B,IAAI,EAAEtB,MAAM,EAAES,KAAK,EAAE0C,IAAI,CAAC;AACzE;;AAQA,eAAe5B,cAAc,CAC3BvB,MAAoB,EACpBoD,MAAkB,EAClBrD,OAAuB,EACG;EAAA;EAC1B,MAAMsD,SAAS,GAAGrD,MAAM,CAACC,MAAM,GAAGmD,MAAM,CAACE,oBAAoB;EAC7D,MAAMC,UAAU,4BAAGH,MAAM,CAACI,gBAAgB,0DAAvB,sBAAyBC,UAAU;;EAGtD,IAAIC,UAAU,GAAG1D,MAAM;EAEvB,IAAID,OAAO,CAAC4D,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAMrE,UAAU,CAChCQ,OAAO,CAAC4D,WAAW,EACnB3D,MAAM,CAACF,MAAM,CAAC+D,KAAK,CAAC7D,MAAM,CAACC,MAAM,EAAEoD,SAAS,CAAC,EAC7CD,MAAM,CAACU,sBAAsB,CAC9B;IACDJ,UAAU,GAAG;MACX5D,MAAM,EAAE8D,SAAS;MACjB3D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE0D,SAAS,CAACzD;IAClB,CAAC;IACDH,MAAM,CAACC,MAAM,GAAGoD,SAAS;EAC3B;;EAGA,MAAMU,cAAc,GAAGpE,aAAa,CAClCR,QAAQ,4BACRiE,MAAM,CAACI,gBAAgB,2DAAvB,uBAAyBQ,yBAAyB,CACnC;EAEjB,IAAIC,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EAEnC,IAAIxD,OAAO,CAACoE,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGhB,YAAY,CAACzD,kBAAkB,EAAEuE,cAAc,EAAEL,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAEzE,WAAW,CAACG,OAAO,CAACoE,MAAM,CAACC,SAAS,CAAC;MAC/CE,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;;EAGA,MAAMC,cAAc,GAAG7E,aAAa,CAClCR,QAAQ,4BACRiE,MAAM,CAACI,gBAAgB,2DAAvB,uBAAyBiB,yBAAyB,CACnC;EAEjB,IAAIC,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAIxD,OAAO,CAACoE,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAGzB,YAAY,CAACzD,kBAAkB,EAAEgF,cAAc,EAAEd,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAEzE,WAAW,CAACG,OAAO,CAACoE,MAAM,CAACQ,SAAS,CAAC;MAC/CL,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EACA,IAAIK,iBAAiB,GAAG,CAAC;EACzB,KAAK,MAAMC,IAAI,IAAIH,OAAO,EAAE;IAC1B,IAAIG,IAAI,KAAK9E,OAAO,CAACoE,MAAM,CAACQ,SAAS,EAAE;MACrCC,iBAAiB,EAAE;IACrB;EACF;;EAGA,MAAME,aAAa,GAAGnF,aAAa,CAACR,QAAQ,4BAAEiE,MAAM,CAACI,gBAAgB,2DAAvB,uBAAyBN,QAAQ,CAAkB;EACjG,MAAM6B,aAAa,GAAG;IACpBnC,UAAU,EAAE7C,OAAO,CAACoE,MAAM,CAACvB,UAAU;IACrCyB,QAAQ,EAAEtE,OAAO,CAACoE,MAAM,CAACvB;EAC3B,CAAC;EAED,MAAMrC,MAAM,GAAG0C,YAAY,CACzBlD,OAAO,CAACoE,MAAM,CAACa,aAAa,EAC5BF,aAAa,EACbpB,UAAU,EACVkB,iBAAiB,EACjBG,aAAa,CACd;EAED,OAAO;IACLzE,OAAO,EAAEoE,OAAO;IAChBrE,OAAO,EAAE4D,OAAO;IAChB1D,MAAM;IACNE,KAAK,EAAE8C,UAAW;IAClBnC,UAAU,EAAEgC;EACd,CAAC;AACH;;AASA,eAAe5B,gBAAgB,CAC7BxB,MAAoB,EACpBoD,MAAkB,EAClBD,IAAS,EACiB;EAAA;EAC1B,MAAME,SAAS,GAAGrD,MAAM,CAACC,MAAM,GAAGmD,MAAM,CAACE,oBAAoB;EAE7D,MAAMC,UAAU,6BAAGH,MAAM,CAAC6B,mBAAmB,2DAA1B,uBAA4BxB,UAAU;EAEzD,MAAMmB,iBAAiB,GAAGrB,UAAU,8BAAGH,MAAM,CAAC6B,mBAAmB,2DAA1B,uBAA4BC,SAAS;EAC5E,MAAMJ,aAAa,GAAGnF,aAAa,CACjCR,QAAQ,4BACRiE,MAAM,CAAC6B,mBAAmB,2DAA1B,uBAA4B/B,QAAQ,CACrB;;EAIjB,IAAIe,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EACnC,IAAIJ,IAAI,CAACgB,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAC7BH,OAAO,GAAGhB,YAAY,CAACzD,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAEuD,UAAU,EAAG;MACtFc,QAAQ,EAAEzE,WAAW,CAACuD,IAAI,CAACgB,MAAM,CAACC,SAAS,CAAC;MAC5CE,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;;EAIA,IAAIG,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAIJ,IAAI,CAACgB,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAC7BD,OAAO,GAAGzB,YAAY,CAACzD,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAEuD,UAAU,EAAG;MACtFc,QAAQ,EAAEzE,WAAW,CAACuD,IAAI,CAACgB,MAAM,CAACQ,SAAS,CAAC;MAC5CL,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;;EAGA,IAAIY,eAAe,GAAGnF,MAAM;EAE5B,8BAAIoD,MAAM,CAAC6B,mBAAmB,mDAA1B,uBAA4BG,aAAa,EAAE;IAC7C,MAAMxB,SAAS,GAAG,MAAMrE,UAAU,CAChC4D,IAAI,CAACQ,WAAW,EAChB3D,MAAM,CAACF,MAAM,CAAC+D,KAAK,CAAC7D,MAAM,CAACC,MAAM,EAAEoD,SAAS,CAAC,EAC7CD,MAAM,CAACU,sBAAsB,CAC9B;IAEDqB,eAAe,GAAG;MAChBrF,MAAM,EAAE8D,SAAS;MACjB3D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE0D,SAAS,CAACzD;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGoD,SAAS;EAC3B;EAEA,MAAM0B,aAAa,GAAG;IACpBnC,UAAU,EAAEO,IAAI,CAACgB,MAAM,CAACvB,UAAU;IAClCyB,QAAQ,EAAElB,IAAI,CAACgB,MAAM,CAACvB;EACxB,CAAC;EAED,MAAMrC,MAAM,GAAG0C,YAAY,CACzBE,IAAI,CAACgB,MAAM,CAACa,aAAa,EACzBF,aAAa,EACbK,eAAe,EACfP,iBAAiB,EACjBG,aAAa,CACd;EAED,OAAO;IACLzE,OAAO,EAAEoE,OAAO;IAChBrE,OAAO,EAAE4D,OAAO;IAChB1D,MAAM;IACNE,KAAK,EAAE8C,UAAW;IAClBnC,UAAU,EAAEgC;EACd,CAAC;AACH;;AAQA,eAAe3B,oBAAoB,CACjCzB,MAAoB,EACpBoB,UAAsB,EACtBrB,OAAuB,EACJ;EAAA;EACnB,MAAMsD,SAAS,GAAGrD,MAAM,CAACC,MAAM,GAAGmB,UAAU,CAACkC,oBAAoB;EAEjE,IAAI+B,UAAU,GAAG;IACfpF,MAAM,EAAE,CAAC;IACTH,MAAM,EAAEE,MAAM,CAACF,MAAM,CAAC+D,KAAK,CAAC7D,MAAM,CAACC,MAAM,EAAEoD,SAAS,CAAC;IACrDnD,IAAI,EAAEmD,SAAS,GAAGrD,MAAM,CAACC;EAC3B,CAAC;EAEDD,MAAM,CAACC,MAAM,GAAGoD,SAAS;EAEzB,IAAItD,OAAO,CAAC4D,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAMrE,UAAU,CAChCQ,OAAO,CAAC4D,WAAW,EACnB0B,UAAU,CAACvF,MAAM,CAAC+D,KAAK,CAACwB,UAAU,CAACpF,MAAM,EAAEoD,SAAS,CAAC,EACrDjC,UAAU,CAAC0C,sBAAsB,CAClC;IAEDuB,UAAU,GAAG;MACXvF,MAAM,EAAE8D,SAAS;MACjB3D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE0D,SAAS,CAACzD;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGoD,SAAS;EAC3B;EAEA,MAAM1C,SAAS,GAAG,CAAAS,UAAU,aAAVA,UAAU,gDAAVA,UAAU,CAAEkE,sBAAsB,0DAAlC,sBAAoC7B,UAAU,KAAI,CAAC;EAErE,OAAOR,YAAY,CACjBlD,OAAO,CAACoE,MAAM,CAACa,aAAa,EAC5BjF,OAAO,CAACoE,MAAM,CAACjB,QAAQ,EACvBmC,UAAU,EACV1E,SAAS,EACTZ,OAAO,CACR,CAACgB,GAAG,CAAEwE,CAAC,IAAKA,CAAC,CAACC,QAAQ,EAAE,CAAC;AAC5B"}
1	+ {"version":3,"file":"decoders.js","names":["PARQUET_CODECS","ConvertedType","Encoding","FieldRepetitionType","PageType","Type","decompress","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","decodePageHeader","getThriftEnum","getBitWidth","decodeDataPages","buffer","options","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","pageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","concat","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","repetition_type","optional","repeated","num_children","res","name","fields","logicalType","converted_type","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","header","_header$data_page_hea","_header$data_page_hea2","_header$data_page_hea3","_header$data_page_hea4","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","slice","uncompressed_page_size","rLevelEncoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","_header$data_page_hea5","_header$data_page_hea6","_header$data_page_hea7","_header$data_page_hea8","data_page_header_v2","num_nulls","valuesBufCursor","is_compressed","_pageHeader$dictionar","dictCursor","dictionary_page_header","d","toString"],"sources":["../../../../src/parquetjs/parser/decoders.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n ParquetCodec,\n ParquetData,\n ParquetOptions,\n ParquetPageData,\n ParquetType,\n PrimitiveType,\n SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n ConvertedType,\n Encoding,\n FieldRepetitionType,\n PageHeader,\n PageType,\n SchemaElement,\n Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/*\n Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n /\nexport async function decodeDataPages(\n buffer: Buffer,\n options: ParquetOptions\n): Promise<ParquetData> {\n const cursor: CursorBuffer = {\n buffer,\n offset: 0,\n size: buffer.length\n };\n\n const data: ParquetData = {\n rlevels: [],\n dlevels: [],\n values: [],\n pageHeaders: [],\n count: 0\n };\n\n let dictionary = options.dictionary \|\| [];\n\n while (\n // @ts-ignore size can be undefined\n cursor.offset < cursor.size &&\n (!options.numValues \|\| data.dlevels.length < Number(options.numValues))\n ) {\n // Looks like we have to decode these in sequence due to cursor updates?\n const page = await decodePage(cursor, options);\n\n if (page.dictionary) {\n dictionary = page.dictionary;\n // eslint-disable-next-line no-continue\n continue;\n }\n\n if (dictionary.length) {\n // eslint-disable-next-line no-loop-func\n page.values = page.values.map((value) => dictionary[value]);\n }\n\n for (let index = 0; index < page.rlevels.length; index++) {\n data.rlevels.push(page.rlevels[index]);\n data.dlevels.push(page.dlevels[index]);\n const value = page.values[index];\n\n if (value !== undefined) {\n data.values.push(value);\n }\n }\n\n data.count += page.count;\n data.pageHeaders.push(page.pageHeader);\n }\n\n return data;\n}\n\n/\n Decode parquet page based on page type\n * @param cursor\n * @param options\n /\nexport async function decodePage(\n cursor: CursorBuffer,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n let page;\n const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n cursor.offset += length;\n\n const pageType = getThriftEnum(PageType, pageHeader.type);\n\n switch (pageType) {\n case 'DATA_PAGE':\n page = await decodeDataPage(cursor, pageHeader, options);\n break;\n case 'DATA_PAGE_V2':\n page = await decodeDataPageV2(cursor, pageHeader, options);\n break;\n case 'DICTIONARY_PAGE':\n page = {\n dictionary: await decodeDictionaryPage(cursor, pageHeader, options),\n pageHeader\n };\n break;\n default:\n throw new Error(`invalid page type: ${pageType}`);\n }\n\n return page;\n}\n\n/\n Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n * result.next - offset at the end of function\n * result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n /\nexport function decodeSchema(\n schemaElements: SchemaElement[],\n offset: number,\n len: number\n): {\n offset: number;\n next: number;\n schema: SchemaDefinition;\n} {\n const schema: SchemaDefinition = {};\n let next = offset;\n for (let i = 0; i < len; i++) {\n const schemaElement = schemaElements[next];\n\n const repetitionType =\n next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n let optional = false;\n let repeated = false;\n switch (repetitionType) {\n case 'REQUIRED':\n break;\n case 'OPTIONAL':\n optional = true;\n break;\n case 'REPEATED':\n repeated = true;\n break;\n default:\n throw new Error('parquet: unknown repetition type');\n }\n\n if (schemaElement.num_children! > 0) {\n const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n next = res.next;\n schema[schemaElement.name] = {\n // type: undefined,\n optional,\n repeated,\n fields: res.schema\n };\n } else {\n const type = getThriftEnum(Type, schemaElement.type!);\n let logicalType = type;\n\n if (schemaElement.converted_type) {\n logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n }\n\n switch (logicalType) {\n case 'DECIMAL':\n logicalType = `${logicalType}_${type}` as ParquetType;\n break;\n default:\n }\n\n schema[schemaElement.name] = {\n type: logicalType as ParquetType,\n typeLength: schemaElement.type_length,\n presision: schemaElement.precision,\n scale: schemaElement.scale,\n optional,\n repeated\n };\n next++;\n }\n }\n return {schema, offset, next};\n}\n\n/\n Decode a consecutive array of data using one of the parquet encodings\n /\nfunction decodeValues(\n type: PrimitiveType,\n encoding: ParquetCodec,\n cursor: CursorBuffer,\n count: number,\n opts: ParquetCodecOptions\n): any[] {\n if (!(encoding in PARQUET_CODECS)) {\n throw new Error(`invalid encoding: ${encoding}`);\n }\n return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/\n Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n /\nasync function decodeDataPage(\n cursor: CursorBuffer,\n header: PageHeader,\n options: ParquetOptions\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n const valueCount = header.data_page_header?.num_values;\n\n / uncompress page /\n let dataCursor = cursor;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n dataCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n cursor.offset = cursorEnd;\n }\n\n / read repetition levels /\n const rLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.repetition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n\n if (options.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.rLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n rLevels.fill(0);\n }\n\n / read definition levels /\n const dLevelEncoding = getThriftEnum(\n Encoding,\n header.data_page_header?.definition_level_encoding!\n ) as ParquetCodec;\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (options.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n bitWidth: getBitWidth(options.column.dLevelMax),\n disableEnvelope: false\n // column: opts.column\n });\n } else {\n dLevels.fill(0);\n }\n let valueCountNonNull = 0;\n for (const dlvl of dLevels) {\n if (dlvl === options.column.dLevelMax) {\n valueCountNonNull++;\n }\n }\n\n / read values /\n const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n const decodeOptions = {\n typeLength: options.column.typeLength,\n bitWidth: options.column.typeLength\n };\n\n const values = decodeValues(\n options.column.primitiveType!,\n valueEncoding,\n dataCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/\n Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n /\nasync function decodeDataPageV2(\n cursor: CursorBuffer,\n header: PageHeader,\n opts: any\n): Promise<ParquetPageData> {\n const cursorEnd = cursor.offset + header.compressed_page_size;\n\n const valueCount = header.data_page_header_v2?.num_values;\n // @ts-ignore\n const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n const valueEncoding = getThriftEnum(\n Encoding,\n header.data_page_header_v2?.encoding!\n ) as ParquetCodec;\n\n / read repetition levels /\n // tslint:disable-next-line:prefer-array-literal\n let rLevels = new Array(valueCount);\n if (opts.column.rLevelMax > 0) {\n rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.rLevelMax),\n disableEnvelope: true\n });\n } else {\n rLevels.fill(0);\n }\n\n / read definition levels /\n // tslint:disable-next-line:prefer-array-literal\n let dLevels = new Array(valueCount);\n if (opts.column.dLevelMax > 0) {\n dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n bitWidth: getBitWidth(opts.column.dLevelMax),\n disableEnvelope: true\n });\n } else {\n dLevels.fill(0);\n }\n\n / read values /\n let valuesBufCursor = cursor;\n\n if (header.data_page_header_v2?.is_compressed) {\n const valuesBuf = await decompress(\n opts.compression,\n cursor.buffer.slice(cursor.offset, cursorEnd),\n header.uncompressed_page_size\n );\n\n valuesBufCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const decodeOptions = {\n typeLength: opts.column.typeLength,\n bitWidth: opts.column.typeLength\n };\n\n const values = decodeValues(\n opts.column.primitiveType!,\n valueEncoding,\n valuesBufCursor,\n valueCountNonNull,\n decodeOptions\n );\n\n return {\n dlevels: dLevels,\n rlevels: rLevels,\n values,\n count: valueCount!,\n pageHeader: header\n };\n}\n\n/\n Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param options\n */\nasync function decodeDictionaryPage(\n cursor: CursorBuffer,\n pageHeader: PageHeader,\n options: ParquetOptions\n): Promise<string[]> {\n const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n let dictCursor = {\n offset: 0,\n buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n size: cursorEnd - cursor.offset\n };\n\n cursor.offset = cursorEnd;\n\n if (options.compression !== 'UNCOMPRESSED') {\n const valuesBuf = await decompress(\n options.compression,\n dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n pageHeader.uncompressed_page_size\n );\n\n dictCursor = {\n buffer: valuesBuf,\n offset: 0,\n size: valuesBuf.length\n };\n\n cursor.offset = cursorEnd;\n }\n\n const numValues = pageHeader?.dictionary_page_header?.num_values \|\| 0;\n\n return decodeValues(\n options.column.primitiveType!,\n options.column.encoding!,\n dictCursor,\n numValues,\n options as ParquetCodecOptions\n ).map((d) => d.toString());\n}\n"],"mappings":"AAUA,SAA2CA,cAAc,QAAO,WAAW;AAC3E,SACEC,aAAa,EACbC,QAAQ,EACRC,mBAAmB,EAEnBC,QAAQ,EAERC,IAAI,QACC,mBAAmB;AAC1B,SAAQC,UAAU,QAAO,gBAAgB;AACzC,SAAQC,kBAAkB,EAAEC,sBAAsB,QAAO,iBAAiB;AAC1E,SAAQC,gBAAgB,EAAEC,aAAa,EAAEC,WAAW,QAAO,qBAAqB;AAShF,OAAO,eAAeC,eAAeA,CACnCC,MAAc,EACdC,OAAuB,EACD;EACtB,MAAMC,MAAoB,GAAG;IAC3BF,MAAM;IACNG,MAAM,EAAE,CAAC;IACTC,IAAI,EAAEJ,MAAM,CAACK;EACf,CAAC;EAED,MAAMC,IAAiB,GAAG;IACxBC,OAAO,EAAE,EAAE;IACXC,OAAO,EAAE,EAAE;IACXC,MAAM,EAAE,EAAE;IACVC,WAAW,EAAE,EAAE;IACfC,KAAK,EAAE;EACT,CAAC;EAED,IAAIC,UAAU,GAAGX,OAAO,CAACW,UAAU,IAAI,EAAE;EAEzC,OAEEV,MAAM,CAACC,MAAM,GAAGD,MAAM,CAACE,IAAI,KAC1B,CAACH,OAAO,CAACY,SAAS,IAAIP,IAAI,CAACE,OAAO,CAACH,MAAM,GAAGS,MAAM,CAACb,OAAO,CAACY,SAAS,CAAC,CAAC,EACvE;IAEA,MAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAM,EAAED,OAAO,CAAC;IAE9C,IAAIc,IAAI,CAACH,UAAU,EAAE;MACnBA,UAAU,GAAGG,IAAI,CAACH,UAAU;MAE5B;IACF;IAEA,IAAIA,UAAU,CAACP,MAAM,EAAE;MAErBU,IAAI,CAACN,MAAM,GAAGM,IAAI,CAACN,MAAM,CAACQ,GAAG,CAAEC,KAAK,IAAKN,UAAU,CAACM,KAAK,CAAC,CAAC;IAC7D;IAEA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGJ,IAAI,CAACR,OAAO,CAACF,MAAM,EAAEc,KAAK,EAAE,EAAE;MACxDb,IAAI,CAACC,OAAO,CAACa,IAAI,CAACL,IAAI,CAACR,OAAO,CAACY,KAAK,CAAC,CAAC;MACtCb,IAAI,CAACE,OAAO,CAACY,IAAI,CAACL,IAAI,CAACP,OAAO,CAACW,KAAK,CAAC,CAAC;MACtC,MAAMD,KAAK,GAAGH,IAAI,CAACN,MAAM,CAACU,KAAK,CAAC;MAEhC,IAAID,KAAK,KAAKG,SAAS,EAAE;QACvBf,IAAI,CAACG,MAAM,CAACW,IAAI,CAACF,KAAK,CAAC;MACzB;IACF;IAEAZ,IAAI,CAACK,KAAK,IAAII,IAAI,CAACJ,KAAK;IACxBL,IAAI,CAACI,WAAW,CAACU,IAAI,CAACL,IAAI,CAACO,UAAU,CAAC;EACxC;EAEA,OAAOhB,IAAI;AACb;AAOA,OAAO,eAAeU,UAAUA,CAC9Bd,MAAoB,EACpBD,OAAuB,EACG;EAC1B,IAAIc,IAAI;EACR,MAAM;IAACO,UAAU;IAAEjB;EAAM,CAAC,GAAGT,gBAAgB,CAACM,MAAM,CAACF,MAAM,EAAEE,MAAM,CAACC,MAAM,CAAC;EAC3ED,MAAM,CAACC,MAAM,IAAIE,MAAM;EAEvB,MAAMkB,QAAQ,GAAG1B,aAAa,CAACN,QAAQ,EAAE+B,UAAU,CAACE,IAAI,CAAC;EAEzD,QAAQD,QAAQ;IACd,KAAK,WAAW;MACdR,IAAI,GAAG,MAAMU,cAAc,CAACvB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MACxD;IACF,KAAK,cAAc;MACjBc,IAAI,GAAG,MAAMW,gBAAgB,CAACxB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MAC1D;IACF,KAAK,iBAAiB;MACpBc,IAAI,GAAG;QACLH,UAAU,EAAE,MAAMe,oBAAoB,CAACzB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;QACnEqB;MACF,CAAC;MACD;IACF;MACE,MAAM,IAAIM,KAAK,uBAAAC,MAAA,CAAuBN,QAAQ,EAAG;EAAC;EAGtD,OAAOR,IAAI;AACb;AAYA,OAAO,SAASe,YAAYA,CAC1BC,cAA+B,EAC/B5B,MAAc,EACd6B,GAAW,EAKX;EACA,MAAMC,MAAwB,GAAG,CAAC,CAAC;EACnC,IAAIC,IAAI,GAAG/B,MAAM;EACjB,KAAK,IAAIgC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,GAAG,EAAEG,CAAC,EAAE,EAAE;IAC5B,MAAMC,aAAa,GAAGL,cAAc,CAACG,IAAI,CAAC;IAE1C,MAAMG,cAAc,GAClBH,IAAI,GAAG,CAAC,GAAGrC,aAAa,CAACP,mBAAmB,EAAE8C,aAAa,CAACE,eAAe,CAAE,GAAG,MAAM;IAExF,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,QAAQ,GAAG,KAAK;IACpB,QAAQH,cAAc;MACpB,KAAK,UAAU;QACb;MACF,KAAK,UAAU;QACbE,QAAQ,GAAG,IAAI;QACf;MACF,KAAK,UAAU;QACbC,QAAQ,GAAG,IAAI;QACf;MACF;QACE,MAAM,IAAIZ,KAAK,CAAC,kCAAkC,CAAC;IAAC;IAGxD,IAAIQ,aAAa,CAACK,YAAY,GAAI,CAAC,EAAE;MACnC,MAAMC,GAAG,GAAGZ,YAAY,CAACC,cAAc,EAAEG,IAAI,GAAG,CAAC,EAAEE,aAAa,CAACK,YAAY,CAAE;MAC/EP,IAAI,GAAGQ,GAAG,CAACR,IAAI;MACfD,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAE3BJ,QAAQ;QACRC,QAAQ;QACRI,MAAM,EAAEF,GAAG,CAACT;MACd,CAAC;IACH,CAAC,MAAM;MACL,MAAMT,IAAI,GAAG3B,aAAa,CAACL,IAAI,EAAE4C,aAAa,CAACZ,IAAI,CAAE;MACrD,IAAIqB,WAAW,GAAGrB,IAAI;MAEtB,IAAIY,aAAa,CAACU,cAAc,EAAE;QAChCD,WAAW,GAAGhD,aAAa,CAACT,aAAa,EAAEgD,aAAa,CAACU,cAAc,CAAC;MAC1E;MAEA,QAAQD,WAAW;QACjB,KAAK,SAAS;UACZA,WAAW,MAAAhB,MAAA,CAAMgB,WAAW,OAAAhB,MAAA,CAAIL,IAAI,CAAiB;UACrD;QACF;MAAQ;MAGVS,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAC3BnB,IAAI,EAAEqB,WAA0B;QAChCE,UAAU,EAAEX,aAAa,CAACY,WAAW;QACrCC,SAAS,EAAEb,aAAa,CAACc,SAAS;QAClCC,KAAK,EAAEf,aAAa,CAACe,KAAK;QAC1BZ,QAAQ;QACRC;MACF,CAAC;MACDN,IAAI,EAAE;IACR;EACF;EACA,OAAO;IAACD,MAAM;IAAE9B,MAAM;IAAE+B;EAAI,CAAC;AAC/B;AAKA,SAASkB,YAAYA,CACnB5B,IAAmB,EACnB6B,QAAsB,EACtBnD,MAAoB,EACpBS,KAAa,EACb2C,IAAyB,EAClB;EACP,IAAI,EAAED,QAAQ,IAAIlE,cAAc,CAAC,EAAE;IACjC,MAAM,IAAIyC,KAAK,sBAAAC,MAAA,CAAsBwB,QAAQ,EAAG;EAClD;EACA,OAAOlE,cAAc,CAACkE,QAAQ,CAAC,CAACD,YAAY,CAAC5B,IAAI,EAAEtB,MAAM,EAAES,KAAK,EAAE2C,IAAI,CAAC;AACzE;AAQA,eAAe7B,cAAcA,CAC3BvB,MAAoB,EACpBqD,MAAkB,EAClBtD,OAAuB,EACG;EAAA,IAAAuD,qBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAMC,SAAS,GAAG1D,MAAM,CAACC,MAAM,GAAGoD,MAAM,CAACM,oBAAoB;EAC7D,MAAMC,UAAU,IAAAN,qBAAA,GAAGD,MAAM,CAACQ,gBAAgB,cAAAP,qBAAA,uBAAvBA,qBAAA,CAAyBQ,UAAU;EAGtD,IAAIC,UAAU,GAAG/D,MAAM;EAEvB,IAAID,OAAO,CAACiE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAM1E,UAAU,CAChCQ,OAAO,CAACiE,WAAW,EACnBhE,MAAM,CAACF,MAAM,CAACoE,KAAK,CAAClE,MAAM,CAACC,MAAM,EAAEyD,SAAS,CAAC,EAC7CL,MAAM,CAACc,sBAAsB,CAC9B;IACDJ,UAAU,GAAG;MACXjE,MAAM,EAAEmE,SAAS;MACjBhE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE+D,SAAS,CAAC9D;IAClB,CAAC;IACDH,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAC3B;EAGA,MAAMU,cAAc,GAAGzE,aAAa,CAClCR,QAAQ,GAAAoE,sBAAA,GACRF,MAAM,CAACQ,gBAAgB,cAAAN,sBAAA,uBAAvBA,sBAAA,CAAyBc,yBAAyB,CACnC;EAEjB,IAAIC,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EAEnC,IAAI7D,OAAO,CAACyE,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGpB,YAAY,CAAC1D,kBAAkB,EAAE4E,cAAc,EAAEL,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAE9E,WAAW,CAACG,OAAO,CAACyE,MAAM,CAACC,SAAS,CAAC;MAC/CE,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,MAAMC,cAAc,GAAGlF,aAAa,CAClCR,QAAQ,GAAAqE,sBAAA,GACRH,MAAM,CAACQ,gBAAgB,cAAAL,sBAAA,uBAAvBA,sBAAA,CAAyBsB,yBAAyB,CACnC;EAEjB,IAAIC,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI7D,OAAO,CAACyE,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAG7B,YAAY,CAAC1D,kBAAkB,EAAEqF,cAAc,EAAEd,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAE9E,WAAW,CAACG,OAAO,CAACyE,MAAM,CAACQ,SAAS,CAAC;MAC/CL,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EACA,IAAIK,iBAAiB,GAAG,CAAC;EACzB,KAAK,MAAMC,IAAI,IAAIH,OAAO,EAAE;IAC1B,IAAIG,IAAI,KAAKnF,OAAO,CAACyE,MAAM,CAACQ,SAAS,EAAE;MACrCC,iBAAiB,EAAE;IACrB;EACF;EAGA,MAAME,aAAa,GAAGxF,aAAa,CAACR,QAAQ,GAAAsE,sBAAA,GAAEJ,MAAM,CAACQ,gBAAgB,cAAAJ,sBAAA,uBAAvBA,sBAAA,CAAyBN,QAAQ,CAAkB;EACjG,MAAMiC,aAAa,GAAG;IACpBvC,UAAU,EAAE9C,OAAO,CAACyE,MAAM,CAAC3B,UAAU;IACrC6B,QAAQ,EAAE3E,OAAO,CAACyE,MAAM,CAAC3B;EAC3B,CAAC;EAED,MAAMtC,MAAM,GAAG2C,YAAY,CACzBnD,OAAO,CAACyE,MAAM,CAACa,aAAa,EAC5BF,aAAa,EACbpB,UAAU,EACVkB,iBAAiB,EACjBG,aAAa,CACd;EAED,OAAO;IACL9E,OAAO,EAAEyE,OAAO;IAChB1E,OAAO,EAAEiE,OAAO;IAChB/D,MAAM;IACNE,KAAK,EAAEmD,UAAW;IAClBxC,UAAU,EAAEiC;EACd,CAAC;AACH;AASA,eAAe7B,gBAAgBA,CAC7BxB,MAAoB,EACpBqD,MAAkB,EAClBD,IAAS,EACiB;EAAA,IAAAkC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAM/B,SAAS,GAAG1D,MAAM,CAACC,MAAM,GAAGoD,MAAM,CAACM,oBAAoB;EAE7D,MAAMC,UAAU,IAAA0B,sBAAA,GAAGjC,MAAM,CAACqC,mBAAmB,cAAAJ,sBAAA,uBAA1BA,sBAAA,CAA4BxB,UAAU;EAEzD,MAAMmB,iBAAiB,GAAGrB,UAAU,KAAA2B,sBAAA,GAAGlC,MAAM,CAACqC,mBAAmB,cAAAH,sBAAA,uBAA1BA,sBAAA,CAA4BI,SAAS;EAC5E,MAAMR,aAAa,GAAGxF,aAAa,CACjCR,QAAQ,GAAAqG,sBAAA,GACRnC,MAAM,CAACqC,mBAAmB,cAAAF,sBAAA,uBAA1BA,sBAAA,CAA4BrC,QAAQ,CACrB;EAIjB,IAAImB,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EACnC,IAAIR,IAAI,CAACoB,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAC7BH,OAAO,GAAGpB,YAAY,CAAC1D,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAE4D,UAAU,EAAG;MACtFc,QAAQ,EAAE9E,WAAW,CAACwD,IAAI,CAACoB,MAAM,CAACC,SAAS,CAAC;MAC5CE,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;EAIA,IAAIG,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAIR,IAAI,CAACoB,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAC7BD,OAAO,GAAG7B,YAAY,CAAC1D,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAE4D,UAAU,EAAG;MACtFc,QAAQ,EAAE9E,WAAW,CAACwD,IAAI,CAACoB,MAAM,CAACQ,SAAS,CAAC;MAC5CL,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,IAAIgB,eAAe,GAAG5F,MAAM;EAE5B,KAAAyF,sBAAA,GAAIpC,MAAM,CAACqC,mBAAmB,cAAAD,sBAAA,eAA1BA,sBAAA,CAA4BI,aAAa,EAAE;IAC7C,MAAM5B,SAAS,GAAG,MAAM1E,UAAU,CAChC6D,IAAI,CAACY,WAAW,EAChBhE,MAAM,CAACF,MAAM,CAACoE,KAAK,CAAClE,MAAM,CAACC,MAAM,EAAEyD,SAAS,CAAC,EAC7CL,MAAM,CAACc,sBAAsB,CAC9B;IAEDyB,eAAe,GAAG;MAChB9F,MAAM,EAAEmE,SAAS;MACjBhE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE+D,SAAS,CAAC9D;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAC3B;EAEA,MAAM0B,aAAa,GAAG;IACpBvC,UAAU,EAAEO,IAAI,CAACoB,MAAM,CAAC3B,UAAU;IAClC6B,QAAQ,EAAEtB,IAAI,CAACoB,MAAM,CAAC3B;EACxB,CAAC;EAED,MAAMtC,MAAM,GAAG2C,YAAY,CACzBE,IAAI,CAACoB,MAAM,CAACa,aAAa,EACzBF,aAAa,EACbS,eAAe,EACfX,iBAAiB,EACjBG,aAAa,CACd;EAED,OAAO;IACL9E,OAAO,EAAEyE,OAAO;IAChB1E,OAAO,EAAEiE,OAAO;IAChB/D,MAAM;IACNE,KAAK,EAAEmD,UAAW;IAClBxC,UAAU,EAAEiC;EACd,CAAC;AACH;AAQA,eAAe5B,oBAAoBA,CACjCzB,MAAoB,EACpBoB,UAAsB,EACtBrB,OAAuB,EACJ;EAAA,IAAA+F,qBAAA;EACnB,MAAMpC,SAAS,GAAG1D,MAAM,CAACC,MAAM,GAAGmB,UAAU,CAACuC,oBAAoB;EAEjE,IAAIoC,UAAU,GAAG;IACf9F,MAAM,EAAE,CAAC;IACTH,MAAM,EAAEE,MAAM,CAACF,MAAM,CAACoE,KAAK,CAAClE,MAAM,CAACC,MAAM,EAAEyD,SAAS,CAAC;IACrDxD,IAAI,EAAEwD,SAAS,GAAG1D,MAAM,CAACC;EAC3B,CAAC;EAEDD,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAEzB,IAAI3D,OAAO,CAACiE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAM1E,UAAU,CAChCQ,OAAO,CAACiE,WAAW,EACnB+B,UAAU,CAACjG,MAAM,CAACoE,KAAK,CAAC6B,UAAU,CAAC9F,MAAM,EAAEyD,SAAS,CAAC,EACrDtC,UAAU,CAAC+C,sBAAsB,CAClC;IAED4B,UAAU,GAAG;MACXjG,MAAM,EAAEmE,SAAS;MACjBhE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE+D,SAAS,CAAC9D;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAC3B;EAEA,MAAM/C,SAAS,GAAG,CAAAS,UAAU,aAAVA,UAAU,wBAAA0E,qBAAA,GAAV1E,UAAU,CAAE4E,sBAAsB,cAAAF,qBAAA,uBAAlCA,qBAAA,CAAoChC,UAAU,KAAI,CAAC;EAErE,OAAOZ,YAAY,CACjBnD,OAAO,CAACyE,MAAM,CAACa,aAAa,EAC5BtF,OAAO,CAACyE,MAAM,CAACrB,QAAQ,EACvB4C,UAAU,EACVpF,SAAS,EACTZ,OAAO,CACR,CAACgB,GAAG,CAAEkF,CAAC,IAAKA,CAAC,CAACC,QAAQ,EAAE,CAAC;AAC5B"}

package/dist/esm/parquetjs/parser/parquet-reader.js CHANGED Viewed

@@ -1,5 +1,4 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 import { ParquetSchema } from '../schema/schema';
 import { decodeSchema } from './decoders';
 import { materializeRecords } from '../schema/shred';
@@ -10,7 +9,6 @@ import { decodeDataPages, decodePage } from './decoders';
 const DEFAULT_PROPS = {
   defaultDictionarySize: 1e6
 };
 export class ParquetReader {
   constructor(file, props) {
     _defineProperty(this, "props", void 0);
@@ -25,7 +23,6 @@ export class ParquetReader {
   close() {
     this.file.close();
   }
   async *rowIterator(props) {
     for await (const rows of this.rowBatchIterator(props)) {
       for (const row of rows) {
@@ -33,14 +30,12 @@ export class ParquetReader {
       }
     }
   }
   async *rowBatchIterator(props) {
     const schema = await this.getSchema();
     for await (const rowGroup of this.rowGroupIterator(props)) {
       yield materializeRecords(schema, rowGroup);
     }
   }
   async *rowGroupIterator(props) {
     const columnList = ((props === null || props === void 0 ? void 0 : props.columnList) || []).map(x => Array.isArray(x) ? x : [x]);
     const metadata = await this.getFileMetadata();
@@ -64,7 +59,6 @@ export class ParquetReader {
     const schema = new ParquetSchema(schemaDefinition);
     return schema;
   }
   async getSchemaMetadata() {
     const metadata = await this.getFileMetadata();
     const md = {};
@@ -80,7 +74,6 @@ export class ParquetReader {
     }
     return this.metadata;
   }
   async readHeader() {
     const buffer = await this.file.read(0, PARQUET_MAGIC.length);
     const magic = buffer.toString();
@@ -93,7 +86,6 @@ export class ParquetReader {
         throw new Error("Invalid parquet file (magic=".concat(magic, ")"));
     }
   }
   async readFooter() {
     const trailerLen = PARQUET_MAGIC.length + 4;
     const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);
@@ -112,7 +104,6 @@ export class ParquetReader {
     } = decodeFileMetadata(metadataBuf);
     return metadata;
   }
   async readRowGroup(schema, rowGroup, columnList) {
     const buffer = {
       rowCount: Number(rowGroup.num_rows),
@@ -124,12 +115,10 @@ export class ParquetReader {
       if (columnList.length > 0 && fieldIndexOf(columnList, colKey) < 0) {
         continue;
       }
       buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
     }
     return buffer;
   }
   async readColumnChunk(schema, colChunk) {
     var _colChunk$meta_data, _colChunk$meta_data2, _colChunk$meta_data3, _colChunk$meta_data4, _colChunk$meta_data5, _colChunk$meta_data7, _colChunk$meta_data8, _options$dictionary;
     if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
@@ -169,10 +158,8 @@ export class ParquetReader {
       dictionary
     });
   }
   async getDictionary(dictionaryPageOffset, options, pagesOffset) {
     if (dictionaryPageOffset === 0) {
       return [];
     }
     const dictionarySize = Math.min(this.file.size - dictionaryPageOffset, this.props.defaultDictionarySize);

package/dist/esm/parquetjs/parser/parquet-reader.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"parquet-reader.js","names":["ParquetSchema","decodeSchema","materializeRecords","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","DEFAULT_PROPS","defaultDictionarySize","ParquetReader","constructor","file","props","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","columnList","map","x","Array","isArray","metadata","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","num_children","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","buffer","read","magic","toString","Error","trailerLen","trailerBuf","size","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","options","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","decodedPage"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRecords} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n ParquetBuffer,\n ParquetCompression,\n ParquetData,\n PrimitiveType,\n ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader /\nexport type ParquetIterationProps = {\n /* Filter allowing some columns to be dropped /\n columnList?: string[] \| string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n defaultDictionarySize: 1e6\n};\n\n/\n The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n /\nexport class ParquetReader {\n props: Required<ParquetReaderProps>;\n file: ReadableFile;\n metadata: Promise<FileMetaData> \| null = null;\n\n constructor(file: ReadableFile, props?: ParquetReaderProps) {\n this.file = file;\n this.props = {...DEFAULT_PROPS, ...props};\n }\n\n close(): void {\n // eslint-disable-next-line @typescript-eslint/no-floating-promises\n this.file.close();\n }\n\n // HIGH LEVEL METHODS\n\n /* Yield one row at a time /\n async rowIterator(props?: ParquetIterationProps) {\n for await (const rows of this.rowBatchIterator(props)) {\n // yield rows\n for (const row of rows) {\n yield row;\n }\n }\n }\n\n /* Yield one batch of rows at a time /\n async rowBatchIterator(props?: ParquetIterationProps) {\n const schema = await this.getSchema();\n for await (const rowGroup of this.rowGroupIterator(props)) {\n yield materializeRecords(schema, rowGroup);\n }\n }\n\n /** Iterate over the raw row groups /\n async rowGroupIterator(props?: ParquetIterationProps) {\n // Ensure strings are nested in arrays\n const columnList: string[][] = (props?.columnList \|\| []).map((x) =>\n Array.isArray(x) ? x : [x]\n );\n\n const metadata = await this.getFileMetadata();\n const schema = await this.getSchema();\n\n const rowGroupCount = metadata?.row_groups.length \|\| 0;\n\n for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n const rowGroup = await this.readRowGroup(\n schema,\n metadata.row_groups[rowGroupIndex],\n columnList\n );\n yield rowGroup;\n }\n }\n\n async getRowCount(): Promise<number> {\n const metadata = await this.getFileMetadata();\n return Number(metadata.num_rows);\n }\n\n async getSchema(): Promise<ParquetSchema> {\n const metadata = await this.getFileMetadata();\n const root = metadata.schema[0];\n const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n const schema = new ParquetSchema(schemaDefinition);\n return schema;\n }\n\n /*\n Returns the user (key/value) metadata for this file\n * In parquet this is not stored on the schema like it is in arrow\n /\n async getSchemaMetadata(): Promise<Record<string, string>> {\n const metadata = await this.getFileMetadata();\n const md: Record<string, string> = {};\n for (const kv of metadata.key_value_metadata!) {\n md[kv.key] = kv.value!;\n }\n return md;\n }\n\n async getFileMetadata(): Promise<FileMetaData> {\n if (!this.metadata) {\n await this.readHeader();\n this.metadata = this.readFooter();\n }\n return this.metadata;\n }\n\n // LOW LEVEL METHODS\n\n /* Metadata is stored in the footer /\n async readHeader(): Promise<void> {\n const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n const magic = buffer.toString();\n switch (magic) {\n case PARQUET_MAGIC:\n break;\n case PARQUET_MAGIC_ENCRYPTED:\n throw new Error('Encrypted parquet file not supported');\n default:\n throw new Error(`Invalid parquet file (magic=${magic})`);\n }\n }\n\n /* Metadata is stored in the footer /\n async readFooter(): Promise<FileMetaData> {\n const trailerLen = PARQUET_MAGIC.length + 4;\n const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n const magic = trailerBuf.slice(4).toString();\n if (magic !== PARQUET_MAGIC) {\n throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n }\n\n const metadataSize = trailerBuf.readUInt32LE(0);\n const metadataOffset = this.file.size - metadataSize - trailerLen;\n if (metadataOffset < PARQUET_MAGIC.length) {\n throw new Error(`Invalid metadata size ${metadataOffset}`);\n }\n\n const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n // let metadata = new parquet_thrift.FileMetaData();\n // parquet_util.decodeThrift(metadata, metadataBuf);\n const {metadata} = decodeFileMetadata(metadataBuf);\n return metadata;\n }\n\n /* Data is stored in row groups (similar to Apache Arrow record batches) /\n async readRowGroup(\n schema: ParquetSchema,\n rowGroup: RowGroup,\n columnList: string[][]\n ): Promise<ParquetBuffer> {\n const buffer: ParquetBuffer = {\n rowCount: Number(rowGroup.num_rows),\n columnData: {}\n };\n for (const colChunk of rowGroup.columns) {\n const colMetadata = colChunk.meta_data;\n const colKey = colMetadata?.path_in_schema;\n if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n continue; // eslint-disable-line no-continue\n }\n buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n }\n return buffer;\n }\n\n /\n Each row group contains column chunks for all the columns.\n /\n async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n throw new Error('external references are not supported');\n }\n\n const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n if (type !== field.primitiveType) {\n throw new Error(`chunk type not matching schema: ${type}`);\n }\n\n const compression: ParquetCompression = getThriftEnum(\n CompressionCodec,\n colChunk.meta_data?.codec!\n ) as any;\n\n const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n if (!colChunk.file_path) {\n pagesSize = Math.min(\n this.file.size - pagesOffset,\n Number(colChunk.meta_data?.total_compressed_size)\n );\n }\n\n const options: ParquetOptions = {\n type,\n rLevelMax: field.rLevelMax,\n dLevelMax: field.dLevelMax,\n compression,\n column: field,\n numValues: colChunk.meta_data?.num_values,\n dictionary: []\n };\n\n let dictionary;\n\n const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n if (dictionaryPageOffset) {\n const dictionaryOffset = Number(dictionaryPageOffset);\n // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n }\n\n dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n return await decodeDataPages(pagesBuf, {...options, dictionary});\n }\n\n /\n Getting dictionary for allows to flatten values by indices.\n * @param dictionaryPageOffset\n * @param options\n * @param pagesOffset\n * @returns\n */\n async getDictionary(\n dictionaryPageOffset: number,\n options: ParquetOptions,\n pagesOffset: number\n ): Promise<string[]> {\n if (dictionaryPageOffset === 0) {\n // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n // In this case we are working with parquet-mr files format. Problem is described below:\n // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n // We need to get dictionary page from column chunk if it exists.\n // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n return [];\n }\n\n const dictionarySize = Math.min(\n this.file.size - dictionaryPageOffset,\n this.props.defaultDictionarySize\n );\n const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n const decodedPage = await decodePage(cursor, options);\n\n return decodedPage.dictionary!;\n }\n}\n"],"mappings":";;AAGA,SAAQA,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,YAAY,QAAO,YAAY;AACvC,SAAQC,kBAAkB,QAAO,iBAAiB;AAElD,SAAQC,aAAa,EAAEC,uBAAuB,QAAO,iBAAiB;AACtE,SAAqBC,gBAAgB,EAA0BC,IAAI,QAAO,mBAAmB;AAQ7F,SAAQC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY,QAAO,qBAAqB;AACnF,SAAQC,eAAe,EAAEC,UAAU,QAAO,YAAY;AAYtD,MAAMC,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;;AAQD,OAAO,MAAMC,aAAa,CAAC;EAKzBC,WAAW,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAA;IAAA;IAAA,kCAFnB,IAAI;IAG3C,IAAI,CAACD,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGL,aAAa;MAAE,GAAGK;IAAK,CAAC;EAC3C;EAEAC,KAAK,GAAS;IAEZ,IAAI,CAACF,IAAI,CAACE,KAAK,EAAE;EACnB;;EAKA,OAAOC,WAAW,CAACF,KAA6B,EAAE;IAChD,WAAW,MAAMG,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACJ,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMK,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;;EAGA,OAAOD,gBAAgB,CAACJ,KAA6B,EAAE;IACrD,MAAMM,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,EAAE;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACT,KAAK,CAAC,EAAE;MACzD,MAAMf,kBAAkB,CAACqB,MAAM,EAAEE,QAAQ,CAAC;IAC5C;EACF;;EAGA,OAAOC,gBAAgB,CAACT,KAA6B,EAAE;IAErD,MAAMU,UAAsB,GAAG,CAAC,CAAAV,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEU,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC,CAC3B;IAED,MAAMG,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMV,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,EAAE;IAErC,MAAMU,aAAa,GAAG,CAAAF,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEG,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMZ,QAAQ,GAAG,MAAM,IAAI,CAACa,YAAY,CACtCf,MAAM,EACNS,QAAQ,CAACG,UAAU,CAACE,aAAa,CAAC,EAClCV,UAAU,CACX;MACD,MAAMF,QAAQ;IAChB;EACF;EAEA,MAAMc,WAAW,GAAoB;IACnC,MAAMP,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,OAAOO,MAAM,CAACR,QAAQ,CAACS,QAAQ,CAAC;EAClC;EAEA,MAAMjB,SAAS,GAA2B;IACxC,MAAMQ,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMS,IAAI,GAAGV,QAAQ,CAACT,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEoB;IAAgB,CAAC,GAAG1C,YAAY,CAAC+B,QAAQ,CAACT,MAAM,EAAE,CAAC,EAAEmB,IAAI,CAACE,YAAY,CAAE;IACvF,MAAMrB,MAAM,GAAG,IAAIvB,aAAa,CAAC2C,gBAAgB,CAAC;IAClD,OAAOpB,MAAM;EACf;;EAMA,MAAMsB,iBAAiB,GAAoC;IACzD,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMa,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAIf,QAAQ,CAACgB,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMb,eAAe,GAA0B;IAC7C,IAAI,CAAC,IAAI,CAACD,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACmB,UAAU,EAAE;MACvB,IAAI,CAACnB,QAAQ,GAAG,IAAI,CAACoB,UAAU,EAAE;IACnC;IACA,OAAO,IAAI,CAACpB,QAAQ;EACtB;;EAKA,MAAMmB,UAAU,GAAkB;IAChC,MAAME,MAAM,GAAG,MAAM,IAAI,CAACrC,IAAI,CAACsC,IAAI,CAAC,CAAC,EAAEnD,aAAa,CAACiC,MAAM,CAAC;IAC5D,MAAMmB,KAAK,GAAGF,MAAM,CAACG,QAAQ,EAAE;IAC/B,QAAQD,KAAK;MACX,KAAKpD,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAIqD,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,uCAAgCF,KAAK,OAAI;IAAC;EAE/D;;EAGA,MAAMH,UAAU,GAA0B;IACxC,MAAMM,UAAU,GAAGvD,aAAa,CAACiC,MAAM,GAAG,CAAC;IAC3C,MAAMuB,UAAU,GAAG,MAAM,IAAI,CAAC3C,IAAI,CAACsC,IAAI,CAAC,IAAI,CAACtC,IAAI,CAAC4C,IAAI,GAAGF,UAAU,EAAEA,UAAU,CAAC;IAEhF,MAAMH,KAAK,GAAGI,UAAU,CAACE,KAAK,CAAC,CAAC,CAAC,CAACL,QAAQ,EAAE;IAC5C,IAAID,KAAK,KAAKpD,aAAa,EAAE;MAC3B,MAAM,IAAIsD,KAAK,6CAAqCF,KAAK,OAAI;IAC/D;IAEA,MAAMO,YAAY,GAAGH,UAAU,CAACI,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAAChD,IAAI,CAAC4C,IAAI,GAAGE,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAG7D,aAAa,CAACiC,MAAM,EAAE;MACzC,MAAM,IAAIqB,KAAK,iCAA0BO,cAAc,EAAG;IAC5D;IAEA,MAAMC,WAAW,GAAG,MAAM,IAAI,CAACjD,IAAI,CAACsC,IAAI,CAACU,cAAc,EAAEF,YAAY,CAAC;IAGtE,MAAM;MAAC9B;IAAQ,CAAC,GAAGzB,kBAAkB,CAAC0D,WAAW,CAAC;IAClD,OAAOjC,QAAQ;EACjB;;EAGA,MAAMM,YAAY,CAChBf,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB,EACE;IACxB,MAAM0B,MAAqB,GAAG;MAC5Ba,QAAQ,EAAE1B,MAAM,CAACf,QAAQ,CAACgB,QAAQ,CAAC;MACnC0B,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAI3C,QAAQ,CAAC4C,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAI9C,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI3B,YAAY,CAACkB,UAAU,EAAE6C,MAAM,CAAE,GAAG,CAAC,EAAE;QAClE;MACF;;MACAnB,MAAM,CAACc,UAAU,CAACK,MAAM,CAAEE,IAAI,EAAE,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACpD,MAAM,EAAE6C,QAAQ,CAAC;IAClF;IACA,OAAOf,MAAM;EACf;;EAKA,MAAMsB,eAAe,CAACpD,MAAqB,EAAE6C,QAAqB,EAAwB;IAAA;IACxF,IAAIA,QAAQ,CAACQ,SAAS,KAAKC,SAAS,IAAIT,QAAQ,CAACQ,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAInB,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAMqB,KAAK,GAAGvD,MAAM,CAACwD,SAAS,wBAACX,QAAQ,CAACG,SAAS,wDAAlB,oBAAoBE,cAAc,CAAE;IACnE,MAAMO,IAAmB,GAAGxE,aAAa,CAACF,IAAI,0BAAE8D,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBS,IAAI,CAAS;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAIxB,KAAK,2CAAoCuB,IAAI,EAAG;IAC5D;IAEA,MAAME,WAA+B,GAAG1E,aAAa,CACnDH,gBAAgB,0BAChB+D,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBY,KAAK,CACnB;IAER,MAAMC,WAAW,GAAG5C,MAAM,yBAAC4B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBc,gBAAgB,CAAE;IACjE,IAAIC,SAAS,GAAG9C,MAAM,yBAAC4B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBgB,qBAAqB,CAAE;IAElE,IAAI,CAACnB,QAAQ,CAACQ,SAAS,EAAE;MAAA;MACvBU,SAAS,GAAGE,IAAI,CAACC,GAAG,CAClB,IAAI,CAACzE,IAAI,CAAC4C,IAAI,GAAGwB,WAAW,EAC5B5C,MAAM,yBAAC4B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBgB,qBAAqB,CAAC,CAClD;IACH;IAEA,MAAMG,OAAuB,GAAG;MAC9BV,IAAI;MACJW,SAAS,EAAEb,KAAK,CAACa,SAAS;MAC1BC,SAAS,EAAEd,KAAK,CAACc,SAAS;MAC1BV,WAAW;MACXW,MAAM,EAAEf,KAAK;MACbgB,SAAS,0BAAE1B,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBwB,UAAU;MACzCC,UAAU,EAAE;IACd,CAAC;IAED,IAAIA,UAAU;IAEd,MAAMC,oBAAoB,GAAG7B,QAAQ,aAARA,QAAQ,+CAARA,QAAQ,CAAEG,SAAS,yDAAnB,qBAAqB2B,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAG3D,MAAM,CAACyD,oBAAoB,CAAC;MAErDD,UAAU,GAAG,MAAM,IAAI,CAACI,aAAa,CAACD,gBAAgB,EAAET,OAAO,EAAEN,WAAW,CAAC;IAC/E;IAEAY,UAAU,GAAG,uBAAAN,OAAO,CAACM,UAAU,gDAAlB,oBAAoB5D,MAAM,GAAGsD,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMK,QAAQ,GAAG,MAAM,IAAI,CAACrF,IAAI,CAACsC,IAAI,CAAC8B,WAAW,EAAEE,SAAS,CAAC;IAC7D,OAAO,MAAM5E,eAAe,CAAC2F,QAAQ,EAAE;MAAC,GAAGX,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;;EASA,MAAMI,aAAa,CACjBH,oBAA4B,EAC5BP,OAAuB,EACvBN,WAAmB,EACA;IACnB,IAAIa,oBAAoB,KAAK,CAAC,EAAE;;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGd,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACzE,IAAI,CAAC4C,IAAI,GAAGqC,oBAAoB,EACrC,IAAI,CAAChF,KAAK,CAACJ,qBAAqB,CACjC;IACD,MAAMwF,QAAQ,GAAG,MAAM,IAAI,CAACrF,IAAI,CAACsC,IAAI,CAAC2C,oBAAoB,EAAEK,cAAc,CAAC;IAE3E,MAAMC,MAAM,GAAG;MAAClD,MAAM,EAAEgD,QAAQ;MAAEG,MAAM,EAAE,CAAC;MAAE5C,IAAI,EAAEyC,QAAQ,CAACjE;IAAM,CAAC;IACnE,MAAMqE,WAAW,GAAG,MAAM9F,UAAU,CAAC4F,MAAM,EAAEb,OAAO,CAAC;IAErD,OAAOe,WAAW,CAACT,UAAU;EAC/B;AACF"}
1	+ {"version":3,"file":"parquet-reader.js","names":["ParquetSchema","decodeSchema","materializeRecords","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","DEFAULT_PROPS","defaultDictionarySize","ParquetReader","constructor","file","props","_defineProperty","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","columnList","map","x","Array","isArray","metadata","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","num_children","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","buffer","read","magic","toString","Error","concat","trailerLen","trailerBuf","size","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_options$dictionary","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","_colChunk$meta_data6","Math","min","options","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","decodedPage"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRecords} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n ParquetBuffer,\n ParquetCompression,\n ParquetData,\n PrimitiveType,\n ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader /\nexport type ParquetIterationProps = {\n /* Filter allowing some columns to be dropped /\n columnList?: string[] \| string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n defaultDictionarySize: 1e6\n};\n\n/\n The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n /\nexport class ParquetReader {\n props: Required<ParquetReaderProps>;\n file: ReadableFile;\n metadata: Promise<FileMetaData> \| null = null;\n\n constructor(file: ReadableFile, props?: ParquetReaderProps) {\n this.file = file;\n this.props = {...DEFAULT_PROPS, ...props};\n }\n\n close(): void {\n // eslint-disable-next-line @typescript-eslint/no-floating-promises\n this.file.close();\n }\n\n // HIGH LEVEL METHODS\n\n /* Yield one row at a time /\n async rowIterator(props?: ParquetIterationProps) {\n for await (const rows of this.rowBatchIterator(props)) {\n // yield rows\n for (const row of rows) {\n yield row;\n }\n }\n }\n\n /* Yield one batch of rows at a time /\n async rowBatchIterator(props?: ParquetIterationProps) {\n const schema = await this.getSchema();\n for await (const rowGroup of this.rowGroupIterator(props)) {\n yield materializeRecords(schema, rowGroup);\n }\n }\n\n /** Iterate over the raw row groups /\n async rowGroupIterator(props?: ParquetIterationProps) {\n // Ensure strings are nested in arrays\n const columnList: string[][] = (props?.columnList \|\| []).map((x) =>\n Array.isArray(x) ? x : [x]\n );\n\n const metadata = await this.getFileMetadata();\n const schema = await this.getSchema();\n\n const rowGroupCount = metadata?.row_groups.length \|\| 0;\n\n for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n const rowGroup = await this.readRowGroup(\n schema,\n metadata.row_groups[rowGroupIndex],\n columnList\n );\n yield rowGroup;\n }\n }\n\n async getRowCount(): Promise<number> {\n const metadata = await this.getFileMetadata();\n return Number(metadata.num_rows);\n }\n\n async getSchema(): Promise<ParquetSchema> {\n const metadata = await this.getFileMetadata();\n const root = metadata.schema[0];\n const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n const schema = new ParquetSchema(schemaDefinition);\n return schema;\n }\n\n /*\n Returns the user (key/value) metadata for this file\n * In parquet this is not stored on the schema like it is in arrow\n /\n async getSchemaMetadata(): Promise<Record<string, string>> {\n const metadata = await this.getFileMetadata();\n const md: Record<string, string> = {};\n for (const kv of metadata.key_value_metadata!) {\n md[kv.key] = kv.value!;\n }\n return md;\n }\n\n async getFileMetadata(): Promise<FileMetaData> {\n if (!this.metadata) {\n await this.readHeader();\n this.metadata = this.readFooter();\n }\n return this.metadata;\n }\n\n // LOW LEVEL METHODS\n\n /* Metadata is stored in the footer /\n async readHeader(): Promise<void> {\n const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n const magic = buffer.toString();\n switch (magic) {\n case PARQUET_MAGIC:\n break;\n case PARQUET_MAGIC_ENCRYPTED:\n throw new Error('Encrypted parquet file not supported');\n default:\n throw new Error(`Invalid parquet file (magic=${magic})`);\n }\n }\n\n /* Metadata is stored in the footer /\n async readFooter(): Promise<FileMetaData> {\n const trailerLen = PARQUET_MAGIC.length + 4;\n const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n const magic = trailerBuf.slice(4).toString();\n if (magic !== PARQUET_MAGIC) {\n throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n }\n\n const metadataSize = trailerBuf.readUInt32LE(0);\n const metadataOffset = this.file.size - metadataSize - trailerLen;\n if (metadataOffset < PARQUET_MAGIC.length) {\n throw new Error(`Invalid metadata size ${metadataOffset}`);\n }\n\n const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n // let metadata = new parquet_thrift.FileMetaData();\n // parquet_util.decodeThrift(metadata, metadataBuf);\n const {metadata} = decodeFileMetadata(metadataBuf);\n return metadata;\n }\n\n /* Data is stored in row groups (similar to Apache Arrow record batches) /\n async readRowGroup(\n schema: ParquetSchema,\n rowGroup: RowGroup,\n columnList: string[][]\n ): Promise<ParquetBuffer> {\n const buffer: ParquetBuffer = {\n rowCount: Number(rowGroup.num_rows),\n columnData: {}\n };\n for (const colChunk of rowGroup.columns) {\n const colMetadata = colChunk.meta_data;\n const colKey = colMetadata?.path_in_schema;\n if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n continue; // eslint-disable-line no-continue\n }\n buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n }\n return buffer;\n }\n\n /\n Each row group contains column chunks for all the columns.\n /\n async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n throw new Error('external references are not supported');\n }\n\n const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n if (type !== field.primitiveType) {\n throw new Error(`chunk type not matching schema: ${type}`);\n }\n\n const compression: ParquetCompression = getThriftEnum(\n CompressionCodec,\n colChunk.meta_data?.codec!\n ) as any;\n\n const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n if (!colChunk.file_path) {\n pagesSize = Math.min(\n this.file.size - pagesOffset,\n Number(colChunk.meta_data?.total_compressed_size)\n );\n }\n\n const options: ParquetOptions = {\n type,\n rLevelMax: field.rLevelMax,\n dLevelMax: field.dLevelMax,\n compression,\n column: field,\n numValues: colChunk.meta_data?.num_values,\n dictionary: []\n };\n\n let dictionary;\n\n const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n if (dictionaryPageOffset) {\n const dictionaryOffset = Number(dictionaryPageOffset);\n // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n }\n\n dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n return await decodeDataPages(pagesBuf, {...options, dictionary});\n }\n\n /\n Getting dictionary for allows to flatten values by indices.\n * @param dictionaryPageOffset\n * @param options\n * @param pagesOffset\n * @returns\n */\n async getDictionary(\n dictionaryPageOffset: number,\n options: ParquetOptions,\n pagesOffset: number\n ): Promise<string[]> {\n if (dictionaryPageOffset === 0) {\n // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n // In this case we are working with parquet-mr files format. Problem is described below:\n // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n // We need to get dictionary page from column chunk if it exists.\n // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n return [];\n }\n\n const dictionarySize = Math.min(\n this.file.size - dictionaryPageOffset,\n this.props.defaultDictionarySize\n );\n const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n const decodedPage = await decodePage(cursor, options);\n\n return decodedPage.dictionary!;\n }\n}\n"],"mappings":";AAGA,SAAQA,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,YAAY,QAAO,YAAY;AACvC,SAAQC,kBAAkB,QAAO,iBAAiB;AAElD,SAAQC,aAAa,EAAEC,uBAAuB,QAAO,iBAAiB;AACtE,SAAqBC,gBAAgB,EAA0BC,IAAI,QAAO,mBAAmB;AAQ7F,SAAQC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY,QAAO,qBAAqB;AACnF,SAAQC,eAAe,EAAEC,UAAU,QAAO,YAAY;AAYtD,MAAMC,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;AAQD,OAAO,MAAMC,aAAa,CAAC;EAKzBC,WAAWA,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAAC,eAAA;IAAAA,eAAA;IAAAA,eAAA,mBAFnB,IAAI;IAG3C,IAAI,CAACF,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGL,aAAa;MAAE,GAAGK;IAAK,CAAC;EAC3C;EAEAE,KAAKA,CAAA,EAAS;IAEZ,IAAI,CAACH,IAAI,CAACG,KAAK,EAAE;EACnB;EAKA,OAAOC,WAAWA,CAACH,KAA6B,EAAE;IAChD,WAAW,MAAMI,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACL,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMM,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;EAGA,OAAOD,gBAAgBA,CAACL,KAA6B,EAAE;IACrD,MAAMO,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,EAAE;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACV,KAAK,CAAC,EAAE;MACzD,MAAMf,kBAAkB,CAACsB,MAAM,EAAEE,QAAQ,CAAC;IAC5C;EACF;EAGA,OAAOC,gBAAgBA,CAACV,KAA6B,EAAE;IAErD,MAAMW,UAAsB,GAAG,CAAC,CAAAX,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEW,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC,CAC3B;IAED,MAAMG,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMV,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,EAAE;IAErC,MAAMU,aAAa,GAAG,CAAAF,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEG,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMZ,QAAQ,GAAG,MAAM,IAAI,CAACa,YAAY,CACtCf,MAAM,EACNS,QAAQ,CAACG,UAAU,CAACE,aAAa,CAAC,EAClCV,UAAU,CACX;MACD,MAAMF,QAAQ;IAChB;EACF;EAEA,MAAMc,WAAWA,CAAA,EAAoB;IACnC,MAAMP,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,OAAOO,MAAM,CAACR,QAAQ,CAACS,QAAQ,CAAC;EAClC;EAEA,MAAMjB,SAASA,CAAA,EAA2B;IACxC,MAAMQ,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMS,IAAI,GAAGV,QAAQ,CAACT,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEoB;IAAgB,CAAC,GAAG3C,YAAY,CAACgC,QAAQ,CAACT,MAAM,EAAE,CAAC,EAAEmB,IAAI,CAACE,YAAY,CAAE;IACvF,MAAMrB,MAAM,GAAG,IAAIxB,aAAa,CAAC4C,gBAAgB,CAAC;IAClD,OAAOpB,MAAM;EACf;EAMA,MAAMsB,iBAAiBA,CAAA,EAAoC;IACzD,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,EAAE;IAC7C,MAAMa,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAIf,QAAQ,CAACgB,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMb,eAAeA,CAAA,EAA0B;IAC7C,IAAI,CAAC,IAAI,CAACD,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACmB,UAAU,EAAE;MACvB,IAAI,CAACnB,QAAQ,GAAG,IAAI,CAACoB,UAAU,EAAE;IACnC;IACA,OAAO,IAAI,CAACpB,QAAQ;EACtB;EAKA,MAAMmB,UAAUA,CAAA,EAAkB;IAChC,MAAME,MAAM,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAAC,CAAC,EAAEpD,aAAa,CAACkC,MAAM,CAAC;IAC5D,MAAMmB,KAAK,GAAGF,MAAM,CAACG,QAAQ,EAAE;IAC/B,QAAQD,KAAK;MACX,KAAKrD,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAIsD,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,gCAAAC,MAAA,CAAgCH,KAAK,OAAI;IAAC;EAE/D;EAGA,MAAMH,UAAUA,CAAA,EAA0B;IACxC,MAAMO,UAAU,GAAGzD,aAAa,CAACkC,MAAM,GAAG,CAAC;IAC3C,MAAMwB,UAAU,GAAG,MAAM,IAAI,CAAC7C,IAAI,CAACuC,IAAI,CAAC,IAAI,CAACvC,IAAI,CAAC8C,IAAI,GAAGF,UAAU,EAAEA,UAAU,CAAC;IAEhF,MAAMJ,KAAK,GAAGK,UAAU,CAACE,KAAK,CAAC,CAAC,CAAC,CAACN,QAAQ,EAAE;IAC5C,IAAID,KAAK,KAAKrD,aAAa,EAAE;MAC3B,MAAM,IAAIuD,KAAK,sCAAAC,MAAA,CAAqCH,KAAK,OAAI;IAC/D;IAEA,MAAMQ,YAAY,GAAGH,UAAU,CAACI,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAAClD,IAAI,CAAC8C,IAAI,GAAGE,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAG/D,aAAa,CAACkC,MAAM,EAAE;MACzC,MAAM,IAAIqB,KAAK,0BAAAC,MAAA,CAA0BO,cAAc,EAAG;IAC5D;IAEA,MAAMC,WAAW,GAAG,MAAM,IAAI,CAACnD,IAAI,CAACuC,IAAI,CAACW,cAAc,EAAEF,YAAY,CAAC;IAGtE,MAAM;MAAC/B;IAAQ,CAAC,GAAG1B,kBAAkB,CAAC4D,WAAW,CAAC;IAClD,OAAOlC,QAAQ;EACjB;EAGA,MAAMM,YAAYA,CAChBf,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB,EACE;IACxB,MAAM0B,MAAqB,GAAG;MAC5Bc,QAAQ,EAAE3B,MAAM,CAACf,QAAQ,CAACgB,QAAQ,CAAC;MACnC2B,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAI5C,QAAQ,CAAC6C,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAI/C,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI5B,YAAY,CAACmB,UAAU,EAAE8C,MAAM,CAAE,GAAG,CAAC,EAAE;QAClE;MACF;MACApB,MAAM,CAACe,UAAU,CAACK,MAAM,CAAEE,IAAI,EAAE,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACrD,MAAM,EAAE8C,QAAQ,CAAC;IAClF;IACA,OAAOhB,MAAM;EACf;EAKA,MAAMuB,eAAeA,CAACrD,MAAqB,EAAE8C,QAAqB,EAAwB;IAAA,IAAAQ,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;IACxF,IAAIf,QAAQ,CAACgB,SAAS,KAAKC,SAAS,IAAIjB,QAAQ,CAACgB,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAI5B,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAM8B,KAAK,GAAGhE,MAAM,CAACiE,SAAS,EAAAX,mBAAA,GAACR,QAAQ,CAACG,SAAS,cAAAK,mBAAA,uBAAlBA,mBAAA,CAAoBH,cAAc,CAAE;IACnE,MAAMe,IAAmB,GAAGlF,aAAa,CAACF,IAAI,GAAAyE,oBAAA,GAAET,QAAQ,CAACG,SAAS,cAAAM,oBAAA,uBAAlBA,oBAAA,CAAoBW,IAAI,CAAS;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAIjC,KAAK,oCAAAC,MAAA,CAAoC+B,IAAI,EAAG;IAC5D;IAEA,MAAME,WAA+B,GAAGpF,aAAa,CACnDH,gBAAgB,GAAA2E,oBAAA,GAChBV,QAAQ,CAACG,SAAS,cAAAO,oBAAA,uBAAlBA,oBAAA,CAAoBa,KAAK,CACnB;IAER,MAAMC,WAAW,GAAGrD,MAAM,EAAAwC,oBAAA,GAACX,QAAQ,CAACG,SAAS,cAAAQ,oBAAA,uBAAlBA,oBAAA,CAAoBc,gBAAgB,CAAE;IACjE,IAAIC,SAAS,GAAGvD,MAAM,EAAAyC,oBAAA,GAACZ,QAAQ,CAACG,SAAS,cAAAS,oBAAA,uBAAlBA,oBAAA,CAAoBe,qBAAqB,CAAE;IAElE,IAAI,CAAC3B,QAAQ,CAACgB,SAAS,EAAE;MAAA,IAAAY,oBAAA;MACvBF,SAAS,GAAGG,IAAI,CAACC,GAAG,CAClB,IAAI,CAACpF,IAAI,CAAC8C,IAAI,GAAGgC,WAAW,EAC5BrD,MAAM,EAAAyD,oBAAA,GAAC5B,QAAQ,CAACG,SAAS,cAAAyB,oBAAA,uBAAlBA,oBAAA,CAAoBD,qBAAqB,CAAC,CAClD;IACH;IAEA,MAAMI,OAAuB,GAAG;MAC9BX,IAAI;MACJY,SAAS,EAAEd,KAAK,CAACc,SAAS;MAC1BC,SAAS,EAAEf,KAAK,CAACe,SAAS;MAC1BX,WAAW;MACXY,MAAM,EAAEhB,KAAK;MACbiB,SAAS,GAAAtB,oBAAA,GAAEb,QAAQ,CAACG,SAAS,cAAAU,oBAAA,uBAAlBA,oBAAA,CAAoBuB,UAAU;MACzCC,UAAU,EAAE;IACd,CAAC;IAED,IAAIA,UAAU;IAEd,MAAMC,oBAAoB,GAAGtC,QAAQ,aAARA,QAAQ,wBAAAc,oBAAA,GAARd,QAAQ,CAAEG,SAAS,cAAAW,oBAAA,uBAAnBA,oBAAA,CAAqByB,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAGrE,MAAM,CAACmE,oBAAoB,CAAC;MAErDD,UAAU,GAAG,MAAM,IAAI,CAACI,aAAa,CAACD,gBAAgB,EAAET,OAAO,EAAEP,WAAW,CAAC;IAC/E;IAEAa,UAAU,GAAG,CAAAtB,mBAAA,GAAAgB,OAAO,CAACM,UAAU,cAAAtB,mBAAA,eAAlBA,mBAAA,CAAoBhD,MAAM,GAAGgE,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMK,QAAQ,GAAG,MAAM,IAAI,CAAChG,IAAI,CAACuC,IAAI,CAACuC,WAAW,EAAEE,SAAS,CAAC;IAC7D,OAAO,MAAMtF,eAAe,CAACsG,QAAQ,EAAE;MAAC,GAAGX,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;EASA,MAAMI,aAAaA,CACjBH,oBAA4B,EAC5BP,OAAuB,EACvBP,WAAmB,EACA;IACnB,IAAIc,oBAAoB,KAAK,CAAC,EAAE;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGd,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACpF,IAAI,CAAC8C,IAAI,GAAG8C,oBAAoB,EACrC,IAAI,CAAC3F,KAAK,CAACJ,qBAAqB,CACjC;IACD,MAAMmG,QAAQ,GAAG,MAAM,IAAI,CAAChG,IAAI,CAACuC,IAAI,CAACqD,oBAAoB,EAAEK,cAAc,CAAC;IAE3E,MAAMC,MAAM,GAAG;MAAC5D,MAAM,EAAE0D,QAAQ;MAAEG,MAAM,EAAE,CAAC;MAAErD,IAAI,EAAEkD,QAAQ,CAAC3E;IAAM,CAAC;IACnE,MAAM+E,WAAW,GAAG,MAAMzG,UAAU,CAACuG,MAAM,EAAEb,OAAO,CAAC;IAErD,OAAOe,WAAW,CAACT,UAAU;EAC/B;AACF"}

package/dist/esm/parquetjs/schema/declare.js CHANGED Viewed

@@ -1,7 +1,5 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 export class ParquetBuffer {
   constructor() {
     let rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
     let columnData = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};

package/dist/esm/parquetjs/schema/declare.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"declare.js","names":["ParquetBuffer","constructor","rowCount","columnData"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' \| 'RLE' \| 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n \| 'UNCOMPRESSED'\n \| 'GZIP'\n \| 'SNAPPY'\n \| 'LZO'\n \| 'BROTLI'\n \| 'LZ4'\n \| 'LZ4_RAW'\n \| 'ZSTD';\nexport type RepetitionType = 'REQUIRED' \| 'OPTIONAL' \| 'REPEATED';\nexport type ParquetType = PrimitiveType \| OriginalType;\n\n/*\n Physical type\n /\nexport type PrimitiveType =\n // Base Types\n \| 'BOOLEAN' // 0\n \| 'INT32' // 1\n \| 'INT64' // 2\n \| 'INT96' // 3\n \| 'FLOAT' // 4\n \| 'DOUBLE' // 5\n \| 'BYTE_ARRAY' // 6,\n \| 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/\n Logical type\n /\nexport type OriginalType =\n // Converted Types\n \| 'UTF8' // 0\n // \| 'MAP' // 1\n // \| 'MAP_KEY_VALUE' // 2\n // \| 'LIST' // 3\n // \| 'ENUM' // 4\n // \| 'DECIMAL' // 5\n \| 'DECIMAL_INT32' // 5\n \| 'DECIMAL_INT64' // 5\n \| 'DECIMAL_BYTE_ARRAY' // 5\n \| 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n \| 'DATE' // 6\n \| 'TIME_MILLIS' // 7\n \| 'TIME_MICROS' // 8\n \| 'TIMESTAMP_MILLIS' // 9\n \| 'TIMESTAMP_MICROS' // 10\n \| 'UINT_8' // 11\n \| 'UINT_16' // 12\n \| 'UINT_32' // 13\n \| 'UINT_64' // 14\n \| 'INT_8' // 15\n \| 'INT_16' // 16\n \| 'INT_32' // 17\n \| 'INT_64' // 18\n \| 'JSON' // 19\n \| 'BSON' // 20\n \| 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n type?: ParquetType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n optional?: boolean;\n repeated?: boolean;\n fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n name: string;\n path: string[];\n key: string;\n primitiveType?: PrimitiveType;\n originalType?: OriginalType;\n repetitionType: RepetitionType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n rLevelMax: number;\n dLevelMax: number;\n isNested?: boolean;\n fieldCount?: number;\n fields?: Record<string, ParquetField>;\n}\n\n/* @todo better name, this is an internal type? /\nexport interface ParquetOptions {\n type: ParquetType;\n rLevelMax: number;\n dLevelMax: number;\n compression: ParquetCompression;\n column: ParquetField;\n numValues?: Int64;\n dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n dlevels: number[];\n rlevels: number[];\n /* Actual column chunks /\n values: any[]; // ArrayLike<any>;\n count: number;\n dictionary?: ParquetDictionary;\n /* The \"raw\" page header from the file /\n pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n [key: string]: any;\n}\n\n/* @\n * Holds data for one row group (column chunks) /\nexport class ParquetBuffer {\n /* Number of rows in this page /\n rowCount: number;\n\n columnData: Record<string, ParquetData>;\n constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n this.rowCount = rowCount;\n this.columnData = columnData;\n }\n}\n\n/* Holds the data for one column chunk */\nexport interface ParquetData {\n dlevels: number[];\n rlevels: number[];\n values: any[];\n count: number;\n pageHeaders: PageHeader[];\n}\n"],"mappings":";;AAgIA,OAAO,MAAMA,aAAa,CAAC;;EAKzBC,~~WAAW~~,~~GAAqE~~;IAAA,IAApEC,QAAgB,~~uEAAG~~,CAAC;IAAA,~~IAAEC~~,UAAuC,~~uEAAG~~,CAAC,CAAC;~~IAAA~~;~~IAAA~~;IAC5E,IAAI,~~CAACD~~,QAAQ,GAAGA,QAAQ;IACxB,IAAI,~~CAACC~~,UAAU,GAAGA,UAAU;EAC9B;AACF"}
1	+ {"version":3,"file":"declare.js","names":["ParquetBuffer","constructor","rowCount","arguments","length","undefined","columnData","_defineProperty"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' \| 'RLE' \| 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n \| 'UNCOMPRESSED'\n \| 'GZIP'\n \| 'SNAPPY'\n \| 'LZO'\n \| 'BROTLI'\n \| 'LZ4'\n \| 'LZ4_RAW'\n \| 'ZSTD';\nexport type RepetitionType = 'REQUIRED' \| 'OPTIONAL' \| 'REPEATED';\nexport type ParquetType = PrimitiveType \| OriginalType;\n\n/*\n Physical type\n /\nexport type PrimitiveType =\n // Base Types\n \| 'BOOLEAN' // 0\n \| 'INT32' // 1\n \| 'INT64' // 2\n \| 'INT96' // 3\n \| 'FLOAT' // 4\n \| 'DOUBLE' // 5\n \| 'BYTE_ARRAY' // 6,\n \| 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/\n Logical type\n /\nexport type OriginalType =\n // Converted Types\n \| 'UTF8' // 0\n // \| 'MAP' // 1\n // \| 'MAP_KEY_VALUE' // 2\n // \| 'LIST' // 3\n // \| 'ENUM' // 4\n // \| 'DECIMAL' // 5\n \| 'DECIMAL_INT32' // 5\n \| 'DECIMAL_INT64' // 5\n \| 'DECIMAL_BYTE_ARRAY' // 5\n \| 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n \| 'DATE' // 6\n \| 'TIME_MILLIS' // 7\n \| 'TIME_MICROS' // 8\n \| 'TIMESTAMP_MILLIS' // 9\n \| 'TIMESTAMP_MICROS' // 10\n \| 'UINT_8' // 11\n \| 'UINT_16' // 12\n \| 'UINT_32' // 13\n \| 'UINT_64' // 14\n \| 'INT_8' // 15\n \| 'INT_16' // 16\n \| 'INT_32' // 17\n \| 'INT_64' // 18\n \| 'JSON' // 19\n \| 'BSON' // 20\n \| 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n type?: ParquetType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n optional?: boolean;\n repeated?: boolean;\n fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n name: string;\n path: string[];\n key: string;\n primitiveType?: PrimitiveType;\n originalType?: OriginalType;\n repetitionType: RepetitionType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n rLevelMax: number;\n dLevelMax: number;\n isNested?: boolean;\n fieldCount?: number;\n fields?: Record<string, ParquetField>;\n}\n\n/* @todo better name, this is an internal type? /\nexport interface ParquetOptions {\n type: ParquetType;\n rLevelMax: number;\n dLevelMax: number;\n compression: ParquetCompression;\n column: ParquetField;\n numValues?: Int64;\n dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n dlevels: number[];\n rlevels: number[];\n /* Actual column chunks /\n values: any[]; // ArrayLike<any>;\n count: number;\n dictionary?: ParquetDictionary;\n /* The \"raw\" page header from the file /\n pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n [key: string]: any;\n}\n\n/* @\n * Holds data for one row group (column chunks) /\nexport class ParquetBuffer {\n /* Number of rows in this page /\n rowCount: number;\n\n columnData: Record<string, ParquetData>;\n constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n this.rowCount = rowCount;\n this.columnData = columnData;\n }\n}\n\n/* Holds the data for one column chunk */\nexport interface ParquetData {\n dlevels: number[];\n rlevels: number[];\n values: any[];\n count: number;\n pageHeaders: PageHeader[];\n}\n"],"mappings":";AAgIA,OAAO,MAAMA,aAAa,CAAC;EAKzBC,WAAWA,CAAA,EAAqE;IAAA,IAApEC,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;IAAA,IAAEG,UAAuC,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;IAAAI,eAAA;IAAAA,eAAA;IAC5E,IAAI,CAACL,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;EAC9B;AACF"}

package/dist/esm/parquetjs/schema/schema.js CHANGED Viewed

@@ -1,10 +1,8 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 import { PARQUET_CODECS } from '../codecs';
 import { PARQUET_COMPRESSION_METHODS } from '../compression';
 import { materializeRecords, shredBuffer, shredRecord } from './shred';
 import { PARQUET_LOGICAL_TYPES } from './types';
 export class ParquetSchema {
   constructor(schema) {
     _defineProperty(this, "schema", void 0);
@@ -14,21 +12,18 @@ export class ParquetSchema {
     this.fields = buildFields(schema, 0, 0, []);
     this.fieldList = listFields(this.fields);
   }
   findField(path) {
     if (typeof path === 'string') {
       path = path.split(',');
     } else {
       path = path.slice(0);
     }
     let n = this.fields;
     for (; path.length > 1; path.shift()) {
       n = n[path[0]].fields;
     }
     return n[path[0]];
   }
   findFieldBranch(path) {
     if (typeof path === 'string') {
       path = path.split(',');
@@ -68,12 +63,10 @@ function setCompress(schema, type) {
     }
   }
 }
 function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
   const fieldList = {};
   for (const name in schema) {
     const opts = schema[name];
     const required = !opts.optional;
     const repeated = Boolean(opts.repeated);
     let rLevelMax = rLevelParentMax;
@@ -88,7 +81,6 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
       rLevelMax++;
       if (required) dLevelMax++;
     }
     if (opts.fields) {
       const cpath = path.concat([name]);
       fieldList[name] = {
@@ -104,7 +96,6 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
       };
       continue;
     }
     const typeDef = PARQUET_LOGICAL_TYPES[opts.type];
     if (!typeDef) {
       throw new Error("invalid parquet type: ".concat(opts.type));
@@ -117,7 +108,6 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
     if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {
       throw new Error("unsupported compression method: ".concat(opts.compression));
     }
     const cpath = path.concat([name]);
     fieldList[name] = {
       name,

package/dist/esm/parquetjs/schema/schema.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"schema.js","names":["PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","materializeRecords","shredBuffer","shredRecord","PARQUET_LOGICAL_TYPES","ParquetSchema","constructor","schema","fields","buildFields","fieldList","listFields","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","record","buffer","compress","type","setCompress","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","Error","encoding","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n FieldDefinition,\n ParquetBuffer,\n ParquetCompression,\n ParquetField,\n ParquetRecord,\n RepetitionType,\n SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/*\n A parquet file schema\n /\nexport class ParquetSchema {\n public schema: Record<string, FieldDefinition>;\n public fields: Record<string, ParquetField>;\n public fieldList: ParquetField[];\n\n /\n Create a new schema from a JSON schema definition\n /\n constructor(schema: SchemaDefinition) {\n this.schema = schema;\n this.fields = buildFields(schema, 0, 0, []);\n this.fieldList = listFields(this.fields);\n }\n\n /\n Retrieve a field definition\n /\n findField(path: string \| string[]): ParquetField {\n if (typeof path === 'string') {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.split(',');\n } else {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.slice(0); // clone array\n }\n\n let n = this.fields;\n for (; path.length > 1; path.shift()) {\n n = n[path[0]].fields as Record<string, ParquetField>;\n }\n\n return n[path[0]];\n }\n\n /\n Retrieve a field definition and all the field's ancestors\n /\n findFieldBranch(path: string \| string[]): ParquetField[] {\n if (typeof path === 'string') {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.split(',');\n }\n const branch: ParquetField[] = [];\n let n = this.fields;\n for (; path.length > 0; path.shift()) {\n branch.push(n[path[0]]);\n if (path.length > 1) {\n n = n[path[0]].fields as Record<string, ParquetField>;\n }\n }\n return branch;\n }\n\n shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n shredRecord(this, record, buffer);\n }\n\n materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n return materializeRecords(this, buffer);\n }\n\n compress(type: ParquetCompression): this {\n setCompress(this.schema, type);\n setCompress(this.fields, type);\n return this;\n }\n\n buffer(): ParquetBuffer {\n return shredBuffer(this);\n }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n for (const name in schema) {\n const node = schema[name];\n if (node.fields) {\n setCompress(node.fields, type);\n } else {\n node.compression = type;\n }\n }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n schema: SchemaDefinition,\n rLevelParentMax: number,\n dLevelParentMax: number,\n path: string[]\n): Record<string, ParquetField> {\n const fieldList: Record<string, ParquetField> = {};\n\n for (const name in schema) {\n const opts = schema[name];\n\n / field repetition type /\n const required = !opts.optional;\n const repeated = Boolean(opts.repeated);\n let rLevelMax = rLevelParentMax;\n let dLevelMax = dLevelParentMax;\n\n let repetitionType: RepetitionType = 'REQUIRED';\n if (!required) {\n repetitionType = 'OPTIONAL';\n dLevelMax++;\n }\n if (repeated) {\n repetitionType = 'REPEATED';\n rLevelMax++;\n if (required) dLevelMax++;\n }\n\n / nested field /\n if (opts.fields) {\n const cpath = path.concat([name]);\n fieldList[name] = {\n name,\n path: cpath,\n key: cpath.join(),\n repetitionType,\n rLevelMax,\n dLevelMax,\n isNested: true,\n fieldCount: Object.keys(opts.fields).length,\n fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n };\n continue; // eslint-disable-line no-continue\n }\n\n const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n if (!typeDef) {\n throw new Error(`invalid parquet type: ${opts.type}`);\n }\n\n opts.encoding = opts.encoding \|\| 'PLAIN';\n if (!(opts.encoding in PARQUET_CODECS)) {\n throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n }\n\n opts.compression = opts.compression \|\| 'UNCOMPRESSED';\n if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n throw new Error(`unsupported compression method: ${opts.compression}`);\n }\n\n / add to schema */\n const cpath = path.concat([name]);\n fieldList[name] = {\n name,\n primitiveType: typeDef.primitiveType,\n originalType: typeDef.originalType,\n path: cpath,\n key: cpath.join(),\n repetitionType,\n encoding: opts.encoding,\n compression: opts.compression,\n typeLength: opts.typeLength \|\| typeDef.typeLength,\n presision: opts.presision,\n scale: opts.scale,\n rLevelMax,\n dLevelMax\n };\n }\n return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n let list: ParquetField[] = [];\n for (const k in fields) {\n list.push(fields[k]);\n if (fields[k].isNested) {\n list = list.concat(listFields(fields[k].fields!));\n }\n }\n return list;\n}\n"],"mappings":";;AAEA,SAAQA,cAAc,QAAO,WAAW;AACxC,SAAQC,2BAA2B,QAAO,gBAAgB;AAU1D,SAAQC,kBAAkB,EAAEC,WAAW,EAAEC,WAAW,QAAO,SAAS;AACpE,SAAQC,qBAAqB,QAAO,SAAS;;AAK7C,OAAO,MAAMC,aAAa,CAAC;EAQzBC,WAAW,CAACC,MAAwB,EAAE;IAAA;IAAA;IAAA;IACpC,IAAI,CAACA,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACC,MAAM,GAAGC,WAAW,CAACF,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACG,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;;EAKAI,SAAS,CAACC,IAAuB,EAAgB;IAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;MAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;IACxB,CAAC,MAAM;MAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;IACtB;;IAEA,IAAIC,CAAC,GAAG,IAAI,CAACR,MAAM;IACnB,OAAOK,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,EAAE,EAAE;MACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACL,MAAsC;IACvD;IAEA,OAAOQ,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;EACnB;;EAKAM,eAAe,CAACN,IAAuB,EAAkB;IACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;MAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;IACxB;IACA,MAAMM,MAAsB,GAAG,EAAE;IACjC,IAAIJ,CAAC,GAAG,IAAI,CAACR,MAAM;IACnB,OAAOK,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,EAAE,EAAE;MACpCE,MAAM,CAACC,IAAI,CAACL,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;MACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;QACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACL,MAAsC;MACvD;IACF;IACA,OAAOY,MAAM;EACf;EAEAjB,WAAW,CAACmB,MAAqB,EAAEC,MAAqB,EAAQ;IAC9DpB,WAAW,CAAC,IAAI,EAAEmB,MAAM,EAAEC,MAAM,CAAC;EACnC;EAEAtB,kBAAkB,CAACsB,MAAqB,EAAmB;IACzD,OAAOtB,kBAAkB,CAAC,IAAI,EAAEsB,MAAM,CAAC;EACzC;EAEAC,QAAQ,CAACC,IAAwB,EAAQ;IACvCC,WAAW,CAAC,IAAI,CAACnB,MAAM,EAAEkB,IAAI,CAAC;IAC9BC,WAAW,CAAC,IAAI,CAAClB,MAAM,EAAEiB,IAAI,CAAC;IAC9B,OAAO,IAAI;EACb;EAEAF,MAAM,GAAkB;IACtB,OAAOrB,WAAW,CAAC,IAAI,CAAC;EAC1B;AACF;AAEA,SAASwB,WAAW,CAACnB,MAAW,EAAEkB,IAAwB,EAAE;EAC1D,KAAK,MAAME,IAAI,IAAIpB,MAAM,EAAE;IACzB,MAAMqB,IAAI,GAAGrB,MAAM,CAACoB,IAAI,CAAC;IACzB,IAAIC,IAAI,CAACpB,MAAM,EAAE;MACfkB,WAAW,CAACE,IAAI,CAACpB,MAAM,EAAEiB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLG,IAAI,CAACC,WAAW,GAAGJ,IAAI;IACzB;EACF;AACF;;AAGA,SAAShB,WAAW,CAClBF,MAAwB,EACxBuB,eAAuB,EACvBC,eAAuB,EACvBlB,IAAc,EACgB;EAC9B,MAAMH,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,MAAMiB,IAAI,IAAIpB,MAAM,EAAE;IACzB,MAAMyB,IAAI,GAAGzB,MAAM,CAACoB,IAAI,CAAC;;IAGzB,MAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,MAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;;IAGA,IAAIN,IAAI,CAACxB,MAAM,EAAE;MACf,MAAMgC,KAAK,GAAG3B,IAAI,CAAC4B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;QAChBA,IAAI;QACJd,IAAI,EAAE2B,KAAK;QACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,EAAE;QACjBJ,cAAc;QACdF,SAAS;QACTC,SAAS;QACTM,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACf,IAAI,CAACxB,MAAM,CAAC,CAACS,MAAM;QAC3CT,MAAM,EAAEC,WAAW,CAACuB,IAAI,CAACxB,MAAM,EAAE6B,SAAS,EAAEC,SAAS,EAAEE,KAAK;MAC9D,CAAC;MACD;IACF;;IAEA,MAAMQ,OAAY,GAAG5C,qBAAqB,CAAC4B,IAAI,CAACP,IAAI,CAAE;IACtD,IAAI,CAACuB,OAAO,EAAE;MACZ,MAAM,IAAIC,KAAK,iCAA0BjB,IAAI,CAACP,IAAI,EAAG;IACvD;IAEAO,IAAI,CAACkB,QAAQ,GAAGlB,IAAI,CAACkB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAElB,IAAI,CAACkB,QAAQ,IAAInD,cAAc,CAAC,EAAE;MACtC,MAAM,IAAIkD,KAAK,yCAAkCjB,IAAI,CAACkB,QAAQ,EAAG;IACnE;IAEAlB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAI7B,2BAA2B,CAAC,EAAE;MACtD,MAAM,IAAIiD,KAAK,2CAAoCjB,IAAI,CAACH,WAAW,EAAG;IACxE;;IAGA,MAAMW,KAAK,GAAG3B,IAAI,CAAC4B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;MAChBA,IAAI;MACJwB,aAAa,EAAEH,OAAO,CAACG,aAAa;MACpCC,YAAY,EAAEJ,OAAO,CAACI,YAAY;MAClCvC,IAAI,EAAE2B,KAAK;MACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,EAAE;MACjBJ,cAAc;MACdW,QAAQ,EAAElB,IAAI,CAACkB,QAAQ;MACvBrB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7BwB,UAAU,EAAErB,IAAI,CAACqB,UAAU,IAAIL,OAAO,CAACK,UAAU;MACjDC,SAAS,EAAEtB,IAAI,CAACsB,SAAS;MACzBC,KAAK,EAAEvB,IAAI,CAACuB,KAAK;MACjBlB,SAAS;MACTC;IACF,CAAC;EACH;EACA,OAAO5B,SAAS;AAClB;AAEA,SAASC,UAAU,CAACH,MAAoC,EAAkB;EACxE,IAAIgD,IAAoB,GAAG,EAAE;EAC7B,KAAK,MAAMC,CAAC,IAAIjD,MAAM,EAAE;IACtBgD,IAAI,CAACnC,IAAI,CAACb,MAAM,CAACiD,CAAC,CAAC,CAAC;IACpB,IAAIjD,MAAM,CAACiD,CAAC,CAAC,CAACb,QAAQ,EAAE;MACtBY,IAAI,GAAGA,IAAI,CAACf,MAAM,CAAC9B,UAAU,CAACH,MAAM,CAACiD,CAAC,CAAC,CAACjD,MAAM,CAAE,CAAC;IACnD;EACF;EACA,OAAOgD,IAAI;AACb"}
1	+ {"version":3,"file":"schema.js","names":["PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","materializeRecords","shredBuffer","shredRecord","PARQUET_LOGICAL_TYPES","ParquetSchema","constructor","schema","_defineProperty","fields","buildFields","fieldList","listFields","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","record","buffer","compress","type","setCompress","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","Error","encoding","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n FieldDefinition,\n ParquetBuffer,\n ParquetCompression,\n ParquetField,\n ParquetRecord,\n RepetitionType,\n SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/*\n A parquet file schema\n /\nexport class ParquetSchema {\n public schema: Record<string, FieldDefinition>;\n public fields: Record<string, ParquetField>;\n public fieldList: ParquetField[];\n\n /\n Create a new schema from a JSON schema definition\n /\n constructor(schema: SchemaDefinition) {\n this.schema = schema;\n this.fields = buildFields(schema, 0, 0, []);\n this.fieldList = listFields(this.fields);\n }\n\n /\n Retrieve a field definition\n /\n findField(path: string \| string[]): ParquetField {\n if (typeof path === 'string') {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.split(',');\n } else {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.slice(0); // clone array\n }\n\n let n = this.fields;\n for (; path.length > 1; path.shift()) {\n n = n[path[0]].fields as Record<string, ParquetField>;\n }\n\n return n[path[0]];\n }\n\n /\n Retrieve a field definition and all the field's ancestors\n /\n findFieldBranch(path: string \| string[]): ParquetField[] {\n if (typeof path === 'string') {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.split(',');\n }\n const branch: ParquetField[] = [];\n let n = this.fields;\n for (; path.length > 0; path.shift()) {\n branch.push(n[path[0]]);\n if (path.length > 1) {\n n = n[path[0]].fields as Record<string, ParquetField>;\n }\n }\n return branch;\n }\n\n shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n shredRecord(this, record, buffer);\n }\n\n materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n return materializeRecords(this, buffer);\n }\n\n compress(type: ParquetCompression): this {\n setCompress(this.schema, type);\n setCompress(this.fields, type);\n return this;\n }\n\n buffer(): ParquetBuffer {\n return shredBuffer(this);\n }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n for (const name in schema) {\n const node = schema[name];\n if (node.fields) {\n setCompress(node.fields, type);\n } else {\n node.compression = type;\n }\n }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n schema: SchemaDefinition,\n rLevelParentMax: number,\n dLevelParentMax: number,\n path: string[]\n): Record<string, ParquetField> {\n const fieldList: Record<string, ParquetField> = {};\n\n for (const name in schema) {\n const opts = schema[name];\n\n / field repetition type /\n const required = !opts.optional;\n const repeated = Boolean(opts.repeated);\n let rLevelMax = rLevelParentMax;\n let dLevelMax = dLevelParentMax;\n\n let repetitionType: RepetitionType = 'REQUIRED';\n if (!required) {\n repetitionType = 'OPTIONAL';\n dLevelMax++;\n }\n if (repeated) {\n repetitionType = 'REPEATED';\n rLevelMax++;\n if (required) dLevelMax++;\n }\n\n / nested field /\n if (opts.fields) {\n const cpath = path.concat([name]);\n fieldList[name] = {\n name,\n path: cpath,\n key: cpath.join(),\n repetitionType,\n rLevelMax,\n dLevelMax,\n isNested: true,\n fieldCount: Object.keys(opts.fields).length,\n fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n };\n continue; // eslint-disable-line no-continue\n }\n\n const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n if (!typeDef) {\n throw new Error(`invalid parquet type: ${opts.type}`);\n }\n\n opts.encoding = opts.encoding \|\| 'PLAIN';\n if (!(opts.encoding in PARQUET_CODECS)) {\n throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n }\n\n opts.compression = opts.compression \|\| 'UNCOMPRESSED';\n if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n throw new Error(`unsupported compression method: ${opts.compression}`);\n }\n\n / add to schema */\n const cpath = path.concat([name]);\n fieldList[name] = {\n name,\n primitiveType: typeDef.primitiveType,\n originalType: typeDef.originalType,\n path: cpath,\n key: cpath.join(),\n repetitionType,\n encoding: opts.encoding,\n compression: opts.compression,\n typeLength: opts.typeLength \|\| typeDef.typeLength,\n presision: opts.presision,\n scale: opts.scale,\n rLevelMax,\n dLevelMax\n };\n }\n return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n let list: ParquetField[] = [];\n for (const k in fields) {\n list.push(fields[k]);\n if (fields[k].isNested) {\n list = list.concat(listFields(fields[k].fields!));\n }\n }\n return list;\n}\n"],"mappings":";AAEA,SAAQA,cAAc,QAAO,WAAW;AACxC,SAAQC,2BAA2B,QAAO,gBAAgB;AAU1D,SAAQC,kBAAkB,EAAEC,WAAW,EAAEC,WAAW,QAAO,SAAS;AACpE,SAAQC,qBAAqB,QAAO,SAAS;AAK7C,OAAO,MAAMC,aAAa,CAAC;EAQzBC,WAAWA,CAACC,MAAwB,EAAE;IAAAC,eAAA;IAAAA,eAAA;IAAAA,eAAA;IACpC,IAAI,CAACD,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACE,MAAM,GAAGC,WAAW,CAACH,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACI,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;EAKAI,SAASA,CAACC,IAAuB,EAAgB;IAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;MAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;IACxB,CAAC,MAAM;MAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;IACtB;IAEA,IAAIC,CAAC,GAAG,IAAI,CAACR,MAAM;IACnB,OAAOK,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,EAAE,EAAE;MACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACL,MAAsC;IACvD;IAEA,OAAOQ,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;EACnB;EAKAM,eAAeA,CAACN,IAAuB,EAAkB;IACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;MAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;IACxB;IACA,MAAMM,MAAsB,GAAG,EAAE;IACjC,IAAIJ,CAAC,GAAG,IAAI,CAACR,MAAM;IACnB,OAAOK,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,EAAE,EAAE;MACpCE,MAAM,CAACC,IAAI,CAACL,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;MACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;QACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACL,MAAsC;MACvD;IACF;IACA,OAAOY,MAAM;EACf;EAEAlB,WAAWA,CAACoB,MAAqB,EAAEC,MAAqB,EAAQ;IAC9DrB,WAAW,CAAC,IAAI,EAAEoB,MAAM,EAAEC,MAAM,CAAC;EACnC;EAEAvB,kBAAkBA,CAACuB,MAAqB,EAAmB;IACzD,OAAOvB,kBAAkB,CAAC,IAAI,EAAEuB,MAAM,CAAC;EACzC;EAEAC,QAAQA,CAACC,IAAwB,EAAQ;IACvCC,WAAW,CAAC,IAAI,CAACpB,MAAM,EAAEmB,IAAI,CAAC;IAC9BC,WAAW,CAAC,IAAI,CAAClB,MAAM,EAAEiB,IAAI,CAAC;IAC9B,OAAO,IAAI;EACb;EAEAF,MAAMA,CAAA,EAAkB;IACtB,OAAOtB,WAAW,CAAC,IAAI,CAAC;EAC1B;AACF;AAEA,SAASyB,WAAWA,CAACpB,MAAW,EAAEmB,IAAwB,EAAE;EAC1D,KAAK,MAAME,IAAI,IAAIrB,MAAM,EAAE;IACzB,MAAMsB,IAAI,GAAGtB,MAAM,CAACqB,IAAI,CAAC;IACzB,IAAIC,IAAI,CAACpB,MAAM,EAAE;MACfkB,WAAW,CAACE,IAAI,CAACpB,MAAM,EAAEiB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLG,IAAI,CAACC,WAAW,GAAGJ,IAAI;IACzB;EACF;AACF;AAGA,SAAShB,WAAWA,CAClBH,MAAwB,EACxBwB,eAAuB,EACvBC,eAAuB,EACvBlB,IAAc,EACgB;EAC9B,MAAMH,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,MAAMiB,IAAI,IAAIrB,MAAM,EAAE;IACzB,MAAM0B,IAAI,GAAG1B,MAAM,CAACqB,IAAI,CAAC;IAGzB,MAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,MAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;IAGA,IAAIN,IAAI,CAACxB,MAAM,EAAE;MACf,MAAMgC,KAAK,GAAG3B,IAAI,CAAC4B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;QAChBA,IAAI;QACJd,IAAI,EAAE2B,KAAK;QACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,EAAE;QACjBJ,cAAc;QACdF,SAAS;QACTC,SAAS;QACTM,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACf,IAAI,CAACxB,MAAM,CAAC,CAACS,MAAM;QAC3CT,MAAM,EAAEC,WAAW,CAACuB,IAAI,CAACxB,MAAM,EAAE6B,SAAS,EAAEC,SAAS,EAAEE,KAAK;MAC9D,CAAC;MACD;IACF;IAEA,MAAMQ,OAAY,GAAG7C,qBAAqB,CAAC6B,IAAI,CAACP,IAAI,CAAE;IACtD,IAAI,CAACuB,OAAO,EAAE;MACZ,MAAM,IAAIC,KAAK,0BAAAR,MAAA,CAA0BT,IAAI,CAACP,IAAI,EAAG;IACvD;IAEAO,IAAI,CAACkB,QAAQ,GAAGlB,IAAI,CAACkB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAElB,IAAI,CAACkB,QAAQ,IAAIpD,cAAc,CAAC,EAAE;MACtC,MAAM,IAAImD,KAAK,kCAAAR,MAAA,CAAkCT,IAAI,CAACkB,QAAQ,EAAG;IACnE;IAEAlB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAI9B,2BAA2B,CAAC,EAAE;MACtD,MAAM,IAAIkD,KAAK,oCAAAR,MAAA,CAAoCT,IAAI,CAACH,WAAW,EAAG;IACxE;IAGA,MAAMW,KAAK,GAAG3B,IAAI,CAAC4B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;MAChBA,IAAI;MACJwB,aAAa,EAAEH,OAAO,CAACG,aAAa;MACpCC,YAAY,EAAEJ,OAAO,CAACI,YAAY;MAClCvC,IAAI,EAAE2B,KAAK;MACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,EAAE;MACjBJ,cAAc;MACdW,QAAQ,EAAElB,IAAI,CAACkB,QAAQ;MACvBrB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7BwB,UAAU,EAAErB,IAAI,CAACqB,UAAU,IAAIL,OAAO,CAACK,UAAU;MACjDC,SAAS,EAAEtB,IAAI,CAACsB,SAAS;MACzBC,KAAK,EAAEvB,IAAI,CAACuB,KAAK;MACjBlB,SAAS;MACTC;IACF,CAAC;EACH;EACA,OAAO5B,SAAS;AAClB;AAEA,SAASC,UAAUA,CAACH,MAAoC,EAAkB;EACxE,IAAIgD,IAAoB,GAAG,EAAE;EAC7B,KAAK,MAAMC,CAAC,IAAIjD,MAAM,EAAE;IACtBgD,IAAI,CAACnC,IAAI,CAACb,MAAM,CAACiD,CAAC,CAAC,CAAC;IACpB,IAAIjD,MAAM,CAACiD,CAAC,CAAC,CAACb,QAAQ,EAAE;MACtBY,IAAI,GAAGA,IAAI,CAACf,MAAM,CAAC9B,UAAU,CAACH,MAAM,CAACiD,CAAC,CAAC,CAACjD,MAAM,CAAE,CAAC;IACnD;EACF;EACA,OAAOgD,IAAI;AACb"}