npm - @loaders.gl/parquet - Versions diffs - 3.4.0-alpha.2 → 3.4.0-alpha.3 - Mend

@loaders.gl/parquet 3.4.0-alpha.2 → 3.4.0-alpha.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (247) hide show

package/dist/es5/parquetjs/parser/parquet-reader.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"parquet-reader.js","names":["DEFAULT_PROPS","defaultDictionarySize","ParquetReader","file","props","close","rowBatchIterator","rows","row","getSchema","schema","rowGroupIterator","rowGroup","materializeRecords","columnList","map","x","Array","isArray","getFileMetadata","metadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","Number","num_rows","root","decodeSchema","num_children","schemaDefinition","ParquetSchema","md","key_value_metadata","kv","key","value","readHeader","readFooter","read","PARQUET_MAGIC","buffer","magic","toString","PARQUET_MAGIC_ENCRYPTED","Error","trailerLen","size","trailerBuf","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","decodeFileMetadata","rowCount","columnData","columns","colChunk","colMetadata","meta_data","colKey","path_in_schema","fieldIndexOf","readColumnChunk","join","file_path","undefined","field","findField","type","getThriftEnum","Type","primitiveType","compression","CompressionCodec","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","options","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","decodeDataPages","dictionarySize","cursor","offset","decodePage","decodedPage"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRecords} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n ParquetBuffer,\n ParquetCompression,\n ParquetData,\n PrimitiveType,\n ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader /\nexport type ParquetIterationProps = {\n /* Filter allowing some columns to be dropped /\n columnList?: string[] \| string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n defaultDictionarySize: 1e6\n};\n\n/\n The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n /\nexport class ParquetReader {\n props: Required<ParquetReaderProps>;\n file: ReadableFile;\n metadata: Promise<FileMetaData> \| null = null;\n\n constructor(file: ReadableFile, props?: ParquetReaderProps) {\n this.file = file;\n this.props = {...DEFAULT_PROPS, ...props};\n }\n\n close(): void {\n // eslint-disable-next-line @typescript-eslint/no-floating-promises\n this.file.close();\n }\n\n // HIGH LEVEL METHODS\n\n /* Yield one row at a time /\n async rowIterator(props?: ParquetIterationProps) {\n for await (const rows of this.rowBatchIterator(props)) {\n // yield rows\n for (const row of rows) {\n yield row;\n }\n }\n }\n\n /* Yield one batch of rows at a time /\n async rowBatchIterator(props?: ParquetIterationProps) {\n const schema = await this.getSchema();\n for await (const rowGroup of this.rowGroupIterator(props)) {\n yield materializeRecords(schema, rowGroup);\n }\n }\n\n /** Iterate over the raw row groups /\n async rowGroupIterator(props?: ParquetIterationProps) {\n // Ensure strings are nested in arrays\n const columnList: string[][] = (props?.columnList \|\| []).map((x) =>\n Array.isArray(x) ? x : [x]\n );\n\n const metadata = await this.getFileMetadata();\n const schema = await this.getSchema();\n\n const rowGroupCount = metadata?.row_groups.length \|\| 0;\n\n for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n const rowGroup = await this.readRowGroup(\n schema,\n metadata.row_groups[rowGroupIndex],\n columnList\n );\n yield rowGroup;\n }\n }\n\n async getRowCount(): Promise<number> {\n const metadata = await this.getFileMetadata();\n return Number(metadata.num_rows);\n }\n\n async getSchema(): Promise<ParquetSchema> {\n const metadata = await this.getFileMetadata();\n const root = metadata.schema[0];\n const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n const schema = new ParquetSchema(schemaDefinition);\n return schema;\n }\n\n /*\n Returns the user (key/value) metadata for this file\n * In parquet this is not stored on the schema like it is in arrow\n /\n async getSchemaMetadata(): Promise<Record<string, string>> {\n const metadata = await this.getFileMetadata();\n const md: Record<string, string> = {};\n for (const kv of metadata.key_value_metadata!) {\n md[kv.key] = kv.value!;\n }\n return md;\n }\n\n async getFileMetadata(): Promise<FileMetaData> {\n if (!this.metadata) {\n await this.readHeader();\n this.metadata = this.readFooter();\n }\n return this.metadata;\n }\n\n // LOW LEVEL METHODS\n\n /* Metadata is stored in the footer /\n async readHeader(): Promise<void> {\n const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n const magic = buffer.toString();\n switch (magic) {\n case PARQUET_MAGIC:\n break;\n case PARQUET_MAGIC_ENCRYPTED:\n throw new Error('Encrypted parquet file not supported');\n default:\n throw new Error(`Invalid parquet file (magic=${magic})`);\n }\n }\n\n /* Metadata is stored in the footer /\n async readFooter(): Promise<FileMetaData> {\n const trailerLen = PARQUET_MAGIC.length + 4;\n const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n const magic = trailerBuf.slice(4).toString();\n if (magic !== PARQUET_MAGIC) {\n throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n }\n\n const metadataSize = trailerBuf.readUInt32LE(0);\n const metadataOffset = this.file.size - metadataSize - trailerLen;\n if (metadataOffset < PARQUET_MAGIC.length) {\n throw new Error(`Invalid metadata size ${metadataOffset}`);\n }\n\n const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n // let metadata = new parquet_thrift.FileMetaData();\n // parquet_util.decodeThrift(metadata, metadataBuf);\n const {metadata} = decodeFileMetadata(metadataBuf);\n return metadata;\n }\n\n /* Data is stored in row groups (similar to Apache Arrow record batches) /\n async readRowGroup(\n schema: ParquetSchema,\n rowGroup: RowGroup,\n columnList: string[][]\n ): Promise<ParquetBuffer> {\n const buffer: ParquetBuffer = {\n rowCount: Number(rowGroup.num_rows),\n columnData: {}\n };\n for (const colChunk of rowGroup.columns) {\n const colMetadata = colChunk.meta_data;\n const colKey = colMetadata?.path_in_schema;\n if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n continue; // eslint-disable-line no-continue\n }\n buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n }\n return buffer;\n }\n\n /\n Each row group contains column chunks for all the columns.\n /\n async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n throw new Error('external references are not supported');\n }\n\n const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n if (type !== field.primitiveType) {\n throw new Error(`chunk type not matching schema: ${type}`);\n }\n\n const compression: ParquetCompression = getThriftEnum(\n CompressionCodec,\n colChunk.meta_data?.codec!\n ) as any;\n\n const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n if (!colChunk.file_path) {\n pagesSize = Math.min(\n this.file.size - pagesOffset,\n Number(colChunk.meta_data?.total_compressed_size)\n );\n }\n\n const options: ParquetOptions = {\n type,\n rLevelMax: field.rLevelMax,\n dLevelMax: field.dLevelMax,\n compression,\n column: field,\n numValues: colChunk.meta_data?.num_values,\n dictionary: []\n };\n\n let dictionary;\n\n const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n if (dictionaryPageOffset) {\n const dictionaryOffset = Number(dictionaryPageOffset);\n // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n }\n\n dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n return await decodeDataPages(pagesBuf, {...options, dictionary});\n }\n\n /\n Getting dictionary for allows to flatten values by indices.\n * @param dictionaryPageOffset\n * @param options\n * @param pagesOffset\n * @returns\n */\n async getDictionary(\n dictionaryPageOffset: number,\n options: ParquetOptions,\n pagesOffset: number\n ): Promise<string[]> {\n if (dictionaryPageOffset === 0) {\n // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n // In this case we are working with parquet-mr files format. Problem is described below:\n // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n // We need to get dictionary page from column chunk if it exists.\n // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n return [];\n }\n\n const dictionarySize = Math.min(\n this.file.size - dictionaryPageOffset,\n this.props.defaultDictionarySize\n );\n const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n const decodedPage = await decodePage(cursor, options);\n\n return decodedPage.dictionary!;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;AAGA;AACA;AACA;AAEA;AACA;AAQA;AAAoF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAapF,IAAMA,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;;AAAC,IAQWC,aAAa;EAKxB,uBAAYC,IAAkB,EAAEC,KAA0B,EAAE;IAAA;IAAA;IAAA;IAAA,gDAFnB,IAAI;IAG3C,IAAI,CAACD,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,mCAAOJ,aAAa,GAAKI,KAAK,CAAC;EAC3C;EAAC;IAAA;IAAA,OAED,iBAAc;MAEZ,IAAI,CAACD,IAAI,CAACE,KAAK,EAAE;IACnB;;EAAC;IAAA;IAAA;;IAKD,qBAAmBD,KAA6B,EAAE;MAAA;MAAA;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA;gBAAA;gBAAA,2BACvB,KAAI,CAACE,gBAAgB,CAACF,KAAK,CAAC;cAAA;gBAAA;gBAAA;cAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAApCG,IAAI;gBAAA,wCAEDA,IAAI;gBAAA;gBAAA;cAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAXC,GAAG;gBAAA;gBACZ,OAAMA,GAAG;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAA;cAAA;gBAAA;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA;IAGf;;EAAC;IAAA;IAAA;IAGD,0BAAwBJ,KAA6B,EAAE;MAAA;MAAA;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,0CAChC,MAAI,CAACK,SAAS,EAAE;cAAA;gBAA/BC,MAAM;gBAAA;gBAAA;gBAAA;gBAAA,4BACiB,MAAI,CAACC,gBAAgB,CAACP,KAAK,CAAC;cAAA;gBAAA;gBAAA;cAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAxCQ,QAAQ;gBAAA;gBACvB,OAAM,IAAAC,yBAAkB,EAACH,MAAM,EAAEE,QAAQ,CAAC;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAA;cAAA;gBAAA;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA;IAE9C;;EAAC;IAAA;IAAA;IAGD,0BAAwBR,KAA6B,EAAE;MAAA;MAAA;QAAA;QAAA;UAAA;YAAA;cAAA;gBAE/CU,UAAsB,GAAG,CAAC,CAAAV,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEU,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAC,UAACC,CAAC;kBAAA,OAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC;gBAAA,EAC3B;gBAAA;gBAAA,0CAEsB,MAAI,CAACG,eAAe,EAAE;cAAA;gBAAvCC,QAAQ;gBAAA;gBAAA,0CACO,MAAI,CAACX,SAAS,EAAE;cAAA;gBAA/BC,MAAM;gBAENW,aAAa,GAAG,CAAAD,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEE,UAAU,CAACC,MAAM,KAAI,CAAC;gBAE7CC,aAAa,GAAG,CAAC;cAAA;gBAAA,MAAEA,aAAa,GAAGH,aAAa;kBAAA;kBAAA;gBAAA;gBAAA;gBAAA,0CAChC,MAAI,CAACI,YAAY,CACtCf,MAAM,EACNU,QAAQ,CAACE,UAAU,CAACE,aAAa,CAAC,EAClCV,UAAU,CACX;cAAA;gBAJKF,QAAQ;gBAAA;gBAKd,OAAMA,QAAQ;cAAA;gBAN2CY,aAAa,EAAE;gBAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA;IAQ5E;EAAC;IAAA;IAAA;MAAA,6EAED;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACyB,IAAI,CAACL,eAAe,EAAE;cAAA;gBAAvCC,QAAQ;gBAAA,kCACPM,MAAM,CAACN,QAAQ,CAACO,QAAQ,CAAC;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACjC;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,2EAED;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACyB,IAAI,CAACR,eAAe,EAAE;cAAA;gBAAvCC,QAAQ;gBACRQ,IAAI,GAAGR,QAAQ,CAACV,MAAM,CAAC,CAAC,CAAC;gBAAA,gBACI,IAAAmB,sBAAY,EAACT,QAAQ,CAACV,MAAM,EAAE,CAAC,EAAEkB,IAAI,CAACE,YAAY,CAAE,EAAxEC,gBAAgB,iBAAxBrB,MAAM;gBACPA,MAAM,GAAG,IAAIsB,qBAAa,CAACD,gBAAgB,CAAC;gBAAA,kCAC3CrB,MAAM;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACd;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,mFAMD;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACyB,IAAI,CAACS,eAAe,EAAE;cAAA;gBAAvCC,QAAQ;gBACRa,EAA0B,GAAG,CAAC,CAAC;gBAAA,wCACpBb,QAAQ,CAACc,kBAAkB;gBAAA;kBAA5C,uDAA+C;oBAApCC,EAAE;oBACXF,EAAE,CAACE,EAAE,CAACC,GAAG,CAAC,GAAGD,EAAE,CAACE,KAAM;kBACxB;gBAAC;kBAAA;gBAAA;kBAAA;gBAAA;gBAAA,kCACMJ,EAAE;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACV;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,iFAED;QAAA;UAAA;YAAA;cAAA;gBAAA,IACO,IAAI,CAACb,QAAQ;kBAAA;kBAAA;gBAAA;gBAAA;gBAAA,OACV,IAAI,CAACkB,UAAU,EAAE;cAAA;gBACvB,IAAI,CAAClB,QAAQ,GAAG,IAAI,CAACmB,UAAU,EAAE;cAAC;gBAAA,kCAE7B,IAAI,CAACnB,QAAQ;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACrB;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,4EAKD;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACuB,IAAI,CAACjB,IAAI,CAACqC,IAAI,CAAC,CAAC,EAAEC,wBAAa,CAAClB,MAAM,CAAC;cAAA;gBAAtDmB,MAAM;gBACNC,KAAK,GAAGD,MAAM,CAACE,QAAQ,EAAE;gBAAA,eACvBD,KAAK;gBAAA,kCACNF,wBAAa,wBAEbI,kCAAuB;gBAAA;cAAA;gBAAA;cAAA;gBAAA,MACpB,IAAIC,KAAK,CAAC,sCAAsC,CAAC;cAAA;gBAAA,MAEjD,IAAIA,KAAK,uCAAgCH,KAAK,OAAI;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAE7D;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,4EAGD;QAAA;QAAA;UAAA;YAAA;cAAA;gBACQI,UAAU,GAAGN,wBAAa,CAAClB,MAAM,GAAG,CAAC;gBAAA;gBAAA,OAClB,IAAI,CAACpB,IAAI,CAACqC,IAAI,CAAC,IAAI,CAACrC,IAAI,CAAC6C,IAAI,GAAGD,UAAU,EAAEA,UAAU,CAAC;cAAA;gBAA1EE,UAAU;gBAEVN,KAAK,GAAGM,UAAU,CAACC,KAAK,CAAC,CAAC,CAAC,CAACN,QAAQ,EAAE;gBAAA,MACxCD,KAAK,KAAKF,wBAAa;kBAAA;kBAAA;gBAAA;gBAAA,MACnB,IAAIK,KAAK,6CAAqCH,KAAK,OAAI;cAAA;gBAGzDQ,YAAY,GAAGF,UAAU,CAACG,YAAY,CAAC,CAAC,CAAC;gBACzCC,cAAc,GAAG,IAAI,CAAClD,IAAI,CAAC6C,IAAI,GAAGG,YAAY,GAAGJ,UAAU;gBAAA,MAC7DM,cAAc,GAAGZ,wBAAa,CAAClB,MAAM;kBAAA;kBAAA;gBAAA;gBAAA,MACjC,IAAIuB,KAAK,iCAA0BO,cAAc,EAAG;cAAA;gBAAA;gBAAA,OAGlC,IAAI,CAAClD,IAAI,CAACqC,IAAI,CAACa,cAAc,EAAEF,YAAY,CAAC;cAAA;gBAAhEG,WAAW;gBAAA,sBAGE,IAAAC,6BAAkB,EAACD,WAAW,CAAC,EAA3ClC,QAAQ,uBAARA,QAAQ;gBAAA,kCACRA,QAAQ;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAChB;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,8EAGD,mBACEV,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB;QAAA;QAAA;UAAA;YAAA;cAAA;gBAEhB4B,MAAqB,GAAG;kBAC5Bc,QAAQ,EAAE9B,MAAM,CAACd,QAAQ,CAACe,QAAQ,CAAC;kBACnC8B,UAAU,EAAE,CAAC;gBACf,CAAC;gBAAA,wCACsB7C,QAAQ,CAAC8C,OAAO;gBAAA;gBAAA;cAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAA5BC,QAAQ;gBACXC,WAAW,GAAGD,QAAQ,CAACE,SAAS;gBAChCC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;gBAAA,MACtCjD,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI,IAAAyC,uBAAY,EAAClD,UAAU,EAAEgD,MAAM,CAAE,GAAG,CAAC;kBAAA;kBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA,OAGxB,IAAI,CAACG,eAAe,CAACvD,MAAM,EAAEiD,QAAQ,CAAC;cAAA;gBAAhFjB,MAAM,CAACe,UAAU,CAACK,MAAM,CAAEI,IAAI,EAAE,CAAC;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA,mCAE5BxB,MAAM;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACd;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,iFAKD,mBAAsBhC,MAAqB,EAAEiD,QAAqB;QAAA;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA,MAC5DA,QAAQ,CAACQ,SAAS,KAAKC,SAAS,IAAIT,QAAQ,CAACQ,SAAS,KAAK,IAAI;kBAAA;kBAAA;gBAAA;gBAAA,MAC3D,IAAIrB,KAAK,CAAC,uCAAuC,CAAC;cAAA;gBAGpDuB,KAAK,GAAG3D,MAAM,CAAC4D,SAAS,wBAACX,QAAQ,CAACE,SAAS,wDAAlB,oBAAoBE,cAAc,CAAE;gBAC7DQ,IAAmB,GAAG,IAAAC,wBAAa,EAACC,mBAAI,0BAAEd,QAAQ,CAACE,SAAS,yDAAlB,qBAAoBU,IAAI,CAAE;gBAAA,MAEtEA,IAAI,KAAKF,KAAK,CAACK,aAAa;kBAAA;kBAAA;gBAAA;gBAAA,MACxB,IAAI5B,KAAK,2CAAoCyB,IAAI,EAAG;cAAA;gBAGtDI,WAA+B,GAAG,IAAAH,wBAAa,EACnDI,+BAAgB,0BAChBjB,QAAQ,CAACE,SAAS,yDAAlB,qBAAoBgB,KAAK,CAC1B;gBAEKC,WAAW,GAAGpD,MAAM,yBAACiC,QAAQ,CAACE,SAAS,yDAAlB,qBAAoBkB,gBAAgB,CAAE;gBAC7DC,SAAS,GAAGtD,MAAM,yBAACiC,QAAQ,CAACE,SAAS,yDAAlB,qBAAoBoB,qBAAqB,CAAE;gBAElE,IAAI,CAACtB,QAAQ,CAACQ,SAAS,EAAE;kBACvBa,SAAS,GAAGE,IAAI,CAACC,GAAG,CAClB,IAAI,CAAChF,IAAI,CAAC6C,IAAI,GAAG8B,WAAW,EAC5BpD,MAAM,yBAACiC,QAAQ,CAACE,SAAS,yDAAlB,qBAAoBoB,qBAAqB,CAAC,CAClD;gBACH;gBAEMG,OAAuB,GAAG;kBAC9Bb,IAAI,EAAJA,IAAI;kBACJc,SAAS,EAAEhB,KAAK,CAACgB,SAAS;kBAC1BC,SAAS,EAAEjB,KAAK,CAACiB,SAAS;kBAC1BX,WAAW,EAAXA,WAAW;kBACXY,MAAM,EAAElB,KAAK;kBACbmB,SAAS,0BAAE7B,QAAQ,CAACE,SAAS,yDAAlB,qBAAoB4B,UAAU;kBACzCC,UAAU,EAAE;gBACd,CAAC;gBAIKC,oBAAoB,GAAGhC,QAAQ,aAARA,QAAQ,+CAARA,QAAQ,CAAEE,SAAS,yDAAnB,qBAAqB+B,sBAAsB;gBAAA,KAEpED,oBAAoB;kBAAA;kBAAA;gBAAA;gBAChBE,gBAAgB,GAAGnE,MAAM,CAACiE,oBAAoB,CAAC;gBAAA;gBAAA,OAElC,IAAI,CAACG,aAAa,CAACD,gBAAgB,EAAET,OAAO,EAAEN,WAAW,CAAC;cAAA;gBAA7EY,UAAU;cAAA;gBAGZA,UAAU,GAAG,uBAAAN,OAAO,CAACM,UAAU,gDAAlB,oBAAoBnE,MAAM,GAAG6D,OAAO,CAACM,UAAU,GAAGA,UAAU;gBAAC;gBAAA,OACnD,IAAI,CAACvF,IAAI,CAACqC,IAAI,CAACsC,WAAW,EAAEE,SAAS,CAAC;cAAA;gBAAvDe,QAAQ;gBAAA;gBAAA,OACD,IAAAC,yBAAe,EAACD,QAAQ,kCAAMX,OAAO;kBAAEM,UAAU,EAAVA;gBAAU,GAAE;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACjE;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,+EASD,mBACEC,oBAA4B,EAC5BP,OAAuB,EACvBN,WAAmB;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA,MAEfa,oBAAoB,KAAK,CAAC;kBAAA;kBAAA;gBAAA;gBAAA,mCAQrB,EAAE;cAAA;gBAGLM,cAAc,GAAGf,IAAI,CAACC,GAAG,CAC7B,IAAI,CAAChF,IAAI,CAAC6C,IAAI,GAAG2C,oBAAoB,EACrC,IAAI,CAACvF,KAAK,CAACH,qBAAqB,CACjC;gBAAA;gBAAA,OACsB,IAAI,CAACE,IAAI,CAACqC,IAAI,CAACmD,oBAAoB,EAAEM,cAAc,CAAC;cAAA;gBAArEF,QAAQ;gBAERG,MAAM,GAAG;kBAACxD,MAAM,EAAEqD,QAAQ;kBAAEI,MAAM,EAAE,CAAC;kBAAEnD,IAAI,EAAE+C,QAAQ,CAACxE;gBAAM,CAAC;gBAAA;gBAAA,OACzC,IAAA6E,oBAAU,EAACF,MAAM,EAAEd,OAAO,CAAC;cAAA;gBAA/CiB,WAAW;gBAAA,mCAEVA,WAAW,CAACX,UAAU;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAC9B;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;EAAA;AAAA;AAAA"}
1	+ {"version":3,"file":"parquet-reader.js","names":["_schema","require","_decoders","_shred","_constants","_parquetThrift","_readUtils","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","call","step","next","_e2","return","minLen","_arrayLikeToArray","Object","prototype","toString","slice","constructor","name","from","test","arr","len","arr2","ownKeys","object","enumerableOnly","keys","getOwnPropertySymbols","symbols","filter","sym","getOwnPropertyDescriptor","enumerable","push","apply","_objectSpread","target","arguments","source","forEach","key","_defineProperty2","default","getOwnPropertyDescriptors","defineProperties","defineProperty","_asyncIterator","iterable","method","async","sync","retry","asyncIterator","AsyncFromSyncIterator","AsyncFromSyncIteratorContinuation","r","Promise","reject","resolve","then","_return","ret","throw","_throw","thr","DEFAULT_PROPS","defaultDictionarySize","ParquetReader","file","props","_classCallCheck2","_createClass2","close","rowIterator","_this","_wrapAsyncGenerator2","_regenerator","mark","_callee","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","rows","_iterator3","_step3","row","wrap","_callee$","_context","prev","rowBatchIterator","_awaitAsyncGenerator2","sent","t0","finish","t1","stop","_this2","_callee2","schema","_iteratorAbruptCompletion2","_didIteratorError2","_iteratorError2","_iterator2","_step2","rowGroup","_callee2$","_context2","getSchema","rowGroupIterator","materializeRecords","_this3","_callee3","columnList","metadata","rowGroupCount","rowGroupIndex","_callee3$","_context3","map","x","getFileMetadata","row_groups","readRowGroup","_getRowCount","_asyncToGenerator2","_callee4","_callee4$","_context4","abrupt","Number","num_rows","getRowCount","_getSchema","_callee5","root","_decodeSchema","schemaDefinition","_callee5$","_context5","decodeSchema","num_children","ParquetSchema","_getSchemaMetadata","_callee6","md","_iterator4","_step4","kv","_callee6$","_context6","key_value_metadata","getSchemaMetadata","_getFileMetadata","_callee7","_callee7$","_context7","readHeader","readFooter","_readHeader","_callee8","buffer","magic","_callee8$","_context8","read","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","Error","concat","_readFooter","_callee9","trailerLen","trailerBuf","metadataSize","metadataOffset","metadataBuf","_decodeFileMetadata","_callee9$","_context9","size","readUInt32LE","decodeFileMetadata","_readRowGroup","_callee10","_iterator5","_step5","colChunk","colMetadata","colKey","_callee10$","_context10","rowCount","columnData","columns","meta_data","path_in_schema","fieldIndexOf","readColumnChunk","join","_x","_x2","_x3","_readColumnChunk","_callee11","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_options$dictionary","field","type","compression","pagesOffset","pagesSize","_colChunk$meta_data6","options","dictionary","dictionaryPageOffset","dictionaryOffset","pagesBuf","_callee11$","_context11","file_path","undefined","findField","getThriftEnum","Type","primitiveType","CompressionCodec","codec","data_page_offset","total_compressed_size","Math","min","rLevelMax","dLevelMax","column","numValues","num_values","dictionary_page_offset","getDictionary","decodeDataPages","_x4","_x5","_getDictionary","_callee12","dictionarySize","cursor","decodedPage","_callee12$","_context12","offset","decodePage","_x6","_x7","_x8","exports"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRecords} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n ParquetBuffer,\n ParquetCompression,\n ParquetData,\n PrimitiveType,\n ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader /\nexport type ParquetIterationProps = {\n /* Filter allowing some columns to be dropped /\n columnList?: string[] \| string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n defaultDictionarySize: 1e6\n};\n\n/\n The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n /\nexport class ParquetReader {\n props: Required<ParquetReaderProps>;\n file: ReadableFile;\n metadata: Promise<FileMetaData> \| null = null;\n\n constructor(file: ReadableFile, props?: ParquetReaderProps) {\n this.file = file;\n this.props = {...DEFAULT_PROPS, ...props};\n }\n\n close(): void {\n // eslint-disable-next-line @typescript-eslint/no-floating-promises\n this.file.close();\n }\n\n // HIGH LEVEL METHODS\n\n /* Yield one row at a time /\n async rowIterator(props?: ParquetIterationProps) {\n for await (const rows of this.rowBatchIterator(props)) {\n // yield rows\n for (const row of rows) {\n yield row;\n }\n }\n }\n\n /* Yield one batch of rows at a time /\n async rowBatchIterator(props?: ParquetIterationProps) {\n const schema = await this.getSchema();\n for await (const rowGroup of this.rowGroupIterator(props)) {\n yield materializeRecords(schema, rowGroup);\n }\n }\n\n /** Iterate over the raw row groups /\n async rowGroupIterator(props?: ParquetIterationProps) {\n // Ensure strings are nested in arrays\n const columnList: string[][] = (props?.columnList \|\| []).map((x) =>\n Array.isArray(x) ? x : [x]\n );\n\n const metadata = await this.getFileMetadata();\n const schema = await this.getSchema();\n\n const rowGroupCount = metadata?.row_groups.length \|\| 0;\n\n for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n const rowGroup = await this.readRowGroup(\n schema,\n metadata.row_groups[rowGroupIndex],\n columnList\n );\n yield rowGroup;\n }\n }\n\n async getRowCount(): Promise<number> {\n const metadata = await this.getFileMetadata();\n return Number(metadata.num_rows);\n }\n\n async getSchema(): Promise<ParquetSchema> {\n const metadata = await this.getFileMetadata();\n const root = metadata.schema[0];\n const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n const schema = new ParquetSchema(schemaDefinition);\n return schema;\n }\n\n /*\n Returns the user (key/value) metadata for this file\n * In parquet this is not stored on the schema like it is in arrow\n /\n async getSchemaMetadata(): Promise<Record<string, string>> {\n const metadata = await this.getFileMetadata();\n const md: Record<string, string> = {};\n for (const kv of metadata.key_value_metadata!) {\n md[kv.key] = kv.value!;\n }\n return md;\n }\n\n async getFileMetadata(): Promise<FileMetaData> {\n if (!this.metadata) {\n await this.readHeader();\n this.metadata = this.readFooter();\n }\n return this.metadata;\n }\n\n // LOW LEVEL METHODS\n\n /* Metadata is stored in the footer /\n async readHeader(): Promise<void> {\n const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n const magic = buffer.toString();\n switch (magic) {\n case PARQUET_MAGIC:\n break;\n case PARQUET_MAGIC_ENCRYPTED:\n throw new Error('Encrypted parquet file not supported');\n default:\n throw new Error(`Invalid parquet file (magic=${magic})`);\n }\n }\n\n /* Metadata is stored in the footer /\n async readFooter(): Promise<FileMetaData> {\n const trailerLen = PARQUET_MAGIC.length + 4;\n const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n const magic = trailerBuf.slice(4).toString();\n if (magic !== PARQUET_MAGIC) {\n throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n }\n\n const metadataSize = trailerBuf.readUInt32LE(0);\n const metadataOffset = this.file.size - metadataSize - trailerLen;\n if (metadataOffset < PARQUET_MAGIC.length) {\n throw new Error(`Invalid metadata size ${metadataOffset}`);\n }\n\n const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n // let metadata = new parquet_thrift.FileMetaData();\n // parquet_util.decodeThrift(metadata, metadataBuf);\n const {metadata} = decodeFileMetadata(metadataBuf);\n return metadata;\n }\n\n /* Data is stored in row groups (similar to Apache Arrow record batches) /\n async readRowGroup(\n schema: ParquetSchema,\n rowGroup: RowGroup,\n columnList: string[][]\n ): Promise<ParquetBuffer> {\n const buffer: ParquetBuffer = {\n rowCount: Number(rowGroup.num_rows),\n columnData: {}\n };\n for (const colChunk of rowGroup.columns) {\n const colMetadata = colChunk.meta_data;\n const colKey = colMetadata?.path_in_schema;\n if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n continue; // eslint-disable-line no-continue\n }\n buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n }\n return buffer;\n }\n\n /\n Each row group contains column chunks for all the columns.\n /\n async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n throw new Error('external references are not supported');\n }\n\n const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n if (type !== field.primitiveType) {\n throw new Error(`chunk type not matching schema: ${type}`);\n }\n\n const compression: ParquetCompression = getThriftEnum(\n CompressionCodec,\n colChunk.meta_data?.codec!\n ) as any;\n\n const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n if (!colChunk.file_path) {\n pagesSize = Math.min(\n this.file.size - pagesOffset,\n Number(colChunk.meta_data?.total_compressed_size)\n );\n }\n\n const options: ParquetOptions = {\n type,\n rLevelMax: field.rLevelMax,\n dLevelMax: field.dLevelMax,\n compression,\n column: field,\n numValues: colChunk.meta_data?.num_values,\n dictionary: []\n };\n\n let dictionary;\n\n const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n if (dictionaryPageOffset) {\n const dictionaryOffset = Number(dictionaryPageOffset);\n // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n }\n\n dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n return await decodeDataPages(pagesBuf, {...options, dictionary});\n }\n\n /\n Getting dictionary for allows to flatten values by indices.\n * @param dictionaryPageOffset\n * @param options\n * @param pagesOffset\n * @returns\n */\n async getDictionary(\n dictionaryPageOffset: number,\n options: ParquetOptions,\n pagesOffset: number\n ): Promise<string[]> {\n if (dictionaryPageOffset === 0) {\n // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n // In this case we are working with parquet-mr files format. Problem is described below:\n // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n // We need to get dictionary page from column chunk if it exists.\n // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n return [];\n }\n\n const dictionarySize = Math.min(\n this.file.size - dictionaryPageOffset,\n this.props.defaultDictionarySize\n );\n const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n const decodedPage = await decodePage(cursor, options);\n\n return decodedPage.dictionary!;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;AAGA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,SAAA,GAAAD,OAAA;AACA,IAAAE,MAAA,GAAAF,OAAA;AAEA,IAAAG,UAAA,GAAAH,OAAA;AACA,IAAAI,cAAA,GAAAJ,OAAA;AAQA,IAAAK,UAAA,GAAAL,OAAA;AAAoF,SAAAM,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAoB,IAAA,CAAAtB,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAW,IAAA,GAAArB,EAAA,CAAAsB,IAAA,IAAAL,gBAAA,GAAAI,IAAA,CAAAV,IAAA,SAAAU,IAAA,KAAAR,CAAA,WAAAA,EAAAU,GAAA,IAAAL,MAAA,SAAAC,GAAA,GAAAI,GAAA,KAAAR,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAwB,MAAA,UAAAxB,EAAA,CAAAwB,MAAA,oBAAAN,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA2B,MAAA,SAAA3B,CAAA,qBAAAA,CAAA,sBAAA4B,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA,OAAAf,CAAA,GAAAiB,MAAA,CAAAC,SAAA,CAAAC,QAAA,CAAAT,IAAA,CAAAtB,CAAA,EAAAgC,KAAA,aAAApB,CAAA,iBAAAZ,CAAA,CAAAiC,WAAA,EAAArB,CAAA,GAAAZ,CAAA,CAAAiC,WAAA,CAAAC,IAAA,MAAAtB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA8B,IAAA,CAAAnC,CAAA,OAAAY,CAAA,+DAAAwB,IAAA,CAAAxB,CAAA,UAAAgB,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA;AAAA,SAAAC,kBAAAS,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA7B,MAAA,EAAA8B,GAAA,GAAAD,GAAA,CAAA7B,MAAA,WAAAC,CAAA,MAAA8B,IAAA,OAAAlC,KAAA,CAAAiC,GAAA,GAAA7B,CAAA,GAAA6B,GAAA,EAAA7B,CAAA,IAAA8B,IAAA,CAAA9B,CAAA,IAAA4B,GAAA,CAAA5B,CAAA,UAAA8B,IAAA;AAAA,SAAAC,QAAAC,MAAA,EAAAC,cAAA,QAAAC,IAAA,GAAAd,MAAA,CAAAc,IAAA,CAAAF,MAAA,OAAAZ,MAAA,CAAAe,qBAAA,QAAAC,OAAA,GAAAhB,MAAA,CAAAe,qBAAA,CAAAH,MAAA,GAAAC,cAAA,KAAAG,OAAA,GAAAA,OAAA,CAAAC,MAAA,WAAAC,GAAA,WAAAlB,MAAA,CAAAmB,wBAAA,CAAAP,MAAA,EAAAM,GAAA,EAAAE,UAAA,OAAAN,IAAA,CAAAO,IAAA,CAAAC,KAAA,CAAAR,IAAA,EAAAE,OAAA,YAAAF,IAAA;AAAA,SAAAS,cAAAC,MAAA,aAAA5C,CAAA,MAAAA,CAAA,GAAA6C,SAAA,CAAA9C,MAAA,EAAAC,CAAA,UAAA8C,MAAA,WAAAD,SAAA,CAAA7C,CAAA,IAAA6C,SAAA,CAAA7C,CAAA,QAAAA,CAAA,OAAA+B,OAAA,CAAAX,MAAA,CAAA0B,MAAA,OAAAC,OAAA,WAAAC,GAAA,QAAAC,gBAAA,CAAAC,OAAA,EAAAN,MAAA,EAAAI,GAAA,EAAAF,MAAA,CAAAE,GAAA,SAAA5B,MAAA,CAAA+B,yBAAA,GAAA/B,MAAA,CAAAgC,gBAAA,CAAAR,MAAA,EAAAxB,MAAA,CAAA+B,yBAAA,CAAAL,MAAA,KAAAf,OAAA,CAAAX,MAAA,CAAA0B,MAAA,GAAAC,OAAA,WAAAC,GAAA,IAAA5B,MAAA,CAAAiC,cAAA,CAAAT,MAAA,EAAAI,GAAA,EAAA5B,MAAA,CAAAmB,wBAAA,CAAAO,MAAA,EAAAE,GAAA,iBAAAJ,MAAA;AAAA,SAAAU,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAjE,MAAA,KAAA+D,KAAA,GAAA/D,MAAA,CAAAkE,aAAA,EAAAF,IAAA,GAAAhE,MAAA,CAAAC,QAAA,GAAAgE,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAA3C,IAAA,CAAA0C,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAG,qBAAA,CAAAL,MAAA,CAAA3C,IAAA,CAAA0C,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAjD,SAAA;AAAA,SAAAoD,sBAAA3D,CAAA,aAAA4D,kCAAAC,CAAA,QAAA3C,MAAA,CAAA2C,CAAA,MAAAA,CAAA,SAAAC,OAAA,CAAAC,MAAA,KAAAxD,SAAA,CAAAsD,CAAA,+BAAA3D,IAAA,GAAA2D,CAAA,CAAA3D,IAAA,SAAA4D,OAAA,CAAAE,OAAA,CAAAH,CAAA,CAAA1D,KAAA,EAAA8D,IAAA,WAAA9D,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAD,IAAA,EAAAA,IAAA,iBAAAyD,qBAAA,YAAAA,sBAAA3D,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAC,CAAA,GAAAD,CAAA,CAAAa,IAAA,KAAA8C,qBAAA,CAAAxC,SAAA,KAAAnB,CAAA,QAAAC,CAAA,QAAAY,IAAA,WAAAA,KAAA,WAAA+C,iCAAA,MAAA3D,CAAA,CAAAuC,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,OAAA5B,MAAA,WAAAmD,QAAA/D,KAAA,QAAAgE,GAAA,QAAAnE,CAAA,CAAAe,MAAA,oBAAAoD,GAAA,GAAAL,OAAA,CAAAE,OAAA,GAAA7D,KAAA,EAAAA,KAAA,EAAAD,IAAA,UAAA0D,iCAAA,CAAAO,GAAA,CAAA3B,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,OAAAyB,KAAA,WAAAC,OAAAlE,KAAA,QAAAmE,GAAA,QAAAtE,CAAA,CAAAe,MAAA,oBAAAuD,GAAA,GAAAR,OAAA,CAAAC,MAAA,CAAA5D,KAAA,IAAAyD,iCAAA,CAAAU,GAAA,CAAA9B,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,aAAAgB,qBAAA,CAAA3D,CAAA;AAapF,IAAMuE,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;AAAC,IAQWC,aAAa;EAKxB,SAAAA,cAAYC,IAAkB,EAAEC,KAA0B,EAAE;IAAA,IAAAC,gBAAA,CAAA5B,OAAA,QAAAyB,aAAA;IAAA,IAAA1B,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA;IAAA,IAAAD,gBAAA,CAAAC,OAAA,oBAFnB,IAAI;IAG3C,IAAI,CAAC0B,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAAlC,aAAA,CAAAA,aAAA,KAAO8B,aAAa,GAAKI,KAAK,CAAC;EAC3C;EAAC,IAAAE,aAAA,CAAA7B,OAAA,EAAAyB,aAAA;IAAA3B,GAAA;IAAA3C,KAAA,EAED,SAAA2E,MAAA,EAAc;MAEZ,IAAI,CAACJ,IAAI,CAACI,KAAK,EAAE;IACnB;EAAC;IAAAhC,GAAA;IAAA3C,KAAA,EAKD,SAAA4E,YAAmBJ,KAA6B,EAAE;MAAA,IAAAK,KAAA;MAAA,WAAAC,oBAAA,CAAAjC,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,UAAAC,QAAA;QAAA,IAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,GAAA;QAAA,OAAAX,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAC,SAAAC,QAAA;UAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAnF,IAAA;YAAA;cAAAwE,yBAAA;cAAAC,iBAAA;cAAAU,QAAA,CAAAC,IAAA;cAAAT,SAAA,GAAApC,cAAA,CACvB4B,KAAI,CAACkB,gBAAgB,CAACvB,KAAK,CAAC;YAAA;cAAAqB,QAAA,CAAAnF,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAAwC,SAAA,CAAA3E,IAAA;YAAA;cAAA,MAAAwE,yBAAA,KAAAI,KAAA,GAAAO,QAAA,CAAAI,IAAA,EAAAlG,IAAA;gBAAA8F,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAApC6E,IAAI,GAAAD,KAAA,CAAAtF,KAAA;cAAAwF,UAAA,GAAAvG,0BAAA,CAEDsG,IAAI;cAAAM,QAAA,CAAAC,IAAA;cAAAN,UAAA,CAAA3F,CAAA;YAAA;cAAA,KAAA4F,MAAA,GAAAD,UAAA,CAAA1F,CAAA,IAAAC,IAAA;gBAAA8F,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAAXgF,GAAG,GAAAD,MAAA,CAAAzF,KAAA;cAAA6F,QAAA,CAAAnF,IAAA;cACZ,OAAMgF,GAAG;YAAA;cAAAG,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAC,IAAA;cAAAD,QAAA,CAAAK,EAAA,GAAAL,QAAA;cAAAL,UAAA,CAAAvF,CAAA,CAAA4F,QAAA,CAAAK,EAAA;YAAA;cAAAL,QAAA,CAAAC,IAAA;cAAAN,UAAA,CAAArF,CAAA;cAAA,OAAA0F,QAAA,CAAAM,MAAA;YAAA;cAAAjB,yBAAA;cAAAW,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAnF,IAAA;cAAA;YAAA;cAAAmF,QAAA,CAAAC,IAAA;cAAAD,QAAA,CAAAO,EAAA,GAAAP,QAAA;cAAAV,iBAAA;cAAAC,cAAA,GAAAS,QAAA,CAAAO,EAAA;YAAA;cAAAP,QAAA,CAAAC,IAAA;cAAAD,QAAA,CAAAC,IAAA;cAAA,MAAAZ,yBAAA,IAAAG,SAAA,CAAAzE,MAAA;gBAAAiF,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAAAmF,QAAA,CAAAnF,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAAwC,SAAA,CAAAzE,MAAA;YAAA;cAAAiF,QAAA,CAAAC,IAAA;cAAA,KAAAX,iBAAA;gBAAAU,QAAA,CAAAnF,IAAA;gBAAA;cAAA;cAAA,MAAA0E,cAAA;YAAA;cAAA,OAAAS,QAAA,CAAAM,MAAA;YAAA;cAAA,OAAAN,QAAA,CAAAM,MAAA;YAAA;YAAA;cAAA,OAAAN,QAAA,CAAAQ,IAAA;UAAA;QAAA,GAAApB,OAAA;MAAA;IAGf;EAAC;IAAAtC,GAAA;IAAA3C,KAAA,EAGD,SAAA+F,iBAAwBvB,KAA6B,EAAE;MAAA,IAAA8B,MAAA;MAAA,WAAAxB,oBAAA,CAAAjC,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,UAAAuB,SAAA;QAAA,IAAAC,MAAA,EAAAC,0BAAA,EAAAC,kBAAA,EAAAC,eAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,QAAA;QAAA,OAAA/B,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAoB,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAlB,IAAA,GAAAkB,SAAA,CAAAtG,IAAA;YAAA;cAAAsG,SAAA,CAAAtG,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAChCyD,MAAI,CAACW,SAAS,EAAE;YAAA;cAA/BT,MAAM,GAAAQ,SAAA,CAAAf,IAAA;cAAAQ,0BAAA;cAAAC,kBAAA;cAAAM,SAAA,CAAAlB,IAAA;cAAAc,UAAA,GAAA3D,cAAA,CACiBqD,MAAI,CAACY,gBAAgB,CAAC1C,KAAK,CAAC;YAAA;cAAAwC,SAAA,CAAAtG,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAA+D,UAAA,CAAAlG,IAAA;YAAA;cAAA,MAAA+F,0BAAA,KAAAI,MAAA,GAAAG,SAAA,CAAAf,IAAA,EAAAlG,IAAA;gBAAAiH,SAAA,CAAAtG,IAAA;gBAAA;cAAA;cAAxCoG,QAAQ,GAAAD,MAAA,CAAA7G,KAAA;cAAAgH,SAAA,CAAAtG,IAAA;cACvB,OAAM,IAAAyG,yBAAkB,EAACX,MAAM,EAAEM,QAAQ,CAAC;YAAA;cAAAL,0BAAA;cAAAO,SAAA,CAAAtG,IAAA;cAAA;YAAA;cAAAsG,SAAA,CAAAtG,IAAA;cAAA;YAAA;cAAAsG,SAAA,CAAAlB,IAAA;cAAAkB,SAAA,CAAAd,EAAA,GAAAc,SAAA;cAAAN,kBAAA;cAAAC,eAAA,GAAAK,SAAA,CAAAd,EAAA;YAAA;cAAAc,SAAA,CAAAlB,IAAA;cAAAkB,SAAA,CAAAlB,IAAA;cAAA,MAAAW,0BAAA,IAAAG,UAAA,CAAAhG,MAAA;gBAAAoG,SAAA,CAAAtG,IAAA;gBAAA;cAAA;cAAAsG,SAAA,CAAAtG,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAAA+D,UAAA,CAAAhG,MAAA;YAAA;cAAAoG,SAAA,CAAAlB,IAAA;cAAA,KAAAY,kBAAA;gBAAAM,SAAA,CAAAtG,IAAA;gBAAA;cAAA;cAAA,MAAAiG,eAAA;YAAA;cAAA,OAAAK,SAAA,CAAAb,MAAA;YAAA;cAAA,OAAAa,SAAA,CAAAb,MAAA;YAAA;YAAA;cAAA,OAAAa,SAAA,CAAAX,IAAA;UAAA;QAAA,GAAAE,QAAA;MAAA;IAE9C;EAAC;IAAA5D,GAAA;IAAA3C,KAAA,EAGD,SAAAkH,iBAAwB1C,KAA6B,EAAE;MAAA,IAAA4C,MAAA;MAAA,WAAAtC,oBAAA,CAAAjC,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,UAAAqC,SAAA;QAAA,IAAAC,UAAA,EAAAC,QAAA,EAAAf,MAAA,EAAAgB,aAAA,EAAAC,aAAA,EAAAX,QAAA;QAAA,OAAA/B,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA+B,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA7B,IAAA,GAAA6B,SAAA,CAAAjH,IAAA;YAAA;cAE/C4G,UAAsB,GAAG,CAAC,CAAA9C,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAE8C,UAAU,KAAI,EAAE,EAAEM,GAAG,CAAC,UAACC,CAAC;gBAAA,OAC7DtI,KAAK,CAACC,OAAO,CAACqI,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC;cAAA,EAC3B;cAAAF,SAAA,CAAAjH,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAEsBuE,MAAI,CAACU,eAAe,EAAE;YAAA;cAAvCP,QAAQ,GAAAI,SAAA,CAAA1B,IAAA;cAAA0B,SAAA,CAAAjH,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EACOuE,MAAI,CAACH,SAAS,EAAE;YAAA;cAA/BT,MAAM,GAAAmB,SAAA,CAAA1B,IAAA;cAENuB,aAAa,GAAG,CAAAD,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEQ,UAAU,CAACrI,MAAM,KAAI,CAAC;cAE7C+H,aAAa,GAAG,CAAC;YAAA;cAAA,MAAEA,aAAa,GAAGD,aAAa;gBAAAG,SAAA,CAAAjH,IAAA;gBAAA;cAAA;cAAAiH,SAAA,CAAAjH,IAAA;cAAA,WAAAsF,qBAAA,CAAAnD,OAAA,EAChCuE,MAAI,CAACY,YAAY,CACtCxB,MAAM,EACNe,QAAQ,CAACQ,UAAU,CAACN,aAAa,CAAC,EAClCH,UAAU,CACX;YAAA;cAJKR,QAAQ,GAAAa,SAAA,CAAA1B,IAAA;cAAA0B,SAAA,CAAAjH,IAAA;cAKd,OAAMoG,QAAQ;YAAA;cAN2CW,aAAa,EAAE;cAAAE,SAAA,CAAAjH,IAAA;cAAA;YAAA;YAAA;cAAA,OAAAiH,SAAA,CAAAtB,IAAA;UAAA;QAAA,GAAAgB,QAAA;MAAA;IAQ5E;EAAC;IAAA1E,GAAA;IAAA3C,KAAA;MAAA,IAAAiI,YAAA,OAAAC,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAED,SAAAmD,SAAA;QAAA,IAAAZ,QAAA;QAAA,OAAAxC,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAyC,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAvC,IAAA,GAAAuC,SAAA,CAAA3H,IAAA;YAAA;cAAA2H,SAAA,CAAA3H,IAAA;cAAA,OACyB,IAAI,CAACoH,eAAe,EAAE;YAAA;cAAvCP,QAAQ,GAAAc,SAAA,CAAApC,IAAA;cAAA,OAAAoC,SAAA,CAAAC,MAAA,WACPC,MAAM,CAAChB,QAAQ,CAACiB,QAAQ,CAAC;YAAA;YAAA;cAAA,OAAAH,SAAA,CAAAhC,IAAA;UAAA;QAAA,GAAA8B,QAAA;MAAA,CACjC;MAAA,SAAAM,YAAA;QAAA,OAAAR,YAAA,CAAA5F,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAiG,WAAA;IAAA;EAAA;IAAA9F,GAAA;IAAA3C,KAAA;MAAA,IAAA0I,UAAA,OAAAR,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAED,SAAA2D,SAAA;QAAA,IAAApB,QAAA,EAAAqB,IAAA,EAAAC,aAAA,EAAAC,gBAAA,EAAAtC,MAAA;QAAA,OAAAzB,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAoD,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAlD,IAAA,GAAAkD,SAAA,CAAAtI,IAAA;YAAA;cAAAsI,SAAA,CAAAtI,IAAA;cAAA,OACyB,IAAI,CAACoH,eAAe,EAAE;YAAA;cAAvCP,QAAQ,GAAAyB,SAAA,CAAA/C,IAAA;cACR2C,IAAI,GAAGrB,QAAQ,CAACf,MAAM,CAAC,CAAC,CAAC;cAAAqC,aAAA,GACI,IAAAI,sBAAY,EAAC1B,QAAQ,CAACf,MAAM,EAAE,CAAC,EAAEoC,IAAI,CAACM,YAAY,CAAE,EAAxEJ,gBAAgB,GAAAD,aAAA,CAAxBrC,MAAM;cACPA,MAAM,GAAG,IAAI2C,qBAAa,CAACL,gBAAgB,CAAC;cAAA,OAAAE,SAAA,CAAAV,MAAA,WAC3C9B,MAAM;YAAA;YAAA;cAAA,OAAAwC,SAAA,CAAA3C,IAAA;UAAA;QAAA,GAAAsC,QAAA;MAAA,CACd;MAAA,SAAA1B,UAAA;QAAA,OAAAyB,UAAA,CAAArG,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAyE,SAAA;IAAA;EAAA;IAAAtE,GAAA;IAAA3C,KAAA;MAAA,IAAAoJ,kBAAA,OAAAlB,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAMD,SAAAqE,SAAA;QAAA,IAAA9B,QAAA,EAAA+B,EAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,EAAA;QAAA,OAAA1E,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA+D,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA7D,IAAA,GAAA6D,SAAA,CAAAjJ,IAAA;YAAA;cAAAiJ,SAAA,CAAAjJ,IAAA;cAAA,OACyB,IAAI,CAACoH,eAAe,EAAE;YAAA;cAAvCP,QAAQ,GAAAoC,SAAA,CAAA1D,IAAA;cACRqD,EAA0B,GAAG,CAAC,CAAC;cAAAC,UAAA,GAAAtK,0BAAA,CACpBsI,QAAQ,CAACqC,kBAAkB;cAAA;gBAA5C,KAAAL,UAAA,CAAA1J,CAAA,MAAA2J,MAAA,GAAAD,UAAA,CAAAzJ,CAAA,IAAAC,IAAA,GAA+C;kBAApC0J,EAAE,GAAAD,MAAA,CAAAxJ,KAAA;kBACXsJ,EAAE,CAACG,EAAE,CAAC9G,GAAG,CAAC,GAAG8G,EAAE,CAACzJ,KAAM;gBACxB;cAAC,SAAAO,GAAA;gBAAAgJ,UAAA,CAAAtJ,CAAA,CAAAM,GAAA;cAAA;gBAAAgJ,UAAA,CAAApJ,CAAA;cAAA;cAAA,OAAAwJ,SAAA,CAAArB,MAAA,WACMgB,EAAE;YAAA;YAAA;cAAA,OAAAK,SAAA,CAAAtD,IAAA;UAAA;QAAA,GAAAgD,QAAA;MAAA,CACV;MAAA,SAAAQ,kBAAA;QAAA,OAAAT,kBAAA,CAAA/G,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAqH,iBAAA;IAAA;EAAA;IAAAlH,GAAA;IAAA3C,KAAA;MAAA,IAAA8J,gBAAA,OAAA5B,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAED,SAAA+E,SAAA;QAAA,OAAAhF,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAqE,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAAnE,IAAA,GAAAmE,SAAA,CAAAvJ,IAAA;YAAA;cAAA,IACO,IAAI,CAAC6G,QAAQ;gBAAA0C,SAAA,CAAAvJ,IAAA;gBAAA;cAAA;cAAAuJ,SAAA,CAAAvJ,IAAA;cAAA,OACV,IAAI,CAACwJ,UAAU,EAAE;YAAA;cACvB,IAAI,CAAC3C,QAAQ,GAAG,IAAI,CAAC4C,UAAU,EAAE;YAAC;cAAA,OAAAF,SAAA,CAAA3B,MAAA,WAE7B,IAAI,CAACf,QAAQ;YAAA;YAAA;cAAA,OAAA0C,SAAA,CAAA5D,IAAA;UAAA;QAAA,GAAA0D,QAAA;MAAA,CACrB;MAAA,SAAAjC,gBAAA;QAAA,OAAAgC,gBAAA,CAAAzH,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAsF,eAAA;IAAA;EAAA;IAAAnF,GAAA;IAAA3C,KAAA;MAAA,IAAAoK,WAAA,OAAAlC,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAKD,SAAAqF,SAAA;QAAA,IAAAC,MAAA,EAAAC,KAAA;QAAA,OAAAxF,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA6E,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA3E,IAAA,GAAA2E,SAAA,CAAA/J,IAAA;YAAA;cAAA+J,SAAA,CAAA/J,IAAA;cAAA,OACuB,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAAC,CAAC,EAAEC,wBAAa,CAACjL,MAAM,CAAC;YAAA;cAAtD4K,MAAM,GAAAG,SAAA,CAAAxE,IAAA;cACNsE,KAAK,GAAGD,MAAM,CAACrJ,QAAQ,EAAE;cAAAwJ,SAAA,CAAAvE,EAAA,GACvBqE,KAAK;cAAAE,SAAA,CAAA/J,IAAA,GAAA+J,SAAA,CAAAvE,EAAA,KACNyE,wBAAa,OAAAF,SAAA,CAAAvE,EAAA,KAEb0E,kCAAuB;cAAA;YAAA;cAAA,OAAAH,SAAA,CAAAnC,MAAA;YAAA;cAAA,MACpB,IAAIuC,KAAK,CAAC,sCAAsC,CAAC;YAAA;cAAA,MAEjD,IAAIA,KAAK,gCAAAC,MAAA,CAAgCP,KAAK,OAAI;YAAA;YAAA;cAAA,OAAAE,SAAA,CAAApE,IAAA;UAAA;QAAA,GAAAgE,QAAA;MAAA,CAE7D;MAAA,SAAAH,WAAA;QAAA,OAAAE,WAAA,CAAA/H,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAA0H,UAAA;IAAA;EAAA;IAAAvH,GAAA;IAAA3C,KAAA;MAAA,IAAA+K,WAAA,OAAA7C,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAGD,SAAAgG,SAAA;QAAA,IAAAC,UAAA,EAAAC,UAAA,EAAAX,KAAA,EAAAY,YAAA,EAAAC,cAAA,EAAAC,WAAA,EAAAC,mBAAA,EAAA/D,QAAA;QAAA,OAAAxC,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA4F,UAAAC,SAAA;UAAA,kBAAAA,SAAA,CAAA1F,IAAA,GAAA0F,SAAA,CAAA9K,IAAA;YAAA;cACQuK,UAAU,GAAGN,wBAAa,CAACjL,MAAM,GAAG,CAAC;cAAA8L,SAAA,CAAA9K,IAAA;cAAA,OAClB,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAAC,IAAI,CAACnG,IAAI,CAACkH,IAAI,GAAGR,UAAU,EAAEA,UAAU,CAAC;YAAA;cAA1EC,UAAU,GAAAM,SAAA,CAAAvF,IAAA;cAEVsE,KAAK,GAAGW,UAAU,CAAChK,KAAK,CAAC,CAAC,CAAC,CAACD,QAAQ,EAAE;cAAA,MACxCsJ,KAAK,KAAKI,wBAAa;gBAAAa,SAAA,CAAA9K,IAAA;gBAAA;cAAA;cAAA,MACnB,IAAImK,KAAK,sCAAAC,MAAA,CAAqCP,KAAK,OAAI;YAAA;cAGzDY,YAAY,GAAGD,UAAU,CAACQ,YAAY,CAAC,CAAC,CAAC;cACzCN,cAAc,GAAG,IAAI,CAAC7G,IAAI,CAACkH,IAAI,GAAGN,YAAY,GAAGF,UAAU;cAAA,MAC7DG,cAAc,GAAGT,wBAAa,CAACjL,MAAM;gBAAA8L,SAAA,CAAA9K,IAAA;gBAAA;cAAA;cAAA,MACjC,IAAImK,KAAK,0BAAAC,MAAA,CAA0BM,cAAc,EAAG;YAAA;cAAAI,SAAA,CAAA9K,IAAA;cAAA,OAGlC,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAACU,cAAc,EAAED,YAAY,CAAC;YAAA;cAAhEE,WAAW,GAAAG,SAAA,CAAAvF,IAAA;cAAAqF,mBAAA,GAGE,IAAAK,6BAAkB,EAACN,WAAW,CAAC,EAA3C9D,QAAQ,GAAA+D,mBAAA,CAAR/D,QAAQ;cAAA,OAAAiE,SAAA,CAAAlD,MAAA,WACRf,QAAQ;YAAA;YAAA;cAAA,OAAAiE,SAAA,CAAAnF,IAAA;UAAA;QAAA,GAAA2E,QAAA;MAAA,CAChB;MAAA,SAAAb,WAAA;QAAA,OAAAY,WAAA,CAAA1I,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAA2H,UAAA;IAAA;EAAA;IAAAxH,GAAA;IAAA3C,KAAA;MAAA,IAAA4L,aAAA,OAAA1D,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAGD,SAAA6G,UACErF,MAAqB,EACrBM,QAAkB,EAClBQ,UAAsB;QAAA,IAAAgD,MAAA,EAAAwB,UAAA,EAAAC,MAAA,EAAAC,QAAA,EAAAC,WAAA,EAAAC,MAAA;QAAA,OAAAnH,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAwG,WAAAC,UAAA;UAAA,kBAAAA,UAAA,CAAAtG,IAAA,GAAAsG,UAAA,CAAA1L,IAAA;YAAA;cAEhB4J,MAAqB,GAAG;gBAC5B+B,QAAQ,EAAE9D,MAAM,CAACzB,QAAQ,CAAC0B,QAAQ,CAAC;gBACnC8D,UAAU,EAAE,CAAC;cACf,CAAC;cAAAR,UAAA,GAAA7M,0BAAA,CACsB6H,QAAQ,CAACyF,OAAO;cAAAH,UAAA,CAAAtG,IAAA;cAAAgG,UAAA,CAAAjM,CAAA;YAAA;cAAA,KAAAkM,MAAA,GAAAD,UAAA,CAAAhM,CAAA,IAAAC,IAAA;gBAAAqM,UAAA,CAAA1L,IAAA;gBAAA;cAAA;cAA5BsL,QAAQ,GAAAD,MAAA,CAAA/L,KAAA;cACXiM,WAAW,GAAGD,QAAQ,CAACQ,SAAS;cAChCN,MAAM,GAAGD,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEQ,cAAc;cAAA,MACtCnF,UAAU,CAAC5H,MAAM,GAAG,CAAC,IAAI,IAAAgN,uBAAY,EAACpF,UAAU,EAAE4E,MAAM,CAAE,GAAG,CAAC;gBAAAE,UAAA,CAAA1L,IAAA;gBAAA;cAAA;cAAA,OAAA0L,UAAA,CAAA9D,MAAA;YAAA;cAAA8D,UAAA,CAAA1L,IAAA;cAAA,OAGxB,IAAI,CAACiM,eAAe,CAACnG,MAAM,EAAEwF,QAAQ,CAAC;YAAA;cAAhF1B,MAAM,CAACgC,UAAU,CAACJ,MAAM,CAAEU,IAAI,EAAE,CAAC,GAAAR,UAAA,CAAAnG,IAAA;YAAA;cAAAmG,UAAA,CAAA1L,IAAA;cAAA;YAAA;cAAA0L,UAAA,CAAA1L,IAAA;cAAA;YAAA;cAAA0L,UAAA,CAAAtG,IAAA;cAAAsG,UAAA,CAAAlG,EAAA,GAAAkG,UAAA;cAAAN,UAAA,CAAA7L,CAAA,CAAAmM,UAAA,CAAAlG,EAAA;YAAA;cAAAkG,UAAA,CAAAtG,IAAA;cAAAgG,UAAA,CAAA3L,CAAA;cAAA,OAAAiM,UAAA,CAAAjG,MAAA;YAAA;cAAA,OAAAiG,UAAA,CAAA9D,MAAA,WAE5BgC,MAAM;YAAA;YAAA;cAAA,OAAA8B,UAAA,CAAA/F,IAAA;UAAA;QAAA,GAAAwF,SAAA;MAAA,CACd;MAAA,SAAA7D,aAAA6E,EAAA,EAAAC,GAAA,EAAAC,GAAA;QAAA,OAAAnB,aAAA,CAAAvJ,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAwF,YAAA;IAAA;EAAA;IAAArF,GAAA;IAAA3C,KAAA;MAAA,IAAAgN,gBAAA,OAAA9E,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CAKD,SAAAiI,UAAsBzG,MAAqB,EAAEwF,QAAqB;QAAA,IAAAkB,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;QAAA,IAAAC,KAAA,EAAAC,IAAA,EAAAC,WAAA,EAAAC,WAAA,EAAAC,SAAA,EAAAC,oBAAA,EAAAC,OAAA,EAAAC,UAAA,EAAAC,oBAAA,EAAAC,gBAAA,EAAAC,QAAA;QAAA,OAAArJ,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAA0I,WAAAC,UAAA;UAAA,kBAAAA,UAAA,CAAAxI,IAAA,GAAAwI,UAAA,CAAA5N,IAAA;YAAA;cAAA,MAC5DsL,QAAQ,CAACuC,SAAS,KAAKC,SAAS,IAAIxC,QAAQ,CAACuC,SAAS,KAAK,IAAI;gBAAAD,UAAA,CAAA5N,IAAA;gBAAA;cAAA;cAAA,MAC3D,IAAImK,KAAK,CAAC,uCAAuC,CAAC;YAAA;cAGpD6C,KAAK,GAAGlH,MAAM,CAACiI,SAAS,EAAAvB,mBAAA,GAAClB,QAAQ,CAACQ,SAAS,cAAAU,mBAAA,uBAAlBA,mBAAA,CAAoBT,cAAc,CAAE;cAC7DkB,IAAmB,GAAG,IAAAe,wBAAa,EAACC,mBAAI,GAAAxB,oBAAA,GAAEnB,QAAQ,CAACQ,SAAS,cAAAW,oBAAA,uBAAlBA,oBAAA,CAAoBQ,IAAI,CAAE;cAAA,MAEtEA,IAAI,KAAKD,KAAK,CAACkB,aAAa;gBAAAN,UAAA,CAAA5N,IAAA;gBAAA;cAAA;cAAA,MACxB,IAAImK,KAAK,oCAAAC,MAAA,CAAoC6C,IAAI,EAAG;YAAA;cAGtDC,WAA+B,GAAG,IAAAc,wBAAa,EACnDG,+BAAgB,GAAAzB,oBAAA,GAChBpB,QAAQ,CAACQ,SAAS,cAAAY,oBAAA,uBAAlBA,oBAAA,CAAoB0B,KAAK,CAC1B;cAEKjB,WAAW,GAAGtF,MAAM,EAAA8E,oBAAA,GAACrB,QAAQ,CAACQ,SAAS,cAAAa,oBAAA,uBAAlBA,oBAAA,CAAoB0B,gBAAgB,CAAE;cAC7DjB,SAAS,GAAGvF,MAAM,EAAA+E,oBAAA,GAACtB,QAAQ,CAACQ,SAAS,cAAAc,oBAAA,uBAAlBA,oBAAA,CAAoB0B,qBAAqB,CAAE;cAElE,IAAI,CAAChD,QAAQ,CAACuC,SAAS,EAAE;gBACvBT,SAAS,GAAGmB,IAAI,CAACC,GAAG,CAClB,IAAI,CAAC3K,IAAI,CAACkH,IAAI,GAAGoC,WAAW,EAC5BtF,MAAM,EAAAwF,oBAAA,GAAC/B,QAAQ,CAACQ,SAAS,cAAAuB,oBAAA,uBAAlBA,oBAAA,CAAoBiB,qBAAqB,CAAC,CAClD;cACH;cAEMhB,OAAuB,GAAG;gBAC9BL,IAAI,EAAJA,IAAI;gBACJwB,SAAS,EAAEzB,KAAK,CAACyB,SAAS;gBAC1BC,SAAS,EAAE1B,KAAK,CAAC0B,SAAS;gBAC1BxB,WAAW,EAAXA,WAAW;gBACXyB,MAAM,EAAE3B,KAAK;gBACb4B,SAAS,GAAA/B,oBAAA,GAAEvB,QAAQ,CAACQ,SAAS,cAAAe,oBAAA,uBAAlBA,oBAAA,CAAoBgC,UAAU;gBACzCtB,UAAU,EAAE;cACd,CAAC;cAIKC,oBAAoB,GAAGlC,QAAQ,aAARA,QAAQ,wBAAAwB,oBAAA,GAARxB,QAAQ,CAAEQ,SAAS,cAAAgB,oBAAA,uBAAnBA,oBAAA,CAAqBgC,sBAAsB;cAAA,KAEpEtB,oBAAoB;gBAAAI,UAAA,CAAA5N,IAAA;gBAAA;cAAA;cAChByN,gBAAgB,GAAG5F,MAAM,CAAC2F,oBAAoB,CAAC;cAAAI,UAAA,CAAA5N,IAAA;cAAA,OAElC,IAAI,CAAC+O,aAAa,CAACtB,gBAAgB,EAAEH,OAAO,EAAEH,WAAW,CAAC;YAAA;cAA7EI,UAAU,GAAAK,UAAA,CAAArI,IAAA;YAAA;cAGZgI,UAAU,GAAG,CAAAR,mBAAA,GAAAO,OAAO,CAACC,UAAU,cAAAR,mBAAA,eAAlBA,mBAAA,CAAoB/N,MAAM,GAAGsO,OAAO,CAACC,UAAU,GAAGA,UAAU;cAACK,UAAA,CAAA5N,IAAA;cAAA,OACnD,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAACmD,WAAW,EAAEC,SAAS,CAAC;YAAA;cAAvDM,QAAQ,GAAAE,UAAA,CAAArI,IAAA;cAAAqI,UAAA,CAAA5N,IAAA;cAAA,OACD,IAAAgP,yBAAe,EAACtB,QAAQ,EAAA9L,aAAA,CAAAA,aAAA,KAAM0L,OAAO;gBAAEC,UAAU,EAAVA;cAAU,GAAE;YAAA;cAAA,OAAAK,UAAA,CAAAhG,MAAA,WAAAgG,UAAA,CAAArI,IAAA;YAAA;YAAA;cAAA,OAAAqI,UAAA,CAAAjI,IAAA;UAAA;QAAA,GAAA4G,SAAA;MAAA,CACjE;MAAA,SAAAN,gBAAAgD,GAAA,EAAAC,GAAA;QAAA,OAAA5C,gBAAA,CAAA3K,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAmK,eAAA;IAAA;EAAA;IAAAhK,GAAA;IAAA3C,KAAA;MAAA,IAAA6P,cAAA,OAAA3H,kBAAA,CAAArF,OAAA,EAAAkC,YAAA,CAAAlC,OAAA,CAAAmC,IAAA,CASD,SAAA8K,UACE5B,oBAA4B,EAC5BF,OAAuB,EACvBH,WAAmB;QAAA,IAAAkC,cAAA,EAAA3B,QAAA,EAAA4B,MAAA,EAAAC,WAAA;QAAA,OAAAlL,YAAA,CAAAlC,OAAA,CAAA8C,IAAA,UAAAuK,WAAAC,UAAA;UAAA,kBAAAA,UAAA,CAAArK,IAAA,GAAAqK,UAAA,CAAAzP,IAAA;YAAA;cAAA,MAEfwN,oBAAoB,KAAK,CAAC;gBAAAiC,UAAA,CAAAzP,IAAA;gBAAA;cAAA;cAAA,OAAAyP,UAAA,CAAA7H,MAAA,WAQrB,EAAE;YAAA;cAGLyH,cAAc,GAAGd,IAAI,CAACC,GAAG,CAC7B,IAAI,CAAC3K,IAAI,CAACkH,IAAI,GAAGyC,oBAAoB,EACrC,IAAI,CAAC1J,KAAK,CAACH,qBAAqB,CACjC;cAAA8L,UAAA,CAAAzP,IAAA;cAAA,OACsB,IAAI,CAAC6D,IAAI,CAACmG,IAAI,CAACwD,oBAAoB,EAAE6B,cAAc,CAAC;YAAA;cAArE3B,QAAQ,GAAA+B,UAAA,CAAAlK,IAAA;cAER+J,MAAM,GAAG;gBAAC1F,MAAM,EAAE8D,QAAQ;gBAAEgC,MAAM,EAAE,CAAC;gBAAE3E,IAAI,EAAE2C,QAAQ,CAAC1O;cAAM,CAAC;cAAAyQ,UAAA,CAAAzP,IAAA;cAAA,OACzC,IAAA2P,oBAAU,EAACL,MAAM,EAAEhC,OAAO,CAAC;YAAA;cAA/CiC,WAAW,GAAAE,UAAA,CAAAlK,IAAA;cAAA,OAAAkK,UAAA,CAAA7H,MAAA,WAEV2H,WAAW,CAAChC,UAAU;YAAA;YAAA;cAAA,OAAAkC,UAAA,CAAA9J,IAAA;UAAA;QAAA,GAAAyJ,SAAA;MAAA,CAC9B;MAAA,SAAAL,cAAAa,GAAA,EAAAC,GAAA,EAAAC,GAAA;QAAA,OAAAX,cAAA,CAAAxN,KAAA,OAAAG,SAAA;MAAA;MAAA,OAAAiN,aAAA;IAAA;EAAA;EAAA,OAAAnL,aAAA;AAAA;AAAAmM,OAAA,CAAAnM,aAAA,GAAAA,aAAA"}

package/dist/es5/parquetjs/schema/declare.js CHANGED Viewed

@@ -8,9 +8,7 @@ exports.ParquetBuffer = void 0;
 var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
 var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-var ParquetBuffer = (0, _createClass2.default)(
-function ParquetBuffer() {
+var ParquetBuffer = (0, _createClass2.default)(function ParquetBuffer() {
   var rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
   var columnData = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
   (0, _classCallCheck2.default)(this, ParquetBuffer);

package/dist/es5/parquetjs/schema/declare.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"declare.js","names":["ParquetBuffer","rowCount","columnData"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' \| 'RLE' \| 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n \| 'UNCOMPRESSED'\n \| 'GZIP'\n \| 'SNAPPY'\n \| 'LZO'\n \| 'BROTLI'\n \| 'LZ4'\n \| 'LZ4_RAW'\n \| 'ZSTD';\nexport type RepetitionType = 'REQUIRED' \| 'OPTIONAL' \| 'REPEATED';\nexport type ParquetType = PrimitiveType \| OriginalType;\n\n/*\n Physical type\n /\nexport type PrimitiveType =\n // Base Types\n \| 'BOOLEAN' // 0\n \| 'INT32' // 1\n \| 'INT64' // 2\n \| 'INT96' // 3\n \| 'FLOAT' // 4\n \| 'DOUBLE' // 5\n \| 'BYTE_ARRAY' // 6,\n \| 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/\n Logical type\n /\nexport type OriginalType =\n // Converted Types\n \| 'UTF8' // 0\n // \| 'MAP' // 1\n // \| 'MAP_KEY_VALUE' // 2\n // \| 'LIST' // 3\n // \| 'ENUM' // 4\n // \| 'DECIMAL' // 5\n \| 'DECIMAL_INT32' // 5\n \| 'DECIMAL_INT64' // 5\n \| 'DECIMAL_BYTE_ARRAY' // 5\n \| 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n \| 'DATE' // 6\n \| 'TIME_MILLIS' // 7\n \| 'TIME_MICROS' // 8\n \| 'TIMESTAMP_MILLIS' // 9\n \| 'TIMESTAMP_MICROS' // 10\n \| 'UINT_8' // 11\n \| 'UINT_16' // 12\n \| 'UINT_32' // 13\n \| 'UINT_64' // 14\n \| 'INT_8' // 15\n \| 'INT_16' // 16\n \| 'INT_32' // 17\n \| 'INT_64' // 18\n \| 'JSON' // 19\n \| 'BSON' // 20\n \| 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n type?: ParquetType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n optional?: boolean;\n repeated?: boolean;\n fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n name: string;\n path: string[];\n key: string;\n primitiveType?: PrimitiveType;\n originalType?: OriginalType;\n repetitionType: RepetitionType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n rLevelMax: number;\n dLevelMax: number;\n isNested?: boolean;\n fieldCount?: number;\n fields?: Record<string, ParquetField>;\n}\n\n/* @todo better name, this is an internal type? /\nexport interface ParquetOptions {\n type: ParquetType;\n rLevelMax: number;\n dLevelMax: number;\n compression: ParquetCompression;\n column: ParquetField;\n numValues?: Int64;\n dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n dlevels: number[];\n rlevels: number[];\n /* Actual column chunks /\n values: any[]; // ArrayLike<any>;\n count: number;\n dictionary?: ParquetDictionary;\n /* The \"raw\" page header from the file /\n pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n [key: string]: any;\n}\n\n/* @\n * Holds data for one row group (column chunks) /\nexport class ParquetBuffer {\n /* Number of rows in this page /\n rowCount: number;\n\n columnData: Record<string, ParquetData>;\n constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n this.rowCount = rowCount;\n this.columnData = columnData;\n }\n}\n\n/* Holds the data for one column chunk */\nexport interface ParquetData {\n dlevels: number[];\n rlevels: number[];\n values: any[];\n count: number;\n pageHeaders: PageHeader[];\n}\n"],"mappings":";;;;;;;;;;IAgIaA,aAAa~~;;AAKxB~~,~~yBAAgF~~;EAAA,~~IAApEC~~,QAAgB,~~uEAAG~~,CAAC;EAAA,~~IAAEC~~,UAAuC,~~uEAAG~~,CAAC,CAAC;EAAA;EAAA;EAAA;EAC5E,IAAI,~~CAACD~~,QAAQ,GAAGA,QAAQ;EACxB,IAAI,~~CAACC~~,UAAU,GAAGA,UAAU;AAC9B,CAAC;~~AAAA~~"}
1	+ {"version":3,"file":"declare.js","names":["ParquetBuffer","_createClass2","default","rowCount","arguments","length","undefined","columnData","_classCallCheck2","_defineProperty2","exports"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' \| 'RLE' \| 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n \| 'UNCOMPRESSED'\n \| 'GZIP'\n \| 'SNAPPY'\n \| 'LZO'\n \| 'BROTLI'\n \| 'LZ4'\n \| 'LZ4_RAW'\n \| 'ZSTD';\nexport type RepetitionType = 'REQUIRED' \| 'OPTIONAL' \| 'REPEATED';\nexport type ParquetType = PrimitiveType \| OriginalType;\n\n/*\n Physical type\n /\nexport type PrimitiveType =\n // Base Types\n \| 'BOOLEAN' // 0\n \| 'INT32' // 1\n \| 'INT64' // 2\n \| 'INT96' // 3\n \| 'FLOAT' // 4\n \| 'DOUBLE' // 5\n \| 'BYTE_ARRAY' // 6,\n \| 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/\n Logical type\n /\nexport type OriginalType =\n // Converted Types\n \| 'UTF8' // 0\n // \| 'MAP' // 1\n // \| 'MAP_KEY_VALUE' // 2\n // \| 'LIST' // 3\n // \| 'ENUM' // 4\n // \| 'DECIMAL' // 5\n \| 'DECIMAL_INT32' // 5\n \| 'DECIMAL_INT64' // 5\n \| 'DECIMAL_BYTE_ARRAY' // 5\n \| 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n \| 'DATE' // 6\n \| 'TIME_MILLIS' // 7\n \| 'TIME_MICROS' // 8\n \| 'TIMESTAMP_MILLIS' // 9\n \| 'TIMESTAMP_MICROS' // 10\n \| 'UINT_8' // 11\n \| 'UINT_16' // 12\n \| 'UINT_32' // 13\n \| 'UINT_64' // 14\n \| 'INT_8' // 15\n \| 'INT_16' // 16\n \| 'INT_32' // 17\n \| 'INT_64' // 18\n \| 'JSON' // 19\n \| 'BSON' // 20\n \| 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n type?: ParquetType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n optional?: boolean;\n repeated?: boolean;\n fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n name: string;\n path: string[];\n key: string;\n primitiveType?: PrimitiveType;\n originalType?: OriginalType;\n repetitionType: RepetitionType;\n typeLength?: number;\n presision?: number;\n scale?: number;\n encoding?: ParquetCodec;\n compression?: ParquetCompression;\n rLevelMax: number;\n dLevelMax: number;\n isNested?: boolean;\n fieldCount?: number;\n fields?: Record<string, ParquetField>;\n}\n\n/* @todo better name, this is an internal type? /\nexport interface ParquetOptions {\n type: ParquetType;\n rLevelMax: number;\n dLevelMax: number;\n compression: ParquetCompression;\n column: ParquetField;\n numValues?: Int64;\n dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n dlevels: number[];\n rlevels: number[];\n /* Actual column chunks /\n values: any[]; // ArrayLike<any>;\n count: number;\n dictionary?: ParquetDictionary;\n /* The \"raw\" page header from the file /\n pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n [key: string]: any;\n}\n\n/* @\n * Holds data for one row group (column chunks) /\nexport class ParquetBuffer {\n /* Number of rows in this page /\n rowCount: number;\n\n columnData: Record<string, ParquetData>;\n constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n this.rowCount = rowCount;\n this.columnData = columnData;\n }\n}\n\n/* Holds the data for one column chunk */\nexport interface ParquetData {\n dlevels: number[];\n rlevels: number[];\n values: any[];\n count: number;\n pageHeaders: PageHeader[];\n}\n"],"mappings":";;;;;;;;;;IAgIaA,aAAa,OAAAC,aAAA,CAAAC,OAAA,EAKxB,SAAAF,cAAA,EAAgF;EAAA,IAApEG,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;EAAA,IAAEG,UAAuC,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;EAAA,IAAAI,gBAAA,CAAAN,OAAA,QAAAF,aAAA;EAAA,IAAAS,gBAAA,CAAAP,OAAA;EAAA,IAAAO,gBAAA,CAAAP,OAAA;EAC5E,IAAI,CAACC,QAAQ,GAAGA,QAAQ;EACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;AAC9B,CAAC;AAAAG,OAAA,CAAAV,aAAA,GAAAA,aAAA"}

package/dist/es5/parquetjs/schema/schema.js CHANGED Viewed

@@ -22,28 +22,23 @@ var ParquetSchema = function () {
     this.fields = buildFields(schema, 0, 0, []);
     this.fieldList = listFields(this.fields);
   }
   (0, _createClass2.default)(ParquetSchema, [{
     key: "findField",
-    value:
-    function findField(path) {
+    value: function findField(path) {
       if (typeof path === 'string') {
         path = path.split(',');
       } else {
         path = path.slice(0);
       }
       var n = this.fields;
       for (; path.length > 1; path.shift()) {
         n = n[path[0]].fields;
       }
       return n[path[0]];
     }
   }, {
     key: "findFieldBranch",
-    value:
-    function findFieldBranch(path) {
+    value: function findFieldBranch(path) {
       if (typeof path === 'string') {
         path = path.split(',');
       }
@@ -93,12 +88,10 @@ function setCompress(schema, type) {
     }
   }
 }
 function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
   var fieldList = {};
   for (var name in schema) {
     var opts = schema[name];
     var required = !opts.optional;
     var repeated = Boolean(opts.repeated);
     var rLevelMax = rLevelParentMax;
@@ -113,7 +106,6 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
       rLevelMax++;
       if (required) dLevelMax++;
     }
     if (opts.fields) {
       var _cpath = path.concat([name]);
       fieldList[name] = {
@@ -129,7 +121,6 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
       };
       continue;
     }
     var typeDef = _types.PARQUET_LOGICAL_TYPES[opts.type];
     if (!typeDef) {
       throw new Error("invalid parquet type: ".concat(opts.type));
@@ -142,7 +133,6 @@ function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
     if (!(opts.compression in _compression.PARQUET_COMPRESSION_METHODS)) {
       throw new Error("unsupported compression method: ".concat(opts.compression));
     }
     var cpath = path.concat([name]);
     fieldList[name] = {
       name: name,

package/dist/es5/parquetjs/schema/schema.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"schema.js","names":["ParquetSchema","schema","fields","buildFields","fieldList","listFields","path","split","slice","n","length","shift","branch","push","record","buffer","shredRecord","materializeRecords","type","setCompress","shredBuffer","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n FieldDefinition,\n ParquetBuffer,\n ParquetCompression,\n ParquetField,\n ParquetRecord,\n RepetitionType,\n SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/*\n A parquet file schema\n /\nexport class ParquetSchema {\n public schema: Record<string, FieldDefinition>;\n public fields: Record<string, ParquetField>;\n public fieldList: ParquetField[];\n\n /\n Create a new schema from a JSON schema definition\n /\n constructor(schema: SchemaDefinition) {\n this.schema = schema;\n this.fields = buildFields(schema, 0, 0, []);\n this.fieldList = listFields(this.fields);\n }\n\n /\n Retrieve a field definition\n /\n findField(path: string \| string[]): ParquetField {\n if (typeof path === 'string') {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.split(',');\n } else {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.slice(0); // clone array\n }\n\n let n = this.fields;\n for (; path.length > 1; path.shift()) {\n n = n[path[0]].fields as Record<string, ParquetField>;\n }\n\n return n[path[0]];\n }\n\n /\n Retrieve a field definition and all the field's ancestors\n /\n findFieldBranch(path: string \| string[]): ParquetField[] {\n if (typeof path === 'string') {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.split(',');\n }\n const branch: ParquetField[] = [];\n let n = this.fields;\n for (; path.length > 0; path.shift()) {\n branch.push(n[path[0]]);\n if (path.length > 1) {\n n = n[path[0]].fields as Record<string, ParquetField>;\n }\n }\n return branch;\n }\n\n shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n shredRecord(this, record, buffer);\n }\n\n materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n return materializeRecords(this, buffer);\n }\n\n compress(type: ParquetCompression): this {\n setCompress(this.schema, type);\n setCompress(this.fields, type);\n return this;\n }\n\n buffer(): ParquetBuffer {\n return shredBuffer(this);\n }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n for (const name in schema) {\n const node = schema[name];\n if (node.fields) {\n setCompress(node.fields, type);\n } else {\n node.compression = type;\n }\n }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n schema: SchemaDefinition,\n rLevelParentMax: number,\n dLevelParentMax: number,\n path: string[]\n): Record<string, ParquetField> {\n const fieldList: Record<string, ParquetField> = {};\n\n for (const name in schema) {\n const opts = schema[name];\n\n / field repetition type /\n const required = !opts.optional;\n const repeated = Boolean(opts.repeated);\n let rLevelMax = rLevelParentMax;\n let dLevelMax = dLevelParentMax;\n\n let repetitionType: RepetitionType = 'REQUIRED';\n if (!required) {\n repetitionType = 'OPTIONAL';\n dLevelMax++;\n }\n if (repeated) {\n repetitionType = 'REPEATED';\n rLevelMax++;\n if (required) dLevelMax++;\n }\n\n / nested field /\n if (opts.fields) {\n const cpath = path.concat([name]);\n fieldList[name] = {\n name,\n path: cpath,\n key: cpath.join(),\n repetitionType,\n rLevelMax,\n dLevelMax,\n isNested: true,\n fieldCount: Object.keys(opts.fields).length,\n fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n };\n continue; // eslint-disable-line no-continue\n }\n\n const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n if (!typeDef) {\n throw new Error(`invalid parquet type: ${opts.type}`);\n }\n\n opts.encoding = opts.encoding \|\| 'PLAIN';\n if (!(opts.encoding in PARQUET_CODECS)) {\n throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n }\n\n opts.compression = opts.compression \|\| 'UNCOMPRESSED';\n if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n throw new Error(`unsupported compression method: ${opts.compression}`);\n }\n\n / add to schema */\n const cpath = path.concat([name]);\n fieldList[name] = {\n name,\n primitiveType: typeDef.primitiveType,\n originalType: typeDef.originalType,\n path: cpath,\n key: cpath.join(),\n repetitionType,\n encoding: opts.encoding,\n compression: opts.compression,\n typeLength: opts.typeLength \|\| typeDef.typeLength,\n presision: opts.presision,\n scale: opts.scale,\n rLevelMax,\n dLevelMax\n };\n }\n return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n let list: ParquetField[] = [];\n for (const k in fields) {\n list.push(fields[k]);\n if (fields[k].isNested) {\n list = list.concat(listFields(fields[k].fields!));\n }\n }\n return list;\n}\n"],"mappings":";;;;;;;;;;AAEA;AACA;AAUA;AACA;AAA8C,IAKjCA,aAAa;EAQxB,uBAAYC,MAAwB,EAAE;IAAA;IAAA;IAAA;IAAA;IACpC,IAAI,CAACA,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACC,MAAM,GAAGC,WAAW,CAACF,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACG,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;;EAAC;IAAA;IAAA;IAKD,mBAAUI,IAAuB,EAAgB;MAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB,CAAC,MAAM;QAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;MACtB;;MAEA,IAAIC,CAAC,GAAG,IAAI,CAACP,MAAM;MACnB,OAAOI,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,EAAE,EAAE;QACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACJ,MAAsC;MACvD;MAEA,OAAOO,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;IACnB;;EAAC;IAAA;IAAA;IAKD,yBAAgBA,IAAuB,EAAkB;MACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB;MACA,IAAMK,MAAsB,GAAG,EAAE;MACjC,IAAIH,CAAC,GAAG,IAAI,CAACP,MAAM;MACnB,OAAOI,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,EAAE,EAAE;QACpCC,MAAM,CAACC,IAAI,CAACJ,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;UACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACJ,MAAsC;QACvD;MACF;MACA,OAAOU,MAAM;IACf;EAAC;IAAA;IAAA,OAED,qBAAYE,MAAqB,EAAEC,MAAqB,EAAQ;MAC9D,IAAAC,kBAAW,EAAC,IAAI,EAAEF,MAAM,EAAEC,MAAM,CAAC;IACnC;EAAC;IAAA;IAAA,OAED,4BAAmBA,MAAqB,EAAmB;MACzD,OAAO,IAAAE,yBAAkB,EAAC,IAAI,EAAEF,MAAM,CAAC;IACzC;EAAC;IAAA;IAAA,OAED,kBAASG,IAAwB,EAAQ;MACvCC,WAAW,CAAC,IAAI,CAAClB,MAAM,EAAEiB,IAAI,CAAC;MAC9BC,WAAW,CAAC,IAAI,CAACjB,MAAM,EAAEgB,IAAI,CAAC;MAC9B,OAAO,IAAI;IACb;EAAC;IAAA;IAAA,OAED,kBAAwB;MACtB,OAAO,IAAAE,kBAAW,EAAC,IAAI,CAAC;IAC1B;EAAC;EAAA;AAAA;AAAA;AAGH,SAASD,WAAW,CAAClB,MAAW,EAAEiB,IAAwB,EAAE;EAC1D,KAAK,IAAMG,IAAI,IAAIpB,MAAM,EAAE;IACzB,IAAMqB,IAAI,GAAGrB,MAAM,CAACoB,IAAI,CAAC;IACzB,IAAIC,IAAI,CAACpB,MAAM,EAAE;MACfiB,WAAW,CAACG,IAAI,CAACpB,MAAM,EAAEgB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLI,IAAI,CAACC,WAAW,GAAGL,IAAI;IACzB;EACF;AACF;;AAGA,SAASf,WAAW,CAClBF,MAAwB,EACxBuB,eAAuB,EACvBC,eAAuB,EACvBnB,IAAc,EACgB;EAC9B,IAAMF,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,IAAMiB,IAAI,IAAIpB,MAAM,EAAE;IACzB,IAAMyB,IAAI,GAAGzB,MAAM,CAACoB,IAAI,CAAC;;IAGzB,IAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,IAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;;IAGA,IAAIN,IAAI,CAACxB,MAAM,EAAE;MACf,IAAMgC,MAAK,GAAG5B,IAAI,CAAC6B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;QAChBA,IAAI,EAAJA,IAAI;QACJf,IAAI,EAAE4B,MAAK;QACXE,GAAG,EAAEF,MAAK,CAACG,IAAI,EAAE;QACjBJ,cAAc,EAAdA,cAAc;QACdF,SAAS,EAATA,SAAS;QACTC,SAAS,EAATA,SAAS;QACTM,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACf,IAAI,CAACxB,MAAM,CAAC,CAACQ,MAAM;QAC3CR,MAAM,EAAEC,WAAW,CAACuB,IAAI,CAACxB,MAAM,EAAE6B,SAAS,EAAEC,SAAS,EAAEE,MAAK;MAC9D,CAAC;MACD;IACF;;IAEA,IAAMQ,OAAY,GAAGC,4BAAqB,CAACjB,IAAI,CAACR,IAAI,CAAE;IACtD,IAAI,CAACwB,OAAO,EAAE;MACZ,MAAM,IAAIE,KAAK,iCAA0BlB,IAAI,CAACR,IAAI,EAAG;IACvD;IAEAQ,IAAI,CAACmB,QAAQ,GAAGnB,IAAI,CAACmB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAEnB,IAAI,CAACmB,QAAQ,IAAIC,sBAAc,CAAC,EAAE;MACtC,MAAM,IAAIF,KAAK,yCAAkClB,IAAI,CAACmB,QAAQ,EAAG;IACnE;IAEAnB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAIwB,wCAA2B,CAAC,EAAE;MACtD,MAAM,IAAIH,KAAK,2CAAoClB,IAAI,CAACH,WAAW,EAAG;IACxE;;IAGA,IAAMW,KAAK,GAAG5B,IAAI,CAAC6B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;MAChBA,IAAI,EAAJA,IAAI;MACJ2B,aAAa,EAAEN,OAAO,CAACM,aAAa;MACpCC,YAAY,EAAEP,OAAO,CAACO,YAAY;MAClC3C,IAAI,EAAE4B,KAAK;MACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,EAAE;MACjBJ,cAAc,EAAdA,cAAc;MACdY,QAAQ,EAAEnB,IAAI,CAACmB,QAAQ;MACvBtB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7B2B,UAAU,EAAExB,IAAI,CAACwB,UAAU,IAAIR,OAAO,CAACQ,UAAU;MACjDC,SAAS,EAAEzB,IAAI,CAACyB,SAAS;MACzBC,KAAK,EAAE1B,IAAI,CAAC0B,KAAK;MACjBrB,SAAS,EAATA,SAAS;MACTC,SAAS,EAATA;IACF,CAAC;EACH;EACA,OAAO5B,SAAS;AAClB;AAEA,SAASC,UAAU,CAACH,MAAoC,EAAkB;EACxE,IAAImD,IAAoB,GAAG,EAAE;EAC7B,KAAK,IAAMC,CAAC,IAAIpD,MAAM,EAAE;IACtBmD,IAAI,CAACxC,IAAI,CAACX,MAAM,CAACoD,CAAC,CAAC,CAAC;IACpB,IAAIpD,MAAM,CAACoD,CAAC,CAAC,CAAChB,QAAQ,EAAE;MACtBe,IAAI,GAAGA,IAAI,CAAClB,MAAM,CAAC9B,UAAU,CAACH,MAAM,CAACoD,CAAC,CAAC,CAACpD,MAAM,CAAE,CAAC;IACnD;EACF;EACA,OAAOmD,IAAI;AACb"}
1	+ {"version":3,"file":"schema.js","names":["_codecs","require","_compression","_shred","_types","ParquetSchema","schema","_classCallCheck2","default","_defineProperty2","fields","buildFields","fieldList","listFields","_createClass2","key","value","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","shredRecord","record","buffer","materializeRecords","compress","type","setCompress","shredBuffer","exports","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n FieldDefinition,\n ParquetBuffer,\n ParquetCompression,\n ParquetField,\n ParquetRecord,\n RepetitionType,\n SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/*\n A parquet file schema\n /\nexport class ParquetSchema {\n public schema: Record<string, FieldDefinition>;\n public fields: Record<string, ParquetField>;\n public fieldList: ParquetField[];\n\n /\n Create a new schema from a JSON schema definition\n /\n constructor(schema: SchemaDefinition) {\n this.schema = schema;\n this.fields = buildFields(schema, 0, 0, []);\n this.fieldList = listFields(this.fields);\n }\n\n /\n Retrieve a field definition\n /\n findField(path: string \| string[]): ParquetField {\n if (typeof path === 'string') {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.split(',');\n } else {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.slice(0); // clone array\n }\n\n let n = this.fields;\n for (; path.length > 1; path.shift()) {\n n = n[path[0]].fields as Record<string, ParquetField>;\n }\n\n return n[path[0]];\n }\n\n /\n Retrieve a field definition and all the field's ancestors\n /\n findFieldBranch(path: string \| string[]): ParquetField[] {\n if (typeof path === 'string') {\n // tslint:disable-next-line:no-parameter-reassignment\n path = path.split(',');\n }\n const branch: ParquetField[] = [];\n let n = this.fields;\n for (; path.length > 0; path.shift()) {\n branch.push(n[path[0]]);\n if (path.length > 1) {\n n = n[path[0]].fields as Record<string, ParquetField>;\n }\n }\n return branch;\n }\n\n shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n shredRecord(this, record, buffer);\n }\n\n materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n return materializeRecords(this, buffer);\n }\n\n compress(type: ParquetCompression): this {\n setCompress(this.schema, type);\n setCompress(this.fields, type);\n return this;\n }\n\n buffer(): ParquetBuffer {\n return shredBuffer(this);\n }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n for (const name in schema) {\n const node = schema[name];\n if (node.fields) {\n setCompress(node.fields, type);\n } else {\n node.compression = type;\n }\n }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n schema: SchemaDefinition,\n rLevelParentMax: number,\n dLevelParentMax: number,\n path: string[]\n): Record<string, ParquetField> {\n const fieldList: Record<string, ParquetField> = {};\n\n for (const name in schema) {\n const opts = schema[name];\n\n / field repetition type /\n const required = !opts.optional;\n const repeated = Boolean(opts.repeated);\n let rLevelMax = rLevelParentMax;\n let dLevelMax = dLevelParentMax;\n\n let repetitionType: RepetitionType = 'REQUIRED';\n if (!required) {\n repetitionType = 'OPTIONAL';\n dLevelMax++;\n }\n if (repeated) {\n repetitionType = 'REPEATED';\n rLevelMax++;\n if (required) dLevelMax++;\n }\n\n / nested field /\n if (opts.fields) {\n const cpath = path.concat([name]);\n fieldList[name] = {\n name,\n path: cpath,\n key: cpath.join(),\n repetitionType,\n rLevelMax,\n dLevelMax,\n isNested: true,\n fieldCount: Object.keys(opts.fields).length,\n fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n };\n continue; // eslint-disable-line no-continue\n }\n\n const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n if (!typeDef) {\n throw new Error(`invalid parquet type: ${opts.type}`);\n }\n\n opts.encoding = opts.encoding \|\| 'PLAIN';\n if (!(opts.encoding in PARQUET_CODECS)) {\n throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n }\n\n opts.compression = opts.compression \|\| 'UNCOMPRESSED';\n if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n throw new Error(`unsupported compression method: ${opts.compression}`);\n }\n\n / add to schema */\n const cpath = path.concat([name]);\n fieldList[name] = {\n name,\n primitiveType: typeDef.primitiveType,\n originalType: typeDef.originalType,\n path: cpath,\n key: cpath.join(),\n repetitionType,\n encoding: opts.encoding,\n compression: opts.compression,\n typeLength: opts.typeLength \|\| typeDef.typeLength,\n presision: opts.presision,\n scale: opts.scale,\n rLevelMax,\n dLevelMax\n };\n }\n return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n let list: ParquetField[] = [];\n for (const k in fields) {\n list.push(fields[k]);\n if (fields[k].isNested) {\n list = list.concat(listFields(fields[k].fields!));\n }\n }\n return list;\n}\n"],"mappings":";;;;;;;;;;AAEA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AAUA,IAAAE,MAAA,GAAAF,OAAA;AACA,IAAAG,MAAA,GAAAH,OAAA;AAA8C,IAKjCI,aAAa;EAQxB,SAAAA,cAAYC,MAAwB,EAAE;IAAA,IAAAC,gBAAA,CAAAC,OAAA,QAAAH,aAAA;IAAA,IAAAI,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA;IACpC,IAAI,CAACF,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACI,MAAM,GAAGC,WAAW,CAACL,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACM,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;EAAC,IAAAI,aAAA,CAAAN,OAAA,EAAAH,aAAA;IAAAU,GAAA;IAAAC,KAAA,EAKD,SAAAC,UAAUC,IAAuB,EAAgB;MAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB,CAAC,MAAM;QAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;MACtB;MAEA,IAAIC,CAAC,GAAG,IAAI,CAACX,MAAM;MACnB,OAAOQ,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,EAAE,EAAE;QACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACR,MAAsC;MACvD;MAEA,OAAOW,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;IACnB;EAAC;IAAAH,GAAA;IAAAC,KAAA,EAKD,SAAAQ,gBAAgBN,IAAuB,EAAkB;MACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB;MACA,IAAMM,MAAsB,GAAG,EAAE;MACjC,IAAIJ,CAAC,GAAG,IAAI,CAACX,MAAM;MACnB,OAAOQ,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,EAAE,EAAE;QACpCE,MAAM,CAACC,IAAI,CAACL,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;UACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACR,MAAsC;QACvD;MACF;MACA,OAAOe,MAAM;IACf;EAAC;IAAAV,GAAA;IAAAC,KAAA,EAED,SAAAW,YAAYC,MAAqB,EAAEC,MAAqB,EAAQ;MAC9D,IAAAF,kBAAW,EAAC,IAAI,EAAEC,MAAM,EAAEC,MAAM,CAAC;IACnC;EAAC;IAAAd,GAAA;IAAAC,KAAA,EAED,SAAAc,mBAAmBD,MAAqB,EAAmB;MACzD,OAAO,IAAAC,yBAAkB,EAAC,IAAI,EAAED,MAAM,CAAC;IACzC;EAAC;IAAAd,GAAA;IAAAC,KAAA,EAED,SAAAe,SAASC,IAAwB,EAAQ;MACvCC,WAAW,CAAC,IAAI,CAAC3B,MAAM,EAAE0B,IAAI,CAAC;MAC9BC,WAAW,CAAC,IAAI,CAACvB,MAAM,EAAEsB,IAAI,CAAC;MAC9B,OAAO,IAAI;IACb;EAAC;IAAAjB,GAAA;IAAAC,KAAA,EAED,SAAAa,OAAA,EAAwB;MACtB,OAAO,IAAAK,kBAAW,EAAC,IAAI,CAAC;IAC1B;EAAC;EAAA,OAAA7B,aAAA;AAAA;AAAA8B,OAAA,CAAA9B,aAAA,GAAAA,aAAA;AAGH,SAAS4B,WAAWA,CAAC3B,MAAW,EAAE0B,IAAwB,EAAE;EAC1D,KAAK,IAAMI,IAAI,IAAI9B,MAAM,EAAE;IACzB,IAAM+B,IAAI,GAAG/B,MAAM,CAAC8B,IAAI,CAAC;IACzB,IAAIC,IAAI,CAAC3B,MAAM,EAAE;MACfuB,WAAW,CAACI,IAAI,CAAC3B,MAAM,EAAEsB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLK,IAAI,CAACC,WAAW,GAAGN,IAAI;IACzB;EACF;AACF;AAGA,SAASrB,WAAWA,CAClBL,MAAwB,EACxBiC,eAAuB,EACvBC,eAAuB,EACvBtB,IAAc,EACgB;EAC9B,IAAMN,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,IAAMwB,IAAI,IAAI9B,MAAM,EAAE;IACzB,IAAMmC,IAAI,GAAGnC,MAAM,CAAC8B,IAAI,CAAC;IAGzB,IAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,IAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;IAGA,IAAIN,IAAI,CAAC/B,MAAM,EAAE;MACf,IAAMuC,MAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCxB,SAAS,CAACwB,IAAI,CAAC,GAAG;QAChBA,IAAI,EAAJA,IAAI;QACJlB,IAAI,EAAE+B,MAAK;QACXlC,GAAG,EAAEkC,MAAK,CAACE,IAAI,EAAE;QACjBH,cAAc,EAAdA,cAAc;QACdF,SAAS,EAATA,SAAS;QACTC,SAAS,EAATA,SAAS;QACTK,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACd,IAAI,CAAC/B,MAAM,CAAC,CAACY,MAAM;QAC3CZ,MAAM,EAAEC,WAAW,CAAC8B,IAAI,CAAC/B,MAAM,EAAEoC,SAAS,EAAEC,SAAS,EAAEE,MAAK;MAC9D,CAAC;MACD;IACF;IAEA,IAAMO,OAAY,GAAGC,4BAAqB,CAAChB,IAAI,CAACT,IAAI,CAAE;IACtD,IAAI,CAACwB,OAAO,EAAE;MACZ,MAAM,IAAIE,KAAK,0BAAAR,MAAA,CAA0BT,IAAI,CAACT,IAAI,EAAG;IACvD;IAEAS,IAAI,CAACkB,QAAQ,GAAGlB,IAAI,CAACkB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAElB,IAAI,CAACkB,QAAQ,IAAIC,sBAAc,CAAC,EAAE;MACtC,MAAM,IAAIF,KAAK,kCAAAR,MAAA,CAAkCT,IAAI,CAACkB,QAAQ,EAAG;IACnE;IAEAlB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAIuB,wCAA2B,CAAC,EAAE;MACtD,MAAM,IAAIH,KAAK,oCAAAR,MAAA,CAAoCT,IAAI,CAACH,WAAW,EAAG;IACxE;IAGA,IAAMW,KAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCxB,SAAS,CAACwB,IAAI,CAAC,GAAG;MAChBA,IAAI,EAAJA,IAAI;MACJ0B,aAAa,EAAEN,OAAO,CAACM,aAAa;MACpCC,YAAY,EAAEP,OAAO,CAACO,YAAY;MAClC7C,IAAI,EAAE+B,KAAK;MACXlC,GAAG,EAAEkC,KAAK,CAACE,IAAI,EAAE;MACjBH,cAAc,EAAdA,cAAc;MACdW,QAAQ,EAAElB,IAAI,CAACkB,QAAQ;MACvBrB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7B0B,UAAU,EAAEvB,IAAI,CAACuB,UAAU,IAAIR,OAAO,CAACQ,UAAU;MACjDC,SAAS,EAAExB,IAAI,CAACwB,SAAS;MACzBC,KAAK,EAAEzB,IAAI,CAACyB,KAAK;MACjBpB,SAAS,EAATA,SAAS;MACTC,SAAS,EAATA;IACF,CAAC;EACH;EACA,OAAOnC,SAAS;AAClB;AAEA,SAASC,UAAUA,CAACH,MAAoC,EAAkB;EACxE,IAAIyD,IAAoB,GAAG,EAAE;EAC7B,KAAK,IAAMC,CAAC,IAAI1D,MAAM,EAAE;IACtByD,IAAI,CAACzC,IAAI,CAAChB,MAAM,CAAC0D,CAAC,CAAC,CAAC;IACpB,IAAI1D,MAAM,CAAC0D,CAAC,CAAC,CAAChB,QAAQ,EAAE;MACtBe,IAAI,GAAGA,IAAI,CAACjB,MAAM,CAACrC,UAAU,CAACH,MAAM,CAAC0D,CAAC,CAAC,CAAC1D,MAAM,CAAE,CAAC;IACnD;EACF;EACA,OAAOyD,IAAI;AACb"}

package/dist/es5/parquetjs/schema/shred.js CHANGED Viewed

@@ -19,7 +19,7 @@ function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "functio
 function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
 function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
 function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
-function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) arr2[i] = arr[i]; return arr2; }
 function shredBuffer(schema) {
   var columnData = {};
   var _iterator = _createForOfIteratorHelper(schema.fieldList),
@@ -45,11 +45,9 @@ function shredBuffer(schema) {
     columnData: columnData
   };
 }
 function shredRecord(schema, record, buffer) {
   var data = shredBuffer(schema).columnData;
   shredRecordFields(schema.fields, record, data, 0, 0);
   if (buffer.rowCount === 0) {
     buffer.rowCount = 1;
     buffer.columnData = data;
@@ -72,11 +70,9 @@ function shredRecord(schema, record, buffer) {
     _iterator2.f();
   }
 }
 function shredRecordFields(fields, record, data, rLevel, dLevel) {
   for (var name in fields) {
     var field = fields[name];
     var values = [];
     if (record && field.name in record && record[field.name] !== undefined && record[field.name] !== null) {
       if (record[field.name].constructor === Array) {
@@ -91,7 +87,6 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
     if (values.length > 1 && field.repetitionType !== 'REPEATED') {
       throw new Error("too many values for field: ".concat(field.name));
     }
     if (values.length === 0) {
       if (field.isNested) {
         shredRecordFields(field.fields, null, data, rLevel, dLevel);
@@ -102,7 +97,6 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
       }
       continue;
     }
     for (var i = 0; i < values.length; i++) {
       var rlvl = i === 0 ? rLevel : field.rLevelMax;
       if (field.isNested) {
@@ -116,7 +110,6 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
     }
   }
 }
 function materializeRecords(schema, buffer) {
   var records = [];
   for (var i = 0; i < buffer.rowCount; i++) {
@@ -130,11 +123,9 @@ function materializeRecords(schema, buffer) {
   }
   return records;
 }
 function materializeColumn(schema, columnData, key, records) {
   var field = schema.findField(key);
   var branch = schema.findFieldBranch(key);
   var rLevels = new Array(field.rLevelMax + 1).fill(0);
   var vIndex = 0;
   for (var i = 0; i < columnData.count; i++) {
@@ -144,7 +135,6 @@ function materializeColumn(schema, columnData, key, records) {
     rLevels.fill(0, rLevel + 1);
     var rIndex = 0;
     var record = records[rLevels[rIndex++] - 1];
     var _iterator3 = _createForOfIteratorHelper(branch),
       _step3;
     try {
@@ -169,15 +159,13 @@ function materializeColumn(schema, columnData, key, records) {
             record = record[step.name];
         }
       }
     } catch (err) {
       _iterator3.e(err);
     } finally {
       _iterator3.f();
     }
     if (dLevel === field.dLevelMax) {
-      var value = Types.fromPrimitive(
-      field.originalType || field.primitiveType, columnData.values[vIndex], field);
+      var value = Types.fromPrimitive(field.originalType || field.primitiveType, columnData.values[vIndex], field);
       vIndex++;
       switch (field.repetitionType) {
         case 'REPEATED':

package/dist/es5/parquetjs/schema/shred.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"shred.js","names":["shredBuffer","schema","columnData","fieldList","field","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","isNested","i","rlvl","rLevelMax","dLevelMax","Types","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n const columnData: Record<string, ParquetData> = {};\n for (const field of schema.fieldList) {\n columnData[field.key] = {\n dlevels: [],\n rlevels: [],\n values: [],\n pageHeaders: [],\n count: 0\n };\n }\n return {rowCount: 0, columnData};\n}\n\n/*\n 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n \n The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n \n The format in which the shredded records will be stored in the buffer is as\n * follows:\n \n buffer = {\n * columnData: [\n * 'my_col': {\n * dlevels: [d1, d2, .. dN],\n * rlevels: [r1, r2, .. rN],\n * values: [v1, v2, .. vN],\n * }, ...\n * ],\n * rowCount: X,\n * }\n /\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n / shred the record, this may raise an exception /\n const data = shredBuffer(schema).columnData;\n\n shredRecordFields(schema.fields, record, data, 0, 0);\n\n / if no error during shredding, add the shredded record to the buffer /\n if (buffer.rowCount === 0) {\n buffer.rowCount = 1;\n buffer.columnData = data;\n return;\n }\n buffer.rowCount += 1;\n for (const field of schema.fieldList) {\n Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n buffer.columnData[field.key].count += data[field.key].count;\n }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n fields: Record<string, ParquetField>,\n record: any,\n data: Record<string, ParquetData>,\n rLevel: number,\n dLevel: number\n) {\n for (const name in fields) {\n const field = fields[name];\n\n // fetch values\n let values: any[] = [];\n if (\n record &&\n field.name in record &&\n record[field.name] !== undefined &&\n record[field.name] !== null\n ) {\n if (record[field.name].constructor === Array) {\n values = record[field.name];\n } else {\n values.push(record[field.name]);\n }\n }\n // check values\n if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n throw new Error(`missing required field: ${field.name}`);\n }\n if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n throw new Error(`too many values for field: ${field.name}`);\n }\n\n // push null\n if (values.length === 0) {\n if (field.isNested) {\n shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n } else {\n data[field.key].count += 1;\n data[field.key].rlevels.push(rLevel);\n data[field.key].dlevels.push(dLevel);\n }\n continue; // eslint-disable-line no-continue\n }\n\n // push values\n for (let i = 0; i < values.length; i++) {\n const rlvl = i === 0 ? rLevel : field.rLevelMax;\n if (field.isNested) {\n shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n } else {\n data[field.key].count += 1;\n data[field.key].rlevels.push(rlvl);\n data[field.key].dlevels.push(field.dLevelMax);\n data[field.key].values.push(\n Types.toPrimitive((field.originalType \|\| field.primitiveType)!, values[i])\n );\n }\n }\n }\n}\n\n/\n 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n \n The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n \n buffer = {\n * columnData: [\n * 'my_col': {\n * dlevels: [d1, d2, .. dN],\n * rlevels: [r1, r2, .. rN],\n * values: [v1, v2, .. vN],\n * }, ...\n * ],\n * rowCount: X,\n * }\n /\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n const records: ParquetRecord[] = [];\n for (let i = 0; i < buffer.rowCount; i++) {\n records.push({});\n }\n for (const key in buffer.columnData) {\n const columnData = buffer.columnData[key];\n if (columnData.count) {\n materializeColumn(schema, columnData, key, records);\n }\n }\n return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n schema: ParquetSchema,\n columnData: ParquetData,\n key: string,\n records: ParquetRecord[]\n): void {\n const field = schema.findField(key);\n const branch = schema.findFieldBranch(key);\n\n // tslint:disable-next-line:prefer-array-literal\n const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n let vIndex = 0;\n for (let i = 0; i < columnData.count; i++) {\n const dLevel = columnData.dlevels[i];\n const rLevel = columnData.rlevels[i];\n rLevels[rLevel]++;\n rLevels.fill(0, rLevel + 1);\n\n let rIndex = 0;\n let record = records[rLevels[rIndex++] - 1];\n\n // Internal nodes - Build a nested row object\n for (const step of branch) {\n if (step === field \|\| dLevel < step.dLevelMax) {\n break;\n }\n\n switch (step.repetitionType) {\n case 'REPEATED':\n if (!(step.name in record)) {\n // eslint-disable max-depth\n record[step.name] = [];\n }\n const ix = rLevels[rIndex++];\n while (record[step.name].length <= ix) {\n // eslint-disable max-depth\n record[step.name].push({});\n }\n record = record[step.name][ix];\n break;\n\n default:\n record[step.name] = record[step.name] \|\| {};\n record = record[step.name];\n }\n }\n\n // Leaf node - Add the value\n if (dLevel === field.dLevelMax) {\n const value = Types.fromPrimitive(\n // @ts-ignore\n field.originalType \|\| field.primitiveType,\n columnData.values[vIndex],\n field\n );\n vIndex++;\n\n switch (field.repetitionType) {\n case 'REPEATED':\n if (!(field.name in record)) {\n // eslint-disable max-depth\n record[field.name] = [];\n }\n const ix = rLevels[rIndex];\n while (record[field.name].length <= ix) {\n // eslint-disable max-depth\n record[field.name].push(null);\n }\n record[field.name][ix] = value;\n break;\n\n default:\n record[field.name] = value;\n }\n }\n }\n}\n\n// Columnar export\n\n/\n 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n \n The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n \n buffer = {\n * columnData: [\n * 'my_col': {\n * dlevels: [d1, d2, .. dN],\n * rlevels: [r1, r2, .. rN],\n * values: [v1, v2, .. vN],\n * }, ...\n * ],\n * rowCount: X,\n * }\n \nexport function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {\n const columns: ParquetRecord = {};\n for (const key in buffer.columnData) {\n const columnData = buffer.columnData[key];\n if (columnData.count) {\n extractColumn(schema, columnData, key, columns);\n }\n }\n return columns;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction extractColumn(\n schema: ParquetSchema,\n columnData: ParquetData,\n key: string,\n columns: Record<string, unknown> \n) {\n if (columnData.count <= 0) {\n return;\n }\n\n const record = columns;\n\n const field = schema.findField(key);\n const branch = schema.findFieldBranch(key);\n\n // tslint:disable-next-line:prefer-array-literal\n const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n let vIndex = 0;\n\n let i = 0;\n const dLevel = columnData.dlevels[i];\n const rLevel = columnData.rlevels[i];\n rLevels[rLevel]++;\n rLevels.fill(0, rLevel + 1);\n\n let rIndex = 0;\n let record = records[rLevels[rIndex++] - 1];\n\n // Internal nodes\n for (const step of branch) {\n if (step === field \|\| dLevel < step.dLevelMax) {\n break;\n }\n\n switch (step.repetitionType) {\n case 'REPEATED':\n if (!(step.name in record)) {\n // eslint-disable max-depth\n record[step.name] = [];\n }\n const ix = rLevels[rIndex++];\n while (record[step.name].length <= ix) {\n // eslint-disable max-depth\n record[step.name].push({});\n }\n record = record[step.name][ix];\n break;\n\n default:\n record[step.name] = record[step.name] \|\| {};\n record = record[step.name];\n }\n }\n\n // Leaf node\n if (dLevel === field.dLevelMax) {\n const value = Types.fromPrimitive(\n // @ts-ignore\n field.originalType \|\| field.primitiveType,\n columnData.values[vIndex],\n field\n );\n vIndex++;\n\n switch (field.repetitionType) {\n case 'REPEATED':\n if (!(field.name in record)) {\n // eslint-disable max-depth\n record[field.name] = [];\n }\n const ix = rLevels[rIndex];\n while (record[field.name].length <= ix) {\n // eslint-disable max-depth\n record[field.name].push(null);\n }\n record[field.name][ix] = value;\n break;\n\n default:\n record[field.name] = value;\n }\n }\n}\n/\n"],"mappings":";;;;;;;;;;;;;;;AAEA;AAEA;AAAiC;AAAA;AAAA;AAAA;AAAA;AAI1B,SAASA,WAAW,CAACC,MAAqB,EAAiB;EAChE,IAAMC,UAAuC,GAAG,CAAC,CAAC;EAAC,2CAC/BD,MAAM,CAACE,SAAS;IAAA;EAAA;IAApC,oDAAsC;MAAA,IAA3BC,KAAK;MACdF,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,GAAG;QACtBC,OAAO,EAAE,EAAE;QACXC,OAAO,EAAE,EAAE;QACXC,MAAM,EAAE,EAAE;QACVC,WAAW,EAAE,EAAE;QACfC,KAAK,EAAE;MACT,CAAC;IACH;EAAC;IAAA;EAAA;IAAA;EAAA;EACD,OAAO;IAACC,QAAQ,EAAE,CAAC;IAAET,UAAU,EAAVA;EAAU,CAAC;AAClC;;AAwBO,SAASU,WAAW,CAACX,MAAqB,EAAEY,MAAW,EAAEC,MAAqB,EAAQ;EAE3F,IAAMC,IAAI,GAAGf,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Cc,iBAAiB,CAACf,MAAM,CAACgB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;;EAGpD,IAAID,MAAM,CAACH,QAAQ,KAAK,CAAC,EAAE;IACzBG,MAAM,CAACH,QAAQ,GAAG,CAAC;IACnBG,MAAM,CAACZ,UAAU,GAAGa,IAAI;IACxB;EACF;EACAD,MAAM,CAACH,QAAQ,IAAI,CAAC;EAAC,4CACDV,MAAM,CAACE,SAAS;IAAA;EAAA;IAApC,uDAAsC;MAAA,IAA3BC,KAAK;MACdc,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,EAAEQ,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,CAAC;MACzFW,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,EAAES,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,CAAC;MACzFY,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACG,MAAM,EAAEO,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACG,MAAM,CAAC;MACvFM,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK,IAAIK,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK;IAC7D;EAAC;IAAA;EAAA;IAAA;EAAA;AACH;;AAGA,SAASM,iBAAiB,CACxBC,MAAoC,EACpCJ,MAAW,EACXE,IAAiC,EACjCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,IAAMC,IAAI,IAAIP,MAAM,EAAE;IACzB,IAAMb,KAAK,GAAGa,MAAM,CAACO,IAAI,CAAC;;IAG1B,IAAIhB,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNT,KAAK,CAACoB,IAAI,IAAIX,MAAM,IACpBA,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,KAAKC,SAAS,IAChCZ,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIX,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAACE,WAAW,KAAKR,KAAK,EAAE;QAC5CV,MAAM,GAAGK,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLhB,MAAM,CAACY,IAAI,CAACP,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,IAAIC,OAAO,CAACf,MAAM,CAAC,IAAIT,KAAK,CAACyB,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,mCAA4B1B,KAAK,CAACoB,IAAI,EAAG;IAC1D;IACA,IAAIhB,MAAM,CAACmB,MAAM,GAAG,CAAC,IAAIvB,KAAK,CAACyB,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,sCAA+B1B,KAAK,CAACoB,IAAI,EAAG;IAC7D;;IAGA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,EAAE;MACvB,IAAIvB,KAAK,CAAC2B,QAAQ,EAAE;QAClBf,iBAAiB,CAACZ,KAAK,CAACa,MAAM,EAAG,IAAI,EAAEF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC9D,CAAC,MAAM;QACLR,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;;IAGA,KAAK,IAAIS,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGxB,MAAM,CAACmB,MAAM,EAAEK,CAAC,EAAE,EAAE;MACtC,IAAMC,IAAI,GAAGD,CAAC,KAAK,CAAC,GAAGV,MAAM,GAAGlB,KAAK,CAAC8B,SAAS;MAC/C,IAAI9B,KAAK,CAAC2B,QAAQ,EAAE;QAClBf,iBAAiB,CAACZ,KAAK,CAACa,MAAM,EAAGT,MAAM,CAACwB,CAAC,CAAC,EAAEjB,IAAI,EAAEkB,IAAI,EAAE7B,KAAK,CAAC+B,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLpB,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACa,IAAI,CAAC;QAClClB,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAAChB,KAAK,CAAC+B,SAAS,CAAC;QAC7CpB,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACG,MAAM,CAACY,IAAI,CACzBgB,KAAK,CAACC,WAAW,CAAEjC,KAAK,CAACkC,YAAY,IAAIlC,KAAK,CAACmC,aAAa,EAAI/B,MAAM,CAACwB,CAAC,CAAC,CAAC,CAC3E;MACH;IACF;EACF;AACF;;AAqBO,SAASQ,kBAAkB,CAACvC,MAAqB,EAAEa,MAAqB,EAAmB;EAChG,IAAM2B,OAAwB,GAAG,EAAE;EACnC,KAAK,IAAIT,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGlB,MAAM,CAACH,QAAQ,EAAEqB,CAAC,EAAE,EAAE;IACxCS,OAAO,CAACrB,IAAI,CAAC,CAAC,CAAC,CAAC;EAClB;EACA,KAAK,IAAMf,GAAG,IAAIS,MAAM,CAACZ,UAAU,EAAE;IACnC,IAAMA,UAAU,GAAGY,MAAM,CAACZ,UAAU,CAACG,GAAG,CAAC;IACzC,IAAIH,UAAU,CAACQ,KAAK,EAAE;MACpBgC,iBAAiB,CAACzC,MAAM,EAAEC,UAAU,EAAEG,GAAG,EAAEoC,OAAO,CAAC;IACrD;EACF;EACA,OAAOA,OAAO;AAChB;;AAGA,SAASC,iBAAiB,CACxBzC,MAAqB,EACrBC,UAAuB,EACvBG,GAAW,EACXoC,OAAwB,EAClB;EACN,IAAMrC,KAAK,GAAGH,MAAM,CAAC0C,SAAS,CAACtC,GAAG,CAAC;EACnC,IAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAe,CAACxC,GAAG,CAAC;;EAG1C,IAAMyC,OAAiB,GAAG,IAAI5B,KAAK,CAACd,KAAK,CAAC8B,SAAS,GAAG,CAAC,CAAC,CAACa,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIhB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG9B,UAAU,CAACQ,KAAK,EAAEsB,CAAC,EAAE,EAAE;IACzC,IAAMT,MAAM,GAAGrB,UAAU,CAACI,OAAO,CAAC0B,CAAC,CAAC;IACpC,IAAMV,MAAM,GAAGpB,UAAU,CAACK,OAAO,CAACyB,CAAC,CAAC;IACpCc,OAAO,CAACxB,MAAM,CAAC,EAAE;IACjBwB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAEzB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI2B,MAAM,GAAG,CAAC;IACd,IAAIpC,MAAM,GAAG4B,OAAO,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;;IAAC,4CAGzBL,MAAM;MAAA;IAAA;MAAzB,uDAA2B;QAAA,IAAhBM,IAAI;QACb,IAAIA,IAAI,KAAK9C,KAAK,IAAImB,MAAM,GAAG2B,IAAI,CAACf,SAAS,EAAE;UAC7C;QACF;QAEA,QAAQe,IAAI,CAACrB,cAAc;UACzB,KAAK,UAAU;YACb,IAAI,EAAEqB,IAAI,CAAC1B,IAAI,IAAIX,MAAM,CAAC,EAAE;cAE1BA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAG,EAAE;YACxB;YACA,IAAM2B,GAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;YAC5B,OAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACG,MAAM,IAAIwB,GAAE,EAAE;cAErCtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B;YACAP,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAAC2B,GAAE,CAAC;YAC9B;UAEF;YACEtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAGX,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3CX,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC;QAAC;MAEjC;;IAAC;MAAA;IAAA;MAAA;IAAA;IAGD,IAAID,MAAM,KAAKnB,KAAK,CAAC+B,SAAS,EAAE;MAC9B,IAAMiB,KAAK,GAAGhB,KAAK,CAACiB,aAAa;MAE/BjD,KAAK,CAACkC,YAAY,IAAIlC,KAAK,CAACmC,aAAa,EACzCrC,UAAU,CAACM,MAAM,CAACwC,MAAM,CAAC,EACzB5C,KAAK,CACN;MACD4C,MAAM,EAAE;MAER,QAAQ5C,KAAK,CAACyB,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAEzB,KAAK,CAACoB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,IAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOpC,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAEtCtC,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAAC2B,EAAE,CAAC,GAAGC,KAAK;UAC9B;QAEF;UACEvC,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,GAAG4B,KAAK;MAAC;IAEjC;EACF;AACF"}
1	+ {"version":3,"file":"shred.js","names":["_declare","require","Types","_interopRequireWildcard","_getRequireWildcardCache","nodeInterop","WeakMap","cacheBabelInterop","cacheNodeInterop","obj","__esModule","_typeof","default","cache","has","get","newObj","hasPropertyDescriptor","Object","defineProperty","getOwnPropertyDescriptor","key","prototype","hasOwnProperty","call","desc","set","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","step","next","_e2","return","minLen","_arrayLikeToArray","toString","slice","constructor","name","from","test","arr","len","arr2","shredBuffer","schema","columnData","_iterator","fieldList","_step","field","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","_iterator2","_step2","push","apply","rLevel","dLevel","undefined","Boolean","repetitionType","Error","concat","isNested","rlvl","rLevelMax","dLevelMax","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","_iterator3","_step3","ix","fromPrimitive"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n const columnData: Record<string, ParquetData> = {};\n for (const field of schema.fieldList) {\n columnData[field.key] = {\n dlevels: [],\n rlevels: [],\n values: [],\n pageHeaders: [],\n count: 0\n };\n }\n return {rowCount: 0, columnData};\n}\n\n/*\n 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n \n The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n \n The format in which the shredded records will be stored in the buffer is as\n * follows:\n \n buffer = {\n * columnData: [\n * 'my_col': {\n * dlevels: [d1, d2, .. dN],\n * rlevels: [r1, r2, .. rN],\n * values: [v1, v2, .. vN],\n * }, ...\n * ],\n * rowCount: X,\n * }\n /\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n / shred the record, this may raise an exception /\n const data = shredBuffer(schema).columnData;\n\n shredRecordFields(schema.fields, record, data, 0, 0);\n\n / if no error during shredding, add the shredded record to the buffer /\n if (buffer.rowCount === 0) {\n buffer.rowCount = 1;\n buffer.columnData = data;\n return;\n }\n buffer.rowCount += 1;\n for (const field of schema.fieldList) {\n Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n buffer.columnData[field.key].count += data[field.key].count;\n }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n fields: Record<string, ParquetField>,\n record: any,\n data: Record<string, ParquetData>,\n rLevel: number,\n dLevel: number\n) {\n for (const name in fields) {\n const field = fields[name];\n\n // fetch values\n let values: any[] = [];\n if (\n record &&\n field.name in record &&\n record[field.name] !== undefined &&\n record[field.name] !== null\n ) {\n if (record[field.name].constructor === Array) {\n values = record[field.name];\n } else {\n values.push(record[field.name]);\n }\n }\n // check values\n if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n throw new Error(`missing required field: ${field.name}`);\n }\n if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n throw new Error(`too many values for field: ${field.name}`);\n }\n\n // push null\n if (values.length === 0) {\n if (field.isNested) {\n shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n } else {\n data[field.key].count += 1;\n data[field.key].rlevels.push(rLevel);\n data[field.key].dlevels.push(dLevel);\n }\n continue; // eslint-disable-line no-continue\n }\n\n // push values\n for (let i = 0; i < values.length; i++) {\n const rlvl = i === 0 ? rLevel : field.rLevelMax;\n if (field.isNested) {\n shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n } else {\n data[field.key].count += 1;\n data[field.key].rlevels.push(rlvl);\n data[field.key].dlevels.push(field.dLevelMax);\n data[field.key].values.push(\n Types.toPrimitive((field.originalType \|\| field.primitiveType)!, values[i])\n );\n }\n }\n }\n}\n\n/\n 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n \n The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n \n buffer = {\n * columnData: [\n * 'my_col': {\n * dlevels: [d1, d2, .. dN],\n * rlevels: [r1, r2, .. rN],\n * values: [v1, v2, .. vN],\n * }, ...\n * ],\n * rowCount: X,\n * }\n /\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n const records: ParquetRecord[] = [];\n for (let i = 0; i < buffer.rowCount; i++) {\n records.push({});\n }\n for (const key in buffer.columnData) {\n const columnData = buffer.columnData[key];\n if (columnData.count) {\n materializeColumn(schema, columnData, key, records);\n }\n }\n return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n schema: ParquetSchema,\n columnData: ParquetData,\n key: string,\n records: ParquetRecord[]\n): void {\n const field = schema.findField(key);\n const branch = schema.findFieldBranch(key);\n\n // tslint:disable-next-line:prefer-array-literal\n const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n let vIndex = 0;\n for (let i = 0; i < columnData.count; i++) {\n const dLevel = columnData.dlevels[i];\n const rLevel = columnData.rlevels[i];\n rLevels[rLevel]++;\n rLevels.fill(0, rLevel + 1);\n\n let rIndex = 0;\n let record = records[rLevels[rIndex++] - 1];\n\n // Internal nodes - Build a nested row object\n for (const step of branch) {\n if (step === field \|\| dLevel < step.dLevelMax) {\n break;\n }\n\n switch (step.repetitionType) {\n case 'REPEATED':\n if (!(step.name in record)) {\n // eslint-disable max-depth\n record[step.name] = [];\n }\n const ix = rLevels[rIndex++];\n while (record[step.name].length <= ix) {\n // eslint-disable max-depth\n record[step.name].push({});\n }\n record = record[step.name][ix];\n break;\n\n default:\n record[step.name] = record[step.name] \|\| {};\n record = record[step.name];\n }\n }\n\n // Leaf node - Add the value\n if (dLevel === field.dLevelMax) {\n const value = Types.fromPrimitive(\n // @ts-ignore\n field.originalType \|\| field.primitiveType,\n columnData.values[vIndex],\n field\n );\n vIndex++;\n\n switch (field.repetitionType) {\n case 'REPEATED':\n if (!(field.name in record)) {\n // eslint-disable max-depth\n record[field.name] = [];\n }\n const ix = rLevels[rIndex];\n while (record[field.name].length <= ix) {\n // eslint-disable max-depth\n record[field.name].push(null);\n }\n record[field.name][ix] = value;\n break;\n\n default:\n record[field.name] = value;\n }\n }\n }\n}\n\n// Columnar export\n\n/\n 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n \n The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n \n buffer = {\n * columnData: [\n * 'my_col': {\n * dlevels: [d1, d2, .. dN],\n * rlevels: [r1, r2, .. rN],\n * values: [v1, v2, .. vN],\n * }, ...\n * ],\n * rowCount: X,\n * }\n \nexport function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {\n const columns: ParquetRecord = {};\n for (const key in buffer.columnData) {\n const columnData = buffer.columnData[key];\n if (columnData.count) {\n extractColumn(schema, columnData, key, columns);\n }\n }\n return columns;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction extractColumn(\n schema: ParquetSchema,\n columnData: ParquetData,\n key: string,\n columns: Record<string, unknown> \n) {\n if (columnData.count <= 0) {\n return;\n }\n\n const record = columns;\n\n const field = schema.findField(key);\n const branch = schema.findFieldBranch(key);\n\n // tslint:disable-next-line:prefer-array-literal\n const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n let vIndex = 0;\n\n let i = 0;\n const dLevel = columnData.dlevels[i];\n const rLevel = columnData.rlevels[i];\n rLevels[rLevel]++;\n rLevels.fill(0, rLevel + 1);\n\n let rIndex = 0;\n let record = records[rLevels[rIndex++] - 1];\n\n // Internal nodes\n for (const step of branch) {\n if (step === field \|\| dLevel < step.dLevelMax) {\n break;\n }\n\n switch (step.repetitionType) {\n case 'REPEATED':\n if (!(step.name in record)) {\n // eslint-disable max-depth\n record[step.name] = [];\n }\n const ix = rLevels[rIndex++];\n while (record[step.name].length <= ix) {\n // eslint-disable max-depth\n record[step.name].push({});\n }\n record = record[step.name][ix];\n break;\n\n default:\n record[step.name] = record[step.name] \|\| {};\n record = record[step.name];\n }\n }\n\n // Leaf node\n if (dLevel === field.dLevelMax) {\n const value = Types.fromPrimitive(\n // @ts-ignore\n field.originalType \|\| field.primitiveType,\n columnData.values[vIndex],\n field\n );\n vIndex++;\n\n switch (field.repetitionType) {\n case 'REPEATED':\n if (!(field.name in record)) {\n // eslint-disable max-depth\n record[field.name] = [];\n }\n const ix = rLevels[rIndex];\n while (record[field.name].length <= ix) {\n // eslint-disable max-depth\n record[field.name].push(null);\n }\n record[field.name][ix] = value;\n break;\n\n default:\n record[field.name] = value;\n }\n }\n}\n/\n"],"mappings":";;;;;;;;;;;;;;;AAEA,IAAAA,QAAA,GAAAC,OAAA;AAEA,IAAAC,KAAA,GAAAC,uBAAA,CAAAF,OAAA;AAAiC,SAAAG,yBAAAC,WAAA,eAAAC,OAAA,kCAAAC,iBAAA,OAAAD,OAAA,QAAAE,gBAAA,OAAAF,OAAA,YAAAF,wBAAA,YAAAA,yBAAAC,WAAA,WAAAA,WAAA,GAAAG,gBAAA,GAAAD,iBAAA,KAAAF,WAAA;AAAA,SAAAF,wBAAAM,GAAA,EAAAJ,WAAA,SAAAA,WAAA,IAAAI,GAAA,IAAAA,GAAA,CAAAC,UAAA,WAAAD,GAAA,QAAAA,GAAA,aAAAE,OAAA,CAAAF,GAAA,yBAAAA,GAAA,4BAAAG,OAAA,EAAAH,GAAA,UAAAI,KAAA,GAAAT,wBAAA,CAAAC,WAAA,OAAAQ,KAAA,IAAAA,KAAA,CAAAC,GAAA,CAAAL,GAAA,YAAAI,KAAA,CAAAE,GAAA,CAAAN,GAAA,SAAAO,MAAA,WAAAC,qBAAA,GAAAC,MAAA,CAAAC,cAAA,IAAAD,MAAA,CAAAE,wBAAA,WAAAC,GAAA,IAAAZ,GAAA,QAAAY,GAAA,kBAAAH,MAAA,CAAAI,SAAA,CAAAC,cAAA,CAAAC,IAAA,CAAAf,GAAA,EAAAY,GAAA,SAAAI,IAAA,GAAAR,qBAAA,GAAAC,MAAA,CAAAE,wBAAA,CAAAX,GAAA,EAAAY,GAAA,cAAAI,IAAA,KAAAA,IAAA,CAAAV,GAAA,IAAAU,IAAA,CAAAC,GAAA,KAAAR,MAAA,CAAAC,cAAA,CAAAH,MAAA,EAAAK,GAAA,EAAAI,IAAA,YAAAT,MAAA,CAAAK,GAAA,IAAAZ,GAAA,CAAAY,GAAA,SAAAL,MAAA,CAAAJ,OAAA,GAAAH,GAAA,MAAAI,KAAA,IAAAA,KAAA,CAAAa,GAAA,CAAAjB,GAAA,EAAAO,MAAA,YAAAA,MAAA;AAAA,SAAAW,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAN,IAAA,CAAAI,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAU,IAAA,GAAApB,EAAA,CAAAqB,IAAA,IAAAJ,gBAAA,GAAAG,IAAA,CAAAT,IAAA,SAAAS,IAAA,KAAAP,CAAA,WAAAA,EAAAS,GAAA,IAAAJ,MAAA,SAAAC,GAAA,GAAAG,GAAA,KAAAP,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAuB,MAAA,UAAAvB,EAAA,CAAAuB,MAAA,oBAAAL,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA0B,MAAA,SAAA1B,CAAA,qBAAAA,CAAA,sBAAA2B,iBAAA,CAAA3B,CAAA,EAAA0B,MAAA,OAAAd,CAAA,GAAAtB,MAAA,CAAAI,SAAA,CAAAkC,QAAA,CAAAhC,IAAA,CAAAI,CAAA,EAAA6B,KAAA,aAAAjB,CAAA,iBAAAZ,CAAA,CAAA8B,WAAA,EAAAlB,CAAA,GAAAZ,CAAA,CAAA8B,WAAA,CAAAC,IAAA,MAAAnB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA2B,IAAA,CAAAhC,CAAA,OAAAY,CAAA,+DAAAqB,IAAA,CAAArB,CAAA,UAAAe,iBAAA,CAAA3B,CAAA,EAAA0B,MAAA;AAAA,SAAAC,kBAAAO,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA1B,MAAA,EAAA2B,GAAA,GAAAD,GAAA,CAAA1B,MAAA,WAAAC,CAAA,MAAA2B,IAAA,OAAA/B,KAAA,CAAA8B,GAAA,GAAA1B,CAAA,GAAA0B,GAAA,EAAA1B,CAAA,IAAA2B,IAAA,CAAA3B,CAAA,IAAAyB,GAAA,CAAAzB,CAAA,UAAA2B,IAAA;AAI1B,SAASC,WAAWA,CAACC,MAAqB,EAAiB;EAChE,IAAMC,UAAuC,GAAG,CAAC,CAAC;EAAC,IAAAC,SAAA,GAAAzC,0BAAA,CAC/BuC,MAAM,CAACG,SAAS;IAAAC,KAAA;EAAA;IAApC,KAAAF,SAAA,CAAA7B,CAAA,MAAA+B,KAAA,GAAAF,SAAA,CAAA5B,CAAA,IAAAC,IAAA,GAAsC;MAAA,IAA3B8B,KAAK,GAAAD,KAAA,CAAA5B,KAAA;MACdyB,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,GAAG;QACtBmD,OAAO,EAAE,EAAE;QACXC,OAAO,EAAE,EAAE;QACXC,MAAM,EAAE,EAAE;QACVC,WAAW,EAAE,EAAE;QACfC,KAAK,EAAE;MACT,CAAC;IACH;EAAC,SAAA3B,GAAA;IAAAmB,SAAA,CAAAzB,CAAA,CAAAM,GAAA;EAAA;IAAAmB,SAAA,CAAAvB,CAAA;EAAA;EACD,OAAO;IAACgC,QAAQ,EAAE,CAAC;IAAEV,UAAU,EAAVA;EAAU,CAAC;AAClC;AAwBO,SAASW,WAAWA,CAACZ,MAAqB,EAAEa,MAAW,EAAEC,MAAqB,EAAQ;EAE3F,IAAMC,IAAI,GAAGhB,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Ce,iBAAiB,CAAChB,MAAM,CAACiB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;EAGpD,IAAID,MAAM,CAACH,QAAQ,KAAK,CAAC,EAAE;IACzBG,MAAM,CAACH,QAAQ,GAAG,CAAC;IACnBG,MAAM,CAACb,UAAU,GAAGc,IAAI;IACxB;EACF;EACAD,MAAM,CAACH,QAAQ,IAAI,CAAC;EAAC,IAAAO,UAAA,GAAAzD,0BAAA,CACDuC,MAAM,CAACG,SAAS;IAAAgB,MAAA;EAAA;IAApC,KAAAD,UAAA,CAAA7C,CAAA,MAAA8C,MAAA,GAAAD,UAAA,CAAA5C,CAAA,IAAAC,IAAA,GAAsC;MAAA,IAA3B8B,KAAK,GAAAc,MAAA,CAAA3C,KAAA;MACdT,KAAK,CAACX,SAAS,CAACgE,IAAI,CAACC,KAAK,CAACP,MAAM,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,EAAEQ,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,CAAC;MACzFxC,KAAK,CAACX,SAAS,CAACgE,IAAI,CAACC,KAAK,CAACP,MAAM,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,EAAES,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,CAAC;MACzFvC,KAAK,CAACX,SAAS,CAACgE,IAAI,CAACC,KAAK,CAACP,MAAM,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACqD,MAAM,EAAEO,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACqD,MAAM,CAAC;MACvFM,MAAM,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK,IAAIK,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK;IAC7D;EAAC,SAAA3B,GAAA;IAAAmC,UAAA,CAAAzC,CAAA,CAAAM,GAAA;EAAA;IAAAmC,UAAA,CAAAvC,CAAA;EAAA;AACH;AAGA,SAASqC,iBAAiBA,CACxBC,MAAoC,EACpCJ,MAAW,EACXE,IAAiC,EACjCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,IAAM9B,IAAI,IAAIwB,MAAM,EAAE;IACzB,IAAMZ,KAAK,GAAGY,MAAM,CAACxB,IAAI,CAAC;IAG1B,IAAIe,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNR,KAAK,CAACZ,IAAI,IAAIoB,MAAM,IACpBA,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,KAAK+B,SAAS,IAChCX,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIoB,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACD,WAAW,KAAKzB,KAAK,EAAE;QAC5CyC,MAAM,GAAGK,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLe,MAAM,CAACY,IAAI,CAACP,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIe,MAAM,CAACtC,MAAM,KAAK,CAAC,IAAIuD,OAAO,CAACZ,MAAM,CAAC,IAAIR,KAAK,CAACqB,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,4BAAAC,MAAA,CAA4BvB,KAAK,CAACZ,IAAI,EAAG;IAC1D;IACA,IAAIe,MAAM,CAACtC,MAAM,GAAG,CAAC,IAAImC,KAAK,CAACqB,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,+BAAAC,MAAA,CAA+BvB,KAAK,CAACZ,IAAI,EAAG;IAC7D;IAGA,IAAIe,MAAM,CAACtC,MAAM,KAAK,CAAC,EAAE;MACvB,IAAImC,KAAK,CAACwB,QAAQ,EAAE;QAClBb,iBAAiB,CAACX,KAAK,CAACY,MAAM,EAAG,IAAI,EAAEF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC9D,CAAC,MAAM;QACLR,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;IAGA,KAAK,IAAIpD,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGqC,MAAM,CAACtC,MAAM,EAAEC,CAAC,EAAE,EAAE;MACtC,IAAM2D,IAAI,GAAG3D,CAAC,KAAK,CAAC,GAAGmD,MAAM,GAAGjB,KAAK,CAAC0B,SAAS;MAC/C,IAAI1B,KAAK,CAACwB,QAAQ,EAAE;QAClBb,iBAAiB,CAACX,KAAK,CAACY,MAAM,EAAGT,MAAM,CAACrC,CAAC,CAAC,EAAE4C,IAAI,EAAEe,IAAI,EAAEzB,KAAK,CAAC2B,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLjB,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,CAACa,IAAI,CAACU,IAAI,CAAC;QAClCf,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,CAACc,IAAI,CAACf,KAAK,CAAC2B,SAAS,CAAC;QAC7CjB,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACqD,MAAM,CAACY,IAAI,CACzBpF,KAAK,CAACiG,WAAW,CAAE5B,KAAK,CAAC6B,YAAY,IAAI7B,KAAK,CAAC8B,aAAa,EAAI3B,MAAM,CAACrC,CAAC,CAAC,CAAC,CAC3E;MACH;IACF;EACF;AACF;AAqBO,SAASiE,kBAAkBA,CAACpC,MAAqB,EAAEc,MAAqB,EAAmB;EAChG,IAAMuB,OAAwB,GAAG,EAAE;EACnC,KAAK,IAAIlE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG2C,MAAM,CAACH,QAAQ,EAAExC,CAAC,EAAE,EAAE;IACxCkE,OAAO,CAACjB,IAAI,CAAC,CAAC,CAAC,CAAC;EAClB;EACA,KAAK,IAAMjE,GAAG,IAAI2D,MAAM,CAACb,UAAU,EAAE;IACnC,IAAMA,UAAU,GAAGa,MAAM,CAACb,UAAU,CAAC9C,GAAG,CAAC;IACzC,IAAI8C,UAAU,CAACS,KAAK,EAAE;MACpB4B,iBAAiB,CAACtC,MAAM,EAAEC,UAAU,EAAE9C,GAAG,EAAEkF,OAAO,CAAC;IACrD;EACF;EACA,OAAOA,OAAO;AAChB;AAGA,SAASC,iBAAiBA,CACxBtC,MAAqB,EACrBC,UAAuB,EACvB9C,GAAW,EACXkF,OAAwB,EAClB;EACN,IAAMhC,KAAK,GAAGL,MAAM,CAACuC,SAAS,CAACpF,GAAG,CAAC;EACnC,IAAMqF,MAAM,GAAGxC,MAAM,CAACyC,eAAe,CAACtF,GAAG,CAAC;EAG1C,IAAMuF,OAAiB,GAAG,IAAI3E,KAAK,CAACsC,KAAK,CAAC0B,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIzE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG8B,UAAU,CAACS,KAAK,EAAEvC,CAAC,EAAE,EAAE;IACzC,IAAMoD,MAAM,GAAGtB,UAAU,CAACK,OAAO,CAACnC,CAAC,CAAC;IACpC,IAAMmD,MAAM,GAAGrB,UAAU,CAACM,OAAO,CAACpC,CAAC,CAAC;IACpCuE,OAAO,CAACpB,MAAM,CAAC,EAAE;IACjBoB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAErB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAIuB,MAAM,GAAG,CAAC;IACd,IAAIhC,MAAM,GAAGwB,OAAO,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;IAAC,IAAAC,UAAA,GAAArF,0BAAA,CAGzB+E,MAAM;MAAAO,MAAA;IAAA;MAAzB,KAAAD,UAAA,CAAAzE,CAAA,MAAA0E,MAAA,GAAAD,UAAA,CAAAxE,CAAA,IAAAC,IAAA,GAA2B;QAAA,IAAhBS,IAAI,GAAA+D,MAAA,CAAAvE,KAAA;QACb,IAAIQ,IAAI,KAAKqB,KAAK,IAAIkB,MAAM,GAAGvC,IAAI,CAACgD,SAAS,EAAE;UAC7C;QACF;QAEA,QAAQhD,IAAI,CAAC0C,cAAc;UACzB,KAAK,UAAU;YACb,IAAI,EAAE1C,IAAI,CAACS,IAAI,IAAIoB,MAAM,CAAC,EAAE;cAE1BA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,GAAG,EAAE;YACxB;YACA,IAAMuD,GAAE,GAAGN,OAAO,CAACG,MAAM,EAAE,CAAC;YAC5B,OAAOhC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAACvB,MAAM,IAAI8E,GAAE,EAAE;cAErCnC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAAC2B,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B;YACAP,MAAM,GAAGA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAACuD,GAAE,CAAC;YAC9B;UAEF;YACEnC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,GAAGoB,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3CoB,MAAM,GAAGA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC;QAAC;MAEjC;IAAC,SAAAV,GAAA;MAAA+D,UAAA,CAAArE,CAAA,CAAAM,GAAA;IAAA;MAAA+D,UAAA,CAAAnE,CAAA;IAAA;IAGD,IAAI4C,MAAM,KAAKlB,KAAK,CAAC2B,SAAS,EAAE;MAC9B,IAAMxD,KAAK,GAAGxC,KAAK,CAACiH,aAAa,CAE/B5C,KAAK,CAAC6B,YAAY,IAAI7B,KAAK,CAAC8B,aAAa,EACzClC,UAAU,CAACO,MAAM,CAACoC,MAAM,CAAC,EACzBvC,KAAK,CACN;MACDuC,MAAM,EAAE;MAER,QAAQvC,KAAK,CAACqB,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAErB,KAAK,CAACZ,IAAI,IAAIoB,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,IAAMuD,EAAE,GAAGN,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOhC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACvB,MAAM,IAAI8E,EAAE,EAAE;YAEtCnC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAAC2B,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACuD,EAAE,CAAC,GAAGxE,KAAK;UAC9B;QAEF;UACEqC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,GAAGjB,KAAK;MAAC;IAEjC;EACF;AACF"}

package/dist/es5/parquetjs/schema/types.js CHANGED Viewed

@@ -1,14 +1,12 @@
 "use strict";
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.PARQUET_LOGICAL_TYPES = void 0;
 exports.fromPrimitive = fromPrimitive;
 exports.toPrimitive = toPrimitive;
-var _bson = _interopRequireDefault(require("bson"));
+var _bson = require("@loaders.gl/bson");
 var PARQUET_LOGICAL_TYPES = {
   BOOLEAN: {
     primitiveType: 'BOOLEAN',
@@ -161,7 +159,6 @@ var PARQUET_LOGICAL_TYPES = {
     fromPrimitive: decimalFromPrimitive_BYTE_ARRAY
   }
 };
 exports.PARQUET_LOGICAL_TYPES = PARQUET_LOGICAL_TYPES;
 function toPrimitive(type, value, field) {
   if (!(type in PARQUET_LOGICAL_TYPES)) {
@@ -169,7 +166,6 @@ function toPrimitive(type, value, field) {
   }
   return PARQUET_LOGICAL_TYPES[type].toPrimitive(value, field);
 }
 function fromPrimitive(type, value, field) {
   if (!(type in PARQUET_LOGICAL_TYPES)) {
     throw new Error("invalid type: ".concat(type));
@@ -178,7 +174,6 @@ function fromPrimitive(type, value, field) {
     var _PARQUET_LOGICAL_TYPE, _PARQUET_LOGICAL_TYPE2;
     return (_PARQUET_LOGICAL_TYPE = (_PARQUET_LOGICAL_TYPE2 = PARQUET_LOGICAL_TYPES[type]).fromPrimitive) === null || _PARQUET_LOGICAL_TYPE === void 0 ? void 0 : _PARQUET_LOGICAL_TYPE.call(_PARQUET_LOGICAL_TYPE2, value, field);
   }
   return value;
 }
 function toPrimitive_BOOLEAN(value) {
@@ -299,10 +294,13 @@ function fromPrimitive_JSON(value) {
   return JSON.parse(value);
 }
 function toPrimitive_BSON(value) {
-  return Buffer.from(_bson.default.serialize(value));
+  var _BSONWriter$encodeSyn;
+  var arrayBuffer = (_BSONWriter$encodeSyn = _bson.BSONWriter.encodeSync) === null || _BSONWriter$encodeSyn === void 0 ? void 0 : _BSONWriter$encodeSyn.call(_bson.BSONWriter, value);
+  return Buffer.from(arrayBuffer);
 }
 function fromPrimitive_BSON(value) {
-  return _bson.default.deserialize(value);
+  var _BSONLoader$parseSync;
+  return (_BSONLoader$parseSync = _bson.BSONLoader.parseSync) === null || _BSONLoader$parseSync === void 0 ? void 0 : _BSONLoader$parseSync.call(_bson.BSONLoader, value);
 }
 function toPrimitive_TIME_MILLIS(value) {
   var v = parseInt(value, 10);
@@ -323,7 +321,6 @@ function toPrimitive_DATE(value) {
   if (value instanceof Date) {
     return value.getTime() / kMillisPerDay;
   }
   {
     var v = parseInt(value, 10);
     if (v < 0 || isNaN(v)) {
@@ -339,7 +336,6 @@ function toPrimitive_TIMESTAMP_MILLIS(value) {
   if (value instanceof Date) {
     return value.getTime();
   }
   {
     var v = parseInt(value, 10);
     if (v < 0 || isNaN(v)) {
@@ -355,7 +351,6 @@ function toPrimitive_TIMESTAMP_MICROS(value) {
   if (value instanceof Date) {
     return value.getTime() * 1000;
   }
   {
     var v = parseInt(value, 10);
     if (v < 0 || isNaN(v)) {