@loaders.gl/parquet 4.0.0-alpha.9 → 4.0.0-beta.1

package/dist/esm/parquetjs/parser/parquet-reader.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-reader.js","names":["ParquetSchema","decodeSchema","materializeRows","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","DEFAULT_PROPS","defaultDictionarySize","ParquetReader","constructor","file","props","_defineProperty","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","columnList","map","x","Array","isArray","metadata","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","num_children","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","buffer","read","magic","toString","Error","concat","trailerLen","trailerBuf","size","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_options$dictionary","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","_colChunk$meta_data6","Math","min","options","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","decodedPage"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRows} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetRowGroup,\n  ParquetCompression,\n  ParquetColumnChunk,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n  defaultDictionarySize: 1e6\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...DEFAULT_PROPS, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRows(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetRowGroup> {\n    const buffer: ParquetRowGroup = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetColumnChunk> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":";AAGA,SAAQA,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,YAAY,QAAO,YAAY;AACvC,SAAQC,eAAe,QAAO,iBAAiB;AAE/C,SAAQC,aAAa,EAAEC,uBAAuB,QAAO,iBAAiB;AACtE,SAAqBC,gBAAgB,EAA0BC,IAAI,QAAO,mBAAmB;AAQ7F,SAAQC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY,QAAO,qBAAqB;AACnF,SAAQC,eAAe,EAAEC,UAAU,QAAO,YAAY;AAYtD,MAAMC,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;AAQD,OAAO,MAAMC,aAAa,CAAC;EAKzBC,WAAWA,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAAC,eAAA;IAAAA,eAAA;IAAAA,eAAA,mBAFnB,IAAI;IAG3C,IAAI,CAACF,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGL,aAAa;MAAE,GAAGK;IAAK,CAAC;EAC3C;EAEAE,KAAKA,CAAA,EAAS;IAEZ,IAAI,CAACH,IAAI,CAACG,KAAK,CAAC,CAAC;EACnB;EAKA,OAAOC,WAAWA,CAACH,KAA6B,EAAE;IAChD,WAAW,MAAMI,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACL,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMM,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;EAGA,OAAOD,gBAAgBA,CAACL,KAA6B,EAAE;IACrD,MAAMO,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACV,KAAK,CAAC,EAAE;MACzD,MAAMf,eAAe,CAACsB,MAAM,EAAEE,QAAQ,CAAC;IACzC;EACF;EAGA,OAAOC,gBAAgBA,CAACV,KAA6B,EAAE;IAErD,MAAMW,UAAsB,GAAG,CAAC,CAAAX,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEW,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAC3B,CAAC;IAED,MAAMG,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMV,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IAErC,MAAMU,aAAa,GAAG,CAAAF,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEG,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMZ,QAAQ,GAAG,MAAM,IAAI,CAACa,YAAY,CACtCf,MAAM,EACNS,QAAQ,CAACG,UAAU,CAACE,aAAa,CAAC,EAClCV,UACF,CAAC;MACD,MAAMF,QAAQ;IAChB;EACF;EAEA,MAAMc,WAAWA,CAAA,EAAoB;IACnC,MAAMP,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,OAAOO,MAAM,CAACR,QAAQ,CAACS,QAAQ,CAAC;EAClC;EAEA,MAAMjB,SAASA,CAAA,EAA2B;IACxC,MAAMQ,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMS,IAAI,GAAGV,QAAQ,CAACT,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEoB;IAAgB,CAAC,GAAG3C,YAAY,CAACgC,QAAQ,CAACT,MAAM,EAAE,CAAC,EAAEmB,IAAI,CAACE,YAAa,CAAC;IACvF,MAAMrB,MAAM,GAAG,IAAIxB,aAAa,CAAC4C,gBAAgB,CAAC;IAClD,OAAOpB,MAAM;EACf;EAMA,MAAMsB,iBAAiBA,CAAA,EAAoC;IACzD,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMa,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAIf,QAAQ,CAACgB,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMb,eAAeA,CAAA,EAA0B;IAC7C,IAAI,CAAC,IAAI,CAACD,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACmB,UAAU,CAAC,CAAC;MACvB,IAAI,CAACnB,QAAQ,GAAG,IAAI,CAACoB,UAAU,CAAC,CAAC;IACnC;IACA,OAAO,IAAI,CAACpB,QAAQ;EACtB;EAKA,MAAMmB,UAAUA,CAAA,EAAkB;IAChC,MAAME,MAAM,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAAC,CAAC,EAAEpD,aAAa,CAACkC,MAAM,CAAC;IAC5D,MAAMmB,KAAK,GAAGF,MAAM,CAACG,QAAQ,CAAC,CAAC;IAC/B,QAAQD,KAAK;MACX,KAAKrD,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAIsD,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,gCAAAC,MAAA,CAAgCH,KAAK,MAAG,CAAC;IAC5D;EACF;EAGA,MAAMH,UAAUA,CAAA,EAA0B;IACxC,MAAMO,UAAU,GAAGzD,aAAa,CAACkC,MAAM,GAAG,CAAC;IAC3C,MAAMwB,UAAU,GAAG,MAAM,IAAI,CAAC7C,IAAI,CAACuC,IAAI,CAAC,IAAI,CAACvC,IAAI,CAAC8C,IAAI,GAAGF,UAAU,EAAEA,UAAU,CAAC;IAEhF,MAAMJ,KAAK,GAAGK,UAAU,CAACE,KAAK,CAAC,CAAC,CAAC,CAACN,QAAQ,CAAC,CAAC;IAC5C,IAAID,KAAK,KAAKrD,aAAa,EAAE;MAC3B,MAAM,IAAIuD,KAAK,sCAAAC,MAAA,CAAqCH,KAAK,MAAG,CAAC;IAC/D;IAEA,MAAMQ,YAAY,GAAGH,UAAU,CAACI,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAAClD,IAAI,CAAC8C,IAAI,GAAGE,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAG/D,aAAa,CAACkC,MAAM,EAAE;MACzC,MAAM,IAAIqB,KAAK,0BAAAC,MAAA,CAA0BO,cAAc,CAAE,CAAC;IAC5D;IAEA,MAAMC,WAAW,GAAG,MAAM,IAAI,CAACnD,IAAI,CAACuC,IAAI,CAACW,cAAc,EAAEF,YAAY,CAAC;IAGtE,MAAM;MAAC/B;IAAQ,CAAC,GAAG1B,kBAAkB,CAAC4D,WAAW,CAAC;IAClD,OAAOlC,QAAQ;EACjB;EAGA,MAAMM,YAAYA,CAChBf,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB,EACI;IAC1B,MAAM0B,MAAuB,GAAG;MAC9Bc,QAAQ,EAAE3B,MAAM,CAACf,QAAQ,CAACgB,QAAQ,CAAC;MACnC2B,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAI5C,QAAQ,CAAC6C,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAI/C,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI5B,YAAY,CAACmB,UAAU,EAAE8C,MAAO,CAAC,GAAG,CAAC,EAAE;QAClE;MACF;MACApB,MAAM,CAACe,UAAU,CAACK,MAAM,CAAEE,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACrD,MAAM,EAAE8C,QAAQ,CAAC;IAClF;IACA,OAAOhB,MAAM;EACf;EAKA,MAAMuB,eAAeA,CAACrD,MAAqB,EAAE8C,QAAqB,EAA+B;IAAA,IAAAQ,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;IAC/F,IAAIf,QAAQ,CAACgB,SAAS,KAAKC,SAAS,IAAIjB,QAAQ,CAACgB,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAI5B,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAM8B,KAAK,GAAGhE,MAAM,CAACiE,SAAS,EAAAX,mBAAA,GAACR,QAAQ,CAACG,SAAS,cAAAK,mBAAA,uBAAlBA,mBAAA,CAAoBH,cAAe,CAAC;IACnE,MAAMe,IAAmB,GAAGlF,aAAa,CAACF,IAAI,GAAAyE,oBAAA,GAAET,QAAQ,CAACG,SAAS,cAAAM,oBAAA,uBAAlBA,oBAAA,CAAoBW,IAAK,CAAQ;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAIjC,KAAK,oCAAAC,MAAA,CAAoC+B,IAAI,CAAE,CAAC;IAC5D;IAEA,MAAME,WAA+B,GAAGpF,aAAa,CACnDH,gBAAgB,GAAA2E,oBAAA,GAChBV,QAAQ,CAACG,SAAS,cAAAO,oBAAA,uBAAlBA,oBAAA,CAAoBa,KACtB,CAAQ;IAER,MAAMC,WAAW,GAAGrD,MAAM,EAAAwC,oBAAA,GAACX,QAAQ,CAACG,SAAS,cAAAQ,oBAAA,uBAAlBA,oBAAA,CAAoBc,gBAAiB,CAAC;IACjE,IAAIC,SAAS,GAAGvD,MAAM,EAAAyC,oBAAA,GAACZ,QAAQ,CAACG,SAAS,cAAAS,oBAAA,uBAAlBA,oBAAA,CAAoBe,qBAAsB,CAAC;IAElE,IAAI,CAAC3B,QAAQ,CAACgB,SAAS,EAAE;MAAA,IAAAY,oBAAA;MACvBF,SAAS,GAAGG,IAAI,CAACC,GAAG,CAClB,IAAI,CAACpF,IAAI,CAAC8C,IAAI,GAAGgC,WAAW,EAC5BrD,MAAM,EAAAyD,oBAAA,GAAC5B,QAAQ,CAACG,SAAS,cAAAyB,oBAAA,uBAAlBA,oBAAA,CAAoBD,qBAAqB,CAClD,CAAC;IACH;IAEA,MAAMI,OAAuB,GAAG;MAC9BX,IAAI;MACJY,SAAS,EAAEd,KAAK,CAACc,SAAS;MAC1BC,SAAS,EAAEf,KAAK,CAACe,SAAS;MAC1BX,WAAW;MACXY,MAAM,EAAEhB,KAAK;MACbiB,SAAS,GAAAtB,oBAAA,GAAEb,QAAQ,CAACG,SAAS,cAAAU,oBAAA,uBAAlBA,oBAAA,CAAoBuB,UAAU;MACzCC,UAAU,EAAE;IACd,CAAC;IAED,IAAIA,UAAU;IAEd,MAAMC,oBAAoB,GAAGtC,QAAQ,aAARA,QAAQ,wBAAAc,oBAAA,GAARd,QAAQ,CAAEG,SAAS,cAAAW,oBAAA,uBAAnBA,oBAAA,CAAqByB,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAGrE,MAAM,CAACmE,oBAAoB,CAAC;MAErDD,UAAU,GAAG,MAAM,IAAI,CAACI,aAAa,CAACD,gBAAgB,EAAET,OAAO,EAAEP,WAAW,CAAC;IAC/E;IAEAa,UAAU,GAAG,CAAAtB,mBAAA,GAAAgB,OAAO,CAACM,UAAU,cAAAtB,mBAAA,eAAlBA,mBAAA,CAAoBhD,MAAM,GAAGgE,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMK,QAAQ,GAAG,MAAM,IAAI,CAAChG,IAAI,CAACuC,IAAI,CAACuC,WAAW,EAAEE,SAAS,CAAC;IAC7D,OAAO,MAAMtF,eAAe,CAACsG,QAAQ,EAAE;MAAC,GAAGX,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;EASA,MAAMI,aAAaA,CACjBH,oBAA4B,EAC5BP,OAAuB,EACvBP,WAAmB,EACA;IACnB,IAAIc,oBAAoB,KAAK,CAAC,EAAE;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGd,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACpF,IAAI,CAAC8C,IAAI,GAAG8C,oBAAoB,EACrC,IAAI,CAAC3F,KAAK,CAACJ,qBACb,CAAC;IACD,MAAMmG,QAAQ,GAAG,MAAM,IAAI,CAAChG,IAAI,CAACuC,IAAI,CAACqD,oBAAoB,EAAEK,cAAc,CAAC;IAE3E,MAAMC,MAAM,GAAG;MAAC5D,MAAM,EAAE0D,QAAQ;MAAEG,MAAM,EAAE,CAAC;MAAErD,IAAI,EAAEkD,QAAQ,CAAC3E;IAAM,CAAC;IACnE,MAAM+E,WAAW,GAAG,MAAMzG,UAAU,CAACuG,MAAM,EAAEb,OAAO,CAAC;IAErD,OAAOe,WAAW,CAACT,UAAU;EAC/B;AACF"}
+{"version":3,"file":"parquet-reader.js","names":["ParquetSchema","decodeSchema","materializeRows","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","ParquetReader","constructor","file","props","_defineProperty","defaultProps","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","columnList","map","x","Array","isArray","metadata","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","num_children","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","buffer","read","magic","toString","Error","concat","trailerLen","trailerBuf","size","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_context$dictionary","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","_colChunk$meta_data6","Math","min","context","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","preserveBinary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","defaultDictionarySize","cursor","offset","decodedPage"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRows} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetRowGroup,\n  ParquetCompression,\n  ParquetColumnChunk,\n  PrimitiveType,\n  ParquetReaderContext\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n  preserveBinary?: boolean;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  static defaultProps: Required<ParquetReaderProps> = {\n    defaultDictionarySize: 1e6,\n    preserveBinary: false\n  };\n\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...ParquetReader.defaultProps, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRows(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetRowGroup> {\n    const buffer: ParquetRowGroup = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetColumnChunk> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const context: ParquetReaderContext = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: [],\n      // Options - TBD is this the right place for these?\n      preserveBinary: this.props.preserveBinary\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, context, pagesOffset);\n    }\n\n    dictionary = context.dictionary?.length ? context.dictionary : dictionary;\n    const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...context, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param context\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    context: ParquetReaderContext,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, context);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":";AAGA,SAAQA,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,YAAY,QAAO,YAAY;AACvC,SAAQC,eAAe,QAAO,iBAAiB;AAE/C,SAAQC,aAAa,EAAEC,uBAAuB,QAAO,iBAAiB;AACtE,SAAqBC,gBAAgB,EAA0BC,IAAI,QAAO,mBAAmB;AAQ7F,SAAQC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY,QAAO,qBAAqB;AACnF,SAAQC,eAAe,EAAEC,UAAU,QAAO,YAAY;AAmBtD,OAAO,MAAMC,aAAa,CAAC;EAUzBC,WAAWA,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAAC,eAAA;IAAAA,eAAA;IAAAA,eAAA,mBAFnB,IAAI;IAG3C,IAAI,CAACF,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGH,aAAa,CAACK,YAAY;MAAE,GAAGF;IAAK,CAAC;EACxD;EAEAG,KAAKA,CAAA,EAAS;IAEZ,IAAI,CAACJ,IAAI,CAACI,KAAK,CAAC,CAAC;EACnB;EAKA,OAAOC,WAAWA,CAACJ,KAA6B,EAAE;IAChD,WAAW,MAAMK,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACN,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMO,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;EAGA,OAAOD,gBAAgBA,CAACN,KAA6B,EAAE;IACrD,MAAMQ,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACX,KAAK,CAAC,EAAE;MACzD,MAAMb,eAAe,CAACqB,MAAM,EAAEE,QAAQ,CAAC;IACzC;EACF;EAGA,OAAOC,gBAAgBA,CAACX,KAA6B,EAAE;IAErD,MAAMY,UAAsB,GAAG,CAAC,CAAAZ,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEY,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAC3B,CAAC;IAED,MAAMG,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMV,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IAErC,MAAMU,aAAa,GAAG,CAAAF,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEG,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMZ,QAAQ,GAAG,MAAM,IAAI,CAACa,YAAY,CACtCf,MAAM,EACNS,QAAQ,CAACG,UAAU,CAACE,aAAa,CAAC,EAClCV,UACF,CAAC;MACD,MAAMF,QAAQ;IAChB;EACF;EAEA,MAAMc,WAAWA,CAAA,EAAoB;IACnC,MAAMP,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,OAAOO,MAAM,CAACR,QAAQ,CAACS,QAAQ,CAAC;EAClC;EAEA,MAAMjB,SAASA,CAAA,EAA2B;IACxC,MAAMQ,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMS,IAAI,GAAGV,QAAQ,CAACT,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEoB;IAAgB,CAAC,GAAG1C,YAAY,CAAC+B,QAAQ,CAACT,MAAM,EAAE,CAAC,EAAEmB,IAAI,CAACE,YAAa,CAAC;IACvF,MAAMrB,MAAM,GAAG,IAAIvB,aAAa,CAAC2C,gBAAgB,CAAC;IAClD,OAAOpB,MAAM;EACf;EAMA,MAAMsB,iBAAiBA,CAAA,EAAoC;IACzD,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMa,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAIf,QAAQ,CAACgB,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMb,eAAeA,CAAA,EAA0B;IAC7C,IAAI,CAAC,IAAI,CAACD,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACmB,UAAU,CAAC,CAAC;MACvB,IAAI,CAACnB,QAAQ,GAAG,IAAI,CAACoB,UAAU,CAAC,CAAC;IACnC;IACA,OAAO,IAAI,CAACpB,QAAQ;EACtB;EAKA,MAAMmB,UAAUA,CAAA,EAAkB;IAChC,MAAME,MAAM,GAAG,MAAM,IAAI,CAACvC,IAAI,CAACwC,IAAI,CAAC,CAAC,EAAEnD,aAAa,CAACiC,MAAM,CAAC;IAC5D,MAAMmB,KAAK,GAAGF,MAAM,CAACG,QAAQ,CAAC,CAAC;IAC/B,QAAQD,KAAK;MACX,KAAKpD,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAIqD,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,gCAAAC,MAAA,CAAgCH,KAAK,MAAG,CAAC;IAC5D;EACF;EAGA,MAAMH,UAAUA,CAAA,EAA0B;IACxC,MAAMO,UAAU,GAAGxD,aAAa,CAACiC,MAAM,GAAG,CAAC;IAC3C,MAAMwB,UAAU,GAAG,MAAM,IAAI,CAAC9C,IAAI,CAACwC,IAAI,CAAC,IAAI,CAACxC,IAAI,CAAC+C,IAAI,GAAGF,UAAU,EAAEA,UAAU,CAAC;IAEhF,MAAMJ,KAAK,GAAGK,UAAU,CAACE,KAAK,CAAC,CAAC,CAAC,CAACN,QAAQ,CAAC,CAAC;IAC5C,IAAID,KAAK,KAAKpD,aAAa,EAAE;MAC3B,MAAM,IAAIsD,KAAK,sCAAAC,MAAA,CAAqCH,KAAK,MAAG,CAAC;IAC/D;IAEA,MAAMQ,YAAY,GAAGH,UAAU,CAACI,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAACnD,IAAI,CAAC+C,IAAI,GAAGE,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAG9D,aAAa,CAACiC,MAAM,EAAE;MACzC,MAAM,IAAIqB,KAAK,0BAAAC,MAAA,CAA0BO,cAAc,CAAE,CAAC;IAC5D;IAEA,MAAMC,WAAW,GAAG,MAAM,IAAI,CAACpD,IAAI,CAACwC,IAAI,CAACW,cAAc,EAAEF,YAAY,CAAC;IAItE,MAAM;MAAC/B;IAAQ,CAAC,GAAGzB,kBAAkB,CAAC2D,WAAW,CAAC;IAClD,OAAOlC,QAAQ;EACjB;EAGA,MAAMM,YAAYA,CAChBf,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB,EACI;IAC1B,MAAM0B,MAAuB,GAAG;MAC9Bc,QAAQ,EAAE3B,MAAM,CAACf,QAAQ,CAACgB,QAAQ,CAAC;MACnC2B,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAI5C,QAAQ,CAAC6C,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAI/C,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI3B,YAAY,CAACkB,UAAU,EAAE8C,MAAO,CAAC,GAAG,CAAC,EAAE;QAClE;MACF;MACApB,MAAM,CAACe,UAAU,CAACK,MAAM,CAAEE,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACrD,MAAM,EAAE8C,QAAQ,CAAC;IAClF;IACA,OAAOhB,MAAM;EACf;EAKA,MAAMuB,eAAeA,CAACrD,MAAqB,EAAE8C,QAAqB,EAA+B;IAAA,IAAAQ,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;IAC/F,IAAIf,QAAQ,CAACgB,SAAS,KAAKC,SAAS,IAAIjB,QAAQ,CAACgB,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAI5B,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAM8B,KAAK,GAAGhE,MAAM,CAACiE,SAAS,EAAAX,mBAAA,GAACR,QAAQ,CAACG,SAAS,cAAAK,mBAAA,uBAAlBA,mBAAA,CAAoBH,cAAe,CAAC;IACnE,MAAMe,IAAmB,GAAGjF,aAAa,CAACF,IAAI,GAAAwE,oBAAA,GAAET,QAAQ,CAACG,SAAS,cAAAM,oBAAA,uBAAlBA,oBAAA,CAAoBW,IAAK,CAAQ;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAIjC,KAAK,oCAAAC,MAAA,CAAoC+B,IAAI,CAAE,CAAC;IAC5D;IAEA,MAAME,WAA+B,GAAGnF,aAAa,CACnDH,gBAAgB,GAAA0E,oBAAA,GAChBV,QAAQ,CAACG,SAAS,cAAAO,oBAAA,uBAAlBA,oBAAA,CAAoBa,KACtB,CAAQ;IAER,MAAMC,WAAW,GAAGrD,MAAM,EAAAwC,oBAAA,GAACX,QAAQ,CAACG,SAAS,cAAAQ,oBAAA,uBAAlBA,oBAAA,CAAoBc,gBAAiB,CAAC;IACjE,IAAIC,SAAS,GAAGvD,MAAM,EAAAyC,oBAAA,GAACZ,QAAQ,CAACG,SAAS,cAAAS,oBAAA,uBAAlBA,oBAAA,CAAoBe,qBAAsB,CAAC;IAElE,IAAI,CAAC3B,QAAQ,CAACgB,SAAS,EAAE;MAAA,IAAAY,oBAAA;MACvBF,SAAS,GAAGG,IAAI,CAACC,GAAG,CAClB,IAAI,CAACrF,IAAI,CAAC+C,IAAI,GAAGgC,WAAW,EAC5BrD,MAAM,EAAAyD,oBAAA,GAAC5B,QAAQ,CAACG,SAAS,cAAAyB,oBAAA,uBAAlBA,oBAAA,CAAoBD,qBAAqB,CAClD,CAAC;IACH;IAEA,MAAMI,OAA6B,GAAG;MACpCX,IAAI;MACJY,SAAS,EAAEd,KAAK,CAACc,SAAS;MAC1BC,SAAS,EAAEf,KAAK,CAACe,SAAS;MAC1BX,WAAW;MACXY,MAAM,EAAEhB,KAAK;MACbiB,SAAS,GAAAtB,oBAAA,GAAEb,QAAQ,CAACG,SAAS,cAAAU,oBAAA,uBAAlBA,oBAAA,CAAoBuB,UAAU;MACzCC,UAAU,EAAE,EAAE;MAEdC,cAAc,EAAE,IAAI,CAAC5F,KAAK,CAAC4F;IAC7B,CAAC;IAED,IAAID,UAAU;IAEd,MAAME,oBAAoB,GAAGvC,QAAQ,aAARA,QAAQ,wBAAAc,oBAAA,GAARd,QAAQ,CAAEG,SAAS,cAAAW,oBAAA,uBAAnBA,oBAAA,CAAqB0B,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAGtE,MAAM,CAACoE,oBAAoB,CAAC;MAErDF,UAAU,GAAG,MAAM,IAAI,CAACK,aAAa,CAACD,gBAAgB,EAAEV,OAAO,EAAEP,WAAW,CAAC;IAC/E;IAEAa,UAAU,GAAG,CAAAtB,mBAAA,GAAAgB,OAAO,CAACM,UAAU,cAAAtB,mBAAA,eAAlBA,mBAAA,CAAoBhD,MAAM,GAAGgE,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMM,QAAQ,GAAG,MAAM,IAAI,CAAClG,IAAI,CAACwC,IAAI,CAACuC,WAAW,EAAEE,SAAS,CAAC;IAC7D,OAAO,MAAMrF,eAAe,CAACsG,QAAQ,EAAE;MAAC,GAAGZ,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;EASA,MAAMK,aAAaA,CACjBH,oBAA4B,EAC5BR,OAA6B,EAC7BP,WAAmB,EACA;IACnB,IAAIe,oBAAoB,KAAK,CAAC,EAAE;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGf,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACrF,IAAI,CAAC+C,IAAI,GAAG+C,oBAAoB,EACrC,IAAI,CAAC7F,KAAK,CAACmG,qBACb,CAAC;IACD,MAAMF,QAAQ,GAAG,MAAM,IAAI,CAAClG,IAAI,CAACwC,IAAI,CAACsD,oBAAoB,EAAEK,cAAc,CAAC;IAE3E,MAAME,MAAM,GAAG;MAAC9D,MAAM,EAAE2D,QAAQ;MAAEI,MAAM,EAAE,CAAC;MAAEvD,IAAI,EAAEmD,QAAQ,CAAC5E;IAAM,CAAC;IACnE,MAAMiF,WAAW,GAAG,MAAM1G,UAAU,CAACwG,MAAM,EAAEf,OAAO,CAAC;IAErD,OAAOiB,WAAW,CAACX,UAAU;EAC/B;AACF;AAAC1F,eAAA,CAxPYJ,aAAa,kBAC4B;EAClDsG,qBAAqB,EAAE,GAAG;EAC1BP,cAAc,EAAE;AAClB,CAAC"}

package/dist/esm/parquetjs/schema/declare.js.map

@@ -1 +1 @@
-{"version":3,"file":"declare.js","names":["ParquetRowGroup","constructor","rowCount","arguments","length","undefined","columnData","_defineProperty"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\n/** @todo better name, this is an internal type? */\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  /** Actual column chunks */\n  values: any[]; // ArrayLike<any>;\n  count: number;\n  dictionary?: ParquetDictionary;\n  /** The \"raw\" page header from the file */\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRow {\n  [key: string]: any;\n}\n\n/** @\n * Holds data for one row group (column chunks) */\nexport class ParquetRowGroup {\n  /** Number of rows in this page */\n  rowCount: number;\n  /** Map of Column chunks */\n  columnData: Record<string, ParquetColumnChunk>;\n\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetColumnChunk> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n\n/** Holds the data for one column chunk */\nexport interface ParquetColumnChunk {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n"],"mappings":";AAgIA,OAAO,MAAMA,eAAe,CAAC;EAM3BC,WAAWA,CAAA,EAA4E;IAAA,IAA3EC,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;IAAA,IAAEG,UAA8C,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;IAAAI,eAAA;IAAAA,eAAA;IACnF,IAAI,CAACL,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;EAC9B;AACF"}
+{"version":3,"file":"declare.js","names":["ParquetRowGroup","constructor","rowCount","arguments","length","undefined","columnData","_defineProperty"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\n/** @todo better name, this is an internal type? */\nexport interface ParquetReaderContext {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n  /** If true, binary values are not converted to strings */\n  preserveBinary?: boolean;\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  /** Actual column chunks */\n  values: any[]; // ArrayLike<any>;\n  count: number;\n  dictionary?: ParquetDictionary;\n  /** The \"raw\" page header from the file */\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRow {\n  [key: string]: any;\n}\n\n/** @\n * Holds data for one row group (column chunks) */\nexport class ParquetRowGroup {\n  /** Number of rows in this page */\n  rowCount: number;\n  /** Map of Column chunks */\n  columnData: Record<string, ParquetColumnChunk>;\n\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetColumnChunk> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n\n/** Holds the data for one column chunk */\nexport interface ParquetColumnChunk {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n"],"mappings":";AAkIA,OAAO,MAAMA,eAAe,CAAC;EAM3BC,WAAWA,CAAA,EAA4E;IAAA,IAA3EC,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;IAAA,IAAEG,UAA8C,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;IAAAI,eAAA;IAAAA,eAAA;IACnF,IAAI,CAACL,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;EAC9B;AACF"}

package/dist/esm/parquetjs/utils/read-utils.js

@@ -1,5 +1,5 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { TBufferedTransport, TCompactProtocol, TFramedTransport } from 'thrift';
+import { TBufferedTransport, TCompactProtocol, TFramedTransport } from '../parquet-thrift';
 import { FileMetaData, PageHeader } from '../parquet-thrift';
 class UFramedTransport extends TFramedTransport {
   constructor() {

package/dist/esm/parquetjs/utils/read-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"read-utils.js","names":["TBufferedTransport","TCompactProtocol","TFramedTransport","FileMetaData","PageHeader","UFramedTransport","constructor","arguments","_defineProperty","serializeThrift","obj","output","transport","undefined","buf","push","protocol","write","flush","Buffer","concat","decodeThrift","offset","readPos","read","getThriftEnum","klass","value","k","Error","decodeFileMetadata","metadata","length","decodePageHeader","pageHeader","getBitWidth","val","Math","ceil","log2","fieldIndexOf","arr","elem","j","m","i"],"sources":["../../../../src/parquetjs/utils/read-utils.ts"],"sourcesContent":["import {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';\nimport {FileMetaData, PageHeader} from '../parquet-thrift';\n\nclass UFramedTransport extends TFramedTransport {\n  public readPos: number = 0;\n}\n\n/**\n * Helper function that serializes a thrift object into a buffer\n */\nexport function serializeThrift(obj: any): Buffer {\n  const output: Buffer[] = [];\n\n  const transport = new TBufferedTransport(undefined, (buf) => {\n    output.push(buf as Buffer);\n  });\n\n  const protocol = new TCompactProtocol(transport);\n  obj.write(protocol);\n  transport.flush();\n\n  return Buffer.concat(output);\n}\n\nexport function decodeThrift(obj: any, buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  obj.read(protocol);\n  return transport.readPos - offset;\n}\n\n/**\n * FIXME not ideal that this is linear\n */\nexport function getThriftEnum(klass: any, value: number | string): string {\n  for (const k in klass) {\n    if (klass[k] === value) {\n      return k;\n    }\n  }\n  throw new Error('Invalid ENUM value');\n}\n\nexport function decodeFileMetadata(buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  const metadata = FileMetaData.read(protocol);\n  return {length: transport.readPos - offset, metadata};\n}\n\nexport function decodePageHeader(buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  const pageHeader = PageHeader.read(protocol);\n  return {length: transport.readPos - offset, pageHeader};\n}\n\n/**\n * Get the number of bits required to store a given value\n */\nexport function getBitWidth(val: number): number {\n  if (val === 0) {\n    return 0;\n    // tslint:disable-next-line:no-else-after-return\n  }\n  return Math.ceil(Math.log2(val + 1));\n}\n\n// Supports MQTT path wildcards\n// + all immediate children\n// # all descendents\nexport function fieldIndexOf(arr: string[][], elem: string[]): number {\n  for (let j = 0; j < arr.length; j++) {\n    if (arr[j].length > elem.length) {\n      continue; // eslint-disable-line no-continue\n    }\n    let m = true;\n    for (let i = 0; i < elem.length; i++) {\n      if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {\n        continue; // eslint-disable-line no-continue\n      }\n      if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {\n        continue; // eslint-disable-line no-continue\n      }\n      m = false;\n      break;\n    }\n    if (m) return j;\n  }\n  return -1;\n}\n"],"mappings":";AAAA,SAAQA,kBAAkB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAO,QAAQ;AAC7E,SAAQC,YAAY,EAAEC,UAAU,QAAO,mBAAmB;AAE1D,MAAMC,gBAAgB,SAASH,gBAAgB,CAAC;EAAAI,YAAA;IAAA,SAAAC,SAAA;IAAAC,eAAA,kBACrB,CAAC;EAAA;AAC5B;AAKA,OAAO,SAASC,eAAeA,CAACC,GAAQ,EAAU;EAChD,MAAMC,MAAgB,GAAG,EAAE;EAE3B,MAAMC,SAAS,GAAG,IAAIZ,kBAAkB,CAACa,SAAS,EAAGC,GAAG,IAAK;IAC3DH,MAAM,CAACI,IAAI,CAACD,GAAa,CAAC;EAC5B,CAAC,CAAC;EAEF,MAAME,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChDF,GAAG,CAACO,KAAK,CAACD,QAAQ,CAAC;EACnBJ,SAAS,CAACM,KAAK,CAAC,CAAC;EAEjB,OAAOC,MAAM,CAACC,MAAM,CAACT,MAAM,CAAC;AAC9B;AAEA,OAAO,SAASU,YAAYA,CAACX,GAAQ,EAAEI,GAAW,EAAEQ,MAAe,EAAE;EACnE,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,MAAMV,SAAS,GAAG,IAAIP,gBAAgB,CAACS,GAAG,CAAC;EAC3CF,SAAS,CAACW,OAAO,GAAGD,MAAM;EAC1B,MAAMN,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChDF,GAAG,CAACc,IAAI,CAACR,QAAQ,CAAC;EAClB,OAAOJ,SAAS,CAACW,OAAO,GAAGD,MAAM;AACnC;AAKA,OAAO,SAASG,aAAaA,CAACC,KAAU,EAAEC,KAAsB,EAAU;EACxE,KAAK,MAAMC,CAAC,IAAIF,KAAK,EAAE;IACrB,IAAIA,KAAK,CAACE,CAAC,CAAC,KAAKD,KAAK,EAAE;MACtB,OAAOC,CAAC;IACV;EACF;EACA,MAAM,IAAIC,KAAK,CAAC,oBAAoB,CAAC;AACvC;AAEA,OAAO,SAASC,kBAAkBA,CAAChB,GAAW,EAAEQ,MAAe,EAAE;EAC/D,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,MAAMV,SAAS,GAAG,IAAIP,gBAAgB,CAACS,GAAG,CAAC;EAC3CF,SAAS,CAACW,OAAO,GAAGD,MAAM;EAC1B,MAAMN,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChD,MAAMmB,QAAQ,GAAG5B,YAAY,CAACqB,IAAI,CAACR,QAAQ,CAAC;EAC5C,OAAO;IAACgB,MAAM,EAAEpB,SAAS,CAACW,OAAO,GAAGD,MAAM;IAAES;EAAQ,CAAC;AACvD;AAEA,OAAO,SAASE,gBAAgBA,CAACnB,GAAW,EAAEQ,MAAe,EAAE;EAC7D,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,MAAMV,SAAS,GAAG,IAAIP,gBAAgB,CAACS,GAAG,CAAC;EAC3CF,SAAS,CAACW,OAAO,GAAGD,MAAM;EAC1B,MAAMN,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChD,MAAMsB,UAAU,GAAG9B,UAAU,CAACoB,IAAI,CAACR,QAAQ,CAAC;EAC5C,OAAO;IAACgB,MAAM,EAAEpB,SAAS,CAACW,OAAO,GAAGD,MAAM;IAAEY;EAAU,CAAC;AACzD;AAKA,OAAO,SAASC,WAAWA,CAACC,GAAW,EAAU;EAC/C,IAAIA,GAAG,KAAK,CAAC,EAAE;IACb,OAAO,CAAC;EAEV;EACA,OAAOC,IAAI,CAACC,IAAI,CAACD,IAAI,CAACE,IAAI,CAACH,GAAG,GAAG,CAAC,CAAC,CAAC;AACtC;AAKA,OAAO,SAASI,YAAYA,CAACC,GAAe,EAAEC,IAAc,EAAU;EACpE,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGF,GAAG,CAACT,MAAM,EAAEW,CAAC,EAAE,EAAE;IACnC,IAAIF,GAAG,CAACE,CAAC,CAAC,CAACX,MAAM,GAAGU,IAAI,CAACV,MAAM,EAAE;MAC/B;IACF;IACA,IAAIY,CAAC,GAAG,IAAI;IACZ,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,IAAI,CAACV,MAAM,EAAEa,CAAC,EAAE,EAAE;MACpC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAKH,IAAI,CAACG,CAAC,CAAC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAK,GAAG,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAK,GAAG,EAAE;QACnE;MACF;MACA,IAAIA,CAAC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACX,MAAM,IAAIS,GAAG,CAACE,CAAC,CAAC,CAACF,GAAG,CAACE,CAAC,CAAC,CAACX,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE;QAC3D;MACF;MACAY,CAAC,GAAG,KAAK;MACT;IACF;IACA,IAAIA,CAAC,EAAE,OAAOD,CAAC;EACjB;EACA,OAAO,CAAC,CAAC;AACX"}
+{"version":3,"file":"read-utils.js","names":["TBufferedTransport","TCompactProtocol","TFramedTransport","FileMetaData","PageHeader","UFramedTransport","constructor","arguments","_defineProperty","serializeThrift","obj","output","transport","undefined","buf","push","protocol","write","flush","Buffer","concat","decodeThrift","offset","readPos","read","getThriftEnum","klass","value","k","Error","decodeFileMetadata","metadata","length","decodePageHeader","pageHeader","getBitWidth","val","Math","ceil","log2","fieldIndexOf","arr","elem","j","m","i"],"sources":["../../../../src/parquetjs/utils/read-utils.ts"],"sourcesContent":["import {TBufferedTransport, TCompactProtocol, TFramedTransport} from '../parquet-thrift';\nimport {FileMetaData, PageHeader} from '../parquet-thrift';\n\nclass UFramedTransport extends TFramedTransport {\n  public readPos: number = 0;\n}\n\n/**\n * Helper function that serializes a thrift object into a buffer\n */\nexport function serializeThrift(obj: any): Buffer {\n  const output: Buffer[] = [];\n\n  const transport = new TBufferedTransport(undefined, (buf) => {\n    output.push(buf as unknown as Buffer);\n  });\n\n  const protocol = new TCompactProtocol(transport);\n  obj.write(protocol);\n  transport.flush();\n\n  return Buffer.concat(output);\n}\n\nexport function decodeThrift(obj: any, buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  obj.read(protocol);\n  return transport.readPos - offset;\n}\n\n/**\n * FIXME not ideal that this is linear\n */\nexport function getThriftEnum(klass: any, value: number | string): string {\n  for (const k in klass) {\n    if (klass[k] === value) {\n      return k;\n    }\n  }\n  throw new Error('Invalid ENUM value');\n}\n\nexport function decodeFileMetadata(buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  const metadata = FileMetaData.read(protocol);\n  return {length: transport.readPos - offset, metadata};\n}\n\nexport function decodePageHeader(buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  const pageHeader = PageHeader.read(protocol);\n  return {length: transport.readPos - offset, pageHeader};\n}\n\n/**\n * Get the number of bits required to store a given value\n */\nexport function getBitWidth(val: number): number {\n  if (val === 0) {\n    return 0;\n    // tslint:disable-next-line:no-else-after-return\n  }\n  return Math.ceil(Math.log2(val + 1));\n}\n\n// Supports MQTT path wildcards\n// + all immediate children\n// # all descendents\nexport function fieldIndexOf(arr: string[][], elem: string[]): number {\n  for (let j = 0; j < arr.length; j++) {\n    if (arr[j].length > elem.length) {\n      continue; // eslint-disable-line no-continue\n    }\n    let m = true;\n    for (let i = 0; i < elem.length; i++) {\n      if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {\n        continue; // eslint-disable-line no-continue\n      }\n      if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {\n        continue; // eslint-disable-line no-continue\n      }\n      m = false;\n      break;\n    }\n    if (m) return j;\n  }\n  return -1;\n}\n"],"mappings":";AAAA,SAAQA,kBAAkB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAO,mBAAmB;AACxF,SAAQC,YAAY,EAAEC,UAAU,QAAO,mBAAmB;AAE1D,MAAMC,gBAAgB,SAASH,gBAAgB,CAAC;EAAAI,YAAA;IAAA,SAAAC,SAAA;IAAAC,eAAA,kBACrB,CAAC;EAAA;AAC5B;AAKA,OAAO,SAASC,eAAeA,CAACC,GAAQ,EAAU;EAChD,MAAMC,MAAgB,GAAG,EAAE;EAE3B,MAAMC,SAAS,GAAG,IAAIZ,kBAAkB,CAACa,SAAS,EAAGC,GAAG,IAAK;IAC3DH,MAAM,CAACI,IAAI,CAACD,GAAwB,CAAC;EACvC,CAAC,CAAC;EAEF,MAAME,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChDF,GAAG,CAACO,KAAK,CAACD,QAAQ,CAAC;EACnBJ,SAAS,CAACM,KAAK,CAAC,CAAC;EAEjB,OAAOC,MAAM,CAACC,MAAM,CAACT,MAAM,CAAC;AAC9B;AAEA,OAAO,SAASU,YAAYA,CAACX,GAAQ,EAAEI,GAAW,EAAEQ,MAAe,EAAE;EACnE,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,MAAMV,SAAS,GAAG,IAAIP,gBAAgB,CAACS,GAAG,CAAC;EAC3CF,SAAS,CAACW,OAAO,GAAGD,MAAM;EAC1B,MAAMN,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChDF,GAAG,CAACc,IAAI,CAACR,QAAQ,CAAC;EAClB,OAAOJ,SAAS,CAACW,OAAO,GAAGD,MAAM;AACnC;AAKA,OAAO,SAASG,aAAaA,CAACC,KAAU,EAAEC,KAAsB,EAAU;EACxE,KAAK,MAAMC,CAAC,IAAIF,KAAK,EAAE;IACrB,IAAIA,KAAK,CAACE,CAAC,CAAC,KAAKD,KAAK,EAAE;MACtB,OAAOC,CAAC;IACV;EACF;EACA,MAAM,IAAIC,KAAK,CAAC,oBAAoB,CAAC;AACvC;AAEA,OAAO,SAASC,kBAAkBA,CAAChB,GAAW,EAAEQ,MAAe,EAAE;EAC/D,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,MAAMV,SAAS,GAAG,IAAIP,gBAAgB,CAACS,GAAG,CAAC;EAC3CF,SAAS,CAACW,OAAO,GAAGD,MAAM;EAC1B,MAAMN,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChD,MAAMmB,QAAQ,GAAG5B,YAAY,CAACqB,IAAI,CAACR,QAAQ,CAAC;EAC5C,OAAO;IAACgB,MAAM,EAAEpB,SAAS,CAACW,OAAO,GAAGD,MAAM;IAAES;EAAQ,CAAC;AACvD;AAEA,OAAO,SAASE,gBAAgBA,CAACnB,GAAW,EAAEQ,MAAe,EAAE;EAC7D,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,MAAMV,SAAS,GAAG,IAAIP,gBAAgB,CAACS,GAAG,CAAC;EAC3CF,SAAS,CAACW,OAAO,GAAGD,MAAM;EAC1B,MAAMN,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChD,MAAMsB,UAAU,GAAG9B,UAAU,CAACoB,IAAI,CAACR,QAAQ,CAAC;EAC5C,OAAO;IAACgB,MAAM,EAAEpB,SAAS,CAACW,OAAO,GAAGD,MAAM;IAAEY;EAAU,CAAC;AACzD;AAKA,OAAO,SAASC,WAAWA,CAACC,GAAW,EAAU;EAC/C,IAAIA,GAAG,KAAK,CAAC,EAAE;IACb,OAAO,CAAC;EAEV;EACA,OAAOC,IAAI,CAACC,IAAI,CAACD,IAAI,CAACE,IAAI,CAACH,GAAG,GAAG,CAAC,CAAC,CAAC;AACtC;AAKA,OAAO,SAASI,YAAYA,CAACC,GAAe,EAAEC,IAAc,EAAU;EACpE,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGF,GAAG,CAACT,MAAM,EAAEW,CAAC,EAAE,EAAE;IACnC,IAAIF,GAAG,CAACE,CAAC,CAAC,CAACX,MAAM,GAAGU,IAAI,CAACV,MAAM,EAAE;MAC/B;IACF;IACA,IAAIY,CAAC,GAAG,IAAI;IACZ,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,IAAI,CAACV,MAAM,EAAEa,CAAC,EAAE,EAAE;MACpC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAKH,IAAI,CAACG,CAAC,CAAC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAK,GAAG,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAK,GAAG,EAAE;QACnE;MACF;MACA,IAAIA,CAAC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACX,MAAM,IAAIS,GAAG,CAACE,CAAC,CAAC,CAACF,GAAG,CAACE,CAAC,CAAC,CAACX,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE;QAC3D;MACF;MACAY,CAAC,GAAG,KAAK;MACT;IACF;IACA,IAAIA,CAAC,EAAE,OAAOD,CAAC;EACjB;EACA,OAAO,CAAC,CAAC;AACX"}