@loaders.gl/parquet 4.2.0-alpha.2 → 4.2.0-alpha.4

package/dist/parquetjs/parser/decoders.js.map

@@ -1 +1 @@
-{"version":3,"file":"decoders.js","names":["PARQUET_CODECS","ConvertedType","Encoding","FieldRepetitionType","PageType","Type","decompress","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","decodePageHeader","getThriftEnum","getBitWidth","decodeDataPages","buffer","context","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","pageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","repetition_type","optional","repeated","num_children","res","name","fields","logicalType","converted_type","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","header","_header$data_page_hea","_header$data_page_hea2","_header$data_page_hea3","_header$data_page_hea4","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","slice","uncompressed_page_size","rLevelEncoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","_header$data_page_hea5","_header$data_page_hea6","_header$data_page_hea7","_header$data_page_hea8","data_page_header_v2","num_nulls","valuesBufCursor","is_compressed","_pageHeader$dictionar","dictCursor","dictionary_page_header","decodedDictionaryValues","preserveBinary","d","toString","ArrayBuffer","isView","Buffer","isBuffer","byteOffset","byteLength"],"sources":["../../../src/parquetjs/parser/decoders.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n  ParquetCodec,\n  ParquetColumnChunk,\n  ParquetReaderContext,\n  ParquetPageData,\n  ParquetType,\n  PrimitiveType,\n  SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n  ConvertedType,\n  Encoding,\n  FieldRepetitionType,\n  PageHeader,\n  PageType,\n  SchemaElement,\n  Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/**\n * Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n */\nexport async function decodeDataPages(\n  buffer: Buffer,\n  context: ParquetReaderContext\n): Promise<ParquetColumnChunk> {\n  const cursor: CursorBuffer = {\n    buffer,\n    offset: 0,\n    size: buffer.length\n  };\n\n  const data: ParquetColumnChunk = {\n    rlevels: [],\n    dlevels: [],\n    values: [],\n    pageHeaders: [],\n    count: 0\n  };\n\n  let dictionary = context.dictionary || [];\n\n  while (\n    // @ts-ignore size can be undefined\n    cursor.offset < cursor.size &&\n    (!context.numValues || data.dlevels.length < Number(context.numValues))\n  ) {\n    // Looks like we have to decode these in sequence due to cursor updates?\n    const page = await decodePage(cursor, context);\n\n    if (page.dictionary) {\n      dictionary = page.dictionary;\n      // eslint-disable-next-line no-continue\n      continue;\n    }\n\n    if (dictionary.length) {\n      // eslint-disable-next-line no-loop-func\n      page.values = page.values.map((value) => dictionary[value]);\n    }\n\n    for (let index = 0; index < page.rlevels.length; index++) {\n      data.rlevels.push(page.rlevels[index]);\n      data.dlevels.push(page.dlevels[index]);\n      const value = page.values[index];\n\n      if (value !== undefined) {\n        data.values.push(value);\n      }\n    }\n\n    data.count += page.count;\n    data.pageHeaders.push(page.pageHeader);\n  }\n\n  return data;\n}\n\n/**\n * Decode parquet page based on page type\n * @param cursor\n * @param context\n */\nexport async function decodePage(\n  cursor: CursorBuffer,\n  context: ParquetReaderContext\n): Promise<ParquetPageData> {\n  let page;\n\n  const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n  cursor.offset += length;\n\n  const pageType = getThriftEnum(PageType, pageHeader.type);\n\n  switch (pageType) {\n    case 'DATA_PAGE':\n      page = await decodeDataPage(cursor, pageHeader, context);\n      break;\n    case 'DATA_PAGE_V2':\n      page = await decodeDataPageV2(cursor, pageHeader, context);\n      break;\n    case 'DICTIONARY_PAGE':\n      page = {\n        dictionary: await decodeDictionaryPage(cursor, pageHeader, context),\n        pageHeader\n      };\n      break;\n    default:\n      throw new Error(`invalid page type: ${pageType}`);\n  }\n\n  return page;\n}\n\n/**\n * Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n *   result.next - offset at the end of function\n *   result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n */\nexport function decodeSchema(\n  schemaElements: SchemaElement[],\n  offset: number,\n  len: number\n): {\n  offset: number;\n  next: number;\n  schema: SchemaDefinition;\n} {\n  const schema: SchemaDefinition = {};\n  let next = offset;\n  for (let i = 0; i < len; i++) {\n    const schemaElement = schemaElements[next];\n\n    const repetitionType =\n      next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n    let optional = false;\n    let repeated = false;\n    switch (repetitionType) {\n      case 'REQUIRED':\n        break;\n      case 'OPTIONAL':\n        optional = true;\n        break;\n      case 'REPEATED':\n        repeated = true;\n        break;\n      default:\n        throw new Error('parquet: unknown repetition type');\n    }\n\n    if (schemaElement.num_children! > 0) {\n      const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n      next = res.next;\n      schema[schemaElement.name] = {\n        // type: undefined,\n        optional,\n        repeated,\n        fields: res.schema\n      };\n    } else {\n      const type = getThriftEnum(Type, schemaElement.type!);\n      let logicalType = type;\n\n      if (schemaElement.converted_type) {\n        logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n      }\n\n      switch (logicalType) {\n        case 'DECIMAL':\n          logicalType = `${logicalType}_${type}` as ParquetType;\n          break;\n        default:\n      }\n\n      schema[schemaElement.name] = {\n        type: logicalType as ParquetType,\n        typeLength: schemaElement.type_length,\n        presision: schemaElement.precision,\n        scale: schemaElement.scale,\n        optional,\n        repeated\n      };\n      next++;\n    }\n  }\n  return {schema, offset, next};\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): any[] {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/**\n * Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n */\nasync function decodeDataPage(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  context: ParquetReaderContext\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n  const valueCount = header.data_page_header?.num_values;\n\n  /* uncompress page */\n  let dataCursor = cursor;\n\n  if (context.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      context.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n    dataCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n    cursor.offset = cursorEnd;\n  }\n\n  /* read repetition levels */\n  const rLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.repetition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n\n  if (context.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.rLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  const dLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.definition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (context.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.dLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    dLevels.fill(0);\n  }\n  let valueCountNonNull = 0;\n  for (const dlvl of dLevels) {\n    if (dlvl === context.column.dLevelMax) {\n      valueCountNonNull++;\n    }\n  }\n\n  /* read values */\n  const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n  const decodeOptions: ParquetCodecOptions = {\n    typeLength: context.column.typeLength,\n    bitWidth: context.column.typeLength\n  };\n\n  const values = decodeValues(\n    context.column.primitiveType!,\n    valueEncoding,\n    dataCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n */\nasync function decodeDataPageV2(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  context: ParquetReaderContext\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n\n  const valueCount = header.data_page_header_v2?.num_values;\n  // @ts-ignore\n  const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n  const valueEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header_v2?.encoding!\n  ) as ParquetCodec;\n\n  /* read repetition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (context.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.rLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (context.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.dLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    dLevels.fill(0);\n  }\n\n  /* read values */\n  let valuesBufCursor = cursor;\n\n  if (header.data_page_header_v2?.is_compressed) {\n    const valuesBuf = await decompress(\n      context.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n\n    valuesBufCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const decodeOptions = {\n    typeLength: context.column.typeLength,\n    bitWidth: context.column.typeLength\n  };\n\n  const values = decodeValues(\n    context.column.primitiveType!,\n    valueEncoding,\n    valuesBufCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param context\n */\nasync function decodeDictionaryPage(\n  cursor: CursorBuffer,\n  pageHeader: PageHeader,\n  context: ParquetReaderContext\n): Promise<(string | ArrayBuffer)[]> {\n  const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n  let dictCursor = {\n    offset: 0,\n    buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n    size: cursorEnd - cursor.offset\n  };\n\n  cursor.offset = cursorEnd;\n\n  if (context.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      context.compression,\n      dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n      pageHeader.uncompressed_page_size\n    );\n\n    dictCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const numValues = pageHeader?.dictionary_page_header?.num_values || 0;\n\n  const decodedDictionaryValues = decodeValues(\n    context.column.primitiveType!,\n    context.column.encoding!,\n    dictCursor,\n    numValues,\n    // TODO - this looks wrong?\n    context as ParquetCodecOptions\n  );\n\n  // Makes it look a little easier\n  let values: any[];\n  if (context?.preserveBinary) {\n    values = decodedDictionaryValues.map((d) => preserveBinary(d));\n  } else {\n    values = decodedDictionaryValues.map((d) => d.toString());\n  }\n  return values;\n}\n\nfunction preserveBinary(d: any): ArrayBuffer | ArrayBufferView | string {\n  if (ArrayBuffer.isView(d)) {\n    return d;\n  }\n  // Convert to ArrayBuffer\n  if (Buffer.isBuffer(d)) {\n    return d.buffer.slice(d.byteOffset, d.byteLength);\n  }\n  return d.toString();\n}\n"],"mappings":"SAU2CA,cAAc;AAAA,SAEvDC,aAAa,EACbC,QAAQ,EACRC,mBAAmB,EAEnBC,QAAQ,EAERC,IAAI;AAAA,SAEEC,UAAU;AAAA,SACVC,kBAAkB,EAAEC,sBAAsB;AAAA,SAC1CC,gBAAgB,EAAEC,aAAa,EAAEC,WAAW;AASpD,OAAO,eAAeC,eAAeA,CACnCC,MAAc,EACdC,OAA6B,EACA;EAC7B,MAAMC,MAAoB,GAAG;IAC3BF,MAAM;IACNG,MAAM,EAAE,CAAC;IACTC,IAAI,EAAEJ,MAAM,CAACK;EACf,CAAC;EAED,MAAMC,IAAwB,GAAG;IAC/BC,OAAO,EAAE,EAAE;IACXC,OAAO,EAAE,EAAE;IACXC,MAAM,EAAE,EAAE;IACVC,WAAW,EAAE,EAAE;IACfC,KAAK,EAAE;EACT,CAAC;EAED,IAAIC,UAAU,GAAGX,OAAO,CAACW,UAAU,IAAI,EAAE;EAEzC,OAEEV,MAAM,CAACC,MAAM,GAAGD,MAAM,CAACE,IAAI,KAC1B,CAACH,OAAO,CAACY,SAAS,IAAIP,IAAI,CAACE,OAAO,CAACH,MAAM,GAAGS,MAAM,CAACb,OAAO,CAACY,SAAS,CAAC,CAAC,EACvE;IAEA,MAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAM,EAAED,OAAO,CAAC;IAE9C,IAAIc,IAAI,CAACH,UAAU,EAAE;MACnBA,UAAU,GAAGG,IAAI,CAACH,UAAU;MAE5B;IACF;IAEA,IAAIA,UAAU,CAACP,MAAM,EAAE;MAErBU,IAAI,CAACN,MAAM,GAAGM,IAAI,CAACN,MAAM,CAACQ,GAAG,CAAEC,KAAK,IAAKN,UAAU,CAACM,KAAK,CAAC,CAAC;IAC7D;IAEA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGJ,IAAI,CAACR,OAAO,CAACF,MAAM,EAAEc,KAAK,EAAE,EAAE;MACxDb,IAAI,CAACC,OAAO,CAACa,IAAI,CAACL,IAAI,CAACR,OAAO,CAACY,KAAK,CAAC,CAAC;MACtCb,IAAI,CAACE,OAAO,CAACY,IAAI,CAACL,IAAI,CAACP,OAAO,CAACW,KAAK,CAAC,CAAC;MACtC,MAAMD,KAAK,GAAGH,IAAI,CAACN,MAAM,CAACU,KAAK,CAAC;MAEhC,IAAID,KAAK,KAAKG,SAAS,EAAE;QACvBf,IAAI,CAACG,MAAM,CAACW,IAAI,CAACF,KAAK,CAAC;MACzB;IACF;IAEAZ,IAAI,CAACK,KAAK,IAAII,IAAI,CAACJ,KAAK;IACxBL,IAAI,CAACI,WAAW,CAACU,IAAI,CAACL,IAAI,CAACO,UAAU,CAAC;EACxC;EAEA,OAAOhB,IAAI;AACb;AAOA,OAAO,eAAeU,UAAUA,CAC9Bd,MAAoB,EACpBD,OAA6B,EACH;EAC1B,IAAIc,IAAI;EAER,MAAM;IAACO,UAAU;IAAEjB;EAAM,CAAC,GAAGT,gBAAgB,CAACM,MAAM,CAACF,MAAM,EAAEE,MAAM,CAACC,MAAM,CAAC;EAC3ED,MAAM,CAACC,MAAM,IAAIE,MAAM;EAEvB,MAAMkB,QAAQ,GAAG1B,aAAa,CAACN,QAAQ,EAAE+B,UAAU,CAACE,IAAI,CAAC;EAEzD,QAAQD,QAAQ;IACd,KAAK,WAAW;MACdR,IAAI,GAAG,MAAMU,cAAc,CAACvB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MACxD;IACF,KAAK,cAAc;MACjBc,IAAI,GAAG,MAAMW,gBAAgB,CAACxB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MAC1D;IACF,KAAK,iBAAiB;MACpBc,IAAI,GAAG;QACLH,UAAU,EAAE,MAAMe,oBAAoB,CAACzB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;QACnEqB;MACF,CAAC;MACD;IACF;MACE,MAAM,IAAIM,KAAK,CAAE,sBAAqBL,QAAS,EAAC,CAAC;EACrD;EAEA,OAAOR,IAAI;AACb;AAYA,OAAO,SAASc,YAAYA,CAC1BC,cAA+B,EAC/B3B,MAAc,EACd4B,GAAW,EAKX;EACA,MAAMC,MAAwB,GAAG,CAAC,CAAC;EACnC,IAAIC,IAAI,GAAG9B,MAAM;EACjB,KAAK,IAAI+B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,GAAG,EAAEG,CAAC,EAAE,EAAE;IAC5B,MAAMC,aAAa,GAAGL,cAAc,CAACG,IAAI,CAAC;IAE1C,MAAMG,cAAc,GAClBH,IAAI,GAAG,CAAC,GAAGpC,aAAa,CAACP,mBAAmB,EAAE6C,aAAa,CAACE,eAAgB,CAAC,GAAG,MAAM;IAExF,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,QAAQ,GAAG,KAAK;IACpB,QAAQH,cAAc;MACpB,KAAK,UAAU;QACb;MACF,KAAK,UAAU;QACbE,QAAQ,GAAG,IAAI;QACf;MACF,KAAK,UAAU;QACbC,QAAQ,GAAG,IAAI;QACf;MACF;QACE,MAAM,IAAIX,KAAK,CAAC,kCAAkC,CAAC;IACvD;IAEA,IAAIO,aAAa,CAACK,YAAY,GAAI,CAAC,EAAE;MACnC,MAAMC,GAAG,GAAGZ,YAAY,CAACC,cAAc,EAAEG,IAAI,GAAG,CAAC,EAAEE,aAAa,CAACK,YAAa,CAAC;MAC/EP,IAAI,GAAGQ,GAAG,CAACR,IAAI;MACfD,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAE3BJ,QAAQ;QACRC,QAAQ;QACRI,MAAM,EAAEF,GAAG,CAACT;MACd,CAAC;IACH,CAAC,MAAM;MACL,MAAMR,IAAI,GAAG3B,aAAa,CAACL,IAAI,EAAE2C,aAAa,CAACX,IAAK,CAAC;MACrD,IAAIoB,WAAW,GAAGpB,IAAI;MAEtB,IAAIW,aAAa,CAACU,cAAc,EAAE;QAChCD,WAAW,GAAG/C,aAAa,CAACT,aAAa,EAAE+C,aAAa,CAACU,cAAc,CAAC;MAC1E;MAEA,QAAQD,WAAW;QACjB,KAAK,SAAS;UACZA,WAAW,GAAI,GAAEA,WAAY,IAAGpB,IAAK,EAAgB;UACrD;QACF;MACF;MAEAQ,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAC3BlB,IAAI,EAAEoB,WAA0B;QAChCE,UAAU,EAAEX,aAAa,CAACY,WAAW;QACrCC,SAAS,EAAEb,aAAa,CAACc,SAAS;QAClCC,KAAK,EAAEf,aAAa,CAACe,KAAK;QAC1BZ,QAAQ;QACRC;MACF,CAAC;MACDN,IAAI,EAAE;IACR;EACF;EACA,OAAO;IAACD,MAAM;IAAE7B,MAAM;IAAE8B;EAAI,CAAC;AAC/B;AAKA,SAASkB,YAAYA,CACnB3B,IAAmB,EACnB4B,QAAsB,EACtBlD,MAAoB,EACpBS,KAAa,EACb0C,IAAyB,EAClB;EACP,IAAI,EAAED,QAAQ,IAAIjE,cAAc,CAAC,EAAE;IACjC,MAAM,IAAIyC,KAAK,CAAE,qBAAoBwB,QAAS,EAAC,CAAC;EAClD;EACA,OAAOjE,cAAc,CAACiE,QAAQ,CAAC,CAACD,YAAY,CAAC3B,IAAI,EAAEtB,MAAM,EAAES,KAAK,EAAE0C,IAAI,CAAC;AACzE;AAQA,eAAe5B,cAAcA,CAC3BvB,MAAoB,EACpBoD,MAAkB,EAClBrD,OAA6B,EACH;EAAA,IAAAsD,qBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAMC,SAAS,GAAGzD,MAAM,CAACC,MAAM,GAAGmD,MAAM,CAACM,oBAAoB;EAC7D,MAAMC,UAAU,IAAAN,qBAAA,GAAGD,MAAM,CAACQ,gBAAgB,cAAAP,qBAAA,uBAAvBA,qBAAA,CAAyBQ,UAAU;EAGtD,IAAIC,UAAU,GAAG9D,MAAM;EAEvB,IAAID,OAAO,CAACgE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAMzE,UAAU,CAChCQ,OAAO,CAACgE,WAAW,EACnB/D,MAAM,CAACF,MAAM,CAACmE,KAAK,CAACjE,MAAM,CAACC,MAAM,EAAEwD,SAAS,CAAC,EAC7CL,MAAM,CAACc,sBACT,CAAC;IACDJ,UAAU,GAAG;MACXhE,MAAM,EAAEkE,SAAS;MACjB/D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE8D,SAAS,CAAC7D;IAClB,CAAC;IACDH,MAAM,CAACC,MAAM,GAAGwD,SAAS;EAC3B;EAGA,MAAMU,cAAc,GAAGxE,aAAa,CAClCR,QAAQ,GAAAmE,sBAAA,GACRF,MAAM,CAACQ,gBAAgB,cAAAN,sBAAA,uBAAvBA,sBAAA,CAAyBc,yBAC3B,CAAiB;EAEjB,IAAIC,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EAEnC,IAAI5D,OAAO,CAACwE,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGpB,YAAY,CAACzD,kBAAkB,EAAE2E,cAAc,EAAEL,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAE7E,WAAW,CAACG,OAAO,CAACwE,MAAM,CAACC,SAAS,CAAC;MAC/CE,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,MAAMC,cAAc,GAAGjF,aAAa,CAClCR,QAAQ,GAAAoE,sBAAA,GACRH,MAAM,CAACQ,gBAAgB,cAAAL,sBAAA,uBAAvBA,sBAAA,CAAyBsB,yBAC3B,CAAiB;EAEjB,IAAIC,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI5D,OAAO,CAACwE,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAG7B,YAAY,CAACzD,kBAAkB,EAAEoF,cAAc,EAAEd,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAE7E,WAAW,CAACG,OAAO,CAACwE,MAAM,CAACQ,SAAS,CAAC;MAC/CL,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EACA,IAAIK,iBAAiB,GAAG,CAAC;EACzB,KAAK,MAAMC,IAAI,IAAIH,OAAO,EAAE;IAC1B,IAAIG,IAAI,KAAKlF,OAAO,CAACwE,MAAM,CAACQ,SAAS,EAAE;MACrCC,iBAAiB,EAAE;IACrB;EACF;EAGA,MAAME,aAAa,GAAGvF,aAAa,CAACR,QAAQ,GAAAqE,sBAAA,GAAEJ,MAAM,CAACQ,gBAAgB,cAAAJ,sBAAA,uBAAvBA,sBAAA,CAAyBN,QAAS,CAAiB;EACjG,MAAMiC,aAAkC,GAAG;IACzCvC,UAAU,EAAE7C,OAAO,CAACwE,MAAM,CAAC3B,UAAU;IACrC6B,QAAQ,EAAE1E,OAAO,CAACwE,MAAM,CAAC3B;EAC3B,CAAC;EAED,MAAMrC,MAAM,GAAG0C,YAAY,CACzBlD,OAAO,CAACwE,MAAM,CAACa,aAAa,EAC5BF,aAAa,EACbpB,UAAU,EACVkB,iBAAiB,EACjBG,aACF,CAAC;EAED,OAAO;IACL7E,OAAO,EAAEwE,OAAO;IAChBzE,OAAO,EAAEgE,OAAO;IAChB9D,MAAM;IACNE,KAAK,EAAEkD,UAAW;IAClBvC,UAAU,EAAEgC;EACd,CAAC;AACH;AASA,eAAe5B,gBAAgBA,CAC7BxB,MAAoB,EACpBoD,MAAkB,EAClBrD,OAA6B,EACH;EAAA,IAAAsF,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAM/B,SAAS,GAAGzD,MAAM,CAACC,MAAM,GAAGmD,MAAM,CAACM,oBAAoB;EAE7D,MAAMC,UAAU,IAAA0B,sBAAA,GAAGjC,MAAM,CAACqC,mBAAmB,cAAAJ,sBAAA,uBAA1BA,sBAAA,CAA4BxB,UAAU;EAEzD,MAAMmB,iBAAiB,GAAGrB,UAAU,KAAA2B,sBAAA,GAAGlC,MAAM,CAACqC,mBAAmB,cAAAH,sBAAA,uBAA1BA,sBAAA,CAA4BI,SAAS;EAC5E,MAAMR,aAAa,GAAGvF,aAAa,CACjCR,QAAQ,GAAAoG,sBAAA,GACRnC,MAAM,CAACqC,mBAAmB,cAAAF,sBAAA,uBAA1BA,sBAAA,CAA4BrC,QAC9B,CAAiB;EAIjB,IAAImB,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI5D,OAAO,CAACwE,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGpB,YAAY,CAACzD,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAE2D,UAAU,EAAG;MACtFc,QAAQ,EAAE7E,WAAW,CAACG,OAAO,CAACwE,MAAM,CAACC,SAAS,CAAC;MAC/CE,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;EAIA,IAAIG,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI5D,OAAO,CAACwE,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAG7B,YAAY,CAACzD,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAE2D,UAAU,EAAG;MACtFc,QAAQ,EAAE7E,WAAW,CAACG,OAAO,CAACwE,MAAM,CAACQ,SAAS,CAAC;MAC/CL,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,IAAIgB,eAAe,GAAG3F,MAAM;EAE5B,KAAAwF,sBAAA,GAAIpC,MAAM,CAACqC,mBAAmB,cAAAD,sBAAA,eAA1BA,sBAAA,CAA4BI,aAAa,EAAE;IAC7C,MAAM5B,SAAS,GAAG,MAAMzE,UAAU,CAChCQ,OAAO,CAACgE,WAAW,EACnB/D,MAAM,CAACF,MAAM,CAACmE,KAAK,CAACjE,MAAM,CAACC,MAAM,EAAEwD,SAAS,CAAC,EAC7CL,MAAM,CAACc,sBACT,CAAC;IAEDyB,eAAe,GAAG;MAChB7F,MAAM,EAAEkE,SAAS;MACjB/D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE8D,SAAS,CAAC7D;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGwD,SAAS;EAC3B;EAEA,MAAM0B,aAAa,GAAG;IACpBvC,UAAU,EAAE7C,OAAO,CAACwE,MAAM,CAAC3B,UAAU;IACrC6B,QAAQ,EAAE1E,OAAO,CAACwE,MAAM,CAAC3B;EAC3B,CAAC;EAED,MAAMrC,MAAM,GAAG0C,YAAY,CACzBlD,OAAO,CAACwE,MAAM,CAACa,aAAa,EAC5BF,aAAa,EACbS,eAAe,EACfX,iBAAiB,EACjBG,aACF,CAAC;EAED,OAAO;IACL7E,OAAO,EAAEwE,OAAO;IAChBzE,OAAO,EAAEgE,OAAO;IAChB9D,MAAM;IACNE,KAAK,EAAEkD,UAAW;IAClBvC,UAAU,EAAEgC;EACd,CAAC;AACH;AAQA,eAAe3B,oBAAoBA,CACjCzB,MAAoB,EACpBoB,UAAsB,EACtBrB,OAA6B,EACM;EAAA,IAAA8F,qBAAA;EACnC,MAAMpC,SAAS,GAAGzD,MAAM,CAACC,MAAM,GAAGmB,UAAU,CAACsC,oBAAoB;EAEjE,IAAIoC,UAAU,GAAG;IACf7F,MAAM,EAAE,CAAC;IACTH,MAAM,EAAEE,MAAM,CAACF,MAAM,CAACmE,KAAK,CAACjE,MAAM,CAACC,MAAM,EAAEwD,SAAS,CAAC;IACrDvD,IAAI,EAAEuD,SAAS,GAAGzD,MAAM,CAACC;EAC3B,CAAC;EAEDD,MAAM,CAACC,MAAM,GAAGwD,SAAS;EAEzB,IAAI1D,OAAO,CAACgE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAMzE,UAAU,CAChCQ,OAAO,CAACgE,WAAW,EACnB+B,UAAU,CAAChG,MAAM,CAACmE,KAAK,CAAC6B,UAAU,CAAC7F,MAAM,EAAEwD,SAAS,CAAC,EACrDrC,UAAU,CAAC8C,sBACb,CAAC;IAED4B,UAAU,GAAG;MACXhG,MAAM,EAAEkE,SAAS;MACjB/D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE8D,SAAS,CAAC7D;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGwD,SAAS;EAC3B;EAEA,MAAM9C,SAAS,GAAG,CAAAS,UAAU,aAAVA,UAAU,wBAAAyE,qBAAA,GAAVzE,UAAU,CAAE2E,sBAAsB,cAAAF,qBAAA,uBAAlCA,qBAAA,CAAoChC,UAAU,KAAI,CAAC;EAErE,MAAMmC,uBAAuB,GAAG/C,YAAY,CAC1ClD,OAAO,CAACwE,MAAM,CAACa,aAAa,EAC5BrF,OAAO,CAACwE,MAAM,CAACrB,QAAQ,EACvB4C,UAAU,EACVnF,SAAS,EAETZ,OACF,CAAC;EAGD,IAAIQ,MAAa;EACjB,IAAIR,OAAO,aAAPA,OAAO,eAAPA,OAAO,CAAEkG,cAAc,EAAE;IAC3B1F,MAAM,GAAGyF,uBAAuB,CAACjF,GAAG,CAAEmF,CAAC,IAAKD,cAAc,CAACC,CAAC,CAAC,CAAC;EAChE,CAAC,MAAM;IACL3F,MAAM,GAAGyF,uBAAuB,CAACjF,GAAG,CAAEmF,CAAC,IAAKA,CAAC,CAACC,QAAQ,CAAC,CAAC,CAAC;EAC3D;EACA,OAAO5F,MAAM;AACf;AAEA,SAAS0F,cAAcA,CAACC,CAAM,EAA0C;EACtE,IAAIE,WAAW,CAACC,MAAM,CAACH,CAAC,CAAC,EAAE;IACzB,OAAOA,CAAC;EACV;EAEA,IAAII,MAAM,CAACC,QAAQ,CAACL,CAAC,CAAC,EAAE;IACtB,OAAOA,CAAC,CAACpG,MAAM,CAACmE,KAAK,CAACiC,CAAC,CAACM,UAAU,EAAEN,CAAC,CAACO,UAAU,CAAC;EACnD;EACA,OAAOP,CAAC,CAACC,QAAQ,CAAC,CAAC;AACrB"}
+{"version":3,"file":"decoders.js","names":["PARQUET_CODECS","ConvertedType","Encoding","FieldRepetitionType","PageType","Type","decompress","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","decodePageHeader","getThriftEnum","getBitWidth","decodeDataPages","buffer","context","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","pageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","repetition_type","optional","repeated","num_children","res","name","fields","logicalType","converted_type","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","header","_header$data_page_hea","_header$data_page_hea2","_header$data_page_hea3","_header$data_page_hea4","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","slice","uncompressed_page_size","rLevelEncoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","_header$data_page_hea5","_header$data_page_hea6","_header$data_page_hea7","_header$data_page_hea8","data_page_header_v2","num_nulls","valuesBufCursor","is_compressed","_pageHeader$dictionar","dictCursor","dictionary_page_header","decodedDictionaryValues","preserveBinary","d","toString","ArrayBuffer","isView","Buffer","isBuffer","byteOffset","byteLength"],"sources":["../../../src/parquetjs/parser/decoders.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n  ParquetCodec,\n  ParquetColumnChunk,\n  ParquetReaderContext,\n  ParquetPageData,\n  ParquetType,\n  PrimitiveType,\n  SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n  ConvertedType,\n  Encoding,\n  FieldRepetitionType,\n  PageHeader,\n  PageType,\n  SchemaElement,\n  Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../lib/constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/**\n * Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n */\nexport async function decodeDataPages(\n  buffer: Buffer,\n  context: ParquetReaderContext\n): Promise<ParquetColumnChunk> {\n  const cursor: CursorBuffer = {\n    buffer,\n    offset: 0,\n    size: buffer.length\n  };\n\n  const data: ParquetColumnChunk = {\n    rlevels: [],\n    dlevels: [],\n    values: [],\n    pageHeaders: [],\n    count: 0\n  };\n\n  let dictionary = context.dictionary || [];\n\n  while (\n    // @ts-ignore size can be undefined\n    cursor.offset < cursor.size &&\n    (!context.numValues || data.dlevels.length < Number(context.numValues))\n  ) {\n    // Looks like we have to decode these in sequence due to cursor updates?\n    const page = await decodePage(cursor, context);\n\n    if (page.dictionary) {\n      dictionary = page.dictionary;\n      // eslint-disable-next-line no-continue\n      continue;\n    }\n\n    if (dictionary.length) {\n      // eslint-disable-next-line no-loop-func\n      page.values = page.values.map((value) => dictionary[value]);\n    }\n\n    for (let index = 0; index < page.rlevels.length; index++) {\n      data.rlevels.push(page.rlevels[index]);\n      data.dlevels.push(page.dlevels[index]);\n      const value = page.values[index];\n\n      if (value !== undefined) {\n        data.values.push(value);\n      }\n    }\n\n    data.count += page.count;\n    data.pageHeaders.push(page.pageHeader);\n  }\n\n  return data;\n}\n\n/**\n * Decode parquet page based on page type\n * @param cursor\n * @param context\n */\nexport async function decodePage(\n  cursor: CursorBuffer,\n  context: ParquetReaderContext\n): Promise<ParquetPageData> {\n  let page;\n\n  const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n  cursor.offset += length;\n\n  const pageType = getThriftEnum(PageType, pageHeader.type);\n\n  switch (pageType) {\n    case 'DATA_PAGE':\n      page = await decodeDataPage(cursor, pageHeader, context);\n      break;\n    case 'DATA_PAGE_V2':\n      page = await decodeDataPageV2(cursor, pageHeader, context);\n      break;\n    case 'DICTIONARY_PAGE':\n      page = {\n        dictionary: await decodeDictionaryPage(cursor, pageHeader, context),\n        pageHeader\n      };\n      break;\n    default:\n      throw new Error(`invalid page type: ${pageType}`);\n  }\n\n  return page;\n}\n\n/**\n * Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n *   result.next - offset at the end of function\n *   result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n */\nexport function decodeSchema(\n  schemaElements: SchemaElement[],\n  offset: number,\n  len: number\n): {\n  offset: number;\n  next: number;\n  schema: SchemaDefinition;\n} {\n  const schema: SchemaDefinition = {};\n  let next = offset;\n  for (let i = 0; i < len; i++) {\n    const schemaElement = schemaElements[next];\n\n    const repetitionType =\n      next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n    let optional = false;\n    let repeated = false;\n    switch (repetitionType) {\n      case 'REQUIRED':\n        break;\n      case 'OPTIONAL':\n        optional = true;\n        break;\n      case 'REPEATED':\n        repeated = true;\n        break;\n      default:\n        throw new Error('parquet: unknown repetition type');\n    }\n\n    if (schemaElement.num_children! > 0) {\n      const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n      next = res.next;\n      schema[schemaElement.name] = {\n        // type: undefined,\n        optional,\n        repeated,\n        fields: res.schema\n      };\n    } else {\n      const type = getThriftEnum(Type, schemaElement.type!);\n      let logicalType = type;\n\n      if (schemaElement.converted_type) {\n        logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n      }\n\n      switch (logicalType) {\n        case 'DECIMAL':\n          logicalType = `${logicalType}_${type}` as ParquetType;\n          break;\n        default:\n      }\n\n      schema[schemaElement.name] = {\n        type: logicalType as ParquetType,\n        typeLength: schemaElement.type_length,\n        presision: schemaElement.precision,\n        scale: schemaElement.scale,\n        optional,\n        repeated\n      };\n      next++;\n    }\n  }\n  return {schema, offset, next};\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): any[] {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/**\n * Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n */\nasync function decodeDataPage(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  context: ParquetReaderContext\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n  const valueCount = header.data_page_header?.num_values;\n\n  /* uncompress page */\n  let dataCursor = cursor;\n\n  if (context.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      context.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n    dataCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n    cursor.offset = cursorEnd;\n  }\n\n  /* read repetition levels */\n  const rLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.repetition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n\n  if (context.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.rLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  const dLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.definition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (context.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.dLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    dLevels.fill(0);\n  }\n  let valueCountNonNull = 0;\n  for (const dlvl of dLevels) {\n    if (dlvl === context.column.dLevelMax) {\n      valueCountNonNull++;\n    }\n  }\n\n  /* read values */\n  const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n  const decodeOptions: ParquetCodecOptions = {\n    typeLength: context.column.typeLength,\n    bitWidth: context.column.typeLength\n  };\n\n  const values = decodeValues(\n    context.column.primitiveType!,\n    valueEncoding,\n    dataCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n */\nasync function decodeDataPageV2(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  context: ParquetReaderContext\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n\n  const valueCount = header.data_page_header_v2?.num_values;\n  // @ts-ignore\n  const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n  const valueEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header_v2?.encoding!\n  ) as ParquetCodec;\n\n  /* read repetition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (context.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.rLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (context.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.dLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    dLevels.fill(0);\n  }\n\n  /* read values */\n  let valuesBufCursor = cursor;\n\n  if (header.data_page_header_v2?.is_compressed) {\n    const valuesBuf = await decompress(\n      context.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n\n    valuesBufCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const decodeOptions = {\n    typeLength: context.column.typeLength,\n    bitWidth: context.column.typeLength\n  };\n\n  const values = decodeValues(\n    context.column.primitiveType!,\n    valueEncoding,\n    valuesBufCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param context\n */\nasync function decodeDictionaryPage(\n  cursor: CursorBuffer,\n  pageHeader: PageHeader,\n  context: ParquetReaderContext\n): Promise<(string | ArrayBuffer)[]> {\n  const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n  let dictCursor = {\n    offset: 0,\n    buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n    size: cursorEnd - cursor.offset\n  };\n\n  cursor.offset = cursorEnd;\n\n  if (context.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      context.compression,\n      dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n      pageHeader.uncompressed_page_size\n    );\n\n    dictCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const numValues = pageHeader?.dictionary_page_header?.num_values || 0;\n\n  const decodedDictionaryValues = decodeValues(\n    context.column.primitiveType!,\n    context.column.encoding!,\n    dictCursor,\n    numValues,\n    // TODO - this looks wrong?\n    context as ParquetCodecOptions\n  );\n\n  // Makes it look a little easier\n  let values: any[];\n  if (context?.preserveBinary) {\n    values = decodedDictionaryValues.map((d) => preserveBinary(d));\n  } else {\n    values = decodedDictionaryValues.map((d) => d.toString());\n  }\n  return values;\n}\n\nfunction preserveBinary(d: any): ArrayBuffer | ArrayBufferView | string {\n  if (ArrayBuffer.isView(d)) {\n    return d;\n  }\n  // Convert to ArrayBuffer\n  if (Buffer.isBuffer(d)) {\n    return d.buffer.slice(d.byteOffset, d.byteLength);\n  }\n  return d.toString();\n}\n"],"mappings":"SAU2CA,cAAc;AAAA,SAEvDC,aAAa,EACbC,QAAQ,EACRC,mBAAmB,EAEnBC,QAAQ,EAERC,IAAI;AAAA,SAEEC,UAAU;AAAA,SACVC,kBAAkB,EAAEC,sBAAsB;AAAA,SAC1CC,gBAAgB,EAAEC,aAAa,EAAEC,WAAW;AASpD,OAAO,eAAeC,eAAeA,CACnCC,MAAc,EACdC,OAA6B,EACA;EAC7B,MAAMC,MAAoB,GAAG;IAC3BF,MAAM;IACNG,MAAM,EAAE,CAAC;IACTC,IAAI,EAAEJ,MAAM,CAACK;EACf,CAAC;EAED,MAAMC,IAAwB,GAAG;IAC/BC,OAAO,EAAE,EAAE;IACXC,OAAO,EAAE,EAAE;IACXC,MAAM,EAAE,EAAE;IACVC,WAAW,EAAE,EAAE;IACfC,KAAK,EAAE;EACT,CAAC;EAED,IAAIC,UAAU,GAAGX,OAAO,CAACW,UAAU,IAAI,EAAE;EAEzC,OAEEV,MAAM,CAACC,MAAM,GAAGD,MAAM,CAACE,IAAI,KAC1B,CAACH,OAAO,CAACY,SAAS,IAAIP,IAAI,CAACE,OAAO,CAACH,MAAM,GAAGS,MAAM,CAACb,OAAO,CAACY,SAAS,CAAC,CAAC,EACvE;IAEA,MAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAM,EAAED,OAAO,CAAC;IAE9C,IAAIc,IAAI,CAACH,UAAU,EAAE;MACnBA,UAAU,GAAGG,IAAI,CAACH,UAAU;MAE5B;IACF;IAEA,IAAIA,UAAU,CAACP,MAAM,EAAE;MAErBU,IAAI,CAACN,MAAM,GAAGM,IAAI,CAACN,MAAM,CAACQ,GAAG,CAAEC,KAAK,IAAKN,UAAU,CAACM,KAAK,CAAC,CAAC;IAC7D;IAEA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGJ,IAAI,CAACR,OAAO,CAACF,MAAM,EAAEc,KAAK,EAAE,EAAE;MACxDb,IAAI,CAACC,OAAO,CAACa,IAAI,CAACL,IAAI,CAACR,OAAO,CAACY,KAAK,CAAC,CAAC;MACtCb,IAAI,CAACE,OAAO,CAACY,IAAI,CAACL,IAAI,CAACP,OAAO,CAACW,KAAK,CAAC,CAAC;MACtC,MAAMD,KAAK,GAAGH,IAAI,CAACN,MAAM,CAACU,KAAK,CAAC;MAEhC,IAAID,KAAK,KAAKG,SAAS,EAAE;QACvBf,IAAI,CAACG,MAAM,CAACW,IAAI,CAACF,KAAK,CAAC;MACzB;IACF;IAEAZ,IAAI,CAACK,KAAK,IAAII,IAAI,CAACJ,KAAK;IACxBL,IAAI,CAACI,WAAW,CAACU,IAAI,CAACL,IAAI,CAACO,UAAU,CAAC;EACxC;EAEA,OAAOhB,IAAI;AACb;AAOA,OAAO,eAAeU,UAAUA,CAC9Bd,MAAoB,EACpBD,OAA6B,EACH;EAC1B,IAAIc,IAAI;EAER,MAAM;IAACO,UAAU;IAAEjB;EAAM,CAAC,GAAGT,gBAAgB,CAACM,MAAM,CAACF,MAAM,EAAEE,MAAM,CAACC,MAAM,CAAC;EAC3ED,MAAM,CAACC,MAAM,IAAIE,MAAM;EAEvB,MAAMkB,QAAQ,GAAG1B,aAAa,CAACN,QAAQ,EAAE+B,UAAU,CAACE,IAAI,CAAC;EAEzD,QAAQD,QAAQ;IACd,KAAK,WAAW;MACdR,IAAI,GAAG,MAAMU,cAAc,CAACvB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MACxD;IACF,KAAK,cAAc;MACjBc,IAAI,GAAG,MAAMW,gBAAgB,CAACxB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MAC1D;IACF,KAAK,iBAAiB;MACpBc,IAAI,GAAG;QACLH,UAAU,EAAE,MAAMe,oBAAoB,CAACzB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;QACnEqB;MACF,CAAC;MACD;IACF;MACE,MAAM,IAAIM,KAAK,CAAE,sBAAqBL,QAAS,EAAC,CAAC;EACrD;EAEA,OAAOR,IAAI;AACb;AAYA,OAAO,SAASc,YAAYA,CAC1BC,cAA+B,EAC/B3B,MAAc,EACd4B,GAAW,EAKX;EACA,MAAMC,MAAwB,GAAG,CAAC,CAAC;EACnC,IAAIC,IAAI,GAAG9B,MAAM;EACjB,KAAK,IAAI+B,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,GAAG,EAAEG,CAAC,EAAE,EAAE;IAC5B,MAAMC,aAAa,GAAGL,cAAc,CAACG,IAAI,CAAC;IAE1C,MAAMG,cAAc,GAClBH,IAAI,GAAG,CAAC,GAAGpC,aAAa,CAACP,mBAAmB,EAAE6C,aAAa,CAACE,eAAgB,CAAC,GAAG,MAAM;IAExF,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,QAAQ,GAAG,KAAK;IACpB,QAAQH,cAAc;MACpB,KAAK,UAAU;QACb;MACF,KAAK,UAAU;QACbE,QAAQ,GAAG,IAAI;QACf;MACF,KAAK,UAAU;QACbC,QAAQ,GAAG,IAAI;QACf;MACF;QACE,MAAM,IAAIX,KAAK,CAAC,kCAAkC,CAAC;IACvD;IAEA,IAAIO,aAAa,CAACK,YAAY,GAAI,CAAC,EAAE;MACnC,MAAMC,GAAG,GAAGZ,YAAY,CAACC,cAAc,EAAEG,IAAI,GAAG,CAAC,EAAEE,aAAa,CAACK,YAAa,CAAC;MAC/EP,IAAI,GAAGQ,GAAG,CAACR,IAAI;MACfD,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAE3BJ,QAAQ;QACRC,QAAQ;QACRI,MAAM,EAAEF,GAAG,CAACT;MACd,CAAC;IACH,CAAC,MAAM;MACL,MAAMR,IAAI,GAAG3B,aAAa,CAACL,IAAI,EAAE2C,aAAa,CAACX,IAAK,CAAC;MACrD,IAAIoB,WAAW,GAAGpB,IAAI;MAEtB,IAAIW,aAAa,CAACU,cAAc,EAAE;QAChCD,WAAW,GAAG/C,aAAa,CAACT,aAAa,EAAE+C,aAAa,CAACU,cAAc,CAAC;MAC1E;MAEA,QAAQD,WAAW;QACjB,KAAK,SAAS;UACZA,WAAW,GAAI,GAAEA,WAAY,IAAGpB,IAAK,EAAgB;UACrD;QACF;MACF;MAEAQ,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAC3BlB,IAAI,EAAEoB,WAA0B;QAChCE,UAAU,EAAEX,aAAa,CAACY,WAAW;QACrCC,SAAS,EAAEb,aAAa,CAACc,SAAS;QAClCC,KAAK,EAAEf,aAAa,CAACe,KAAK;QAC1BZ,QAAQ;QACRC;MACF,CAAC;MACDN,IAAI,EAAE;IACR;EACF;EACA,OAAO;IAACD,MAAM;IAAE7B,MAAM;IAAE8B;EAAI,CAAC;AAC/B;AAKA,SAASkB,YAAYA,CACnB3B,IAAmB,EACnB4B,QAAsB,EACtBlD,MAAoB,EACpBS,KAAa,EACb0C,IAAyB,EAClB;EACP,IAAI,EAAED,QAAQ,IAAIjE,cAAc,CAAC,EAAE;IACjC,MAAM,IAAIyC,KAAK,CAAE,qBAAoBwB,QAAS,EAAC,CAAC;EAClD;EACA,OAAOjE,cAAc,CAACiE,QAAQ,CAAC,CAACD,YAAY,CAAC3B,IAAI,EAAEtB,MAAM,EAAES,KAAK,EAAE0C,IAAI,CAAC;AACzE;AAQA,eAAe5B,cAAcA,CAC3BvB,MAAoB,EACpBoD,MAAkB,EAClBrD,OAA6B,EACH;EAAA,IAAAsD,qBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAMC,SAAS,GAAGzD,MAAM,CAACC,MAAM,GAAGmD,MAAM,CAACM,oBAAoB;EAC7D,MAAMC,UAAU,IAAAN,qBAAA,GAAGD,MAAM,CAACQ,gBAAgB,cAAAP,qBAAA,uBAAvBA,qBAAA,CAAyBQ,UAAU;EAGtD,IAAIC,UAAU,GAAG9D,MAAM;EAEvB,IAAID,OAAO,CAACgE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAMzE,UAAU,CAChCQ,OAAO,CAACgE,WAAW,EACnB/D,MAAM,CAACF,MAAM,CAACmE,KAAK,CAACjE,MAAM,CAACC,MAAM,EAAEwD,SAAS,CAAC,EAC7CL,MAAM,CAACc,sBACT,CAAC;IACDJ,UAAU,GAAG;MACXhE,MAAM,EAAEkE,SAAS;MACjB/D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE8D,SAAS,CAAC7D;IAClB,CAAC;IACDH,MAAM,CAACC,MAAM,GAAGwD,SAAS;EAC3B;EAGA,MAAMU,cAAc,GAAGxE,aAAa,CAClCR,QAAQ,GAAAmE,sBAAA,GACRF,MAAM,CAACQ,gBAAgB,cAAAN,sBAAA,uBAAvBA,sBAAA,CAAyBc,yBAC3B,CAAiB;EAEjB,IAAIC,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EAEnC,IAAI5D,OAAO,CAACwE,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGpB,YAAY,CAACzD,kBAAkB,EAAE2E,cAAc,EAAEL,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAE7E,WAAW,CAACG,OAAO,CAACwE,MAAM,CAACC,SAAS,CAAC;MAC/CE,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,MAAMC,cAAc,GAAGjF,aAAa,CAClCR,QAAQ,GAAAoE,sBAAA,GACRH,MAAM,CAACQ,gBAAgB,cAAAL,sBAAA,uBAAvBA,sBAAA,CAAyBsB,yBAC3B,CAAiB;EAEjB,IAAIC,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI5D,OAAO,CAACwE,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAG7B,YAAY,CAACzD,kBAAkB,EAAEoF,cAAc,EAAEd,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAE7E,WAAW,CAACG,OAAO,CAACwE,MAAM,CAACQ,SAAS,CAAC;MAC/CL,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EACA,IAAIK,iBAAiB,GAAG,CAAC;EACzB,KAAK,MAAMC,IAAI,IAAIH,OAAO,EAAE;IAC1B,IAAIG,IAAI,KAAKlF,OAAO,CAACwE,MAAM,CAACQ,SAAS,EAAE;MACrCC,iBAAiB,EAAE;IACrB;EACF;EAGA,MAAME,aAAa,GAAGvF,aAAa,CAACR,QAAQ,GAAAqE,sBAAA,GAAEJ,MAAM,CAACQ,gBAAgB,cAAAJ,sBAAA,uBAAvBA,sBAAA,CAAyBN,QAAS,CAAiB;EACjG,MAAMiC,aAAkC,GAAG;IACzCvC,UAAU,EAAE7C,OAAO,CAACwE,MAAM,CAAC3B,UAAU;IACrC6B,QAAQ,EAAE1E,OAAO,CAACwE,MAAM,CAAC3B;EAC3B,CAAC;EAED,MAAMrC,MAAM,GAAG0C,YAAY,CACzBlD,OAAO,CAACwE,MAAM,CAACa,aAAa,EAC5BF,aAAa,EACbpB,UAAU,EACVkB,iBAAiB,EACjBG,aACF,CAAC;EAED,OAAO;IACL7E,OAAO,EAAEwE,OAAO;IAChBzE,OAAO,EAAEgE,OAAO;IAChB9D,MAAM;IACNE,KAAK,EAAEkD,UAAW;IAClBvC,UAAU,EAAEgC;EACd,CAAC;AACH;AASA,eAAe5B,gBAAgBA,CAC7BxB,MAAoB,EACpBoD,MAAkB,EAClBrD,OAA6B,EACH;EAAA,IAAAsF,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAM/B,SAAS,GAAGzD,MAAM,CAACC,MAAM,GAAGmD,MAAM,CAACM,oBAAoB;EAE7D,MAAMC,UAAU,IAAA0B,sBAAA,GAAGjC,MAAM,CAACqC,mBAAmB,cAAAJ,sBAAA,uBAA1BA,sBAAA,CAA4BxB,UAAU;EAEzD,MAAMmB,iBAAiB,GAAGrB,UAAU,KAAA2B,sBAAA,GAAGlC,MAAM,CAACqC,mBAAmB,cAAAH,sBAAA,uBAA1BA,sBAAA,CAA4BI,SAAS;EAC5E,MAAMR,aAAa,GAAGvF,aAAa,CACjCR,QAAQ,GAAAoG,sBAAA,GACRnC,MAAM,CAACqC,mBAAmB,cAAAF,sBAAA,uBAA1BA,sBAAA,CAA4BrC,QAC9B,CAAiB;EAIjB,IAAImB,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI5D,OAAO,CAACwE,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGpB,YAAY,CAACzD,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAE2D,UAAU,EAAG;MACtFc,QAAQ,EAAE7E,WAAW,CAACG,OAAO,CAACwE,MAAM,CAACC,SAAS,CAAC;MAC/CE,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;EAIA,IAAIG,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI5D,OAAO,CAACwE,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAG7B,YAAY,CAACzD,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAE2D,UAAU,EAAG;MACtFc,QAAQ,EAAE7E,WAAW,CAACG,OAAO,CAACwE,MAAM,CAACQ,SAAS,CAAC;MAC/CL,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,IAAIgB,eAAe,GAAG3F,MAAM;EAE5B,KAAAwF,sBAAA,GAAIpC,MAAM,CAACqC,mBAAmB,cAAAD,sBAAA,eAA1BA,sBAAA,CAA4BI,aAAa,EAAE;IAC7C,MAAM5B,SAAS,GAAG,MAAMzE,UAAU,CAChCQ,OAAO,CAACgE,WAAW,EACnB/D,MAAM,CAACF,MAAM,CAACmE,KAAK,CAACjE,MAAM,CAACC,MAAM,EAAEwD,SAAS,CAAC,EAC7CL,MAAM,CAACc,sBACT,CAAC;IAEDyB,eAAe,GAAG;MAChB7F,MAAM,EAAEkE,SAAS;MACjB/D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE8D,SAAS,CAAC7D;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGwD,SAAS;EAC3B;EAEA,MAAM0B,aAAa,GAAG;IACpBvC,UAAU,EAAE7C,OAAO,CAACwE,MAAM,CAAC3B,UAAU;IACrC6B,QAAQ,EAAE1E,OAAO,CAACwE,MAAM,CAAC3B;EAC3B,CAAC;EAED,MAAMrC,MAAM,GAAG0C,YAAY,CACzBlD,OAAO,CAACwE,MAAM,CAACa,aAAa,EAC5BF,aAAa,EACbS,eAAe,EACfX,iBAAiB,EACjBG,aACF,CAAC;EAED,OAAO;IACL7E,OAAO,EAAEwE,OAAO;IAChBzE,OAAO,EAAEgE,OAAO;IAChB9D,MAAM;IACNE,KAAK,EAAEkD,UAAW;IAClBvC,UAAU,EAAEgC;EACd,CAAC;AACH;AAQA,eAAe3B,oBAAoBA,CACjCzB,MAAoB,EACpBoB,UAAsB,EACtBrB,OAA6B,EACM;EAAA,IAAA8F,qBAAA;EACnC,MAAMpC,SAAS,GAAGzD,MAAM,CAACC,MAAM,GAAGmB,UAAU,CAACsC,oBAAoB;EAEjE,IAAIoC,UAAU,GAAG;IACf7F,MAAM,EAAE,CAAC;IACTH,MAAM,EAAEE,MAAM,CAACF,MAAM,CAACmE,KAAK,CAACjE,MAAM,CAACC,MAAM,EAAEwD,SAAS,CAAC;IACrDvD,IAAI,EAAEuD,SAAS,GAAGzD,MAAM,CAACC;EAC3B,CAAC;EAEDD,MAAM,CAACC,MAAM,GAAGwD,SAAS;EAEzB,IAAI1D,OAAO,CAACgE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAMzE,UAAU,CAChCQ,OAAO,CAACgE,WAAW,EACnB+B,UAAU,CAAChG,MAAM,CAACmE,KAAK,CAAC6B,UAAU,CAAC7F,MAAM,EAAEwD,SAAS,CAAC,EACrDrC,UAAU,CAAC8C,sBACb,CAAC;IAED4B,UAAU,GAAG;MACXhG,MAAM,EAAEkE,SAAS;MACjB/D,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE8D,SAAS,CAAC7D;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGwD,SAAS;EAC3B;EAEA,MAAM9C,SAAS,GAAG,CAAAS,UAAU,aAAVA,UAAU,wBAAAyE,qBAAA,GAAVzE,UAAU,CAAE2E,sBAAsB,cAAAF,qBAAA,uBAAlCA,qBAAA,CAAoChC,UAAU,KAAI,CAAC;EAErE,MAAMmC,uBAAuB,GAAG/C,YAAY,CAC1ClD,OAAO,CAACwE,MAAM,CAACa,aAAa,EAC5BrF,OAAO,CAACwE,MAAM,CAACrB,QAAQ,EACvB4C,UAAU,EACVnF,SAAS,EAETZ,OACF,CAAC;EAGD,IAAIQ,MAAa;EACjB,IAAIR,OAAO,aAAPA,OAAO,eAAPA,OAAO,CAAEkG,cAAc,EAAE;IAC3B1F,MAAM,GAAGyF,uBAAuB,CAACjF,GAAG,CAAEmF,CAAC,IAAKD,cAAc,CAACC,CAAC,CAAC,CAAC;EAChE,CAAC,MAAM;IACL3F,MAAM,GAAGyF,uBAAuB,CAACjF,GAAG,CAAEmF,CAAC,IAAKA,CAAC,CAACC,QAAQ,CAAC,CAAC,CAAC;EAC3D;EACA,OAAO5F,MAAM;AACf;AAEA,SAAS0F,cAAcA,CAACC,CAAM,EAA0C;EACtE,IAAIE,WAAW,CAACC,MAAM,CAACH,CAAC,CAAC,EAAE;IACzB,OAAOA,CAAC;EACV;EAEA,IAAII,MAAM,CAACC,QAAQ,CAACL,CAAC,CAAC,EAAE;IACtB,OAAOA,CAAC,CAACpG,MAAM,CAACmE,KAAK,CAACiC,CAAC,CAACM,UAAU,EAAEN,CAAC,CAACO,UAAU,CAAC;EACnD;EACA,OAAOP,CAAC,CAACC,QAAQ,CAAC,CAAC;AACrB"}

package/dist/parquetjs/parser/parquet-reader.js

@@ -1,7 +1,7 @@
 import { ParquetSchema } from "../schema/schema.js";
 import { decodeSchema } from "./decoders.js";
 import { materializeRows } from "../schema/shred.js";
-import { PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED } from "../../constants.js";
+import { PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED } from "../../lib/constants.js";
 import { CompressionCodec, Type } from "../parquet-thrift/index.js";
 import { decodeFileMetadata, getThriftEnum, fieldIndexOf } from "../utils/read-utils.js";
 import { decodeDataPages, decodePage } from "./decoders.js";

package/dist/parquetjs/parser/parquet-reader.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-reader.js","names":["ParquetSchema","decodeSchema","materializeRows","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","ParquetReader","constructor","file","props","metadata","defaultProps","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","columnList","map","x","Array","isArray","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","num_children","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","arrayBuffer","read","buffer","Buffer","from","magic","toString","Error","trailerLen","size","trailerBuf","slice","metadataSize","readUInt32LE","metadataOffset","arrayBuffer2","metadataBuf","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_context$dictionary","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","_colChunk$meta_data6","Math","min","context","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","preserveBinary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","defaultDictionarySize","cursor","offset","decodedPage"],"sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRows} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetRowGroup,\n  ParquetCompression,\n  ParquetColumnChunk,\n  PrimitiveType,\n  ParquetReaderContext\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n  preserveBinary?: boolean;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  static defaultProps: Required<ParquetReaderProps> = {\n    defaultDictionarySize: 1e6,\n    preserveBinary: false\n  };\n\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...ParquetReader.defaultProps, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRows(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const arrayBuffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const buffer = Buffer.from(arrayBuffer);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const arrayBuffer = await this.file.read(this.file.size - trailerLen, trailerLen);\n    const trailerBuf = Buffer.from(arrayBuffer);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const arrayBuffer2 = await this.file.read(metadataOffset, metadataSize);\n    const metadataBuf = Buffer.from(arrayBuffer2);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetRowGroup> {\n    const buffer: ParquetRowGroup = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetColumnChunk> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const context: ParquetReaderContext = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: [],\n      // Options - TBD is this the right place for these?\n      preserveBinary: this.props.preserveBinary\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, context, pagesOffset);\n    }\n\n    dictionary = context.dictionary?.length ? context.dictionary : dictionary;\n    const arrayBuffer = await this.file.read(pagesOffset, pagesSize);\n    const pagesBuf = Buffer.from(arrayBuffer);\n    return await decodeDataPages(pagesBuf, {...context, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param context\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    context: ParquetReaderContext,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const arrayBuffer = await this.file.read(dictionaryPageOffset, dictionarySize);\n    const pagesBuf = Buffer.from(arrayBuffer);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, context);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":"SAGQA,aAAa;AAAA,SACbC,YAAY;AAAA,SACZC,eAAe;AAAA,SAEfC,aAAa,EAAEC,uBAAuB;AAAA,SACzBC,gBAAgB,EAA0BC,IAAI;AAAA,SAQ3DC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY;AAAA,SAC/CC,eAAe,EAAEC,UAAU;AAmBnC,OAAO,MAAMC,aAAa,CAAC;EAUzBC,WAAWA,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAA,KAJ5DA,KAAK;IAAA,KACLD,IAAI;IAAA,KACJE,QAAQ,GAAiC,IAAI;IAG3C,IAAI,CAACF,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGH,aAAa,CAACK,YAAY;MAAE,GAAGF;IAAK,CAAC;EACxD;EAEAG,KAAKA,CAAA,EAAS;IAEZ,IAAI,CAACJ,IAAI,CAACI,KAAK,CAAC,CAAC;EACnB;EAKA,OAAOC,WAAWA,CAACJ,KAA6B,EAAE;IAChD,WAAW,MAAMK,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACN,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMO,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;EAGA,OAAOD,gBAAgBA,CAACN,KAA6B,EAAE;IACrD,MAAMQ,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACX,KAAK,CAAC,EAAE;MACzD,MAAMb,eAAe,CAACqB,MAAM,EAAEE,QAAQ,CAAC;IACzC;EACF;EAGA,OAAOC,gBAAgBA,CAACX,KAA6B,EAAE;IAErD,MAAMY,UAAsB,GAAG,CAAC,CAAAZ,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEY,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAC3B,CAAC;IAED,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACgB,eAAe,CAAC,CAAC;IAC7C,MAAMT,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IAErC,MAAMS,aAAa,GAAG,CAAAjB,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEkB,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMX,QAAQ,GAAG,MAAM,IAAI,CAACY,YAAY,CACtCd,MAAM,EACNP,QAAQ,CAACkB,UAAU,CAACE,aAAa,CAAC,EAClCT,UACF,CAAC;MACD,MAAMF,QAAQ;IAChB;EACF;EAEA,MAAMa,WAAWA,CAAA,EAAoB;IACnC,MAAMtB,QAAQ,GAAG,MAAM,IAAI,CAACgB,eAAe,CAAC,CAAC;IAC7C,OAAOO,MAAM,CAACvB,QAAQ,CAACwB,QAAQ,CAAC;EAClC;EAEA,MAAMhB,SAASA,CAAA,EAA2B;IACxC,MAAMR,QAAQ,GAAG,MAAM,IAAI,CAACgB,eAAe,CAAC,CAAC;IAC7C,MAAMS,IAAI,GAAGzB,QAAQ,CAACO,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEmB;IAAgB,CAAC,GAAGzC,YAAY,CAACe,QAAQ,CAACO,MAAM,EAAE,CAAC,EAAEkB,IAAI,CAACE,YAAa,CAAC;IACvF,MAAMpB,MAAM,GAAG,IAAIvB,aAAa,CAAC0C,gBAAgB,CAAC;IAClD,OAAOnB,MAAM;EACf;EAMA,MAAMqB,iBAAiBA,CAAA,EAAoC;IACzD,MAAM5B,QAAQ,GAAG,MAAM,IAAI,CAACgB,eAAe,CAAC,CAAC;IAC7C,MAAMa,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAI9B,QAAQ,CAAC+B,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMb,eAAeA,CAAA,EAA0B;IAC7C,IAAI,CAAC,IAAI,CAAChB,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACkC,UAAU,CAAC,CAAC;MACvB,IAAI,CAAClC,QAAQ,GAAG,IAAI,CAACmC,UAAU,CAAC,CAAC;IACnC;IACA,OAAO,IAAI,CAACnC,QAAQ;EACtB;EAKA,MAAMkC,UAAUA,CAAA,EAAkB;IAChC,MAAME,WAAW,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAAC,CAAC,EAAElD,aAAa,CAACgC,MAAM,CAAC;IACjE,MAAMmB,MAAM,GAAGC,MAAM,CAACC,IAAI,CAACJ,WAAW,CAAC;IACvC,MAAMK,KAAK,GAAGH,MAAM,CAACI,QAAQ,CAAC,CAAC;IAC/B,QAAQD,KAAK;MACX,KAAKtD,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAIuD,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,CAAE,+BAA8BF,KAAM,GAAE,CAAC;IAC5D;EACF;EAGA,MAAMN,UAAUA,CAAA,EAA0B;IACxC,MAAMS,UAAU,GAAGzD,aAAa,CAACgC,MAAM,GAAG,CAAC;IAC3C,MAAMiB,WAAW,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAAC,IAAI,CAACvC,IAAI,CAAC+C,IAAI,GAAGD,UAAU,EAAEA,UAAU,CAAC;IACjF,MAAME,UAAU,GAAGP,MAAM,CAACC,IAAI,CAACJ,WAAW,CAAC;IAE3C,MAAMK,KAAK,GAAGK,UAAU,CAACC,KAAK,CAAC,CAAC,CAAC,CAACL,QAAQ,CAAC,CAAC;IAC5C,IAAID,KAAK,KAAKtD,aAAa,EAAE;MAC3B,MAAM,IAAIwD,KAAK,CAAE,oCAAmCF,KAAM,GAAE,CAAC;IAC/D;IAEA,MAAMO,YAAY,GAAGF,UAAU,CAACG,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAACpD,IAAI,CAAC+C,IAAI,GAAGG,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAG/D,aAAa,CAACgC,MAAM,EAAE;MACzC,MAAM,IAAIwB,KAAK,CAAE,yBAAwBO,cAAe,EAAC,CAAC;IAC5D;IAEA,MAAMC,YAAY,GAAG,MAAM,IAAI,CAACrD,IAAI,CAACuC,IAAI,CAACa,cAAc,EAAEF,YAAY,CAAC;IACvE,MAAMI,WAAW,GAAGb,MAAM,CAACC,IAAI,CAACW,YAAY,CAAC;IAI7C,MAAM;MAACnD;IAAQ,CAAC,GAAGT,kBAAkB,CAAC6D,WAAW,CAAC;IAClD,OAAOpD,QAAQ;EACjB;EAGA,MAAMqB,YAAYA,CAChBd,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB,EACI;IAC1B,MAAM2B,MAAuB,GAAG;MAC9Be,QAAQ,EAAE9B,MAAM,CAACd,QAAQ,CAACe,QAAQ,CAAC;MACnC8B,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAI9C,QAAQ,CAAC+C,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAIjD,UAAU,CAACQ,MAAM,GAAG,CAAC,IAAI1B,YAAY,CAACkB,UAAU,EAAEgD,MAAO,CAAC,GAAG,CAAC,EAAE;QAClE;MACF;MACArB,MAAM,CAACgB,UAAU,CAACK,MAAM,CAAEE,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACvD,MAAM,EAAEgD,QAAQ,CAAC;IAClF;IACA,OAAOjB,MAAM;EACf;EAKA,MAAMwB,eAAeA,CAACvD,MAAqB,EAAEgD,QAAqB,EAA+B;IAAA,IAAAQ,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;IAC/F,IAAIf,QAAQ,CAACgB,SAAS,KAAKC,SAAS,IAAIjB,QAAQ,CAACgB,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAI5B,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAM8B,KAAK,GAAGlE,MAAM,CAACmE,SAAS,EAAAX,mBAAA,GAACR,QAAQ,CAACG,SAAS,cAAAK,mBAAA,uBAAlBA,mBAAA,CAAoBH,cAAe,CAAC;IACnE,MAAMe,IAAmB,GAAGnF,aAAa,CAACF,IAAI,GAAA0E,oBAAA,GAAET,QAAQ,CAACG,SAAS,cAAAM,oBAAA,uBAAlBA,oBAAA,CAAoBW,IAAK,CAAQ;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAIjC,KAAK,CAAE,mCAAkCgC,IAAK,EAAC,CAAC;IAC5D;IAEA,MAAME,WAA+B,GAAGrF,aAAa,CACnDH,gBAAgB,GAAA4E,oBAAA,GAChBV,QAAQ,CAACG,SAAS,cAAAO,oBAAA,uBAAlBA,oBAAA,CAAoBa,KACtB,CAAQ;IAER,MAAMC,WAAW,GAAGxD,MAAM,EAAA2C,oBAAA,GAACX,QAAQ,CAACG,SAAS,cAAAQ,oBAAA,uBAAlBA,oBAAA,CAAoBc,gBAAiB,CAAC;IACjE,IAAIC,SAAS,GAAG1D,MAAM,EAAA4C,oBAAA,GAACZ,QAAQ,CAACG,SAAS,cAAAS,oBAAA,uBAAlBA,oBAAA,CAAoBe,qBAAsB,CAAC;IAElE,IAAI,CAAC3B,QAAQ,CAACgB,SAAS,EAAE;MAAA,IAAAY,oBAAA;MACvBF,SAAS,GAAGG,IAAI,CAACC,GAAG,CAClB,IAAI,CAACvF,IAAI,CAAC+C,IAAI,GAAGkC,WAAW,EAC5BxD,MAAM,EAAA4D,oBAAA,GAAC5B,QAAQ,CAACG,SAAS,cAAAyB,oBAAA,uBAAlBA,oBAAA,CAAoBD,qBAAqB,CAClD,CAAC;IACH;IAEA,MAAMI,OAA6B,GAAG;MACpCX,IAAI;MACJY,SAAS,EAAEd,KAAK,CAACc,SAAS;MAC1BC,SAAS,EAAEf,KAAK,CAACe,SAAS;MAC1BX,WAAW;MACXY,MAAM,EAAEhB,KAAK;MACbiB,SAAS,GAAAtB,oBAAA,GAAEb,QAAQ,CAACG,SAAS,cAAAU,oBAAA,uBAAlBA,oBAAA,CAAoBuB,UAAU;MACzCC,UAAU,EAAE,EAAE;MAEdC,cAAc,EAAE,IAAI,CAAC9F,KAAK,CAAC8F;IAC7B,CAAC;IAED,IAAID,UAAU;IAEd,MAAME,oBAAoB,GAAGvC,QAAQ,aAARA,QAAQ,wBAAAc,oBAAA,GAARd,QAAQ,CAAEG,SAAS,cAAAW,oBAAA,uBAAnBA,oBAAA,CAAqB0B,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAGzE,MAAM,CAACuE,oBAAoB,CAAC;MAErDF,UAAU,GAAG,MAAM,IAAI,CAACK,aAAa,CAACD,gBAAgB,EAAEV,OAAO,EAAEP,WAAW,CAAC;IAC/E;IAEAa,UAAU,GAAG,CAAAtB,mBAAA,GAAAgB,OAAO,CAACM,UAAU,cAAAtB,mBAAA,eAAlBA,mBAAA,CAAoBnD,MAAM,GAAGmE,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMxD,WAAW,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAAC0C,WAAW,EAAEE,SAAS,CAAC;IAChE,MAAMiB,QAAQ,GAAG3D,MAAM,CAACC,IAAI,CAACJ,WAAW,CAAC;IACzC,OAAO,MAAM1C,eAAe,CAACwG,QAAQ,EAAE;MAAC,GAAGZ,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;EASA,MAAMK,aAAaA,CACjBH,oBAA4B,EAC5BR,OAA6B,EAC7BP,WAAmB,EACA;IACnB,IAAIe,oBAAoB,KAAK,CAAC,EAAE;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGf,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACvF,IAAI,CAAC+C,IAAI,GAAGiD,oBAAoB,EACrC,IAAI,CAAC/F,KAAK,CAACqG,qBACb,CAAC;IACD,MAAMhE,WAAW,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAACyD,oBAAoB,EAAEK,cAAc,CAAC;IAC9E,MAAMD,QAAQ,GAAG3D,MAAM,CAACC,IAAI,CAACJ,WAAW,CAAC;IAEzC,MAAMiE,MAAM,GAAG;MAAC/D,MAAM,EAAE4D,QAAQ;MAAEI,MAAM,EAAE,CAAC;MAAEzD,IAAI,EAAEqD,QAAQ,CAAC/E;IAAM,CAAC;IACnE,MAAMoF,WAAW,GAAG,MAAM5G,UAAU,CAAC0G,MAAM,EAAEf,OAAO,CAAC;IAErD,OAAOiB,WAAW,CAACX,UAAU;EAC/B;AACF;AA7PahG,aAAa,CACjBK,YAAY,GAAiC;EAClDmG,qBAAqB,EAAE,GAAG;EAC1BP,cAAc,EAAE;AAClB,CAAC"}
+{"version":3,"file":"parquet-reader.js","names":["ParquetSchema","decodeSchema","materializeRows","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","ParquetReader","constructor","file","props","metadata","defaultProps","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","columnList","map","x","Array","isArray","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","num_children","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","arrayBuffer","read","buffer","Buffer","from","magic","toString","Error","trailerLen","size","trailerBuf","slice","metadataSize","readUInt32LE","metadataOffset","arrayBuffer2","metadataBuf","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_context$dictionary","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","_colChunk$meta_data6","Math","min","context","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","preserveBinary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","defaultDictionarySize","cursor","offset","decodedPage"],"sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRows} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../lib/constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetRowGroup,\n  ParquetCompression,\n  ParquetColumnChunk,\n  PrimitiveType,\n  ParquetReaderContext\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n  preserveBinary?: boolean;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  static defaultProps: Required<ParquetReaderProps> = {\n    defaultDictionarySize: 1e6,\n    preserveBinary: false\n  };\n\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...ParquetReader.defaultProps, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRows(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const arrayBuffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const buffer = Buffer.from(arrayBuffer);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const arrayBuffer = await this.file.read(this.file.size - trailerLen, trailerLen);\n    const trailerBuf = Buffer.from(arrayBuffer);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const arrayBuffer2 = await this.file.read(metadataOffset, metadataSize);\n    const metadataBuf = Buffer.from(arrayBuffer2);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetRowGroup> {\n    const buffer: ParquetRowGroup = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetColumnChunk> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const context: ParquetReaderContext = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: [],\n      // Options - TBD is this the right place for these?\n      preserveBinary: this.props.preserveBinary\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, context, pagesOffset);\n    }\n\n    dictionary = context.dictionary?.length ? context.dictionary : dictionary;\n    const arrayBuffer = await this.file.read(pagesOffset, pagesSize);\n    const pagesBuf = Buffer.from(arrayBuffer);\n    return await decodeDataPages(pagesBuf, {...context, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param context\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    context: ParquetReaderContext,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const arrayBuffer = await this.file.read(dictionaryPageOffset, dictionarySize);\n    const pagesBuf = Buffer.from(arrayBuffer);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, context);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":"SAGQA,aAAa;AAAA,SACbC,YAAY;AAAA,SACZC,eAAe;AAAA,SAEfC,aAAa,EAAEC,uBAAuB;AAAA,SACzBC,gBAAgB,EAA0BC,IAAI;AAAA,SAQ3DC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY;AAAA,SAC/CC,eAAe,EAAEC,UAAU;AAmBnC,OAAO,MAAMC,aAAa,CAAC;EAUzBC,WAAWA,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAA,KAJ5DA,KAAK;IAAA,KACLD,IAAI;IAAA,KACJE,QAAQ,GAAiC,IAAI;IAG3C,IAAI,CAACF,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGH,aAAa,CAACK,YAAY;MAAE,GAAGF;IAAK,CAAC;EACxD;EAEAG,KAAKA,CAAA,EAAS;IAEZ,IAAI,CAACJ,IAAI,CAACI,KAAK,CAAC,CAAC;EACnB;EAKA,OAAOC,WAAWA,CAACJ,KAA6B,EAAE;IAChD,WAAW,MAAMK,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACN,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMO,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;EAGA,OAAOD,gBAAgBA,CAACN,KAA6B,EAAE;IACrD,MAAMQ,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACX,KAAK,CAAC,EAAE;MACzD,MAAMb,eAAe,CAACqB,MAAM,EAAEE,QAAQ,CAAC;IACzC;EACF;EAGA,OAAOC,gBAAgBA,CAACX,KAA6B,EAAE;IAErD,MAAMY,UAAsB,GAAG,CAAC,CAAAZ,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEY,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAC3B,CAAC;IAED,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACgB,eAAe,CAAC,CAAC;IAC7C,MAAMT,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IAErC,MAAMS,aAAa,GAAG,CAAAjB,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEkB,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMX,QAAQ,GAAG,MAAM,IAAI,CAACY,YAAY,CACtCd,MAAM,EACNP,QAAQ,CAACkB,UAAU,CAACE,aAAa,CAAC,EAClCT,UACF,CAAC;MACD,MAAMF,QAAQ;IAChB;EACF;EAEA,MAAMa,WAAWA,CAAA,EAAoB;IACnC,MAAMtB,QAAQ,GAAG,MAAM,IAAI,CAACgB,eAAe,CAAC,CAAC;IAC7C,OAAOO,MAAM,CAACvB,QAAQ,CAACwB,QAAQ,CAAC;EAClC;EAEA,MAAMhB,SAASA,CAAA,EAA2B;IACxC,MAAMR,QAAQ,GAAG,MAAM,IAAI,CAACgB,eAAe,CAAC,CAAC;IAC7C,MAAMS,IAAI,GAAGzB,QAAQ,CAACO,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEmB;IAAgB,CAAC,GAAGzC,YAAY,CAACe,QAAQ,CAACO,MAAM,EAAE,CAAC,EAAEkB,IAAI,CAACE,YAAa,CAAC;IACvF,MAAMpB,MAAM,GAAG,IAAIvB,aAAa,CAAC0C,gBAAgB,CAAC;IAClD,OAAOnB,MAAM;EACf;EAMA,MAAMqB,iBAAiBA,CAAA,EAAoC;IACzD,MAAM5B,QAAQ,GAAG,MAAM,IAAI,CAACgB,eAAe,CAAC,CAAC;IAC7C,MAAMa,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAI9B,QAAQ,CAAC+B,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMb,eAAeA,CAAA,EAA0B;IAC7C,IAAI,CAAC,IAAI,CAAChB,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACkC,UAAU,CAAC,CAAC;MACvB,IAAI,CAAClC,QAAQ,GAAG,IAAI,CAACmC,UAAU,CAAC,CAAC;IACnC;IACA,OAAO,IAAI,CAACnC,QAAQ;EACtB;EAKA,MAAMkC,UAAUA,CAAA,EAAkB;IAChC,MAAME,WAAW,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAAC,CAAC,EAAElD,aAAa,CAACgC,MAAM,CAAC;IACjE,MAAMmB,MAAM,GAAGC,MAAM,CAACC,IAAI,CAACJ,WAAW,CAAC;IACvC,MAAMK,KAAK,GAAGH,MAAM,CAACI,QAAQ,CAAC,CAAC;IAC/B,QAAQD,KAAK;MACX,KAAKtD,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAIuD,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,CAAE,+BAA8BF,KAAM,GAAE,CAAC;IAC5D;EACF;EAGA,MAAMN,UAAUA,CAAA,EAA0B;IACxC,MAAMS,UAAU,GAAGzD,aAAa,CAACgC,MAAM,GAAG,CAAC;IAC3C,MAAMiB,WAAW,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAAC,IAAI,CAACvC,IAAI,CAAC+C,IAAI,GAAGD,UAAU,EAAEA,UAAU,CAAC;IACjF,MAAME,UAAU,GAAGP,MAAM,CAACC,IAAI,CAACJ,WAAW,CAAC;IAE3C,MAAMK,KAAK,GAAGK,UAAU,CAACC,KAAK,CAAC,CAAC,CAAC,CAACL,QAAQ,CAAC,CAAC;IAC5C,IAAID,KAAK,KAAKtD,aAAa,EAAE;MAC3B,MAAM,IAAIwD,KAAK,CAAE,oCAAmCF,KAAM,GAAE,CAAC;IAC/D;IAEA,MAAMO,YAAY,GAAGF,UAAU,CAACG,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAACpD,IAAI,CAAC+C,IAAI,GAAGG,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAG/D,aAAa,CAACgC,MAAM,EAAE;MACzC,MAAM,IAAIwB,KAAK,CAAE,yBAAwBO,cAAe,EAAC,CAAC;IAC5D;IAEA,MAAMC,YAAY,GAAG,MAAM,IAAI,CAACrD,IAAI,CAACuC,IAAI,CAACa,cAAc,EAAEF,YAAY,CAAC;IACvE,MAAMI,WAAW,GAAGb,MAAM,CAACC,IAAI,CAACW,YAAY,CAAC;IAI7C,MAAM;MAACnD;IAAQ,CAAC,GAAGT,kBAAkB,CAAC6D,WAAW,CAAC;IAClD,OAAOpD,QAAQ;EACjB;EAGA,MAAMqB,YAAYA,CAChBd,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB,EACI;IAC1B,MAAM2B,MAAuB,GAAG;MAC9Be,QAAQ,EAAE9B,MAAM,CAACd,QAAQ,CAACe,QAAQ,CAAC;MACnC8B,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAI9C,QAAQ,CAAC+C,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAIjD,UAAU,CAACQ,MAAM,GAAG,CAAC,IAAI1B,YAAY,CAACkB,UAAU,EAAEgD,MAAO,CAAC,GAAG,CAAC,EAAE;QAClE;MACF;MACArB,MAAM,CAACgB,UAAU,CAACK,MAAM,CAAEE,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACvD,MAAM,EAAEgD,QAAQ,CAAC;IAClF;IACA,OAAOjB,MAAM;EACf;EAKA,MAAMwB,eAAeA,CAACvD,MAAqB,EAAEgD,QAAqB,EAA+B;IAAA,IAAAQ,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;IAC/F,IAAIf,QAAQ,CAACgB,SAAS,KAAKC,SAAS,IAAIjB,QAAQ,CAACgB,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAI5B,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAM8B,KAAK,GAAGlE,MAAM,CAACmE,SAAS,EAAAX,mBAAA,GAACR,QAAQ,CAACG,SAAS,cAAAK,mBAAA,uBAAlBA,mBAAA,CAAoBH,cAAe,CAAC;IACnE,MAAMe,IAAmB,GAAGnF,aAAa,CAACF,IAAI,GAAA0E,oBAAA,GAAET,QAAQ,CAACG,SAAS,cAAAM,oBAAA,uBAAlBA,oBAAA,CAAoBW,IAAK,CAAQ;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAIjC,KAAK,CAAE,mCAAkCgC,IAAK,EAAC,CAAC;IAC5D;IAEA,MAAME,WAA+B,GAAGrF,aAAa,CACnDH,gBAAgB,GAAA4E,oBAAA,GAChBV,QAAQ,CAACG,SAAS,cAAAO,oBAAA,uBAAlBA,oBAAA,CAAoBa,KACtB,CAAQ;IAER,MAAMC,WAAW,GAAGxD,MAAM,EAAA2C,oBAAA,GAACX,QAAQ,CAACG,SAAS,cAAAQ,oBAAA,uBAAlBA,oBAAA,CAAoBc,gBAAiB,CAAC;IACjE,IAAIC,SAAS,GAAG1D,MAAM,EAAA4C,oBAAA,GAACZ,QAAQ,CAACG,SAAS,cAAAS,oBAAA,uBAAlBA,oBAAA,CAAoBe,qBAAsB,CAAC;IAElE,IAAI,CAAC3B,QAAQ,CAACgB,SAAS,EAAE;MAAA,IAAAY,oBAAA;MACvBF,SAAS,GAAGG,IAAI,CAACC,GAAG,CAClB,IAAI,CAACvF,IAAI,CAAC+C,IAAI,GAAGkC,WAAW,EAC5BxD,MAAM,EAAA4D,oBAAA,GAAC5B,QAAQ,CAACG,SAAS,cAAAyB,oBAAA,uBAAlBA,oBAAA,CAAoBD,qBAAqB,CAClD,CAAC;IACH;IAEA,MAAMI,OAA6B,GAAG;MACpCX,IAAI;MACJY,SAAS,EAAEd,KAAK,CAACc,SAAS;MAC1BC,SAAS,EAAEf,KAAK,CAACe,SAAS;MAC1BX,WAAW;MACXY,MAAM,EAAEhB,KAAK;MACbiB,SAAS,GAAAtB,oBAAA,GAAEb,QAAQ,CAACG,SAAS,cAAAU,oBAAA,uBAAlBA,oBAAA,CAAoBuB,UAAU;MACzCC,UAAU,EAAE,EAAE;MAEdC,cAAc,EAAE,IAAI,CAAC9F,KAAK,CAAC8F;IAC7B,CAAC;IAED,IAAID,UAAU;IAEd,MAAME,oBAAoB,GAAGvC,QAAQ,aAARA,QAAQ,wBAAAc,oBAAA,GAARd,QAAQ,CAAEG,SAAS,cAAAW,oBAAA,uBAAnBA,oBAAA,CAAqB0B,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAGzE,MAAM,CAACuE,oBAAoB,CAAC;MAErDF,UAAU,GAAG,MAAM,IAAI,CAACK,aAAa,CAACD,gBAAgB,EAAEV,OAAO,EAAEP,WAAW,CAAC;IAC/E;IAEAa,UAAU,GAAG,CAAAtB,mBAAA,GAAAgB,OAAO,CAACM,UAAU,cAAAtB,mBAAA,eAAlBA,mBAAA,CAAoBnD,MAAM,GAAGmE,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMxD,WAAW,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAAC0C,WAAW,EAAEE,SAAS,CAAC;IAChE,MAAMiB,QAAQ,GAAG3D,MAAM,CAACC,IAAI,CAACJ,WAAW,CAAC;IACzC,OAAO,MAAM1C,eAAe,CAACwG,QAAQ,EAAE;MAAC,GAAGZ,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;EASA,MAAMK,aAAaA,CACjBH,oBAA4B,EAC5BR,OAA6B,EAC7BP,WAAmB,EACA;IACnB,IAAIe,oBAAoB,KAAK,CAAC,EAAE;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGf,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACvF,IAAI,CAAC+C,IAAI,GAAGiD,oBAAoB,EACrC,IAAI,CAAC/F,KAAK,CAACqG,qBACb,CAAC;IACD,MAAMhE,WAAW,GAAG,MAAM,IAAI,CAACtC,IAAI,CAACuC,IAAI,CAACyD,oBAAoB,EAAEK,cAAc,CAAC;IAC9E,MAAMD,QAAQ,GAAG3D,MAAM,CAACC,IAAI,CAACJ,WAAW,CAAC;IAEzC,MAAMiE,MAAM,GAAG;MAAC/D,MAAM,EAAE4D,QAAQ;MAAEI,MAAM,EAAE,CAAC;MAAEzD,IAAI,EAAEqD,QAAQ,CAAC/E;IAAM,CAAC;IACnE,MAAMoF,WAAW,GAAG,MAAM5G,UAAU,CAAC0G,MAAM,EAAEf,OAAO,CAAC;IAErD,OAAOiB,WAAW,CAACX,UAAU;EAC/B;AACF;AA7PahG,aAAa,CACjBK,YAAY,GAAiC;EAClDmG,qBAAqB,EAAE,GAAG;EAC1BP,cAAc,EAAE;AAClB,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "4.2.0-alpha.2",
+  "version": "4.2.0-alpha.4",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "type": "module",
@@ -53,21 +53,19 @@
     "util": false,
     "events": false,
     "./src/polyfills/buffer/buffer-polyfill.node.ts": "./src/polyfills/buffer/buffer-polyfill.browser.ts",
-    "./dist/polyfills/buffer/buffer-polyfill.node.js": "./dist/polyfills/buffer/buffer-polyfill.browser.js",
-    "./src/lib/wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm-browser.ts",
-    "./dist/lib/wasm/load-wasm-node.js": "./dist/lib/wasm/load-wasm-browser.js"
+    "./dist/polyfills/buffer/buffer-polyfill.node.js": "./dist/polyfills/buffer/buffer-polyfill.browser.js"
   },
   "comments": [
     "base64-js and ieee754 are used by buffer polyfill"
   ],
   "dependencies": {
-    "@loaders.gl/arrow": "4.2.0-alpha.2",
-    "@loaders.gl/bson": "4.2.0-alpha.2",
-    "@loaders.gl/compression": "4.2.0-alpha.2",
-    "@loaders.gl/gis": "4.2.0-alpha.2",
-    "@loaders.gl/loader-utils": "4.2.0-alpha.2",
-    "@loaders.gl/schema": "4.2.0-alpha.2",
-    "@loaders.gl/wkt": "4.2.0-alpha.2",
+    "@loaders.gl/arrow": "4.2.0-alpha.4",
+    "@loaders.gl/bson": "4.2.0-alpha.4",
+    "@loaders.gl/compression": "4.2.0-alpha.4",
+    "@loaders.gl/gis": "4.2.0-alpha.4",
+    "@loaders.gl/loader-utils": "4.2.0-alpha.4",
+    "@loaders.gl/schema": "4.2.0-alpha.4",
+    "@loaders.gl/wkt": "4.2.0-alpha.4",
     "async-mutex": "^0.2.2",
     "base64-js": "^1.3.1",
     "brotli": "^1.3.2",
@@ -76,7 +74,7 @@
     "lz4js": "^0.2.0",
     "node-int64": "^0.4.0",
     "object-stream": "0.0.1",
-    "parquet-wasm": "^0.3.1",
+    "parquet-wasm": "^0.6.0-beta.1",
     "snappyjs": "^0.6.0",
     "thrift": "^0.19.0",
     "util": "^0.12.5",
@@ -93,5 +91,5 @@
     "@types/varint": "^5.0.0",
     "apache-arrow": "^15.0.0"
   },
-  "gitHead": "d66a6a4626ea84c5f2cad5fa5cf7ebb6943c57c8"
+  "gitHead": "6c52dee5c3f005648a394cc4aee7fc37005c8e83"
 }

package/src/index.ts

@@ -15,12 +15,13 @@ export {
   ParquetColumnarLoader
 } from './parquet-loader';
 
-// import type {ParquetWasmLoaderOptions} from './lib/wasm/parse-parquet-wasm';
-// import {parseParquetWasm} from './lib/wasm/parse-parquet-wasm';
-// import {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';
-
 export {ParquetWriter as _ParquetWriter} from './parquet-writer';
-// export {ParquetWasmWriter} from './parquet-wasm-writer';
+
+// EXPERIMENTAL - expose Parquet WASM loaders/writer
+
+export type {ParquetWasmLoaderOptions} from './parquet-wasm-loader';
+export {ParquetWasmLoader, ParquetWasmWorkerLoader} from './parquet-wasm-loader';
+export {ParquetWasmWriter} from './parquet-wasm-writer';
 
 // EXPERIMENTAL - expose the internal parquetjs API
 

package/src/{constants.ts → lib/constants.ts}

@@ -2,7 +2,11 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+export const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+export const PARQUET_WASM_URL = 'https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm';
+
 /**
  * Parquet File Magic String
  */

package/src/lib/wasm/encode-parquet-wasm.ts

@@ -2,39 +2,67 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {WriterOptions} from '@loaders.gl/loader-utils';
 import type {ArrowTable} from '@loaders.gl/arrow';
 
 import * as arrow from 'apache-arrow';
 import {loadWasm} from './load-wasm';
 
-export type ParquetWriterOptions = WriterOptions & {
-  parquet?: {
-    wasmUrl?: string;
-  };
-};
+import type {ParquetWriterOptions} from '../../parquet-wasm-writer';
 
 /**
  * Encode Arrow arrow.Table to Parquet buffer
  */
 export async function encode(
   table: ArrowTable,
-  options?: ParquetWriterOptions
+  options: ParquetWriterOptions
 ): Promise<ArrayBuffer> {
-  const wasmUrl = options?.parquet?.wasmUrl;
+  const wasmUrl = options.parquet?.wasmUrl!;
   const wasm = await loadWasm(wasmUrl);
 
+  // Serialize the table to the IPC format.
   const arrowTable: arrow.Table = table.data;
+  const ipcStream = arrow.tableToIPC(arrowTable);
 
-  // Serialize a table to the IPC format.
-  const writer = arrow.RecordBatchStreamWriter.writeAll(arrowTable);
-  const arrowIPCBytes = writer.toUint8Array(true);
-
-  // TODO: provide options for how to write table.
-  const writerProperties = new wasm.WriterPropertiesBuilder().build();
-  const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);
-  return parquetBytes.buffer.slice(
-    parquetBytes.byteOffset,
-    parquetBytes.byteLength + parquetBytes.byteOffset
-  );
+  // Pass the IPC stream to the Parquet writer.
+  const wasmTable = wasm.Table.fromIPCStream(ipcStream);
+  const wasmProperties = new wasm.WriterPropertiesBuilder().build();
+  try {
+    const parquetBytes = wasm.writeParquet(wasmTable, wasmProperties);
+    // const parquetBytes = wasm.writeParquet(wasmTable, wasmProperties);
+    return parquetBytes.buffer.slice(
+      parquetBytes.byteOffset,
+      parquetBytes.byteLength + parquetBytes.byteOffset
+    );
+  } finally {
+    // wasmTable.free();
+    // wasmProperties.free();
+  }
 }
+
+// type WriteOptions = {
+//   compression?: number;
+//   dictionaryEnabled?: boolean;
+//   encoding?: number;
+//   maxRowGroupSize?: number;
+//   maxStatisticsSize?: number;
+//   statisticsEnabled?: boolean;
+//   writeBatchSize?: number;
+//   dataPageSizeLimit?: number;
+//   dictionaryPageSizeLimit?: number;
+// };
+
+// columnCompression: Record<string, number>;
+// columnDictionaryEnabled: Record<string, boolean>;
+// columnEncoding: Record<string, number>;
+// columnMaxStatisticsSize
+// compression:Record<string, number>;
+// setCreatedBy
+// setDataPageSizeLimit
+// setDictionaryEnabled
+// setDictionaryPageSizeLimit
+// setEncoding
+// setMaxRowGroupSize
+// setMaxStatisticsSize
+// setStatisticsEnabled
+// setWriteBatchSize
+// setWriterVersion

package/src/lib/wasm/load-wasm.ts

@@ -2,4 +2,21 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-export {loadWasm} from './load-wasm-node';
+// eslint-disable-next-line import/default
+import initWasm from 'parquet-wasm';
+import * as parquetWasm from 'parquet-wasm';
+import {PARQUET_WASM_URL} from '../constants';
+
+let initializePromise: any;
+
+export async function loadWasm(wasmUrl: string = PARQUET_WASM_URL) {
+  if (!initializePromise && typeof initWasm === 'function') {
+    if (!wasmUrl) {
+      throw new Error('ParquetLoader: No wasmUrl provided');
+    }
+    // @ts-ignore
+    initializePromise = initWasm(wasmUrl);
+  }
+  await initializePromise;
+  return parquetWasm;
+}

package/src/lib/wasm/parse-parquet-wasm.ts

@@ -3,43 +3,31 @@
 // Copyright (c) vis.gl contributors
 
 // eslint-disable
-import type {LoaderOptions} from '@loaders.gl/loader-utils';
 import type {ArrowTable} from '@loaders.gl/arrow';
 import {serializeArrowSchema} from '@loaders.gl/arrow';
-import * as arrow from 'apache-arrow';
+import type {ParquetWasmLoaderOptions} from '../../parquet-wasm-loader';
 import {loadWasm} from './load-wasm';
-
-export type ParquetWasmLoaderOptions = LoaderOptions & {
-  parquet?: {
-    type?: 'arrow-table';
-    wasmUrl?: string;
-  };
-};
+import * as arrow from 'apache-arrow';
 
 export async function parseParquetWasm(
   arrayBuffer: ArrayBuffer,
-  options?: ParquetWasmLoaderOptions
+  options: ParquetWasmLoaderOptions
 ): Promise<ArrowTable> {
+  const arr = new Uint8Array(arrayBuffer);
+
   const wasmUrl = options?.parquet?.wasmUrl;
   const wasm = await loadWasm(wasmUrl);
+  const wasmTable = wasm.readParquet(arr);
+  try {
+    const ipcStream = wasmTable.intoIPCStream();
+    const arrowTable = arrow.tableFromIPC(ipcStream);
 
-  const arr = new Uint8Array(arrayBuffer);
-  const arrowIPCUint8Arr = wasm.readParquet(arr);
-  const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(
-    arrowIPCUint8Arr.byteOffset,
-    arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset
-  );
-
-  const reader = arrow.RecordBatchStreamReader.from(arrowIPCBuffer);
-  const recordBatches: arrow.RecordBatch[] = [];
-  for (const recordBatch of reader) {
-    recordBatches.push(recordBatch);
+    return {
+      shape: 'arrow-table',
+      schema: serializeArrowSchema(arrowTable.schema),
+      data: arrowTable
+    };
+  } finally {
+    // wasmTable.free();
   }
-  const arrowTable = new arrow.Table(recordBatches);
-
-  return {
-    shape: 'arrow-table',
-    schema: serializeArrowSchema(arrowTable.schema),
-    data: arrowTable
-  };
 }

package/src/parquet-wasm-loader.ts

@@ -6,10 +6,7 @@ import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-u
 import type {ArrowTable} from '@loaders.gl/arrow';
 
 import {parseParquetWasm} from './lib/wasm/parse-parquet-wasm';
-
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+import {VERSION, PARQUET_WASM_URL} from './lib/constants';
 
 /** Parquet WASM loader options */
 export type ParquetWasmLoaderOptions = LoaderOptions & {
@@ -34,7 +31,7 @@ export const ParquetWasmWorkerLoader: Loader<ArrowTable, never, ParquetWasmLoade
   options: {
     parquet: {
       type: 'arrow-table',
-      wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
+      wasmUrl: PARQUET_WASM_URL
     }
   }
 };
@@ -42,5 +39,8 @@ export const ParquetWasmWorkerLoader: Loader<ArrowTable, never, ParquetWasmLoade
 /** Parquet WASM table loader */
 export const ParquetWasmLoader: LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions> = {
   ...ParquetWasmWorkerLoader,
-  parse: parseParquetWasm
+  parse(arrayBuffer: ArrayBuffer, options?: ParquetWasmLoaderOptions) {
+    options = {parquet: {...ParquetWasmLoader.options.parquet, ...options?.parquet}, ...options};
+    return parseParquetWasm(arrayBuffer, options);
+  }
 };

package/src/parquet-wasm-writer.ts

@@ -4,11 +4,16 @@
 
 import type {WriterWithEncoder} from '@loaders.gl/loader-utils';
 import type {ArrowTable} from '@loaders.gl/arrow';
-import {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';
+import {encode} from './lib/wasm/encode-parquet-wasm';
+import type {WriterOptions} from '@loaders.gl/loader-utils';
 
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+import {VERSION, PARQUET_WASM_URL} from './lib/constants';
+
+export type ParquetWriterOptions = WriterOptions & {
+  parquet?: {
+    wasmUrl?: string;
+  };
+};
 
 /** Parquet WASM writer */
 export const ParquetWasmWriter: WriterWithEncoder<ArrowTable, never, ParquetWriterOptions> = {
@@ -21,8 +26,11 @@ export const ParquetWasmWriter: WriterWithEncoder<ArrowTable, never, ParquetWrit
   binary: true,
   options: {
     parquet: {
-      wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
+      wasmUrl: PARQUET_WASM_URL
     }
   },
-  encode
+  encode(arrowTable: ArrowTable, options?: ParquetWriterOptions) {
+    options = {parquet: {...ParquetWasmWriter.options.parquet, ...options?.parquet}, ...options};
+    return encode(arrowTable, options);
+  }
 };

package/src/parquetjs/parser/decoders.ts

@@ -19,7 +19,7 @@ import {
   Type
 } from '../parquet-thrift';
 import {decompress} from '../compression';
-import {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';
+import {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../lib/constants';
 import {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';
 
 /**

package/src/parquetjs/parser/parquet-reader.ts

@@ -5,7 +5,7 @@ import {ParquetSchema} from '../schema/schema';
 import {decodeSchema} from './decoders';
 import {materializeRows} from '../schema/shred';
 
-import {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';
+import {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../lib/constants';
 import {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';
 import {
   ParquetRowGroup,

package/dist/constants.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../src/constants.ts"],"names":[],"mappings":"AAKA;;GAEG;AACH,eAAO,MAAM,aAAa,SAAS,CAAC;AACpC,eAAO,MAAM,uBAAuB,SAAS,CAAC;AAE9C;;GAEG;AACH,eAAO,MAAM,eAAe,IAAI,CAAC;AAEjC;;GAEG;AACH,eAAO,MAAM,kBAAkB,UAAU,CAAC;AAC1C,eAAO,MAAM,sBAAsB,QAAQ,CAAC"}

package/dist/constants.js

@@ -1,6 +0,0 @@
-export const PARQUET_MAGIC = 'PAR1';
-export const PARQUET_MAGIC_ENCRYPTED = 'PARE';
-export const PARQUET_VERSION = 1;
-export const PARQUET_RDLVL_TYPE = 'INT32';
-export const PARQUET_RDLVL_ENCODING = 'RLE';
-//# sourceMappingURL=constants.js.map

package/dist/constants.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"constants.js","names":["PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING"],"sources":["../src/constants.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/**\n * Parquet File Magic String\n */\nexport const PARQUET_MAGIC = 'PAR1';\nexport const PARQUET_MAGIC_ENCRYPTED = 'PARE';\n\n/**\n * Parquet File Format Version\n */\nexport const PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nexport const PARQUET_RDLVL_TYPE = 'INT32';\nexport const PARQUET_RDLVL_ENCODING = 'RLE';\n"],"mappings":"AAQA,OAAO,MAAMA,aAAa,GAAG,MAAM;AACnC,OAAO,MAAMC,uBAAuB,GAAG,MAAM;AAK7C,OAAO,MAAMC,eAAe,GAAG,CAAC;AAKhC,OAAO,MAAMC,kBAAkB,GAAG,OAAO;AACzC,OAAO,MAAMC,sBAAsB,GAAG,KAAK"}

package/dist/lib/wasm/load-wasm-browser.d.ts

@@ -1,3 +0,0 @@
-import * as wasmEsm from 'parquet-wasm/esm2/arrow1';
-export declare function loadWasm(wasmUrl?: string): Promise<typeof wasmEsm>;
-//# sourceMappingURL=load-wasm-browser.d.ts.map

package/dist/lib/wasm/load-wasm-browser.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"load-wasm-browser.d.ts","sourceRoot":"","sources":["../../../src/lib/wasm/load-wasm-browser.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,OAAO,MAAM,0BAA0B,CAAC;AAIpD,wBAAsB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,2BAU9C"}

package/dist/lib/wasm/load-wasm-browser.js

@@ -1,11 +0,0 @@
-import * as wasmEsm from 'parquet-wasm/esm2/arrow1';
-let cached = null;
-export async function loadWasm(wasmUrl) {
-  if (cached !== null) {
-    return cached;
-  }
-  await wasmEsm.default(wasmUrl);
-  cached = wasmEsm;
-  return wasmEsm;
-}
-//# sourceMappingURL=load-wasm-browser.js.map

package/dist/lib/wasm/load-wasm-browser.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"load-wasm-browser.js","names":["wasmEsm","cached","loadWasm","wasmUrl","default"],"sources":["../../../src/lib/wasm/load-wasm-browser.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport * as wasmEsm from 'parquet-wasm/esm2/arrow1';\n\nlet cached: typeof wasmEsm | null = null;\n\nexport async function loadWasm(wasmUrl?: string) {\n  if (cached !== null) {\n    return cached;\n  }\n\n  // For ESM bundles, need to await the default export, which loads the WASM\n  await wasmEsm.default(wasmUrl);\n  cached = wasmEsm;\n\n  return wasmEsm;\n}\n"],"mappings":"AAIA,OAAO,KAAKA,OAAO,MAAM,0BAA0B;AAEnD,IAAIC,MAA6B,GAAG,IAAI;AAExC,OAAO,eAAeC,QAAQA,CAACC,OAAgB,EAAE;EAC/C,IAAIF,MAAM,KAAK,IAAI,EAAE;IACnB,OAAOA,MAAM;EACf;EAGA,MAAMD,OAAO,CAACI,OAAO,CAACD,OAAO,CAAC;EAC9BF,MAAM,GAAGD,OAAO;EAEhB,OAAOA,OAAO;AAChB"}