@loaders.gl/parquet 4.0.0-beta.2 → 4.0.0-beta.3

package/dist/esm/parquetjs/parquet-thrift/index.js

@@ -1,46 +0,0 @@
-export { Buffer } from '../../buffer-polyfill/install-buffer-polyfill';
-export { TBufferedTransport, TCompactProtocol, TFramedTransport } from 'thrift';
-export * from './Type';
-export * from './ConvertedType';
-export * from './FieldRepetitionType';
-export * from './Encoding';
-export * from './CompressionCodec';
-export * from './PageType';
-export * from './BoundaryOrder';
-export * from './Statistics';
-export * from './StringType';
-export * from './UUIDType';
-export * from './MapType';
-export * from './ListType';
-export * from './EnumType';
-export * from './DateType';
-export * from './NullType';
-export * from './DecimalType';
-export * from './MilliSeconds';
-export * from './MicroSeconds';
-export * from './TimestampType';
-export * from './TimeType';
-export * from './IntType';
-export * from './JsonType';
-export * from './BsonType';
-export * from './SchemaElement';
-export * from './DataPageHeader';
-export * from './IndexPageHeader';
-export * from './DictionaryPageHeader';
-export * from './DataPageHeaderV2';
-export * from './PageHeader';
-export * from './KeyValue';
-export * from './SortingColumn';
-export * from './PageEncodingStats';
-export * from './ColumnMetaData';
-export * from './ColumnChunk';
-export * from './RowGroup';
-export * from './TypeDefinedOrder';
-export * from './PageLocation';
-export * from './OffsetIndex';
-export * from './ColumnIndex';
-export * from './FileMetaData';
-export * from './TimeUnit';
-export * from './LogicalType';
-export * from './ColumnOrder';
-//# sourceMappingURL=index.js.map

package/dist/esm/parquetjs/parquet-thrift/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","names":["Buffer","TBufferedTransport","TCompactProtocol","TFramedTransport"],"sources":["../../../../src/parquetjs/parquet-thrift/index.ts"],"sourcesContent":["/* tslint:disable */\n/* eslint-disable */\n/*\n * Autogenerated by @creditkarma/thrift-typescript v3.7.2\n * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING\n */\nexport {Buffer} from '../../buffer-polyfill/install-buffer-polyfill';\n\nexport {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';\n\nexport * from './Type';\nexport * from './ConvertedType';\nexport * from './FieldRepetitionType';\nexport * from './Encoding';\nexport * from './CompressionCodec';\nexport * from './PageType';\nexport * from './BoundaryOrder';\nexport * from './Statistics';\nexport * from './StringType';\nexport * from './UUIDType';\nexport * from './MapType';\nexport * from './ListType';\nexport * from './EnumType';\nexport * from './DateType';\nexport * from './NullType';\nexport * from './DecimalType';\nexport * from './MilliSeconds';\nexport * from './MicroSeconds';\nexport * from './TimestampType';\nexport * from './TimeType';\nexport * from './IntType';\nexport * from './JsonType';\nexport * from './BsonType';\nexport * from './SchemaElement';\nexport * from './DataPageHeader';\nexport * from './IndexPageHeader';\nexport * from './DictionaryPageHeader';\nexport * from './DataPageHeaderV2';\nexport * from './PageHeader';\nexport * from './KeyValue';\nexport * from './SortingColumn';\nexport * from './PageEncodingStats';\nexport * from './ColumnMetaData';\nexport * from './ColumnChunk';\nexport * from './RowGroup';\nexport * from './TypeDefinedOrder';\nexport * from './PageLocation';\nexport * from './OffsetIndex';\nexport * from './ColumnIndex';\nexport * from './FileMetaData';\nexport * from './TimeUnit';\nexport * from './LogicalType';\nexport * from './ColumnOrder';\n"],"mappings":"AAMA,SAAQA,MAAM,QAAO,+CAA+C;AAEpE,SAAQC,kBAAkB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAO,QAAQ;AAE7E,cAAc,QAAQ;AACtB,cAAc,iBAAiB;AAC/B,cAAc,uBAAuB;AACrC,cAAc,YAAY;AAC1B,cAAc,oBAAoB;AAClC,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,cAAc;AAC5B,cAAc,cAAc;AAC5B,cAAc,YAAY;AAC1B,cAAc,WAAW;AACzB,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,eAAe;AAC7B,cAAc,gBAAgB;AAC9B,cAAc,gBAAgB;AAC9B,cAAc,iBAAiB;AAC/B,cAAc,YAAY;AAC1B,cAAc,WAAW;AACzB,cAAc,YAAY;AAC1B,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,kBAAkB;AAChC,cAAc,mBAAmB;AACjC,cAAc,wBAAwB;AACtC,cAAc,oBAAoB;AAClC,cAAc,cAAc;AAC5B,cAAc,YAAY;AAC1B,cAAc,iBAAiB;AAC/B,cAAc,qBAAqB;AACnC,cAAc,kBAAkB;AAChC,cAAc,eAAe;AAC7B,cAAc,YAAY;AAC1B,cAAc,oBAAoB;AAClC,cAAc,gBAAgB;AAC9B,cAAc,eAAe;AAC7B,cAAc,eAAe;AAC7B,cAAc,gBAAgB;AAC9B,cAAc,YAAY;AAC1B,cAAc,eAAe;AAC7B,cAAc,eAAe"}

package/dist/esm/parquetjs/parser/decoders.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"decoders.js","names":["PARQUET_CODECS","ConvertedType","Encoding","FieldRepetitionType","PageType","Type","decompress","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING","decodePageHeader","getThriftEnum","getBitWidth","decodeDataPages","buffer","context","cursor","offset","size","length","data","rlevels","dlevels","values","pageHeaders","count","dictionary","numValues","Number","page","decodePage","map","value","index","push","undefined","pageHeader","pageType","type","decodeDataPage","decodeDataPageV2","decodeDictionaryPage","Error","concat","decodeSchema","schemaElements","len","schema","next","i","schemaElement","repetitionType","repetition_type","optional","repeated","num_children","res","name","fields","logicalType","converted_type","typeLength","type_length","presision","precision","scale","decodeValues","encoding","opts","header","_header$data_page_hea","_header$data_page_hea2","_header$data_page_hea3","_header$data_page_hea4","cursorEnd","compressed_page_size","valueCount","data_page_header","num_values","dataCursor","compression","valuesBuf","slice","uncompressed_page_size","rLevelEncoding","repetition_level_encoding","rLevels","Array","column","rLevelMax","bitWidth","disableEnvelope","fill","dLevelEncoding","definition_level_encoding","dLevels","dLevelMax","valueCountNonNull","dlvl","valueEncoding","decodeOptions","primitiveType","_header$data_page_hea5","_header$data_page_hea6","_header$data_page_hea7","_header$data_page_hea8","data_page_header_v2","num_nulls","valuesBufCursor","is_compressed","_pageHeader$dictionar","dictCursor","dictionary_page_header","decodedDictionaryValues","preserveBinary","d","toString","ArrayBuffer","isView","Buffer","isBuffer","byteOffset","byteLength"],"sources":["../../../../src/parquetjs/parser/decoders.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {\n  ParquetCodec,\n  ParquetColumnChunk,\n  ParquetReaderContext,\n  ParquetPageData,\n  ParquetType,\n  PrimitiveType,\n  SchemaDefinition\n} from '../schema/declare';\nimport {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';\nimport {\n  ConvertedType,\n  Encoding,\n  FieldRepetitionType,\n  PageHeader,\n  PageType,\n  SchemaElement,\n  Type\n} from '../parquet-thrift';\nimport {decompress} from '../compression';\nimport {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../constants';\nimport {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';\n\n/**\n * Decode data pages\n * @param buffer - input data\n * @param column - parquet column\n * @param compression - compression type\n * @returns parquet data page data\n */\nexport async function decodeDataPages(\n  buffer: Buffer,\n  context: ParquetReaderContext\n): Promise<ParquetColumnChunk> {\n  const cursor: CursorBuffer = {\n    buffer,\n    offset: 0,\n    size: buffer.length\n  };\n\n  const data: ParquetColumnChunk = {\n    rlevels: [],\n    dlevels: [],\n    values: [],\n    pageHeaders: [],\n    count: 0\n  };\n\n  let dictionary = context.dictionary || [];\n\n  while (\n    // @ts-ignore size can be undefined\n    cursor.offset < cursor.size &&\n    (!context.numValues || data.dlevels.length < Number(context.numValues))\n  ) {\n    // Looks like we have to decode these in sequence due to cursor updates?\n    const page = await decodePage(cursor, context);\n\n    if (page.dictionary) {\n      dictionary = page.dictionary;\n      // eslint-disable-next-line no-continue\n      continue;\n    }\n\n    if (dictionary.length) {\n      // eslint-disable-next-line no-loop-func\n      page.values = page.values.map((value) => dictionary[value]);\n    }\n\n    for (let index = 0; index < page.rlevels.length; index++) {\n      data.rlevels.push(page.rlevels[index]);\n      data.dlevels.push(page.dlevels[index]);\n      const value = page.values[index];\n\n      if (value !== undefined) {\n        data.values.push(value);\n      }\n    }\n\n    data.count += page.count;\n    data.pageHeaders.push(page.pageHeader);\n  }\n\n  return data;\n}\n\n/**\n * Decode parquet page based on page type\n * @param cursor\n * @param context\n */\nexport async function decodePage(\n  cursor: CursorBuffer,\n  context: ParquetReaderContext\n): Promise<ParquetPageData> {\n  let page;\n\n  const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);\n  cursor.offset += length;\n\n  const pageType = getThriftEnum(PageType, pageHeader.type);\n\n  switch (pageType) {\n    case 'DATA_PAGE':\n      page = await decodeDataPage(cursor, pageHeader, context);\n      break;\n    case 'DATA_PAGE_V2':\n      page = await decodeDataPageV2(cursor, pageHeader, context);\n      break;\n    case 'DICTIONARY_PAGE':\n      page = {\n        dictionary: await decodeDictionaryPage(cursor, pageHeader, context),\n        pageHeader\n      };\n      break;\n    default:\n      throw new Error(`invalid page type: ${pageType}`);\n  }\n\n  return page;\n}\n\n/**\n * Decode parquet schema\n * @param schemaElements input schema elements data\n * @param offset offset to read from\n * @param len length of data\n * @returns result.offset\n *   result.next - offset at the end of function\n *   result.schema - schema read from the input data\n * @todo output offset is the same as input - possibly excess output field\n */\nexport function decodeSchema(\n  schemaElements: SchemaElement[],\n  offset: number,\n  len: number\n): {\n  offset: number;\n  next: number;\n  schema: SchemaDefinition;\n} {\n  const schema: SchemaDefinition = {};\n  let next = offset;\n  for (let i = 0; i < len; i++) {\n    const schemaElement = schemaElements[next];\n\n    const repetitionType =\n      next > 0 ? getThriftEnum(FieldRepetitionType, schemaElement.repetition_type!) : 'ROOT';\n\n    let optional = false;\n    let repeated = false;\n    switch (repetitionType) {\n      case 'REQUIRED':\n        break;\n      case 'OPTIONAL':\n        optional = true;\n        break;\n      case 'REPEATED':\n        repeated = true;\n        break;\n      default:\n        throw new Error('parquet: unknown repetition type');\n    }\n\n    if (schemaElement.num_children! > 0) {\n      const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children!);\n      next = res.next;\n      schema[schemaElement.name] = {\n        // type: undefined,\n        optional,\n        repeated,\n        fields: res.schema\n      };\n    } else {\n      const type = getThriftEnum(Type, schemaElement.type!);\n      let logicalType = type;\n\n      if (schemaElement.converted_type) {\n        logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);\n      }\n\n      switch (logicalType) {\n        case 'DECIMAL':\n          logicalType = `${logicalType}_${type}` as ParquetType;\n          break;\n        default:\n      }\n\n      schema[schemaElement.name] = {\n        type: logicalType as ParquetType,\n        typeLength: schemaElement.type_length,\n        presision: schemaElement.precision,\n        scale: schemaElement.scale,\n        optional,\n        repeated\n      };\n      next++;\n    }\n  }\n  return {schema, offset, next};\n}\n\n/**\n * Decode a consecutive array of data using one of the parquet encodings\n */\nfunction decodeValues(\n  type: PrimitiveType,\n  encoding: ParquetCodec,\n  cursor: CursorBuffer,\n  count: number,\n  opts: ParquetCodecOptions\n): any[] {\n  if (!(encoding in PARQUET_CODECS)) {\n    throw new Error(`invalid encoding: ${encoding}`);\n  }\n  return PARQUET_CODECS[encoding].decodeValues(type, cursor, count, opts);\n}\n\n/**\n * Do decoding of parquet dataPage from column chunk\n * @param cursor\n * @param header\n * @param options\n */\nasync function decodeDataPage(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  context: ParquetReaderContext\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n  const valueCount = header.data_page_header?.num_values;\n\n  /* uncompress page */\n  let dataCursor = cursor;\n\n  if (context.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      context.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n    dataCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n    cursor.offset = cursorEnd;\n  }\n\n  /* read repetition levels */\n  const rLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.repetition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n\n  if (context.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.rLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  const dLevelEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header?.definition_level_encoding!\n  ) as ParquetCodec;\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (context.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.dLevelMax),\n      disableEnvelope: false\n      // column: opts.column\n    });\n  } else {\n    dLevels.fill(0);\n  }\n  let valueCountNonNull = 0;\n  for (const dlvl of dLevels) {\n    if (dlvl === context.column.dLevelMax) {\n      valueCountNonNull++;\n    }\n  }\n\n  /* read values */\n  const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding!) as ParquetCodec;\n  const decodeOptions: ParquetCodecOptions = {\n    typeLength: context.column.typeLength,\n    bitWidth: context.column.typeLength\n  };\n\n  const values = decodeValues(\n    context.column.primitiveType!,\n    valueEncoding,\n    dataCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of parquet dataPage in version 2 from column chunk\n * @param cursor\n * @param header\n * @param opts\n * @returns\n */\nasync function decodeDataPageV2(\n  cursor: CursorBuffer,\n  header: PageHeader,\n  context: ParquetReaderContext\n): Promise<ParquetPageData> {\n  const cursorEnd = cursor.offset + header.compressed_page_size;\n\n  const valueCount = header.data_page_header_v2?.num_values;\n  // @ts-ignore\n  const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;\n  const valueEncoding = getThriftEnum(\n    Encoding,\n    header.data_page_header_v2?.encoding!\n  ) as ParquetCodec;\n\n  /* read repetition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let rLevels = new Array(valueCount);\n  if (context.column.rLevelMax > 0) {\n    rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.rLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    rLevels.fill(0);\n  }\n\n  /* read definition levels */\n  // tslint:disable-next-line:prefer-array-literal\n  let dLevels = new Array(valueCount);\n  if (context.column.dLevelMax > 0) {\n    dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount!, {\n      bitWidth: getBitWidth(context.column.dLevelMax),\n      disableEnvelope: true\n    });\n  } else {\n    dLevels.fill(0);\n  }\n\n  /* read values */\n  let valuesBufCursor = cursor;\n\n  if (header.data_page_header_v2?.is_compressed) {\n    const valuesBuf = await decompress(\n      context.compression,\n      cursor.buffer.slice(cursor.offset, cursorEnd),\n      header.uncompressed_page_size\n    );\n\n    valuesBufCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const decodeOptions = {\n    typeLength: context.column.typeLength,\n    bitWidth: context.column.typeLength\n  };\n\n  const values = decodeValues(\n    context.column.primitiveType!,\n    valueEncoding,\n    valuesBufCursor,\n    valueCountNonNull,\n    decodeOptions\n  );\n\n  return {\n    dlevels: dLevels,\n    rlevels: rLevels,\n    values,\n    count: valueCount!,\n    pageHeader: header\n  };\n}\n\n/**\n * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.\n * @param cursor\n * @param pageHeader\n * @param context\n */\nasync function decodeDictionaryPage(\n  cursor: CursorBuffer,\n  pageHeader: PageHeader,\n  context: ParquetReaderContext\n): Promise<(string | ArrayBuffer)[]> {\n  const cursorEnd = cursor.offset + pageHeader.compressed_page_size;\n\n  let dictCursor = {\n    offset: 0,\n    buffer: cursor.buffer.slice(cursor.offset, cursorEnd),\n    size: cursorEnd - cursor.offset\n  };\n\n  cursor.offset = cursorEnd;\n\n  if (context.compression !== 'UNCOMPRESSED') {\n    const valuesBuf = await decompress(\n      context.compression,\n      dictCursor.buffer.slice(dictCursor.offset, cursorEnd),\n      pageHeader.uncompressed_page_size\n    );\n\n    dictCursor = {\n      buffer: valuesBuf,\n      offset: 0,\n      size: valuesBuf.length\n    };\n\n    cursor.offset = cursorEnd;\n  }\n\n  const numValues = pageHeader?.dictionary_page_header?.num_values || 0;\n\n  const decodedDictionaryValues = decodeValues(\n    context.column.primitiveType!,\n    context.column.encoding!,\n    dictCursor,\n    numValues,\n    // TODO - this looks wrong?\n    context as ParquetCodecOptions\n  );\n\n  // Makes it look a little easier\n  let values: any[];\n  if (context?.preserveBinary) {\n    values = decodedDictionaryValues.map((d) => preserveBinary(d));\n  } else {\n    values = decodedDictionaryValues.map((d) => d.toString());\n  }\n  return values;\n}\n\nfunction preserveBinary(d: any): ArrayBuffer | ArrayBufferView | string {\n  if (ArrayBuffer.isView(d)) {\n    return d;\n  }\n  // Convert to ArrayBuffer\n  if (Buffer.isBuffer(d)) {\n    return d.buffer.slice(d.byteOffset, d.byteLength);\n  }\n  return d.toString();\n}\n"],"mappings":"AAUA,SAA2CA,cAAc,QAAO,WAAW;AAC3E,SACEC,aAAa,EACbC,QAAQ,EACRC,mBAAmB,EAEnBC,QAAQ,EAERC,IAAI,QACC,mBAAmB;AAC1B,SAAQC,UAAU,QAAO,gBAAgB;AACzC,SAAQC,kBAAkB,EAAEC,sBAAsB,QAAO,iBAAiB;AAC1E,SAAQC,gBAAgB,EAAEC,aAAa,EAAEC,WAAW,QAAO,qBAAqB;AAShF,OAAO,eAAeC,eAAeA,CACnCC,MAAc,EACdC,OAA6B,EACA;EAC7B,MAAMC,MAAoB,GAAG;IAC3BF,MAAM;IACNG,MAAM,EAAE,CAAC;IACTC,IAAI,EAAEJ,MAAM,CAACK;EACf,CAAC;EAED,MAAMC,IAAwB,GAAG;IAC/BC,OAAO,EAAE,EAAE;IACXC,OAAO,EAAE,EAAE;IACXC,MAAM,EAAE,EAAE;IACVC,WAAW,EAAE,EAAE;IACfC,KAAK,EAAE;EACT,CAAC;EAED,IAAIC,UAAU,GAAGX,OAAO,CAACW,UAAU,IAAI,EAAE;EAEzC,OAEEV,MAAM,CAACC,MAAM,GAAGD,MAAM,CAACE,IAAI,KAC1B,CAACH,OAAO,CAACY,SAAS,IAAIP,IAAI,CAACE,OAAO,CAACH,MAAM,GAAGS,MAAM,CAACb,OAAO,CAACY,SAAS,CAAC,CAAC,EACvE;IAEA,MAAME,IAAI,GAAG,MAAMC,UAAU,CAACd,MAAM,EAAED,OAAO,CAAC;IAE9C,IAAIc,IAAI,CAACH,UAAU,EAAE;MACnBA,UAAU,GAAGG,IAAI,CAACH,UAAU;MAE5B;IACF;IAEA,IAAIA,UAAU,CAACP,MAAM,EAAE;MAErBU,IAAI,CAACN,MAAM,GAAGM,IAAI,CAACN,MAAM,CAACQ,GAAG,CAAEC,KAAK,IAAKN,UAAU,CAACM,KAAK,CAAC,CAAC;IAC7D;IAEA,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGJ,IAAI,CAACR,OAAO,CAACF,MAAM,EAAEc,KAAK,EAAE,EAAE;MACxDb,IAAI,CAACC,OAAO,CAACa,IAAI,CAACL,IAAI,CAACR,OAAO,CAACY,KAAK,CAAC,CAAC;MACtCb,IAAI,CAACE,OAAO,CAACY,IAAI,CAACL,IAAI,CAACP,OAAO,CAACW,KAAK,CAAC,CAAC;MACtC,MAAMD,KAAK,GAAGH,IAAI,CAACN,MAAM,CAACU,KAAK,CAAC;MAEhC,IAAID,KAAK,KAAKG,SAAS,EAAE;QACvBf,IAAI,CAACG,MAAM,CAACW,IAAI,CAACF,KAAK,CAAC;MACzB;IACF;IAEAZ,IAAI,CAACK,KAAK,IAAII,IAAI,CAACJ,KAAK;IACxBL,IAAI,CAACI,WAAW,CAACU,IAAI,CAACL,IAAI,CAACO,UAAU,CAAC;EACxC;EAEA,OAAOhB,IAAI;AACb;AAOA,OAAO,eAAeU,UAAUA,CAC9Bd,MAAoB,EACpBD,OAA6B,EACH;EAC1B,IAAIc,IAAI;EAER,MAAM;IAACO,UAAU;IAAEjB;EAAM,CAAC,GAAGT,gBAAgB,CAACM,MAAM,CAACF,MAAM,EAAEE,MAAM,CAACC,MAAM,CAAC;EAC3ED,MAAM,CAACC,MAAM,IAAIE,MAAM;EAEvB,MAAMkB,QAAQ,GAAG1B,aAAa,CAACN,QAAQ,EAAE+B,UAAU,CAACE,IAAI,CAAC;EAEzD,QAAQD,QAAQ;IACd,KAAK,WAAW;MACdR,IAAI,GAAG,MAAMU,cAAc,CAACvB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MACxD;IACF,KAAK,cAAc;MACjBc,IAAI,GAAG,MAAMW,gBAAgB,CAACxB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;MAC1D;IACF,KAAK,iBAAiB;MACpBc,IAAI,GAAG;QACLH,UAAU,EAAE,MAAMe,oBAAoB,CAACzB,MAAM,EAAEoB,UAAU,EAAErB,OAAO,CAAC;QACnEqB;MACF,CAAC;MACD;IACF;MACE,MAAM,IAAIM,KAAK,uBAAAC,MAAA,CAAuBN,QAAQ,CAAE,CAAC;EACrD;EAEA,OAAOR,IAAI;AACb;AAYA,OAAO,SAASe,YAAYA,CAC1BC,cAA+B,EAC/B5B,MAAc,EACd6B,GAAW,EAKX;EACA,MAAMC,MAAwB,GAAG,CAAC,CAAC;EACnC,IAAIC,IAAI,GAAG/B,MAAM;EACjB,KAAK,IAAIgC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,GAAG,EAAEG,CAAC,EAAE,EAAE;IAC5B,MAAMC,aAAa,GAAGL,cAAc,CAACG,IAAI,CAAC;IAE1C,MAAMG,cAAc,GAClBH,IAAI,GAAG,CAAC,GAAGrC,aAAa,CAACP,mBAAmB,EAAE8C,aAAa,CAACE,eAAgB,CAAC,GAAG,MAAM;IAExF,IAAIC,QAAQ,GAAG,KAAK;IACpB,IAAIC,QAAQ,GAAG,KAAK;IACpB,QAAQH,cAAc;MACpB,KAAK,UAAU;QACb;MACF,KAAK,UAAU;QACbE,QAAQ,GAAG,IAAI;QACf;MACF,KAAK,UAAU;QACbC,QAAQ,GAAG,IAAI;QACf;MACF;QACE,MAAM,IAAIZ,KAAK,CAAC,kCAAkC,CAAC;IACvD;IAEA,IAAIQ,aAAa,CAACK,YAAY,GAAI,CAAC,EAAE;MACnC,MAAMC,GAAG,GAAGZ,YAAY,CAACC,cAAc,EAAEG,IAAI,GAAG,CAAC,EAAEE,aAAa,CAACK,YAAa,CAAC;MAC/EP,IAAI,GAAGQ,GAAG,CAACR,IAAI;MACfD,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAE3BJ,QAAQ;QACRC,QAAQ;QACRI,MAAM,EAAEF,GAAG,CAACT;MACd,CAAC;IACH,CAAC,MAAM;MACL,MAAMT,IAAI,GAAG3B,aAAa,CAACL,IAAI,EAAE4C,aAAa,CAACZ,IAAK,CAAC;MACrD,IAAIqB,WAAW,GAAGrB,IAAI;MAEtB,IAAIY,aAAa,CAACU,cAAc,EAAE;QAChCD,WAAW,GAAGhD,aAAa,CAACT,aAAa,EAAEgD,aAAa,CAACU,cAAc,CAAC;MAC1E;MAEA,QAAQD,WAAW;QACjB,KAAK,SAAS;UACZA,WAAW,MAAAhB,MAAA,CAAMgB,WAAW,OAAAhB,MAAA,CAAIL,IAAI,CAAiB;UACrD;QACF;MACF;MAEAS,MAAM,CAACG,aAAa,CAACO,IAAI,CAAC,GAAG;QAC3BnB,IAAI,EAAEqB,WAA0B;QAChCE,UAAU,EAAEX,aAAa,CAACY,WAAW;QACrCC,SAAS,EAAEb,aAAa,CAACc,SAAS;QAClCC,KAAK,EAAEf,aAAa,CAACe,KAAK;QAC1BZ,QAAQ;QACRC;MACF,CAAC;MACDN,IAAI,EAAE;IACR;EACF;EACA,OAAO;IAACD,MAAM;IAAE9B,MAAM;IAAE+B;EAAI,CAAC;AAC/B;AAKA,SAASkB,YAAYA,CACnB5B,IAAmB,EACnB6B,QAAsB,EACtBnD,MAAoB,EACpBS,KAAa,EACb2C,IAAyB,EAClB;EACP,IAAI,EAAED,QAAQ,IAAIlE,cAAc,CAAC,EAAE;IACjC,MAAM,IAAIyC,KAAK,sBAAAC,MAAA,CAAsBwB,QAAQ,CAAE,CAAC;EAClD;EACA,OAAOlE,cAAc,CAACkE,QAAQ,CAAC,CAACD,YAAY,CAAC5B,IAAI,EAAEtB,MAAM,EAAES,KAAK,EAAE2C,IAAI,CAAC;AACzE;AAQA,eAAe7B,cAAcA,CAC3BvB,MAAoB,EACpBqD,MAAkB,EAClBtD,OAA6B,EACH;EAAA,IAAAuD,qBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAMC,SAAS,GAAG1D,MAAM,CAACC,MAAM,GAAGoD,MAAM,CAACM,oBAAoB;EAC7D,MAAMC,UAAU,IAAAN,qBAAA,GAAGD,MAAM,CAACQ,gBAAgB,cAAAP,qBAAA,uBAAvBA,qBAAA,CAAyBQ,UAAU;EAGtD,IAAIC,UAAU,GAAG/D,MAAM;EAEvB,IAAID,OAAO,CAACiE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAM1E,UAAU,CAChCQ,OAAO,CAACiE,WAAW,EACnBhE,MAAM,CAACF,MAAM,CAACoE,KAAK,CAAClE,MAAM,CAACC,MAAM,EAAEyD,SAAS,CAAC,EAC7CL,MAAM,CAACc,sBACT,CAAC;IACDJ,UAAU,GAAG;MACXjE,MAAM,EAAEmE,SAAS;MACjBhE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE+D,SAAS,CAAC9D;IAClB,CAAC;IACDH,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAC3B;EAGA,MAAMU,cAAc,GAAGzE,aAAa,CAClCR,QAAQ,GAAAoE,sBAAA,GACRF,MAAM,CAACQ,gBAAgB,cAAAN,sBAAA,uBAAvBA,sBAAA,CAAyBc,yBAC3B,CAAiB;EAEjB,IAAIC,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EAEnC,IAAI7D,OAAO,CAACyE,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGpB,YAAY,CAAC1D,kBAAkB,EAAE4E,cAAc,EAAEL,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAE9E,WAAW,CAACG,OAAO,CAACyE,MAAM,CAACC,SAAS,CAAC;MAC/CE,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,MAAMC,cAAc,GAAGlF,aAAa,CAClCR,QAAQ,GAAAqE,sBAAA,GACRH,MAAM,CAACQ,gBAAgB,cAAAL,sBAAA,uBAAvBA,sBAAA,CAAyBsB,yBAC3B,CAAiB;EAEjB,IAAIC,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI7D,OAAO,CAACyE,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAG7B,YAAY,CAAC1D,kBAAkB,EAAEqF,cAAc,EAAEd,UAAU,EAAEH,UAAU,EAAG;MAClFc,QAAQ,EAAE9E,WAAW,CAACG,OAAO,CAACyE,MAAM,CAACQ,SAAS,CAAC;MAC/CL,eAAe,EAAE;IAEnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EACA,IAAIK,iBAAiB,GAAG,CAAC;EACzB,KAAK,MAAMC,IAAI,IAAIH,OAAO,EAAE;IAC1B,IAAIG,IAAI,KAAKnF,OAAO,CAACyE,MAAM,CAACQ,SAAS,EAAE;MACrCC,iBAAiB,EAAE;IACrB;EACF;EAGA,MAAME,aAAa,GAAGxF,aAAa,CAACR,QAAQ,GAAAsE,sBAAA,GAAEJ,MAAM,CAACQ,gBAAgB,cAAAJ,sBAAA,uBAAvBA,sBAAA,CAAyBN,QAAS,CAAiB;EACjG,MAAMiC,aAAkC,GAAG;IACzCvC,UAAU,EAAE9C,OAAO,CAACyE,MAAM,CAAC3B,UAAU;IACrC6B,QAAQ,EAAE3E,OAAO,CAACyE,MAAM,CAAC3B;EAC3B,CAAC;EAED,MAAMtC,MAAM,GAAG2C,YAAY,CACzBnD,OAAO,CAACyE,MAAM,CAACa,aAAa,EAC5BF,aAAa,EACbpB,UAAU,EACVkB,iBAAiB,EACjBG,aACF,CAAC;EAED,OAAO;IACL9E,OAAO,EAAEyE,OAAO;IAChB1E,OAAO,EAAEiE,OAAO;IAChB/D,MAAM;IACNE,KAAK,EAAEmD,UAAW;IAClBxC,UAAU,EAAEiC;EACd,CAAC;AACH;AASA,eAAe7B,gBAAgBA,CAC7BxB,MAAoB,EACpBqD,MAAkB,EAClBtD,OAA6B,EACH;EAAA,IAAAuF,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA,EAAAC,sBAAA;EAC1B,MAAM/B,SAAS,GAAG1D,MAAM,CAACC,MAAM,GAAGoD,MAAM,CAACM,oBAAoB;EAE7D,MAAMC,UAAU,IAAA0B,sBAAA,GAAGjC,MAAM,CAACqC,mBAAmB,cAAAJ,sBAAA,uBAA1BA,sBAAA,CAA4BxB,UAAU;EAEzD,MAAMmB,iBAAiB,GAAGrB,UAAU,KAAA2B,sBAAA,GAAGlC,MAAM,CAACqC,mBAAmB,cAAAH,sBAAA,uBAA1BA,sBAAA,CAA4BI,SAAS;EAC5E,MAAMR,aAAa,GAAGxF,aAAa,CACjCR,QAAQ,GAAAqG,sBAAA,GACRnC,MAAM,CAACqC,mBAAmB,cAAAF,sBAAA,uBAA1BA,sBAAA,CAA4BrC,QAC9B,CAAiB;EAIjB,IAAImB,OAAO,GAAG,IAAIC,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI7D,OAAO,CAACyE,MAAM,CAACC,SAAS,GAAG,CAAC,EAAE;IAChCH,OAAO,GAAGpB,YAAY,CAAC1D,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAE4D,UAAU,EAAG;MACtFc,QAAQ,EAAE9E,WAAW,CAACG,OAAO,CAACyE,MAAM,CAACC,SAAS,CAAC;MAC/CE,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLL,OAAO,CAACM,IAAI,CAAC,CAAC,CAAC;EACjB;EAIA,IAAIG,OAAO,GAAG,IAAIR,KAAK,CAACX,UAAU,CAAC;EACnC,IAAI7D,OAAO,CAACyE,MAAM,CAACQ,SAAS,GAAG,CAAC,EAAE;IAChCD,OAAO,GAAG7B,YAAY,CAAC1D,kBAAkB,EAAEC,sBAAsB,EAAEO,MAAM,EAAE4D,UAAU,EAAG;MACtFc,QAAQ,EAAE9E,WAAW,CAACG,OAAO,CAACyE,MAAM,CAACQ,SAAS,CAAC;MAC/CL,eAAe,EAAE;IACnB,CAAC,CAAC;EACJ,CAAC,MAAM;IACLI,OAAO,CAACH,IAAI,CAAC,CAAC,CAAC;EACjB;EAGA,IAAIgB,eAAe,GAAG5F,MAAM;EAE5B,KAAAyF,sBAAA,GAAIpC,MAAM,CAACqC,mBAAmB,cAAAD,sBAAA,eAA1BA,sBAAA,CAA4BI,aAAa,EAAE;IAC7C,MAAM5B,SAAS,GAAG,MAAM1E,UAAU,CAChCQ,OAAO,CAACiE,WAAW,EACnBhE,MAAM,CAACF,MAAM,CAACoE,KAAK,CAAClE,MAAM,CAACC,MAAM,EAAEyD,SAAS,CAAC,EAC7CL,MAAM,CAACc,sBACT,CAAC;IAEDyB,eAAe,GAAG;MAChB9F,MAAM,EAAEmE,SAAS;MACjBhE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE+D,SAAS,CAAC9D;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAC3B;EAEA,MAAM0B,aAAa,GAAG;IACpBvC,UAAU,EAAE9C,OAAO,CAACyE,MAAM,CAAC3B,UAAU;IACrC6B,QAAQ,EAAE3E,OAAO,CAACyE,MAAM,CAAC3B;EAC3B,CAAC;EAED,MAAMtC,MAAM,GAAG2C,YAAY,CACzBnD,OAAO,CAACyE,MAAM,CAACa,aAAa,EAC5BF,aAAa,EACbS,eAAe,EACfX,iBAAiB,EACjBG,aACF,CAAC;EAED,OAAO;IACL9E,OAAO,EAAEyE,OAAO;IAChB1E,OAAO,EAAEiE,OAAO;IAChB/D,MAAM;IACNE,KAAK,EAAEmD,UAAW;IAClBxC,UAAU,EAAEiC;EACd,CAAC;AACH;AAQA,eAAe5B,oBAAoBA,CACjCzB,MAAoB,EACpBoB,UAAsB,EACtBrB,OAA6B,EACM;EAAA,IAAA+F,qBAAA;EACnC,MAAMpC,SAAS,GAAG1D,MAAM,CAACC,MAAM,GAAGmB,UAAU,CAACuC,oBAAoB;EAEjE,IAAIoC,UAAU,GAAG;IACf9F,MAAM,EAAE,CAAC;IACTH,MAAM,EAAEE,MAAM,CAACF,MAAM,CAACoE,KAAK,CAAClE,MAAM,CAACC,MAAM,EAAEyD,SAAS,CAAC;IACrDxD,IAAI,EAAEwD,SAAS,GAAG1D,MAAM,CAACC;EAC3B,CAAC;EAEDD,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAEzB,IAAI3D,OAAO,CAACiE,WAAW,KAAK,cAAc,EAAE;IAC1C,MAAMC,SAAS,GAAG,MAAM1E,UAAU,CAChCQ,OAAO,CAACiE,WAAW,EACnB+B,UAAU,CAACjG,MAAM,CAACoE,KAAK,CAAC6B,UAAU,CAAC9F,MAAM,EAAEyD,SAAS,CAAC,EACrDtC,UAAU,CAAC+C,sBACb,CAAC;IAED4B,UAAU,GAAG;MACXjG,MAAM,EAAEmE,SAAS;MACjBhE,MAAM,EAAE,CAAC;MACTC,IAAI,EAAE+D,SAAS,CAAC9D;IAClB,CAAC;IAEDH,MAAM,CAACC,MAAM,GAAGyD,SAAS;EAC3B;EAEA,MAAM/C,SAAS,GAAG,CAAAS,UAAU,aAAVA,UAAU,wBAAA0E,qBAAA,GAAV1E,UAAU,CAAE4E,sBAAsB,cAAAF,qBAAA,uBAAlCA,qBAAA,CAAoChC,UAAU,KAAI,CAAC;EAErE,MAAMmC,uBAAuB,GAAG/C,YAAY,CAC1CnD,OAAO,CAACyE,MAAM,CAACa,aAAa,EAC5BtF,OAAO,CAACyE,MAAM,CAACrB,QAAQ,EACvB4C,UAAU,EACVpF,SAAS,EAETZ,OACF,CAAC;EAGD,IAAIQ,MAAa;EACjB,IAAIR,OAAO,aAAPA,OAAO,eAAPA,OAAO,CAAEmG,cAAc,EAAE;IAC3B3F,MAAM,GAAG0F,uBAAuB,CAAClF,GAAG,CAAEoF,CAAC,IAAKD,cAAc,CAACC,CAAC,CAAC,CAAC;EAChE,CAAC,MAAM;IACL5F,MAAM,GAAG0F,uBAAuB,CAAClF,GAAG,CAAEoF,CAAC,IAAKA,CAAC,CAACC,QAAQ,CAAC,CAAC,CAAC;EAC3D;EACA,OAAO7F,MAAM;AACf;AAEA,SAAS2F,cAAcA,CAACC,CAAM,EAA0C;EACtE,IAAIE,WAAW,CAACC,MAAM,CAACH,CAAC,CAAC,EAAE;IACzB,OAAOA,CAAC;EACV;EAEA,IAAII,MAAM,CAACC,QAAQ,CAACL,CAAC,CAAC,EAAE;IACtB,OAAOA,CAAC,CAACrG,MAAM,CAACoE,KAAK,CAACiC,CAAC,CAACM,UAAU,EAAEN,CAAC,CAACO,UAAU,CAAC;EACnD;EACA,OAAOP,CAAC,CAACC,QAAQ,CAAC,CAAC;AACrB"}

package/dist/esm/parquetjs/parser/parquet-reader.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parquet-reader.js","names":["ParquetSchema","decodeSchema","materializeRows","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","ParquetReader","constructor","file","props","_defineProperty","defaultProps","close","rowIterator","rows","rowBatchIterator","row","schema","getSchema","rowGroup","rowGroupIterator","columnList","map","x","Array","isArray","metadata","getFileMetadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","getRowCount","Number","num_rows","root","schemaDefinition","num_children","getSchemaMetadata","md","kv","key_value_metadata","key","value","readHeader","readFooter","buffer","read","magic","toString","Error","concat","trailerLen","trailerBuf","size","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","rowCount","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","_colChunk$meta_data","_colChunk$meta_data2","_colChunk$meta_data3","_colChunk$meta_data4","_colChunk$meta_data5","_colChunk$meta_data7","_colChunk$meta_data8","_context$dictionary","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","_colChunk$meta_data6","Math","min","context","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","preserveBinary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","defaultDictionarySize","cursor","offset","decodedPage"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRows} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetRowGroup,\n  ParquetCompression,\n  ParquetColumnChunk,\n  PrimitiveType,\n  ParquetReaderContext\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n  preserveBinary?: boolean;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  static defaultProps: Required<ParquetReaderProps> = {\n    defaultDictionarySize: 1e6,\n    preserveBinary: false\n  };\n\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...ParquetReader.defaultProps, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRows(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetRowGroup> {\n    const buffer: ParquetRowGroup = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetColumnChunk> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const context: ParquetReaderContext = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: [],\n      // Options - TBD is this the right place for these?\n      preserveBinary: this.props.preserveBinary\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, context, pagesOffset);\n    }\n\n    dictionary = context.dictionary?.length ? context.dictionary : dictionary;\n    const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...context, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param context\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    context: ParquetReaderContext,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, context);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":";AAGA,SAAQA,aAAa,QAAO,kBAAkB;AAC9C,SAAQC,YAAY,QAAO,YAAY;AACvC,SAAQC,eAAe,QAAO,iBAAiB;AAE/C,SAAQC,aAAa,EAAEC,uBAAuB,QAAO,iBAAiB;AACtE,SAAqBC,gBAAgB,EAA0BC,IAAI,QAAO,mBAAmB;AAQ7F,SAAQC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY,QAAO,qBAAqB;AACnF,SAAQC,eAAe,EAAEC,UAAU,QAAO,YAAY;AAmBtD,OAAO,MAAMC,aAAa,CAAC;EAUzBC,WAAWA,CAACC,IAAkB,EAAEC,KAA0B,EAAE;IAAAC,eAAA;IAAAA,eAAA;IAAAA,eAAA,mBAFnB,IAAI;IAG3C,IAAI,CAACF,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAG;MAAC,GAAGH,aAAa,CAACK,YAAY;MAAE,GAAGF;IAAK,CAAC;EACxD;EAEAG,KAAKA,CAAA,EAAS;IAEZ,IAAI,CAACJ,IAAI,CAACI,KAAK,CAAC,CAAC;EACnB;EAKA,OAAOC,WAAWA,CAACJ,KAA6B,EAAE;IAChD,WAAW,MAAMK,IAAI,IAAI,IAAI,CAACC,gBAAgB,CAACN,KAAK,CAAC,EAAE;MAErD,KAAK,MAAMO,GAAG,IAAIF,IAAI,EAAE;QACtB,MAAME,GAAG;MACX;IACF;EACF;EAGA,OAAOD,gBAAgBA,CAACN,KAA6B,EAAE;IACrD,MAAMQ,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IACrC,WAAW,MAAMC,QAAQ,IAAI,IAAI,CAACC,gBAAgB,CAACX,KAAK,CAAC,EAAE;MACzD,MAAMb,eAAe,CAACqB,MAAM,EAAEE,QAAQ,CAAC;IACzC;EACF;EAGA,OAAOC,gBAAgBA,CAACX,KAA6B,EAAE;IAErD,MAAMY,UAAsB,GAAG,CAAC,CAAAZ,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEY,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAEC,CAAC,IAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAC3B,CAAC;IAED,MAAMG,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMV,MAAM,GAAG,MAAM,IAAI,CAACC,SAAS,CAAC,CAAC;IAErC,MAAMU,aAAa,GAAG,CAAAF,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEG,UAAU,CAACC,MAAM,KAAI,CAAC;IAEtD,KAAK,IAAIC,aAAa,GAAG,CAAC,EAAEA,aAAa,GAAGH,aAAa,EAAEG,aAAa,EAAE,EAAE;MAC1E,MAAMZ,QAAQ,GAAG,MAAM,IAAI,CAACa,YAAY,CACtCf,MAAM,EACNS,QAAQ,CAACG,UAAU,CAACE,aAAa,CAAC,EAClCV,UACF,CAAC;MACD,MAAMF,QAAQ;IAChB;EACF;EAEA,MAAMc,WAAWA,CAAA,EAAoB;IACnC,MAAMP,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,OAAOO,MAAM,CAACR,QAAQ,CAACS,QAAQ,CAAC;EAClC;EAEA,MAAMjB,SAASA,CAAA,EAA2B;IACxC,MAAMQ,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMS,IAAI,GAAGV,QAAQ,CAACT,MAAM,CAAC,CAAC,CAAC;IAC/B,MAAM;MAACA,MAAM,EAAEoB;IAAgB,CAAC,GAAG1C,YAAY,CAAC+B,QAAQ,CAACT,MAAM,EAAE,CAAC,EAAEmB,IAAI,CAACE,YAAa,CAAC;IACvF,MAAMrB,MAAM,GAAG,IAAIvB,aAAa,CAAC2C,gBAAgB,CAAC;IAClD,OAAOpB,MAAM;EACf;EAMA,MAAMsB,iBAAiBA,CAAA,EAAoC;IACzD,MAAMb,QAAQ,GAAG,MAAM,IAAI,CAACC,eAAe,CAAC,CAAC;IAC7C,MAAMa,EAA0B,GAAG,CAAC,CAAC;IACrC,KAAK,MAAMC,EAAE,IAAIf,QAAQ,CAACgB,kBAAkB,EAAG;MAC7CF,EAAE,CAACC,EAAE,CAACE,GAAG,CAAC,GAAGF,EAAE,CAACG,KAAM;IACxB;IACA,OAAOJ,EAAE;EACX;EAEA,MAAMb,eAAeA,CAAA,EAA0B;IAC7C,IAAI,CAAC,IAAI,CAACD,QAAQ,EAAE;MAClB,MAAM,IAAI,CAACmB,UAAU,CAAC,CAAC;MACvB,IAAI,CAACnB,QAAQ,GAAG,IAAI,CAACoB,UAAU,CAAC,CAAC;IACnC;IACA,OAAO,IAAI,CAACpB,QAAQ;EACtB;EAKA,MAAMmB,UAAUA,CAAA,EAAkB;IAChC,MAAME,MAAM,GAAG,MAAM,IAAI,CAACvC,IAAI,CAACwC,IAAI,CAAC,CAAC,EAAEnD,aAAa,CAACiC,MAAM,CAAC;IAC5D,MAAMmB,KAAK,GAAGF,MAAM,CAACG,QAAQ,CAAC,CAAC;IAC/B,QAAQD,KAAK;MACX,KAAKpD,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAIqD,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,gCAAAC,MAAA,CAAgCH,KAAK,MAAG,CAAC;IAC5D;EACF;EAGA,MAAMH,UAAUA,CAAA,EAA0B;IACxC,MAAMO,UAAU,GAAGxD,aAAa,CAACiC,MAAM,GAAG,CAAC;IAC3C,MAAMwB,UAAU,GAAG,MAAM,IAAI,CAAC9C,IAAI,CAACwC,IAAI,CAAC,IAAI,CAACxC,IAAI,CAAC+C,IAAI,GAAGF,UAAU,EAAEA,UAAU,CAAC;IAEhF,MAAMJ,KAAK,GAAGK,UAAU,CAACE,KAAK,CAAC,CAAC,CAAC,CAACN,QAAQ,CAAC,CAAC;IAC5C,IAAID,KAAK,KAAKpD,aAAa,EAAE;MAC3B,MAAM,IAAIsD,KAAK,sCAAAC,MAAA,CAAqCH,KAAK,MAAG,CAAC;IAC/D;IAEA,MAAMQ,YAAY,GAAGH,UAAU,CAACI,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAACnD,IAAI,CAAC+C,IAAI,GAAGE,YAAY,GAAGJ,UAAU;IACjE,IAAIM,cAAc,GAAG9D,aAAa,CAACiC,MAAM,EAAE;MACzC,MAAM,IAAIqB,KAAK,0BAAAC,MAAA,CAA0BO,cAAc,CAAE,CAAC;IAC5D;IAEA,MAAMC,WAAW,GAAG,MAAM,IAAI,CAACpD,IAAI,CAACwC,IAAI,CAACW,cAAc,EAAEF,YAAY,CAAC;IAItE,MAAM;MAAC/B;IAAQ,CAAC,GAAGzB,kBAAkB,CAAC2D,WAAW,CAAC;IAClD,OAAOlC,QAAQ;EACjB;EAGA,MAAMM,YAAYA,CAChBf,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB,EACI;IAC1B,MAAM0B,MAAuB,GAAG;MAC9Bc,QAAQ,EAAE3B,MAAM,CAACf,QAAQ,CAACgB,QAAQ,CAAC;MACnC2B,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAI5C,QAAQ,CAAC6C,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAI/C,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI3B,YAAY,CAACkB,UAAU,EAAE8C,MAAO,CAAC,GAAG,CAAC,EAAE;QAClE;MACF;MACApB,MAAM,CAACe,UAAU,CAACK,MAAM,CAAEE,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACrD,MAAM,EAAE8C,QAAQ,CAAC;IAClF;IACA,OAAOhB,MAAM;EACf;EAKA,MAAMuB,eAAeA,CAACrD,MAAqB,EAAE8C,QAAqB,EAA+B;IAAA,IAAAQ,mBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,oBAAA,EAAAC,mBAAA;IAC/F,IAAIf,QAAQ,CAACgB,SAAS,KAAKC,SAAS,IAAIjB,QAAQ,CAACgB,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAI5B,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAM8B,KAAK,GAAGhE,MAAM,CAACiE,SAAS,EAAAX,mBAAA,GAACR,QAAQ,CAACG,SAAS,cAAAK,mBAAA,uBAAlBA,mBAAA,CAAoBH,cAAe,CAAC;IACnE,MAAMe,IAAmB,GAAGjF,aAAa,CAACF,IAAI,GAAAwE,oBAAA,GAAET,QAAQ,CAACG,SAAS,cAAAM,oBAAA,uBAAlBA,oBAAA,CAAoBW,IAAK,CAAQ;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAIjC,KAAK,oCAAAC,MAAA,CAAoC+B,IAAI,CAAE,CAAC;IAC5D;IAEA,MAAME,WAA+B,GAAGnF,aAAa,CACnDH,gBAAgB,GAAA0E,oBAAA,GAChBV,QAAQ,CAACG,SAAS,cAAAO,oBAAA,uBAAlBA,oBAAA,CAAoBa,KACtB,CAAQ;IAER,MAAMC,WAAW,GAAGrD,MAAM,EAAAwC,oBAAA,GAACX,QAAQ,CAACG,SAAS,cAAAQ,oBAAA,uBAAlBA,oBAAA,CAAoBc,gBAAiB,CAAC;IACjE,IAAIC,SAAS,GAAGvD,MAAM,EAAAyC,oBAAA,GAACZ,QAAQ,CAACG,SAAS,cAAAS,oBAAA,uBAAlBA,oBAAA,CAAoBe,qBAAsB,CAAC;IAElE,IAAI,CAAC3B,QAAQ,CAACgB,SAAS,EAAE;MAAA,IAAAY,oBAAA;MACvBF,SAAS,GAAGG,IAAI,CAACC,GAAG,CAClB,IAAI,CAACrF,IAAI,CAAC+C,IAAI,GAAGgC,WAAW,EAC5BrD,MAAM,EAAAyD,oBAAA,GAAC5B,QAAQ,CAACG,SAAS,cAAAyB,oBAAA,uBAAlBA,oBAAA,CAAoBD,qBAAqB,CAClD,CAAC;IACH;IAEA,MAAMI,OAA6B,GAAG;MACpCX,IAAI;MACJY,SAAS,EAAEd,KAAK,CAACc,SAAS;MAC1BC,SAAS,EAAEf,KAAK,CAACe,SAAS;MAC1BX,WAAW;MACXY,MAAM,EAAEhB,KAAK;MACbiB,SAAS,GAAAtB,oBAAA,GAAEb,QAAQ,CAACG,SAAS,cAAAU,oBAAA,uBAAlBA,oBAAA,CAAoBuB,UAAU;MACzCC,UAAU,EAAE,EAAE;MAEdC,cAAc,EAAE,IAAI,CAAC5F,KAAK,CAAC4F;IAC7B,CAAC;IAED,IAAID,UAAU;IAEd,MAAME,oBAAoB,GAAGvC,QAAQ,aAARA,QAAQ,wBAAAc,oBAAA,GAARd,QAAQ,CAAEG,SAAS,cAAAW,oBAAA,uBAAnBA,oBAAA,CAAqB0B,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAGtE,MAAM,CAACoE,oBAAoB,CAAC;MAErDF,UAAU,GAAG,MAAM,IAAI,CAACK,aAAa,CAACD,gBAAgB,EAAEV,OAAO,EAAEP,WAAW,CAAC;IAC/E;IAEAa,UAAU,GAAG,CAAAtB,mBAAA,GAAAgB,OAAO,CAACM,UAAU,cAAAtB,mBAAA,eAAlBA,mBAAA,CAAoBhD,MAAM,GAAGgE,OAAO,CAACM,UAAU,GAAGA,UAAU;IACzE,MAAMM,QAAQ,GAAG,MAAM,IAAI,CAAClG,IAAI,CAACwC,IAAI,CAACuC,WAAW,EAAEE,SAAS,CAAC;IAC7D,OAAO,MAAMrF,eAAe,CAACsG,QAAQ,EAAE;MAAC,GAAGZ,OAAO;MAAEM;IAAU,CAAC,CAAC;EAClE;EASA,MAAMK,aAAaA,CACjBH,oBAA4B,EAC5BR,OAA6B,EAC7BP,WAAmB,EACA;IACnB,IAAIe,oBAAoB,KAAK,CAAC,EAAE;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGf,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACrF,IAAI,CAAC+C,IAAI,GAAG+C,oBAAoB,EACrC,IAAI,CAAC7F,KAAK,CAACmG,qBACb,CAAC;IACD,MAAMF,QAAQ,GAAG,MAAM,IAAI,CAAClG,IAAI,CAACwC,IAAI,CAACsD,oBAAoB,EAAEK,cAAc,CAAC;IAE3E,MAAME,MAAM,GAAG;MAAC9D,MAAM,EAAE2D,QAAQ;MAAEI,MAAM,EAAE,CAAC;MAAEvD,IAAI,EAAEmD,QAAQ,CAAC5E;IAAM,CAAC;IACnE,MAAMiF,WAAW,GAAG,MAAM1G,UAAU,CAACwG,MAAM,EAAEf,OAAO,CAAC;IAErD,OAAOiB,WAAW,CAACX,UAAU;EAC/B;AACF;AAAC1F,eAAA,CAxPYJ,aAAa,kBAC4B;EAClDsG,qBAAqB,EAAE,GAAG;EAC1BP,cAAc,EAAE;AAClB,CAAC"}

package/dist/esm/parquetjs/schema/declare.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"declare.js","names":["ParquetRowGroup","constructor","rowCount","arguments","length","undefined","columnData","_defineProperty"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\n/** @todo better name, this is an internal type? */\nexport interface ParquetReaderContext {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n  /** If true, binary values are not converted to strings */\n  preserveBinary?: boolean;\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  /** Actual column chunks */\n  values: any[]; // ArrayLike<any>;\n  count: number;\n  dictionary?: ParquetDictionary;\n  /** The \"raw\" page header from the file */\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRow {\n  [key: string]: any;\n}\n\n/** @\n * Holds data for one row group (column chunks) */\nexport class ParquetRowGroup {\n  /** Number of rows in this page */\n  rowCount: number;\n  /** Map of Column chunks */\n  columnData: Record<string, ParquetColumnChunk>;\n\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetColumnChunk> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n\n/** Holds the data for one column chunk */\nexport interface ParquetColumnChunk {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n"],"mappings":";AAkIA,OAAO,MAAMA,eAAe,CAAC;EAM3BC,WAAWA,CAAA,EAA4E;IAAA,IAA3EC,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;IAAA,IAAEG,UAA8C,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;IAAAI,eAAA;IAAAA,eAAA;IACnF,IAAI,CAACL,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;EAC9B;AACF"}

package/dist/esm/parquetjs/schema/schema.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"schema.js","names":["PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","materializeRows","shredBuffer","shredRecord","PARQUET_LOGICAL_TYPES","ParquetSchema","constructor","schema","_defineProperty","fields","buildFields","fieldList","listFields","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","row","rowGroup","compress","type","setCompress","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","Error","encoding","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetRowGroup,\n  ParquetCompression,\n  ParquetField,\n  ParquetRow,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRows, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(row: ParquetRow, rowGroup: ParquetRowGroup): void {\n    shredRecord(this, row, rowGroup);\n  }\n\n  materializeRows(rowGroup: ParquetRowGroup): ParquetRow[] {\n    return materializeRows(this, rowGroup);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  rowGroup(): ParquetRowGroup {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"mappings":";AAEA,SAAQA,cAAc,QAAO,WAAW;AACxC,SAAQC,2BAA2B,QAAO,gBAAgB;AAU1D,SAAQC,eAAe,EAAEC,WAAW,EAAEC,WAAW,QAAO,SAAS;AACjE,SAAQC,qBAAqB,QAAO,SAAS;AAK7C,OAAO,MAAMC,aAAa,CAAC;EAQzBC,WAAWA,CAACC,MAAwB,EAAE;IAAAC,eAAA;IAAAA,eAAA;IAAAA,eAAA;IACpC,IAAI,CAACD,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACE,MAAM,GAAGC,WAAW,CAACH,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACI,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;EAKAI,SAASA,CAACC,IAAuB,EAAgB;IAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;MAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;IACxB,CAAC,MAAM;MAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;IACtB;IAEA,IAAIC,CAAC,GAAG,IAAI,CAACR,MAAM;IACnB,OAAOK,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;MACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACL,MAAsC;IACvD;IAEA,OAAOQ,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;EACnB;EAKAM,eAAeA,CAACN,IAAuB,EAAkB;IACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;MAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;IACxB;IACA,MAAMM,MAAsB,GAAG,EAAE;IACjC,IAAIJ,CAAC,GAAG,IAAI,CAACR,MAAM;IACnB,OAAOK,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;MACpCE,MAAM,CAACC,IAAI,CAACL,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;MACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;QACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACL,MAAsC;MACvD;IACF;IACA,OAAOY,MAAM;EACf;EAEAlB,WAAWA,CAACoB,GAAe,EAAEC,QAAyB,EAAQ;IAC5DrB,WAAW,CAAC,IAAI,EAAEoB,GAAG,EAAEC,QAAQ,CAAC;EAClC;EAEAvB,eAAeA,CAACuB,QAAyB,EAAgB;IACvD,OAAOvB,eAAe,CAAC,IAAI,EAAEuB,QAAQ,CAAC;EACxC;EAEAC,QAAQA,CAACC,IAAwB,EAAQ;IACvCC,WAAW,CAAC,IAAI,CAACpB,MAAM,EAAEmB,IAAI,CAAC;IAC9BC,WAAW,CAAC,IAAI,CAAClB,MAAM,EAAEiB,IAAI,CAAC;IAC9B,OAAO,IAAI;EACb;EAEAF,QAAQA,CAAA,EAAoB;IAC1B,OAAOtB,WAAW,CAAC,IAAI,CAAC;EAC1B;AACF;AAEA,SAASyB,WAAWA,CAACpB,MAAW,EAAEmB,IAAwB,EAAE;EAC1D,KAAK,MAAME,IAAI,IAAIrB,MAAM,EAAE;IACzB,MAAMsB,IAAI,GAAGtB,MAAM,CAACqB,IAAI,CAAC;IACzB,IAAIC,IAAI,CAACpB,MAAM,EAAE;MACfkB,WAAW,CAACE,IAAI,CAACpB,MAAM,EAAEiB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLG,IAAI,CAACC,WAAW,GAAGJ,IAAI;IACzB;EACF;AACF;AAGA,SAAShB,WAAWA,CAClBH,MAAwB,EACxBwB,eAAuB,EACvBC,eAAuB,EACvBlB,IAAc,EACgB;EAC9B,MAAMH,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,MAAMiB,IAAI,IAAIrB,MAAM,EAAE;IACzB,MAAM0B,IAAI,GAAG1B,MAAM,CAACqB,IAAI,CAAC;IAGzB,MAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,MAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;IAGA,IAAIN,IAAI,CAACxB,MAAM,EAAE;MACf,MAAMgC,KAAK,GAAG3B,IAAI,CAAC4B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;QAChBA,IAAI;QACJd,IAAI,EAAE2B,KAAK;QACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,CAAC,CAAC;QACjBJ,cAAc;QACdF,SAAS;QACTC,SAAS;QACTM,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACf,IAAI,CAACxB,MAAM,CAAC,CAACS,MAAM;QAC3CT,MAAM,EAAEC,WAAW,CAACuB,IAAI,CAACxB,MAAM,EAAE6B,SAAS,EAAEC,SAAS,EAAEE,KAAK;MAC9D,CAAC;MACD;IACF;IAEA,MAAMQ,OAAY,GAAG7C,qBAAqB,CAAC6B,IAAI,CAACP,IAAI,CAAE;IACtD,IAAI,CAACuB,OAAO,EAAE;MACZ,MAAM,IAAIC,KAAK,0BAAAR,MAAA,CAA0BT,IAAI,CAACP,IAAI,CAAE,CAAC;IACvD;IAEAO,IAAI,CAACkB,QAAQ,GAAGlB,IAAI,CAACkB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAElB,IAAI,CAACkB,QAAQ,IAAIpD,cAAc,CAAC,EAAE;MACtC,MAAM,IAAImD,KAAK,kCAAAR,MAAA,CAAkCT,IAAI,CAACkB,QAAQ,CAAE,CAAC;IACnE;IAEAlB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAI9B,2BAA2B,CAAC,EAAE;MACtD,MAAM,IAAIkD,KAAK,oCAAAR,MAAA,CAAoCT,IAAI,CAACH,WAAW,CAAE,CAAC;IACxE;IAGA,MAAMW,KAAK,GAAG3B,IAAI,CAAC4B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;MAChBA,IAAI;MACJwB,aAAa,EAAEH,OAAO,CAACG,aAAa;MACpCC,YAAY,EAAEJ,OAAO,CAACI,YAAY;MAClCvC,IAAI,EAAE2B,KAAK;MACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,CAAC,CAAC;MACjBJ,cAAc;MACdW,QAAQ,EAAElB,IAAI,CAACkB,QAAQ;MACvBrB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7BwB,UAAU,EAAErB,IAAI,CAACqB,UAAU,IAAIL,OAAO,CAACK,UAAU;MACjDC,SAAS,EAAEtB,IAAI,CAACsB,SAAS;MACzBC,KAAK,EAAEvB,IAAI,CAACuB,KAAK;MACjBlB,SAAS;MACTC;IACF,CAAC;EACH;EACA,OAAO5B,SAAS;AAClB;AAEA,SAASC,UAAUA,CAACH,MAAoC,EAAkB;EACxE,IAAIgD,IAAoB,GAAG,EAAE;EAC7B,KAAK,MAAMC,CAAC,IAAIjD,MAAM,EAAE;IACtBgD,IAAI,CAACnC,IAAI,CAACb,MAAM,CAACiD,CAAC,CAAC,CAAC;IACpB,IAAIjD,MAAM,CAACiD,CAAC,CAAC,CAACb,QAAQ,EAAE;MACtBY,IAAI,GAAGA,IAAI,CAACf,MAAM,CAAC9B,UAAU,CAACH,MAAM,CAACiD,CAAC,CAAC,CAACjD,MAAO,CAAC,CAAC;IACnD;EACF;EACA,OAAOgD,IAAI;AACb"}

package/dist/esm/parquetjs/schema/shred.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"shred.js","names":["ParquetRowGroup","Types","shredBuffer","schema","columnData","field","fieldList","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","rowGroup","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","concat","isNested","i","rlvl","rLevelMax","dLevelMax","toPrimitive","originalType","primitiveType","materializeRows","rows","materializeColumnAsRows","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive","materializeColumns","columns","materializeColumnAsColumnarArray","columnName","column"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ArrayType} from '@loaders.gl/schema';\nimport {ParquetRowGroup, ParquetColumnChunk, ParquetField, ParquetRow} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetRowGroup};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetRowGroup {\n  const columnData: Record<string, ParquetColumnChunk> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The rowGroup argument must point to an object into which the shredded record\n * will be returned. You may re-use the rowGroup for repeated calls to this function\n * to append to an existing rowGroup, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the rowGroup is as\n * follows:\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(\n  schema: ParquetSchema,\n  record: ParquetRow,\n  rowGroup: ParquetRowGroup\n): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the rowGroup */\n  if (rowGroup.rowCount === 0) {\n    rowGroup.rowCount = 1;\n    rowGroup.columnData = data;\n    return;\n  }\n  rowGroup.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(rowGroup.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(rowGroup.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(rowGroup.columnData[field.key].values, data[field.key].values);\n    rowGroup.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: ParquetRow,\n  data: Record<string, ParquetColumnChunk>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null!, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The rowGroup argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRows(schema: ParquetSchema, rowGroup: ParquetRowGroup): ParquetRow[] {\n  const rows: ParquetRow[] = [];\n  // rows = new Array(rowGroup.rowCount).fill({})'\n  for (let i = 0; i < rowGroup.rowCount; i++) {\n    rows.push({});\n  }\n  for (const key in rowGroup.columnData) {\n    const columnData = rowGroup.columnData[key];\n    if (columnData.count) {\n      materializeColumnAsRows(schema, columnData, key, rows);\n    }\n  }\n  return rows;\n}\n\n/** Populate record fields for one column */\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumnAsRows(\n  schema: ParquetSchema,\n  columnData: ParquetColumnChunk,\n  key: string,\n  rows: ParquetRow[]\n): void {\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = rows[rLevels[rIndex++] - 1];\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n}\n\n// Columnar export\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The rowGroup argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeColumns(\n  schema: ParquetSchema,\n  rowGroup: ParquetRowGroup\n): Record<string, ArrayType> {\n  const columns: Record<string, ArrayType> = {};\n  for (const key in rowGroup.columnData) {\n    const columnData = rowGroup.columnData[key];\n    if (columnData.count) {\n      materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);\n    }\n  }\n  return columns;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumnAsColumnarArray(\n  schema: ParquetSchema,\n  columnData: ParquetColumnChunk,\n  rowCount: number,\n  key: string,\n  columns: Record<string, ArrayType<any>>\n) {\n  if (columnData.count <= 0) {\n    return;\n  }\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  const columnName = branch[0].name;\n\n  let column: ArrayType | undefined;\n  const {values} = columnData;\n  if (values.length === rowCount && branch[0].primitiveType) {\n    // if (branch[0].repetitionType === `REQUIRED`) {\n    //   switch (branch[0].primitiveType) {\n    //     case 'INT32': return values instanceof Int32Array ? values : new Int32Array(values);\n    //   }\n    // }\n    column = values;\n  }\n\n  if (column) {\n    columns[columnName] = column;\n    return;\n  }\n\n  column = new Array(rowCount);\n  for (let i = 0; i < rowCount; i++) {\n    column[i] = {};\n  }\n  columns[columnName] = column;\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = column[rLevels[rIndex++] - 1] as ParquetRow;\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n\n  // Remove one level of nesting\n  for (let i = 0; i < rowCount; ++i) {\n    if (columnName in (column[i] as object)) {\n      column[i] = (column[i] as object)[columnName];\n    }\n  }\n}\n"],"mappings":"AAGA,SAAQA,eAAe,QAAqD,WAAW;AAEvF,OAAO,KAAKC,KAAK,MAAM,SAAS;AAEhC,SAAQD,eAAe;AAEvB,OAAO,SAASE,WAAWA,CAACC,MAAqB,EAAmB;EAClE,MAAMC,UAA8C,GAAG,CAAC,CAAC;EACzD,KAAK,MAAMC,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCF,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,GAAG;MACtBC,OAAO,EAAE,EAAE;MACXC,OAAO,EAAE,EAAE;MACXC,MAAM,EAAE,EAAE;MACVC,WAAW,EAAE,EAAE;MACfC,KAAK,EAAE;IACT,CAAC;EACH;EACA,OAAO;IAACC,QAAQ,EAAE,CAAC;IAAET;EAAU,CAAC;AAClC;AAwBA,OAAO,SAASU,WAAWA,CACzBX,MAAqB,EACrBY,MAAkB,EAClBC,QAAyB,EACnB;EAEN,MAAMC,IAAI,GAAGf,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Cc,iBAAiB,CAACf,MAAM,CAACgB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;EAGpD,IAAID,QAAQ,CAACH,QAAQ,KAAK,CAAC,EAAE;IAC3BG,QAAQ,CAACH,QAAQ,GAAG,CAAC;IACrBG,QAAQ,CAACZ,UAAU,GAAGa,IAAI;IAC1B;EACF;EACAD,QAAQ,CAACH,QAAQ,IAAI,CAAC;EACtB,KAAK,MAAMR,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCc,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,EAAEQ,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAAC;IAC3FW,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,EAAES,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAAC;IAC3FY,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,EAAEO,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAAC;IACzFM,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAIK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK;EAC/D;AACF;AAGA,SAASM,iBAAiBA,CACxBC,MAAoC,EACpCJ,MAAkB,EAClBE,IAAwC,EACxCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,MAAMC,IAAI,IAAIP,MAAM,EAAE;IACzB,MAAMd,KAAK,GAAGc,MAAM,CAACO,IAAI,CAAC;IAG1B,IAAIhB,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNV,KAAK,CAACqB,IAAI,IAAIX,MAAM,IACpBA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAKC,SAAS,IAChCZ,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIX,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACE,WAAW,KAAKR,KAAK,EAAE;QAC5CV,MAAM,GAAGK,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLhB,MAAM,CAACY,IAAI,CAACP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,IAAIC,OAAO,CAACf,MAAM,CAAC,IAAIV,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,4BAAAC,MAAA,CAA4B5B,KAAK,CAACqB,IAAI,CAAE,CAAC;IAC1D;IACA,IAAIhB,MAAM,CAACmB,MAAM,GAAG,CAAC,IAAIxB,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,+BAAAC,MAAA,CAA+B5B,KAAK,CAACqB,IAAI,CAAE,CAAC;IAC7D;IAGA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,EAAE;MACvB,IAAIxB,KAAK,CAAC6B,QAAQ,EAAE;QAClBhB,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAG,IAAI,EAAGF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC/D,CAAC,MAAM;QACLR,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;IAGA,KAAK,IAAIU,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGzB,MAAM,CAACmB,MAAM,EAAEM,CAAC,EAAE,EAAE;MACtC,MAAMC,IAAI,GAAGD,CAAC,KAAK,CAAC,GAAGX,MAAM,GAAGnB,KAAK,CAACgC,SAAS;MAC/C,IAAIhC,KAAK,CAAC6B,QAAQ,EAAE;QAClBhB,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAGT,MAAM,CAACyB,CAAC,CAAC,EAAElB,IAAI,EAAEmB,IAAI,EAAE/B,KAAK,CAACiC,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLrB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACc,IAAI,CAAC;QAClCnB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACjB,KAAK,CAACiC,SAAS,CAAC;QAC7CrB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAACY,IAAI,CACzBrB,KAAK,CAACsC,WAAW,CAAElC,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EAAI/B,MAAM,CAACyB,CAAC,CAAC,CAC3E,CAAC;MACH;IACF;EACF;AACF;AAqBA,OAAO,SAASO,eAAeA,CAACvC,MAAqB,EAAEa,QAAyB,EAAgB;EAC9F,MAAM2B,IAAkB,GAAG,EAAE;EAE7B,KAAK,IAAIR,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGnB,QAAQ,CAACH,QAAQ,EAAEsB,CAAC,EAAE,EAAE;IAC1CQ,IAAI,CAACrB,IAAI,CAAC,CAAC,CAAC,CAAC;EACf;EACA,KAAK,MAAMf,GAAG,IAAIS,QAAQ,CAACZ,UAAU,EAAE;IACrC,MAAMA,UAAU,GAAGY,QAAQ,CAACZ,UAAU,CAACG,GAAG,CAAC;IAC3C,IAAIH,UAAU,CAACQ,KAAK,EAAE;MACpBgC,uBAAuB,CAACzC,MAAM,EAAEC,UAAU,EAAEG,GAAG,EAAEoC,IAAI,CAAC;IACxD;EACF;EACA,OAAOA,IAAI;AACb;AAIA,SAASC,uBAAuBA,CAC9BzC,MAAqB,EACrBC,UAA8B,EAC9BG,GAAW,EACXoC,IAAkB,EACZ;EACN,MAAMtC,KAAK,GAAGF,MAAM,CAAC0C,SAAS,CAACtC,GAAG,CAAC;EACnC,MAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAe,CAACxC,GAAG,CAAC;EAG1C,MAAMyC,OAAiB,GAAG,IAAI5B,KAAK,CAACf,KAAK,CAACgC,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIf,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG/B,UAAU,CAACQ,KAAK,EAAEuB,CAAC,EAAE,EAAE;IACzC,MAAMV,MAAM,GAAGrB,UAAU,CAACI,OAAO,CAAC2B,CAAC,CAAC;IACpC,MAAMX,MAAM,GAAGpB,UAAU,CAACK,OAAO,CAAC0B,CAAC,CAAC;IACpCa,OAAO,CAACxB,MAAM,CAAC,EAAE;IACjBwB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAEzB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI2B,MAAM,GAAG,CAAC;IACd,IAAIpC,MAAM,GAAG4B,IAAI,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;IAGxC,KAAK,MAAMC,IAAI,IAAIN,MAAM,EAAE;MACzB,IAAIM,IAAI,KAAK/C,KAAK,IAAIoB,MAAM,GAAG2B,IAAI,CAACd,SAAS,EAAE;QAC7C;MACF;MAEA,QAAQc,IAAI,CAACrB,cAAc;QACzB,KAAK,UAAU;UACb,IAAI,EAAEqB,IAAI,CAAC1B,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE1BA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAG,EAAE;UACxB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;UAC5B,OAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAErCtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;UAC5B;UACAP,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAAC2B,EAAE,CAAC;UAC9B;QAEF;UACEtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAGX,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,IAAI,CAAC,CAAC;UAC3CX,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC;MAC9B;IACF;IAGA,IAAID,MAAM,KAAKpB,KAAK,CAACiC,SAAS,EAAE;MAC9B,MAAMgB,KAAK,GAAGrD,KAAK,CAACsD,aAAa,CAE/BlD,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EACzCrC,UAAU,CAACM,MAAM,CAACwC,MAAM,CAAC,EACzB7C,KACF,CAAC;MACD6C,MAAM,EAAE;MAER,QAAQ7C,KAAK,CAAC0B,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAE1B,KAAK,CAACqB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOpC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAEtCtC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC2B,EAAE,CAAC,GAAGC,KAAK;UAC9B;QAEF;UACEvC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG4B,KAAK;MAC9B;IACF;EACF;AACF;AAuBA,OAAO,SAASE,kBAAkBA,CAChCrD,MAAqB,EACrBa,QAAyB,EACE;EAC3B,MAAMyC,OAAkC,GAAG,CAAC,CAAC;EAC7C,KAAK,MAAMlD,GAAG,IAAIS,QAAQ,CAACZ,UAAU,EAAE;IACrC,MAAMA,UAAU,GAAGY,QAAQ,CAACZ,UAAU,CAACG,GAAG,CAAC;IAC3C,IAAIH,UAAU,CAACQ,KAAK,EAAE;MACpB8C,gCAAgC,CAACvD,MAAM,EAAEC,UAAU,EAAEY,QAAQ,CAACH,QAAQ,EAAEN,GAAG,EAAEkD,OAAO,CAAC;IACvF;EACF;EACA,OAAOA,OAAO;AAChB;AAGA,SAASC,gCAAgCA,CACvCvD,MAAqB,EACrBC,UAA8B,EAC9BS,QAAgB,EAChBN,GAAW,EACXkD,OAAuC,EACvC;EACA,IAAIrD,UAAU,CAACQ,KAAK,IAAI,CAAC,EAAE;IACzB;EACF;EAEA,MAAMP,KAAK,GAAGF,MAAM,CAAC0C,SAAS,CAACtC,GAAG,CAAC;EACnC,MAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAe,CAACxC,GAAG,CAAC;EAE1C,MAAMoD,UAAU,GAAGb,MAAM,CAAC,CAAC,CAAC,CAACpB,IAAI;EAEjC,IAAIkC,MAA6B;EACjC,MAAM;IAAClD;EAAM,CAAC,GAAGN,UAAU;EAC3B,IAAIM,MAAM,CAACmB,MAAM,KAAKhB,QAAQ,IAAIiC,MAAM,CAAC,CAAC,CAAC,CAACL,aAAa,EAAE;IAMzDmB,MAAM,GAAGlD,MAAM;EACjB;EAEA,IAAIkD,MAAM,EAAE;IACVH,OAAO,CAACE,UAAU,CAAC,GAAGC,MAAM;IAC5B;EACF;EAEAA,MAAM,GAAG,IAAIxC,KAAK,CAACP,QAAQ,CAAC;EAC5B,KAAK,IAAIsB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGtB,QAAQ,EAAEsB,CAAC,EAAE,EAAE;IACjCyB,MAAM,CAACzB,CAAC,CAAC,GAAG,CAAC,CAAC;EAChB;EACAsB,OAAO,CAACE,UAAU,CAAC,GAAGC,MAAM;EAG5B,MAAMZ,OAAiB,GAAG,IAAI5B,KAAK,CAACf,KAAK,CAACgC,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIf,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG/B,UAAU,CAACQ,KAAK,EAAEuB,CAAC,EAAE,EAAE;IACzC,MAAMV,MAAM,GAAGrB,UAAU,CAACI,OAAO,CAAC2B,CAAC,CAAC;IACpC,MAAMX,MAAM,GAAGpB,UAAU,CAACK,OAAO,CAAC0B,CAAC,CAAC;IACpCa,OAAO,CAACxB,MAAM,CAAC,EAAE;IACjBwB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAEzB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI2B,MAAM,GAAG,CAAC;IACd,IAAIpC,MAAM,GAAG6C,MAAM,CAACZ,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAe;IAGxD,KAAK,MAAMC,IAAI,IAAIN,MAAM,EAAE;MACzB,IAAIM,IAAI,KAAK/C,KAAK,IAAIoB,MAAM,GAAG2B,IAAI,CAACd,SAAS,EAAE;QAC7C;MACF;MAEA,QAAQc,IAAI,CAACrB,cAAc;QACzB,KAAK,UAAU;UACb,IAAI,EAAEqB,IAAI,CAAC1B,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE1BA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAG,EAAE;UACxB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;UAC5B,OAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAErCtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;UAC5B;UACAP,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAAC2B,EAAE,CAAC;UAC9B;QAEF;UACEtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAGX,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,IAAI,CAAC,CAAC;UAC3CX,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC;MAC9B;IACF;IAGA,IAAID,MAAM,KAAKpB,KAAK,CAACiC,SAAS,EAAE;MAC9B,MAAMgB,KAAK,GAAGrD,KAAK,CAACsD,aAAa,CAE/BlD,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EACzCrC,UAAU,CAACM,MAAM,CAACwC,MAAM,CAAC,EACzB7C,KACF,CAAC;MACD6C,MAAM,EAAE;MAER,QAAQ7C,KAAK,CAAC0B,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAE1B,KAAK,CAACqB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOpC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAEtCtC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC2B,EAAE,CAAC,GAAGC,KAAK;UAC9B;QAEF;UACEvC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG4B,KAAK;MAC9B;IACF;EACF;EAGA,KAAK,IAAInB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGtB,QAAQ,EAAE,EAAEsB,CAAC,EAAE;IACjC,IAAIwB,UAAU,IAAKC,MAAM,CAACzB,CAAC,CAAY,EAAE;MACvCyB,MAAM,CAACzB,CAAC,CAAC,GAAIyB,MAAM,CAACzB,CAAC,CAAC,CAAYwB,UAAU,CAAC;IAC/C;EACF;AACF"}

package/dist/esm/parquetjs/schema/types.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.js","names":["BSONLoader","BSONWriter","PARQUET_LOGICAL_TYPES","BOOLEAN","primitiveType","toPrimitive","toPrimitive_BOOLEAN","fromPrimitive","fromPrimitive_BOOLEAN","INT32","toPrimitive_INT32","INT64","toPrimitive_INT64","INT96","toPrimitive_INT96","FLOAT","toPrimitive_FLOAT","DOUBLE","toPrimitive_DOUBLE","BYTE_ARRAY","toPrimitive_BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","originalType","toPrimitive_UTF8","fromPrimitive_UTF8","TIME_MILLIS","toPrimitive_TIME_MILLIS","TIME_MICROS","toPrimitive_TIME_MICROS","DATE","toPrimitive_DATE","fromPrimitive_DATE","TIMESTAMP_MILLIS","toPrimitive_TIMESTAMP_MILLIS","fromPrimitive_TIMESTAMP_MILLIS","TIMESTAMP_MICROS","toPrimitive_TIMESTAMP_MICROS","fromPrimitive_TIMESTAMP_MICROS","UINT_8","toPrimitive_UINT8","UINT_16","toPrimitive_UINT16","UINT_32","toPrimitive_UINT32","UINT_64","toPrimitive_UINT64","INT_8","toPrimitive_INT8","INT_16","toPrimitive_INT16","INT_32","INT_64","JSON","toPrimitive_JSON","fromPrimitive_JSON","BSON","toPrimitive_BSON","fromPrimitive_BSON","INTERVAL","typeLength","toPrimitive_INTERVAL","fromPrimitive_INTERVAL","DECIMAL_INT32","decimalToPrimitive_INT32","decimalFromPrimitive_INT","DECIMAL_INT64","decimalToPrimitive_INT64","DECIMAL_BYTE_ARRAY","decimalToPrimitive_BYTE_ARRAY","decimalFromPrimitive_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","type","value","field","Error","concat","_PARQUET_LOGICAL_TYPE","_PARQUET_LOGICAL_TYPE2","call","Boolean","v","parseFloat","isNaN","parseInt","primitiveValue","scale","Math","round","presision","Buffer","from","toString","stringify","parse","_BSONWriter$encodeSyn","arrayBuffer","encodeSync","_BSONLoader$parseSync","parseSync","kMillisPerDay","Date","getTime","months","days","milliseconds","buf","alloc","writeUInt32LE","readUInt32LE","millis","presisionInt","number","length","i","component"],"sources":["../../../../src/parquetjs/schema/types.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport {BSONLoader, BSONWriter} from '@loaders.gl/bson';\nimport {OriginalType, ParquetField, ParquetType, PrimitiveType} from './declare';\n\nexport interface ParquetTypeKit {\n  primitiveType: PrimitiveType;\n  originalType?: OriginalType;\n  typeLength?: number;\n  toPrimitive: Function;\n  fromPrimitive?: Function;\n}\n\nexport const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {\n  BOOLEAN: {\n    primitiveType: 'BOOLEAN',\n    toPrimitive: toPrimitive_BOOLEAN,\n    fromPrimitive: fromPrimitive_BOOLEAN\n  },\n  INT32: {\n    primitiveType: 'INT32',\n    toPrimitive: toPrimitive_INT32\n  },\n  INT64: {\n    primitiveType: 'INT64',\n    toPrimitive: toPrimitive_INT64\n  },\n  INT96: {\n    primitiveType: 'INT96',\n    toPrimitive: toPrimitive_INT96\n  },\n  FLOAT: {\n    primitiveType: 'FLOAT',\n    toPrimitive: toPrimitive_FLOAT\n  },\n  DOUBLE: {\n    primitiveType: 'DOUBLE',\n    toPrimitive: toPrimitive_DOUBLE\n  },\n  BYTE_ARRAY: {\n    primitiveType: 'BYTE_ARRAY',\n    toPrimitive: toPrimitive_BYTE_ARRAY\n  },\n  FIXED_LEN_BYTE_ARRAY: {\n    primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n    toPrimitive: toPrimitive_BYTE_ARRAY\n  },\n  UTF8: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'UTF8',\n    toPrimitive: toPrimitive_UTF8,\n    fromPrimitive: fromPrimitive_UTF8\n  },\n  TIME_MILLIS: {\n    primitiveType: 'INT32',\n    originalType: 'TIME_MILLIS',\n    toPrimitive: toPrimitive_TIME_MILLIS\n  },\n  TIME_MICROS: {\n    primitiveType: 'INT64',\n    originalType: 'TIME_MICROS',\n    toPrimitive: toPrimitive_TIME_MICROS\n  },\n  DATE: {\n    primitiveType: 'INT32',\n    originalType: 'DATE',\n    toPrimitive: toPrimitive_DATE,\n    fromPrimitive: fromPrimitive_DATE\n  },\n  TIMESTAMP_MILLIS: {\n    primitiveType: 'INT64',\n    originalType: 'TIMESTAMP_MILLIS',\n    toPrimitive: toPrimitive_TIMESTAMP_MILLIS,\n    fromPrimitive: fromPrimitive_TIMESTAMP_MILLIS\n  },\n  TIMESTAMP_MICROS: {\n    primitiveType: 'INT64',\n    originalType: 'TIMESTAMP_MICROS',\n    toPrimitive: toPrimitive_TIMESTAMP_MICROS,\n    fromPrimitive: fromPrimitive_TIMESTAMP_MICROS\n  },\n  UINT_8: {\n    primitiveType: 'INT32',\n    originalType: 'UINT_8',\n    toPrimitive: toPrimitive_UINT8\n  },\n  UINT_16: {\n    primitiveType: 'INT32',\n    originalType: 'UINT_16',\n    toPrimitive: toPrimitive_UINT16\n  },\n  UINT_32: {\n    primitiveType: 'INT32',\n    originalType: 'UINT_32',\n    toPrimitive: toPrimitive_UINT32\n  },\n  UINT_64: {\n    primitiveType: 'INT64',\n    originalType: 'UINT_64',\n    toPrimitive: toPrimitive_UINT64\n  },\n  INT_8: {\n    primitiveType: 'INT32',\n    originalType: 'INT_8',\n    toPrimitive: toPrimitive_INT8\n  },\n  INT_16: {\n    primitiveType: 'INT32',\n    originalType: 'INT_16',\n    toPrimitive: toPrimitive_INT16\n  },\n  INT_32: {\n    primitiveType: 'INT32',\n    originalType: 'INT_32',\n    toPrimitive: toPrimitive_INT32\n  },\n  INT_64: {\n    primitiveType: 'INT64',\n    originalType: 'INT_64',\n    toPrimitive: toPrimitive_INT64\n  },\n  JSON: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'JSON',\n    toPrimitive: toPrimitive_JSON,\n    fromPrimitive: fromPrimitive_JSON\n  },\n  BSON: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'BSON',\n    toPrimitive: toPrimitive_BSON,\n    fromPrimitive: fromPrimitive_BSON\n  },\n  INTERVAL: {\n    primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n    originalType: 'INTERVAL',\n    typeLength: 12,\n    toPrimitive: toPrimitive_INTERVAL,\n    fromPrimitive: fromPrimitive_INTERVAL\n  },\n  DECIMAL_INT32: {\n    primitiveType: 'INT32',\n    originalType: 'DECIMAL_INT32',\n    toPrimitive: decimalToPrimitive_INT32,\n    fromPrimitive: decimalFromPrimitive_INT\n  },\n  DECIMAL_INT64: {\n    primitiveType: 'INT64',\n    originalType: 'DECIMAL_INT64',\n    toPrimitive: decimalToPrimitive_INT64,\n    fromPrimitive: decimalFromPrimitive_INT\n  },\n  DECIMAL_BYTE_ARRAY: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'DECIMAL_BYTE_ARRAY',\n    toPrimitive: decimalToPrimitive_BYTE_ARRAY,\n    fromPrimitive: decimalFromPrimitive_BYTE_ARRAY\n  },\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: {\n    primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n    originalType: 'DECIMAL_FIXED_LEN_BYTE_ARRAY',\n    toPrimitive: decimalToPrimitive_BYTE_ARRAY,\n    fromPrimitive: decimalFromPrimitive_BYTE_ARRAY\n  }\n};\n\n/**\n * Convert a value from it's native representation to the internal/underlying\n * primitive type\n */\nexport function toPrimitive(type: ParquetType, value: unknown, field?: ParquetField): unknown {\n  if (!(type in PARQUET_LOGICAL_TYPES)) {\n    throw new Error(`invalid type: ${type}`);\n  }\n\n  return PARQUET_LOGICAL_TYPES[type].toPrimitive(value, field);\n}\n\n/**\n * Convert a value from it's internal/underlying primitive representation to\n * the native representation\n */\nexport function fromPrimitive(type: ParquetType, value: unknown, field?: ParquetField) {\n  if (!(type in PARQUET_LOGICAL_TYPES)) {\n    throw new Error(`invalid type: ${type}`);\n  }\n\n  if ('fromPrimitive' in PARQUET_LOGICAL_TYPES[type]) {\n    return PARQUET_LOGICAL_TYPES[type].fromPrimitive?.(value, field);\n    // tslint:disable-next-line:no-else-after-return\n  }\n  return value;\n}\n\nfunction toPrimitive_BOOLEAN(value: unknown): boolean {\n  return Boolean(value);\n}\n\nfunction fromPrimitive_BOOLEAN(value: any): boolean {\n  return Boolean(value);\n}\n\nfunction toPrimitive_FLOAT(value: any): number {\n  const v = parseFloat(value);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for FLOAT: ${value}`);\n  }\n  return v;\n}\n\nfunction toPrimitive_DOUBLE(value: any): number {\n  const v = parseFloat(value);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for DOUBLE: ${value}`);\n  }\n  return v;\n}\n\nfunction toPrimitive_INT8(value: any) {\n  const v = parseInt(value, 10);\n  if (v < -0x80 || v > 0x7f || isNaN(v)) {\n    throw new Error(`invalid value for INT8: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_UINT8(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xff || isNaN(v)) {\n    throw new Error(`invalid value for UINT8: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT16(value: any) {\n  const v = parseInt(value, 10);\n  if (v < -0x8000 || v > 0x7fff || isNaN(v)) {\n    throw new Error(`invalid value for INT16: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_UINT16(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xffff || isNaN(v)) {\n    throw new Error(`invalid value for UINT16: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT32(value: any) {\n  const v = parseInt(value, 10);\n  if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {\n    throw new Error(`invalid value for INT32: ${value}`);\n  }\n\n  return v;\n}\n\nfunction decimalToPrimitive_INT32(value: number, field: ParquetField): number {\n  const primitiveValue = value * 10 ** (field.scale || 0);\n  const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n  if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {\n    throw new Error(`invalid value for INT32: ${value}`);\n  }\n  return v;\n}\n\nfunction toPrimitive_UINT32(value: any): number {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xffffffffffff || isNaN(v)) {\n    throw new Error(`invalid value for UINT32: ${value}`);\n  }\n  return v;\n}\n\nfunction toPrimitive_INT64(value: any): number {\n  const v = parseInt(value, 10);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for INT64: ${value}`);\n  }\n  return v;\n}\n\nfunction decimalToPrimitive_INT64(value: number, field: ParquetField) {\n  const primitiveValue = value * 10 ** (field.scale || 0);\n  const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for INT64: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_UINT64(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || isNaN(v)) {\n    throw new Error(`invalid value for UINT64: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT96(value: any) {\n  const v = parseInt(value, 10);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for INT96: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_BYTE_ARRAY(value: any): Buffer {\n  return Buffer.from(value);\n}\n\nfunction decimalToPrimitive_BYTE_ARRAY(value: any): Buffer {\n  // TBD\n  return Buffer.from(value);\n}\n\nfunction toPrimitive_UTF8(value: any): Buffer {\n  return Buffer.from(value, 'utf8');\n}\n\nfunction fromPrimitive_UTF8(value: any): string {\n  return value.toString();\n}\n\nfunction toPrimitive_JSON(value: any): Buffer {\n  return Buffer.from(JSON.stringify(value));\n}\n\nfunction fromPrimitive_JSON(value: any): unknown {\n  return JSON.parse(value);\n}\n\nfunction toPrimitive_BSON(value: any): Buffer {\n  const arrayBuffer = BSONWriter.encodeSync?.(value) as ArrayBuffer;\n  return Buffer.from(arrayBuffer);\n}\n\nfunction fromPrimitive_BSON(value: any) {\n  return BSONLoader.parseSync?.(value);\n}\n\nfunction toPrimitive_TIME_MILLIS(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xffffffffffffffff || isNaN(v)) {\n    throw new Error(`invalid value for TIME_MILLIS: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_TIME_MICROS(value: any): number {\n  const v = parseInt(value, 10);\n  if (v < 0 || isNaN(v)) {\n    throw new Error(`invalid value for TIME_MICROS: ${value}`);\n  }\n  return v;\n}\n\nconst kMillisPerDay = 86400000;\n\nfunction toPrimitive_DATE(value: any): number {\n  /* convert from date */\n  if (value instanceof Date) {\n    return value.getTime() / kMillisPerDay;\n  }\n\n  /* convert from integer */\n  {\n    const v = parseInt(value, 10);\n    if (v < 0 || isNaN(v)) {\n      throw new Error(`invalid value for DATE: ${value}`);\n    }\n\n    return v;\n  }\n}\n\nfunction fromPrimitive_DATE(value: any): Date {\n  return new Date(value * kMillisPerDay);\n}\n\nfunction toPrimitive_TIMESTAMP_MILLIS(value: any): number {\n  /* convert from date */\n  if (value instanceof Date) {\n    return value.getTime();\n  }\n\n  /* convert from integer */\n  {\n    const v = parseInt(value, 10);\n    if (v < 0 || isNaN(v)) {\n      throw new Error(`invalid value for TIMESTAMP_MILLIS: ${value}`);\n    }\n\n    return v;\n  }\n}\n\nfunction fromPrimitive_TIMESTAMP_MILLIS(value: any): Date {\n  return new Date(value);\n}\n\nfunction toPrimitive_TIMESTAMP_MICROS(value: any) {\n  /* convert from date */\n  if (value instanceof Date) {\n    return value.getTime() * 1000;\n  }\n\n  /* convert from integer */\n  {\n    const v = parseInt(value, 10);\n    if (v < 0 || isNaN(v)) {\n      throw new Error(`invalid value for TIMESTAMP_MICROS: ${value}`);\n    }\n\n    return v;\n  }\n}\n\nfunction fromPrimitive_TIMESTAMP_MICROS(value: any) {\n  return new Date(value / 1000);\n}\n\nfunction toPrimitive_INTERVAL(value: any) {\n  if (!value.months || !value.days || !value.milliseconds) {\n    throw new Error(\n      'value for INTERVAL must be object { months: ..., days: ..., milliseconds: ... }'\n    );\n  }\n\n  const buf = Buffer.alloc(12);\n\n  buf.writeUInt32LE(value.months, 0);\n  buf.writeUInt32LE(value.days, 4);\n  buf.writeUInt32LE(value.milliseconds, 8);\n  return buf;\n}\n\nfunction fromPrimitive_INTERVAL(value: any) {\n  const buf = Buffer.from(value);\n  const months = buf.readUInt32LE(0);\n  const days = buf.readUInt32LE(4);\n  const millis = buf.readUInt32LE(8);\n\n  return {months, days, milliseconds: millis};\n}\n\nfunction decimalFromPrimitive_INT(value: any, field: ParquetField) {\n  const presisionInt = Math.round(((value * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n  return presisionInt * 10 ** -(field.scale || 0);\n}\n\nfunction decimalFromPrimitive_BYTE_ARRAY(value: any, field: ParquetField) {\n  let number = 0;\n  if (value.length <= 4) {\n    // Bytewise operators faster. Use them if it is possible\n    for (let i = 0; i < value.length; i++) {\n      // `value.length - i - 1` bytes have reverse order (big-endian)\n      const component = value[i] << (8 * (value.length - i - 1));\n      number += component;\n    }\n  } else {\n    for (let i = 0; i < value.length; i++) {\n      // `value.length - i - 1` bytes have reverse order (big-endian)\n      const component = value[i] * 2 ** (8 * (value.length - 1 - i));\n      number += component;\n    }\n  }\n\n  const presisionInt = Math.round(\n    ((number * 10 ** -field.presision!) % 1) * 10 ** field.presision!\n  );\n  return presisionInt * 10 ** -(field.scale || 0);\n}\n"],"mappings":"AAEA,SAAQA,UAAU,EAAEC,UAAU,QAAO,kBAAkB;AAWvD,OAAO,MAAMC,qBAA0D,GAAG;EACxEC,OAAO,EAAE;IACPC,aAAa,EAAE,SAAS;IACxBC,WAAW,EAAEC,mBAAmB;IAChCC,aAAa,EAAEC;EACjB,CAAC;EACDC,KAAK,EAAE;IACLL,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAEK;EACf,CAAC;EACDC,KAAK,EAAE;IACLP,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAEO;EACf,CAAC;EACDC,KAAK,EAAE;IACLT,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAES;EACf,CAAC;EACDC,KAAK,EAAE;IACLX,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAEW;EACf,CAAC;EACDC,MAAM,EAAE;IACNb,aAAa,EAAE,QAAQ;IACvBC,WAAW,EAAEa;EACf,CAAC;EACDC,UAAU,EAAE;IACVf,aAAa,EAAE,YAAY;IAC3BC,WAAW,EAAEe;EACf,CAAC;EACDC,oBAAoB,EAAE;IACpBjB,aAAa,EAAE,sBAAsB;IACrCC,WAAW,EAAEe;EACf,CAAC;EACDE,IAAI,EAAE;IACJlB,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAEmB,gBAAgB;IAC7BjB,aAAa,EAAEkB;EACjB,CAAC;EACDC,WAAW,EAAE;IACXtB,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,aAAa;IAC3BlB,WAAW,EAAEsB;EACf,CAAC;EACDC,WAAW,EAAE;IACXxB,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,aAAa;IAC3BlB,WAAW,EAAEwB;EACf,CAAC;EACDC,IAAI,EAAE;IACJ1B,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAE0B,gBAAgB;IAC7BxB,aAAa,EAAEyB;EACjB,CAAC;EACDC,gBAAgB,EAAE;IAChB7B,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,kBAAkB;IAChClB,WAAW,EAAE6B,4BAA4B;IACzC3B,aAAa,EAAE4B;EACjB,CAAC;EACDC,gBAAgB,EAAE;IAChBhC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,kBAAkB;IAChClB,WAAW,EAAEgC,4BAA4B;IACzC9B,aAAa,EAAE+B;EACjB,CAAC;EACDC,MAAM,EAAE;IACNnC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAEmC;EACf,CAAC;EACDC,OAAO,EAAE;IACPrC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,SAAS;IACvBlB,WAAW,EAAEqC;EACf,CAAC;EACDC,OAAO,EAAE;IACPvC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,SAAS;IACvBlB,WAAW,EAAEuC;EACf,CAAC;EACDC,OAAO,EAAE;IACPzC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,SAAS;IACvBlB,WAAW,EAAEyC;EACf,CAAC;EACDC,KAAK,EAAE;IACL3C,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,OAAO;IACrBlB,WAAW,EAAE2C;EACf,CAAC;EACDC,MAAM,EAAE;IACN7C,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAE6C;EACf,CAAC;EACDC,MAAM,EAAE;IACN/C,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAEK;EACf,CAAC;EACD0C,MAAM,EAAE;IACNhD,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAEO;EACf,CAAC;EACDyC,IAAI,EAAE;IACJjD,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAEiD,gBAAgB;IAC7B/C,aAAa,EAAEgD;EACjB,CAAC;EACDC,IAAI,EAAE;IACJpD,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAEoD,gBAAgB;IAC7BlD,aAAa,EAAEmD;EACjB,CAAC;EACDC,QAAQ,EAAE;IACRvD,aAAa,EAAE,sBAAsB;IACrCmB,YAAY,EAAE,UAAU;IACxBqC,UAAU,EAAE,EAAE;IACdvD,WAAW,EAAEwD,oBAAoB;IACjCtD,aAAa,EAAEuD;EACjB,CAAC;EACDC,aAAa,EAAE;IACb3D,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,eAAe;IAC7BlB,WAAW,EAAE2D,wBAAwB;IACrCzD,aAAa,EAAE0D;EACjB,CAAC;EACDC,aAAa,EAAE;IACb9D,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,eAAe;IAC7BlB,WAAW,EAAE8D,wBAAwB;IACrC5D,aAAa,EAAE0D;EACjB,CAAC;EACDG,kBAAkB,EAAE;IAClBhE,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,oBAAoB;IAClClB,WAAW,EAAEgE,6BAA6B;IAC1C9D,aAAa,EAAE+D;EACjB,CAAC;EACDC,4BAA4B,EAAE;IAC5BnE,aAAa,EAAE,sBAAsB;IACrCmB,YAAY,EAAE,8BAA8B;IAC5ClB,WAAW,EAAEgE,6BAA6B;IAC1C9D,aAAa,EAAE+D;EACjB;AACF,CAAC;AAMD,OAAO,SAASjE,WAAWA,CAACmE,IAAiB,EAAEC,KAAc,EAAEC,KAAoB,EAAW;EAC5F,IAAI,EAAEF,IAAI,IAAItE,qBAAqB,CAAC,EAAE;IACpC,MAAM,IAAIyE,KAAK,kBAAAC,MAAA,CAAkBJ,IAAI,CAAE,CAAC;EAC1C;EAEA,OAAOtE,qBAAqB,CAACsE,IAAI,CAAC,CAACnE,WAAW,CAACoE,KAAK,EAAEC,KAAK,CAAC;AAC9D;AAMA,OAAO,SAASnE,aAAaA,CAACiE,IAAiB,EAAEC,KAAc,EAAEC,KAAoB,EAAE;EACrF,IAAI,EAAEF,IAAI,IAAItE,qBAAqB,CAAC,EAAE;IACpC,MAAM,IAAIyE,KAAK,kBAAAC,MAAA,CAAkBJ,IAAI,CAAE,CAAC;EAC1C;EAEA,IAAI,eAAe,IAAItE,qBAAqB,CAACsE,IAAI,CAAC,EAAE;IAAA,IAAAK,qBAAA,EAAAC,sBAAA;IAClD,QAAAD,qBAAA,GAAO,CAAAC,sBAAA,GAAA5E,qBAAqB,CAACsE,IAAI,CAAC,EAACjE,aAAa,cAAAsE,qBAAA,uBAAzCA,qBAAA,CAAAE,IAAA,CAAAD,sBAAA,EAA4CL,KAAK,EAAEC,KAAK,CAAC;EAElE;EACA,OAAOD,KAAK;AACd;AAEA,SAASnE,mBAAmBA,CAACmE,KAAc,EAAW;EACpD,OAAOO,OAAO,CAACP,KAAK,CAAC;AACvB;AAEA,SAASjE,qBAAqBA,CAACiE,KAAU,EAAW;EAClD,OAAOO,OAAO,CAACP,KAAK,CAAC;AACvB;AAEA,SAASzD,iBAAiBA,CAACyD,KAAU,EAAU;EAC7C,MAAMQ,CAAC,GAAGC,UAAU,CAACT,KAAK,CAAC;EAC3B,IAAIU,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIN,KAAK,6BAAAC,MAAA,CAA6BH,KAAK,CAAE,CAAC;EACtD;EACA,OAAOQ,CAAC;AACV;AAEA,SAAS/D,kBAAkBA,CAACuD,KAAU,EAAU;EAC9C,MAAMQ,CAAC,GAAGC,UAAU,CAACT,KAAK,CAAC;EAC3B,IAAIU,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIN,KAAK,8BAAAC,MAAA,CAA8BH,KAAK,CAAE,CAAC;EACvD;EACA,OAAOQ,CAAC;AACV;AAEA,SAASjC,gBAAgBA,CAACyB,KAAU,EAAE;EACpC,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIQ,CAAC,GAAG,CAAC,IAAI,IAAIA,CAAC,GAAG,IAAI,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACrC,MAAM,IAAIN,KAAK,4BAAAC,MAAA,CAA4BH,KAAK,CAAE,CAAC;EACrD;EAEA,OAAOQ,CAAC;AACV;AAEA,SAASzC,iBAAiBA,CAACiC,KAAU,EAAE;EACrC,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIQ,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,IAAI,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACjC,MAAM,IAAIN,KAAK,6BAAAC,MAAA,CAA6BH,KAAK,CAAE,CAAC;EACtD;EAEA,OAAOQ,CAAC;AACV;AAEA,SAAS/B,iBAAiBA,CAACuB,KAAU,EAAE;EACrC,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIQ,CAAC,GAAG,CAAC,MAAM,IAAIA,CAAC,GAAG,MAAM,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACzC,MAAM,IAAIN,KAAK,6BAAAC,MAAA,CAA6BH,KAAK,CAAE,CAAC;EACtD;EAEA,OAAOQ,CAAC;AACV;AAEA,SAASvC,kBAAkBA,CAAC+B,KAAU,EAAE;EACtC,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIQ,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,MAAM,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACnC,MAAM,IAAIN,KAAK,8BAAAC,MAAA,CAA8BH,KAAK,CAAE,CAAC;EACvD;EAEA,OAAOQ,CAAC;AACV;AAEA,SAASvE,iBAAiBA,CAAC+D,KAAU,EAAE;EACrC,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIQ,CAAC,GAAG,CAAC,UAAU,IAAIA,CAAC,GAAG,UAAU,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACjD,MAAM,IAAIN,KAAK,6BAAAC,MAAA,CAA6BH,KAAK,CAAE,CAAC;EACtD;EAEA,OAAOQ,CAAC;AACV;AAEA,SAASjB,wBAAwBA,CAACS,KAAa,EAAEC,KAAmB,EAAU;EAC5E,MAAMW,cAAc,GAAGZ,KAAK,GAAG,EAAE,KAAKC,KAAK,CAACY,KAAK,IAAI,CAAC,CAAC;EACvD,MAAML,CAAC,GAAGM,IAAI,CAACC,KAAK,CAAGH,cAAc,GAAG,EAAE,IAAI,CAACX,KAAK,CAACe,SAAU,GAAI,CAAC,GAAI,EAAE,IAAIf,KAAK,CAACe,SAAU,CAAC;EAC/F,IAAIR,CAAC,GAAG,CAAC,UAAU,IAAIA,CAAC,GAAG,UAAU,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACjD,MAAM,IAAIN,KAAK,6BAAAC,MAAA,CAA6BH,KAAK,CAAE,CAAC;EACtD;EACA,OAAOQ,CAAC;AACV;AAEA,SAASrC,kBAAkBA,CAAC6B,KAAU,EAAU;EAC9C,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIQ,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,cAAc,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IAC3C,MAAM,IAAIN,KAAK,8BAAAC,MAAA,CAA8BH,KAAK,CAAE,CAAC;EACvD;EACA,OAAOQ,CAAC;AACV;AAEA,SAASrE,iBAAiBA,CAAC6D,KAAU,EAAU;EAC7C,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIU,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIN,KAAK,6BAAAC,MAAA,CAA6BH,KAAK,CAAE,CAAC;EACtD;EACA,OAAOQ,CAAC;AACV;AAEA,SAASd,wBAAwBA,CAACM,KAAa,EAAEC,KAAmB,EAAE;EACpE,MAAMW,cAAc,GAAGZ,KAAK,GAAG,EAAE,KAAKC,KAAK,CAACY,KAAK,IAAI,CAAC,CAAC;EACvD,MAAML,CAAC,GAAGM,IAAI,CAACC,KAAK,CAAGH,cAAc,GAAG,EAAE,IAAI,CAACX,KAAK,CAACe,SAAU,GAAI,CAAC,GAAI,EAAE,IAAIf,KAAK,CAACe,SAAU,CAAC;EAC/F,IAAIN,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIN,KAAK,6BAAAC,MAAA,CAA6BH,KAAK,CAAE,CAAC;EACtD;EAEA,OAAOQ,CAAC;AACV;AAEA,SAASnC,kBAAkBA,CAAC2B,KAAU,EAAE;EACtC,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIQ,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACrB,MAAM,IAAIN,KAAK,8BAAAC,MAAA,CAA8BH,KAAK,CAAE,CAAC;EACvD;EAEA,OAAOQ,CAAC;AACV;AAEA,SAASnE,iBAAiBA,CAAC2D,KAAU,EAAE;EACrC,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIU,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIN,KAAK,6BAAAC,MAAA,CAA6BH,KAAK,CAAE,CAAC;EACtD;EAEA,OAAOQ,CAAC;AACV;AAEA,SAAS7D,sBAAsBA,CAACqD,KAAU,EAAU;EAClD,OAAOiB,MAAM,CAACC,IAAI,CAAClB,KAAK,CAAC;AAC3B;AAEA,SAASJ,6BAA6BA,CAACI,KAAU,EAAU;EAEzD,OAAOiB,MAAM,CAACC,IAAI,CAAClB,KAAK,CAAC;AAC3B;AAEA,SAASjD,gBAAgBA,CAACiD,KAAU,EAAU;EAC5C,OAAOiB,MAAM,CAACC,IAAI,CAAClB,KAAK,EAAE,MAAM,CAAC;AACnC;AAEA,SAAShD,kBAAkBA,CAACgD,KAAU,EAAU;EAC9C,OAAOA,KAAK,CAACmB,QAAQ,CAAC,CAAC;AACzB;AAEA,SAAStC,gBAAgBA,CAACmB,KAAU,EAAU;EAC5C,OAAOiB,MAAM,CAACC,IAAI,CAACtC,IAAI,CAACwC,SAAS,CAACpB,KAAK,CAAC,CAAC;AAC3C;AAEA,SAASlB,kBAAkBA,CAACkB,KAAU,EAAW;EAC/C,OAAOpB,IAAI,CAACyC,KAAK,CAACrB,KAAK,CAAC;AAC1B;AAEA,SAAShB,gBAAgBA,CAACgB,KAAU,EAAU;EAAA,IAAAsB,qBAAA;EAC5C,MAAMC,WAAW,IAAAD,qBAAA,GAAG9F,UAAU,CAACgG,UAAU,cAAAF,qBAAA,uBAArBA,qBAAA,CAAAhB,IAAA,CAAA9E,UAAU,EAAcwE,KAAK,CAAgB;EACjE,OAAOiB,MAAM,CAACC,IAAI,CAACK,WAAW,CAAC;AACjC;AAEA,SAAStC,kBAAkBA,CAACe,KAAU,EAAE;EAAA,IAAAyB,qBAAA;EACtC,QAAAA,qBAAA,GAAOlG,UAAU,CAACmG,SAAS,cAAAD,qBAAA,uBAApBA,qBAAA,CAAAnB,IAAA,CAAA/E,UAAU,EAAayE,KAAK,CAAC;AACtC;AAEA,SAAS9C,uBAAuBA,CAAC8C,KAAU,EAAE;EAC3C,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIQ,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,kBAAkB,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IAC/C,MAAM,IAAIN,KAAK,mCAAAC,MAAA,CAAmCH,KAAK,CAAE,CAAC;EAC5D;EAEA,OAAOQ,CAAC;AACV;AAEA,SAASpD,uBAAuBA,CAAC4C,KAAU,EAAU;EACnD,MAAMQ,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIQ,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACrB,MAAM,IAAIN,KAAK,mCAAAC,MAAA,CAAmCH,KAAK,CAAE,CAAC;EAC5D;EACA,OAAOQ,CAAC;AACV;AAEA,MAAMmB,aAAa,GAAG,QAAQ;AAE9B,SAASrE,gBAAgBA,CAAC0C,KAAU,EAAU;EAE5C,IAAIA,KAAK,YAAY4B,IAAI,EAAE;IACzB,OAAO5B,KAAK,CAAC6B,OAAO,CAAC,CAAC,GAAGF,aAAa;EACxC;EAGA;IACE,MAAMnB,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;IAC7B,IAAIQ,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;MACrB,MAAM,IAAIN,KAAK,4BAAAC,MAAA,CAA4BH,KAAK,CAAE,CAAC;IACrD;IAEA,OAAOQ,CAAC;EACV;AACF;AAEA,SAASjD,kBAAkBA,CAACyC,KAAU,EAAQ;EAC5C,OAAO,IAAI4B,IAAI,CAAC5B,KAAK,GAAG2B,aAAa,CAAC;AACxC;AAEA,SAASlE,4BAA4BA,CAACuC,KAAU,EAAU;EAExD,IAAIA,KAAK,YAAY4B,IAAI,EAAE;IACzB,OAAO5B,KAAK,CAAC6B,OAAO,CAAC,CAAC;EACxB;EAGA;IACE,MAAMrB,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;IAC7B,IAAIQ,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;MACrB,MAAM,IAAIN,KAAK,wCAAAC,MAAA,CAAwCH,KAAK,CAAE,CAAC;IACjE;IAEA,OAAOQ,CAAC;EACV;AACF;AAEA,SAAS9C,8BAA8BA,CAACsC,KAAU,EAAQ;EACxD,OAAO,IAAI4B,IAAI,CAAC5B,KAAK,CAAC;AACxB;AAEA,SAASpC,4BAA4BA,CAACoC,KAAU,EAAE;EAEhD,IAAIA,KAAK,YAAY4B,IAAI,EAAE;IACzB,OAAO5B,KAAK,CAAC6B,OAAO,CAAC,CAAC,GAAG,IAAI;EAC/B;EAGA;IACE,MAAMrB,CAAC,GAAGG,QAAQ,CAACX,KAAK,EAAE,EAAE,CAAC;IAC7B,IAAIQ,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;MACrB,MAAM,IAAIN,KAAK,wCAAAC,MAAA,CAAwCH,KAAK,CAAE,CAAC;IACjE;IAEA,OAAOQ,CAAC;EACV;AACF;AAEA,SAAS3C,8BAA8BA,CAACmC,KAAU,EAAE;EAClD,OAAO,IAAI4B,IAAI,CAAC5B,KAAK,GAAG,IAAI,CAAC;AAC/B;AAEA,SAASZ,oBAAoBA,CAACY,KAAU,EAAE;EACxC,IAAI,CAACA,KAAK,CAAC8B,MAAM,IAAI,CAAC9B,KAAK,CAAC+B,IAAI,IAAI,CAAC/B,KAAK,CAACgC,YAAY,EAAE;IACvD,MAAM,IAAI9B,KAAK,CACb,iFACF,CAAC;EACH;EAEA,MAAM+B,GAAG,GAAGhB,MAAM,CAACiB,KAAK,CAAC,EAAE,CAAC;EAE5BD,GAAG,CAACE,aAAa,CAACnC,KAAK,CAAC8B,MAAM,EAAE,CAAC,CAAC;EAClCG,GAAG,CAACE,aAAa,CAACnC,KAAK,CAAC+B,IAAI,EAAE,CAAC,CAAC;EAChCE,GAAG,CAACE,aAAa,CAACnC,KAAK,CAACgC,YAAY,EAAE,CAAC,CAAC;EACxC,OAAOC,GAAG;AACZ;AAEA,SAAS5C,sBAAsBA,CAACW,KAAU,EAAE;EAC1C,MAAMiC,GAAG,GAAGhB,MAAM,CAACC,IAAI,CAAClB,KAAK,CAAC;EAC9B,MAAM8B,MAAM,GAAGG,GAAG,CAACG,YAAY,CAAC,CAAC,CAAC;EAClC,MAAML,IAAI,GAAGE,GAAG,CAACG,YAAY,CAAC,CAAC,CAAC;EAChC,MAAMC,MAAM,GAAGJ,GAAG,CAACG,YAAY,CAAC,CAAC,CAAC;EAElC,OAAO;IAACN,MAAM;IAAEC,IAAI;IAAEC,YAAY,EAAEK;EAAM,CAAC;AAC7C;AAEA,SAAS7C,wBAAwBA,CAACQ,KAAU,EAAEC,KAAmB,EAAE;EACjE,MAAMqC,YAAY,GAAGxB,IAAI,CAACC,KAAK,CAAGf,KAAK,GAAG,EAAE,IAAI,CAACC,KAAK,CAACe,SAAU,GAAI,CAAC,GAAI,EAAE,IAAIf,KAAK,CAACe,SAAU,CAAC;EACjG,OAAOsB,YAAY,GAAG,EAAE,IAAI,EAAErC,KAAK,CAACY,KAAK,IAAI,CAAC,CAAC;AACjD;AAEA,SAAShB,+BAA+BA,CAACG,KAAU,EAAEC,KAAmB,EAAE;EACxE,IAAIsC,MAAM,GAAG,CAAC;EACd,IAAIvC,KAAK,CAACwC,MAAM,IAAI,CAAC,EAAE;IAErB,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGzC,KAAK,CAACwC,MAAM,EAAEC,CAAC,EAAE,EAAE;MAErC,MAAMC,SAAS,GAAG1C,KAAK,CAACyC,CAAC,CAAC,IAAK,CAAC,IAAIzC,KAAK,CAACwC,MAAM,GAAGC,CAAC,GAAG,CAAC,CAAE;MAC1DF,MAAM,IAAIG,SAAS;IACrB;EACF,CAAC,MAAM;IACL,KAAK,IAAID,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGzC,KAAK,CAACwC,MAAM,EAAEC,CAAC,EAAE,EAAE;MAErC,MAAMC,SAAS,GAAG1C,KAAK,CAACyC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAIzC,KAAK,CAACwC,MAAM,GAAG,CAAC,GAAGC,CAAC,CAAC,CAAC;MAC9DF,MAAM,IAAIG,SAAS;IACrB;EACF;EAEA,MAAMJ,YAAY,GAAGxB,IAAI,CAACC,KAAK,CAC3BwB,MAAM,GAAG,EAAE,IAAI,CAACtC,KAAK,CAACe,SAAU,GAAI,CAAC,GAAI,EAAE,IAAIf,KAAK,CAACe,SACzD,CAAC;EACD,OAAOsB,YAAY,GAAG,EAAE,IAAI,EAAErC,KAAK,CAACY,KAAK,IAAI,CAAC,CAAC;AACjD"}

package/dist/esm/parquetjs/utils/file-utils.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"file-utils.js","names":["fs","load","name","module","global","require","oswrite","os","buf","Promise","resolve","reject","write","err","osclose","close","osopen","path","opts","outputStream","createWriteStream","once","fd"],"sources":["../../../../src/parquetjs/utils/file-utils.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {fs, stream} from '@loaders.gl/loader-utils';\n\nexport function load(name: string): any {\n  return (module || (global as any)).require(name);\n}\nexport interface WriteStreamOptions {\n  flags?: string;\n  encoding?: string;\n  fd?: number;\n  mode?: number;\n  autoClose?: boolean;\n  start?: number;\n}\n\nexport function oswrite(os: stream.Writable, buf: Buffer): Promise<void> {\n  return new Promise((resolve, reject) => {\n    os.write(buf, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osclose(os: stream.Writable): Promise<void> {\n  return new Promise((resolve, reject) => {\n    (os as any).close((err: any) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {\n  return new Promise((resolve, reject) => {\n    const outputStream = fs.createWriteStream(path, opts as any);\n    outputStream.once('open', (fd) => resolve(outputStream));\n    outputStream.once('error', (err) => reject(err));\n  });\n}\n"],"mappings":"AACA,SAAQA,EAAE,QAAe,0BAA0B;AAEnD,OAAO,SAASC,IAAIA,CAACC,IAAY,EAAO;EACtC,OAAO,CAACC,MAAM,IAAKC,MAAc,EAAEC,OAAO,CAACH,IAAI,CAAC;AAClD;AAUA,OAAO,SAASI,OAAOA,CAACC,EAAmB,EAAEC,GAAW,EAAiB;EACvE,OAAO,IAAIC,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;IACtCJ,EAAE,CAACK,KAAK,CAACJ,GAAG,EAAGK,GAAG,IAAK;MACrB,IAAIA,GAAG,EAAE;QACPF,MAAM,CAACE,GAAG,CAAC;MACb,CAAC,MAAM;QACLH,OAAO,CAAC,CAAC;MACX;IACF,CAAC,CAAC;EACJ,CAAC,CAAC;AACJ;AAEA,OAAO,SAASI,OAAOA,CAACP,EAAmB,EAAiB;EAC1D,OAAO,IAAIE,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;IACrCJ,EAAE,CAASQ,KAAK,CAAEF,GAAQ,IAAK;MAC9B,IAAIA,GAAG,EAAE;QACPF,MAAM,CAACE,GAAG,CAAC;MACb,CAAC,MAAM;QACLH,OAAO,CAAC,CAAC;MACX;IACF,CAAC,CAAC;EACJ,CAAC,CAAC;AACJ;AAEA,OAAO,SAASM,MAAMA,CAACC,IAAY,EAAEC,IAAyB,EAA2B;EACvF,OAAO,IAAIT,OAAO,CAAC,CAACC,OAAO,EAAEC,MAAM,KAAK;IACtC,MAAMQ,YAAY,GAAGnB,EAAE,CAACoB,iBAAiB,CAACH,IAAI,EAAEC,IAAW,CAAC;IAC5DC,YAAY,CAACE,IAAI,CAAC,MAAM,EAAGC,EAAE,IAAKZ,OAAO,CAACS,YAAY,CAAC,CAAC;IACxDA,YAAY,CAACE,IAAI,CAAC,OAAO,EAAGR,GAAG,IAAKF,MAAM,CAACE,GAAG,CAAC,CAAC;EAClD,CAAC,CAAC;AACJ"}

package/dist/esm/parquetjs/utils/read-utils.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"read-utils.js","names":["TBufferedTransport","TCompactProtocol","TFramedTransport","FileMetaData","PageHeader","UFramedTransport","constructor","arguments","_defineProperty","serializeThrift","obj","output","transport","undefined","buf","push","protocol","write","flush","Buffer","concat","decodeThrift","offset","readPos","read","getThriftEnum","klass","value","k","Error","decodeFileMetadata","metadata","length","decodePageHeader","pageHeader","getBitWidth","val","Math","ceil","log2","fieldIndexOf","arr","elem","j","m","i"],"sources":["../../../../src/parquetjs/utils/read-utils.ts"],"sourcesContent":["import {TBufferedTransport, TCompactProtocol, TFramedTransport} from '../parquet-thrift';\nimport {FileMetaData, PageHeader} from '../parquet-thrift';\n\nclass UFramedTransport extends TFramedTransport {\n  public readPos: number = 0;\n}\n\n/**\n * Helper function that serializes a thrift object into a buffer\n */\nexport function serializeThrift(obj: any): Buffer {\n  const output: Buffer[] = [];\n\n  const transport = new TBufferedTransport(undefined, (buf) => {\n    output.push(buf as unknown as Buffer);\n  });\n\n  const protocol = new TCompactProtocol(transport);\n  obj.write(protocol);\n  transport.flush();\n\n  return Buffer.concat(output);\n}\n\nexport function decodeThrift(obj: any, buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  obj.read(protocol);\n  return transport.readPos - offset;\n}\n\n/**\n * FIXME not ideal that this is linear\n */\nexport function getThriftEnum(klass: any, value: number | string): string {\n  for (const k in klass) {\n    if (klass[k] === value) {\n      return k;\n    }\n  }\n  throw new Error('Invalid ENUM value');\n}\n\nexport function decodeFileMetadata(buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  const metadata = FileMetaData.read(protocol);\n  return {length: transport.readPos - offset, metadata};\n}\n\nexport function decodePageHeader(buf: Buffer, offset?: number) {\n  if (!offset) {\n    // tslint:disable-next-line:no-parameter-reassignment\n    offset = 0;\n  }\n\n  const transport = new UFramedTransport(buf);\n  transport.readPos = offset;\n  const protocol = new TCompactProtocol(transport);\n  const pageHeader = PageHeader.read(protocol);\n  return {length: transport.readPos - offset, pageHeader};\n}\n\n/**\n * Get the number of bits required to store a given value\n */\nexport function getBitWidth(val: number): number {\n  if (val === 0) {\n    return 0;\n    // tslint:disable-next-line:no-else-after-return\n  }\n  return Math.ceil(Math.log2(val + 1));\n}\n\n// Supports MQTT path wildcards\n// + all immediate children\n// # all descendents\nexport function fieldIndexOf(arr: string[][], elem: string[]): number {\n  for (let j = 0; j < arr.length; j++) {\n    if (arr[j].length > elem.length) {\n      continue; // eslint-disable-line no-continue\n    }\n    let m = true;\n    for (let i = 0; i < elem.length; i++) {\n      if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {\n        continue; // eslint-disable-line no-continue\n      }\n      if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {\n        continue; // eslint-disable-line no-continue\n      }\n      m = false;\n      break;\n    }\n    if (m) return j;\n  }\n  return -1;\n}\n"],"mappings":";AAAA,SAAQA,kBAAkB,EAAEC,gBAAgB,EAAEC,gBAAgB,QAAO,mBAAmB;AACxF,SAAQC,YAAY,EAAEC,UAAU,QAAO,mBAAmB;AAE1D,MAAMC,gBAAgB,SAASH,gBAAgB,CAAC;EAAAI,YAAA;IAAA,SAAAC,SAAA;IAAAC,eAAA,kBACrB,CAAC;EAAA;AAC5B;AAKA,OAAO,SAASC,eAAeA,CAACC,GAAQ,EAAU;EAChD,MAAMC,MAAgB,GAAG,EAAE;EAE3B,MAAMC,SAAS,GAAG,IAAIZ,kBAAkB,CAACa,SAAS,EAAGC,GAAG,IAAK;IAC3DH,MAAM,CAACI,IAAI,CAACD,GAAwB,CAAC;EACvC,CAAC,CAAC;EAEF,MAAME,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChDF,GAAG,CAACO,KAAK,CAACD,QAAQ,CAAC;EACnBJ,SAAS,CAACM,KAAK,CAAC,CAAC;EAEjB,OAAOC,MAAM,CAACC,MAAM,CAACT,MAAM,CAAC;AAC9B;AAEA,OAAO,SAASU,YAAYA,CAACX,GAAQ,EAAEI,GAAW,EAAEQ,MAAe,EAAE;EACnE,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,MAAMV,SAAS,GAAG,IAAIP,gBAAgB,CAACS,GAAG,CAAC;EAC3CF,SAAS,CAACW,OAAO,GAAGD,MAAM;EAC1B,MAAMN,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChDF,GAAG,CAACc,IAAI,CAACR,QAAQ,CAAC;EAClB,OAAOJ,SAAS,CAACW,OAAO,GAAGD,MAAM;AACnC;AAKA,OAAO,SAASG,aAAaA,CAACC,KAAU,EAAEC,KAAsB,EAAU;EACxE,KAAK,MAAMC,CAAC,IAAIF,KAAK,EAAE;IACrB,IAAIA,KAAK,CAACE,CAAC,CAAC,KAAKD,KAAK,EAAE;MACtB,OAAOC,CAAC;IACV;EACF;EACA,MAAM,IAAIC,KAAK,CAAC,oBAAoB,CAAC;AACvC;AAEA,OAAO,SAASC,kBAAkBA,CAAChB,GAAW,EAAEQ,MAAe,EAAE;EAC/D,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,MAAMV,SAAS,GAAG,IAAIP,gBAAgB,CAACS,GAAG,CAAC;EAC3CF,SAAS,CAACW,OAAO,GAAGD,MAAM;EAC1B,MAAMN,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChD,MAAMmB,QAAQ,GAAG5B,YAAY,CAACqB,IAAI,CAACR,QAAQ,CAAC;EAC5C,OAAO;IAACgB,MAAM,EAAEpB,SAAS,CAACW,OAAO,GAAGD,MAAM;IAAES;EAAQ,CAAC;AACvD;AAEA,OAAO,SAASE,gBAAgBA,CAACnB,GAAW,EAAEQ,MAAe,EAAE;EAC7D,IAAI,CAACA,MAAM,EAAE;IAEXA,MAAM,GAAG,CAAC;EACZ;EAEA,MAAMV,SAAS,GAAG,IAAIP,gBAAgB,CAACS,GAAG,CAAC;EAC3CF,SAAS,CAACW,OAAO,GAAGD,MAAM;EAC1B,MAAMN,QAAQ,GAAG,IAAIf,gBAAgB,CAACW,SAAS,CAAC;EAChD,MAAMsB,UAAU,GAAG9B,UAAU,CAACoB,IAAI,CAACR,QAAQ,CAAC;EAC5C,OAAO;IAACgB,MAAM,EAAEpB,SAAS,CAACW,OAAO,GAAGD,MAAM;IAAEY;EAAU,CAAC;AACzD;AAKA,OAAO,SAASC,WAAWA,CAACC,GAAW,EAAU;EAC/C,IAAIA,GAAG,KAAK,CAAC,EAAE;IACb,OAAO,CAAC;EAEV;EACA,OAAOC,IAAI,CAACC,IAAI,CAACD,IAAI,CAACE,IAAI,CAACH,GAAG,GAAG,CAAC,CAAC,CAAC;AACtC;AAKA,OAAO,SAASI,YAAYA,CAACC,GAAe,EAAEC,IAAc,EAAU;EACpE,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGF,GAAG,CAACT,MAAM,EAAEW,CAAC,EAAE,EAAE;IACnC,IAAIF,GAAG,CAACE,CAAC,CAAC,CAACX,MAAM,GAAGU,IAAI,CAACV,MAAM,EAAE;MAC/B;IACF;IACA,IAAIY,CAAC,GAAG,IAAI;IACZ,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGH,IAAI,CAACV,MAAM,EAAEa,CAAC,EAAE,EAAE;MACpC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAKH,IAAI,CAACG,CAAC,CAAC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAK,GAAG,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACE,CAAC,CAAC,KAAK,GAAG,EAAE;QACnE;MACF;MACA,IAAIA,CAAC,IAAIJ,GAAG,CAACE,CAAC,CAAC,CAACX,MAAM,IAAIS,GAAG,CAACE,CAAC,CAAC,CAACF,GAAG,CAACE,CAAC,CAAC,CAACX,MAAM,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE;QAC3D;MACF;MACAY,CAAC,GAAG,KAAK;MACT;IACF;IACA,IAAIA,CAAC,EAAE,OAAOD,CAAC;EACjB;EACA,OAAO,CAAC,CAAC;AACX"}

package/dist/esm/workers/parquet-worker.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parquet-worker.js","names":["createLoaderWorker","ParquetLoader"],"sources":["../../../src/workers/parquet-worker.ts"],"sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ParquetLoader} from '../index';\n\ncreateLoaderWorker(ParquetLoader);\n"],"mappings":"AAAA,SAAQA,kBAAkB,QAAO,0BAA0B;AAC3D,SAAQC,aAAa,QAAO,UAAU;AAEtCD,kBAAkB,CAACC,aAAa,CAAC"}