@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.2

package/dist/es5/parquetjs/parser/parquet-reader.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-reader.js","names":["Symbol","asyncIterator","ParquetReader","metadata","envelopeReader","version","PARQUET_VERSION","Error","root","schema","decodeSchema","num_children","ParquetSchema","close","columnList","map","x","Array","isArray","ParquetCursor","Number","num_rows","md","key_value_metadata","kv","key","value","getCursor","blob","readFn","start","length","slice","arrayBuffer","Buffer","from","closeFn","size","ParquetEnvelopeReader","readHeader","readFooter","byteLength","buffer","openBuffer"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetCursor} from './parquet-cursor';\nimport {PARQUET_VERSION} from '../../constants';\nimport {decodeSchema} from './decoders';\n\n/**\n * A parquet reader allows retrieving the rows from a parquet file in order.\n * The basic usage is to create a reader and then retrieve a cursor/iterator\n * which allows you to consume row after row until all rows have been read. It is\n * important that you call close() after you are finished reading the file to\n * avoid leaking file descriptors.\n */\nexport class ParquetReader<T> implements AsyncIterable<T> {\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openBlob<T>(blob: Blob): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => {\n      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();\n      return Buffer.from(arrayBuffer);\n    };\n    const closeFn = async () => {};\n    const size = blob.size;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  /**\n   * return a new parquet reader initialized with a read function\n   */\n  static async openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>> {\n    const readFn = async (start: number, length: number) => Buffer.from(arrayBuffer, start, length);\n    const closeFn = async () => {};\n    const size = arrayBuffer.byteLength;\n    const envelopeReader = new ParquetEnvelopeReader(readFn, closeFn, size);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {\n    const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);\n    try {\n      await envelopeReader.readHeader();\n      const metadata = await envelopeReader.readFooter();\n      return new ParquetReader<T>(metadata, envelopeReader);\n    } catch (err) {\n      await envelopeReader.close();\n      throw err;\n    }\n  }\n\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is not recommended to call this constructor directly except for advanced\n   * and internal use cases. Consider using one of the open{File,Buffer} methods\n   * instead\n   */\n  constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader) {\n    if (metadata.version !== PARQUET_VERSION) {\n      throw new Error('invalid parquet version');\n    }\n\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    const root = this.metadata.schema[0];\n    const {schema} = decodeSchema(this.metadata.schema, 1, root.num_children!);\n    this.schema = new ParquetSchema(schema);\n  }\n\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  async close(): Promise<void> {\n    await this.envelopeReader.close();\n    // this.envelopeReader = null;\n    // this.metadata = null;\n  }\n\n  /**\n   * Return a cursor to the file. You may open more than one cursor and use\n   * them concurrently. All cursors become invalid once close() is called on\n   * the reader object.\n   *\n   * The required_columns parameter controls which columns are actually read\n   * from disk. An empty array or no value implies all columns. A list of column\n   * names means that only those columns should be loaded from disk.\n   */\n  getCursor(): ParquetCursor<T>;\n  // @ts-ignore\n  getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;\n  getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;\n  getCursor(columnList?: (string | string[])[]): ParquetCursor<Partial<T>> {\n    if (!columnList) {\n      // tslint:disable-next-line:no-parameter-reassignment\n      columnList = [];\n    }\n\n    // tslint:disable-next-line:no-parameter-reassignment\n    columnList = columnList.map((x) => (Array.isArray(x) ? x : [x]));\n\n    return new ParquetCursor<T>(\n      this.metadata,\n      this.envelopeReader,\n      this.schema,\n      columnList as string[][]\n    );\n  }\n\n  /**\n   * Return the number of rows in this file. Note that the number of rows is\n   * not neccessarily equal to the number of rows in each column.\n   */\n  getRowCount(): number {\n    return Number(this.metadata.num_rows);\n  }\n\n  /**\n   * Returns the ParquetSchema for this file\n   */\n  getSchema(): ParquetSchema {\n    return this.schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   */\n  getMetadata(): Record<string, string> {\n    const md: Record<string, string> = {};\n    for (const kv of this.metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    return this.getCursor()[Symbol.asyncIterator]();\n  }\n}\n"],"mappings":";;;;;;;;;;;;AACA;AAEA;AACA;AACA;AACA;AAAwC;AAAA;AAAA;AAAA;AAAA,wBAyJrCA,MAAM,CAACC,aAAa;AAAA,IAhJVC,aAAa;EA8DxB,uBAAYC,QAAsB,EAAEC,cAAqC,EAAE;IAAA;IAAA;IAAA;IAAA;IACzE,IAAID,QAAQ,CAACE,OAAO,KAAKC,0BAAe,EAAE;MACxC,MAAM,IAAIC,KAAK,CAAC,yBAAyB,CAAC;IAC5C;IAEA,IAAI,CAACJ,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACC,cAAc,GAAGA,cAAc;IACpC,IAAMI,IAAI,GAAG,IAAI,CAACL,QAAQ,CAACM,MAAM,CAAC,CAAC,CAAC;IACpC,oBAAiB,IAAAC,sBAAY,EAAC,IAAI,CAACP,QAAQ,CAACM,MAAM,EAAE,CAAC,EAAED,IAAI,CAACG,YAAY,CAAE;MAAnEF,MAAM,iBAANA,MAAM;IACb,IAAI,CAACA,MAAM,GAAG,IAAIG,qBAAa,CAACH,MAAM,CAAC;EACzC;;EAAC;IAAA;IAAA;MAAA,uEAMD;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACQ,IAAI,CAACL,cAAc,CAACS,KAAK,EAAE;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAGlC;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA,OAeD,mBAAUC,UAAkC,EAA6B;MACvE,IAAI,CAACA,UAAU,EAAE;QAEfA,UAAU,GAAG,EAAE;MACjB;;MAGAA,UAAU,GAAGA,UAAU,CAACC,GAAG,CAAC,UAACC,CAAC;QAAA,OAAMC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC;MAAA,CAAC,CAAC;MAEhE,OAAO,IAAIG,4BAAa,CACtB,IAAI,CAAChB,QAAQ,EACb,IAAI,CAACC,cAAc,EACnB,IAAI,CAACK,MAAM,EACXK,UAAU,CACX;IACH;;EAAC;IAAA;IAAA;IAMD,uBAAsB;MACpB,OAAOM,MAAM,CAAC,IAAI,CAACjB,QAAQ,CAACkB,QAAQ,CAAC;IACvC;;EAAC;IAAA;IAAA;IAKD,qBAA2B;MACzB,OAAO,IAAI,CAACZ,MAAM;IACpB;;EAAC;IAAA;IAAA;IAKD,uBAAsC;MACpC,IAAMa,EAA0B,GAAG,CAAC,CAAC;MAAC,2CACrB,IAAI,CAACnB,QAAQ,CAACoB,kBAAkB;QAAA;MAAA;QAAjD,oDAAoD;UAAA,IAAzCC,EAAE;UACXF,EAAE,CAACE,EAAE,CAACC,GAAG,CAAC,GAAGD,EAAE,CAACE,KAAM;QACxB;MAAC;QAAA;MAAA;QAAA;MAAA;MACD,OAAOJ,EAAE;IACX;;EAAC;IAAA;IAAA;IAMD,iBAA2C;MACzC,OAAO,IAAI,CAACK,SAAS,EAAE,CAAC3B,MAAM,CAACC,aAAa,CAAC,EAAE;IACjD;EAAC;IAAA;IAAA;MAAA,0EA9ID,kBAAyB2B,IAAU;QAAA;QAAA;UAAA;YAAA;cAAA;gBAC3BC,MAAM;kBAAA,qEAAG,kBAAOC,KAAa,EAAEC,MAAc;oBAAA;oBAAA;sBAAA;wBAAA;0BAAA;4BAAA;4BAAA,OACvBH,IAAI,CAACI,KAAK,CAACF,KAAK,EAAEA,KAAK,GAAGC,MAAM,CAAC,CAACE,WAAW,EAAE;0BAAA;4BAAnEA,WAAW;4BAAA,kCACVC,MAAM,CAACC,IAAI,CAACF,WAAW,CAAC;0BAAA;0BAAA;4BAAA;wBAAA;sBAAA;oBAAA;kBAAA,CAChC;kBAAA,gBAHKJ,MAAM;oBAAA;kBAAA;gBAAA;gBAINO,OAAO;kBAAA,sEAAG;oBAAA;sBAAA;wBAAA;0BAAA;0BAAA;4BAAA;wBAAA;sBAAA;oBAAA;kBAAA,CAAc;kBAAA,gBAAxBA,OAAO;oBAAA;kBAAA;gBAAA;gBACPC,IAAI,GAAGT,IAAI,CAACS,IAAI;gBAChBjC,cAAc,GAAG,IAAIkC,4CAAqB,CAACT,MAAM,EAAEO,OAAO,EAAEC,IAAI,CAAC;gBAAA;gBAAA;gBAAA,OAE/DjC,cAAc,CAACmC,UAAU,EAAE;cAAA;gBAAA;gBAAA,OACVnC,cAAc,CAACoC,UAAU,EAAE;cAAA;gBAA5CrC,QAAQ;gBAAA,kCACP,IAAID,aAAa,CAACC,QAAQ,EAAEC,cAAc,CAAC;cAAA;gBAAA;gBAAA;gBAAA;gBAAA,OAE5CA,cAAc,CAACS,KAAK,EAAE;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAG/B;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,iFAKD,kBAAgCoB,WAAwB;QAAA;QAAA;UAAA;YAAA;cAAA;gBAChDJ,MAAM;kBAAA,sEAAG,kBAAOC,KAAa,EAAEC,MAAc;oBAAA;sBAAA;wBAAA;0BAAA;4BAAA,kCAAKG,MAAM,CAACC,IAAI,CAACF,WAAW,EAAEH,KAAK,EAAEC,MAAM,CAAC;0BAAA;0BAAA;4BAAA;wBAAA;sBAAA;oBAAA;kBAAA;kBAAA,gBAAzFF,MAAM;oBAAA;kBAAA;gBAAA;gBACNO,OAAO;kBAAA,sEAAG;oBAAA;sBAAA;wBAAA;0BAAA;0BAAA;4BAAA;wBAAA;sBAAA;oBAAA;kBAAA,CAAc;kBAAA,gBAAxBA,OAAO;oBAAA;kBAAA;gBAAA;gBACPC,IAAI,GAAGJ,WAAW,CAACQ,UAAU;gBAC7BrC,cAAc,GAAG,IAAIkC,4CAAqB,CAACT,MAAM,EAAEO,OAAO,EAAEC,IAAI,CAAC;gBAAA;gBAAA;gBAAA,OAE/DjC,cAAc,CAACmC,UAAU,EAAE;cAAA;gBAAA;gBAAA,OACVnC,cAAc,CAACoC,UAAU,EAAE;cAAA;gBAA5CrC,QAAQ;gBAAA,kCACP,IAAID,aAAa,CAACC,QAAQ,EAAEC,cAAc,CAAC;cAAA;gBAAA;gBAAA;gBAAA;gBAAA,OAE5CA,cAAc,CAACS,KAAK,EAAE;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAG/B;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,4EAED,kBAA2B6B,MAAc;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACVJ,4CAAqB,CAACK,UAAU,CAACD,MAAM,CAAC;cAAA;gBAA/DtC,cAAc;gBAAA;gBAAA;gBAAA,OAEZA,cAAc,CAACmC,UAAU,EAAE;cAAA;gBAAA;gBAAA,OACVnC,cAAc,CAACoC,UAAU,EAAE;cAAA;gBAA5CrC,QAAQ;gBAAA,kCACP,IAAID,aAAa,CAAIC,QAAQ,EAAEC,cAAc,CAAC;cAAA;gBAAA;gBAAA;gBAAA;gBAAA,OAE/CA,cAAc,CAACS,KAAK,EAAE;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAG/B;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;EAAA;AAAA;AAAA"}
+{"version":3,"file":"parquet-reader.js","names":["DEFAULT_PROPS","defaultDictionarySize","ParquetReader","file","props","close","rowBatchIterator","rows","row","getSchema","schema","rowGroupIterator","rowGroup","materializeRecords","columnList","map","x","Array","isArray","getFileMetadata","metadata","rowGroupCount","row_groups","length","rowGroupIndex","readRowGroup","Number","num_rows","root","decodeSchema","num_children","schemaDefinition","ParquetSchema","md","key_value_metadata","kv","key","value","readHeader","readFooter","read","PARQUET_MAGIC","buffer","magic","toString","PARQUET_MAGIC_ENCRYPTED","Error","trailerLen","size","trailerBuf","slice","metadataSize","readUInt32LE","metadataOffset","metadataBuf","decodeFileMetadata","rowCount","columnData","columns","colChunk","colMetadata","meta_data","colKey","path_in_schema","fieldIndexOf","readColumnChunk","join","file_path","undefined","field","findField","type","getThriftEnum","Type","primitiveType","compression","CompressionCodec","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","options","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","decodeDataPages","dictionarySize","cursor","offset","decodePage","decodedPage"],"sources":["../../../../src/parquetjs/parser/parquet-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\n\nimport {ParquetSchema} from '../schema/schema';\nimport {decodeSchema} from './decoders';\nimport {materializeRecords} from '../schema/shred';\n\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nexport type ParquetReaderProps = {\n  defaultDictionarySize?: number;\n};\n\n/** Properties for initializing a ParquetRowGroupReader */\nexport type ParquetIterationProps = {\n  /** Filter allowing some columns to be dropped */\n  columnList?: string[] | string[][];\n};\n\nconst DEFAULT_PROPS: Required<ParquetReaderProps> = {\n  defaultDictionarySize: 1e6\n};\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetReader {\n  props: Required<ParquetReaderProps>;\n  file: ReadableFile;\n  metadata: Promise<FileMetaData> | null = null;\n\n  constructor(file: ReadableFile, props?: ParquetReaderProps) {\n    this.file = file;\n    this.props = {...DEFAULT_PROPS, ...props};\n  }\n\n  close(): void {\n    // eslint-disable-next-line @typescript-eslint/no-floating-promises\n    this.file.close();\n  }\n\n  // HIGH LEVEL METHODS\n\n  /** Yield one row at a time */\n  async *rowIterator(props?: ParquetIterationProps) {\n    for await (const rows of this.rowBatchIterator(props)) {\n      // yield *rows\n      for (const row of rows) {\n        yield row;\n      }\n    }\n  }\n\n  /** Yield one batch of rows at a time */\n  async *rowBatchIterator(props?: ParquetIterationProps) {\n    const schema = await this.getSchema();\n    for await (const rowGroup of this.rowGroupIterator(props)) {\n      yield materializeRecords(schema, rowGroup);\n    }\n  }\n\n  /** Iterate over the raw row groups */\n  async *rowGroupIterator(props?: ParquetIterationProps) {\n    // Ensure strings are nested in arrays\n    const columnList: string[][] = (props?.columnList || []).map((x) =>\n      Array.isArray(x) ? x : [x]\n    );\n\n    const metadata = await this.getFileMetadata();\n    const schema = await this.getSchema();\n\n    const rowGroupCount = metadata?.row_groups.length || 0;\n\n    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {\n      const rowGroup = await this.readRowGroup(\n        schema,\n        metadata.row_groups[rowGroupIndex],\n        columnList\n      );\n      yield rowGroup;\n    }\n  }\n\n  async getRowCount(): Promise<number> {\n    const metadata = await this.getFileMetadata();\n    return Number(metadata.num_rows);\n  }\n\n  async getSchema(): Promise<ParquetSchema> {\n    const metadata = await this.getFileMetadata();\n    const root = metadata.schema[0];\n    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);\n    const schema = new ParquetSchema(schemaDefinition);\n    return schema;\n  }\n\n  /**\n   * Returns the user (key/value) metadata for this file\n   * In parquet this is not stored on the schema like it is in arrow\n   */\n  async getSchemaMetadata(): Promise<Record<string, string>> {\n    const metadata = await this.getFileMetadata();\n    const md: Record<string, string> = {};\n    for (const kv of metadata.key_value_metadata!) {\n      md[kv.key] = kv.value!;\n    }\n    return md;\n  }\n\n  async getFileMetadata(): Promise<FileMetaData> {\n    if (!this.metadata) {\n      await this.readHeader();\n      this.metadata = this.readFooter();\n    }\n    return this.metadata;\n  }\n\n  // LOW LEVEL METHODS\n\n  /** Metadata is stored in the footer */\n  async readHeader(): Promise<void> {\n    const buffer = await this.file.read(0, PARQUET_MAGIC.length);\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  /** Metadata is stored in the footer */\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.file.size - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.file.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n\n  /** Data is stored in row groups (similar to Apache Arrow record batches) */\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Each row group contains column chunks for all the columns.\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.file.size - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.file.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.file.size - dictionaryPageOffset,\n      this.props.defaultDictionarySize\n    );\n    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;AAGA;AACA;AACA;AAEA;AACA;AAQA;AAAoF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAapF,IAAMA,aAA2C,GAAG;EAClDC,qBAAqB,EAAE;AACzB,CAAC;;AAAC,IAQWC,aAAa;EAKxB,uBAAYC,IAAkB,EAAEC,KAA0B,EAAE;IAAA;IAAA;IAAA;IAAA,gDAFnB,IAAI;IAG3C,IAAI,CAACD,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,mCAAOJ,aAAa,GAAKI,KAAK,CAAC;EAC3C;EAAC;IAAA;IAAA,OAED,iBAAc;MAEZ,IAAI,CAACD,IAAI,CAACE,KAAK,EAAE;IACnB;;EAAC;IAAA;IAAA;;IAKD,qBAAmBD,KAA6B,EAAE;MAAA;MAAA;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA;gBAAA;gBAAA,2BACvB,KAAI,CAACE,gBAAgB,CAACF,KAAK,CAAC;cAAA;gBAAA;gBAAA;cAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAApCG,IAAI;gBAAA,wCAEDA,IAAI;gBAAA;gBAAA;cAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAXC,GAAG;gBAAA;gBACZ,OAAMA,GAAG;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAA;cAAA;gBAAA;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA;IAGf;;EAAC;IAAA;IAAA;IAGD,0BAAwBJ,KAA6B,EAAE;MAAA;MAAA;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,0CAChC,MAAI,CAACK,SAAS,EAAE;cAAA;gBAA/BC,MAAM;gBAAA;gBAAA;gBAAA;gBAAA,4BACiB,MAAI,CAACC,gBAAgB,CAACP,KAAK,CAAC;cAAA;gBAAA;gBAAA;cAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAxCQ,QAAQ;gBAAA;gBACvB,OAAM,IAAAC,yBAAkB,EAACH,MAAM,EAAEE,QAAQ,CAAC;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAAA;cAAA;gBAAA;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA;IAE9C;;EAAC;IAAA;IAAA;IAGD,0BAAwBR,KAA6B,EAAE;MAAA;MAAA;QAAA;QAAA;UAAA;YAAA;cAAA;gBAE/CU,UAAsB,GAAG,CAAC,CAAAV,KAAK,aAALA,KAAK,uBAALA,KAAK,CAAEU,UAAU,KAAI,EAAE,EAAEC,GAAG,CAAC,UAACC,CAAC;kBAAA,OAC7DC,KAAK,CAACC,OAAO,CAACF,CAAC,CAAC,GAAGA,CAAC,GAAG,CAACA,CAAC,CAAC;gBAAA,EAC3B;gBAAA;gBAAA,0CAEsB,MAAI,CAACG,eAAe,EAAE;cAAA;gBAAvCC,QAAQ;gBAAA;gBAAA,0CACO,MAAI,CAACX,SAAS,EAAE;cAAA;gBAA/BC,MAAM;gBAENW,aAAa,GAAG,CAAAD,QAAQ,aAARA,QAAQ,uBAARA,QAAQ,CAAEE,UAAU,CAACC,MAAM,KAAI,CAAC;gBAE7CC,aAAa,GAAG,CAAC;cAAA;gBAAA,MAAEA,aAAa,GAAGH,aAAa;kBAAA;kBAAA;gBAAA;gBAAA;gBAAA,0CAChC,MAAI,CAACI,YAAY,CACtCf,MAAM,EACNU,QAAQ,CAACE,UAAU,CAACE,aAAa,CAAC,EAClCV,UAAU,CACX;cAAA;gBAJKF,QAAQ;gBAAA;gBAKd,OAAMA,QAAQ;cAAA;gBAN2CY,aAAa,EAAE;gBAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA;IAQ5E;EAAC;IAAA;IAAA;MAAA,6EAED;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACyB,IAAI,CAACL,eAAe,EAAE;cAAA;gBAAvCC,QAAQ;gBAAA,kCACPM,MAAM,CAACN,QAAQ,CAACO,QAAQ,CAAC;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACjC;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,2EAED;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACyB,IAAI,CAACR,eAAe,EAAE;cAAA;gBAAvCC,QAAQ;gBACRQ,IAAI,GAAGR,QAAQ,CAACV,MAAM,CAAC,CAAC,CAAC;gBAAA,gBACI,IAAAmB,sBAAY,EAACT,QAAQ,CAACV,MAAM,EAAE,CAAC,EAAEkB,IAAI,CAACE,YAAY,CAAE,EAAxEC,gBAAgB,iBAAxBrB,MAAM;gBACPA,MAAM,GAAG,IAAIsB,qBAAa,CAACD,gBAAgB,CAAC;gBAAA,kCAC3CrB,MAAM;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACd;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,mFAMD;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACyB,IAAI,CAACS,eAAe,EAAE;cAAA;gBAAvCC,QAAQ;gBACRa,EAA0B,GAAG,CAAC,CAAC;gBAAA,wCACpBb,QAAQ,CAACc,kBAAkB;gBAAA;kBAA5C,uDAA+C;oBAApCC,EAAE;oBACXF,EAAE,CAACE,EAAE,CAACC,GAAG,CAAC,GAAGD,EAAE,CAACE,KAAM;kBACxB;gBAAC;kBAAA;gBAAA;kBAAA;gBAAA;gBAAA,kCACMJ,EAAE;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACV;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,iFAED;QAAA;UAAA;YAAA;cAAA;gBAAA,IACO,IAAI,CAACb,QAAQ;kBAAA;kBAAA;gBAAA;gBAAA;gBAAA,OACV,IAAI,CAACkB,UAAU,EAAE;cAAA;gBACvB,IAAI,CAAClB,QAAQ,GAAG,IAAI,CAACmB,UAAU,EAAE;cAAC;gBAAA,kCAE7B,IAAI,CAACnB,QAAQ;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACrB;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,4EAKD;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA;gBAAA,OACuB,IAAI,CAACjB,IAAI,CAACqC,IAAI,CAAC,CAAC,EAAEC,wBAAa,CAAClB,MAAM,CAAC;cAAA;gBAAtDmB,MAAM;gBACNC,KAAK,GAAGD,MAAM,CAACE,QAAQ,EAAE;gBAAA,eACvBD,KAAK;gBAAA,kCACNF,wBAAa,wBAEbI,kCAAuB;gBAAA;cAAA;gBAAA;cAAA;gBAAA,MACpB,IAAIC,KAAK,CAAC,sCAAsC,CAAC;cAAA;gBAAA,MAEjD,IAAIA,KAAK,uCAAgCH,KAAK,OAAI;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAE7D;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,4EAGD;QAAA;QAAA;UAAA;YAAA;cAAA;gBACQI,UAAU,GAAGN,wBAAa,CAAClB,MAAM,GAAG,CAAC;gBAAA;gBAAA,OAClB,IAAI,CAACpB,IAAI,CAACqC,IAAI,CAAC,IAAI,CAACrC,IAAI,CAAC6C,IAAI,GAAGD,UAAU,EAAEA,UAAU,CAAC;cAAA;gBAA1EE,UAAU;gBAEVN,KAAK,GAAGM,UAAU,CAACC,KAAK,CAAC,CAAC,CAAC,CAACN,QAAQ,EAAE;gBAAA,MACxCD,KAAK,KAAKF,wBAAa;kBAAA;kBAAA;gBAAA;gBAAA,MACnB,IAAIK,KAAK,6CAAqCH,KAAK,OAAI;cAAA;gBAGzDQ,YAAY,GAAGF,UAAU,CAACG,YAAY,CAAC,CAAC,CAAC;gBACzCC,cAAc,GAAG,IAAI,CAAClD,IAAI,CAAC6C,IAAI,GAAGG,YAAY,GAAGJ,UAAU;gBAAA,MAC7DM,cAAc,GAAGZ,wBAAa,CAAClB,MAAM;kBAAA;kBAAA;gBAAA;gBAAA,MACjC,IAAIuB,KAAK,iCAA0BO,cAAc,EAAG;cAAA;gBAAA;gBAAA,OAGlC,IAAI,CAAClD,IAAI,CAACqC,IAAI,CAACa,cAAc,EAAEF,YAAY,CAAC;cAAA;gBAAhEG,WAAW;gBAAA,sBAGE,IAAAC,6BAAkB,EAACD,WAAW,CAAC,EAA3ClC,QAAQ,uBAARA,QAAQ;gBAAA,kCACRA,QAAQ;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAChB;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,8EAGD,mBACEV,MAAqB,EACrBE,QAAkB,EAClBE,UAAsB;QAAA;QAAA;UAAA;YAAA;cAAA;gBAEhB4B,MAAqB,GAAG;kBAC5Bc,QAAQ,EAAE9B,MAAM,CAACd,QAAQ,CAACe,QAAQ,CAAC;kBACnC8B,UAAU,EAAE,CAAC;gBACf,CAAC;gBAAA,wCACsB7C,QAAQ,CAAC8C,OAAO;gBAAA;gBAAA;cAAA;gBAAA;kBAAA;kBAAA;gBAAA;gBAA5BC,QAAQ;gBACXC,WAAW,GAAGD,QAAQ,CAACE,SAAS;gBAChCC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;gBAAA,MACtCjD,UAAU,CAACS,MAAM,GAAG,CAAC,IAAI,IAAAyC,uBAAY,EAAClD,UAAU,EAAEgD,MAAM,CAAE,GAAG,CAAC;kBAAA;kBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA,OAGxB,IAAI,CAACG,eAAe,CAACvD,MAAM,EAAEiD,QAAQ,CAAC;cAAA;gBAAhFjB,MAAM,CAACe,UAAU,CAACK,MAAM,CAAEI,IAAI,EAAE,CAAC;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA;gBAAA;gBAAA;cAAA;gBAAA,mCAE5BxB,MAAM;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACd;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,iFAKD,mBAAsBhC,MAAqB,EAAEiD,QAAqB;QAAA;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA,MAC5DA,QAAQ,CAACQ,SAAS,KAAKC,SAAS,IAAIT,QAAQ,CAACQ,SAAS,KAAK,IAAI;kBAAA;kBAAA;gBAAA;gBAAA,MAC3D,IAAIrB,KAAK,CAAC,uCAAuC,CAAC;cAAA;gBAGpDuB,KAAK,GAAG3D,MAAM,CAAC4D,SAAS,wBAACX,QAAQ,CAACE,SAAS,wDAAlB,oBAAoBE,cAAc,CAAE;gBAC7DQ,IAAmB,GAAG,IAAAC,wBAAa,EAACC,mBAAI,0BAAEd,QAAQ,CAACE,SAAS,yDAAlB,qBAAoBU,IAAI,CAAE;gBAAA,MAEtEA,IAAI,KAAKF,KAAK,CAACK,aAAa;kBAAA;kBAAA;gBAAA;gBAAA,MACxB,IAAI5B,KAAK,2CAAoCyB,IAAI,EAAG;cAAA;gBAGtDI,WAA+B,GAAG,IAAAH,wBAAa,EACnDI,+BAAgB,0BAChBjB,QAAQ,CAACE,SAAS,yDAAlB,qBAAoBgB,KAAK,CAC1B;gBAEKC,WAAW,GAAGpD,MAAM,yBAACiC,QAAQ,CAACE,SAAS,yDAAlB,qBAAoBkB,gBAAgB,CAAE;gBAC7DC,SAAS,GAAGtD,MAAM,yBAACiC,QAAQ,CAACE,SAAS,yDAAlB,qBAAoBoB,qBAAqB,CAAE;gBAElE,IAAI,CAACtB,QAAQ,CAACQ,SAAS,EAAE;kBACvBa,SAAS,GAAGE,IAAI,CAACC,GAAG,CAClB,IAAI,CAAChF,IAAI,CAAC6C,IAAI,GAAG8B,WAAW,EAC5BpD,MAAM,yBAACiC,QAAQ,CAACE,SAAS,yDAAlB,qBAAoBoB,qBAAqB,CAAC,CAClD;gBACH;gBAEMG,OAAuB,GAAG;kBAC9Bb,IAAI,EAAJA,IAAI;kBACJc,SAAS,EAAEhB,KAAK,CAACgB,SAAS;kBAC1BC,SAAS,EAAEjB,KAAK,CAACiB,SAAS;kBAC1BX,WAAW,EAAXA,WAAW;kBACXY,MAAM,EAAElB,KAAK;kBACbmB,SAAS,0BAAE7B,QAAQ,CAACE,SAAS,yDAAlB,qBAAoB4B,UAAU;kBACzCC,UAAU,EAAE;gBACd,CAAC;gBAIKC,oBAAoB,GAAGhC,QAAQ,aAARA,QAAQ,+CAARA,QAAQ,CAAEE,SAAS,yDAAnB,qBAAqB+B,sBAAsB;gBAAA,KAEpED,oBAAoB;kBAAA;kBAAA;gBAAA;gBAChBE,gBAAgB,GAAGnE,MAAM,CAACiE,oBAAoB,CAAC;gBAAA;gBAAA,OAElC,IAAI,CAACG,aAAa,CAACD,gBAAgB,EAAET,OAAO,EAAEN,WAAW,CAAC;cAAA;gBAA7EY,UAAU;cAAA;gBAGZA,UAAU,GAAG,uBAAAN,OAAO,CAACM,UAAU,gDAAlB,oBAAoBnE,MAAM,GAAG6D,OAAO,CAACM,UAAU,GAAGA,UAAU;gBAAC;gBAAA,OACnD,IAAI,CAACvF,IAAI,CAACqC,IAAI,CAACsC,WAAW,EAAEE,SAAS,CAAC;cAAA;gBAAvDe,QAAQ;gBAAA;gBAAA,OACD,IAAAC,yBAAe,EAACD,QAAQ,kCAAMX,OAAO;kBAAEM,UAAU,EAAVA;gBAAU,GAAE;cAAA;gBAAA;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CACjE;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;IAAA;IAAA;MAAA,+EASD,mBACEC,oBAA4B,EAC5BP,OAAuB,EACvBN,WAAmB;QAAA;QAAA;UAAA;YAAA;cAAA;gBAAA,MAEfa,oBAAoB,KAAK,CAAC;kBAAA;kBAAA;gBAAA;gBAAA,mCAQrB,EAAE;cAAA;gBAGLM,cAAc,GAAGf,IAAI,CAACC,GAAG,CAC7B,IAAI,CAAChF,IAAI,CAAC6C,IAAI,GAAG2C,oBAAoB,EACrC,IAAI,CAACvF,KAAK,CAACH,qBAAqB,CACjC;gBAAA;gBAAA,OACsB,IAAI,CAACE,IAAI,CAACqC,IAAI,CAACmD,oBAAoB,EAAEM,cAAc,CAAC;cAAA;gBAArEF,QAAQ;gBAERG,MAAM,GAAG;kBAACxD,MAAM,EAAEqD,QAAQ;kBAAEI,MAAM,EAAE,CAAC;kBAAEnD,IAAI,EAAE+C,QAAQ,CAACxE;gBAAM,CAAC;gBAAA;gBAAA,OACzC,IAAA6E,oBAAU,EAACF,MAAM,EAAEd,OAAO,CAAC;cAAA;gBAA/CiB,WAAW;gBAAA,mCAEVA,WAAW,CAACX,UAAU;cAAA;cAAA;gBAAA;YAAA;UAAA;QAAA;MAAA,CAC9B;MAAA;QAAA;MAAA;MAAA;IAAA;EAAA;EAAA;AAAA;AAAA"}

package/dist/es5/parquetjs/schema/declare.js

@@ -8,7 +8,9 @@ exports.ParquetBuffer = void 0;
 var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
 var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-var ParquetBuffer = (0, _createClass2.default)(function ParquetBuffer() {
+var ParquetBuffer = (0, _createClass2.default)(
+
+function ParquetBuffer() {
   var rowCount = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 0;
   var columnData = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
   (0, _classCallCheck2.default)(this, ParquetBuffer);

package/dist/es5/parquetjs/schema/declare.js.map

@@ -1 +1 @@
-{"version":3,"file":"declare.js","names":["ParquetBuffer","rowCount","columnData"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  dictionary?: ParquetDictionary;\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\nexport class ParquetBuffer {\n  rowCount: number;\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n"],"mappings":";;;;;;;;;;IAmIaA,aAAa,8BAGxB,yBAAgF;EAAA,IAApEC,QAAgB,uEAAG,CAAC;EAAA,IAAEC,UAAuC,uEAAG,CAAC,CAAC;EAAA;EAAA;EAAA;EAC5E,IAAI,CAACD,QAAQ,GAAGA,QAAQ;EACxB,IAAI,CAACC,UAAU,GAAGA,UAAU;AAC9B,CAAC;AAAA"}
+{"version":3,"file":"declare.js","names":["ParquetBuffer","rowCount","columnData"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\n/** @todo better name, this is an internal type? */\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  /** Actual column chunks */\n  values: any[]; // ArrayLike<any>;\n  count: number;\n  dictionary?: ParquetDictionary;\n  /** The \"raw\" page header from the file */\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\n/** @\n * Holds data for one row group (column chunks) */\nexport class ParquetBuffer {\n  /** Number of rows in this page */\n  rowCount: number;\n\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n\n/** Holds the data for one column chunk */\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n"],"mappings":";;;;;;;;;;IAgIaA,aAAa;;AAKxB,yBAAgF;EAAA,IAApEC,QAAgB,uEAAG,CAAC;EAAA,IAAEC,UAAuC,uEAAG,CAAC,CAAC;EAAA;EAAA;EAAA;EAC5E,IAAI,CAACD,QAAQ,GAAGA,QAAQ;EACxB,IAAI,CAACC,UAAU,GAAGA,UAAU;AAC9B,CAAC;AAAA"}

package/dist/es5/parquetjs/schema/shred.js

@@ -123,22 +123,23 @@ function materializeRecords(schema, buffer) {
     records.push({});
   }
   for (var key in buffer.columnData) {
-    materializeColumn(schema, buffer, key, records);
+    var columnData = buffer.columnData[key];
+    if (columnData.count) {
+      materializeColumn(schema, columnData, key, records);
+    }
   }
   return records;
 }
 
-function materializeColumn(schema, buffer, key, records) {
-  var data = buffer.columnData[key];
-  if (!data.count) return;
+function materializeColumn(schema, columnData, key, records) {
   var field = schema.findField(key);
   var branch = schema.findFieldBranch(key);
 
   var rLevels = new Array(field.rLevelMax + 1).fill(0);
   var vIndex = 0;
-  for (var i = 0; i < data.count; i++) {
-    var dLevel = data.dlevels[i];
-    var rLevel = data.rlevels[i];
+  for (var i = 0; i < columnData.count; i++) {
+    var dLevel = columnData.dlevels[i];
+    var rLevel = columnData.rlevels[i];
     rLevels[rLevel]++;
     rLevels.fill(0, rLevel + 1);
     var rIndex = 0;
@@ -149,20 +150,23 @@ function materializeColumn(schema, buffer, key, records) {
     try {
       for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
         var step = _step3.value;
-        if (step === field) break;
-        if (dLevel < step.dLevelMax) break;
-        if (step.repetitionType === 'REPEATED') {
-          if (!(step.name in record)) {
-            record[step.name] = [];
-          }
-          var _ix = rLevels[rIndex++];
-          while (record[step.name].length <= _ix) {
-            record[step.name].push({});
-          }
-          record = record[step.name][_ix];
-        } else {
-          record[step.name] = record[step.name] || {};
-          record = record[step.name];
+        if (step === field || dLevel < step.dLevelMax) {
+          break;
+        }
+        switch (step.repetitionType) {
+          case 'REPEATED':
+            if (!(step.name in record)) {
+              record[step.name] = [];
+            }
+            var _ix = rLevels[rIndex++];
+            while (record[step.name].length <= _ix) {
+              record[step.name].push({});
+            }
+            record = record[step.name][_ix];
+            break;
+          default:
+            record[step.name] = record[step.name] || {};
+            record = record[step.name];
         }
       }
 
@@ -173,19 +177,21 @@ function materializeColumn(schema, buffer, key, records) {
     }
     if (dLevel === field.dLevelMax) {
       var value = Types.fromPrimitive(
-      field.originalType || field.primitiveType, data.values[vIndex], field);
+      field.originalType || field.primitiveType, columnData.values[vIndex], field);
       vIndex++;
-      if (field.repetitionType === 'REPEATED') {
-        if (!(field.name in record)) {
-          record[field.name] = [];
-        }
-        var ix = rLevels[rIndex];
-        while (record[field.name].length <= ix) {
-          record[field.name].push(null);
-        }
-        record[field.name][ix] = value;
-      } else {
-        record[field.name] = value;
+      switch (field.repetitionType) {
+        case 'REPEATED':
+          if (!(field.name in record)) {
+            record[field.name] = [];
+          }
+          var ix = rLevels[rIndex];
+          while (record[field.name].length <= ix) {
+            record[field.name].push(null);
+          }
+          record[field.name][ix] = value;
+          break;
+        default:
+          record[field.name] = value;
       }
     }
   }

package/dist/es5/parquetjs/schema/shred.js.map

@@ -1 +1 @@
-{"version":3,"file":"shred.js","names":["shredBuffer","schema","columnData","fieldList","field","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","isNested","i","rlvl","rLevelMax","dLevelMax","Types","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n  const columnData: Record<string, ParquetData> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the buffer is as\n * follows:\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the buffer */\n  if (buffer.rowCount === 0) {\n    buffer.rowCount = 1;\n    buffer.columnData = data;\n    return;\n  }\n  buffer.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n    buffer.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: any,\n  data: Record<string, ParquetData>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n  const records: ParquetRecord[] = [];\n  for (let i = 0; i < buffer.rowCount; i++) records.push({});\n  for (const key in buffer.columnData) {\n    materializeColumn(schema, buffer, key, records);\n  }\n  return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n  schema: ParquetSchema,\n  buffer: ParquetBuffer,\n  key: string,\n  records: ParquetRecord[]\n) {\n  const data = buffer.columnData[key];\n  if (!data.count) return;\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < data.count; i++) {\n    const dLevel = data.dlevels[i];\n    const rLevel = data.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = records[rLevels[rIndex++] - 1];\n\n    // Internal nodes\n    for (const step of branch) {\n      if (step === field) break;\n      if (dLevel < step.dLevelMax) break;\n      if (step.repetitionType === 'REPEATED') {\n        if (!(step.name in record)) {\n          // eslint-disable max-depth\n          record[step.name] = [];\n        }\n        const ix = rLevels[rIndex++];\n        while (record[step.name].length <= ix) {\n          // eslint-disable max-depth\n          record[step.name].push({});\n        }\n        record = record[step.name][ix];\n      } else {\n        record[step.name] = record[step.name] || {};\n        record = record[step.name];\n      }\n    }\n\n    // Leaf node\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        data.values[vIndex],\n        field\n      );\n      vIndex++;\n      if (field.repetitionType === 'REPEATED') {\n        if (!(field.name in record)) {\n          // eslint-disable max-depth\n          record[field.name] = [];\n        }\n        const ix = rLevels[rIndex];\n        while (record[field.name].length <= ix) {\n          // eslint-disable max-depth\n          record[field.name].push(null);\n        }\n        record[field.name][ix] = value;\n      } else {\n        record[field.name] = value;\n      }\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAEA;AAEA;AAAiC;AAAA;AAAA;AAAA;AAAA;AAI1B,SAASA,WAAW,CAACC,MAAqB,EAAiB;EAChE,IAAMC,UAAuC,GAAG,CAAC,CAAC;EAAC,2CAC/BD,MAAM,CAACE,SAAS;IAAA;EAAA;IAApC,oDAAsC;MAAA,IAA3BC,KAAK;MACdF,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,GAAG;QACtBC,OAAO,EAAE,EAAE;QACXC,OAAO,EAAE,EAAE;QACXC,MAAM,EAAE,EAAE;QACVC,WAAW,EAAE,EAAE;QACfC,KAAK,EAAE;MACT,CAAC;IACH;EAAC;IAAA;EAAA;IAAA;EAAA;EACD,OAAO;IAACC,QAAQ,EAAE,CAAC;IAAET,UAAU,EAAVA;EAAU,CAAC;AAClC;;AAwBO,SAASU,WAAW,CAACX,MAAqB,EAAEY,MAAW,EAAEC,MAAqB,EAAQ;EAE3F,IAAMC,IAAI,GAAGf,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Cc,iBAAiB,CAACf,MAAM,CAACgB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;;EAGpD,IAAID,MAAM,CAACH,QAAQ,KAAK,CAAC,EAAE;IACzBG,MAAM,CAACH,QAAQ,GAAG,CAAC;IACnBG,MAAM,CAACZ,UAAU,GAAGa,IAAI;IACxB;EACF;EACAD,MAAM,CAACH,QAAQ,IAAI,CAAC;EAAC,4CACDV,MAAM,CAACE,SAAS;IAAA;EAAA;IAApC,uDAAsC;MAAA,IAA3BC,KAAK;MACdc,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,EAAEQ,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,CAAC;MACzFW,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,EAAES,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,CAAC;MACzFY,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACG,MAAM,EAAEO,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACG,MAAM,CAAC;MACvFM,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK,IAAIK,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK;IAC7D;EAAC;IAAA;EAAA;IAAA;EAAA;AACH;;AAGA,SAASM,iBAAiB,CACxBC,MAAoC,EACpCJ,MAAW,EACXE,IAAiC,EACjCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,IAAMC,IAAI,IAAIP,MAAM,EAAE;IACzB,IAAMb,KAAK,GAAGa,MAAM,CAACO,IAAI,CAAC;;IAG1B,IAAIhB,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNT,KAAK,CAACoB,IAAI,IAAIX,MAAM,IACpBA,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,KAAKC,SAAS,IAChCZ,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIX,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAACE,WAAW,KAAKR,KAAK,EAAE;QAC5CV,MAAM,GAAGK,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLhB,MAAM,CAACY,IAAI,CAACP,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,IAAIC,OAAO,CAACf,MAAM,CAAC,IAAIT,KAAK,CAACyB,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,mCAA4B1B,KAAK,CAACoB,IAAI,EAAG;IAC1D;IACA,IAAIhB,MAAM,CAACmB,MAAM,GAAG,CAAC,IAAIvB,KAAK,CAACyB,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,sCAA+B1B,KAAK,CAACoB,IAAI,EAAG;IAC7D;;IAGA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,EAAE;MACvB,IAAIvB,KAAK,CAAC2B,QAAQ,EAAE;QAClBf,iBAAiB,CAACZ,KAAK,CAACa,MAAM,EAAG,IAAI,EAAEF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC9D,CAAC,MAAM;QACLR,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;;IAGA,KAAK,IAAIS,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGxB,MAAM,CAACmB,MAAM,EAAEK,CAAC,EAAE,EAAE;MACtC,IAAMC,IAAI,GAAGD,CAAC,KAAK,CAAC,GAAGV,MAAM,GAAGlB,KAAK,CAAC8B,SAAS;MAC/C,IAAI9B,KAAK,CAAC2B,QAAQ,EAAE;QAClBf,iBAAiB,CAACZ,KAAK,CAACa,MAAM,EAAGT,MAAM,CAACwB,CAAC,CAAC,EAAEjB,IAAI,EAAEkB,IAAI,EAAE7B,KAAK,CAAC+B,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLpB,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACa,IAAI,CAAC;QAClClB,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAAChB,KAAK,CAAC+B,SAAS,CAAC;QAC7CpB,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACG,MAAM,CAACY,IAAI,CACzBgB,KAAK,CAACC,WAAW,CAAEjC,KAAK,CAACkC,YAAY,IAAIlC,KAAK,CAACmC,aAAa,EAAI/B,MAAM,CAACwB,CAAC,CAAC,CAAC,CAC3E;MACH;IACF;EACF;AACF;;AAqBO,SAASQ,kBAAkB,CAACvC,MAAqB,EAAEa,MAAqB,EAAmB;EAChG,IAAM2B,OAAwB,GAAG,EAAE;EACnC,KAAK,IAAIT,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGlB,MAAM,CAACH,QAAQ,EAAEqB,CAAC,EAAE;IAAES,OAAO,CAACrB,IAAI,CAAC,CAAC,CAAC,CAAC;EAAC;EAC3D,KAAK,IAAMf,GAAG,IAAIS,MAAM,CAACZ,UAAU,EAAE;IACnCwC,iBAAiB,CAACzC,MAAM,EAAEa,MAAM,EAAET,GAAG,EAAEoC,OAAO,CAAC;EACjD;EACA,OAAOA,OAAO;AAChB;;AAGA,SAASC,iBAAiB,CACxBzC,MAAqB,EACrBa,MAAqB,EACrBT,GAAW,EACXoC,OAAwB,EACxB;EACA,IAAM1B,IAAI,GAAGD,MAAM,CAACZ,UAAU,CAACG,GAAG,CAAC;EACnC,IAAI,CAACU,IAAI,CAACL,KAAK,EAAE;EAEjB,IAAMN,KAAK,GAAGH,MAAM,CAAC0C,SAAS,CAACtC,GAAG,CAAC;EACnC,IAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAe,CAACxC,GAAG,CAAC;;EAG1C,IAAMyC,OAAiB,GAAG,IAAI5B,KAAK,CAACd,KAAK,CAAC8B,SAAS,GAAG,CAAC,CAAC,CAACa,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIhB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGjB,IAAI,CAACL,KAAK,EAAEsB,CAAC,EAAE,EAAE;IACnC,IAAMT,MAAM,GAAGR,IAAI,CAACT,OAAO,CAAC0B,CAAC,CAAC;IAC9B,IAAMV,MAAM,GAAGP,IAAI,CAACR,OAAO,CAACyB,CAAC,CAAC;IAC9Bc,OAAO,CAACxB,MAAM,CAAC,EAAE;IACjBwB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAEzB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI2B,MAAM,GAAG,CAAC;IACd,IAAIpC,MAAM,GAAG4B,OAAO,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;;IAAC,4CAGzBL,MAAM;MAAA;IAAA;MAAzB,uDAA2B;QAAA,IAAhBM,IAAI;QACb,IAAIA,IAAI,KAAK9C,KAAK,EAAE;QACpB,IAAImB,MAAM,GAAG2B,IAAI,CAACf,SAAS,EAAE;QAC7B,IAAIe,IAAI,CAACrB,cAAc,KAAK,UAAU,EAAE;UACtC,IAAI,EAAEqB,IAAI,CAAC1B,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE1BA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAG,EAAE;UACxB;UACA,IAAM2B,GAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;UAC5B,OAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACG,MAAM,IAAIwB,GAAE,EAAE;YAErCtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;UAC5B;UACAP,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAAC2B,GAAE,CAAC;QAChC,CAAC,MAAM;UACLtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAGX,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,IAAI,CAAC,CAAC;UAC3CX,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC;QAC5B;MACF;;IAAC;MAAA;IAAA;MAAA;IAAA;IAGD,IAAID,MAAM,KAAKnB,KAAK,CAAC+B,SAAS,EAAE;MAC9B,IAAMiB,KAAK,GAAGhB,KAAK,CAACiB,aAAa;MAE/BjD,KAAK,CAACkC,YAAY,IAAIlC,KAAK,CAACmC,aAAa,EACzCxB,IAAI,CAACP,MAAM,CAACwC,MAAM,CAAC,EACnB5C,KAAK,CACN;MACD4C,MAAM,EAAE;MACR,IAAI5C,KAAK,CAACyB,cAAc,KAAK,UAAU,EAAE;QACvC,IAAI,EAAEzB,KAAK,CAACoB,IAAI,IAAIX,MAAM,CAAC,EAAE;UAE3BA,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,GAAG,EAAE;QACzB;QACA,IAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;QAC1B,OAAOpC,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;UAEtCtC,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;QAC/B;QACAP,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAAC2B,EAAE,CAAC,GAAGC,KAAK;MAChC,CAAC,MAAM;QACLvC,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,GAAG4B,KAAK;MAC5B;IACF;EACF;AACF"}
+{"version":3,"file":"shred.js","names":["shredBuffer","schema","columnData","fieldList","field","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","isNested","i","rlvl","rLevelMax","dLevelMax","Types","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n  const columnData: Record<string, ParquetData> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the buffer is as\n * follows:\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the buffer */\n  if (buffer.rowCount === 0) {\n    buffer.rowCount = 1;\n    buffer.columnData = data;\n    return;\n  }\n  buffer.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n    buffer.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: any,\n  data: Record<string, ParquetData>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n  const records: ParquetRecord[] = [];\n  for (let i = 0; i < buffer.rowCount; i++) {\n    records.push({});\n  }\n  for (const key in buffer.columnData) {\n    const columnData = buffer.columnData[key];\n    if (columnData.count) {\n      materializeColumn(schema, columnData, key, records);\n    }\n  }\n  return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n  schema: ParquetSchema,\n  columnData: ParquetData,\n  key: string,\n  records: ParquetRecord[]\n): void {\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = records[rLevels[rIndex++] - 1];\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n}\n\n// Columnar export\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n *\nexport function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {\n  const columns: ParquetRecord = {};\n  for (const key in buffer.columnData) {\n    const columnData = buffer.columnData[key];\n    if (columnData.count) {\n      extractColumn(schema, columnData, key, columns);\n    }\n  }\n  return columns;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction extractColumn(\n  schema: ParquetSchema,\n  columnData: ParquetData,\n  key: string,\n  columns: Record<string, unknown> \n) {\n  if (columnData.count <= 0) {\n    return;\n  }\n\n  const record = columns;\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n\n  let i = 0;\n  const dLevel = columnData.dlevels[i];\n  const rLevel = columnData.rlevels[i];\n  rLevels[rLevel]++;\n  rLevels.fill(0, rLevel + 1);\n\n  let rIndex = 0;\n  let record = records[rLevels[rIndex++] - 1];\n\n  // Internal nodes\n  for (const step of branch) {\n    if (step === field || dLevel < step.dLevelMax) {\n      break;\n    }\n\n    switch (step.repetitionType) {\n      case 'REPEATED':\n        if (!(step.name in record)) {\n          // eslint-disable max-depth\n          record[step.name] = [];\n        }\n        const ix = rLevels[rIndex++];\n        while (record[step.name].length <= ix) {\n          // eslint-disable max-depth\n          record[step.name].push({});\n        }\n        record = record[step.name][ix];\n        break;\n\n      default:\n        record[step.name] = record[step.name] || {};\n        record = record[step.name];\n    }\n  }\n\n  // Leaf node\n  if (dLevel === field.dLevelMax) {\n    const value = Types.fromPrimitive(\n      // @ts-ignore\n      field.originalType || field.primitiveType,\n      columnData.values[vIndex],\n      field\n    );\n    vIndex++;\n\n    switch (field.repetitionType) {\n      case 'REPEATED':\n        if (!(field.name in record)) {\n          // eslint-disable max-depth\n          record[field.name] = [];\n        }\n        const ix = rLevels[rIndex];\n        while (record[field.name].length <= ix) {\n          // eslint-disable max-depth\n          record[field.name].push(null);\n        }\n        record[field.name][ix] = value;\n        break;\n\n      default:\n        record[field.name] = value;\n    }\n  }\n}\n*/\n"],"mappings":";;;;;;;;;;;;;;;AAEA;AAEA;AAAiC;AAAA;AAAA;AAAA;AAAA;AAI1B,SAASA,WAAW,CAACC,MAAqB,EAAiB;EAChE,IAAMC,UAAuC,GAAG,CAAC,CAAC;EAAC,2CAC/BD,MAAM,CAACE,SAAS;IAAA;EAAA;IAApC,oDAAsC;MAAA,IAA3BC,KAAK;MACdF,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,GAAG;QACtBC,OAAO,EAAE,EAAE;QACXC,OAAO,EAAE,EAAE;QACXC,MAAM,EAAE,EAAE;QACVC,WAAW,EAAE,EAAE;QACfC,KAAK,EAAE;MACT,CAAC;IACH;EAAC;IAAA;EAAA;IAAA;EAAA;EACD,OAAO;IAACC,QAAQ,EAAE,CAAC;IAAET,UAAU,EAAVA;EAAU,CAAC;AAClC;;AAwBO,SAASU,WAAW,CAACX,MAAqB,EAAEY,MAAW,EAAEC,MAAqB,EAAQ;EAE3F,IAAMC,IAAI,GAAGf,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Cc,iBAAiB,CAACf,MAAM,CAACgB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;;EAGpD,IAAID,MAAM,CAACH,QAAQ,KAAK,CAAC,EAAE;IACzBG,MAAM,CAACH,QAAQ,GAAG,CAAC;IACnBG,MAAM,CAACZ,UAAU,GAAGa,IAAI;IACxB;EACF;EACAD,MAAM,CAACH,QAAQ,IAAI,CAAC;EAAC,4CACDV,MAAM,CAACE,SAAS;IAAA;EAAA;IAApC,uDAAsC;MAAA,IAA3BC,KAAK;MACdc,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,EAAEQ,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,CAAC;MACzFW,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,EAAES,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,CAAC;MACzFY,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACG,MAAM,EAAEO,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACG,MAAM,CAAC;MACvFM,MAAM,CAACZ,UAAU,CAACE,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK,IAAIK,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK;IAC7D;EAAC;IAAA;EAAA;IAAA;EAAA;AACH;;AAGA,SAASM,iBAAiB,CACxBC,MAAoC,EACpCJ,MAAW,EACXE,IAAiC,EACjCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,IAAMC,IAAI,IAAIP,MAAM,EAAE;IACzB,IAAMb,KAAK,GAAGa,MAAM,CAACO,IAAI,CAAC;;IAG1B,IAAIhB,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNT,KAAK,CAACoB,IAAI,IAAIX,MAAM,IACpBA,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,KAAKC,SAAS,IAChCZ,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIX,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAACE,WAAW,KAAKR,KAAK,EAAE;QAC5CV,MAAM,GAAGK,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLhB,MAAM,CAACY,IAAI,CAACP,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,IAAIC,OAAO,CAACf,MAAM,CAAC,IAAIT,KAAK,CAACyB,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,mCAA4B1B,KAAK,CAACoB,IAAI,EAAG;IAC1D;IACA,IAAIhB,MAAM,CAACmB,MAAM,GAAG,CAAC,IAAIvB,KAAK,CAACyB,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,sCAA+B1B,KAAK,CAACoB,IAAI,EAAG;IAC7D;;IAGA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,EAAE;MACvB,IAAIvB,KAAK,CAAC2B,QAAQ,EAAE;QAClBf,iBAAiB,CAACZ,KAAK,CAACa,MAAM,EAAG,IAAI,EAAEF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC9D,CAAC,MAAM;QACLR,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;;IAGA,KAAK,IAAIS,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGxB,MAAM,CAACmB,MAAM,EAAEK,CAAC,EAAE,EAAE;MACtC,IAAMC,IAAI,GAAGD,CAAC,KAAK,CAAC,GAAGV,MAAM,GAAGlB,KAAK,CAAC8B,SAAS;MAC/C,IAAI9B,KAAK,CAAC2B,QAAQ,EAAE;QAClBf,iBAAiB,CAACZ,KAAK,CAACa,MAAM,EAAGT,MAAM,CAACwB,CAAC,CAAC,EAAEjB,IAAI,EAAEkB,IAAI,EAAE7B,KAAK,CAAC+B,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLpB,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACa,IAAI,CAAC;QAClClB,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAAChB,KAAK,CAAC+B,SAAS,CAAC;QAC7CpB,IAAI,CAACX,KAAK,CAACC,GAAG,CAAC,CAACG,MAAM,CAACY,IAAI,CACzBgB,KAAK,CAACC,WAAW,CAAEjC,KAAK,CAACkC,YAAY,IAAIlC,KAAK,CAACmC,aAAa,EAAI/B,MAAM,CAACwB,CAAC,CAAC,CAAC,CAC3E;MACH;IACF;EACF;AACF;;AAqBO,SAASQ,kBAAkB,CAACvC,MAAqB,EAAEa,MAAqB,EAAmB;EAChG,IAAM2B,OAAwB,GAAG,EAAE;EACnC,KAAK,IAAIT,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGlB,MAAM,CAACH,QAAQ,EAAEqB,CAAC,EAAE,EAAE;IACxCS,OAAO,CAACrB,IAAI,CAAC,CAAC,CAAC,CAAC;EAClB;EACA,KAAK,IAAMf,GAAG,IAAIS,MAAM,CAACZ,UAAU,EAAE;IACnC,IAAMA,UAAU,GAAGY,MAAM,CAACZ,UAAU,CAACG,GAAG,CAAC;IACzC,IAAIH,UAAU,CAACQ,KAAK,EAAE;MACpBgC,iBAAiB,CAACzC,MAAM,EAAEC,UAAU,EAAEG,GAAG,EAAEoC,OAAO,CAAC;IACrD;EACF;EACA,OAAOA,OAAO;AAChB;;AAGA,SAASC,iBAAiB,CACxBzC,MAAqB,EACrBC,UAAuB,EACvBG,GAAW,EACXoC,OAAwB,EAClB;EACN,IAAMrC,KAAK,GAAGH,MAAM,CAAC0C,SAAS,CAACtC,GAAG,CAAC;EACnC,IAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAe,CAACxC,GAAG,CAAC;;EAG1C,IAAMyC,OAAiB,GAAG,IAAI5B,KAAK,CAACd,KAAK,CAAC8B,SAAS,GAAG,CAAC,CAAC,CAACa,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIhB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG9B,UAAU,CAACQ,KAAK,EAAEsB,CAAC,EAAE,EAAE;IACzC,IAAMT,MAAM,GAAGrB,UAAU,CAACI,OAAO,CAAC0B,CAAC,CAAC;IACpC,IAAMV,MAAM,GAAGpB,UAAU,CAACK,OAAO,CAACyB,CAAC,CAAC;IACpCc,OAAO,CAACxB,MAAM,CAAC,EAAE;IACjBwB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAEzB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI2B,MAAM,GAAG,CAAC;IACd,IAAIpC,MAAM,GAAG4B,OAAO,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;;IAAC,4CAGzBL,MAAM;MAAA;IAAA;MAAzB,uDAA2B;QAAA,IAAhBM,IAAI;QACb,IAAIA,IAAI,KAAK9C,KAAK,IAAImB,MAAM,GAAG2B,IAAI,CAACf,SAAS,EAAE;UAC7C;QACF;QAEA,QAAQe,IAAI,CAACrB,cAAc;UACzB,KAAK,UAAU;YACb,IAAI,EAAEqB,IAAI,CAAC1B,IAAI,IAAIX,MAAM,CAAC,EAAE;cAE1BA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAG,EAAE;YACxB;YACA,IAAM2B,GAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;YAC5B,OAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACG,MAAM,IAAIwB,GAAE,EAAE;cAErCtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B;YACAP,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAAC2B,GAAE,CAAC;YAC9B;UAEF;YACEtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAGX,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3CX,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC;QAAC;MAEjC;;IAAC;MAAA;IAAA;MAAA;IAAA;IAGD,IAAID,MAAM,KAAKnB,KAAK,CAAC+B,SAAS,EAAE;MAC9B,IAAMiB,KAAK,GAAGhB,KAAK,CAACiB,aAAa;MAE/BjD,KAAK,CAACkC,YAAY,IAAIlC,KAAK,CAACmC,aAAa,EACzCrC,UAAU,CAACM,MAAM,CAACwC,MAAM,CAAC,EACzB5C,KAAK,CACN;MACD4C,MAAM,EAAE;MAER,QAAQ5C,KAAK,CAACyB,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAEzB,KAAK,CAACoB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,IAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOpC,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAEtCtC,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,CAAC2B,EAAE,CAAC,GAAGC,KAAK;UAC9B;QAEF;UACEvC,MAAM,CAACT,KAAK,CAACoB,IAAI,CAAC,GAAG4B,KAAK;MAAC;IAEjC;EACF;AACF"}

package/dist/es5/parquetjs/schema/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","names":["PARQUET_LOGICAL_TYPES","BOOLEAN","primitiveType","toPrimitive","toPrimitive_BOOLEAN","fromPrimitive","fromPrimitive_BOOLEAN","INT32","toPrimitive_INT32","INT64","toPrimitive_INT64","INT96","toPrimitive_INT96","FLOAT","toPrimitive_FLOAT","DOUBLE","toPrimitive_DOUBLE","BYTE_ARRAY","toPrimitive_BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","originalType","toPrimitive_UTF8","fromPrimitive_UTF8","TIME_MILLIS","toPrimitive_TIME_MILLIS","TIME_MICROS","toPrimitive_TIME_MICROS","DATE","toPrimitive_DATE","fromPrimitive_DATE","TIMESTAMP_MILLIS","toPrimitive_TIMESTAMP_MILLIS","fromPrimitive_TIMESTAMP_MILLIS","TIMESTAMP_MICROS","toPrimitive_TIMESTAMP_MICROS","fromPrimitive_TIMESTAMP_MICROS","UINT_8","toPrimitive_UINT8","UINT_16","toPrimitive_UINT16","UINT_32","toPrimitive_UINT32","UINT_64","toPrimitive_UINT64","INT_8","toPrimitive_INT8","INT_16","toPrimitive_INT16","INT_32","INT_64","JSON","toPrimitive_JSON","fromPrimitive_JSON","BSON","toPrimitive_BSON","fromPrimitive_BSON","INTERVAL","typeLength","toPrimitive_INTERVAL","fromPrimitive_INTERVAL","DECIMAL_INT32","decimalToPrimitive_INT32","decimalFromPrimitive_INT","DECIMAL_INT64","decimalToPrimitive_INT64","DECIMAL_BYTE_ARRAY","decimalToPrimitive_BYTE_ARRAY","decimalFromPrimitive_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","type","value","field","Error","Boolean","v","parseFloat","isNaN","parseInt","primitiveValue","scale","Math","round","presision","Buffer","from","toString","stringify","parse","serialize","deserialize","kMillisPerDay","Date","getTime","months","days","milliseconds","buf","alloc","writeUInt32LE","readUInt32LE","millis","presisionInt","number","length","i","component"],"sources":["../../../../src/parquetjs/schema/types.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport BSON from 'bson';\nimport {OriginalType, ParquetField, ParquetType, PrimitiveType} from './declare';\n\nexport interface ParquetTypeKit {\n  primitiveType: PrimitiveType;\n  originalType?: OriginalType;\n  typeLength?: number;\n  toPrimitive: Function;\n  fromPrimitive?: Function;\n}\n\nexport const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {\n  BOOLEAN: {\n    primitiveType: 'BOOLEAN',\n    toPrimitive: toPrimitive_BOOLEAN,\n    fromPrimitive: fromPrimitive_BOOLEAN\n  },\n  INT32: {\n    primitiveType: 'INT32',\n    toPrimitive: toPrimitive_INT32\n  },\n  INT64: {\n    primitiveType: 'INT64',\n    toPrimitive: toPrimitive_INT64\n  },\n  INT96: {\n    primitiveType: 'INT96',\n    toPrimitive: toPrimitive_INT96\n  },\n  FLOAT: {\n    primitiveType: 'FLOAT',\n    toPrimitive: toPrimitive_FLOAT\n  },\n  DOUBLE: {\n    primitiveType: 'DOUBLE',\n    toPrimitive: toPrimitive_DOUBLE\n  },\n  BYTE_ARRAY: {\n    primitiveType: 'BYTE_ARRAY',\n    toPrimitive: toPrimitive_BYTE_ARRAY\n  },\n  FIXED_LEN_BYTE_ARRAY: {\n    primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n    toPrimitive: toPrimitive_BYTE_ARRAY\n  },\n  UTF8: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'UTF8',\n    toPrimitive: toPrimitive_UTF8,\n    fromPrimitive: fromPrimitive_UTF8\n  },\n  TIME_MILLIS: {\n    primitiveType: 'INT32',\n    originalType: 'TIME_MILLIS',\n    toPrimitive: toPrimitive_TIME_MILLIS\n  },\n  TIME_MICROS: {\n    primitiveType: 'INT64',\n    originalType: 'TIME_MICROS',\n    toPrimitive: toPrimitive_TIME_MICROS\n  },\n  DATE: {\n    primitiveType: 'INT32',\n    originalType: 'DATE',\n    toPrimitive: toPrimitive_DATE,\n    fromPrimitive: fromPrimitive_DATE\n  },\n  TIMESTAMP_MILLIS: {\n    primitiveType: 'INT64',\n    originalType: 'TIMESTAMP_MILLIS',\n    toPrimitive: toPrimitive_TIMESTAMP_MILLIS,\n    fromPrimitive: fromPrimitive_TIMESTAMP_MILLIS\n  },\n  TIMESTAMP_MICROS: {\n    primitiveType: 'INT64',\n    originalType: 'TIMESTAMP_MICROS',\n    toPrimitive: toPrimitive_TIMESTAMP_MICROS,\n    fromPrimitive: fromPrimitive_TIMESTAMP_MICROS\n  },\n  UINT_8: {\n    primitiveType: 'INT32',\n    originalType: 'UINT_8',\n    toPrimitive: toPrimitive_UINT8\n  },\n  UINT_16: {\n    primitiveType: 'INT32',\n    originalType: 'UINT_16',\n    toPrimitive: toPrimitive_UINT16\n  },\n  UINT_32: {\n    primitiveType: 'INT32',\n    originalType: 'UINT_32',\n    toPrimitive: toPrimitive_UINT32\n  },\n  UINT_64: {\n    primitiveType: 'INT64',\n    originalType: 'UINT_64',\n    toPrimitive: toPrimitive_UINT64\n  },\n  INT_8: {\n    primitiveType: 'INT32',\n    originalType: 'INT_8',\n    toPrimitive: toPrimitive_INT8\n  },\n  INT_16: {\n    primitiveType: 'INT32',\n    originalType: 'INT_16',\n    toPrimitive: toPrimitive_INT16\n  },\n  INT_32: {\n    primitiveType: 'INT32',\n    originalType: 'INT_32',\n    toPrimitive: toPrimitive_INT32\n  },\n  INT_64: {\n    primitiveType: 'INT64',\n    originalType: 'INT_64',\n    toPrimitive: toPrimitive_INT64\n  },\n  JSON: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'JSON',\n    toPrimitive: toPrimitive_JSON,\n    fromPrimitive: fromPrimitive_JSON\n  },\n  BSON: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'BSON',\n    toPrimitive: toPrimitive_BSON,\n    fromPrimitive: fromPrimitive_BSON\n  },\n  INTERVAL: {\n    primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n    originalType: 'INTERVAL',\n    typeLength: 12,\n    toPrimitive: toPrimitive_INTERVAL,\n    fromPrimitive: fromPrimitive_INTERVAL\n  },\n  DECIMAL_INT32: {\n    primitiveType: 'INT32',\n    originalType: 'DECIMAL_INT32',\n    toPrimitive: decimalToPrimitive_INT32,\n    fromPrimitive: decimalFromPrimitive_INT\n  },\n  DECIMAL_INT64: {\n    primitiveType: 'INT64',\n    originalType: 'DECIMAL_INT64',\n    toPrimitive: decimalToPrimitive_INT64,\n    fromPrimitive: decimalFromPrimitive_INT\n  },\n  DECIMAL_BYTE_ARRAY: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'DECIMAL_BYTE_ARRAY',\n    toPrimitive: decimalToPrimitive_BYTE_ARRAY,\n    fromPrimitive: decimalFromPrimitive_BYTE_ARRAY\n  },\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: {\n    primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n    originalType: 'DECIMAL_FIXED_LEN_BYTE_ARRAY',\n    toPrimitive: decimalToPrimitive_BYTE_ARRAY,\n    fromPrimitive: decimalFromPrimitive_BYTE_ARRAY\n  }\n};\n\n/**\n * Convert a value from it's native representation to the internal/underlying\n * primitive type\n */\nexport function toPrimitive(type: ParquetType, value: any, field?: ParquetField) {\n  if (!(type in PARQUET_LOGICAL_TYPES)) {\n    throw new Error(`invalid type: ${type}`);\n  }\n\n  return PARQUET_LOGICAL_TYPES[type].toPrimitive(value, field);\n}\n\n/**\n * Convert a value from it's internal/underlying primitive representation to\n * the native representation\n */\nexport function fromPrimitive(type: ParquetType, value: any, field?: ParquetField) {\n  if (!(type in PARQUET_LOGICAL_TYPES)) {\n    throw new Error(`invalid type: ${type}`);\n  }\n\n  if ('fromPrimitive' in PARQUET_LOGICAL_TYPES[type]) {\n    return PARQUET_LOGICAL_TYPES[type].fromPrimitive?.(value, field);\n    // tslint:disable-next-line:no-else-after-return\n  }\n  return value;\n}\n\nfunction toPrimitive_BOOLEAN(value: any) {\n  return Boolean(value);\n}\n\nfunction fromPrimitive_BOOLEAN(value: any) {\n  return Boolean(value);\n}\n\nfunction toPrimitive_FLOAT(value: any) {\n  const v = parseFloat(value);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for FLOAT: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_DOUBLE(value: any) {\n  const v = parseFloat(value);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for DOUBLE: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT8(value: any) {\n  const v = parseInt(value, 10);\n  if (v < -0x80 || v > 0x7f || isNaN(v)) {\n    throw new Error(`invalid value for INT8: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_UINT8(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xff || isNaN(v)) {\n    throw new Error(`invalid value for UINT8: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT16(value: any) {\n  const v = parseInt(value, 10);\n  if (v < -0x8000 || v > 0x7fff || isNaN(v)) {\n    throw new Error(`invalid value for INT16: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_UINT16(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xffff || isNaN(v)) {\n    throw new Error(`invalid value for UINT16: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT32(value: any) {\n  const v = parseInt(value, 10);\n  if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {\n    throw new Error(`invalid value for INT32: ${value}`);\n  }\n\n  return v;\n}\n\nfunction decimalToPrimitive_INT32(value: number, field: ParquetField) {\n  const primitiveValue = value * 10 ** (field.scale || 0);\n  const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n  if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {\n    throw new Error(`invalid value for INT32: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_UINT32(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xffffffffffff || isNaN(v)) {\n    throw new Error(`invalid value for UINT32: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT64(value: any) {\n  const v = parseInt(value, 10);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for INT64: ${value}`);\n  }\n\n  return v;\n}\n\nfunction decimalToPrimitive_INT64(value: number, field: ParquetField) {\n  const primitiveValue = value * 10 ** (field.scale || 0);\n  const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for INT64: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_UINT64(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || isNaN(v)) {\n    throw new Error(`invalid value for UINT64: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT96(value: any) {\n  const v = parseInt(value, 10);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for INT96: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_BYTE_ARRAY(value: any) {\n  return Buffer.from(value);\n}\n\nfunction decimalToPrimitive_BYTE_ARRAY(value: any) {\n  // TBD\n  return Buffer.from(value);\n}\n\nfunction toPrimitive_UTF8(value: any) {\n  return Buffer.from(value, 'utf8');\n}\n\nfunction fromPrimitive_UTF8(value: any) {\n  return value.toString();\n}\n\nfunction toPrimitive_JSON(value: any) {\n  return Buffer.from(JSON.stringify(value));\n}\n\nfunction fromPrimitive_JSON(value: any) {\n  return JSON.parse(value);\n}\n\nfunction toPrimitive_BSON(value: any) {\n  return Buffer.from(BSON.serialize(value));\n}\n\nfunction fromPrimitive_BSON(value: any) {\n  return BSON.deserialize(value);\n}\n\nfunction toPrimitive_TIME_MILLIS(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xffffffffffffffff || isNaN(v)) {\n    throw new Error(`invalid value for TIME_MILLIS: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_TIME_MICROS(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || isNaN(v)) {\n    throw new Error(`invalid value for TIME_MICROS: ${value}`);\n  }\n\n  return v;\n}\n\nconst kMillisPerDay = 86400000;\n\nfunction toPrimitive_DATE(value: any) {\n  /* convert from date */\n  if (value instanceof Date) {\n    return value.getTime() / kMillisPerDay;\n  }\n\n  /* convert from integer */\n  {\n    const v = parseInt(value, 10);\n    if (v < 0 || isNaN(v)) {\n      throw new Error(`invalid value for DATE: ${value}`);\n    }\n\n    return v;\n  }\n}\n\nfunction fromPrimitive_DATE(value: any) {\n  return new Date(value * kMillisPerDay);\n}\n\nfunction toPrimitive_TIMESTAMP_MILLIS(value: any) {\n  /* convert from date */\n  if (value instanceof Date) {\n    return value.getTime();\n  }\n\n  /* convert from integer */\n  {\n    const v = parseInt(value, 10);\n    if (v < 0 || isNaN(v)) {\n      throw new Error(`invalid value for TIMESTAMP_MILLIS: ${value}`);\n    }\n\n    return v;\n  }\n}\n\nfunction fromPrimitive_TIMESTAMP_MILLIS(value: any) {\n  return new Date(value);\n}\n\nfunction toPrimitive_TIMESTAMP_MICROS(value: any) {\n  /* convert from date */\n  if (value instanceof Date) {\n    return value.getTime() * 1000;\n  }\n\n  /* convert from integer */\n  {\n    const v = parseInt(value, 10);\n    if (v < 0 || isNaN(v)) {\n      throw new Error(`invalid value for TIMESTAMP_MICROS: ${value}`);\n    }\n\n    return v;\n  }\n}\n\nfunction fromPrimitive_TIMESTAMP_MICROS(value: any) {\n  return new Date(value / 1000);\n}\n\nfunction toPrimitive_INTERVAL(value: any) {\n  if (!value.months || !value.days || !value.milliseconds) {\n    throw new Error(\n      'value for INTERVAL must be object { months: ..., days: ..., milliseconds: ... }'\n    );\n  }\n\n  const buf = Buffer.alloc(12);\n\n  buf.writeUInt32LE(value.months, 0);\n  buf.writeUInt32LE(value.days, 4);\n  buf.writeUInt32LE(value.milliseconds, 8);\n  return buf;\n}\n\nfunction fromPrimitive_INTERVAL(value: any) {\n  const buf = Buffer.from(value);\n  const months = buf.readUInt32LE(0);\n  const days = buf.readUInt32LE(4);\n  const millis = buf.readUInt32LE(8);\n\n  return {months, days, milliseconds: millis};\n}\n\nfunction decimalFromPrimitive_INT(value: any, field: ParquetField) {\n  const presisionInt = Math.round(((value * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n  return presisionInt * 10 ** -(field.scale || 0);\n}\n\nfunction decimalFromPrimitive_BYTE_ARRAY(value: any, field: ParquetField) {\n  let number = 0;\n  if (value.length <= 4) {\n    // Bytewise operators faster. Use them if it is possible\n    for (let i = 0; i < value.length; i++) {\n      // `value.length - i - 1` bytes have reverse order (big-endian)\n      const component = value[i] << (8 * (value.length - i - 1));\n      number += component;\n    }\n  } else {\n    for (let i = 0; i < value.length; i++) {\n      // `value.length - i - 1` bytes have reverse order (big-endian)\n      const component = value[i] * 2 ** (8 * (value.length - 1 - i));\n      number += component;\n    }\n  }\n\n  const presisionInt = Math.round(\n    ((number * 10 ** -field.presision!) % 1) * 10 ** field.presision!\n  );\n  return presisionInt * 10 ** -(field.scale || 0);\n}\n"],"mappings":";;;;;;;;;AAEA;;AAWO,IAAMA,qBAA0D,GAAG;EACxEC,OAAO,EAAE;IACPC,aAAa,EAAE,SAAS;IACxBC,WAAW,EAAEC,mBAAmB;IAChCC,aAAa,EAAEC;EACjB,CAAC;EACDC,KAAK,EAAE;IACLL,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAEK;EACf,CAAC;EACDC,KAAK,EAAE;IACLP,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAEO;EACf,CAAC;EACDC,KAAK,EAAE;IACLT,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAES;EACf,CAAC;EACDC,KAAK,EAAE;IACLX,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAEW;EACf,CAAC;EACDC,MAAM,EAAE;IACNb,aAAa,EAAE,QAAQ;IACvBC,WAAW,EAAEa;EACf,CAAC;EACDC,UAAU,EAAE;IACVf,aAAa,EAAE,YAAY;IAC3BC,WAAW,EAAEe;EACf,CAAC;EACDC,oBAAoB,EAAE;IACpBjB,aAAa,EAAE,sBAAsB;IACrCC,WAAW,EAAEe;EACf,CAAC;EACDE,IAAI,EAAE;IACJlB,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAEmB,gBAAgB;IAC7BjB,aAAa,EAAEkB;EACjB,CAAC;EACDC,WAAW,EAAE;IACXtB,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,aAAa;IAC3BlB,WAAW,EAAEsB;EACf,CAAC;EACDC,WAAW,EAAE;IACXxB,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,aAAa;IAC3BlB,WAAW,EAAEwB;EACf,CAAC;EACDC,IAAI,EAAE;IACJ1B,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAE0B,gBAAgB;IAC7BxB,aAAa,EAAEyB;EACjB,CAAC;EACDC,gBAAgB,EAAE;IAChB7B,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,kBAAkB;IAChClB,WAAW,EAAE6B,4BAA4B;IACzC3B,aAAa,EAAE4B;EACjB,CAAC;EACDC,gBAAgB,EAAE;IAChBhC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,kBAAkB;IAChClB,WAAW,EAAEgC,4BAA4B;IACzC9B,aAAa,EAAE+B;EACjB,CAAC;EACDC,MAAM,EAAE;IACNnC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAEmC;EACf,CAAC;EACDC,OAAO,EAAE;IACPrC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,SAAS;IACvBlB,WAAW,EAAEqC;EACf,CAAC;EACDC,OAAO,EAAE;IACPvC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,SAAS;IACvBlB,WAAW,EAAEuC;EACf,CAAC;EACDC,OAAO,EAAE;IACPzC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,SAAS;IACvBlB,WAAW,EAAEyC;EACf,CAAC;EACDC,KAAK,EAAE;IACL3C,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,OAAO;IACrBlB,WAAW,EAAE2C;EACf,CAAC;EACDC,MAAM,EAAE;IACN7C,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAE6C;EACf,CAAC;EACDC,MAAM,EAAE;IACN/C,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAEK;EACf,CAAC;EACD0C,MAAM,EAAE;IACNhD,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAEO;EACf,CAAC;EACDyC,IAAI,EAAE;IACJjD,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAEiD,gBAAgB;IAC7B/C,aAAa,EAAEgD;EACjB,CAAC;EACDC,IAAI,EAAE;IACJpD,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAEoD,gBAAgB;IAC7BlD,aAAa,EAAEmD;EACjB,CAAC;EACDC,QAAQ,EAAE;IACRvD,aAAa,EAAE,sBAAsB;IACrCmB,YAAY,EAAE,UAAU;IACxBqC,UAAU,EAAE,EAAE;IACdvD,WAAW,EAAEwD,oBAAoB;IACjCtD,aAAa,EAAEuD;EACjB,CAAC;EACDC,aAAa,EAAE;IACb3D,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,eAAe;IAC7BlB,WAAW,EAAE2D,wBAAwB;IACrCzD,aAAa,EAAE0D;EACjB,CAAC;EACDC,aAAa,EAAE;IACb9D,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,eAAe;IAC7BlB,WAAW,EAAE8D,wBAAwB;IACrC5D,aAAa,EAAE0D;EACjB,CAAC;EACDG,kBAAkB,EAAE;IAClBhE,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,oBAAoB;IAClClB,WAAW,EAAEgE,6BAA6B;IAC1C9D,aAAa,EAAE+D;EACjB,CAAC;EACDC,4BAA4B,EAAE;IAC5BnE,aAAa,EAAE,sBAAsB;IACrCmB,YAAY,EAAE,8BAA8B;IAC5ClB,WAAW,EAAEgE,6BAA6B;IAC1C9D,aAAa,EAAE+D;EACjB;AACF,CAAC;;AAAC;AAMK,SAASjE,WAAW,CAACmE,IAAiB,EAAEC,KAAU,EAAEC,KAAoB,EAAE;EAC/E,IAAI,EAAEF,IAAI,IAAItE,qBAAqB,CAAC,EAAE;IACpC,MAAM,IAAIyE,KAAK,yBAAkBH,IAAI,EAAG;EAC1C;EAEA,OAAOtE,qBAAqB,CAACsE,IAAI,CAAC,CAACnE,WAAW,CAACoE,KAAK,EAAEC,KAAK,CAAC;AAC9D;;AAMO,SAASnE,aAAa,CAACiE,IAAiB,EAAEC,KAAU,EAAEC,KAAoB,EAAE;EACjF,IAAI,EAAEF,IAAI,IAAItE,qBAAqB,CAAC,EAAE;IACpC,MAAM,IAAIyE,KAAK,yBAAkBH,IAAI,EAAG;EAC1C;EAEA,IAAI,eAAe,IAAItE,qBAAqB,CAACsE,IAAI,CAAC,EAAE;IAAA;IAClD,gCAAO,0BAAAtE,qBAAqB,CAACsE,IAAI,CAAC,EAACjE,aAAa,0DAAzC,mDAA4CkE,KAAK,EAAEC,KAAK,CAAC;EAElE;;EACA,OAAOD,KAAK;AACd;AAEA,SAASnE,mBAAmB,CAACmE,KAAU,EAAE;EACvC,OAAOG,OAAO,CAACH,KAAK,CAAC;AACvB;AAEA,SAASjE,qBAAqB,CAACiE,KAAU,EAAE;EACzC,OAAOG,OAAO,CAACH,KAAK,CAAC;AACvB;AAEA,SAASzD,iBAAiB,CAACyD,KAAU,EAAE;EACrC,IAAMI,CAAC,GAAGC,UAAU,CAACL,KAAK,CAAC;EAC3B,IAAIM,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAAS3D,kBAAkB,CAACuD,KAAU,EAAE;EACtC,IAAMI,CAAC,GAAGC,UAAU,CAACL,KAAK,CAAC;EAC3B,IAAIM,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIF,KAAK,qCAA8BF,KAAK,EAAG;EACvD;EAEA,OAAOI,CAAC;AACV;AAEA,SAAS7B,gBAAgB,CAACyB,KAAU,EAAE;EACpC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAI,IAAIA,CAAC,GAAG,IAAI,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACrC,MAAM,IAAIF,KAAK,mCAA4BF,KAAK,EAAG;EACrD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASrC,iBAAiB,CAACiC,KAAU,EAAE;EACrC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,IAAI,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACjC,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAAS3B,iBAAiB,CAACuB,KAAU,EAAE;EACrC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,MAAM,IAAIA,CAAC,GAAG,MAAM,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACzC,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASnC,kBAAkB,CAAC+B,KAAU,EAAE;EACtC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,MAAM,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACnC,MAAM,IAAIF,KAAK,qCAA8BF,KAAK,EAAG;EACvD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASnE,iBAAiB,CAAC+D,KAAU,EAAE;EACrC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,UAAU,IAAIA,CAAC,GAAG,UAAU,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACjD,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASb,wBAAwB,CAACS,KAAa,EAAEC,KAAmB,EAAE;EACpE,IAAMO,cAAc,GAAGR,KAAK,YAAG,EAAE,EAAKC,KAAK,CAACQ,KAAK,IAAI,CAAC,CAAC;EACvD,IAAML,CAAC,GAAGM,IAAI,CAACC,KAAK,CAAGH,cAAc,YAAG,EAAE,EAAI,CAACP,KAAK,CAACW,SAAU,IAAI,CAAC,YAAI,EAAE,EAAIX,KAAK,CAACW,SAAS,CAAC,CAAC;EAC/F,IAAIR,CAAC,GAAG,CAAC,UAAU,IAAIA,CAAC,GAAG,UAAU,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACjD,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASjC,kBAAkB,CAAC6B,KAAU,EAAE;EACtC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,cAAc,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IAC3C,MAAM,IAAIF,KAAK,qCAA8BF,KAAK,EAAG;EACvD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASjE,iBAAiB,CAAC6D,KAAU,EAAE;EACrC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIM,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASV,wBAAwB,CAACM,KAAa,EAAEC,KAAmB,EAAE;EACpE,IAAMO,cAAc,GAAGR,KAAK,YAAG,EAAE,EAAKC,KAAK,CAACQ,KAAK,IAAI,CAAC,CAAC;EACvD,IAAML,CAAC,GAAGM,IAAI,CAACC,KAAK,CAAGH,cAAc,YAAG,EAAE,EAAI,CAACP,KAAK,CAACW,SAAU,IAAI,CAAC,YAAI,EAAE,EAAIX,KAAK,CAACW,SAAS,CAAC,CAAC;EAC/F,IAAIN,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAAS/B,kBAAkB,CAAC2B,KAAU,EAAE;EACtC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACrB,MAAM,IAAIF,KAAK,qCAA8BF,KAAK,EAAG;EACvD;EAEA,OAAOI,CAAC;AACV;AAEA,SAAS/D,iBAAiB,CAAC2D,KAAU,EAAE;EACrC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIM,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASzD,sBAAsB,CAACqD,KAAU,EAAE;EAC1C,OAAOa,MAAM,CAACC,IAAI,CAACd,KAAK,CAAC;AAC3B;AAEA,SAASJ,6BAA6B,CAACI,KAAU,EAAE;EAEjD,OAAOa,MAAM,CAACC,IAAI,CAACd,KAAK,CAAC;AAC3B;AAEA,SAASjD,gBAAgB,CAACiD,KAAU,EAAE;EACpC,OAAOa,MAAM,CAACC,IAAI,CAACd,KAAK,EAAE,MAAM,CAAC;AACnC;AAEA,SAAShD,kBAAkB,CAACgD,KAAU,EAAE;EACtC,OAAOA,KAAK,CAACe,QAAQ,EAAE;AACzB;AAEA,SAASlC,gBAAgB,CAACmB,KAAU,EAAE;EACpC,OAAOa,MAAM,CAACC,IAAI,CAAClC,IAAI,CAACoC,SAAS,CAAChB,KAAK,CAAC,CAAC;AAC3C;AAEA,SAASlB,kBAAkB,CAACkB,KAAU,EAAE;EACtC,OAAOpB,IAAI,CAACqC,KAAK,CAACjB,KAAK,CAAC;AAC1B;AAEA,SAAShB,gBAAgB,CAACgB,KAAU,EAAE;EACpC,OAAOa,MAAM,CAACC,IAAI,CAAC/B,aAAI,CAACmC,SAAS,CAAClB,KAAK,CAAC,CAAC;AAC3C;AAEA,SAASf,kBAAkB,CAACe,KAAU,EAAE;EACtC,OAAOjB,aAAI,CAACoC,WAAW,CAACnB,KAAK,CAAC;AAChC;AAEA,SAAS9C,uBAAuB,CAAC8C,KAAU,EAAE;EAC3C,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,kBAAkB,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IAC/C,MAAM,IAAIF,KAAK,0CAAmCF,KAAK,EAAG;EAC5D;EAEA,OAAOI,CAAC;AACV;AAEA,SAAShD,uBAAuB,CAAC4C,KAAU,EAAE;EAC3C,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACrB,MAAM,IAAIF,KAAK,0CAAmCF,KAAK,EAAG;EAC5D;EAEA,OAAOI,CAAC;AACV;AAEA,IAAMgB,aAAa,GAAG,QAAQ;AAE9B,SAAS9D,gBAAgB,CAAC0C,KAAU,EAAE;EAEpC,IAAIA,KAAK,YAAYqB,IAAI,EAAE;IACzB,OAAOrB,KAAK,CAACsB,OAAO,EAAE,GAAGF,aAAa;EACxC;;EAGA;IACE,IAAMhB,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;IAC7B,IAAII,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;MACrB,MAAM,IAAIF,KAAK,mCAA4BF,KAAK,EAAG;IACrD;IAEA,OAAOI,CAAC;EACV;AACF;AAEA,SAAS7C,kBAAkB,CAACyC,KAAU,EAAE;EACtC,OAAO,IAAIqB,IAAI,CAACrB,KAAK,GAAGoB,aAAa,CAAC;AACxC;AAEA,SAAS3D,4BAA4B,CAACuC,KAAU,EAAE;EAEhD,IAAIA,KAAK,YAAYqB,IAAI,EAAE;IACzB,OAAOrB,KAAK,CAACsB,OAAO,EAAE;EACxB;;EAGA;IACE,IAAMlB,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;IAC7B,IAAII,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;MACrB,MAAM,IAAIF,KAAK,+CAAwCF,KAAK,EAAG;IACjE;IAEA,OAAOI,CAAC;EACV;AACF;AAEA,SAAS1C,8BAA8B,CAACsC,KAAU,EAAE;EAClD,OAAO,IAAIqB,IAAI,CAACrB,KAAK,CAAC;AACxB;AAEA,SAASpC,4BAA4B,CAACoC,KAAU,EAAE;EAEhD,IAAIA,KAAK,YAAYqB,IAAI,EAAE;IACzB,OAAOrB,KAAK,CAACsB,OAAO,EAAE,GAAG,IAAI;EAC/B;;EAGA;IACE,IAAMlB,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;IAC7B,IAAII,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;MACrB,MAAM,IAAIF,KAAK,+CAAwCF,KAAK,EAAG;IACjE;IAEA,OAAOI,CAAC;EACV;AACF;AAEA,SAASvC,8BAA8B,CAACmC,KAAU,EAAE;EAClD,OAAO,IAAIqB,IAAI,CAACrB,KAAK,GAAG,IAAI,CAAC;AAC/B;AAEA,SAASZ,oBAAoB,CAACY,KAAU,EAAE;EACxC,IAAI,CAACA,KAAK,CAACuB,MAAM,IAAI,CAACvB,KAAK,CAACwB,IAAI,IAAI,CAACxB,KAAK,CAACyB,YAAY,EAAE;IACvD,MAAM,IAAIvB,KAAK,CACb,iFAAiF,CAClF;EACH;EAEA,IAAMwB,GAAG,GAAGb,MAAM,CAACc,KAAK,CAAC,EAAE,CAAC;EAE5BD,GAAG,CAACE,aAAa,CAAC5B,KAAK,CAACuB,MAAM,EAAE,CAAC,CAAC;EAClCG,GAAG,CAACE,aAAa,CAAC5B,KAAK,CAACwB,IAAI,EAAE,CAAC,CAAC;EAChCE,GAAG,CAACE,aAAa,CAAC5B,KAAK,CAACyB,YAAY,EAAE,CAAC,CAAC;EACxC,OAAOC,GAAG;AACZ;AAEA,SAASrC,sBAAsB,CAACW,KAAU,EAAE;EAC1C,IAAM0B,GAAG,GAAGb,MAAM,CAACC,IAAI,CAACd,KAAK,CAAC;EAC9B,IAAMuB,MAAM,GAAGG,GAAG,CAACG,YAAY,CAAC,CAAC,CAAC;EAClC,IAAML,IAAI,GAAGE,GAAG,CAACG,YAAY,CAAC,CAAC,CAAC;EAChC,IAAMC,MAAM,GAAGJ,GAAG,CAACG,YAAY,CAAC,CAAC,CAAC;EAElC,OAAO;IAACN,MAAM,EAANA,MAAM;IAAEC,IAAI,EAAJA,IAAI;IAAEC,YAAY,EAAEK;EAAM,CAAC;AAC7C;AAEA,SAAStC,wBAAwB,CAACQ,KAAU,EAAEC,KAAmB,EAAE;EACjE,IAAM8B,YAAY,GAAGrB,IAAI,CAACC,KAAK,CAAGX,KAAK,YAAG,EAAE,EAAI,CAACC,KAAK,CAACW,SAAU,IAAI,CAAC,YAAI,EAAE,EAAIX,KAAK,CAACW,SAAS,CAAC,CAAC;EACjG,OAAOmB,YAAY,YAAG,EAAE,EAAI,EAAE9B,KAAK,CAACQ,KAAK,IAAI,CAAC,CAAC;AACjD;AAEA,SAASZ,+BAA+B,CAACG,KAAU,EAAEC,KAAmB,EAAE;EACxE,IAAI+B,MAAM,GAAG,CAAC;EACd,IAAIhC,KAAK,CAACiC,MAAM,IAAI,CAAC,EAAE;IAErB,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGlC,KAAK,CAACiC,MAAM,EAAEC,CAAC,EAAE,EAAE;MAErC,IAAMC,SAAS,GAAGnC,KAAK,CAACkC,CAAC,CAAC,IAAK,CAAC,IAAIlC,KAAK,CAACiC,MAAM,GAAGC,CAAC,GAAG,CAAC,CAAE;MAC1DF,MAAM,IAAIG,SAAS;IACrB;EACF,CAAC,MAAM;IACL,KAAK,IAAID,EAAC,GAAG,CAAC,EAAEA,EAAC,GAAGlC,KAAK,CAACiC,MAAM,EAAEC,EAAC,EAAE,EAAE;MAErC,IAAMC,UAAS,GAAGnC,KAAK,CAACkC,EAAC,CAAC,YAAG,CAAC,EAAK,CAAC,IAAIlC,KAAK,CAACiC,MAAM,GAAG,CAAC,GAAGC,EAAC,CAAC,CAAC;MAC9DF,MAAM,IAAIG,UAAS;IACrB;EACF;EAEA,IAAMJ,YAAY,GAAGrB,IAAI,CAACC,KAAK,CAC3BqB,MAAM,YAAG,EAAE,EAAI,CAAC/B,KAAK,CAACW,SAAU,IAAI,CAAC,YAAI,EAAE,EAAIX,KAAK,CAACW,SAAS,CAAC,CAClE;EACD,OAAOmB,YAAY,YAAG,EAAE,EAAI,EAAE9B,KAAK,CAACQ,KAAK,IAAI,CAAC,CAAC;AACjD"}
+{"version":3,"file":"types.js","names":["PARQUET_LOGICAL_TYPES","BOOLEAN","primitiveType","toPrimitive","toPrimitive_BOOLEAN","fromPrimitive","fromPrimitive_BOOLEAN","INT32","toPrimitive_INT32","INT64","toPrimitive_INT64","INT96","toPrimitive_INT96","FLOAT","toPrimitive_FLOAT","DOUBLE","toPrimitive_DOUBLE","BYTE_ARRAY","toPrimitive_BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","originalType","toPrimitive_UTF8","fromPrimitive_UTF8","TIME_MILLIS","toPrimitive_TIME_MILLIS","TIME_MICROS","toPrimitive_TIME_MICROS","DATE","toPrimitive_DATE","fromPrimitive_DATE","TIMESTAMP_MILLIS","toPrimitive_TIMESTAMP_MILLIS","fromPrimitive_TIMESTAMP_MILLIS","TIMESTAMP_MICROS","toPrimitive_TIMESTAMP_MICROS","fromPrimitive_TIMESTAMP_MICROS","UINT_8","toPrimitive_UINT8","UINT_16","toPrimitive_UINT16","UINT_32","toPrimitive_UINT32","UINT_64","toPrimitive_UINT64","INT_8","toPrimitive_INT8","INT_16","toPrimitive_INT16","INT_32","INT_64","JSON","toPrimitive_JSON","fromPrimitive_JSON","BSON","toPrimitive_BSON","fromPrimitive_BSON","INTERVAL","typeLength","toPrimitive_INTERVAL","fromPrimitive_INTERVAL","DECIMAL_INT32","decimalToPrimitive_INT32","decimalFromPrimitive_INT","DECIMAL_INT64","decimalToPrimitive_INT64","DECIMAL_BYTE_ARRAY","decimalToPrimitive_BYTE_ARRAY","decimalFromPrimitive_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","type","value","field","Error","Boolean","v","parseFloat","isNaN","parseInt","primitiveValue","scale","Math","round","presision","Buffer","from","toString","stringify","parse","serialize","deserialize","kMillisPerDay","Date","getTime","months","days","milliseconds","buf","alloc","writeUInt32LE","readUInt32LE","millis","presisionInt","number","length","i","component"],"sources":["../../../../src/parquetjs/schema/types.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\nimport BSON from 'bson';\nimport {OriginalType, ParquetField, ParquetType, PrimitiveType} from './declare';\n\nexport interface ParquetTypeKit {\n  primitiveType: PrimitiveType;\n  originalType?: OriginalType;\n  typeLength?: number;\n  toPrimitive: Function;\n  fromPrimitive?: Function;\n}\n\nexport const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {\n  BOOLEAN: {\n    primitiveType: 'BOOLEAN',\n    toPrimitive: toPrimitive_BOOLEAN,\n    fromPrimitive: fromPrimitive_BOOLEAN\n  },\n  INT32: {\n    primitiveType: 'INT32',\n    toPrimitive: toPrimitive_INT32\n  },\n  INT64: {\n    primitiveType: 'INT64',\n    toPrimitive: toPrimitive_INT64\n  },\n  INT96: {\n    primitiveType: 'INT96',\n    toPrimitive: toPrimitive_INT96\n  },\n  FLOAT: {\n    primitiveType: 'FLOAT',\n    toPrimitive: toPrimitive_FLOAT\n  },\n  DOUBLE: {\n    primitiveType: 'DOUBLE',\n    toPrimitive: toPrimitive_DOUBLE\n  },\n  BYTE_ARRAY: {\n    primitiveType: 'BYTE_ARRAY',\n    toPrimitive: toPrimitive_BYTE_ARRAY\n  },\n  FIXED_LEN_BYTE_ARRAY: {\n    primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n    toPrimitive: toPrimitive_BYTE_ARRAY\n  },\n  UTF8: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'UTF8',\n    toPrimitive: toPrimitive_UTF8,\n    fromPrimitive: fromPrimitive_UTF8\n  },\n  TIME_MILLIS: {\n    primitiveType: 'INT32',\n    originalType: 'TIME_MILLIS',\n    toPrimitive: toPrimitive_TIME_MILLIS\n  },\n  TIME_MICROS: {\n    primitiveType: 'INT64',\n    originalType: 'TIME_MICROS',\n    toPrimitive: toPrimitive_TIME_MICROS\n  },\n  DATE: {\n    primitiveType: 'INT32',\n    originalType: 'DATE',\n    toPrimitive: toPrimitive_DATE,\n    fromPrimitive: fromPrimitive_DATE\n  },\n  TIMESTAMP_MILLIS: {\n    primitiveType: 'INT64',\n    originalType: 'TIMESTAMP_MILLIS',\n    toPrimitive: toPrimitive_TIMESTAMP_MILLIS,\n    fromPrimitive: fromPrimitive_TIMESTAMP_MILLIS\n  },\n  TIMESTAMP_MICROS: {\n    primitiveType: 'INT64',\n    originalType: 'TIMESTAMP_MICROS',\n    toPrimitive: toPrimitive_TIMESTAMP_MICROS,\n    fromPrimitive: fromPrimitive_TIMESTAMP_MICROS\n  },\n  UINT_8: {\n    primitiveType: 'INT32',\n    originalType: 'UINT_8',\n    toPrimitive: toPrimitive_UINT8\n  },\n  UINT_16: {\n    primitiveType: 'INT32',\n    originalType: 'UINT_16',\n    toPrimitive: toPrimitive_UINT16\n  },\n  UINT_32: {\n    primitiveType: 'INT32',\n    originalType: 'UINT_32',\n    toPrimitive: toPrimitive_UINT32\n  },\n  UINT_64: {\n    primitiveType: 'INT64',\n    originalType: 'UINT_64',\n    toPrimitive: toPrimitive_UINT64\n  },\n  INT_8: {\n    primitiveType: 'INT32',\n    originalType: 'INT_8',\n    toPrimitive: toPrimitive_INT8\n  },\n  INT_16: {\n    primitiveType: 'INT32',\n    originalType: 'INT_16',\n    toPrimitive: toPrimitive_INT16\n  },\n  INT_32: {\n    primitiveType: 'INT32',\n    originalType: 'INT_32',\n    toPrimitive: toPrimitive_INT32\n  },\n  INT_64: {\n    primitiveType: 'INT64',\n    originalType: 'INT_64',\n    toPrimitive: toPrimitive_INT64\n  },\n  JSON: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'JSON',\n    toPrimitive: toPrimitive_JSON,\n    fromPrimitive: fromPrimitive_JSON\n  },\n  BSON: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'BSON',\n    toPrimitive: toPrimitive_BSON,\n    fromPrimitive: fromPrimitive_BSON\n  },\n  INTERVAL: {\n    primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n    originalType: 'INTERVAL',\n    typeLength: 12,\n    toPrimitive: toPrimitive_INTERVAL,\n    fromPrimitive: fromPrimitive_INTERVAL\n  },\n  DECIMAL_INT32: {\n    primitiveType: 'INT32',\n    originalType: 'DECIMAL_INT32',\n    toPrimitive: decimalToPrimitive_INT32,\n    fromPrimitive: decimalFromPrimitive_INT\n  },\n  DECIMAL_INT64: {\n    primitiveType: 'INT64',\n    originalType: 'DECIMAL_INT64',\n    toPrimitive: decimalToPrimitive_INT64,\n    fromPrimitive: decimalFromPrimitive_INT\n  },\n  DECIMAL_BYTE_ARRAY: {\n    primitiveType: 'BYTE_ARRAY',\n    originalType: 'DECIMAL_BYTE_ARRAY',\n    toPrimitive: decimalToPrimitive_BYTE_ARRAY,\n    fromPrimitive: decimalFromPrimitive_BYTE_ARRAY\n  },\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: {\n    primitiveType: 'FIXED_LEN_BYTE_ARRAY',\n    originalType: 'DECIMAL_FIXED_LEN_BYTE_ARRAY',\n    toPrimitive: decimalToPrimitive_BYTE_ARRAY,\n    fromPrimitive: decimalFromPrimitive_BYTE_ARRAY\n  }\n};\n\n/**\n * Convert a value from it's native representation to the internal/underlying\n * primitive type\n */\nexport function toPrimitive(type: ParquetType, value: unknown, field?: ParquetField): unknown {\n  if (!(type in PARQUET_LOGICAL_TYPES)) {\n    throw new Error(`invalid type: ${type}`);\n  }\n\n  return PARQUET_LOGICAL_TYPES[type].toPrimitive(value, field);\n}\n\n/**\n * Convert a value from it's internal/underlying primitive representation to\n * the native representation\n */\nexport function fromPrimitive(type: ParquetType, value: unknown, field?: ParquetField) {\n  if (!(type in PARQUET_LOGICAL_TYPES)) {\n    throw new Error(`invalid type: ${type}`);\n  }\n\n  if ('fromPrimitive' in PARQUET_LOGICAL_TYPES[type]) {\n    return PARQUET_LOGICAL_TYPES[type].fromPrimitive?.(value, field);\n    // tslint:disable-next-line:no-else-after-return\n  }\n  return value;\n}\n\nfunction toPrimitive_BOOLEAN(value: unknown): boolean {\n  return Boolean(value);\n}\n\nfunction fromPrimitive_BOOLEAN(value: any): boolean {\n  return Boolean(value);\n}\n\nfunction toPrimitive_FLOAT(value: any): number {\n  const v = parseFloat(value);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for FLOAT: ${value}`);\n  }\n  return v;\n}\n\nfunction toPrimitive_DOUBLE(value: any): number {\n  const v = parseFloat(value);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for DOUBLE: ${value}`);\n  }\n  return v;\n}\n\nfunction toPrimitive_INT8(value: any) {\n  const v = parseInt(value, 10);\n  if (v < -0x80 || v > 0x7f || isNaN(v)) {\n    throw new Error(`invalid value for INT8: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_UINT8(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xff || isNaN(v)) {\n    throw new Error(`invalid value for UINT8: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT16(value: any) {\n  const v = parseInt(value, 10);\n  if (v < -0x8000 || v > 0x7fff || isNaN(v)) {\n    throw new Error(`invalid value for INT16: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_UINT16(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xffff || isNaN(v)) {\n    throw new Error(`invalid value for UINT16: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT32(value: any) {\n  const v = parseInt(value, 10);\n  if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {\n    throw new Error(`invalid value for INT32: ${value}`);\n  }\n\n  return v;\n}\n\nfunction decimalToPrimitive_INT32(value: number, field: ParquetField): number {\n  const primitiveValue = value * 10 ** (field.scale || 0);\n  const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n  if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {\n    throw new Error(`invalid value for INT32: ${value}`);\n  }\n  return v;\n}\n\nfunction toPrimitive_UINT32(value: any): number {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xffffffffffff || isNaN(v)) {\n    throw new Error(`invalid value for UINT32: ${value}`);\n  }\n  return v;\n}\n\nfunction toPrimitive_INT64(value: any): number {\n  const v = parseInt(value, 10);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for INT64: ${value}`);\n  }\n  return v;\n}\n\nfunction decimalToPrimitive_INT64(value: number, field: ParquetField) {\n  const primitiveValue = value * 10 ** (field.scale || 0);\n  const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for INT64: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_UINT64(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || isNaN(v)) {\n    throw new Error(`invalid value for UINT64: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_INT96(value: any) {\n  const v = parseInt(value, 10);\n  if (isNaN(v)) {\n    throw new Error(`invalid value for INT96: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_BYTE_ARRAY(value: any): Buffer {\n  return Buffer.from(value);\n}\n\nfunction decimalToPrimitive_BYTE_ARRAY(value: any): Buffer {\n  // TBD\n  return Buffer.from(value);\n}\n\nfunction toPrimitive_UTF8(value: any): Buffer {\n  return Buffer.from(value, 'utf8');\n}\n\nfunction fromPrimitive_UTF8(value: any): string {\n  return value.toString();\n}\n\nfunction toPrimitive_JSON(value: any): Buffer {\n  return Buffer.from(JSON.stringify(value));\n}\n\nfunction fromPrimitive_JSON(value: any): unknown {\n  return JSON.parse(value);\n}\n\nfunction toPrimitive_BSON(value: any): Buffer {\n  return Buffer.from(BSON.serialize(value));\n}\n\nfunction fromPrimitive_BSON(value: any) {\n  return BSON.deserialize(value);\n}\n\nfunction toPrimitive_TIME_MILLIS(value: any) {\n  const v = parseInt(value, 10);\n  if (v < 0 || v > 0xffffffffffffffff || isNaN(v)) {\n    throw new Error(`invalid value for TIME_MILLIS: ${value}`);\n  }\n\n  return v;\n}\n\nfunction toPrimitive_TIME_MICROS(value: any): number {\n  const v = parseInt(value, 10);\n  if (v < 0 || isNaN(v)) {\n    throw new Error(`invalid value for TIME_MICROS: ${value}`);\n  }\n  return v;\n}\n\nconst kMillisPerDay = 86400000;\n\nfunction toPrimitive_DATE(value: any): number {\n  /* convert from date */\n  if (value instanceof Date) {\n    return value.getTime() / kMillisPerDay;\n  }\n\n  /* convert from integer */\n  {\n    const v = parseInt(value, 10);\n    if (v < 0 || isNaN(v)) {\n      throw new Error(`invalid value for DATE: ${value}`);\n    }\n\n    return v;\n  }\n}\n\nfunction fromPrimitive_DATE(value: any): Date {\n  return new Date(value * kMillisPerDay);\n}\n\nfunction toPrimitive_TIMESTAMP_MILLIS(value: any): number {\n  /* convert from date */\n  if (value instanceof Date) {\n    return value.getTime();\n  }\n\n  /* convert from integer */\n  {\n    const v = parseInt(value, 10);\n    if (v < 0 || isNaN(v)) {\n      throw new Error(`invalid value for TIMESTAMP_MILLIS: ${value}`);\n    }\n\n    return v;\n  }\n}\n\nfunction fromPrimitive_TIMESTAMP_MILLIS(value: any): Date {\n  return new Date(value);\n}\n\nfunction toPrimitive_TIMESTAMP_MICROS(value: any) {\n  /* convert from date */\n  if (value instanceof Date) {\n    return value.getTime() * 1000;\n  }\n\n  /* convert from integer */\n  {\n    const v = parseInt(value, 10);\n    if (v < 0 || isNaN(v)) {\n      throw new Error(`invalid value for TIMESTAMP_MICROS: ${value}`);\n    }\n\n    return v;\n  }\n}\n\nfunction fromPrimitive_TIMESTAMP_MICROS(value: any) {\n  return new Date(value / 1000);\n}\n\nfunction toPrimitive_INTERVAL(value: any) {\n  if (!value.months || !value.days || !value.milliseconds) {\n    throw new Error(\n      'value for INTERVAL must be object { months: ..., days: ..., milliseconds: ... }'\n    );\n  }\n\n  const buf = Buffer.alloc(12);\n\n  buf.writeUInt32LE(value.months, 0);\n  buf.writeUInt32LE(value.days, 4);\n  buf.writeUInt32LE(value.milliseconds, 8);\n  return buf;\n}\n\nfunction fromPrimitive_INTERVAL(value: any) {\n  const buf = Buffer.from(value);\n  const months = buf.readUInt32LE(0);\n  const days = buf.readUInt32LE(4);\n  const millis = buf.readUInt32LE(8);\n\n  return {months, days, milliseconds: millis};\n}\n\nfunction decimalFromPrimitive_INT(value: any, field: ParquetField) {\n  const presisionInt = Math.round(((value * 10 ** -field.presision!) % 1) * 10 ** field.presision!);\n  return presisionInt * 10 ** -(field.scale || 0);\n}\n\nfunction decimalFromPrimitive_BYTE_ARRAY(value: any, field: ParquetField) {\n  let number = 0;\n  if (value.length <= 4) {\n    // Bytewise operators faster. Use them if it is possible\n    for (let i = 0; i < value.length; i++) {\n      // `value.length - i - 1` bytes have reverse order (big-endian)\n      const component = value[i] << (8 * (value.length - i - 1));\n      number += component;\n    }\n  } else {\n    for (let i = 0; i < value.length; i++) {\n      // `value.length - i - 1` bytes have reverse order (big-endian)\n      const component = value[i] * 2 ** (8 * (value.length - 1 - i));\n      number += component;\n    }\n  }\n\n  const presisionInt = Math.round(\n    ((number * 10 ** -field.presision!) % 1) * 10 ** field.presision!\n  );\n  return presisionInt * 10 ** -(field.scale || 0);\n}\n"],"mappings":";;;;;;;;;AAEA;;AAWO,IAAMA,qBAA0D,GAAG;EACxEC,OAAO,EAAE;IACPC,aAAa,EAAE,SAAS;IACxBC,WAAW,EAAEC,mBAAmB;IAChCC,aAAa,EAAEC;EACjB,CAAC;EACDC,KAAK,EAAE;IACLL,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAEK;EACf,CAAC;EACDC,KAAK,EAAE;IACLP,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAEO;EACf,CAAC;EACDC,KAAK,EAAE;IACLT,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAES;EACf,CAAC;EACDC,KAAK,EAAE;IACLX,aAAa,EAAE,OAAO;IACtBC,WAAW,EAAEW;EACf,CAAC;EACDC,MAAM,EAAE;IACNb,aAAa,EAAE,QAAQ;IACvBC,WAAW,EAAEa;EACf,CAAC;EACDC,UAAU,EAAE;IACVf,aAAa,EAAE,YAAY;IAC3BC,WAAW,EAAEe;EACf,CAAC;EACDC,oBAAoB,EAAE;IACpBjB,aAAa,EAAE,sBAAsB;IACrCC,WAAW,EAAEe;EACf,CAAC;EACDE,IAAI,EAAE;IACJlB,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAEmB,gBAAgB;IAC7BjB,aAAa,EAAEkB;EACjB,CAAC;EACDC,WAAW,EAAE;IACXtB,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,aAAa;IAC3BlB,WAAW,EAAEsB;EACf,CAAC;EACDC,WAAW,EAAE;IACXxB,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,aAAa;IAC3BlB,WAAW,EAAEwB;EACf,CAAC;EACDC,IAAI,EAAE;IACJ1B,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAE0B,gBAAgB;IAC7BxB,aAAa,EAAEyB;EACjB,CAAC;EACDC,gBAAgB,EAAE;IAChB7B,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,kBAAkB;IAChClB,WAAW,EAAE6B,4BAA4B;IACzC3B,aAAa,EAAE4B;EACjB,CAAC;EACDC,gBAAgB,EAAE;IAChBhC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,kBAAkB;IAChClB,WAAW,EAAEgC,4BAA4B;IACzC9B,aAAa,EAAE+B;EACjB,CAAC;EACDC,MAAM,EAAE;IACNnC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAEmC;EACf,CAAC;EACDC,OAAO,EAAE;IACPrC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,SAAS;IACvBlB,WAAW,EAAEqC;EACf,CAAC;EACDC,OAAO,EAAE;IACPvC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,SAAS;IACvBlB,WAAW,EAAEuC;EACf,CAAC;EACDC,OAAO,EAAE;IACPzC,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,SAAS;IACvBlB,WAAW,EAAEyC;EACf,CAAC;EACDC,KAAK,EAAE;IACL3C,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,OAAO;IACrBlB,WAAW,EAAE2C;EACf,CAAC;EACDC,MAAM,EAAE;IACN7C,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAE6C;EACf,CAAC;EACDC,MAAM,EAAE;IACN/C,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAEK;EACf,CAAC;EACD0C,MAAM,EAAE;IACNhD,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,QAAQ;IACtBlB,WAAW,EAAEO;EACf,CAAC;EACDyC,IAAI,EAAE;IACJjD,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAEiD,gBAAgB;IAC7B/C,aAAa,EAAEgD;EACjB,CAAC;EACDC,IAAI,EAAE;IACJpD,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,MAAM;IACpBlB,WAAW,EAAEoD,gBAAgB;IAC7BlD,aAAa,EAAEmD;EACjB,CAAC;EACDC,QAAQ,EAAE;IACRvD,aAAa,EAAE,sBAAsB;IACrCmB,YAAY,EAAE,UAAU;IACxBqC,UAAU,EAAE,EAAE;IACdvD,WAAW,EAAEwD,oBAAoB;IACjCtD,aAAa,EAAEuD;EACjB,CAAC;EACDC,aAAa,EAAE;IACb3D,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,eAAe;IAC7BlB,WAAW,EAAE2D,wBAAwB;IACrCzD,aAAa,EAAE0D;EACjB,CAAC;EACDC,aAAa,EAAE;IACb9D,aAAa,EAAE,OAAO;IACtBmB,YAAY,EAAE,eAAe;IAC7BlB,WAAW,EAAE8D,wBAAwB;IACrC5D,aAAa,EAAE0D;EACjB,CAAC;EACDG,kBAAkB,EAAE;IAClBhE,aAAa,EAAE,YAAY;IAC3BmB,YAAY,EAAE,oBAAoB;IAClClB,WAAW,EAAEgE,6BAA6B;IAC1C9D,aAAa,EAAE+D;EACjB,CAAC;EACDC,4BAA4B,EAAE;IAC5BnE,aAAa,EAAE,sBAAsB;IACrCmB,YAAY,EAAE,8BAA8B;IAC5ClB,WAAW,EAAEgE,6BAA6B;IAC1C9D,aAAa,EAAE+D;EACjB;AACF,CAAC;;AAAC;AAMK,SAASjE,WAAW,CAACmE,IAAiB,EAAEC,KAAc,EAAEC,KAAoB,EAAW;EAC5F,IAAI,EAAEF,IAAI,IAAItE,qBAAqB,CAAC,EAAE;IACpC,MAAM,IAAIyE,KAAK,yBAAkBH,IAAI,EAAG;EAC1C;EAEA,OAAOtE,qBAAqB,CAACsE,IAAI,CAAC,CAACnE,WAAW,CAACoE,KAAK,EAAEC,KAAK,CAAC;AAC9D;;AAMO,SAASnE,aAAa,CAACiE,IAAiB,EAAEC,KAAc,EAAEC,KAAoB,EAAE;EACrF,IAAI,EAAEF,IAAI,IAAItE,qBAAqB,CAAC,EAAE;IACpC,MAAM,IAAIyE,KAAK,yBAAkBH,IAAI,EAAG;EAC1C;EAEA,IAAI,eAAe,IAAItE,qBAAqB,CAACsE,IAAI,CAAC,EAAE;IAAA;IAClD,gCAAO,0BAAAtE,qBAAqB,CAACsE,IAAI,CAAC,EAACjE,aAAa,0DAAzC,mDAA4CkE,KAAK,EAAEC,KAAK,CAAC;EAElE;;EACA,OAAOD,KAAK;AACd;AAEA,SAASnE,mBAAmB,CAACmE,KAAc,EAAW;EACpD,OAAOG,OAAO,CAACH,KAAK,CAAC;AACvB;AAEA,SAASjE,qBAAqB,CAACiE,KAAU,EAAW;EAClD,OAAOG,OAAO,CAACH,KAAK,CAAC;AACvB;AAEA,SAASzD,iBAAiB,CAACyD,KAAU,EAAU;EAC7C,IAAMI,CAAC,GAAGC,UAAU,CAACL,KAAK,CAAC;EAC3B,IAAIM,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EACA,OAAOI,CAAC;AACV;AAEA,SAAS3D,kBAAkB,CAACuD,KAAU,EAAU;EAC9C,IAAMI,CAAC,GAAGC,UAAU,CAACL,KAAK,CAAC;EAC3B,IAAIM,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIF,KAAK,qCAA8BF,KAAK,EAAG;EACvD;EACA,OAAOI,CAAC;AACV;AAEA,SAAS7B,gBAAgB,CAACyB,KAAU,EAAE;EACpC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAI,IAAIA,CAAC,GAAG,IAAI,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACrC,MAAM,IAAIF,KAAK,mCAA4BF,KAAK,EAAG;EACrD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASrC,iBAAiB,CAACiC,KAAU,EAAE;EACrC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,IAAI,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACjC,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAAS3B,iBAAiB,CAACuB,KAAU,EAAE;EACrC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,MAAM,IAAIA,CAAC,GAAG,MAAM,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACzC,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASnC,kBAAkB,CAAC+B,KAAU,EAAE;EACtC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,MAAM,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACnC,MAAM,IAAIF,KAAK,qCAA8BF,KAAK,EAAG;EACvD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASnE,iBAAiB,CAAC+D,KAAU,EAAE;EACrC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,UAAU,IAAIA,CAAC,GAAG,UAAU,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACjD,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASb,wBAAwB,CAACS,KAAa,EAAEC,KAAmB,EAAU;EAC5E,IAAMO,cAAc,GAAGR,KAAK,YAAG,EAAE,EAAKC,KAAK,CAACQ,KAAK,IAAI,CAAC,CAAC;EACvD,IAAML,CAAC,GAAGM,IAAI,CAACC,KAAK,CAAGH,cAAc,YAAG,EAAE,EAAI,CAACP,KAAK,CAACW,SAAU,IAAI,CAAC,YAAI,EAAE,EAAIX,KAAK,CAACW,SAAS,CAAC,CAAC;EAC/F,IAAIR,CAAC,GAAG,CAAC,UAAU,IAAIA,CAAC,GAAG,UAAU,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACjD,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EACA,OAAOI,CAAC;AACV;AAEA,SAASjC,kBAAkB,CAAC6B,KAAU,EAAU;EAC9C,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,cAAc,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IAC3C,MAAM,IAAIF,KAAK,qCAA8BF,KAAK,EAAG;EACvD;EACA,OAAOI,CAAC;AACV;AAEA,SAASjE,iBAAiB,CAAC6D,KAAU,EAAU;EAC7C,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIM,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EACA,OAAOI,CAAC;AACV;AAEA,SAASV,wBAAwB,CAACM,KAAa,EAAEC,KAAmB,EAAE;EACpE,IAAMO,cAAc,GAAGR,KAAK,YAAG,EAAE,EAAKC,KAAK,CAACQ,KAAK,IAAI,CAAC,CAAC;EACvD,IAAML,CAAC,GAAGM,IAAI,CAACC,KAAK,CAAGH,cAAc,YAAG,EAAE,EAAI,CAACP,KAAK,CAACW,SAAU,IAAI,CAAC,YAAI,EAAE,EAAIX,KAAK,CAACW,SAAS,CAAC,CAAC;EAC/F,IAAIN,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAAS/B,kBAAkB,CAAC2B,KAAU,EAAE;EACtC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACrB,MAAM,IAAIF,KAAK,qCAA8BF,KAAK,EAAG;EACvD;EAEA,OAAOI,CAAC;AACV;AAEA,SAAS/D,iBAAiB,CAAC2D,KAAU,EAAE;EACrC,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAIM,KAAK,CAACF,CAAC,CAAC,EAAE;IACZ,MAAM,IAAIF,KAAK,oCAA6BF,KAAK,EAAG;EACtD;EAEA,OAAOI,CAAC;AACV;AAEA,SAASzD,sBAAsB,CAACqD,KAAU,EAAU;EAClD,OAAOa,MAAM,CAACC,IAAI,CAACd,KAAK,CAAC;AAC3B;AAEA,SAASJ,6BAA6B,CAACI,KAAU,EAAU;EAEzD,OAAOa,MAAM,CAACC,IAAI,CAACd,KAAK,CAAC;AAC3B;AAEA,SAASjD,gBAAgB,CAACiD,KAAU,EAAU;EAC5C,OAAOa,MAAM,CAACC,IAAI,CAACd,KAAK,EAAE,MAAM,CAAC;AACnC;AAEA,SAAShD,kBAAkB,CAACgD,KAAU,EAAU;EAC9C,OAAOA,KAAK,CAACe,QAAQ,EAAE;AACzB;AAEA,SAASlC,gBAAgB,CAACmB,KAAU,EAAU;EAC5C,OAAOa,MAAM,CAACC,IAAI,CAAClC,IAAI,CAACoC,SAAS,CAAChB,KAAK,CAAC,CAAC;AAC3C;AAEA,SAASlB,kBAAkB,CAACkB,KAAU,EAAW;EAC/C,OAAOpB,IAAI,CAACqC,KAAK,CAACjB,KAAK,CAAC;AAC1B;AAEA,SAAShB,gBAAgB,CAACgB,KAAU,EAAU;EAC5C,OAAOa,MAAM,CAACC,IAAI,CAAC/B,aAAI,CAACmC,SAAS,CAAClB,KAAK,CAAC,CAAC;AAC3C;AAEA,SAASf,kBAAkB,CAACe,KAAU,EAAE;EACtC,OAAOjB,aAAI,CAACoC,WAAW,CAACnB,KAAK,CAAC;AAChC;AAEA,SAAS9C,uBAAuB,CAAC8C,KAAU,EAAE;EAC3C,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIA,CAAC,GAAG,kBAAkB,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IAC/C,MAAM,IAAIF,KAAK,0CAAmCF,KAAK,EAAG;EAC5D;EAEA,OAAOI,CAAC;AACV;AAEA,SAAShD,uBAAuB,CAAC4C,KAAU,EAAU;EACnD,IAAMI,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;EAC7B,IAAII,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;IACrB,MAAM,IAAIF,KAAK,0CAAmCF,KAAK,EAAG;EAC5D;EACA,OAAOI,CAAC;AACV;AAEA,IAAMgB,aAAa,GAAG,QAAQ;AAE9B,SAAS9D,gBAAgB,CAAC0C,KAAU,EAAU;EAE5C,IAAIA,KAAK,YAAYqB,IAAI,EAAE;IACzB,OAAOrB,KAAK,CAACsB,OAAO,EAAE,GAAGF,aAAa;EACxC;;EAGA;IACE,IAAMhB,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;IAC7B,IAAII,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;MACrB,MAAM,IAAIF,KAAK,mCAA4BF,KAAK,EAAG;IACrD;IAEA,OAAOI,CAAC;EACV;AACF;AAEA,SAAS7C,kBAAkB,CAACyC,KAAU,EAAQ;EAC5C,OAAO,IAAIqB,IAAI,CAACrB,KAAK,GAAGoB,aAAa,CAAC;AACxC;AAEA,SAAS3D,4BAA4B,CAACuC,KAAU,EAAU;EAExD,IAAIA,KAAK,YAAYqB,IAAI,EAAE;IACzB,OAAOrB,KAAK,CAACsB,OAAO,EAAE;EACxB;;EAGA;IACE,IAAMlB,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;IAC7B,IAAII,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;MACrB,MAAM,IAAIF,KAAK,+CAAwCF,KAAK,EAAG;IACjE;IAEA,OAAOI,CAAC;EACV;AACF;AAEA,SAAS1C,8BAA8B,CAACsC,KAAU,EAAQ;EACxD,OAAO,IAAIqB,IAAI,CAACrB,KAAK,CAAC;AACxB;AAEA,SAASpC,4BAA4B,CAACoC,KAAU,EAAE;EAEhD,IAAIA,KAAK,YAAYqB,IAAI,EAAE;IACzB,OAAOrB,KAAK,CAACsB,OAAO,EAAE,GAAG,IAAI;EAC/B;;EAGA;IACE,IAAMlB,CAAC,GAAGG,QAAQ,CAACP,KAAK,EAAE,EAAE,CAAC;IAC7B,IAAII,CAAC,GAAG,CAAC,IAAIE,KAAK,CAACF,CAAC,CAAC,EAAE;MACrB,MAAM,IAAIF,KAAK,+CAAwCF,KAAK,EAAG;IACjE;IAEA,OAAOI,CAAC;EACV;AACF;AAEA,SAASvC,8BAA8B,CAACmC,KAAU,EAAE;EAClD,OAAO,IAAIqB,IAAI,CAACrB,KAAK,GAAG,IAAI,CAAC;AAC/B;AAEA,SAASZ,oBAAoB,CAACY,KAAU,EAAE;EACxC,IAAI,CAACA,KAAK,CAACuB,MAAM,IAAI,CAACvB,KAAK,CAACwB,IAAI,IAAI,CAACxB,KAAK,CAACyB,YAAY,EAAE;IACvD,MAAM,IAAIvB,KAAK,CACb,iFAAiF,CAClF;EACH;EAEA,IAAMwB,GAAG,GAAGb,MAAM,CAACc,KAAK,CAAC,EAAE,CAAC;EAE5BD,GAAG,CAACE,aAAa,CAAC5B,KAAK,CAACuB,MAAM,EAAE,CAAC,CAAC;EAClCG,GAAG,CAACE,aAAa,CAAC5B,KAAK,CAACwB,IAAI,EAAE,CAAC,CAAC;EAChCE,GAAG,CAACE,aAAa,CAAC5B,KAAK,CAACyB,YAAY,EAAE,CAAC,CAAC;EACxC,OAAOC,GAAG;AACZ;AAEA,SAASrC,sBAAsB,CAACW,KAAU,EAAE;EAC1C,IAAM0B,GAAG,GAAGb,MAAM,CAACC,IAAI,CAACd,KAAK,CAAC;EAC9B,IAAMuB,MAAM,GAAGG,GAAG,CAACG,YAAY,CAAC,CAAC,CAAC;EAClC,IAAML,IAAI,GAAGE,GAAG,CAACG,YAAY,CAAC,CAAC,CAAC;EAChC,IAAMC,MAAM,GAAGJ,GAAG,CAACG,YAAY,CAAC,CAAC,CAAC;EAElC,OAAO;IAACN,MAAM,EAANA,MAAM;IAAEC,IAAI,EAAJA,IAAI;IAAEC,YAAY,EAAEK;EAAM,CAAC;AAC7C;AAEA,SAAStC,wBAAwB,CAACQ,KAAU,EAAEC,KAAmB,EAAE;EACjE,IAAM8B,YAAY,GAAGrB,IAAI,CAACC,KAAK,CAAGX,KAAK,YAAG,EAAE,EAAI,CAACC,KAAK,CAACW,SAAU,IAAI,CAAC,YAAI,EAAE,EAAIX,KAAK,CAACW,SAAS,CAAC,CAAC;EACjG,OAAOmB,YAAY,YAAG,EAAE,EAAI,EAAE9B,KAAK,CAACQ,KAAK,IAAI,CAAC,CAAC;AACjD;AAEA,SAASZ,+BAA+B,CAACG,KAAU,EAAEC,KAAmB,EAAE;EACxE,IAAI+B,MAAM,GAAG,CAAC;EACd,IAAIhC,KAAK,CAACiC,MAAM,IAAI,CAAC,EAAE;IAErB,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGlC,KAAK,CAACiC,MAAM,EAAEC,CAAC,EAAE,EAAE;MAErC,IAAMC,SAAS,GAAGnC,KAAK,CAACkC,CAAC,CAAC,IAAK,CAAC,IAAIlC,KAAK,CAACiC,MAAM,GAAGC,CAAC,GAAG,CAAC,CAAE;MAC1DF,MAAM,IAAIG,SAAS;IACrB;EACF,CAAC,MAAM;IACL,KAAK,IAAID,EAAC,GAAG,CAAC,EAAEA,EAAC,GAAGlC,KAAK,CAACiC,MAAM,EAAEC,EAAC,EAAE,EAAE;MAErC,IAAMC,UAAS,GAAGnC,KAAK,CAACkC,EAAC,CAAC,YAAG,CAAC,EAAK,CAAC,IAAIlC,KAAK,CAACiC,MAAM,GAAG,CAAC,GAAGC,EAAC,CAAC,CAAC;MAC9DF,MAAM,IAAIG,UAAS;IACrB;EACF;EAEA,IAAMJ,YAAY,GAAGrB,IAAI,CAACC,KAAK,CAC3BqB,MAAM,YAAG,EAAE,EAAI,CAAC/B,KAAK,CAACW,SAAU,IAAI,CAAC,YAAI,EAAE,EAAIX,KAAK,CAACW,SAAS,CAAC,CAClE;EACD,OAAOmB,YAAY,YAAG,EAAE,EAAI,EAAE9B,KAAK,CAACQ,KAAK,IAAI,CAAC,CAAC;AACjD"}

package/dist/es5/parquetjs/utils/file-utils.js

@@ -1,6 +1,5 @@
 "use strict";
 
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
@@ -8,7 +7,7 @@ exports.load = load;
 exports.osclose = osclose;
 exports.osopen = osopen;
 exports.oswrite = oswrite;
-var _fs = _interopRequireDefault(require("fs"));
+var _loaderUtils = require("@loaders.gl/loader-utils");
 
 function load(name) {
   return (module || global).require(name);
@@ -37,7 +36,7 @@ function osclose(os) {
 }
 function osopen(path, opts) {
   return new Promise(function (resolve, reject) {
-    var outputStream = _fs.default.createWriteStream(path, opts);
+    var outputStream = _loaderUtils.fs.createWriteStream(path, opts);
     outputStream.once('open', function (fd) {
       return resolve(outputStream);
     });

package/dist/es5/parquetjs/utils/file-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"file-utils.js","names":["load","name","module","global","require","oswrite","os","buf","Promise","resolve","reject","write","err","osclose","close","osopen","path","opts","outputStream","fs","createWriteStream","once","fd"],"sources":["../../../../src/parquetjs/utils/file-utils.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport fs from 'fs';\nimport {Writable} from 'stream';\n\nexport function load(name: string): any {\n  return (module || (global as any)).require(name);\n}\nexport interface WriteStreamOptions {\n  flags?: string;\n  encoding?: string;\n  fd?: number;\n  mode?: number;\n  autoClose?: boolean;\n  start?: number;\n}\n\nexport function oswrite(os: Writable, buf: Buffer): Promise<void> {\n  return new Promise((resolve, reject) => {\n    os.write(buf, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osclose(os: Writable): Promise<void> {\n  return new Promise((resolve, reject) => {\n    (os as any).close((err: any) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {\n  return new Promise((resolve, reject) => {\n    const outputStream = fs.createWriteStream(path, opts as any);\n    outputStream.once('open', (fd) => resolve(outputStream));\n    outputStream.once('error', (err) => reject(err));\n  });\n}\n"],"mappings":";;;;;;;;;;AACA;;AAGO,SAASA,IAAI,CAACC,IAAY,EAAO;EACtC,OAAO,CAACC,MAAM,IAAKC,MAAc,EAAEC,OAAO,CAACH,IAAI,CAAC;AAClD;AAUO,SAASI,OAAO,CAACC,EAAY,EAAEC,GAAW,EAAiB;EAChE,OAAO,IAAIC,OAAO,CAAC,UAACC,OAAO,EAAEC,MAAM,EAAK;IACtCJ,EAAE,CAACK,KAAK,CAACJ,GAAG,EAAE,UAACK,GAAG,EAAK;MACrB,IAAIA,GAAG,EAAE;QACPF,MAAM,CAACE,GAAG,CAAC;MACb,CAAC,MAAM;QACLH,OAAO,EAAE;MACX;IACF,CAAC,CAAC;EACJ,CAAC,CAAC;AACJ;AAEO,SAASI,OAAO,CAACP,EAAY,EAAiB;EACnD,OAAO,IAAIE,OAAO,CAAC,UAACC,OAAO,EAAEC,MAAM,EAAK;IACrCJ,EAAE,CAASQ,KAAK,CAAC,UAACF,GAAQ,EAAK;MAC9B,IAAIA,GAAG,EAAE;QACPF,MAAM,CAACE,GAAG,CAAC;MACb,CAAC,MAAM;QACLH,OAAO,EAAE;MACX;IACF,CAAC,CAAC;EACJ,CAAC,CAAC;AACJ;AAEO,SAASM,MAAM,CAACC,IAAY,EAAEC,IAAyB,EAA2B;EACvF,OAAO,IAAIT,OAAO,CAAC,UAACC,OAAO,EAAEC,MAAM,EAAK;IACtC,IAAMQ,YAAY,GAAGC,WAAE,CAACC,iBAAiB,CAACJ,IAAI,EAAEC,IAAI,CAAQ;IAC5DC,YAAY,CAACG,IAAI,CAAC,MAAM,EAAE,UAACC,EAAE;MAAA,OAAKb,OAAO,CAACS,YAAY,CAAC;IAAA,EAAC;IACxDA,YAAY,CAACG,IAAI,CAAC,OAAO,EAAE,UAACT,GAAG;MAAA,OAAKF,MAAM,CAACE,GAAG,CAAC;IAAA,EAAC;EAClD,CAAC,CAAC;AACJ"}
+{"version":3,"file":"file-utils.js","names":["load","name","module","global","require","oswrite","os","buf","Promise","resolve","reject","write","err","osclose","close","osopen","path","opts","outputStream","fs","createWriteStream","once","fd"],"sources":["../../../../src/parquetjs/utils/file-utils.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {fs, stream} from '@loaders.gl/loader-utils';\n\nexport function load(name: string): any {\n  return (module || (global as any)).require(name);\n}\nexport interface WriteStreamOptions {\n  flags?: string;\n  encoding?: string;\n  fd?: number;\n  mode?: number;\n  autoClose?: boolean;\n  start?: number;\n}\n\nexport function oswrite(os: stream.Writable, buf: Buffer): Promise<void> {\n  return new Promise((resolve, reject) => {\n    os.write(buf, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osclose(os: stream.Writable): Promise<void> {\n  return new Promise((resolve, reject) => {\n    (os as any).close((err: any) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {\n  return new Promise((resolve, reject) => {\n    const outputStream = fs.createWriteStream(path, opts as any);\n    outputStream.once('open', (fd) => resolve(outputStream));\n    outputStream.once('error', (err) => reject(err));\n  });\n}\n"],"mappings":";;;;;;;;;AACA;;AAEO,SAASA,IAAI,CAACC,IAAY,EAAO;EACtC,OAAO,CAACC,MAAM,IAAKC,MAAc,EAAEC,OAAO,CAACH,IAAI,CAAC;AAClD;AAUO,SAASI,OAAO,CAACC,EAAmB,EAAEC,GAAW,EAAiB;EACvE,OAAO,IAAIC,OAAO,CAAC,UAACC,OAAO,EAAEC,MAAM,EAAK;IACtCJ,EAAE,CAACK,KAAK,CAACJ,GAAG,EAAE,UAACK,GAAG,EAAK;MACrB,IAAIA,GAAG,EAAE;QACPF,MAAM,CAACE,GAAG,CAAC;MACb,CAAC,MAAM;QACLH,OAAO,EAAE;MACX;IACF,CAAC,CAAC;EACJ,CAAC,CAAC;AACJ;AAEO,SAASI,OAAO,CAACP,EAAmB,EAAiB;EAC1D,OAAO,IAAIE,OAAO,CAAC,UAACC,OAAO,EAAEC,MAAM,EAAK;IACrCJ,EAAE,CAASQ,KAAK,CAAC,UAACF,GAAQ,EAAK;MAC9B,IAAIA,GAAG,EAAE;QACPF,MAAM,CAACE,GAAG,CAAC;MACb,CAAC,MAAM;QACLH,OAAO,EAAE;MACX;IACF,CAAC,CAAC;EACJ,CAAC,CAAC;AACJ;AAEO,SAASM,MAAM,CAACC,IAAY,EAAEC,IAAyB,EAA2B;EACvF,OAAO,IAAIT,OAAO,CAAC,UAACC,OAAO,EAAEC,MAAM,EAAK;IACtC,IAAMQ,YAAY,GAAGC,eAAE,CAACC,iBAAiB,CAACJ,IAAI,EAAEC,IAAI,CAAQ;IAC5DC,YAAY,CAACG,IAAI,CAAC,MAAM,EAAE,UAACC,EAAE;MAAA,OAAKb,OAAO,CAACS,YAAY,CAAC;IAAA,EAAC;IACxDA,YAAY,CAACG,IAAI,CAAC,OAAO,EAAE,UAACT,GAAG;MAAA,OAAKF,MAAM,CAACE,GAAG,CAAC;IAAA,EAAC;EAClD,CAAC,CAAC;AACJ"}

package/dist/esm/index.js

@@ -2,7 +2,8 @@
 
 import { ParquetWasmLoader as ParquetWasmWorkerLoader } from './parquet-wasm-loader';
 import { ParquetLoader as ParquetWorkerLoader } from './parquet-loader';
-import { parseParquet, parseParquetFileInBatches } from './lib/parse-parquet';
+import { parseParquet, parseParquetFileInBatches } from './lib/parsers/parse-parquet-to-rows';
+import { parseParquetInColumns, parseParquetFileInColumnarBatches } from './lib/parsers/parse-parquet-to-columns';
 import { parseParquet as parseParquetWasm } from './lib/wasm/parse-parquet-wasm';
 export { ParquetWorkerLoader, ParquetWasmWorkerLoader };
 
@@ -11,6 +12,12 @@ export const ParquetLoader = {
   parse: parseParquet,
   parseFileInBatches: parseParquetFileInBatches
 };
+
+export const ParquetColumnarLoader = {
+  ...ParquetWorkerLoader,
+  parse: parseParquetInColumns,
+  parseFileInBatches: parseParquetFileInColumnarBatches
+};
 export const ParquetWasmLoader = {
   ...ParquetWasmWorkerLoader,
   parse: parseParquetWasm
@@ -22,8 +29,11 @@ export { ParquetWasmWriter } from './parquet-wasm-writer';
 export { preloadCompressions } from './parquetjs/compression';
 export { ParquetSchema } from './parquetjs/schema/schema';
 export { ParquetReader } from './parquetjs/parser/parquet-reader';
-export { ParquetEnvelopeReader } from './parquetjs/parser/parquet-envelope-reader';
-export { convertParquetToArrowSchema } from './lib/convert-schema';
+export { ParquetEncoder } from './parquetjs/encoder/parquet-encoder';
+export { convertSchemaFromParquet, convertSchemaFromParquet as convertParquetToArrowSchema } from './lib/arrow/convert-schema-from-parquet';
 
 export const _typecheckParquetLoader = ParquetLoader;
+
+export { default as geoJSONSchema } from './lib/geo/geoparquet-schema';
+export { getGeoMetadata, setGeoMetadata, unpackGeoMetadata } from './lib/geo/decode-geo-metadata';
 //# sourceMappingURL=index.js.map

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":["ParquetWasmLoader","ParquetWasmWorkerLoader","ParquetLoader","ParquetWorkerLoader","parseParquet","parseParquetFileInBatches","parseParquetWasm","parse","parseFileInBatches","ParquetWriter","_ParquetWriter","ParquetWasmWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEnvelopeReader","convertParquetToArrowSchema","_typecheckParquetLoader"],"sources":["../../src/index.ts"],"sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\nimport {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';\n\nexport {ParquetWorkerLoader, ParquetWasmWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\nexport const ParquetWasmLoader = {\n  ...ParquetWasmWorkerLoader,\n  parse: parseParquetWasm\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\nexport {ParquetWasmWriter} from './parquet-wasm-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"mappings":";;AAIA,SAAQA,iBAAiB,IAAIC,uBAAuB,QAAO,uBAAuB;AAClF,SAAQC,aAAa,IAAIC,mBAAmB,QAAO,kBAAkB;AACrE,SAAQC,YAAY,EAAEC,yBAAyB,QAAO,qBAAqB;AAC3E,SAAQD,YAAY,IAAIE,gBAAgB,QAAO,+BAA+B;AAE9E,SAAQH,mBAAmB,EAAEF,uBAAuB;;AAGpD,OAAO,MAAMC,aAAa,GAAG;EAC3B,GAAGC,mBAAmB;EACtBI,KAAK,EAAEH,YAAY;EACnBI,kBAAkB,EAAEH;AACtB,CAAC;AAED,OAAO,MAAML,iBAAiB,GAAG;EAC/B,GAAGC,uBAAuB;EAC1BM,KAAK,EAAED;AACT,CAAC;;AAID,SAAQG,aAAa,IAAIC,cAAc,QAAO,kBAAkB;AAChE,SAAQC,iBAAiB,QAAO,uBAAuB;;AAIvD,SAAQC,mBAAmB,QAAO,yBAAyB;AAE3D,SAAQC,aAAa,QAAO,2BAA2B;AACvD,SAAQC,aAAa,QAAO,mCAAmC;AAC/D,SAAQC,qBAAqB,QAAO,4CAA4C;AAEhF,SAAQC,2BAA2B,QAAO,sBAAsB;;AAGhE,OAAO,MAAMC,uBAAyC,GAAGf,aAAa"}
+{"version":3,"file":"index.js","names":["ParquetWasmLoader","ParquetWasmWorkerLoader","ParquetLoader","ParquetWorkerLoader","parseParquet","parseParquetFileInBatches","parseParquetInColumns","parseParquetFileInColumnarBatches","parseParquetWasm","parse","parseFileInBatches","ParquetColumnarLoader","ParquetWriter","_ParquetWriter","ParquetWasmWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEncoder","convertSchemaFromParquet","convertParquetToArrowSchema","_typecheckParquetLoader","default","geoJSONSchema","getGeoMetadata","setGeoMetadata","unpackGeoMetadata"],"sources":["../../src/index.ts"],"sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parsers/parse-parquet-to-rows';\nimport {\n  parseParquetInColumns,\n  parseParquetFileInColumnarBatches\n} from './lib/parsers/parse-parquet-to-columns';\nimport {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';\n\nexport {ParquetWorkerLoader, ParquetWasmWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\n/** ParquetJS table loader */\nexport const ParquetColumnarLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquetInColumns,\n  parseFileInBatches: parseParquetFileInColumnarBatches\n};\n\nexport const ParquetWasmLoader = {\n  ...ParquetWasmWorkerLoader,\n  parse: parseParquetWasm\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\nexport {ParquetWasmWriter} from './parquet-wasm-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';\n\nexport {\n  convertSchemaFromParquet,\n  convertSchemaFromParquet as convertParquetToArrowSchema\n} from './lib/arrow/convert-schema-from-parquet';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n\n// Geo Metadata\nexport {default as geoJSONSchema} from './lib/geo/geoparquet-schema';\n\nexport type {GeoMetadata} from './lib/geo/decode-geo-metadata';\nexport {getGeoMetadata, setGeoMetadata, unpackGeoMetadata} from './lib/geo/decode-geo-metadata';\n"],"mappings":";;AAIA,SAAQA,iBAAiB,IAAIC,uBAAuB,QAAO,uBAAuB;AAClF,SAAQC,aAAa,IAAIC,mBAAmB,QAAO,kBAAkB;AACrE,SAAQC,YAAY,EAAEC,yBAAyB,QAAO,qCAAqC;AAC3F,SACEC,qBAAqB,EACrBC,iCAAiC,QAC5B,wCAAwC;AAC/C,SAAQH,YAAY,IAAII,gBAAgB,QAAO,+BAA+B;AAE9E,SAAQL,mBAAmB,EAAEF,uBAAuB;;AAGpD,OAAO,MAAMC,aAAa,GAAG;EAC3B,GAAGC,mBAAmB;EACtBM,KAAK,EAAEL,YAAY;EACnBM,kBAAkB,EAAEL;AACtB,CAAC;;AAGD,OAAO,MAAMM,qBAAqB,GAAG;EACnC,GAAGR,mBAAmB;EACtBM,KAAK,EAAEH,qBAAqB;EAC5BI,kBAAkB,EAAEH;AACtB,CAAC;AAED,OAAO,MAAMP,iBAAiB,GAAG;EAC/B,GAAGC,uBAAuB;EAC1BQ,KAAK,EAAED;AACT,CAAC;;AAID,SAAQI,aAAa,IAAIC,cAAc,QAAO,kBAAkB;AAChE,SAAQC,iBAAiB,QAAO,uBAAuB;;AAIvD,SAAQC,mBAAmB,QAAO,yBAAyB;AAE3D,SAAQC,aAAa,QAAO,2BAA2B;AACvD,SAAQC,aAAa,QAAO,mCAAmC;AAC/D,SAAQC,cAAc,QAAO,qCAAqC;AAElE,SACEC,wBAAwB,EACxBA,wBAAwB,IAAIC,2BAA2B,QAClD,yCAAyC;;AAGhD,OAAO,MAAMC,uBAAyC,GAAGnB,aAAa;;AAGtE,SAAQoB,OAAO,IAAIC,aAAa,QAAO,6BAA6B;AAGpE,SAAQC,cAAc,EAAEC,cAAc,EAAEC,iBAAiB,QAAO,+BAA+B"}

package/dist/esm/lib/arrow/convert-columns-to-row-group.js

@@ -0,0 +1,2 @@
+
+//# sourceMappingURL=convert-columns-to-row-group.js.map

package/dist/esm/lib/arrow/convert-columns-to-row-group.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-columns-to-row-group.js","names":[],"sources":["../../../../src/lib/arrow/convert-columns-to-row-group.ts"],"sourcesContent":[""],"mappings":""}

package/dist/esm/lib/arrow/convert-row-group-to-columns.js

@@ -0,0 +1,10 @@
+
+
+export function convertParquetRowGroupToColumns(schema, rowGroup) {
+  const columns = {};
+  for (const [columnName, data] of Object.entries(rowGroup.columnData)) {
+    columns[columnName] = columns[columnName] || data.values;
+  }
+  return columns;
+}
+//# sourceMappingURL=convert-row-group-to-columns.js.map

package/dist/esm/lib/arrow/convert-row-group-to-columns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-row-group-to-columns.js","names":["convertParquetRowGroupToColumns","schema","rowGroup","columns","columnName","data","Object","entries","columnData","values"],"sources":["../../../../src/lib/arrow/convert-row-group-to-columns.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport {Schema} from '@loaders.gl/schema';\nimport {ParquetBuffer} from '@loaders.gl/parquet/parquetjs/schema/declare';\n\nexport function convertParquetRowGroupToColumns(\n  schema: Schema,\n  rowGroup: ParquetBuffer\n): Record<string, any[]> {\n  const columns: Record<string, any[]> = {};\n  for (const [columnName, data] of Object.entries(rowGroup.columnData)) {\n    columns[columnName] = columns[columnName] || data.values;\n  }\n  return columns;\n}\n"],"mappings":";;AAKA,OAAO,SAASA,+BAA+B,CAC7CC,MAAc,EACdC,QAAuB,EACA;EACvB,MAAMC,OAA8B,GAAG,CAAC,CAAC;EACzC,KAAK,MAAM,CAACC,UAAU,EAAEC,IAAI,CAAC,IAAIC,MAAM,CAACC,OAAO,CAACL,QAAQ,CAACM,UAAU,CAAC,EAAE;IACpEL,OAAO,CAACC,UAAU,CAAC,GAAGD,OAAO,CAACC,UAAU,CAAC,IAAIC,IAAI,CAACI,MAAM;EAC1D;EACA,OAAON,OAAO;AAChB"}