@loaders.gl/parquet 3.4.6 → 4.0.0-alpha.10

package/dist/es5/parquetjs/schema/declare.js.map

@@ -1 +1 @@
-{"version":3,"file":"declare.js","names":["ParquetBuffer","_createClass2","default","rowCount","arguments","length","undefined","columnData","_classCallCheck2","_defineProperty2","exports"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\n/** @todo better name, this is an internal type? */\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  /** Actual column chunks */\n  values: any[]; // ArrayLike<any>;\n  count: number;\n  dictionary?: ParquetDictionary;\n  /** The \"raw\" page header from the file */\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRecord {\n  [key: string]: any;\n}\n\n/** @\n * Holds data for one row group (column chunks) */\nexport class ParquetBuffer {\n  /** Number of rows in this page */\n  rowCount: number;\n\n  columnData: Record<string, ParquetData>;\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetData> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n\n/** Holds the data for one column chunk */\nexport interface ParquetData {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n"],"mappings":";;;;;;;;;;IAgIaA,aAAa,OAAAC,aAAA,CAAAC,OAAA,EAKxB,SAAAF,cAAA,EAAgF;EAAA,IAApEG,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;EAAA,IAAEG,UAAuC,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;EAAA,IAAAI,gBAAA,CAAAN,OAAA,QAAAF,aAAA;EAAA,IAAAS,gBAAA,CAAAP,OAAA;EAAA,IAAAO,gBAAA,CAAAP,OAAA;EAC5E,IAAI,CAACC,QAAQ,GAAGA,QAAQ;EACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;AAC9B,CAAC;AAAAG,OAAA,CAAAV,aAAA,GAAAA,aAAA"}
+{"version":3,"file":"declare.js","names":["ParquetRowGroup","_createClass2","default","rowCount","arguments","length","undefined","columnData","_classCallCheck2","_defineProperty2","exports"],"sources":["../../../../src/parquetjs/schema/declare.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport Int64 from 'node-int64';\nimport type {PageHeader} from '../parquet-thrift';\n\nexport type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';\nexport type ParquetCompression =\n  | 'UNCOMPRESSED'\n  | 'GZIP'\n  | 'SNAPPY'\n  | 'LZO'\n  | 'BROTLI'\n  | 'LZ4'\n  | 'LZ4_RAW'\n  | 'ZSTD';\nexport type RepetitionType = 'REQUIRED' | 'OPTIONAL' | 'REPEATED';\nexport type ParquetType = PrimitiveType | OriginalType;\n\n/**\n * Physical type\n */\nexport type PrimitiveType =\n  // Base Types\n  | 'BOOLEAN' // 0\n  | 'INT32' // 1\n  | 'INT64' // 2\n  | 'INT96' // 3\n  | 'FLOAT' // 4\n  | 'DOUBLE' // 5\n  | 'BYTE_ARRAY' // 6,\n  | 'FIXED_LEN_BYTE_ARRAY'; // 7\n\n/**\n * Logical type\n */\nexport type OriginalType =\n  // Converted Types\n  | 'UTF8' // 0\n  // | 'MAP' // 1\n  // | 'MAP_KEY_VALUE' // 2\n  // | 'LIST' // 3\n  // | 'ENUM' // 4\n  // | 'DECIMAL' // 5\n  | 'DECIMAL_INT32' // 5\n  | 'DECIMAL_INT64' // 5\n  | 'DECIMAL_BYTE_ARRAY' // 5\n  | 'DECIMAL_FIXED_LEN_BYTE_ARRAY' // 5\n  | 'DATE' // 6\n  | 'TIME_MILLIS' // 7\n  | 'TIME_MICROS' // 8\n  | 'TIMESTAMP_MILLIS' // 9\n  | 'TIMESTAMP_MICROS' // 10\n  | 'UINT_8' // 11\n  | 'UINT_16' // 12\n  | 'UINT_32' // 13\n  | 'UINT_64' // 14\n  | 'INT_8' // 15\n  | 'INT_16' // 16\n  | 'INT_32' // 17\n  | 'INT_64' // 18\n  | 'JSON' // 19\n  | 'BSON' // 20\n  | 'INTERVAL'; // 21\n\nexport type ParquetDictionary = string[];\n\nexport interface SchemaDefinition {\n  [string: string]: FieldDefinition;\n}\n\nexport interface FieldDefinition {\n  type?: ParquetType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  optional?: boolean;\n  repeated?: boolean;\n  fields?: SchemaDefinition;\n}\n\nexport interface ParquetField {\n  name: string;\n  path: string[];\n  key: string;\n  primitiveType?: PrimitiveType;\n  originalType?: OriginalType;\n  repetitionType: RepetitionType;\n  typeLength?: number;\n  presision?: number;\n  scale?: number;\n  encoding?: ParquetCodec;\n  compression?: ParquetCompression;\n  rLevelMax: number;\n  dLevelMax: number;\n  isNested?: boolean;\n  fieldCount?: number;\n  fields?: Record<string, ParquetField>;\n}\n\n/** @todo better name, this is an internal type? */\nexport interface ParquetOptions {\n  type: ParquetType;\n  rLevelMax: number;\n  dLevelMax: number;\n  compression: ParquetCompression;\n  column: ParquetField;\n  numValues?: Int64;\n  dictionary?: ParquetDictionary;\n}\n\nexport interface ParquetPageData {\n  dlevels: number[];\n  rlevels: number[];\n  /** Actual column chunks */\n  values: any[]; // ArrayLike<any>;\n  count: number;\n  dictionary?: ParquetDictionary;\n  /** The \"raw\" page header from the file */\n  pageHeader: PageHeader;\n}\n\nexport interface ParquetRow {\n  [key: string]: any;\n}\n\n/** @\n * Holds data for one row group (column chunks) */\nexport class ParquetRowGroup {\n  /** Number of rows in this page */\n  rowCount: number;\n  /** Map of Column chunks */\n  columnData: Record<string, ParquetColumnChunk>;\n\n  constructor(rowCount: number = 0, columnData: Record<string, ParquetColumnChunk> = {}) {\n    this.rowCount = rowCount;\n    this.columnData = columnData;\n  }\n}\n\n/** Holds the data for one column chunk */\nexport interface ParquetColumnChunk {\n  dlevels: number[];\n  rlevels: number[];\n  values: any[];\n  count: number;\n  pageHeaders: PageHeader[];\n}\n"],"mappings":";;;;;;;;;;IAgIaA,eAAe,OAAAC,aAAA,CAAAC,OAAA,EAM1B,SAAAF,gBAAA,EAAuF;EAAA,IAA3EG,QAAgB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC;EAAA,IAAEG,UAA8C,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,CAAC,CAAC;EAAA,IAAAI,gBAAA,CAAAN,OAAA,QAAAF,eAAA;EAAA,IAAAS,gBAAA,CAAAP,OAAA;EAAA,IAAAO,gBAAA,CAAAP,OAAA;EACnF,IAAI,CAACC,QAAQ,GAAGA,QAAQ;EACxB,IAAI,CAACI,UAAU,GAAGA,UAAU;AAC9B,CAAC;AAAAG,OAAA,CAAAV,eAAA,GAAAA,eAAA"}

package/dist/es5/parquetjs/schema/schema.js

@@ -54,13 +54,13 @@ var ParquetSchema = function () {
     }
   }, {
     key: "shredRecord",
-    value: function shredRecord(record, buffer) {
-      (0, _shred.shredRecord)(this, record, buffer);
+    value: function shredRecord(row, rowGroup) {
+      (0, _shred.shredRecord)(this, row, rowGroup);
     }
   }, {
-    key: "materializeRecords",
-    value: function materializeRecords(buffer) {
-      return (0, _shred.materializeRecords)(this, buffer);
+    key: "materializeRows",
+    value: function materializeRows(rowGroup) {
+      return (0, _shred.materializeRows)(this, rowGroup);
     }
   }, {
     key: "compress",
@@ -70,8 +70,8 @@ var ParquetSchema = function () {
       return this;
     }
   }, {
-    key: "buffer",
-    value: function buffer() {
+    key: "rowGroup",
+    value: function rowGroup() {
       return (0, _shred.shredBuffer)(this);
     }
   }]);

package/dist/es5/parquetjs/schema/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","names":["_codecs","require","_compression","_shred","_types","ParquetSchema","schema","_classCallCheck2","default","_defineProperty2","fields","buildFields","fieldList","listFields","_createClass2","key","value","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","shredRecord","record","buffer","materializeRecords","compress","type","setCompress","shredBuffer","exports","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetField,\n  ParquetRecord,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRecords, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void {\n    shredRecord(this, record, buffer);\n  }\n\n  materializeRecords(buffer: ParquetBuffer): ParquetRecord[] {\n    return materializeRecords(this, buffer);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  buffer(): ParquetBuffer {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"mappings":";;;;;;;;;;AAEA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AAUA,IAAAE,MAAA,GAAAF,OAAA;AACA,IAAAG,MAAA,GAAAH,OAAA;AAA8C,IAKjCI,aAAa;EAQxB,SAAAA,cAAYC,MAAwB,EAAE;IAAA,IAAAC,gBAAA,CAAAC,OAAA,QAAAH,aAAA;IAAA,IAAAI,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA;IACpC,IAAI,CAACF,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACI,MAAM,GAAGC,WAAW,CAACL,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACM,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;EAAC,IAAAI,aAAA,CAAAN,OAAA,EAAAH,aAAA;IAAAU,GAAA;IAAAC,KAAA,EAKD,SAAAC,UAAUC,IAAuB,EAAgB;MAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB,CAAC,MAAM;QAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;MACtB;MAEA,IAAIC,CAAC,GAAG,IAAI,CAACX,MAAM;MACnB,OAAOQ,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;QACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACR,MAAsC;MACvD;MAEA,OAAOW,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;IACnB;EAAC;IAAAH,GAAA;IAAAC,KAAA,EAKD,SAAAQ,gBAAgBN,IAAuB,EAAkB;MACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB;MACA,IAAMM,MAAsB,GAAG,EAAE;MACjC,IAAIJ,CAAC,GAAG,IAAI,CAACX,MAAM;MACnB,OAAOQ,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;QACpCE,MAAM,CAACC,IAAI,CAACL,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;UACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACR,MAAsC;QACvD;MACF;MACA,OAAOe,MAAM;IACf;EAAC;IAAAV,GAAA;IAAAC,KAAA,EAED,SAAAW,YAAYC,MAAqB,EAAEC,MAAqB,EAAQ;MAC9D,IAAAF,kBAAW,EAAC,IAAI,EAAEC,MAAM,EAAEC,MAAM,CAAC;IACnC;EAAC;IAAAd,GAAA;IAAAC,KAAA,EAED,SAAAc,mBAAmBD,MAAqB,EAAmB;MACzD,OAAO,IAAAC,yBAAkB,EAAC,IAAI,EAAED,MAAM,CAAC;IACzC;EAAC;IAAAd,GAAA;IAAAC,KAAA,EAED,SAAAe,SAASC,IAAwB,EAAQ;MACvCC,WAAW,CAAC,IAAI,CAAC3B,MAAM,EAAE0B,IAAI,CAAC;MAC9BC,WAAW,CAAC,IAAI,CAACvB,MAAM,EAAEsB,IAAI,CAAC;MAC9B,OAAO,IAAI;IACb;EAAC;IAAAjB,GAAA;IAAAC,KAAA,EAED,SAAAa,OAAA,EAAwB;MACtB,OAAO,IAAAK,kBAAW,EAAC,IAAI,CAAC;IAC1B;EAAC;EAAA,OAAA7B,aAAA;AAAA;AAAA8B,OAAA,CAAA9B,aAAA,GAAAA,aAAA;AAGH,SAAS4B,WAAWA,CAAC3B,MAAW,EAAE0B,IAAwB,EAAE;EAC1D,KAAK,IAAMI,IAAI,IAAI9B,MAAM,EAAE;IACzB,IAAM+B,IAAI,GAAG/B,MAAM,CAAC8B,IAAI,CAAC;IACzB,IAAIC,IAAI,CAAC3B,MAAM,EAAE;MACfuB,WAAW,CAACI,IAAI,CAAC3B,MAAM,EAAEsB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLK,IAAI,CAACC,WAAW,GAAGN,IAAI;IACzB;EACF;AACF;AAGA,SAASrB,WAAWA,CAClBL,MAAwB,EACxBiC,eAAuB,EACvBC,eAAuB,EACvBtB,IAAc,EACgB;EAC9B,IAAMN,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,IAAMwB,IAAI,IAAI9B,MAAM,EAAE;IACzB,IAAMmC,IAAI,GAAGnC,MAAM,CAAC8B,IAAI,CAAC;IAGzB,IAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,IAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;IAGA,IAAIN,IAAI,CAAC/B,MAAM,EAAE;MACf,IAAMuC,MAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCxB,SAAS,CAACwB,IAAI,CAAC,GAAG;QAChBA,IAAI,EAAJA,IAAI;QACJlB,IAAI,EAAE+B,MAAK;QACXlC,GAAG,EAAEkC,MAAK,CAACE,IAAI,CAAC,CAAC;QACjBH,cAAc,EAAdA,cAAc;QACdF,SAAS,EAATA,SAAS;QACTC,SAAS,EAATA,SAAS;QACTK,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACd,IAAI,CAAC/B,MAAM,CAAC,CAACY,MAAM;QAC3CZ,MAAM,EAAEC,WAAW,CAAC8B,IAAI,CAAC/B,MAAM,EAAEoC,SAAS,EAAEC,SAAS,EAAEE,MAAK;MAC9D,CAAC;MACD;IACF;IAEA,IAAMO,OAAY,GAAGC,4BAAqB,CAAChB,IAAI,CAACT,IAAI,CAAE;IACtD,IAAI,CAACwB,OAAO,EAAE;MACZ,MAAM,IAAIE,KAAK,0BAAAR,MAAA,CAA0BT,IAAI,CAACT,IAAI,CAAE,CAAC;IACvD;IAEAS,IAAI,CAACkB,QAAQ,GAAGlB,IAAI,CAACkB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAElB,IAAI,CAACkB,QAAQ,IAAIC,sBAAc,CAAC,EAAE;MACtC,MAAM,IAAIF,KAAK,kCAAAR,MAAA,CAAkCT,IAAI,CAACkB,QAAQ,CAAE,CAAC;IACnE;IAEAlB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAIuB,wCAA2B,CAAC,EAAE;MACtD,MAAM,IAAIH,KAAK,oCAAAR,MAAA,CAAoCT,IAAI,CAACH,WAAW,CAAE,CAAC;IACxE;IAGA,IAAMW,KAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCxB,SAAS,CAACwB,IAAI,CAAC,GAAG;MAChBA,IAAI,EAAJA,IAAI;MACJ0B,aAAa,EAAEN,OAAO,CAACM,aAAa;MACpCC,YAAY,EAAEP,OAAO,CAACO,YAAY;MAClC7C,IAAI,EAAE+B,KAAK;MACXlC,GAAG,EAAEkC,KAAK,CAACE,IAAI,CAAC,CAAC;MACjBH,cAAc,EAAdA,cAAc;MACdW,QAAQ,EAAElB,IAAI,CAACkB,QAAQ;MACvBrB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7B0B,UAAU,EAAEvB,IAAI,CAACuB,UAAU,IAAIR,OAAO,CAACQ,UAAU;MACjDC,SAAS,EAAExB,IAAI,CAACwB,SAAS;MACzBC,KAAK,EAAEzB,IAAI,CAACyB,KAAK;MACjBpB,SAAS,EAATA,SAAS;MACTC,SAAS,EAATA;IACF,CAAC;EACH;EACA,OAAOnC,SAAS;AAClB;AAEA,SAASC,UAAUA,CAACH,MAAoC,EAAkB;EACxE,IAAIyD,IAAoB,GAAG,EAAE;EAC7B,KAAK,IAAMC,CAAC,IAAI1D,MAAM,EAAE;IACtByD,IAAI,CAACzC,IAAI,CAAChB,MAAM,CAAC0D,CAAC,CAAC,CAAC;IACpB,IAAI1D,MAAM,CAAC0D,CAAC,CAAC,CAAChB,QAAQ,EAAE;MACtBe,IAAI,GAAGA,IAAI,CAACjB,MAAM,CAACrC,UAAU,CAACH,MAAM,CAAC0D,CAAC,CAAC,CAAC1D,MAAO,CAAC,CAAC;IACnD;EACF;EACA,OAAOyD,IAAI;AACb"}
+{"version":3,"file":"schema.js","names":["_codecs","require","_compression","_shred","_types","ParquetSchema","schema","_classCallCheck2","default","_defineProperty2","fields","buildFields","fieldList","listFields","_createClass2","key","value","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","shredRecord","row","rowGroup","materializeRows","compress","type","setCompress","shredBuffer","exports","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","join","isNested","fieldCount","Object","keys","typeDef","PARQUET_LOGICAL_TYPES","Error","encoding","PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetRowGroup,\n  ParquetCompression,\n  ParquetField,\n  ParquetRow,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRows, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(row: ParquetRow, rowGroup: ParquetRowGroup): void {\n    shredRecord(this, row, rowGroup);\n  }\n\n  materializeRows(rowGroup: ParquetRowGroup): ParquetRow[] {\n    return materializeRows(this, rowGroup);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  rowGroup(): ParquetRowGroup {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"mappings":";;;;;;;;;;AAEA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AAUA,IAAAE,MAAA,GAAAF,OAAA;AACA,IAAAG,MAAA,GAAAH,OAAA;AAA8C,IAKjCI,aAAa;EAQxB,SAAAA,cAAYC,MAAwB,EAAE;IAAA,IAAAC,gBAAA,CAAAC,OAAA,QAAAH,aAAA;IAAA,IAAAI,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA;IAAA,IAAAC,gBAAA,CAAAD,OAAA;IACpC,IAAI,CAACF,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACI,MAAM,GAAGC,WAAW,CAACL,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACM,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;EAAC,IAAAI,aAAA,CAAAN,OAAA,EAAAH,aAAA;IAAAU,GAAA;IAAAC,KAAA,EAKD,SAAAC,UAAUC,IAAuB,EAAgB;MAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB,CAAC,MAAM;QAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;MACtB;MAEA,IAAIC,CAAC,GAAG,IAAI,CAACX,MAAM;MACnB,OAAOQ,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;QACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACR,MAAsC;MACvD;MAEA,OAAOW,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;IACnB;EAAC;IAAAH,GAAA;IAAAC,KAAA,EAKD,SAAAQ,gBAAgBN,IAAuB,EAAkB;MACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;QAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;MACxB;MACA,IAAMM,MAAsB,GAAG,EAAE;MACjC,IAAIJ,CAAC,GAAG,IAAI,CAACX,MAAM;MACnB,OAAOQ,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;QACpCE,MAAM,CAACC,IAAI,CAACL,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;UACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACR,MAAsC;QACvD;MACF;MACA,OAAOe,MAAM;IACf;EAAC;IAAAV,GAAA;IAAAC,KAAA,EAED,SAAAW,YAAYC,GAAe,EAAEC,QAAyB,EAAQ;MAC5D,IAAAF,kBAAW,EAAC,IAAI,EAAEC,GAAG,EAAEC,QAAQ,CAAC;IAClC;EAAC;IAAAd,GAAA;IAAAC,KAAA,EAED,SAAAc,gBAAgBD,QAAyB,EAAgB;MACvD,OAAO,IAAAC,sBAAe,EAAC,IAAI,EAAED,QAAQ,CAAC;IACxC;EAAC;IAAAd,GAAA;IAAAC,KAAA,EAED,SAAAe,SAASC,IAAwB,EAAQ;MACvCC,WAAW,CAAC,IAAI,CAAC3B,MAAM,EAAE0B,IAAI,CAAC;MAC9BC,WAAW,CAAC,IAAI,CAACvB,MAAM,EAAEsB,IAAI,CAAC;MAC9B,OAAO,IAAI;IACb;EAAC;IAAAjB,GAAA;IAAAC,KAAA,EAED,SAAAa,SAAA,EAA4B;MAC1B,OAAO,IAAAK,kBAAW,EAAC,IAAI,CAAC;IAC1B;EAAC;EAAA,OAAA7B,aAAA;AAAA;AAAA8B,OAAA,CAAA9B,aAAA,GAAAA,aAAA;AAGH,SAAS4B,WAAWA,CAAC3B,MAAW,EAAE0B,IAAwB,EAAE;EAC1D,KAAK,IAAMI,IAAI,IAAI9B,MAAM,EAAE;IACzB,IAAM+B,IAAI,GAAG/B,MAAM,CAAC8B,IAAI,CAAC;IACzB,IAAIC,IAAI,CAAC3B,MAAM,EAAE;MACfuB,WAAW,CAACI,IAAI,CAAC3B,MAAM,EAAEsB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLK,IAAI,CAACC,WAAW,GAAGN,IAAI;IACzB;EACF;AACF;AAGA,SAASrB,WAAWA,CAClBL,MAAwB,EACxBiC,eAAuB,EACvBC,eAAuB,EACvBtB,IAAc,EACgB;EAC9B,IAAMN,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,IAAMwB,IAAI,IAAI9B,MAAM,EAAE;IACzB,IAAMmC,IAAI,GAAGnC,MAAM,CAAC8B,IAAI,CAAC;IAGzB,IAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,IAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;IAGA,IAAIN,IAAI,CAAC/B,MAAM,EAAE;MACf,IAAMuC,MAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCxB,SAAS,CAACwB,IAAI,CAAC,GAAG;QAChBA,IAAI,EAAJA,IAAI;QACJlB,IAAI,EAAE+B,MAAK;QACXlC,GAAG,EAAEkC,MAAK,CAACE,IAAI,CAAC,CAAC;QACjBH,cAAc,EAAdA,cAAc;QACdF,SAAS,EAATA,SAAS;QACTC,SAAS,EAATA,SAAS;QACTK,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACd,IAAI,CAAC/B,MAAM,CAAC,CAACY,MAAM;QAC3CZ,MAAM,EAAEC,WAAW,CAAC8B,IAAI,CAAC/B,MAAM,EAAEoC,SAAS,EAAEC,SAAS,EAAEE,MAAK;MAC9D,CAAC;MACD;IACF;IAEA,IAAMO,OAAY,GAAGC,4BAAqB,CAAChB,IAAI,CAACT,IAAI,CAAE;IACtD,IAAI,CAACwB,OAAO,EAAE;MACZ,MAAM,IAAIE,KAAK,0BAAAR,MAAA,CAA0BT,IAAI,CAACT,IAAI,CAAE,CAAC;IACvD;IAEAS,IAAI,CAACkB,QAAQ,GAAGlB,IAAI,CAACkB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAElB,IAAI,CAACkB,QAAQ,IAAIC,sBAAc,CAAC,EAAE;MACtC,MAAM,IAAIF,KAAK,kCAAAR,MAAA,CAAkCT,IAAI,CAACkB,QAAQ,CAAE,CAAC;IACnE;IAEAlB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAIuB,wCAA2B,CAAC,EAAE;MACtD,MAAM,IAAIH,KAAK,oCAAAR,MAAA,CAAoCT,IAAI,CAACH,WAAW,CAAE,CAAC;IACxE;IAGA,IAAMW,KAAK,GAAG/B,IAAI,CAACgC,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCxB,SAAS,CAACwB,IAAI,CAAC,GAAG;MAChBA,IAAI,EAAJA,IAAI;MACJ0B,aAAa,EAAEN,OAAO,CAACM,aAAa;MACpCC,YAAY,EAAEP,OAAO,CAACO,YAAY;MAClC7C,IAAI,EAAE+B,KAAK;MACXlC,GAAG,EAAEkC,KAAK,CAACE,IAAI,CAAC,CAAC;MACjBH,cAAc,EAAdA,cAAc;MACdW,QAAQ,EAAElB,IAAI,CAACkB,QAAQ;MACvBrB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7B0B,UAAU,EAAEvB,IAAI,CAACuB,UAAU,IAAIR,OAAO,CAACQ,UAAU;MACjDC,SAAS,EAAExB,IAAI,CAACwB,SAAS;MACzBC,KAAK,EAAEzB,IAAI,CAACyB,KAAK;MACjBpB,SAAS,EAATA,SAAS;MACTC,SAAS,EAATA;IACF,CAAC;EACH;EACA,OAAOnC,SAAS;AAClB;AAEA,SAASC,UAAUA,CAACH,MAAoC,EAAkB;EACxE,IAAIyD,IAAoB,GAAG,EAAE;EAC7B,KAAK,IAAMC,CAAC,IAAI1D,MAAM,EAAE;IACtByD,IAAI,CAACzC,IAAI,CAAChB,MAAM,CAAC0D,CAAC,CAAC,CAAC;IACpB,IAAI1D,MAAM,CAAC0D,CAAC,CAAC,CAAChB,QAAQ,EAAE;MACtBe,IAAI,GAAGA,IAAI,CAACjB,MAAM,CAACrC,UAAU,CAACH,MAAM,CAAC0D,CAAC,CAAC,CAAC1D,MAAO,CAAC,CAAC;IACnD;EACF;EACA,OAAOyD,IAAI;AACb"}

package/dist/es5/parquetjs/schema/shred.js

@@ -4,13 +4,14 @@ var _typeof = require("@babel/runtime/helpers/typeof");
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
-Object.defineProperty(exports, "ParquetBuffer", {
+Object.defineProperty(exports, "ParquetRowGroup", {
   enumerable: true,
   get: function get() {
-    return _declare.ParquetBuffer;
+    return _declare.ParquetRowGroup;
   }
 });
-exports.materializeRecords = materializeRecords;
+exports.materializeColumns = materializeColumns;
+exports.materializeRows = materializeRows;
 exports.shredBuffer = shredBuffer;
 exports.shredRecord = shredRecord;
 var _declare = require("./declare");
@@ -45,24 +46,24 @@ function shredBuffer(schema) {
     columnData: columnData
   };
 }
-function shredRecord(schema, record, buffer) {
+function shredRecord(schema, record, rowGroup) {
   var data = shredBuffer(schema).columnData;
   shredRecordFields(schema.fields, record, data, 0, 0);
-  if (buffer.rowCount === 0) {
-    buffer.rowCount = 1;
-    buffer.columnData = data;
+  if (rowGroup.rowCount === 0) {
+    rowGroup.rowCount = 1;
+    rowGroup.columnData = data;
     return;
   }
-  buffer.rowCount += 1;
+  rowGroup.rowCount += 1;
   var _iterator2 = _createForOfIteratorHelper(schema.fieldList),
     _step2;
   try {
     for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
       var field = _step2.value;
-      Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);
-      Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);
-      Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);
-      buffer.columnData[field.key].count += data[field.key].count;
+      Array.prototype.push.apply(rowGroup.columnData[field.key].rlevels, data[field.key].rlevels);
+      Array.prototype.push.apply(rowGroup.columnData[field.key].dlevels, data[field.key].dlevels);
+      Array.prototype.push.apply(rowGroup.columnData[field.key].values, data[field.key].values);
+      rowGroup.columnData[field.key].count += data[field.key].count;
     }
   } catch (err) {
     _iterator2.e(err);
@@ -110,20 +111,20 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
     }
   }
 }
-function materializeRecords(schema, buffer) {
-  var records = [];
-  for (var i = 0; i < buffer.rowCount; i++) {
-    records.push({});
+function materializeRows(schema, rowGroup) {
+  var rows = [];
+  for (var i = 0; i < rowGroup.rowCount; i++) {
+    rows.push({});
   }
-  for (var key in buffer.columnData) {
-    var columnData = buffer.columnData[key];
+  for (var key in rowGroup.columnData) {
+    var columnData = rowGroup.columnData[key];
     if (columnData.count) {
-      materializeColumn(schema, columnData, key, records);
+      materializeColumnAsRows(schema, columnData, key, rows);
     }
   }
-  return records;
+  return rows;
 }
-function materializeColumn(schema, columnData, key, records) {
+function materializeColumnAsRows(schema, columnData, key, rows) {
   var field = schema.findField(key);
   var branch = schema.findFieldBranch(key);
   var rLevels = new Array(field.rLevelMax + 1).fill(0);
@@ -134,7 +135,7 @@ function materializeColumn(schema, columnData, key, records) {
     rLevels[rLevel]++;
     rLevels.fill(0, rLevel + 1);
     var rIndex = 0;
-    var record = records[rLevels[rIndex++] - 1];
+    var record = rows[rLevels[rIndex++] - 1];
     var _iterator3 = _createForOfIteratorHelper(branch),
       _step3;
     try {
@@ -184,4 +185,98 @@ function materializeColumn(schema, columnData, key, records) {
     }
   }
 }
+function materializeColumns(schema, rowGroup) {
+  var columns = {};
+  for (var key in rowGroup.columnData) {
+    var columnData = rowGroup.columnData[key];
+    if (columnData.count) {
+      materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);
+    }
+  }
+  return columns;
+}
+function materializeColumnAsColumnarArray(schema, columnData, rowCount, key, columns) {
+  if (columnData.count <= 0) {
+    return;
+  }
+  var field = schema.findField(key);
+  var branch = schema.findFieldBranch(key);
+  var columnName = branch[0].name;
+  var column;
+  var values = columnData.values;
+  if (values.length === rowCount && branch[0].primitiveType) {
+    column = values;
+  }
+  if (column) {
+    columns[columnName] = column;
+    return;
+  }
+  column = new Array(rowCount);
+  for (var i = 0; i < rowCount; i++) {
+    column[i] = {};
+  }
+  columns[columnName] = column;
+  var rLevels = new Array(field.rLevelMax + 1).fill(0);
+  var vIndex = 0;
+  for (var _i = 0; _i < columnData.count; _i++) {
+    var dLevel = columnData.dlevels[_i];
+    var rLevel = columnData.rlevels[_i];
+    rLevels[rLevel]++;
+    rLevels.fill(0, rLevel + 1);
+    var rIndex = 0;
+    var record = column[rLevels[rIndex++] - 1];
+    var _iterator4 = _createForOfIteratorHelper(branch),
+      _step4;
+    try {
+      for (_iterator4.s(); !(_step4 = _iterator4.n()).done;) {
+        var step = _step4.value;
+        if (step === field || dLevel < step.dLevelMax) {
+          break;
+        }
+        switch (step.repetitionType) {
+          case 'REPEATED':
+            if (!(step.name in record)) {
+              record[step.name] = [];
+            }
+            var _ix2 = rLevels[rIndex++];
+            while (record[step.name].length <= _ix2) {
+              record[step.name].push({});
+            }
+            record = record[step.name][_ix2];
+            break;
+          default:
+            record[step.name] = record[step.name] || {};
+            record = record[step.name];
+        }
+      }
+    } catch (err) {
+      _iterator4.e(err);
+    } finally {
+      _iterator4.f();
+    }
+    if (dLevel === field.dLevelMax) {
+      var value = Types.fromPrimitive(field.originalType || field.primitiveType, columnData.values[vIndex], field);
+      vIndex++;
+      switch (field.repetitionType) {
+        case 'REPEATED':
+          if (!(field.name in record)) {
+            record[field.name] = [];
+          }
+          var ix = rLevels[rIndex];
+          while (record[field.name].length <= ix) {
+            record[field.name].push(null);
+          }
+          record[field.name][ix] = value;
+          break;
+        default:
+          record[field.name] = value;
+      }
+    }
+  }
+  for (var _i2 = 0; _i2 < rowCount; ++_i2) {
+    if (columnName in column[_i2]) {
+      column[_i2] = column[_i2][columnName];
+    }
+  }
+}
 //# sourceMappingURL=shred.js.map

package/dist/es5/parquetjs/schema/shred.js.map

@@ -1 +1 @@
-{"version":3,"file":"shred.js","names":["_declare","require","Types","_interopRequireWildcard","_getRequireWildcardCache","nodeInterop","WeakMap","cacheBabelInterop","cacheNodeInterop","obj","__esModule","_typeof","default","cache","has","get","newObj","hasPropertyDescriptor","Object","defineProperty","getOwnPropertyDescriptor","key","prototype","hasOwnProperty","call","desc","set","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","step","next","_e2","return","minLen","_arrayLikeToArray","toString","slice","constructor","name","from","test","arr","len","arr2","shredBuffer","schema","columnData","_iterator","fieldList","_step","field","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","_iterator2","_step2","push","apply","rLevel","dLevel","undefined","Boolean","repetitionType","Error","concat","isNested","rlvl","rLevelMax","dLevelMax","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","_iterator3","_step3","ix","fromPrimitive"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n  const columnData: Record<string, ParquetData> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the buffer is as\n * follows:\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the buffer */\n  if (buffer.rowCount === 0) {\n    buffer.rowCount = 1;\n    buffer.columnData = data;\n    return;\n  }\n  buffer.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n    buffer.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: any,\n  data: Record<string, ParquetData>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n  const records: ParquetRecord[] = [];\n  for (let i = 0; i < buffer.rowCount; i++) {\n    records.push({});\n  }\n  for (const key in buffer.columnData) {\n    const columnData = buffer.columnData[key];\n    if (columnData.count) {\n      materializeColumn(schema, columnData, key, records);\n    }\n  }\n  return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n  schema: ParquetSchema,\n  columnData: ParquetData,\n  key: string,\n  records: ParquetRecord[]\n): void {\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = records[rLevels[rIndex++] - 1];\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n}\n\n// Columnar export\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n *\nexport function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {\n  const columns: ParquetRecord = {};\n  for (const key in buffer.columnData) {\n    const columnData = buffer.columnData[key];\n    if (columnData.count) {\n      extractColumn(schema, columnData, key, columns);\n    }\n  }\n  return columns;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction extractColumn(\n  schema: ParquetSchema,\n  columnData: ParquetData,\n  key: string,\n  columns: Record<string, unknown> \n) {\n  if (columnData.count <= 0) {\n    return;\n  }\n\n  const record = columns;\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n\n  let i = 0;\n  const dLevel = columnData.dlevels[i];\n  const rLevel = columnData.rlevels[i];\n  rLevels[rLevel]++;\n  rLevels.fill(0, rLevel + 1);\n\n  let rIndex = 0;\n  let record = records[rLevels[rIndex++] - 1];\n\n  // Internal nodes\n  for (const step of branch) {\n    if (step === field || dLevel < step.dLevelMax) {\n      break;\n    }\n\n    switch (step.repetitionType) {\n      case 'REPEATED':\n        if (!(step.name in record)) {\n          // eslint-disable max-depth\n          record[step.name] = [];\n        }\n        const ix = rLevels[rIndex++];\n        while (record[step.name].length <= ix) {\n          // eslint-disable max-depth\n          record[step.name].push({});\n        }\n        record = record[step.name][ix];\n        break;\n\n      default:\n        record[step.name] = record[step.name] || {};\n        record = record[step.name];\n    }\n  }\n\n  // Leaf node\n  if (dLevel === field.dLevelMax) {\n    const value = Types.fromPrimitive(\n      // @ts-ignore\n      field.originalType || field.primitiveType,\n      columnData.values[vIndex],\n      field\n    );\n    vIndex++;\n\n    switch (field.repetitionType) {\n      case 'REPEATED':\n        if (!(field.name in record)) {\n          // eslint-disable max-depth\n          record[field.name] = [];\n        }\n        const ix = rLevels[rIndex];\n        while (record[field.name].length <= ix) {\n          // eslint-disable max-depth\n          record[field.name].push(null);\n        }\n        record[field.name][ix] = value;\n        break;\n\n      default:\n        record[field.name] = value;\n    }\n  }\n}\n*/\n"],"mappings":";;;;;;;;;;;;;;;AAEA,IAAAA,QAAA,GAAAC,OAAA;AAEA,IAAAC,KAAA,GAAAC,uBAAA,CAAAF,OAAA;AAAiC,SAAAG,yBAAAC,WAAA,eAAAC,OAAA,kCAAAC,iBAAA,OAAAD,OAAA,QAAAE,gBAAA,OAAAF,OAAA,YAAAF,wBAAA,YAAAA,yBAAAC,WAAA,WAAAA,WAAA,GAAAG,gBAAA,GAAAD,iBAAA,KAAAF,WAAA;AAAA,SAAAF,wBAAAM,GAAA,EAAAJ,WAAA,SAAAA,WAAA,IAAAI,GAAA,IAAAA,GAAA,CAAAC,UAAA,WAAAD,GAAA,QAAAA,GAAA,aAAAE,OAAA,CAAAF,GAAA,yBAAAA,GAAA,4BAAAG,OAAA,EAAAH,GAAA,UAAAI,KAAA,GAAAT,wBAAA,CAAAC,WAAA,OAAAQ,KAAA,IAAAA,KAAA,CAAAC,GAAA,CAAAL,GAAA,YAAAI,KAAA,CAAAE,GAAA,CAAAN,GAAA,SAAAO,MAAA,WAAAC,qBAAA,GAAAC,MAAA,CAAAC,cAAA,IAAAD,MAAA,CAAAE,wBAAA,WAAAC,GAAA,IAAAZ,GAAA,QAAAY,GAAA,kBAAAH,MAAA,CAAAI,SAAA,CAAAC,cAAA,CAAAC,IAAA,CAAAf,GAAA,EAAAY,GAAA,SAAAI,IAAA,GAAAR,qBAAA,GAAAC,MAAA,CAAAE,wBAAA,CAAAX,GAAA,EAAAY,GAAA,cAAAI,IAAA,KAAAA,IAAA,CAAAV,GAAA,IAAAU,IAAA,CAAAC,GAAA,KAAAR,MAAA,CAAAC,cAAA,CAAAH,MAAA,EAAAK,GAAA,EAAAI,IAAA,YAAAT,MAAA,CAAAK,GAAA,IAAAZ,GAAA,CAAAY,GAAA,SAAAL,MAAA,CAAAJ,OAAA,GAAAH,GAAA,MAAAI,KAAA,IAAAA,KAAA,CAAAa,GAAA,CAAAjB,GAAA,EAAAO,MAAA,YAAAA,MAAA;AAAA,SAAAW,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAN,IAAA,CAAAI,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAU,IAAA,GAAApB,EAAA,CAAAqB,IAAA,IAAAJ,gBAAA,GAAAG,IAAA,CAAAT,IAAA,SAAAS,IAAA,KAAAP,CAAA,WAAAA,EAAAS,GAAA,IAAAJ,MAAA,SAAAC,GAAA,GAAAG,GAAA,KAAAP,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAuB,MAAA,UAAAvB,EAAA,CAAAuB,MAAA,oBAAAL,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA0B,MAAA,SAAA1B,CAAA,qBAAAA,CAAA,sBAAA2B,iBAAA,CAAA3B,CAAA,EAAA0B,MAAA,OAAAd,CAAA,GAAAtB,MAAA,CAAAI,SAAA,CAAAkC,QAAA,CAAAhC,IAAA,CAAAI,CAAA,EAAA6B,KAAA,aAAAjB,CAAA,iBAAAZ,CAAA,CAAA8B,WAAA,EAAAlB,CAAA,GAAAZ,CAAA,CAAA8B,WAAA,CAAAC,IAAA,MAAAnB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA2B,IAAA,CAAAhC,CAAA,OAAAY,CAAA,+DAAAqB,IAAA,CAAArB,CAAA,UAAAe,iBAAA,CAAA3B,CAAA,EAAA0B,MAAA;AAAA,SAAAC,kBAAAO,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA1B,MAAA,EAAA2B,GAAA,GAAAD,GAAA,CAAA1B,MAAA,WAAAC,CAAA,MAAA2B,IAAA,OAAA/B,KAAA,CAAA8B,GAAA,GAAA1B,CAAA,GAAA0B,GAAA,EAAA1B,CAAA,IAAA2B,IAAA,CAAA3B,CAAA,IAAAyB,GAAA,CAAAzB,CAAA,UAAA2B,IAAA;AAI1B,SAASC,WAAWA,CAACC,MAAqB,EAAiB;EAChE,IAAMC,UAAuC,GAAG,CAAC,CAAC;EAAC,IAAAC,SAAA,GAAAzC,0BAAA,CAC/BuC,MAAM,CAACG,SAAS;IAAAC,KAAA;EAAA;IAApC,KAAAF,SAAA,CAAA7B,CAAA,MAAA+B,KAAA,GAAAF,SAAA,CAAA5B,CAAA,IAAAC,IAAA,GAAsC;MAAA,IAA3B8B,KAAK,GAAAD,KAAA,CAAA5B,KAAA;MACdyB,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,GAAG;QACtBmD,OAAO,EAAE,EAAE;QACXC,OAAO,EAAE,EAAE;QACXC,MAAM,EAAE,EAAE;QACVC,WAAW,EAAE,EAAE;QACfC,KAAK,EAAE;MACT,CAAC;IACH;EAAC,SAAA3B,GAAA;IAAAmB,SAAA,CAAAzB,CAAA,CAAAM,GAAA;EAAA;IAAAmB,SAAA,CAAAvB,CAAA;EAAA;EACD,OAAO;IAACgC,QAAQ,EAAE,CAAC;IAAEV,UAAU,EAAVA;EAAU,CAAC;AAClC;AAwBO,SAASW,WAAWA,CAACZ,MAAqB,EAAEa,MAAW,EAAEC,MAAqB,EAAQ;EAE3F,IAAMC,IAAI,GAAGhB,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Ce,iBAAiB,CAAChB,MAAM,CAACiB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;EAGpD,IAAID,MAAM,CAACH,QAAQ,KAAK,CAAC,EAAE;IACzBG,MAAM,CAACH,QAAQ,GAAG,CAAC;IACnBG,MAAM,CAACb,UAAU,GAAGc,IAAI;IACxB;EACF;EACAD,MAAM,CAACH,QAAQ,IAAI,CAAC;EAAC,IAAAO,UAAA,GAAAzD,0BAAA,CACDuC,MAAM,CAACG,SAAS;IAAAgB,MAAA;EAAA;IAApC,KAAAD,UAAA,CAAA7C,CAAA,MAAA8C,MAAA,GAAAD,UAAA,CAAA5C,CAAA,IAAAC,IAAA,GAAsC;MAAA,IAA3B8B,KAAK,GAAAc,MAAA,CAAA3C,KAAA;MACdT,KAAK,CAACX,SAAS,CAACgE,IAAI,CAACC,KAAK,CAACP,MAAM,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,EAAEQ,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,CAAC;MACzFxC,KAAK,CAACX,SAAS,CAACgE,IAAI,CAACC,KAAK,CAACP,MAAM,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,EAAES,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,CAAC;MACzFvC,KAAK,CAACX,SAAS,CAACgE,IAAI,CAACC,KAAK,CAACP,MAAM,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACqD,MAAM,EAAEO,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACqD,MAAM,CAAC;MACvFM,MAAM,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK,IAAIK,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK;IAC7D;EAAC,SAAA3B,GAAA;IAAAmC,UAAA,CAAAzC,CAAA,CAAAM,GAAA;EAAA;IAAAmC,UAAA,CAAAvC,CAAA;EAAA;AACH;AAGA,SAASqC,iBAAiBA,CACxBC,MAAoC,EACpCJ,MAAW,EACXE,IAAiC,EACjCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,IAAM9B,IAAI,IAAIwB,MAAM,EAAE;IACzB,IAAMZ,KAAK,GAAGY,MAAM,CAACxB,IAAI,CAAC;IAG1B,IAAIe,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNR,KAAK,CAACZ,IAAI,IAAIoB,MAAM,IACpBA,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,KAAK+B,SAAS,IAChCX,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIoB,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACD,WAAW,KAAKzB,KAAK,EAAE;QAC5CyC,MAAM,GAAGK,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLe,MAAM,CAACY,IAAI,CAACP,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIe,MAAM,CAACtC,MAAM,KAAK,CAAC,IAAIuD,OAAO,CAACZ,MAAM,CAAC,IAAIR,KAAK,CAACqB,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,4BAAAC,MAAA,CAA4BvB,KAAK,CAACZ,IAAI,CAAE,CAAC;IAC1D;IACA,IAAIe,MAAM,CAACtC,MAAM,GAAG,CAAC,IAAImC,KAAK,CAACqB,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,+BAAAC,MAAA,CAA+BvB,KAAK,CAACZ,IAAI,CAAE,CAAC;IAC7D;IAGA,IAAIe,MAAM,CAACtC,MAAM,KAAK,CAAC,EAAE;MACvB,IAAImC,KAAK,CAACwB,QAAQ,EAAE;QAClBb,iBAAiB,CAACX,KAAK,CAACY,MAAM,EAAG,IAAI,EAAEF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC9D,CAAC,MAAM;QACLR,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;IAGA,KAAK,IAAIpD,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGqC,MAAM,CAACtC,MAAM,EAAEC,CAAC,EAAE,EAAE;MACtC,IAAM2D,IAAI,GAAG3D,CAAC,KAAK,CAAC,GAAGmD,MAAM,GAAGjB,KAAK,CAAC0B,SAAS;MAC/C,IAAI1B,KAAK,CAACwB,QAAQ,EAAE;QAClBb,iBAAiB,CAACX,KAAK,CAACY,MAAM,EAAGT,MAAM,CAACrC,CAAC,CAAC,EAAE4C,IAAI,EAAEe,IAAI,EAAEzB,KAAK,CAAC2B,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLjB,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,CAACa,IAAI,CAACU,IAAI,CAAC;QAClCf,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,CAACc,IAAI,CAACf,KAAK,CAAC2B,SAAS,CAAC;QAC7CjB,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACqD,MAAM,CAACY,IAAI,CACzBpF,KAAK,CAACiG,WAAW,CAAE5B,KAAK,CAAC6B,YAAY,IAAI7B,KAAK,CAAC8B,aAAa,EAAI3B,MAAM,CAACrC,CAAC,CAAC,CAC3E,CAAC;MACH;IACF;EACF;AACF;AAqBO,SAASiE,kBAAkBA,CAACpC,MAAqB,EAAEc,MAAqB,EAAmB;EAChG,IAAMuB,OAAwB,GAAG,EAAE;EACnC,KAAK,IAAIlE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG2C,MAAM,CAACH,QAAQ,EAAExC,CAAC,EAAE,EAAE;IACxCkE,OAAO,CAACjB,IAAI,CAAC,CAAC,CAAC,CAAC;EAClB;EACA,KAAK,IAAMjE,GAAG,IAAI2D,MAAM,CAACb,UAAU,EAAE;IACnC,IAAMA,UAAU,GAAGa,MAAM,CAACb,UAAU,CAAC9C,GAAG,CAAC;IACzC,IAAI8C,UAAU,CAACS,KAAK,EAAE;MACpB4B,iBAAiB,CAACtC,MAAM,EAAEC,UAAU,EAAE9C,GAAG,EAAEkF,OAAO,CAAC;IACrD;EACF;EACA,OAAOA,OAAO;AAChB;AAGA,SAASC,iBAAiBA,CACxBtC,MAAqB,EACrBC,UAAuB,EACvB9C,GAAW,EACXkF,OAAwB,EAClB;EACN,IAAMhC,KAAK,GAAGL,MAAM,CAACuC,SAAS,CAACpF,GAAG,CAAC;EACnC,IAAMqF,MAAM,GAAGxC,MAAM,CAACyC,eAAe,CAACtF,GAAG,CAAC;EAG1C,IAAMuF,OAAiB,GAAG,IAAI3E,KAAK,CAACsC,KAAK,CAAC0B,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIzE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG8B,UAAU,CAACS,KAAK,EAAEvC,CAAC,EAAE,EAAE;IACzC,IAAMoD,MAAM,GAAGtB,UAAU,CAACK,OAAO,CAACnC,CAAC,CAAC;IACpC,IAAMmD,MAAM,GAAGrB,UAAU,CAACM,OAAO,CAACpC,CAAC,CAAC;IACpCuE,OAAO,CAACpB,MAAM,CAAC,EAAE;IACjBoB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAErB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAIuB,MAAM,GAAG,CAAC;IACd,IAAIhC,MAAM,GAAGwB,OAAO,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;IAAC,IAAAC,UAAA,GAAArF,0BAAA,CAGzB+E,MAAM;MAAAO,MAAA;IAAA;MAAzB,KAAAD,UAAA,CAAAzE,CAAA,MAAA0E,MAAA,GAAAD,UAAA,CAAAxE,CAAA,IAAAC,IAAA,GAA2B;QAAA,IAAhBS,IAAI,GAAA+D,MAAA,CAAAvE,KAAA;QACb,IAAIQ,IAAI,KAAKqB,KAAK,IAAIkB,MAAM,GAAGvC,IAAI,CAACgD,SAAS,EAAE;UAC7C;QACF;QAEA,QAAQhD,IAAI,CAAC0C,cAAc;UACzB,KAAK,UAAU;YACb,IAAI,EAAE1C,IAAI,CAACS,IAAI,IAAIoB,MAAM,CAAC,EAAE;cAE1BA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,GAAG,EAAE;YACxB;YACA,IAAMuD,GAAE,GAAGN,OAAO,CAACG,MAAM,EAAE,CAAC;YAC5B,OAAOhC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAACvB,MAAM,IAAI8E,GAAE,EAAE;cAErCnC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAAC2B,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B;YACAP,MAAM,GAAGA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAACuD,GAAE,CAAC;YAC9B;UAEF;YACEnC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,GAAGoB,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3CoB,MAAM,GAAGA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC;QAC9B;MACF;IAAC,SAAAV,GAAA;MAAA+D,UAAA,CAAArE,CAAA,CAAAM,GAAA;IAAA;MAAA+D,UAAA,CAAAnE,CAAA;IAAA;IAGD,IAAI4C,MAAM,KAAKlB,KAAK,CAAC2B,SAAS,EAAE;MAC9B,IAAMxD,KAAK,GAAGxC,KAAK,CAACiH,aAAa,CAE/B5C,KAAK,CAAC6B,YAAY,IAAI7B,KAAK,CAAC8B,aAAa,EACzClC,UAAU,CAACO,MAAM,CAACoC,MAAM,CAAC,EACzBvC,KACF,CAAC;MACDuC,MAAM,EAAE;MAER,QAAQvC,KAAK,CAACqB,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAErB,KAAK,CAACZ,IAAI,IAAIoB,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,IAAMuD,EAAE,GAAGN,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOhC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACvB,MAAM,IAAI8E,EAAE,EAAE;YAEtCnC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAAC2B,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACuD,EAAE,CAAC,GAAGxE,KAAK;UAC9B;QAEF;UACEqC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,GAAGjB,KAAK;MAC9B;IACF;EACF;AACF"}
+{"version":3,"file":"shred.js","names":["_declare","require","Types","_interopRequireWildcard","_getRequireWildcardCache","nodeInterop","WeakMap","cacheBabelInterop","cacheNodeInterop","obj","__esModule","_typeof","default","cache","has","get","newObj","hasPropertyDescriptor","Object","defineProperty","getOwnPropertyDescriptor","key","prototype","hasOwnProperty","call","desc","set","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","step","next","_e2","return","minLen","_arrayLikeToArray","toString","slice","constructor","name","from","test","arr","len","arr2","shredBuffer","schema","columnData","_iterator","fieldList","_step","field","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","rowGroup","data","shredRecordFields","fields","_iterator2","_step2","push","apply","rLevel","dLevel","undefined","Boolean","repetitionType","Error","concat","isNested","rlvl","rLevelMax","dLevelMax","toPrimitive","originalType","primitiveType","materializeRows","rows","materializeColumnAsRows","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","_iterator3","_step3","ix","fromPrimitive","materializeColumns","columns","materializeColumnAsColumnarArray","columnName","column","_iterator4","_step4"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ArrayType} from '@loaders.gl/schema';\nimport {ParquetRowGroup, ParquetColumnChunk, ParquetField, ParquetRow} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetRowGroup};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetRowGroup {\n  const columnData: Record<string, ParquetColumnChunk> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The rowGroup argument must point to an object into which the shredded record\n * will be returned. You may re-use the rowGroup for repeated calls to this function\n * to append to an existing rowGroup, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the rowGroup is as\n * follows:\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(\n  schema: ParquetSchema,\n  record: ParquetRow,\n  rowGroup: ParquetRowGroup\n): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the rowGroup */\n  if (rowGroup.rowCount === 0) {\n    rowGroup.rowCount = 1;\n    rowGroup.columnData = data;\n    return;\n  }\n  rowGroup.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(rowGroup.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(rowGroup.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(rowGroup.columnData[field.key].values, data[field.key].values);\n    rowGroup.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: ParquetRow,\n  data: Record<string, ParquetColumnChunk>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null!, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The rowGroup argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRows(schema: ParquetSchema, rowGroup: ParquetRowGroup): ParquetRow[] {\n  const rows: ParquetRow[] = [];\n  // rows = new Array(rowGroup.rowCount).fill({})'\n  for (let i = 0; i < rowGroup.rowCount; i++) {\n    rows.push({});\n  }\n  for (const key in rowGroup.columnData) {\n    const columnData = rowGroup.columnData[key];\n    if (columnData.count) {\n      materializeColumnAsRows(schema, columnData, key, rows);\n    }\n  }\n  return rows;\n}\n\n/** Populate record fields for one column */\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumnAsRows(\n  schema: ParquetSchema,\n  columnData: ParquetColumnChunk,\n  key: string,\n  rows: ParquetRow[]\n): void {\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = rows[rLevels[rIndex++] - 1];\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n}\n\n// Columnar export\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The rowGroup argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeColumns(\n  schema: ParquetSchema,\n  rowGroup: ParquetRowGroup\n): Record<string, ArrayType> {\n  const columns: Record<string, ArrayType> = {};\n  for (const key in rowGroup.columnData) {\n    const columnData = rowGroup.columnData[key];\n    if (columnData.count) {\n      materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);\n    }\n  }\n  return columns;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumnAsColumnarArray(\n  schema: ParquetSchema,\n  columnData: ParquetColumnChunk,\n  rowCount: number,\n  key: string,\n  columns: Record<string, ArrayType<any>>\n) {\n  if (columnData.count <= 0) {\n    return;\n  }\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  const columnName = branch[0].name;\n\n  let column: ArrayType | undefined;\n  const {values} = columnData;\n  if (values.length === rowCount && branch[0].primitiveType) {\n    // if (branch[0].repetitionType === `REQUIRED`) {\n    //   switch (branch[0].primitiveType) {\n    //     case 'INT32': return values instanceof Int32Array ? values : new Int32Array(values);\n    //   }\n    // }\n    column = values;\n  }\n\n  if (column) {\n    columns[columnName] = column;\n    return;\n  }\n\n  column = new Array(rowCount);\n  for (let i = 0; i < rowCount; i++) {\n    column[i] = {};\n  }\n  columns[columnName] = column;\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = column[rLevels[rIndex++] - 1] as ParquetRow;\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n\n  // Remove one level of nesting\n  for (let i = 0; i < rowCount; ++i) {\n    if (columnName in (column[i] as object)) {\n      column[i] = (column[i] as object)[columnName];\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;AAGA,IAAAA,QAAA,GAAAC,OAAA;AAEA,IAAAC,KAAA,GAAAC,uBAAA,CAAAF,OAAA;AAAiC,SAAAG,yBAAAC,WAAA,eAAAC,OAAA,kCAAAC,iBAAA,OAAAD,OAAA,QAAAE,gBAAA,OAAAF,OAAA,YAAAF,wBAAA,YAAAA,yBAAAC,WAAA,WAAAA,WAAA,GAAAG,gBAAA,GAAAD,iBAAA,KAAAF,WAAA;AAAA,SAAAF,wBAAAM,GAAA,EAAAJ,WAAA,SAAAA,WAAA,IAAAI,GAAA,IAAAA,GAAA,CAAAC,UAAA,WAAAD,GAAA,QAAAA,GAAA,aAAAE,OAAA,CAAAF,GAAA,yBAAAA,GAAA,4BAAAG,OAAA,EAAAH,GAAA,UAAAI,KAAA,GAAAT,wBAAA,CAAAC,WAAA,OAAAQ,KAAA,IAAAA,KAAA,CAAAC,GAAA,CAAAL,GAAA,YAAAI,KAAA,CAAAE,GAAA,CAAAN,GAAA,SAAAO,MAAA,WAAAC,qBAAA,GAAAC,MAAA,CAAAC,cAAA,IAAAD,MAAA,CAAAE,wBAAA,WAAAC,GAAA,IAAAZ,GAAA,QAAAY,GAAA,kBAAAH,MAAA,CAAAI,SAAA,CAAAC,cAAA,CAAAC,IAAA,CAAAf,GAAA,EAAAY,GAAA,SAAAI,IAAA,GAAAR,qBAAA,GAAAC,MAAA,CAAAE,wBAAA,CAAAX,GAAA,EAAAY,GAAA,cAAAI,IAAA,KAAAA,IAAA,CAAAV,GAAA,IAAAU,IAAA,CAAAC,GAAA,KAAAR,MAAA,CAAAC,cAAA,CAAAH,MAAA,EAAAK,GAAA,EAAAI,IAAA,YAAAT,MAAA,CAAAK,GAAA,IAAAZ,GAAA,CAAAY,GAAA,SAAAL,MAAA,CAAAJ,OAAA,GAAAH,GAAA,MAAAI,KAAA,IAAAA,KAAA,CAAAa,GAAA,CAAAjB,GAAA,EAAAO,MAAA,YAAAA,MAAA;AAAA,SAAAW,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAN,IAAA,CAAAI,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAU,IAAA,GAAApB,EAAA,CAAAqB,IAAA,IAAAJ,gBAAA,GAAAG,IAAA,CAAAT,IAAA,SAAAS,IAAA,KAAAP,CAAA,WAAAA,EAAAS,GAAA,IAAAJ,MAAA,SAAAC,GAAA,GAAAG,GAAA,KAAAP,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAuB,MAAA,UAAAvB,EAAA,CAAAuB,MAAA,oBAAAL,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA0B,MAAA,SAAA1B,CAAA,qBAAAA,CAAA,sBAAA2B,iBAAA,CAAA3B,CAAA,EAAA0B,MAAA,OAAAd,CAAA,GAAAtB,MAAA,CAAAI,SAAA,CAAAkC,QAAA,CAAAhC,IAAA,CAAAI,CAAA,EAAA6B,KAAA,aAAAjB,CAAA,iBAAAZ,CAAA,CAAA8B,WAAA,EAAAlB,CAAA,GAAAZ,CAAA,CAAA8B,WAAA,CAAAC,IAAA,MAAAnB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA2B,IAAA,CAAAhC,CAAA,OAAAY,CAAA,+DAAAqB,IAAA,CAAArB,CAAA,UAAAe,iBAAA,CAAA3B,CAAA,EAAA0B,MAAA;AAAA,SAAAC,kBAAAO,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA1B,MAAA,EAAA2B,GAAA,GAAAD,GAAA,CAAA1B,MAAA,WAAAC,CAAA,MAAA2B,IAAA,OAAA/B,KAAA,CAAA8B,GAAA,GAAA1B,CAAA,GAAA0B,GAAA,EAAA1B,CAAA,IAAA2B,IAAA,CAAA3B,CAAA,IAAAyB,GAAA,CAAAzB,CAAA,UAAA2B,IAAA;AAI1B,SAASC,WAAWA,CAACC,MAAqB,EAAmB;EAClE,IAAMC,UAA8C,GAAG,CAAC,CAAC;EAAC,IAAAC,SAAA,GAAAzC,0BAAA,CACtCuC,MAAM,CAACG,SAAS;IAAAC,KAAA;EAAA;IAApC,KAAAF,SAAA,CAAA7B,CAAA,MAAA+B,KAAA,GAAAF,SAAA,CAAA5B,CAAA,IAAAC,IAAA,GAAsC;MAAA,IAA3B8B,KAAK,GAAAD,KAAA,CAAA5B,KAAA;MACdyB,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,GAAG;QACtBmD,OAAO,EAAE,EAAE;QACXC,OAAO,EAAE,EAAE;QACXC,MAAM,EAAE,EAAE;QACVC,WAAW,EAAE,EAAE;QACfC,KAAK,EAAE;MACT,CAAC;IACH;EAAC,SAAA3B,GAAA;IAAAmB,SAAA,CAAAzB,CAAA,CAAAM,GAAA;EAAA;IAAAmB,SAAA,CAAAvB,CAAA;EAAA;EACD,OAAO;IAACgC,QAAQ,EAAE,CAAC;IAAEV,UAAU,EAAVA;EAAU,CAAC;AAClC;AAwBO,SAASW,WAAWA,CACzBZ,MAAqB,EACrBa,MAAkB,EAClBC,QAAyB,EACnB;EAEN,IAAMC,IAAI,GAAGhB,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Ce,iBAAiB,CAAChB,MAAM,CAACiB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;EAGpD,IAAID,QAAQ,CAACH,QAAQ,KAAK,CAAC,EAAE;IAC3BG,QAAQ,CAACH,QAAQ,GAAG,CAAC;IACrBG,QAAQ,CAACb,UAAU,GAAGc,IAAI;IAC1B;EACF;EACAD,QAAQ,CAACH,QAAQ,IAAI,CAAC;EAAC,IAAAO,UAAA,GAAAzD,0BAAA,CACHuC,MAAM,CAACG,SAAS;IAAAgB,MAAA;EAAA;IAApC,KAAAD,UAAA,CAAA7C,CAAA,MAAA8C,MAAA,GAAAD,UAAA,CAAA5C,CAAA,IAAAC,IAAA,GAAsC;MAAA,IAA3B8B,KAAK,GAAAc,MAAA,CAAA3C,KAAA;MACdT,KAAK,CAACX,SAAS,CAACgE,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,EAAEQ,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,CAAC;MAC3FxC,KAAK,CAACX,SAAS,CAACgE,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,EAAES,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,CAAC;MAC3FvC,KAAK,CAACX,SAAS,CAACgE,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACqD,MAAM,EAAEO,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACqD,MAAM,CAAC;MACzFM,QAAQ,CAACb,UAAU,CAACI,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK,IAAIK,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK;IAC/D;EAAC,SAAA3B,GAAA;IAAAmC,UAAA,CAAAzC,CAAA,CAAAM,GAAA;EAAA;IAAAmC,UAAA,CAAAvC,CAAA;EAAA;AACH;AAGA,SAASqC,iBAAiBA,CACxBC,MAAoC,EACpCJ,MAAkB,EAClBE,IAAwC,EACxCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,IAAM9B,IAAI,IAAIwB,MAAM,EAAE;IACzB,IAAMZ,KAAK,GAAGY,MAAM,CAACxB,IAAI,CAAC;IAG1B,IAAIe,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNR,KAAK,CAACZ,IAAI,IAAIoB,MAAM,IACpBA,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,KAAK+B,SAAS,IAChCX,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIoB,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACD,WAAW,KAAKzB,KAAK,EAAE;QAC5CyC,MAAM,GAAGK,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLe,MAAM,CAACY,IAAI,CAACP,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIe,MAAM,CAACtC,MAAM,KAAK,CAAC,IAAIuD,OAAO,CAACZ,MAAM,CAAC,IAAIR,KAAK,CAACqB,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,4BAAAC,MAAA,CAA4BvB,KAAK,CAACZ,IAAI,CAAE,CAAC;IAC1D;IACA,IAAIe,MAAM,CAACtC,MAAM,GAAG,CAAC,IAAImC,KAAK,CAACqB,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,+BAAAC,MAAA,CAA+BvB,KAAK,CAACZ,IAAI,CAAE,CAAC;IAC7D;IAGA,IAAIe,MAAM,CAACtC,MAAM,KAAK,CAAC,EAAE;MACvB,IAAImC,KAAK,CAACwB,QAAQ,EAAE;QAClBb,iBAAiB,CAACX,KAAK,CAACY,MAAM,EAAG,IAAI,EAAGF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC/D,CAAC,MAAM;QACLR,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;IAGA,KAAK,IAAIpD,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGqC,MAAM,CAACtC,MAAM,EAAEC,CAAC,EAAE,EAAE;MACtC,IAAM2D,IAAI,GAAG3D,CAAC,KAAK,CAAC,GAAGmD,MAAM,GAAGjB,KAAK,CAAC0B,SAAS;MAC/C,IAAI1B,KAAK,CAACwB,QAAQ,EAAE;QAClBb,iBAAiB,CAACX,KAAK,CAACY,MAAM,EAAGT,MAAM,CAACrC,CAAC,CAAC,EAAE4C,IAAI,EAAEe,IAAI,EAAEzB,KAAK,CAAC2B,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLjB,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACuD,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACoD,OAAO,CAACa,IAAI,CAACU,IAAI,CAAC;QAClCf,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACmD,OAAO,CAACc,IAAI,CAACf,KAAK,CAAC2B,SAAS,CAAC;QAC7CjB,IAAI,CAACV,KAAK,CAAClD,GAAG,CAAC,CAACqD,MAAM,CAACY,IAAI,CACzBpF,KAAK,CAACiG,WAAW,CAAE5B,KAAK,CAAC6B,YAAY,IAAI7B,KAAK,CAAC8B,aAAa,EAAI3B,MAAM,CAACrC,CAAC,CAAC,CAC3E,CAAC;MACH;IACF;EACF;AACF;AAqBO,SAASiE,eAAeA,CAACpC,MAAqB,EAAEc,QAAyB,EAAgB;EAC9F,IAAMuB,IAAkB,GAAG,EAAE;EAE7B,KAAK,IAAIlE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG2C,QAAQ,CAACH,QAAQ,EAAExC,CAAC,EAAE,EAAE;IAC1CkE,IAAI,CAACjB,IAAI,CAAC,CAAC,CAAC,CAAC;EACf;EACA,KAAK,IAAMjE,GAAG,IAAI2D,QAAQ,CAACb,UAAU,EAAE;IACrC,IAAMA,UAAU,GAAGa,QAAQ,CAACb,UAAU,CAAC9C,GAAG,CAAC;IAC3C,IAAI8C,UAAU,CAACS,KAAK,EAAE;MACpB4B,uBAAuB,CAACtC,MAAM,EAAEC,UAAU,EAAE9C,GAAG,EAAEkF,IAAI,CAAC;IACxD;EACF;EACA,OAAOA,IAAI;AACb;AAIA,SAASC,uBAAuBA,CAC9BtC,MAAqB,EACrBC,UAA8B,EAC9B9C,GAAW,EACXkF,IAAkB,EACZ;EACN,IAAMhC,KAAK,GAAGL,MAAM,CAACuC,SAAS,CAACpF,GAAG,CAAC;EACnC,IAAMqF,MAAM,GAAGxC,MAAM,CAACyC,eAAe,CAACtF,GAAG,CAAC;EAG1C,IAAMuF,OAAiB,GAAG,IAAI3E,KAAK,CAACsC,KAAK,CAAC0B,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIzE,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG8B,UAAU,CAACS,KAAK,EAAEvC,CAAC,EAAE,EAAE;IACzC,IAAMoD,MAAM,GAAGtB,UAAU,CAACK,OAAO,CAACnC,CAAC,CAAC;IACpC,IAAMmD,MAAM,GAAGrB,UAAU,CAACM,OAAO,CAACpC,CAAC,CAAC;IACpCuE,OAAO,CAACpB,MAAM,CAAC,EAAE;IACjBoB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAErB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAIuB,MAAM,GAAG,CAAC;IACd,IAAIhC,MAAM,GAAGwB,IAAI,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;IAAC,IAAAC,UAAA,GAAArF,0BAAA,CAGtB+E,MAAM;MAAAO,MAAA;IAAA;MAAzB,KAAAD,UAAA,CAAAzE,CAAA,MAAA0E,MAAA,GAAAD,UAAA,CAAAxE,CAAA,IAAAC,IAAA,GAA2B;QAAA,IAAhBS,IAAI,GAAA+D,MAAA,CAAAvE,KAAA;QACb,IAAIQ,IAAI,KAAKqB,KAAK,IAAIkB,MAAM,GAAGvC,IAAI,CAACgD,SAAS,EAAE;UAC7C;QACF;QAEA,QAAQhD,IAAI,CAAC0C,cAAc;UACzB,KAAK,UAAU;YACb,IAAI,EAAE1C,IAAI,CAACS,IAAI,IAAIoB,MAAM,CAAC,EAAE;cAE1BA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,GAAG,EAAE;YACxB;YACA,IAAMuD,GAAE,GAAGN,OAAO,CAACG,MAAM,EAAE,CAAC;YAC5B,OAAOhC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAACvB,MAAM,IAAI8E,GAAE,EAAE;cAErCnC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAAC2B,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B;YACAP,MAAM,GAAGA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAACuD,GAAE,CAAC;YAC9B;UAEF;YACEnC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,GAAGoB,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3CoB,MAAM,GAAGA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC;QAC9B;MACF;IAAC,SAAAV,GAAA;MAAA+D,UAAA,CAAArE,CAAA,CAAAM,GAAA;IAAA;MAAA+D,UAAA,CAAAnE,CAAA;IAAA;IAGD,IAAI4C,MAAM,KAAKlB,KAAK,CAAC2B,SAAS,EAAE;MAC9B,IAAMxD,KAAK,GAAGxC,KAAK,CAACiH,aAAa,CAE/B5C,KAAK,CAAC6B,YAAY,IAAI7B,KAAK,CAAC8B,aAAa,EACzClC,UAAU,CAACO,MAAM,CAACoC,MAAM,CAAC,EACzBvC,KACF,CAAC;MACDuC,MAAM,EAAE;MAER,QAAQvC,KAAK,CAACqB,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAErB,KAAK,CAACZ,IAAI,IAAIoB,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,IAAMuD,EAAE,GAAGN,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOhC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACvB,MAAM,IAAI8E,EAAE,EAAE;YAEtCnC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAAC2B,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACuD,EAAE,CAAC,GAAGxE,KAAK;UAC9B;QAEF;UACEqC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,GAAGjB,KAAK;MAC9B;IACF;EACF;AACF;AAuBO,SAAS0E,kBAAkBA,CAChClD,MAAqB,EACrBc,QAAyB,EACE;EAC3B,IAAMqC,OAAkC,GAAG,CAAC,CAAC;EAC7C,KAAK,IAAMhG,GAAG,IAAI2D,QAAQ,CAACb,UAAU,EAAE;IACrC,IAAMA,UAAU,GAAGa,QAAQ,CAACb,UAAU,CAAC9C,GAAG,CAAC;IAC3C,IAAI8C,UAAU,CAACS,KAAK,EAAE;MACpB0C,gCAAgC,CAACpD,MAAM,EAAEC,UAAU,EAAEa,QAAQ,CAACH,QAAQ,EAAExD,GAAG,EAAEgG,OAAO,CAAC;IACvF;EACF;EACA,OAAOA,OAAO;AAChB;AAGA,SAASC,gCAAgCA,CACvCpD,MAAqB,EACrBC,UAA8B,EAC9BU,QAAgB,EAChBxD,GAAW,EACXgG,OAAuC,EACvC;EACA,IAAIlD,UAAU,CAACS,KAAK,IAAI,CAAC,EAAE;IACzB;EACF;EAEA,IAAML,KAAK,GAAGL,MAAM,CAACuC,SAAS,CAACpF,GAAG,CAAC;EACnC,IAAMqF,MAAM,GAAGxC,MAAM,CAACyC,eAAe,CAACtF,GAAG,CAAC;EAE1C,IAAMkG,UAAU,GAAGb,MAAM,CAAC,CAAC,CAAC,CAAC/C,IAAI;EAEjC,IAAI6D,MAA6B;EACjC,IAAO9C,MAAM,GAAIP,UAAU,CAApBO,MAAM;EACb,IAAIA,MAAM,CAACtC,MAAM,KAAKyC,QAAQ,IAAI6B,MAAM,CAAC,CAAC,CAAC,CAACL,aAAa,EAAE;IAMzDmB,MAAM,GAAG9C,MAAM;EACjB;EAEA,IAAI8C,MAAM,EAAE;IACVH,OAAO,CAACE,UAAU,CAAC,GAAGC,MAAM;IAC5B;EACF;EAEAA,MAAM,GAAG,IAAIvF,KAAK,CAAC4C,QAAQ,CAAC;EAC5B,KAAK,IAAIxC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGwC,QAAQ,EAAExC,CAAC,EAAE,EAAE;IACjCmF,MAAM,CAACnF,CAAC,CAAC,GAAG,CAAC,CAAC;EAChB;EACAgF,OAAO,CAACE,UAAU,CAAC,GAAGC,MAAM;EAG5B,IAAMZ,OAAiB,GAAG,IAAI3E,KAAK,CAACsC,KAAK,CAAC0B,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIzE,EAAC,GAAG,CAAC,EAAEA,EAAC,GAAG8B,UAAU,CAACS,KAAK,EAAEvC,EAAC,EAAE,EAAE;IACzC,IAAMoD,MAAM,GAAGtB,UAAU,CAACK,OAAO,CAACnC,EAAC,CAAC;IACpC,IAAMmD,MAAM,GAAGrB,UAAU,CAACM,OAAO,CAACpC,EAAC,CAAC;IACpCuE,OAAO,CAACpB,MAAM,CAAC,EAAE;IACjBoB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAErB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAIuB,MAAM,GAAG,CAAC;IACd,IAAIhC,MAAM,GAAGyC,MAAM,CAACZ,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAe;IAAC,IAAAU,UAAA,GAAA9F,0BAAA,CAGtC+E,MAAM;MAAAgB,MAAA;IAAA;MAAzB,KAAAD,UAAA,CAAAlF,CAAA,MAAAmF,MAAA,GAAAD,UAAA,CAAAjF,CAAA,IAAAC,IAAA,GAA2B;QAAA,IAAhBS,IAAI,GAAAwE,MAAA,CAAAhF,KAAA;QACb,IAAIQ,IAAI,KAAKqB,KAAK,IAAIkB,MAAM,GAAGvC,IAAI,CAACgD,SAAS,EAAE;UAC7C;QACF;QAEA,QAAQhD,IAAI,CAAC0C,cAAc;UACzB,KAAK,UAAU;YACb,IAAI,EAAE1C,IAAI,CAACS,IAAI,IAAIoB,MAAM,CAAC,EAAE;cAE1BA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,GAAG,EAAE;YACxB;YACA,IAAMuD,IAAE,GAAGN,OAAO,CAACG,MAAM,EAAE,CAAC;YAC5B,OAAOhC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAACvB,MAAM,IAAI8E,IAAE,EAAE;cAErCnC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAAC2B,IAAI,CAAC,CAAC,CAAC,CAAC;YAC5B;YACAP,MAAM,GAAGA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,CAACuD,IAAE,CAAC;YAC9B;UAEF;YACEnC,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,GAAGoB,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3CoB,MAAM,GAAGA,MAAM,CAAC7B,IAAI,CAACS,IAAI,CAAC;QAC9B;MACF;IAAC,SAAAV,GAAA;MAAAwE,UAAA,CAAA9E,CAAA,CAAAM,GAAA;IAAA;MAAAwE,UAAA,CAAA5E,CAAA;IAAA;IAGD,IAAI4C,MAAM,KAAKlB,KAAK,CAAC2B,SAAS,EAAE;MAC9B,IAAMxD,KAAK,GAAGxC,KAAK,CAACiH,aAAa,CAE/B5C,KAAK,CAAC6B,YAAY,IAAI7B,KAAK,CAAC8B,aAAa,EACzClC,UAAU,CAACO,MAAM,CAACoC,MAAM,CAAC,EACzBvC,KACF,CAAC;MACDuC,MAAM,EAAE;MAER,QAAQvC,KAAK,CAACqB,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAErB,KAAK,CAACZ,IAAI,IAAIoB,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,IAAMuD,EAAE,GAAGN,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOhC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACvB,MAAM,IAAI8E,EAAE,EAAE;YAEtCnC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAAC2B,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,CAACuD,EAAE,CAAC,GAAGxE,KAAK;UAC9B;QAEF;UACEqC,MAAM,CAACR,KAAK,CAACZ,IAAI,CAAC,GAAGjB,KAAK;MAC9B;IACF;EACF;EAGA,KAAK,IAAIL,GAAC,GAAG,CAAC,EAAEA,GAAC,GAAGwC,QAAQ,EAAE,EAAExC,GAAC,EAAE;IACjC,IAAIkF,UAAU,IAAKC,MAAM,CAACnF,GAAC,CAAY,EAAE;MACvCmF,MAAM,CAACnF,GAAC,CAAC,GAAImF,MAAM,CAACnF,GAAC,CAAC,CAAYkF,UAAU,CAAC;IAC/C;EACF;AACF"}

package/dist/esm/index.js

@@ -1,8 +1,8 @@
-import { ParquetWasmLoader as ParquetWasmWorkerLoader } from './parquet-wasm-loader';
-import { ParquetLoader as ParquetWorkerLoader } from './parquet-loader';
+import { ParquetLoader as ParquetWorkerLoader, ParquetLoader as ParquetColumnarWorkerLoader } from './parquet-loader';
 import { parseParquet, parseParquetFileInBatches } from './lib/parsers/parse-parquet-to-rows';
 import { parseParquetInColumns, parseParquetFileInColumnarBatches } from './lib/parsers/parse-parquet-to-columns';
-import { parseParquet as parseParquetWasm } from './lib/wasm/parse-parquet-wasm';
+import { parseParquetWasm } from './lib/wasm/parse-parquet-wasm';
+import { ParquetWasmLoader as ParquetWasmWorkerLoader } from './parquet-wasm-loader';
 export { ParquetWorkerLoader, ParquetWasmWorkerLoader };
 export const ParquetLoader = {
   ...ParquetWorkerLoader,
@@ -10,7 +10,7 @@ export const ParquetLoader = {
   parseFileInBatches: parseParquetFileInBatches
 };
 export const ParquetColumnarLoader = {
-  ...ParquetWorkerLoader,
+  ...ParquetColumnarWorkerLoader,
   parse: parseParquetInColumns,
   parseFileInBatches: parseParquetFileInColumnarBatches
 };
@@ -24,7 +24,7 @@ export { preloadCompressions } from './parquetjs/compression';
 export { ParquetSchema } from './parquetjs/schema/schema';
 export { ParquetReader } from './parquetjs/parser/parquet-reader';
 export { ParquetEncoder } from './parquetjs/encoder/parquet-encoder';
-export { convertSchemaFromParquet, convertSchemaFromParquet as convertParquetToArrowSchema } from './lib/arrow/convert-schema-from-parquet';
+export { convertParquetSchema, convertParquetSchema as convertParquetToArrowSchema } from './lib/arrow/convert-schema-from-parquet';
 export const _typecheckParquetLoader = ParquetLoader;
 export { default as geoJSONSchema } from './lib/geo/geoparquet-schema';
 export { getGeoMetadata, setGeoMetadata, unpackGeoMetadata } from './lib/geo/decode-geo-metadata';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","names":["ParquetWasmLoader","ParquetWasmWorkerLoader","ParquetLoader","ParquetWorkerLoader","parseParquet","parseParquetFileInBatches","parseParquetInColumns","parseParquetFileInColumnarBatches","parseParquetWasm","parse","parseFileInBatches","ParquetColumnarLoader","ParquetWriter","_ParquetWriter","ParquetWasmWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEncoder","convertSchemaFromParquet","convertParquetToArrowSchema","_typecheckParquetLoader","default","geoJSONSchema","getGeoMetadata","setGeoMetadata","unpackGeoMetadata"],"sources":["../../src/index.ts"],"sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parsers/parse-parquet-to-rows';\nimport {\n  parseParquetInColumns,\n  parseParquetFileInColumnarBatches\n} from './lib/parsers/parse-parquet-to-columns';\nimport {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';\n\nexport {ParquetWorkerLoader, ParquetWasmWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\n/** ParquetJS table loader */\nexport const ParquetColumnarLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquetInColumns,\n  parseFileInBatches: parseParquetFileInColumnarBatches\n};\n\nexport const ParquetWasmLoader = {\n  ...ParquetWasmWorkerLoader,\n  parse: parseParquetWasm\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\nexport {ParquetWasmWriter} from './parquet-wasm-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';\n\nexport {\n  convertSchemaFromParquet,\n  convertSchemaFromParquet as convertParquetToArrowSchema\n} from './lib/arrow/convert-schema-from-parquet';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n\n// Geo Metadata\nexport {default as geoJSONSchema} from './lib/geo/geoparquet-schema';\n\nexport type {GeoMetadata} from './lib/geo/decode-geo-metadata';\nexport {getGeoMetadata, setGeoMetadata, unpackGeoMetadata} from './lib/geo/decode-geo-metadata';\n"],"mappings":"AAIA,SAAQA,iBAAiB,IAAIC,uBAAuB,QAAO,uBAAuB;AAClF,SAAQC,aAAa,IAAIC,mBAAmB,QAAO,kBAAkB;AACrE,SAAQC,YAAY,EAAEC,yBAAyB,QAAO,qCAAqC;AAC3F,SACEC,qBAAqB,EACrBC,iCAAiC,QAC5B,wCAAwC;AAC/C,SAAQH,YAAY,IAAII,gBAAgB,QAAO,+BAA+B;AAE9E,SAAQL,mBAAmB,EAAEF,uBAAuB;AAGpD,OAAO,MAAMC,aAAa,GAAG;EAC3B,GAAGC,mBAAmB;EACtBM,KAAK,EAAEL,YAAY;EACnBM,kBAAkB,EAAEL;AACtB,CAAC;AAGD,OAAO,MAAMM,qBAAqB,GAAG;EACnC,GAAGR,mBAAmB;EACtBM,KAAK,EAAEH,qBAAqB;EAC5BI,kBAAkB,EAAEH;AACtB,CAAC;AAED,OAAO,MAAMP,iBAAiB,GAAG;EAC/B,GAAGC,uBAAuB;EAC1BQ,KAAK,EAAED;AACT,CAAC;AAID,SAAQI,aAAa,IAAIC,cAAc,QAAO,kBAAkB;AAChE,SAAQC,iBAAiB,QAAO,uBAAuB;AAIvD,SAAQC,mBAAmB,QAAO,yBAAyB;AAE3D,SAAQC,aAAa,QAAO,2BAA2B;AACvD,SAAQC,aAAa,QAAO,mCAAmC;AAC/D,SAAQC,cAAc,QAAO,qCAAqC;AAElE,SACEC,wBAAwB,EACxBA,wBAAwB,IAAIC,2BAA2B,QAClD,yCAAyC;AAGhD,OAAO,MAAMC,uBAAyC,GAAGnB,aAAa;AAGtE,SAAQoB,OAAO,IAAIC,aAAa,QAAO,6BAA6B;AAGpE,SAAQC,cAAc,EAAEC,cAAc,EAAEC,iBAAiB,QAAO,+BAA+B"}
+{"version":3,"file":"index.js","names":["ParquetLoader","ParquetWorkerLoader","ParquetColumnarWorkerLoader","parseParquet","parseParquetFileInBatches","parseParquetInColumns","parseParquetFileInColumnarBatches","parseParquetWasm","ParquetWasmLoader","ParquetWasmWorkerLoader","parse","parseFileInBatches","ParquetColumnarLoader","ParquetWriter","_ParquetWriter","ParquetWasmWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEncoder","convertParquetSchema","convertParquetToArrowSchema","_typecheckParquetLoader","default","geoJSONSchema","getGeoMetadata","setGeoMetadata","unpackGeoMetadata"],"sources":["../../src/index.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {\n  ObjectRowTable,\n  ObjectRowTableBatch,\n  ColumnarTable,\n  ColumnarTableBatch\n} from '@loaders.gl/schema';\nimport type {Table as ArrowTable} from 'apache-arrow';\n\n// ParquetLoader\n\nimport {\n  ParquetLoader as ParquetWorkerLoader,\n  ParquetLoader as ParquetColumnarWorkerLoader,\n  ParquetLoaderOptions\n} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parsers/parse-parquet-to-rows';\nimport {\n  parseParquetInColumns,\n  parseParquetFileInColumnarBatches\n} from './lib/parsers/parse-parquet-to-columns';\n\nimport {parseParquetWasm, ParquetWasmLoaderOptions} from './lib/wasm/parse-parquet-wasm';\nimport {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';\n\nexport {ParquetWorkerLoader, ParquetWasmWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader: LoaderWithParser<\n  ObjectRowTable,\n  ObjectRowTableBatch,\n  ParquetLoaderOptions\n> = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\n/** ParquetJS table loader */\n// @ts-expect-error\nexport const ParquetColumnarLoader: LoaderWithParser<\n  ColumnarTable,\n  ColumnarTableBatch,\n  ParquetLoaderOptions\n> = {\n  ...ParquetColumnarWorkerLoader,\n  parse: parseParquetInColumns,\n  parseFileInBatches: parseParquetFileInColumnarBatches\n};\n\nexport const ParquetWasmLoader: LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions> = {\n  ...ParquetWasmWorkerLoader,\n  parse: parseParquetWasm\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\nexport {ParquetWasmWriter} from './parquet-wasm-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';\n\nexport {\n  convertParquetSchema,\n  convertParquetSchema as convertParquetToArrowSchema\n} from './lib/arrow/convert-schema-from-parquet';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n\n// Geo Metadata\nexport {default as geoJSONSchema} from './lib/geo/geoparquet-schema';\n\nexport type {GeoMetadata} from './lib/geo/decode-geo-metadata';\nexport {getGeoMetadata, setGeoMetadata, unpackGeoMetadata} from './lib/geo/decode-geo-metadata';\n"],"mappings":"AAaA,SACEA,aAAa,IAAIC,mBAAmB,EACpCD,aAAa,IAAIE,2BAA2B,QAEvC,kBAAkB;AACzB,SAAQC,YAAY,EAAEC,yBAAyB,QAAO,qCAAqC;AAC3F,SACEC,qBAAqB,EACrBC,iCAAiC,QAC5B,wCAAwC;AAE/C,SAAQC,gBAAgB,QAAiC,+BAA+B;AACxF,SAAQC,iBAAiB,IAAIC,uBAAuB,QAAO,uBAAuB;AAElF,SAAQR,mBAAmB,EAAEQ,uBAAuB;AAGpD,OAAO,MAAMT,aAIZ,GAAG;EACF,GAAGC,mBAAmB;EACtBS,KAAK,EAAEP,YAAY;EACnBQ,kBAAkB,EAAEP;AACtB,CAAC;AAID,OAAO,MAAMQ,qBAIZ,GAAG;EACF,GAAGV,2BAA2B;EAC9BQ,KAAK,EAAEL,qBAAqB;EAC5BM,kBAAkB,EAAEL;AACtB,CAAC;AAED,OAAO,MAAME,iBAAgF,GAAG;EAC9F,GAAGC,uBAAuB;EAC1BC,KAAK,EAAEH;AACT,CAAC;AAID,SAAQM,aAAa,IAAIC,cAAc,QAAO,kBAAkB;AAChE,SAAQC,iBAAiB,QAAO,uBAAuB;AAIvD,SAAQC,mBAAmB,QAAO,yBAAyB;AAE3D,SAAQC,aAAa,QAAO,2BAA2B;AACvD,SAAQC,aAAa,QAAO,mCAAmC;AAC/D,SAAQC,cAAc,QAAO,qCAAqC;AAElE,SACEC,oBAAoB,EACpBA,oBAAoB,IAAIC,2BAA2B,QAC9C,yCAAyC;AAGhD,OAAO,MAAMC,uBAAyC,GAAGtB,aAAa;AAGtE,SAAQuB,OAAO,IAAIC,aAAa,QAAO,6BAA6B;AAGpE,SAAQC,cAAc,EAAEC,cAAc,EAAEC,iBAAiB,QAAO,+BAA+B"}

package/dist/esm/lib/arrow/convert-row-group-to-columns.js.map

@@ -1 +1 @@
-{"version":3,"file":"convert-row-group-to-columns.js","names":["convertParquetRowGroupToColumns","schema","rowGroup","columns","columnName","data","Object","entries","columnData","values"],"sources":["../../../../src/lib/arrow/convert-row-group-to-columns.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport {Schema} from '@loaders.gl/schema';\nimport {ParquetBuffer} from '@loaders.gl/parquet/parquetjs/schema/declare';\n\nexport function convertParquetRowGroupToColumns(\n  schema: Schema,\n  rowGroup: ParquetBuffer\n): Record<string, any[]> {\n  const columns: Record<string, any[]> = {};\n  for (const [columnName, data] of Object.entries(rowGroup.columnData)) {\n    columns[columnName] = columns[columnName] || data.values;\n  }\n  return columns;\n}\n"],"mappings":"AAKA,OAAO,SAASA,+BAA+BA,CAC7CC,MAAc,EACdC,QAAuB,EACA;EACvB,MAAMC,OAA8B,GAAG,CAAC,CAAC;EACzC,KAAK,MAAM,CAACC,UAAU,EAAEC,IAAI,CAAC,IAAIC,MAAM,CAACC,OAAO,CAACL,QAAQ,CAACM,UAAU,CAAC,EAAE;IACpEL,OAAO,CAACC,UAAU,CAAC,GAAGD,OAAO,CAACC,UAAU,CAAC,IAAIC,IAAI,CAACI,MAAM;EAC1D;EACA,OAAON,OAAO;AAChB"}
+{"version":3,"file":"convert-row-group-to-columns.js","names":["convertParquetRowGroupToColumns","schema","rowGroup","columns","columnName","data","Object","entries","columnData","values"],"sources":["../../../../src/lib/arrow/convert-row-group-to-columns.ts"],"sourcesContent":["// loaders.gl, MIT license\n\nimport {Schema} from '@loaders.gl/schema';\nimport {ParquetRowGroup} from '@loaders.gl/parquet/parquetjs/schema/declare';\n\nexport function convertParquetRowGroupToColumns(\n  schema: Schema,\n  rowGroup: ParquetRowGroup\n): Record<string, any[]> {\n  const columns: Record<string, any[]> = {};\n  for (const [columnName, data] of Object.entries(rowGroup.columnData)) {\n    columns[columnName] = columns[columnName] || data.values;\n  }\n  return columns;\n}\n"],"mappings":"AAKA,OAAO,SAASA,+BAA+BA,CAC7CC,MAAc,EACdC,QAAyB,EACF;EACvB,MAAMC,OAA8B,GAAG,CAAC,CAAC;EACzC,KAAK,MAAM,CAACC,UAAU,EAAEC,IAAI,CAAC,IAAIC,MAAM,CAACC,OAAO,CAACL,QAAQ,CAACM,UAAU,CAAC,EAAE;IACpEL,OAAO,CAACC,UAAU,CAAC,GAAGD,OAAO,CAACC,UAAU,CAAC,IAAIC,IAAI,CAACI,MAAM;EAC1D;EACA,OAAON,OAAO;AAChB"}

package/dist/esm/lib/arrow/convert-schema-from-parquet.js

@@ -1,77 +1,93 @@
-import { Schema, Struct, Field, Bool, Float64, Int32, Float32, Binary, Utf8, Int64, Uint16, Uint32, Uint64, Int8, Int16 } from '@loaders.gl/schema';
 export const PARQUET_TYPE_MAPPING = {
-  BOOLEAN: Bool,
-  INT32: Int32,
-  INT64: Float64,
-  INT96: Float64,
-  FLOAT: Float32,
-  DOUBLE: Float64,
-  BYTE_ARRAY: Binary,
-  FIXED_LEN_BYTE_ARRAY: Binary,
-  UTF8: Utf8,
-  DATE: Int32,
-  TIME_MILLIS: Int64,
-  TIME_MICROS: Int64,
-  TIMESTAMP_MILLIS: Int64,
-  TIMESTAMP_MICROS: Int64,
-  UINT_8: Int32,
-  UINT_16: Uint16,
-  UINT_32: Uint32,
-  UINT_64: Uint64,
-  INT_8: Int8,
-  INT_16: Int16,
-  INT_32: Int32,
-  INT_64: Int64,
-  JSON: Binary,
-  BSON: Binary,
-  INTERVAL: Binary,
-  DECIMAL_INT32: Float32,
-  DECIMAL_INT64: Float64,
-  DECIMAL_BYTE_ARRAY: Float64,
-  DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
+  BOOLEAN: 'bool',
+  INT32: 'int32',
+  INT64: 'float64',
+  INT96: 'float64',
+  FLOAT: 'float32',
+  DOUBLE: 'float64',
+  BYTE_ARRAY: 'binary',
+  FIXED_LEN_BYTE_ARRAY: 'binary',
+  UTF8: 'utf8',
+  DATE: 'int32',
+  TIME_MILLIS: 'int64',
+  TIME_MICROS: 'int64',
+  TIMESTAMP_MILLIS: 'int64',
+  TIMESTAMP_MICROS: 'int64',
+  UINT_8: 'int32',
+  UINT_16: 'uint16',
+  UINT_32: 'uint32',
+  UINT_64: 'uint64',
+  INT_8: 'int8',
+  INT_16: 'int16',
+  INT_32: 'int32',
+  INT_64: 'int64',
+  JSON: 'binary',
+  BSON: 'binary',
+  INTERVAL: 'binary',
+  DECIMAL_INT32: 'float32',
+  DECIMAL_INT64: 'float64',
+  DECIMAL_BYTE_ARRAY: 'float64',
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
 };
-export function convertSchemaFromParquet(parquetSchema, parquetMetadata) {
+export function convertParquetSchema(parquetSchema, parquetMetadata) {
   const fields = getFields(parquetSchema.schema);
   const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
-  return new Schema(fields, metadata);
+  const schema = {
+    fields,
+    metadata: metadata || {}
+  };
+  return schema;
 }
 function getFields(schema) {
   const fields = [];
   for (const name in schema) {
     const field = schema[name];
     if (field.fields) {
-      const childFields = getFields(field.fields);
-      const nestedField = new Field(name, new Struct(childFields), field.optional);
-      fields.push(nestedField);
+      const children = getFields(field.fields);
+      fields.push({
+        name,
+        type: {
+          type: 'struct',
+          children
+        },
+        nullable: field.optional
+      });
     } else {
-      const FieldType = PARQUET_TYPE_MAPPING[field.type];
+      const type = PARQUET_TYPE_MAPPING[field.type];
       const metadata = getFieldMetadata(field);
-      const arrowField = new Field(name, new FieldType(), field.optional, metadata);
+      const arrowField = {
+        name,
+        type,
+        nullable: field.optional,
+        metadata
+      };
       fields.push(arrowField);
     }
   }
   return fields;
 }
 function getFieldMetadata(field) {
-  const metadata = new Map();
+  let metadata;
   for (const key in field) {
     if (key !== 'name') {
       let value = field[key] || '';
       value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
-      metadata.set(key, value);
+      metadata = metadata || {};
+      metadata[key] = value;
     }
   }
   return metadata;
 }
 function getSchemaMetadata(parquetMetadata) {
-  const metadata = new Map();
+  let metadata;
   const keyValueList = parquetMetadata.key_value_metadata || [];
   for (const {
     key,
     value
   } of keyValueList) {
     if (typeof value === 'string') {
-      metadata.set(key, value);
+      metadata = metadata || {};
+      metadata[key] = value;
     }
   }
   return metadata;