@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6

package/dist/esm/parquetjs/schema/schema.js

@@ -0,0 +1,140 @@
+import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
+import { PARQUET_CODECS } from '../codecs';
+import { PARQUET_COMPRESSION_METHODS } from '../compression';
+import { materializeRows, shredBuffer, shredRecord } from './shred';
+import { PARQUET_LOGICAL_TYPES } from './types';
+export class ParquetSchema {
+  constructor(schema) {
+    _defineProperty(this, "schema", void 0);
+    _defineProperty(this, "fields", void 0);
+    _defineProperty(this, "fieldList", void 0);
+    this.schema = schema;
+    this.fields = buildFields(schema, 0, 0, []);
+    this.fieldList = listFields(this.fields);
+  }
+  findField(path) {
+    if (typeof path === 'string') {
+      path = path.split(',');
+    } else {
+      path = path.slice(0);
+    }
+    let n = this.fields;
+    for (; path.length > 1; path.shift()) {
+      n = n[path[0]].fields;
+    }
+    return n[path[0]];
+  }
+  findFieldBranch(path) {
+    if (typeof path === 'string') {
+      path = path.split(',');
+    }
+    const branch = [];
+    let n = this.fields;
+    for (; path.length > 0; path.shift()) {
+      branch.push(n[path[0]]);
+      if (path.length > 1) {
+        n = n[path[0]].fields;
+      }
+    }
+    return branch;
+  }
+  shredRecord(row, rowGroup) {
+    shredRecord(this, row, rowGroup);
+  }
+  materializeRows(rowGroup) {
+    return materializeRows(this, rowGroup);
+  }
+  compress(type) {
+    setCompress(this.schema, type);
+    setCompress(this.fields, type);
+    return this;
+  }
+  rowGroup() {
+    return shredBuffer(this);
+  }
+}
+function setCompress(schema, type) {
+  for (const name in schema) {
+    const node = schema[name];
+    if (node.fields) {
+      setCompress(node.fields, type);
+    } else {
+      node.compression = type;
+    }
+  }
+}
+function buildFields(schema, rLevelParentMax, dLevelParentMax, path) {
+  const fieldList = {};
+  for (const name in schema) {
+    const opts = schema[name];
+    const required = !opts.optional;
+    const repeated = Boolean(opts.repeated);
+    let rLevelMax = rLevelParentMax;
+    let dLevelMax = dLevelParentMax;
+    let repetitionType = 'REQUIRED';
+    if (!required) {
+      repetitionType = 'OPTIONAL';
+      dLevelMax++;
+    }
+    if (repeated) {
+      repetitionType = 'REPEATED';
+      rLevelMax++;
+      if (required) dLevelMax++;
+    }
+    if (opts.fields) {
+      const cpath = path.concat([name]);
+      fieldList[name] = {
+        name,
+        path: cpath,
+        key: cpath.join(),
+        repetitionType,
+        rLevelMax,
+        dLevelMax,
+        isNested: true,
+        fieldCount: Object.keys(opts.fields).length,
+        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)
+      };
+      continue;
+    }
+    const typeDef = PARQUET_LOGICAL_TYPES[opts.type];
+    if (!typeDef) {
+      throw new Error("invalid parquet type: ".concat(opts.type));
+    }
+    opts.encoding = opts.encoding || 'PLAIN';
+    if (!(opts.encoding in PARQUET_CODECS)) {
+      throw new Error("unsupported parquet encoding: ".concat(opts.encoding));
+    }
+    opts.compression = opts.compression || 'UNCOMPRESSED';
+    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {
+      throw new Error("unsupported compression method: ".concat(opts.compression));
+    }
+    const cpath = path.concat([name]);
+    fieldList[name] = {
+      name,
+      primitiveType: typeDef.primitiveType,
+      originalType: typeDef.originalType,
+      path: cpath,
+      key: cpath.join(),
+      repetitionType,
+      encoding: opts.encoding,
+      compression: opts.compression,
+      typeLength: opts.typeLength || typeDef.typeLength,
+      presision: opts.presision,
+      scale: opts.scale,
+      rLevelMax,
+      dLevelMax
+    };
+  }
+  return fieldList;
+}
+function listFields(fields) {
+  let list = [];
+  for (const k in fields) {
+    list.push(fields[k]);
+    if (fields[k].isNested) {
+      list = list.concat(listFields(fields[k].fields));
+    }
+  }
+  return list;
+}
+//# sourceMappingURL=schema.js.map

package/dist/esm/parquetjs/schema/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","names":["PARQUET_CODECS","PARQUET_COMPRESSION_METHODS","materializeRows","shredBuffer","shredRecord","PARQUET_LOGICAL_TYPES","ParquetSchema","constructor","schema","_defineProperty","fields","buildFields","fieldList","listFields","findField","path","split","slice","n","length","shift","findFieldBranch","branch","push","row","rowGroup","compress","type","setCompress","name","node","compression","rLevelParentMax","dLevelParentMax","opts","required","optional","repeated","Boolean","rLevelMax","dLevelMax","repetitionType","cpath","concat","key","join","isNested","fieldCount","Object","keys","typeDef","Error","encoding","primitiveType","originalType","typeLength","presision","scale","list","k"],"sources":["../../../../src/parquetjs/schema/schema.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {PARQUET_CODECS} from '../codecs';\nimport {PARQUET_COMPRESSION_METHODS} from '../compression';\nimport {\n  FieldDefinition,\n  ParquetRowGroup,\n  ParquetCompression,\n  ParquetField,\n  ParquetRow,\n  RepetitionType,\n  SchemaDefinition\n} from './declare';\nimport {materializeRows, shredBuffer, shredRecord} from './shred';\nimport {PARQUET_LOGICAL_TYPES} from './types';\n\n/**\n * A parquet file schema\n */\nexport class ParquetSchema {\n  public schema: Record<string, FieldDefinition>;\n  public fields: Record<string, ParquetField>;\n  public fieldList: ParquetField[];\n\n  /**\n   * Create a new schema from a JSON schema definition\n   */\n  constructor(schema: SchemaDefinition) {\n    this.schema = schema;\n    this.fields = buildFields(schema, 0, 0, []);\n    this.fieldList = listFields(this.fields);\n  }\n\n  /**\n   * Retrieve a field definition\n   */\n  findField(path: string | string[]): ParquetField {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    } else {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.slice(0); // clone array\n    }\n\n    let n = this.fields;\n    for (; path.length > 1; path.shift()) {\n      n = n[path[0]].fields as Record<string, ParquetField>;\n    }\n\n    return n[path[0]];\n  }\n\n  /**\n   * Retrieve a field definition and all the field's ancestors\n   */\n  findFieldBranch(path: string | string[]): ParquetField[] {\n    if (typeof path === 'string') {\n      // tslint:disable-next-line:no-parameter-reassignment\n      path = path.split(',');\n    }\n    const branch: ParquetField[] = [];\n    let n = this.fields;\n    for (; path.length > 0; path.shift()) {\n      branch.push(n[path[0]]);\n      if (path.length > 1) {\n        n = n[path[0]].fields as Record<string, ParquetField>;\n      }\n    }\n    return branch;\n  }\n\n  shredRecord(row: ParquetRow, rowGroup: ParquetRowGroup): void {\n    shredRecord(this, row, rowGroup);\n  }\n\n  materializeRows(rowGroup: ParquetRowGroup): ParquetRow[] {\n    return materializeRows(this, rowGroup);\n  }\n\n  compress(type: ParquetCompression): this {\n    setCompress(this.schema, type);\n    setCompress(this.fields, type);\n    return this;\n  }\n\n  rowGroup(): ParquetRowGroup {\n    return shredBuffer(this);\n  }\n}\n\nfunction setCompress(schema: any, type: ParquetCompression) {\n  for (const name in schema) {\n    const node = schema[name];\n    if (node.fields) {\n      setCompress(node.fields, type);\n    } else {\n      node.compression = type;\n    }\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction buildFields(\n  schema: SchemaDefinition,\n  rLevelParentMax: number,\n  dLevelParentMax: number,\n  path: string[]\n): Record<string, ParquetField> {\n  const fieldList: Record<string, ParquetField> = {};\n\n  for (const name in schema) {\n    const opts = schema[name];\n\n    /* field repetition type */\n    const required = !opts.optional;\n    const repeated = Boolean(opts.repeated);\n    let rLevelMax = rLevelParentMax;\n    let dLevelMax = dLevelParentMax;\n\n    let repetitionType: RepetitionType = 'REQUIRED';\n    if (!required) {\n      repetitionType = 'OPTIONAL';\n      dLevelMax++;\n    }\n    if (repeated) {\n      repetitionType = 'REPEATED';\n      rLevelMax++;\n      if (required) dLevelMax++;\n    }\n\n    /* nested field */\n    if (opts.fields) {\n      const cpath = path.concat([name]);\n      fieldList[name] = {\n        name,\n        path: cpath,\n        key: cpath.join(),\n        repetitionType,\n        rLevelMax,\n        dLevelMax,\n        isNested: true,\n        fieldCount: Object.keys(opts.fields).length,\n        fields: buildFields(opts.fields, rLevelMax, dLevelMax, cpath)\n      };\n      continue; // eslint-disable-line no-continue\n    }\n\n    const typeDef: any = PARQUET_LOGICAL_TYPES[opts.type!];\n    if (!typeDef) {\n      throw new Error(`invalid parquet type: ${opts.type}`);\n    }\n\n    opts.encoding = opts.encoding || 'PLAIN';\n    if (!(opts.encoding in PARQUET_CODECS)) {\n      throw new Error(`unsupported parquet encoding: ${opts.encoding}`);\n    }\n\n    opts.compression = opts.compression || 'UNCOMPRESSED';\n    if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {\n      throw new Error(`unsupported compression method: ${opts.compression}`);\n    }\n\n    /* add to schema */\n    const cpath = path.concat([name]);\n    fieldList[name] = {\n      name,\n      primitiveType: typeDef.primitiveType,\n      originalType: typeDef.originalType,\n      path: cpath,\n      key: cpath.join(),\n      repetitionType,\n      encoding: opts.encoding,\n      compression: opts.compression,\n      typeLength: opts.typeLength || typeDef.typeLength,\n      presision: opts.presision,\n      scale: opts.scale,\n      rLevelMax,\n      dLevelMax\n    };\n  }\n  return fieldList;\n}\n\nfunction listFields(fields: Record<string, ParquetField>): ParquetField[] {\n  let list: ParquetField[] = [];\n  for (const k in fields) {\n    list.push(fields[k]);\n    if (fields[k].isNested) {\n      list = list.concat(listFields(fields[k].fields!));\n    }\n  }\n  return list;\n}\n"],"mappings":";AAEA,SAAQA,cAAc,QAAO,WAAW;AACxC,SAAQC,2BAA2B,QAAO,gBAAgB;AAU1D,SAAQC,eAAe,EAAEC,WAAW,EAAEC,WAAW,QAAO,SAAS;AACjE,SAAQC,qBAAqB,QAAO,SAAS;AAK7C,OAAO,MAAMC,aAAa,CAAC;EAQzBC,WAAWA,CAACC,MAAwB,EAAE;IAAAC,eAAA;IAAAA,eAAA;IAAAA,eAAA;IACpC,IAAI,CAACD,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACE,MAAM,GAAGC,WAAW,CAACH,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;IAC3C,IAAI,CAACI,SAAS,GAAGC,UAAU,CAAC,IAAI,CAACH,MAAM,CAAC;EAC1C;EAKAI,SAASA,CAACC,IAAuB,EAAgB;IAC/C,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;MAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;IACxB,CAAC,MAAM;MAELD,IAAI,GAAGA,IAAI,CAACE,KAAK,CAAC,CAAC,CAAC;IACtB;IAEA,IAAIC,CAAC,GAAG,IAAI,CAACR,MAAM;IACnB,OAAOK,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;MACpCF,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACL,MAAsC;IACvD;IAEA,OAAOQ,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC;EACnB;EAKAM,eAAeA,CAACN,IAAuB,EAAkB;IACvD,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;MAE5BA,IAAI,GAAGA,IAAI,CAACC,KAAK,CAAC,GAAG,CAAC;IACxB;IACA,MAAMM,MAAsB,GAAG,EAAE;IACjC,IAAIJ,CAAC,GAAG,IAAI,CAACR,MAAM;IACnB,OAAOK,IAAI,CAACI,MAAM,GAAG,CAAC,EAAEJ,IAAI,CAACK,KAAK,CAAC,CAAC,EAAE;MACpCE,MAAM,CAACC,IAAI,CAACL,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;MACvB,IAAIA,IAAI,CAACI,MAAM,GAAG,CAAC,EAAE;QACnBD,CAAC,GAAGA,CAAC,CAACH,IAAI,CAAC,CAAC,CAAC,CAAC,CAACL,MAAsC;MACvD;IACF;IACA,OAAOY,MAAM;EACf;EAEAlB,WAAWA,CAACoB,GAAe,EAAEC,QAAyB,EAAQ;IAC5DrB,WAAW,CAAC,IAAI,EAAEoB,GAAG,EAAEC,QAAQ,CAAC;EAClC;EAEAvB,eAAeA,CAACuB,QAAyB,EAAgB;IACvD,OAAOvB,eAAe,CAAC,IAAI,EAAEuB,QAAQ,CAAC;EACxC;EAEAC,QAAQA,CAACC,IAAwB,EAAQ;IACvCC,WAAW,CAAC,IAAI,CAACpB,MAAM,EAAEmB,IAAI,CAAC;IAC9BC,WAAW,CAAC,IAAI,CAAClB,MAAM,EAAEiB,IAAI,CAAC;IAC9B,OAAO,IAAI;EACb;EAEAF,QAAQA,CAAA,EAAoB;IAC1B,OAAOtB,WAAW,CAAC,IAAI,CAAC;EAC1B;AACF;AAEA,SAASyB,WAAWA,CAACpB,MAAW,EAAEmB,IAAwB,EAAE;EAC1D,KAAK,MAAME,IAAI,IAAIrB,MAAM,EAAE;IACzB,MAAMsB,IAAI,GAAGtB,MAAM,CAACqB,IAAI,CAAC;IACzB,IAAIC,IAAI,CAACpB,MAAM,EAAE;MACfkB,WAAW,CAACE,IAAI,CAACpB,MAAM,EAAEiB,IAAI,CAAC;IAChC,CAAC,MAAM;MACLG,IAAI,CAACC,WAAW,GAAGJ,IAAI;IACzB;EACF;AACF;AAGA,SAAShB,WAAWA,CAClBH,MAAwB,EACxBwB,eAAuB,EACvBC,eAAuB,EACvBlB,IAAc,EACgB;EAC9B,MAAMH,SAAuC,GAAG,CAAC,CAAC;EAElD,KAAK,MAAMiB,IAAI,IAAIrB,MAAM,EAAE;IACzB,MAAM0B,IAAI,GAAG1B,MAAM,CAACqB,IAAI,CAAC;IAGzB,MAAMM,QAAQ,GAAG,CAACD,IAAI,CAACE,QAAQ;IAC/B,MAAMC,QAAQ,GAAGC,OAAO,CAACJ,IAAI,CAACG,QAAQ,CAAC;IACvC,IAAIE,SAAS,GAAGP,eAAe;IAC/B,IAAIQ,SAAS,GAAGP,eAAe;IAE/B,IAAIQ,cAA8B,GAAG,UAAU;IAC/C,IAAI,CAACN,QAAQ,EAAE;MACbM,cAAc,GAAG,UAAU;MAC3BD,SAAS,EAAE;IACb;IACA,IAAIH,QAAQ,EAAE;MACZI,cAAc,GAAG,UAAU;MAC3BF,SAAS,EAAE;MACX,IAAIJ,QAAQ,EAAEK,SAAS,EAAE;IAC3B;IAGA,IAAIN,IAAI,CAACxB,MAAM,EAAE;MACf,MAAMgC,KAAK,GAAG3B,IAAI,CAAC4B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;MACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;QAChBA,IAAI;QACJd,IAAI,EAAE2B,KAAK;QACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,CAAC,CAAC;QACjBJ,cAAc;QACdF,SAAS;QACTC,SAAS;QACTM,QAAQ,EAAE,IAAI;QACdC,UAAU,EAAEC,MAAM,CAACC,IAAI,CAACf,IAAI,CAACxB,MAAM,CAAC,CAACS,MAAM;QAC3CT,MAAM,EAAEC,WAAW,CAACuB,IAAI,CAACxB,MAAM,EAAE6B,SAAS,EAAEC,SAAS,EAAEE,KAAK;MAC9D,CAAC;MACD;IACF;IAEA,MAAMQ,OAAY,GAAG7C,qBAAqB,CAAC6B,IAAI,CAACP,IAAI,CAAE;IACtD,IAAI,CAACuB,OAAO,EAAE;MACZ,MAAM,IAAIC,KAAK,0BAAAR,MAAA,CAA0BT,IAAI,CAACP,IAAI,CAAE,CAAC;IACvD;IAEAO,IAAI,CAACkB,QAAQ,GAAGlB,IAAI,CAACkB,QAAQ,IAAI,OAAO;IACxC,IAAI,EAAElB,IAAI,CAACkB,QAAQ,IAAIpD,cAAc,CAAC,EAAE;MACtC,MAAM,IAAImD,KAAK,kCAAAR,MAAA,CAAkCT,IAAI,CAACkB,QAAQ,CAAE,CAAC;IACnE;IAEAlB,IAAI,CAACH,WAAW,GAAGG,IAAI,CAACH,WAAW,IAAI,cAAc;IACrD,IAAI,EAAEG,IAAI,CAACH,WAAW,IAAI9B,2BAA2B,CAAC,EAAE;MACtD,MAAM,IAAIkD,KAAK,oCAAAR,MAAA,CAAoCT,IAAI,CAACH,WAAW,CAAE,CAAC;IACxE;IAGA,MAAMW,KAAK,GAAG3B,IAAI,CAAC4B,MAAM,CAAC,CAACd,IAAI,CAAC,CAAC;IACjCjB,SAAS,CAACiB,IAAI,CAAC,GAAG;MAChBA,IAAI;MACJwB,aAAa,EAAEH,OAAO,CAACG,aAAa;MACpCC,YAAY,EAAEJ,OAAO,CAACI,YAAY;MAClCvC,IAAI,EAAE2B,KAAK;MACXE,GAAG,EAAEF,KAAK,CAACG,IAAI,CAAC,CAAC;MACjBJ,cAAc;MACdW,QAAQ,EAAElB,IAAI,CAACkB,QAAQ;MACvBrB,WAAW,EAAEG,IAAI,CAACH,WAAW;MAC7BwB,UAAU,EAAErB,IAAI,CAACqB,UAAU,IAAIL,OAAO,CAACK,UAAU;MACjDC,SAAS,EAAEtB,IAAI,CAACsB,SAAS;MACzBC,KAAK,EAAEvB,IAAI,CAACuB,KAAK;MACjBlB,SAAS;MACTC;IACF,CAAC;EACH;EACA,OAAO5B,SAAS;AAClB;AAEA,SAASC,UAAUA,CAACH,MAAoC,EAAkB;EACxE,IAAIgD,IAAoB,GAAG,EAAE;EAC7B,KAAK,MAAMC,CAAC,IAAIjD,MAAM,EAAE;IACtBgD,IAAI,CAACnC,IAAI,CAACb,MAAM,CAACiD,CAAC,CAAC,CAAC;IACpB,IAAIjD,MAAM,CAACiD,CAAC,CAAC,CAACb,QAAQ,EAAE;MACtBY,IAAI,GAAGA,IAAI,CAACf,MAAM,CAAC9B,UAAU,CAACH,MAAM,CAACiD,CAAC,CAAC,CAACjD,MAAO,CAAC,CAAC;IACnD;EACF;EACA,OAAOgD,IAAI;AACb"}

package/dist/esm/parquetjs/schema/shred.js

@@ -0,0 +1,228 @@
+import { ParquetRowGroup } from './declare';
+import * as Types from './types';
+export { ParquetRowGroup };
+export function shredBuffer(schema) {
+  const columnData = {};
+  for (const field of schema.fieldList) {
+    columnData[field.key] = {
+      dlevels: [],
+      rlevels: [],
+      values: [],
+      pageHeaders: [],
+      count: 0
+    };
+  }
+  return {
+    rowCount: 0,
+    columnData
+  };
+}
+export function shredRecord(schema, record, rowGroup) {
+  const data = shredBuffer(schema).columnData;
+  shredRecordFields(schema.fields, record, data, 0, 0);
+  if (rowGroup.rowCount === 0) {
+    rowGroup.rowCount = 1;
+    rowGroup.columnData = data;
+    return;
+  }
+  rowGroup.rowCount += 1;
+  for (const field of schema.fieldList) {
+    Array.prototype.push.apply(rowGroup.columnData[field.key].rlevels, data[field.key].rlevels);
+    Array.prototype.push.apply(rowGroup.columnData[field.key].dlevels, data[field.key].dlevels);
+    Array.prototype.push.apply(rowGroup.columnData[field.key].values, data[field.key].values);
+    rowGroup.columnData[field.key].count += data[field.key].count;
+  }
+}
+function shredRecordFields(fields, record, data, rLevel, dLevel) {
+  for (const name in fields) {
+    const field = fields[name];
+    let values = [];
+    if (record && field.name in record && record[field.name] !== undefined && record[field.name] !== null) {
+      if (record[field.name].constructor === Array) {
+        values = record[field.name];
+      } else {
+        values.push(record[field.name]);
+      }
+    }
+    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {
+      throw new Error("missing required field: ".concat(field.name));
+    }
+    if (values.length > 1 && field.repetitionType !== 'REPEATED') {
+      throw new Error("too many values for field: ".concat(field.name));
+    }
+    if (values.length === 0) {
+      if (field.isNested) {
+        shredRecordFields(field.fields, null, data, rLevel, dLevel);
+      } else {
+        data[field.key].count += 1;
+        data[field.key].rlevels.push(rLevel);
+        data[field.key].dlevels.push(dLevel);
+      }
+      continue;
+    }
+    for (let i = 0; i < values.length; i++) {
+      const rlvl = i === 0 ? rLevel : field.rLevelMax;
+      if (field.isNested) {
+        shredRecordFields(field.fields, values[i], data, rlvl, field.dLevelMax);
+      } else {
+        data[field.key].count += 1;
+        data[field.key].rlevels.push(rlvl);
+        data[field.key].dlevels.push(field.dLevelMax);
+        data[field.key].values.push(Types.toPrimitive(field.originalType || field.primitiveType, values[i]));
+      }
+    }
+  }
+}
+export function materializeRows(schema, rowGroup) {
+  const rows = [];
+  for (let i = 0; i < rowGroup.rowCount; i++) {
+    rows.push({});
+  }
+  for (const key in rowGroup.columnData) {
+    const columnData = rowGroup.columnData[key];
+    if (columnData.count) {
+      materializeColumnAsRows(schema, columnData, key, rows);
+    }
+  }
+  return rows;
+}
+function materializeColumnAsRows(schema, columnData, key, rows) {
+  const field = schema.findField(key);
+  const branch = schema.findFieldBranch(key);
+  const rLevels = new Array(field.rLevelMax + 1).fill(0);
+  let vIndex = 0;
+  for (let i = 0; i < columnData.count; i++) {
+    const dLevel = columnData.dlevels[i];
+    const rLevel = columnData.rlevels[i];
+    rLevels[rLevel]++;
+    rLevels.fill(0, rLevel + 1);
+    let rIndex = 0;
+    let record = rows[rLevels[rIndex++] - 1];
+    for (const step of branch) {
+      if (step === field || dLevel < step.dLevelMax) {
+        break;
+      }
+      switch (step.repetitionType) {
+        case 'REPEATED':
+          if (!(step.name in record)) {
+            record[step.name] = [];
+          }
+          const ix = rLevels[rIndex++];
+          while (record[step.name].length <= ix) {
+            record[step.name].push({});
+          }
+          record = record[step.name][ix];
+          break;
+        default:
+          record[step.name] = record[step.name] || {};
+          record = record[step.name];
+      }
+    }
+    if (dLevel === field.dLevelMax) {
+      const value = Types.fromPrimitive(field.originalType || field.primitiveType, columnData.values[vIndex], field);
+      vIndex++;
+      switch (field.repetitionType) {
+        case 'REPEATED':
+          if (!(field.name in record)) {
+            record[field.name] = [];
+          }
+          const ix = rLevels[rIndex];
+          while (record[field.name].length <= ix) {
+            record[field.name].push(null);
+          }
+          record[field.name][ix] = value;
+          break;
+        default:
+          record[field.name] = value;
+      }
+    }
+  }
+}
+export function materializeColumns(schema, rowGroup) {
+  const columns = {};
+  for (const key in rowGroup.columnData) {
+    const columnData = rowGroup.columnData[key];
+    if (columnData.count) {
+      materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);
+    }
+  }
+  return columns;
+}
+function materializeColumnAsColumnarArray(schema, columnData, rowCount, key, columns) {
+  if (columnData.count <= 0) {
+    return;
+  }
+  const field = schema.findField(key);
+  const branch = schema.findFieldBranch(key);
+  const columnName = branch[0].name;
+  let column;
+  const {
+    values
+  } = columnData;
+  if (values.length === rowCount && branch[0].primitiveType) {
+    column = values;
+  }
+  if (column) {
+    columns[columnName] = column;
+    return;
+  }
+  column = new Array(rowCount);
+  for (let i = 0; i < rowCount; i++) {
+    column[i] = {};
+  }
+  columns[columnName] = column;
+  const rLevels = new Array(field.rLevelMax + 1).fill(0);
+  let vIndex = 0;
+  for (let i = 0; i < columnData.count; i++) {
+    const dLevel = columnData.dlevels[i];
+    const rLevel = columnData.rlevels[i];
+    rLevels[rLevel]++;
+    rLevels.fill(0, rLevel + 1);
+    let rIndex = 0;
+    let record = column[rLevels[rIndex++] - 1];
+    for (const step of branch) {
+      if (step === field || dLevel < step.dLevelMax) {
+        break;
+      }
+      switch (step.repetitionType) {
+        case 'REPEATED':
+          if (!(step.name in record)) {
+            record[step.name] = [];
+          }
+          const ix = rLevels[rIndex++];
+          while (record[step.name].length <= ix) {
+            record[step.name].push({});
+          }
+          record = record[step.name][ix];
+          break;
+        default:
+          record[step.name] = record[step.name] || {};
+          record = record[step.name];
+      }
+    }
+    if (dLevel === field.dLevelMax) {
+      const value = Types.fromPrimitive(field.originalType || field.primitiveType, columnData.values[vIndex], field);
+      vIndex++;
+      switch (field.repetitionType) {
+        case 'REPEATED':
+          if (!(field.name in record)) {
+            record[field.name] = [];
+          }
+          const ix = rLevels[rIndex];
+          while (record[field.name].length <= ix) {
+            record[field.name].push(null);
+          }
+          record[field.name][ix] = value;
+          break;
+        default:
+          record[field.name] = value;
+      }
+    }
+  }
+  for (let i = 0; i < rowCount; ++i) {
+    if (columnName in column[i]) {
+      column[i] = column[i][columnName];
+    }
+  }
+}
+//# sourceMappingURL=shred.js.map

package/dist/esm/parquetjs/schema/shred.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shred.js","names":["ParquetRowGroup","Types","shredBuffer","schema","columnData","field","fieldList","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","rowGroup","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","concat","isNested","i","rlvl","rLevelMax","dLevelMax","toPrimitive","originalType","primitiveType","materializeRows","rows","materializeColumnAsRows","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive","materializeColumns","columns","materializeColumnAsColumnarArray","columnName","column"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ArrayType} from '@loaders.gl/schema';\nimport {ParquetRowGroup, ParquetColumnChunk, ParquetField, ParquetRow} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetRowGroup};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetRowGroup {\n  const columnData: Record<string, ParquetColumnChunk> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The rowGroup argument must point to an object into which the shredded record\n * will be returned. You may re-use the rowGroup for repeated calls to this function\n * to append to an existing rowGroup, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the rowGroup is as\n * follows:\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(\n  schema: ParquetSchema,\n  record: ParquetRow,\n  rowGroup: ParquetRowGroup\n): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the rowGroup */\n  if (rowGroup.rowCount === 0) {\n    rowGroup.rowCount = 1;\n    rowGroup.columnData = data;\n    return;\n  }\n  rowGroup.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(rowGroup.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(rowGroup.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(rowGroup.columnData[field.key].values, data[field.key].values);\n    rowGroup.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: ParquetRow,\n  data: Record<string, ParquetColumnChunk>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null!, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The rowGroup argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRows(schema: ParquetSchema, rowGroup: ParquetRowGroup): ParquetRow[] {\n  const rows: ParquetRow[] = [];\n  // rows = new Array(rowGroup.rowCount).fill({})'\n  for (let i = 0; i < rowGroup.rowCount; i++) {\n    rows.push({});\n  }\n  for (const key in rowGroup.columnData) {\n    const columnData = rowGroup.columnData[key];\n    if (columnData.count) {\n      materializeColumnAsRows(schema, columnData, key, rows);\n    }\n  }\n  return rows;\n}\n\n/** Populate record fields for one column */\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumnAsRows(\n  schema: ParquetSchema,\n  columnData: ParquetColumnChunk,\n  key: string,\n  rows: ParquetRow[]\n): void {\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = rows[rLevels[rIndex++] - 1];\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n}\n\n// Columnar export\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The rowGroup argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeColumns(\n  schema: ParquetSchema,\n  rowGroup: ParquetRowGroup\n): Record<string, ArrayType> {\n  const columns: Record<string, ArrayType> = {};\n  for (const key in rowGroup.columnData) {\n    const columnData = rowGroup.columnData[key];\n    if (columnData.count) {\n      materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);\n    }\n  }\n  return columns;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumnAsColumnarArray(\n  schema: ParquetSchema,\n  columnData: ParquetColumnChunk,\n  rowCount: number,\n  key: string,\n  columns: Record<string, ArrayType<any>>\n) {\n  if (columnData.count <= 0) {\n    return;\n  }\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  const columnName = branch[0].name;\n\n  let column: ArrayType | undefined;\n  const {values} = columnData;\n  if (values.length === rowCount && branch[0].primitiveType) {\n    // if (branch[0].repetitionType === `REQUIRED`) {\n    //   switch (branch[0].primitiveType) {\n    //     case 'INT32': return values instanceof Int32Array ? values : new Int32Array(values);\n    //   }\n    // }\n    column = values;\n  }\n\n  if (column) {\n    columns[columnName] = column;\n    return;\n  }\n\n  column = new Array(rowCount);\n  for (let i = 0; i < rowCount; i++) {\n    column[i] = {};\n  }\n  columns[columnName] = column;\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = column[rLevels[rIndex++] - 1] as ParquetRow;\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n\n  // Remove one level of nesting\n  for (let i = 0; i < rowCount; ++i) {\n    if (columnName in (column[i] as object)) {\n      column[i] = (column[i] as object)[columnName];\n    }\n  }\n}\n"],"mappings":"AAGA,SAAQA,eAAe,QAAqD,WAAW;AAEvF,OAAO,KAAKC,KAAK,MAAM,SAAS;AAEhC,SAAQD,eAAe;AAEvB,OAAO,SAASE,WAAWA,CAACC,MAAqB,EAAmB;EAClE,MAAMC,UAA8C,GAAG,CAAC,CAAC;EACzD,KAAK,MAAMC,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCF,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,GAAG;MACtBC,OAAO,EAAE,EAAE;MACXC,OAAO,EAAE,EAAE;MACXC,MAAM,EAAE,EAAE;MACVC,WAAW,EAAE,EAAE;MACfC,KAAK,EAAE;IACT,CAAC;EACH;EACA,OAAO;IAACC,QAAQ,EAAE,CAAC;IAAET;EAAU,CAAC;AAClC;AAwBA,OAAO,SAASU,WAAWA,CACzBX,MAAqB,EACrBY,MAAkB,EAClBC,QAAyB,EACnB;EAEN,MAAMC,IAAI,GAAGf,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Cc,iBAAiB,CAACf,MAAM,CAACgB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;EAGpD,IAAID,QAAQ,CAACH,QAAQ,KAAK,CAAC,EAAE;IAC3BG,QAAQ,CAACH,QAAQ,GAAG,CAAC;IACrBG,QAAQ,CAACZ,UAAU,GAAGa,IAAI;IAC1B;EACF;EACAD,QAAQ,CAACH,QAAQ,IAAI,CAAC;EACtB,KAAK,MAAMR,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCc,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,EAAEQ,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAAC;IAC3FW,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,EAAES,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAAC;IAC3FY,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,EAAEO,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAAC;IACzFM,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAIK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK;EAC/D;AACF;AAGA,SAASM,iBAAiBA,CACxBC,MAAoC,EACpCJ,MAAkB,EAClBE,IAAwC,EACxCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,MAAMC,IAAI,IAAIP,MAAM,EAAE;IACzB,MAAMd,KAAK,GAAGc,MAAM,CAACO,IAAI,CAAC;IAG1B,IAAIhB,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNV,KAAK,CAACqB,IAAI,IAAIX,MAAM,IACpBA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAKC,SAAS,IAChCZ,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIX,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACE,WAAW,KAAKR,KAAK,EAAE;QAC5CV,MAAM,GAAGK,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLhB,MAAM,CAACY,IAAI,CAACP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,IAAIC,OAAO,CAACf,MAAM,CAAC,IAAIV,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,4BAAAC,MAAA,CAA4B5B,KAAK,CAACqB,IAAI,CAAE,CAAC;IAC1D;IACA,IAAIhB,MAAM,CAACmB,MAAM,GAAG,CAAC,IAAIxB,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,+BAAAC,MAAA,CAA+B5B,KAAK,CAACqB,IAAI,CAAE,CAAC;IAC7D;IAGA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,EAAE;MACvB,IAAIxB,KAAK,CAAC6B,QAAQ,EAAE;QAClBhB,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAG,IAAI,EAAGF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC/D,CAAC,MAAM;QACLR,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;IAGA,KAAK,IAAIU,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGzB,MAAM,CAACmB,MAAM,EAAEM,CAAC,EAAE,EAAE;MACtC,MAAMC,IAAI,GAAGD,CAAC,KAAK,CAAC,GAAGX,MAAM,GAAGnB,KAAK,CAACgC,SAAS;MAC/C,IAAIhC,KAAK,CAAC6B,QAAQ,EAAE;QAClBhB,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAGT,MAAM,CAACyB,CAAC,CAAC,EAAElB,IAAI,EAAEmB,IAAI,EAAE/B,KAAK,CAACiC,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLrB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACc,IAAI,CAAC;QAClCnB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACjB,KAAK,CAACiC,SAAS,CAAC;QAC7CrB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAACY,IAAI,CACzBrB,KAAK,CAACsC,WAAW,CAAElC,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EAAI/B,MAAM,CAACyB,CAAC,CAAC,CAC3E,CAAC;MACH;IACF;EACF;AACF;AAqBA,OAAO,SAASO,eAAeA,CAACvC,MAAqB,EAAEa,QAAyB,EAAgB;EAC9F,MAAM2B,IAAkB,GAAG,EAAE;EAE7B,KAAK,IAAIR,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGnB,QAAQ,CAACH,QAAQ,EAAEsB,CAAC,EAAE,EAAE;IAC1CQ,IAAI,CAACrB,IAAI,CAAC,CAAC,CAAC,CAAC;EACf;EACA,KAAK,MAAMf,GAAG,IAAIS,QAAQ,CAACZ,UAAU,EAAE;IACrC,MAAMA,UAAU,GAAGY,QAAQ,CAACZ,UAAU,CAACG,GAAG,CAAC;IAC3C,IAAIH,UAAU,CAACQ,KAAK,EAAE;MACpBgC,uBAAuB,CAACzC,MAAM,EAAEC,UAAU,EAAEG,GAAG,EAAEoC,IAAI,CAAC;IACxD;EACF;EACA,OAAOA,IAAI;AACb;AAIA,SAASC,uBAAuBA,CAC9BzC,MAAqB,EACrBC,UAA8B,EAC9BG,GAAW,EACXoC,IAAkB,EACZ;EACN,MAAMtC,KAAK,GAAGF,MAAM,CAAC0C,SAAS,CAACtC,GAAG,CAAC;EACnC,MAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAe,CAACxC,GAAG,CAAC;EAG1C,MAAMyC,OAAiB,GAAG,IAAI5B,KAAK,CAACf,KAAK,CAACgC,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIf,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG/B,UAAU,CAACQ,KAAK,EAAEuB,CAAC,EAAE,EAAE;IACzC,MAAMV,MAAM,GAAGrB,UAAU,CAACI,OAAO,CAAC2B,CAAC,CAAC;IACpC,MAAMX,MAAM,GAAGpB,UAAU,CAACK,OAAO,CAAC0B,CAAC,CAAC;IACpCa,OAAO,CAACxB,MAAM,CAAC,EAAE;IACjBwB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAEzB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI2B,MAAM,GAAG,CAAC;IACd,IAAIpC,MAAM,GAAG4B,IAAI,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;IAGxC,KAAK,MAAMC,IAAI,IAAIN,MAAM,EAAE;MACzB,IAAIM,IAAI,KAAK/C,KAAK,IAAIoB,MAAM,GAAG2B,IAAI,CAACd,SAAS,EAAE;QAC7C;MACF;MAEA,QAAQc,IAAI,CAACrB,cAAc;QACzB,KAAK,UAAU;UACb,IAAI,EAAEqB,IAAI,CAAC1B,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE1BA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAG,EAAE;UACxB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;UAC5B,OAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAErCtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;UAC5B;UACAP,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAAC2B,EAAE,CAAC;UAC9B;QAEF;UACEtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAGX,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,IAAI,CAAC,CAAC;UAC3CX,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC;MAC9B;IACF;IAGA,IAAID,MAAM,KAAKpB,KAAK,CAACiC,SAAS,EAAE;MAC9B,MAAMgB,KAAK,GAAGrD,KAAK,CAACsD,aAAa,CAE/BlD,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EACzCrC,UAAU,CAACM,MAAM,CAACwC,MAAM,CAAC,EACzB7C,KACF,CAAC;MACD6C,MAAM,EAAE;MAER,QAAQ7C,KAAK,CAAC0B,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAE1B,KAAK,CAACqB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOpC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAEtCtC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC2B,EAAE,CAAC,GAAGC,KAAK;UAC9B;QAEF;UACEvC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG4B,KAAK;MAC9B;IACF;EACF;AACF;AAuBA,OAAO,SAASE,kBAAkBA,CAChCrD,MAAqB,EACrBa,QAAyB,EACE;EAC3B,MAAMyC,OAAkC,GAAG,CAAC,CAAC;EAC7C,KAAK,MAAMlD,GAAG,IAAIS,QAAQ,CAACZ,UAAU,EAAE;IACrC,MAAMA,UAAU,GAAGY,QAAQ,CAACZ,UAAU,CAACG,GAAG,CAAC;IAC3C,IAAIH,UAAU,CAACQ,KAAK,EAAE;MACpB8C,gCAAgC,CAACvD,MAAM,EAAEC,UAAU,EAAEY,QAAQ,CAACH,QAAQ,EAAEN,GAAG,EAAEkD,OAAO,CAAC;IACvF;EACF;EACA,OAAOA,OAAO;AAChB;AAGA,SAASC,gCAAgCA,CACvCvD,MAAqB,EACrBC,UAA8B,EAC9BS,QAAgB,EAChBN,GAAW,EACXkD,OAAuC,EACvC;EACA,IAAIrD,UAAU,CAACQ,KAAK,IAAI,CAAC,EAAE;IACzB;EACF;EAEA,MAAMP,KAAK,GAAGF,MAAM,CAAC0C,SAAS,CAACtC,GAAG,CAAC;EACnC,MAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAe,CAACxC,GAAG,CAAC;EAE1C,MAAMoD,UAAU,GAAGb,MAAM,CAAC,CAAC,CAAC,CAACpB,IAAI;EAEjC,IAAIkC,MAA6B;EACjC,MAAM;IAAClD;EAAM,CAAC,GAAGN,UAAU;EAC3B,IAAIM,MAAM,CAACmB,MAAM,KAAKhB,QAAQ,IAAIiC,MAAM,CAAC,CAAC,CAAC,CAACL,aAAa,EAAE;IAMzDmB,MAAM,GAAGlD,MAAM;EACjB;EAEA,IAAIkD,MAAM,EAAE;IACVH,OAAO,CAACE,UAAU,CAAC,GAAGC,MAAM;IAC5B;EACF;EAEAA,MAAM,GAAG,IAAIxC,KAAK,CAACP,QAAQ,CAAC;EAC5B,KAAK,IAAIsB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGtB,QAAQ,EAAEsB,CAAC,EAAE,EAAE;IACjCyB,MAAM,CAACzB,CAAC,CAAC,GAAG,CAAC,CAAC;EAChB;EACAsB,OAAO,CAACE,UAAU,CAAC,GAAGC,MAAM;EAG5B,MAAMZ,OAAiB,GAAG,IAAI5B,KAAK,CAACf,KAAK,CAACgC,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIf,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG/B,UAAU,CAACQ,KAAK,EAAEuB,CAAC,EAAE,EAAE;IACzC,MAAMV,MAAM,GAAGrB,UAAU,CAACI,OAAO,CAAC2B,CAAC,CAAC;IACpC,MAAMX,MAAM,GAAGpB,UAAU,CAACK,OAAO,CAAC0B,CAAC,CAAC;IACpCa,OAAO,CAACxB,MAAM,CAAC,EAAE;IACjBwB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAEzB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI2B,MAAM,GAAG,CAAC;IACd,IAAIpC,MAAM,GAAG6C,MAAM,CAACZ,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAe;IAGxD,KAAK,MAAMC,IAAI,IAAIN,MAAM,EAAE;MACzB,IAAIM,IAAI,KAAK/C,KAAK,IAAIoB,MAAM,GAAG2B,IAAI,CAACd,SAAS,EAAE;QAC7C;MACF;MAEA,QAAQc,IAAI,CAACrB,cAAc;QACzB,KAAK,UAAU;UACb,IAAI,EAAEqB,IAAI,CAAC1B,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE1BA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAG,EAAE;UACxB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;UAC5B,OAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAErCtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;UAC5B;UACAP,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAAC2B,EAAE,CAAC;UAC9B;QAEF;UACEtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAGX,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,IAAI,CAAC,CAAC;UAC3CX,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC;MAC9B;IACF;IAGA,IAAID,MAAM,KAAKpB,KAAK,CAACiC,SAAS,EAAE;MAC9B,MAAMgB,KAAK,GAAGrD,KAAK,CAACsD,aAAa,CAE/BlD,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EACzCrC,UAAU,CAACM,MAAM,CAACwC,MAAM,CAAC,EACzB7C,KACF,CAAC;MACD6C,MAAM,EAAE;MAER,QAAQ7C,KAAK,CAAC0B,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAE1B,KAAK,CAACqB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOpC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAEtCtC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC2B,EAAE,CAAC,GAAGC,KAAK;UAC9B;QAEF;UACEvC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG4B,KAAK;MAC9B;IACF;EACF;EAGA,KAAK,IAAInB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGtB,QAAQ,EAAE,EAAEsB,CAAC,EAAE;IACjC,IAAIwB,UAAU,IAAKC,MAAM,CAACzB,CAAC,CAAY,EAAE;MACvCyB,MAAM,CAACzB,CAAC,CAAC,GAAIyB,MAAM,CAACzB,CAAC,CAAC,CAAYwB,UAAU,CAAC;IAC/C;EACF;AACF"}