@loaders.gl/parquet 3.4.6 → 4.0.0-alpha.10

package/dist/esm/parquetjs/schema/shred.js

@@ -1,6 +1,6 @@
-import { ParquetBuffer } from './declare';
+import { ParquetRowGroup } from './declare';
 import * as Types from './types';
-export { ParquetBuffer };
+export { ParquetRowGroup };
 export function shredBuffer(schema) {
   const columnData = {};
   for (const field of schema.fieldList) {
@@ -17,20 +17,20 @@ export function shredBuffer(schema) {
     columnData
   };
 }
-export function shredRecord(schema, record, buffer) {
+export function shredRecord(schema, record, rowGroup) {
   const data = shredBuffer(schema).columnData;
   shredRecordFields(schema.fields, record, data, 0, 0);
-  if (buffer.rowCount === 0) {
-    buffer.rowCount = 1;
-    buffer.columnData = data;
+  if (rowGroup.rowCount === 0) {
+    rowGroup.rowCount = 1;
+    rowGroup.columnData = data;
     return;
   }
-  buffer.rowCount += 1;
+  rowGroup.rowCount += 1;
   for (const field of schema.fieldList) {
-    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);
-    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);
-    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);
-    buffer.columnData[field.key].count += data[field.key].count;
+    Array.prototype.push.apply(rowGroup.columnData[field.key].rlevels, data[field.key].rlevels);
+    Array.prototype.push.apply(rowGroup.columnData[field.key].dlevels, data[field.key].dlevels);
+    Array.prototype.push.apply(rowGroup.columnData[field.key].values, data[field.key].values);
+    rowGroup.columnData[field.key].count += data[field.key].count;
   }
 }
 function shredRecordFields(fields, record, data, rLevel, dLevel) {
@@ -73,20 +73,20 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
     }
   }
 }
-export function materializeRecords(schema, buffer) {
-  const records = [];
-  for (let i = 0; i < buffer.rowCount; i++) {
-    records.push({});
+export function materializeRows(schema, rowGroup) {
+  const rows = [];
+  for (let i = 0; i < rowGroup.rowCount; i++) {
+    rows.push({});
   }
-  for (const key in buffer.columnData) {
-    const columnData = buffer.columnData[key];
+  for (const key in rowGroup.columnData) {
+    const columnData = rowGroup.columnData[key];
     if (columnData.count) {
-      materializeColumn(schema, columnData, key, records);
+      materializeColumnAsRows(schema, columnData, key, rows);
     }
   }
-  return records;
+  return rows;
 }
-function materializeColumn(schema, columnData, key, records) {
+function materializeColumnAsRows(schema, columnData, key, rows) {
   const field = schema.findField(key);
   const branch = schema.findFieldBranch(key);
   const rLevels = new Array(field.rLevelMax + 1).fill(0);
@@ -97,7 +97,7 @@ function materializeColumn(schema, columnData, key, records) {
     rLevels[rLevel]++;
     rLevels.fill(0, rLevel + 1);
     let rIndex = 0;
-    let record = records[rLevels[rIndex++] - 1];
+    let record = rows[rLevels[rIndex++] - 1];
     for (const step of branch) {
       if (step === field || dLevel < step.dLevelMax) {
         break;
@@ -138,4 +138,91 @@ function materializeColumn(schema, columnData, key, records) {
     }
   }
 }
+export function materializeColumns(schema, rowGroup) {
+  const columns = {};
+  for (const key in rowGroup.columnData) {
+    const columnData = rowGroup.columnData[key];
+    if (columnData.count) {
+      materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);
+    }
+  }
+  return columns;
+}
+function materializeColumnAsColumnarArray(schema, columnData, rowCount, key, columns) {
+  if (columnData.count <= 0) {
+    return;
+  }
+  const field = schema.findField(key);
+  const branch = schema.findFieldBranch(key);
+  const columnName = branch[0].name;
+  let column;
+  const {
+    values
+  } = columnData;
+  if (values.length === rowCount && branch[0].primitiveType) {
+    column = values;
+  }
+  if (column) {
+    columns[columnName] = column;
+    return;
+  }
+  column = new Array(rowCount);
+  for (let i = 0; i < rowCount; i++) {
+    column[i] = {};
+  }
+  columns[columnName] = column;
+  const rLevels = new Array(field.rLevelMax + 1).fill(0);
+  let vIndex = 0;
+  for (let i = 0; i < columnData.count; i++) {
+    const dLevel = columnData.dlevels[i];
+    const rLevel = columnData.rlevels[i];
+    rLevels[rLevel]++;
+    rLevels.fill(0, rLevel + 1);
+    let rIndex = 0;
+    let record = column[rLevels[rIndex++] - 1];
+    for (const step of branch) {
+      if (step === field || dLevel < step.dLevelMax) {
+        break;
+      }
+      switch (step.repetitionType) {
+        case 'REPEATED':
+          if (!(step.name in record)) {
+            record[step.name] = [];
+          }
+          const ix = rLevels[rIndex++];
+          while (record[step.name].length <= ix) {
+            record[step.name].push({});
+          }
+          record = record[step.name][ix];
+          break;
+        default:
+          record[step.name] = record[step.name] || {};
+          record = record[step.name];
+      }
+    }
+    if (dLevel === field.dLevelMax) {
+      const value = Types.fromPrimitive(field.originalType || field.primitiveType, columnData.values[vIndex], field);
+      vIndex++;
+      switch (field.repetitionType) {
+        case 'REPEATED':
+          if (!(field.name in record)) {
+            record[field.name] = [];
+          }
+          const ix = rLevels[rIndex];
+          while (record[field.name].length <= ix) {
+            record[field.name].push(null);
+          }
+          record[field.name][ix] = value;
+          break;
+        default:
+          record[field.name] = value;
+      }
+    }
+  }
+  for (let i = 0; i < rowCount; ++i) {
+    if (columnName in column[i]) {
+      column[i] = column[i][columnName];
+    }
+  }
+}
 //# sourceMappingURL=shred.js.map

package/dist/esm/parquetjs/schema/shred.js.map

@@ -1 +1 @@
-{"version":3,"file":"shred.js","names":["ParquetBuffer","Types","shredBuffer","schema","columnData","field","fieldList","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","buffer","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","concat","isNested","i","rlvl","rLevelMax","dLevelMax","toPrimitive","originalType","primitiveType","materializeRecords","records","materializeColumn","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetBuffer};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetBuffer {\n  const columnData: Record<string, ParquetData> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The buffer argument must point to an object into which the shredded record\n * will be returned. You may re-use the buffer for repeated calls to this function\n * to append to an existing buffer, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the buffer is as\n * follows:\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the buffer */\n  if (buffer.rowCount === 0) {\n    buffer.rowCount = 1;\n    buffer.columnData = data;\n    return;\n  }\n  buffer.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);\n    buffer.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: any,\n  data: Record<string, ParquetData>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {\n  const records: ParquetRecord[] = [];\n  for (let i = 0; i < buffer.rowCount; i++) {\n    records.push({});\n  }\n  for (const key in buffer.columnData) {\n    const columnData = buffer.columnData[key];\n    if (columnData.count) {\n      materializeColumn(schema, columnData, key, records);\n    }\n  }\n  return records;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumn(\n  schema: ParquetSchema,\n  columnData: ParquetData,\n  key: string,\n  records: ParquetRecord[]\n): void {\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = records[rLevels[rIndex++] - 1];\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n}\n\n// Columnar export\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The buffer argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   buffer = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n *\nexport function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {\n  const columns: ParquetRecord = {};\n  for (const key in buffer.columnData) {\n    const columnData = buffer.columnData[key];\n    if (columnData.count) {\n      extractColumn(schema, columnData, key, columns);\n    }\n  }\n  return columns;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction extractColumn(\n  schema: ParquetSchema,\n  columnData: ParquetData,\n  key: string,\n  columns: Record<string, unknown> \n) {\n  if (columnData.count <= 0) {\n    return;\n  }\n\n  const record = columns;\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n\n  let i = 0;\n  const dLevel = columnData.dlevels[i];\n  const rLevel = columnData.rlevels[i];\n  rLevels[rLevel]++;\n  rLevels.fill(0, rLevel + 1);\n\n  let rIndex = 0;\n  let record = records[rLevels[rIndex++] - 1];\n\n  // Internal nodes\n  for (const step of branch) {\n    if (step === field || dLevel < step.dLevelMax) {\n      break;\n    }\n\n    switch (step.repetitionType) {\n      case 'REPEATED':\n        if (!(step.name in record)) {\n          // eslint-disable max-depth\n          record[step.name] = [];\n        }\n        const ix = rLevels[rIndex++];\n        while (record[step.name].length <= ix) {\n          // eslint-disable max-depth\n          record[step.name].push({});\n        }\n        record = record[step.name][ix];\n        break;\n\n      default:\n        record[step.name] = record[step.name] || {};\n        record = record[step.name];\n    }\n  }\n\n  // Leaf node\n  if (dLevel === field.dLevelMax) {\n    const value = Types.fromPrimitive(\n      // @ts-ignore\n      field.originalType || field.primitiveType,\n      columnData.values[vIndex],\n      field\n    );\n    vIndex++;\n\n    switch (field.repetitionType) {\n      case 'REPEATED':\n        if (!(field.name in record)) {\n          // eslint-disable max-depth\n          record[field.name] = [];\n        }\n        const ix = rLevels[rIndex];\n        while (record[field.name].length <= ix) {\n          // eslint-disable max-depth\n          record[field.name].push(null);\n        }\n        record[field.name][ix] = value;\n        break;\n\n      default:\n        record[field.name] = value;\n    }\n  }\n}\n*/\n"],"mappings":"AAEA,SAAQA,aAAa,QAAiD,WAAW;AAEjF,OAAO,KAAKC,KAAK,MAAM,SAAS;AAEhC,SAAQD,aAAa;AAErB,OAAO,SAASE,WAAWA,CAACC,MAAqB,EAAiB;EAChE,MAAMC,UAAuC,GAAG,CAAC,CAAC;EAClD,KAAK,MAAMC,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCF,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,GAAG;MACtBC,OAAO,EAAE,EAAE;MACXC,OAAO,EAAE,EAAE;MACXC,MAAM,EAAE,EAAE;MACVC,WAAW,EAAE,EAAE;MACfC,KAAK,EAAE;IACT,CAAC;EACH;EACA,OAAO;IAACC,QAAQ,EAAE,CAAC;IAAET;EAAU,CAAC;AAClC;AAwBA,OAAO,SAASU,WAAWA,CAACX,MAAqB,EAAEY,MAAW,EAAEC,MAAqB,EAAQ;EAE3F,MAAMC,IAAI,GAAGf,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Cc,iBAAiB,CAACf,MAAM,CAACgB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;EAGpD,IAAID,MAAM,CAACH,QAAQ,KAAK,CAAC,EAAE;IACzBG,MAAM,CAACH,QAAQ,GAAG,CAAC;IACnBG,MAAM,CAACZ,UAAU,GAAGa,IAAI;IACxB;EACF;EACAD,MAAM,CAACH,QAAQ,IAAI,CAAC;EACpB,KAAK,MAAMR,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCc,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,EAAEQ,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAAC;IACzFW,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,EAAES,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAAC;IACzFY,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,EAAEO,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAAC;IACvFM,MAAM,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAIK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK;EAC7D;AACF;AAGA,SAASM,iBAAiBA,CACxBC,MAAoC,EACpCJ,MAAW,EACXE,IAAiC,EACjCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,MAAMC,IAAI,IAAIP,MAAM,EAAE;IACzB,MAAMd,KAAK,GAAGc,MAAM,CAACO,IAAI,CAAC;IAG1B,IAAIhB,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNV,KAAK,CAACqB,IAAI,IAAIX,MAAM,IACpBA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAKC,SAAS,IAChCZ,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIX,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACE,WAAW,KAAKR,KAAK,EAAE;QAC5CV,MAAM,GAAGK,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLhB,MAAM,CAACY,IAAI,CAACP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,IAAIC,OAAO,CAACf,MAAM,CAAC,IAAIV,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,4BAAAC,MAAA,CAA4B5B,KAAK,CAACqB,IAAI,CAAE,CAAC;IAC1D;IACA,IAAIhB,MAAM,CAACmB,MAAM,GAAG,CAAC,IAAIxB,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,+BAAAC,MAAA,CAA+B5B,KAAK,CAACqB,IAAI,CAAE,CAAC;IAC7D;IAGA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,EAAE;MACvB,IAAIxB,KAAK,CAAC6B,QAAQ,EAAE;QAClBhB,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAG,IAAI,EAAEF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC9D,CAAC,MAAM;QACLR,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;IAGA,KAAK,IAAIU,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGzB,MAAM,CAACmB,MAAM,EAAEM,CAAC,EAAE,EAAE;MACtC,MAAMC,IAAI,GAAGD,CAAC,KAAK,CAAC,GAAGX,MAAM,GAAGnB,KAAK,CAACgC,SAAS;MAC/C,IAAIhC,KAAK,CAAC6B,QAAQ,EAAE;QAClBhB,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAGT,MAAM,CAACyB,CAAC,CAAC,EAAElB,IAAI,EAAEmB,IAAI,EAAE/B,KAAK,CAACiC,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLrB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACc,IAAI,CAAC;QAClCnB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACjB,KAAK,CAACiC,SAAS,CAAC;QAC7CrB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAACY,IAAI,CACzBrB,KAAK,CAACsC,WAAW,CAAElC,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EAAI/B,MAAM,CAACyB,CAAC,CAAC,CAC3E,CAAC;MACH;IACF;EACF;AACF;AAqBA,OAAO,SAASO,kBAAkBA,CAACvC,MAAqB,EAAEa,MAAqB,EAAmB;EAChG,MAAM2B,OAAwB,GAAG,EAAE;EACnC,KAAK,IAAIR,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGnB,MAAM,CAACH,QAAQ,EAAEsB,CAAC,EAAE,EAAE;IACxCQ,OAAO,CAACrB,IAAI,CAAC,CAAC,CAAC,CAAC;EAClB;EACA,KAAK,MAAMf,GAAG,IAAIS,MAAM,CAACZ,UAAU,EAAE;IACnC,MAAMA,UAAU,GAAGY,MAAM,CAACZ,UAAU,CAACG,GAAG,CAAC;IACzC,IAAIH,UAAU,CAACQ,KAAK,EAAE;MACpBgC,iBAAiB,CAACzC,MAAM,EAAEC,UAAU,EAAEG,GAAG,EAAEoC,OAAO,CAAC;IACrD;EACF;EACA,OAAOA,OAAO;AAChB;AAGA,SAASC,iBAAiBA,CACxBzC,MAAqB,EACrBC,UAAuB,EACvBG,GAAW,EACXoC,OAAwB,EAClB;EACN,MAAMtC,KAAK,GAAGF,MAAM,CAAC0C,SAAS,CAACtC,GAAG,CAAC;EACnC,MAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAe,CAACxC,GAAG,CAAC;EAG1C,MAAMyC,OAAiB,GAAG,IAAI5B,KAAK,CAACf,KAAK,CAACgC,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIf,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG/B,UAAU,CAACQ,KAAK,EAAEuB,CAAC,EAAE,EAAE;IACzC,MAAMV,MAAM,GAAGrB,UAAU,CAACI,OAAO,CAAC2B,CAAC,CAAC;IACpC,MAAMX,MAAM,GAAGpB,UAAU,CAACK,OAAO,CAAC0B,CAAC,CAAC;IACpCa,OAAO,CAACxB,MAAM,CAAC,EAAE;IACjBwB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAEzB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI2B,MAAM,GAAG,CAAC;IACd,IAAIpC,MAAM,GAAG4B,OAAO,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;IAG3C,KAAK,MAAMC,IAAI,IAAIN,MAAM,EAAE;MACzB,IAAIM,IAAI,KAAK/C,KAAK,IAAIoB,MAAM,GAAG2B,IAAI,CAACd,SAAS,EAAE;QAC7C;MACF;MAEA,QAAQc,IAAI,CAACrB,cAAc;QACzB,KAAK,UAAU;UACb,IAAI,EAAEqB,IAAI,CAAC1B,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE1BA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAG,EAAE;UACxB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;UAC5B,OAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAErCtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;UAC5B;UACAP,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAAC2B,EAAE,CAAC;UAC9B;QAEF;UACEtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAGX,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,IAAI,CAAC,CAAC;UAC3CX,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC;MAC9B;IACF;IAGA,IAAID,MAAM,KAAKpB,KAAK,CAACiC,SAAS,EAAE;MAC9B,MAAMgB,KAAK,GAAGrD,KAAK,CAACsD,aAAa,CAE/BlD,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EACzCrC,UAAU,CAACM,MAAM,CAACwC,MAAM,CAAC,EACzB7C,KACF,CAAC;MACD6C,MAAM,EAAE;MAER,QAAQ7C,KAAK,CAAC0B,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAE1B,KAAK,CAACqB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOpC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAEtCtC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC2B,EAAE,CAAC,GAAGC,KAAK;UAC9B;QAEF;UACEvC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG4B,KAAK;MAC9B;IACF;EACF;AACF"}
+{"version":3,"file":"shred.js","names":["ParquetRowGroup","Types","shredBuffer","schema","columnData","field","fieldList","key","dlevels","rlevels","values","pageHeaders","count","rowCount","shredRecord","record","rowGroup","data","shredRecordFields","fields","Array","prototype","push","apply","rLevel","dLevel","name","undefined","constructor","length","Boolean","repetitionType","Error","concat","isNested","i","rlvl","rLevelMax","dLevelMax","toPrimitive","originalType","primitiveType","materializeRows","rows","materializeColumnAsRows","findField","branch","findFieldBranch","rLevels","fill","vIndex","rIndex","step","ix","value","fromPrimitive","materializeColumns","columns","materializeColumnAsColumnarArray","columnName","column"],"sources":["../../../../src/parquetjs/schema/shred.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\nimport {ArrayType} from '@loaders.gl/schema';\nimport {ParquetRowGroup, ParquetColumnChunk, ParquetField, ParquetRow} from './declare';\nimport {ParquetSchema} from './schema';\nimport * as Types from './types';\n\nexport {ParquetRowGroup};\n\nexport function shredBuffer(schema: ParquetSchema): ParquetRowGroup {\n  const columnData: Record<string, ParquetColumnChunk> = {};\n  for (const field of schema.fieldList) {\n    columnData[field.key] = {\n      dlevels: [],\n      rlevels: [],\n      values: [],\n      pageHeaders: [],\n      count: 0\n    };\n  }\n  return {rowCount: 0, columnData};\n}\n\n/**\n * 'Shred' a record into a list of <value, repetition_level, definition_level>\n * tuples per column using the Google Dremel Algorithm..\n *\n * The rowGroup argument must point to an object into which the shredded record\n * will be returned. You may re-use the rowGroup for repeated calls to this function\n * to append to an existing rowGroup, as long as the schema is unchanged.\n *\n * The format in which the shredded records will be stored in the rowGroup is as\n * follows:\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function shredRecord(\n  schema: ParquetSchema,\n  record: ParquetRow,\n  rowGroup: ParquetRowGroup\n): void {\n  /* shred the record, this may raise an exception */\n  const data = shredBuffer(schema).columnData;\n\n  shredRecordFields(schema.fields, record, data, 0, 0);\n\n  /* if no error during shredding, add the shredded record to the rowGroup */\n  if (rowGroup.rowCount === 0) {\n    rowGroup.rowCount = 1;\n    rowGroup.columnData = data;\n    return;\n  }\n  rowGroup.rowCount += 1;\n  for (const field of schema.fieldList) {\n    Array.prototype.push.apply(rowGroup.columnData[field.key].rlevels, data[field.key].rlevels);\n    Array.prototype.push.apply(rowGroup.columnData[field.key].dlevels, data[field.key].dlevels);\n    Array.prototype.push.apply(rowGroup.columnData[field.key].values, data[field.key].values);\n    rowGroup.columnData[field.key].count += data[field.key].count;\n  }\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction shredRecordFields(\n  fields: Record<string, ParquetField>,\n  record: ParquetRow,\n  data: Record<string, ParquetColumnChunk>,\n  rLevel: number,\n  dLevel: number\n) {\n  for (const name in fields) {\n    const field = fields[name];\n\n    // fetch values\n    let values: any[] = [];\n    if (\n      record &&\n      field.name in record &&\n      record[field.name] !== undefined &&\n      record[field.name] !== null\n    ) {\n      if (record[field.name].constructor === Array) {\n        values = record[field.name];\n      } else {\n        values.push(record[field.name]);\n      }\n    }\n    // check values\n    if (values.length === 0 && Boolean(record) && field.repetitionType === 'REQUIRED') {\n      throw new Error(`missing required field: ${field.name}`);\n    }\n    if (values.length > 1 && field.repetitionType !== 'REPEATED') {\n      throw new Error(`too many values for field: ${field.name}`);\n    }\n\n    // push null\n    if (values.length === 0) {\n      if (field.isNested) {\n        shredRecordFields(field.fields!, null!, data, rLevel, dLevel);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rLevel);\n        data[field.key].dlevels.push(dLevel);\n      }\n      continue; // eslint-disable-line no-continue\n    }\n\n    // push values\n    for (let i = 0; i < values.length; i++) {\n      const rlvl = i === 0 ? rLevel : field.rLevelMax;\n      if (field.isNested) {\n        shredRecordFields(field.fields!, values[i], data, rlvl, field.dLevelMax);\n      } else {\n        data[field.key].count += 1;\n        data[field.key].rlevels.push(rlvl);\n        data[field.key].dlevels.push(field.dLevelMax);\n        data[field.key].values.push(\n          Types.toPrimitive((field.originalType || field.primitiveType)!, values[i])\n        );\n      }\n    }\n  }\n}\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The rowGroup argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeRows(schema: ParquetSchema, rowGroup: ParquetRowGroup): ParquetRow[] {\n  const rows: ParquetRow[] = [];\n  // rows = new Array(rowGroup.rowCount).fill({})'\n  for (let i = 0; i < rowGroup.rowCount; i++) {\n    rows.push({});\n  }\n  for (const key in rowGroup.columnData) {\n    const columnData = rowGroup.columnData[key];\n    if (columnData.count) {\n      materializeColumnAsRows(schema, columnData, key, rows);\n    }\n  }\n  return rows;\n}\n\n/** Populate record fields for one column */\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumnAsRows(\n  schema: ParquetSchema,\n  columnData: ParquetColumnChunk,\n  key: string,\n  rows: ParquetRow[]\n): void {\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = rows[rLevels[rIndex++] - 1];\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n}\n\n// Columnar export\n\n/**\n * 'Materialize' a list of <value, repetition_level, definition_level>\n * tuples back to nested records (objects/arrays) using the Google Dremel\n * Algorithm..\n *\n * The rowGroup argument must point to an object with the following structure (i.e.\n * the same structure that is returned by shredRecords):\n *\n *   rowGroup = {\n *     columnData: [\n *       'my_col': {\n *          dlevels: [d1, d2, .. dN],\n *          rlevels: [r1, r2, .. rN],\n *          values: [v1, v2, .. vN],\n *        }, ...\n *      ],\n *      rowCount: X,\n *   }\n */\nexport function materializeColumns(\n  schema: ParquetSchema,\n  rowGroup: ParquetRowGroup\n): Record<string, ArrayType> {\n  const columns: Record<string, ArrayType> = {};\n  for (const key in rowGroup.columnData) {\n    const columnData = rowGroup.columnData[key];\n    if (columnData.count) {\n      materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);\n    }\n  }\n  return columns;\n}\n\n// eslint-disable-next-line max-statements, complexity\nfunction materializeColumnAsColumnarArray(\n  schema: ParquetSchema,\n  columnData: ParquetColumnChunk,\n  rowCount: number,\n  key: string,\n  columns: Record<string, ArrayType<any>>\n) {\n  if (columnData.count <= 0) {\n    return;\n  }\n\n  const field = schema.findField(key);\n  const branch = schema.findFieldBranch(key);\n\n  const columnName = branch[0].name;\n\n  let column: ArrayType | undefined;\n  const {values} = columnData;\n  if (values.length === rowCount && branch[0].primitiveType) {\n    // if (branch[0].repetitionType === `REQUIRED`) {\n    //   switch (branch[0].primitiveType) {\n    //     case 'INT32': return values instanceof Int32Array ? values : new Int32Array(values);\n    //   }\n    // }\n    column = values;\n  }\n\n  if (column) {\n    columns[columnName] = column;\n    return;\n  }\n\n  column = new Array(rowCount);\n  for (let i = 0; i < rowCount; i++) {\n    column[i] = {};\n  }\n  columns[columnName] = column;\n\n  // tslint:disable-next-line:prefer-array-literal\n  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);\n  let vIndex = 0;\n  for (let i = 0; i < columnData.count; i++) {\n    const dLevel = columnData.dlevels[i];\n    const rLevel = columnData.rlevels[i];\n    rLevels[rLevel]++;\n    rLevels.fill(0, rLevel + 1);\n\n    let rIndex = 0;\n    let record = column[rLevels[rIndex++] - 1] as ParquetRow;\n\n    // Internal nodes - Build a nested row object\n    for (const step of branch) {\n      if (step === field || dLevel < step.dLevelMax) {\n        break;\n      }\n\n      switch (step.repetitionType) {\n        case 'REPEATED':\n          if (!(step.name in record)) {\n            // eslint-disable max-depth\n            record[step.name] = [];\n          }\n          const ix = rLevels[rIndex++];\n          while (record[step.name].length <= ix) {\n            // eslint-disable max-depth\n            record[step.name].push({});\n          }\n          record = record[step.name][ix];\n          break;\n\n        default:\n          record[step.name] = record[step.name] || {};\n          record = record[step.name];\n      }\n    }\n\n    // Leaf node - Add the value\n    if (dLevel === field.dLevelMax) {\n      const value = Types.fromPrimitive(\n        // @ts-ignore\n        field.originalType || field.primitiveType,\n        columnData.values[vIndex],\n        field\n      );\n      vIndex++;\n\n      switch (field.repetitionType) {\n        case 'REPEATED':\n          if (!(field.name in record)) {\n            // eslint-disable max-depth\n            record[field.name] = [];\n          }\n          const ix = rLevels[rIndex];\n          while (record[field.name].length <= ix) {\n            // eslint-disable max-depth\n            record[field.name].push(null);\n          }\n          record[field.name][ix] = value;\n          break;\n\n        default:\n          record[field.name] = value;\n      }\n    }\n  }\n\n  // Remove one level of nesting\n  for (let i = 0; i < rowCount; ++i) {\n    if (columnName in (column[i] as object)) {\n      column[i] = (column[i] as object)[columnName];\n    }\n  }\n}\n"],"mappings":"AAGA,SAAQA,eAAe,QAAqD,WAAW;AAEvF,OAAO,KAAKC,KAAK,MAAM,SAAS;AAEhC,SAAQD,eAAe;AAEvB,OAAO,SAASE,WAAWA,CAACC,MAAqB,EAAmB;EAClE,MAAMC,UAA8C,GAAG,CAAC,CAAC;EACzD,KAAK,MAAMC,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCF,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,GAAG;MACtBC,OAAO,EAAE,EAAE;MACXC,OAAO,EAAE,EAAE;MACXC,MAAM,EAAE,EAAE;MACVC,WAAW,EAAE,EAAE;MACfC,KAAK,EAAE;IACT,CAAC;EACH;EACA,OAAO;IAACC,QAAQ,EAAE,CAAC;IAAET;EAAU,CAAC;AAClC;AAwBA,OAAO,SAASU,WAAWA,CACzBX,MAAqB,EACrBY,MAAkB,EAClBC,QAAyB,EACnB;EAEN,MAAMC,IAAI,GAAGf,WAAW,CAACC,MAAM,CAAC,CAACC,UAAU;EAE3Cc,iBAAiB,CAACf,MAAM,CAACgB,MAAM,EAAEJ,MAAM,EAAEE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;EAGpD,IAAID,QAAQ,CAACH,QAAQ,KAAK,CAAC,EAAE;IAC3BG,QAAQ,CAACH,QAAQ,GAAG,CAAC;IACrBG,QAAQ,CAACZ,UAAU,GAAGa,IAAI;IAC1B;EACF;EACAD,QAAQ,CAACH,QAAQ,IAAI,CAAC;EACtB,KAAK,MAAMR,KAAK,IAAIF,MAAM,CAACG,SAAS,EAAE;IACpCc,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,EAAEQ,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAAC;IAC3FW,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,EAAES,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAAC;IAC3FY,KAAK,CAACC,SAAS,CAACC,IAAI,CAACC,KAAK,CAACP,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,EAAEO,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAAC;IACzFM,QAAQ,CAACZ,UAAU,CAACC,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAIK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK;EAC/D;AACF;AAGA,SAASM,iBAAiBA,CACxBC,MAAoC,EACpCJ,MAAkB,EAClBE,IAAwC,EACxCO,MAAc,EACdC,MAAc,EACd;EACA,KAAK,MAAMC,IAAI,IAAIP,MAAM,EAAE;IACzB,MAAMd,KAAK,GAAGc,MAAM,CAACO,IAAI,CAAC;IAG1B,IAAIhB,MAAa,GAAG,EAAE;IACtB,IACEK,MAAM,IACNV,KAAK,CAACqB,IAAI,IAAIX,MAAM,IACpBA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAKC,SAAS,IAChCZ,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,KAAK,IAAI,EAC3B;MACA,IAAIX,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACE,WAAW,KAAKR,KAAK,EAAE;QAC5CV,MAAM,GAAGK,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC;MAC7B,CAAC,MAAM;QACLhB,MAAM,CAACY,IAAI,CAACP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC;MACjC;IACF;IAEA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,IAAIC,OAAO,CAACf,MAAM,CAAC,IAAIV,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MACjF,MAAM,IAAIC,KAAK,4BAAAC,MAAA,CAA4B5B,KAAK,CAACqB,IAAI,CAAE,CAAC;IAC1D;IACA,IAAIhB,MAAM,CAACmB,MAAM,GAAG,CAAC,IAAIxB,KAAK,CAAC0B,cAAc,KAAK,UAAU,EAAE;MAC5D,MAAM,IAAIC,KAAK,+BAAAC,MAAA,CAA+B5B,KAAK,CAACqB,IAAI,CAAE,CAAC;IAC7D;IAGA,IAAIhB,MAAM,CAACmB,MAAM,KAAK,CAAC,EAAE;MACvB,IAAIxB,KAAK,CAAC6B,QAAQ,EAAE;QAClBhB,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAG,IAAI,EAAGF,IAAI,EAAEO,MAAM,EAAEC,MAAM,CAAC;MAC/D,CAAC,MAAM;QACLR,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACE,MAAM,CAAC;QACpCP,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACG,MAAM,CAAC;MACtC;MACA;IACF;IAGA,KAAK,IAAIU,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGzB,MAAM,CAACmB,MAAM,EAAEM,CAAC,EAAE,EAAE;MACtC,MAAMC,IAAI,GAAGD,CAAC,KAAK,CAAC,GAAGX,MAAM,GAAGnB,KAAK,CAACgC,SAAS;MAC/C,IAAIhC,KAAK,CAAC6B,QAAQ,EAAE;QAClBhB,iBAAiB,CAACb,KAAK,CAACc,MAAM,EAAGT,MAAM,CAACyB,CAAC,CAAC,EAAElB,IAAI,EAAEmB,IAAI,EAAE/B,KAAK,CAACiC,SAAS,CAAC;MAC1E,CAAC,MAAM;QACLrB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACK,KAAK,IAAI,CAAC;QAC1BK,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACE,OAAO,CAACa,IAAI,CAACc,IAAI,CAAC;QAClCnB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACC,OAAO,CAACc,IAAI,CAACjB,KAAK,CAACiC,SAAS,CAAC;QAC7CrB,IAAI,CAACZ,KAAK,CAACE,GAAG,CAAC,CAACG,MAAM,CAACY,IAAI,CACzBrB,KAAK,CAACsC,WAAW,CAAElC,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EAAI/B,MAAM,CAACyB,CAAC,CAAC,CAC3E,CAAC;MACH;IACF;EACF;AACF;AAqBA,OAAO,SAASO,eAAeA,CAACvC,MAAqB,EAAEa,QAAyB,EAAgB;EAC9F,MAAM2B,IAAkB,GAAG,EAAE;EAE7B,KAAK,IAAIR,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGnB,QAAQ,CAACH,QAAQ,EAAEsB,CAAC,EAAE,EAAE;IAC1CQ,IAAI,CAACrB,IAAI,CAAC,CAAC,CAAC,CAAC;EACf;EACA,KAAK,MAAMf,GAAG,IAAIS,QAAQ,CAACZ,UAAU,EAAE;IACrC,MAAMA,UAAU,GAAGY,QAAQ,CAACZ,UAAU,CAACG,GAAG,CAAC;IAC3C,IAAIH,UAAU,CAACQ,KAAK,EAAE;MACpBgC,uBAAuB,CAACzC,MAAM,EAAEC,UAAU,EAAEG,GAAG,EAAEoC,IAAI,CAAC;IACxD;EACF;EACA,OAAOA,IAAI;AACb;AAIA,SAASC,uBAAuBA,CAC9BzC,MAAqB,EACrBC,UAA8B,EAC9BG,GAAW,EACXoC,IAAkB,EACZ;EACN,MAAMtC,KAAK,GAAGF,MAAM,CAAC0C,SAAS,CAACtC,GAAG,CAAC;EACnC,MAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAe,CAACxC,GAAG,CAAC;EAG1C,MAAMyC,OAAiB,GAAG,IAAI5B,KAAK,CAACf,KAAK,CAACgC,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIf,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG/B,UAAU,CAACQ,KAAK,EAAEuB,CAAC,EAAE,EAAE;IACzC,MAAMV,MAAM,GAAGrB,UAAU,CAACI,OAAO,CAAC2B,CAAC,CAAC;IACpC,MAAMX,MAAM,GAAGpB,UAAU,CAACK,OAAO,CAAC0B,CAAC,CAAC;IACpCa,OAAO,CAACxB,MAAM,CAAC,EAAE;IACjBwB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAEzB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI2B,MAAM,GAAG,CAAC;IACd,IAAIpC,MAAM,GAAG4B,IAAI,CAACK,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC;IAGxC,KAAK,MAAMC,IAAI,IAAIN,MAAM,EAAE;MACzB,IAAIM,IAAI,KAAK/C,KAAK,IAAIoB,MAAM,GAAG2B,IAAI,CAACd,SAAS,EAAE;QAC7C;MACF;MAEA,QAAQc,IAAI,CAACrB,cAAc;QACzB,KAAK,UAAU;UACb,IAAI,EAAEqB,IAAI,CAAC1B,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE1BA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAG,EAAE;UACxB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;UAC5B,OAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAErCtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;UAC5B;UACAP,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAAC2B,EAAE,CAAC;UAC9B;QAEF;UACEtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAGX,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,IAAI,CAAC,CAAC;UAC3CX,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC;MAC9B;IACF;IAGA,IAAID,MAAM,KAAKpB,KAAK,CAACiC,SAAS,EAAE;MAC9B,MAAMgB,KAAK,GAAGrD,KAAK,CAACsD,aAAa,CAE/BlD,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EACzCrC,UAAU,CAACM,MAAM,CAACwC,MAAM,CAAC,EACzB7C,KACF,CAAC;MACD6C,MAAM,EAAE;MAER,QAAQ7C,KAAK,CAAC0B,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAE1B,KAAK,CAACqB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOpC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAEtCtC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC2B,EAAE,CAAC,GAAGC,KAAK;UAC9B;QAEF;UACEvC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG4B,KAAK;MAC9B;IACF;EACF;AACF;AAuBA,OAAO,SAASE,kBAAkBA,CAChCrD,MAAqB,EACrBa,QAAyB,EACE;EAC3B,MAAMyC,OAAkC,GAAG,CAAC,CAAC;EAC7C,KAAK,MAAMlD,GAAG,IAAIS,QAAQ,CAACZ,UAAU,EAAE;IACrC,MAAMA,UAAU,GAAGY,QAAQ,CAACZ,UAAU,CAACG,GAAG,CAAC;IAC3C,IAAIH,UAAU,CAACQ,KAAK,EAAE;MACpB8C,gCAAgC,CAACvD,MAAM,EAAEC,UAAU,EAAEY,QAAQ,CAACH,QAAQ,EAAEN,GAAG,EAAEkD,OAAO,CAAC;IACvF;EACF;EACA,OAAOA,OAAO;AAChB;AAGA,SAASC,gCAAgCA,CACvCvD,MAAqB,EACrBC,UAA8B,EAC9BS,QAAgB,EAChBN,GAAW,EACXkD,OAAuC,EACvC;EACA,IAAIrD,UAAU,CAACQ,KAAK,IAAI,CAAC,EAAE;IACzB;EACF;EAEA,MAAMP,KAAK,GAAGF,MAAM,CAAC0C,SAAS,CAACtC,GAAG,CAAC;EACnC,MAAMuC,MAAM,GAAG3C,MAAM,CAAC4C,eAAe,CAACxC,GAAG,CAAC;EAE1C,MAAMoD,UAAU,GAAGb,MAAM,CAAC,CAAC,CAAC,CAACpB,IAAI;EAEjC,IAAIkC,MAA6B;EACjC,MAAM;IAAClD;EAAM,CAAC,GAAGN,UAAU;EAC3B,IAAIM,MAAM,CAACmB,MAAM,KAAKhB,QAAQ,IAAIiC,MAAM,CAAC,CAAC,CAAC,CAACL,aAAa,EAAE;IAMzDmB,MAAM,GAAGlD,MAAM;EACjB;EAEA,IAAIkD,MAAM,EAAE;IACVH,OAAO,CAACE,UAAU,CAAC,GAAGC,MAAM;IAC5B;EACF;EAEAA,MAAM,GAAG,IAAIxC,KAAK,CAACP,QAAQ,CAAC;EAC5B,KAAK,IAAIsB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGtB,QAAQ,EAAEsB,CAAC,EAAE,EAAE;IACjCyB,MAAM,CAACzB,CAAC,CAAC,GAAG,CAAC,CAAC;EAChB;EACAsB,OAAO,CAACE,UAAU,CAAC,GAAGC,MAAM;EAG5B,MAAMZ,OAAiB,GAAG,IAAI5B,KAAK,CAACf,KAAK,CAACgC,SAAS,GAAG,CAAC,CAAC,CAACY,IAAI,CAAC,CAAC,CAAC;EAChE,IAAIC,MAAM,GAAG,CAAC;EACd,KAAK,IAAIf,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG/B,UAAU,CAACQ,KAAK,EAAEuB,CAAC,EAAE,EAAE;IACzC,MAAMV,MAAM,GAAGrB,UAAU,CAACI,OAAO,CAAC2B,CAAC,CAAC;IACpC,MAAMX,MAAM,GAAGpB,UAAU,CAACK,OAAO,CAAC0B,CAAC,CAAC;IACpCa,OAAO,CAACxB,MAAM,CAAC,EAAE;IACjBwB,OAAO,CAACC,IAAI,CAAC,CAAC,EAAEzB,MAAM,GAAG,CAAC,CAAC;IAE3B,IAAI2B,MAAM,GAAG,CAAC;IACd,IAAIpC,MAAM,GAAG6C,MAAM,CAACZ,OAAO,CAACG,MAAM,EAAE,CAAC,GAAG,CAAC,CAAe;IAGxD,KAAK,MAAMC,IAAI,IAAIN,MAAM,EAAE;MACzB,IAAIM,IAAI,KAAK/C,KAAK,IAAIoB,MAAM,GAAG2B,IAAI,CAACd,SAAS,EAAE;QAC7C;MACF;MAEA,QAAQc,IAAI,CAACrB,cAAc;QACzB,KAAK,UAAU;UACb,IAAI,EAAEqB,IAAI,CAAC1B,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE1BA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAG,EAAE;UACxB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,EAAE,CAAC;UAC5B,OAAOpC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAErCtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAACJ,IAAI,CAAC,CAAC,CAAC,CAAC;UAC5B;UACAP,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,CAAC2B,EAAE,CAAC;UAC9B;QAEF;UACEtC,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,GAAGX,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC,IAAI,CAAC,CAAC;UAC3CX,MAAM,GAAGA,MAAM,CAACqC,IAAI,CAAC1B,IAAI,CAAC;MAC9B;IACF;IAGA,IAAID,MAAM,KAAKpB,KAAK,CAACiC,SAAS,EAAE;MAC9B,MAAMgB,KAAK,GAAGrD,KAAK,CAACsD,aAAa,CAE/BlD,KAAK,CAACmC,YAAY,IAAInC,KAAK,CAACoC,aAAa,EACzCrC,UAAU,CAACM,MAAM,CAACwC,MAAM,CAAC,EACzB7C,KACF,CAAC;MACD6C,MAAM,EAAE;MAER,QAAQ7C,KAAK,CAAC0B,cAAc;QAC1B,KAAK,UAAU;UACb,IAAI,EAAE1B,KAAK,CAACqB,IAAI,IAAIX,MAAM,CAAC,EAAE;YAE3BA,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG,EAAE;UACzB;UACA,MAAM2B,EAAE,GAAGL,OAAO,CAACG,MAAM,CAAC;UAC1B,OAAOpC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACG,MAAM,IAAIwB,EAAE,EAAE;YAEtCtC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAACJ,IAAI,CAAC,IAAI,CAAC;UAC/B;UACAP,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,CAAC2B,EAAE,CAAC,GAAGC,KAAK;UAC9B;QAEF;UACEvC,MAAM,CAACV,KAAK,CAACqB,IAAI,CAAC,GAAG4B,KAAK;MAC9B;IACF;EACF;EAGA,KAAK,IAAInB,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGtB,QAAQ,EAAE,EAAEsB,CAAC,EAAE;IACjC,IAAIwB,UAAU,IAAKC,MAAM,CAACzB,CAAC,CAAY,EAAE;MACvCyB,MAAM,CAACzB,CAAC,CAAC,GAAIyB,MAAM,CAACzB,CAAC,CAAC,CAAYwB,UAAU,CAAC;IAC/C;EACF;AACF"}

package/dist/index.d.ts

@@ -1,63 +1,22 @@
 import type { LoaderWithParser } from '@loaders.gl/loader-utils';
+import type { ObjectRowTable, ObjectRowTableBatch, ColumnarTable, ColumnarTableBatch } from '@loaders.gl/schema';
+import type { Table as ArrowTable } from 'apache-arrow';
+import { ParquetLoader as ParquetWorkerLoader, ParquetLoaderOptions } from './parquet-loader';
+import { ParquetWasmLoaderOptions } from './lib/wasm/parse-parquet-wasm';
 import { ParquetWasmLoader as ParquetWasmWorkerLoader } from './parquet-wasm-loader';
-import { ParquetLoader as ParquetWorkerLoader } from './parquet-loader';
-import { parseParquet, parseParquetFileInBatches } from './lib/parsers/parse-parquet-to-rows';
-import { parseParquetInColumns, parseParquetFileInColumnarBatches } from './lib/parsers/parse-parquet-to-columns';
-import { parseParquet as parseParquetWasm } from './lib/wasm/parse-parquet-wasm';
 export { ParquetWorkerLoader, ParquetWasmWorkerLoader };
 /** ParquetJS table loader */
-export declare const ParquetLoader: {
-    parse: typeof parseParquet;
-    parseFileInBatches: typeof parseParquetFileInBatches;
-    name: string;
-    id: string;
-    module: string; /** ParquetJS table loader */
-    version: any;
-    worker: boolean;
-    category: string;
-    extensions: string[];
-    mimeTypes: string[];
-    binary: boolean;
-    tests: string[];
-    options: import("./parquet-loader").ParquetLoaderOptions;
-};
+export declare const ParquetLoader: LoaderWithParser<ObjectRowTable, ObjectRowTableBatch, ParquetLoaderOptions>;
 /** ParquetJS table loader */
-export declare const ParquetColumnarLoader: {
-    parse: typeof parseParquetInColumns;
-    parseFileInBatches: typeof parseParquetFileInColumnarBatches;
-    name: string;
-    id: string;
-    module: string; /** ParquetJS table loader */
-    version: any;
-    worker: boolean;
-    category: string;
-    extensions: string[];
-    mimeTypes: string[];
-    binary: boolean;
-    tests: string[];
-    options: import("./parquet-loader").ParquetLoaderOptions;
-};
-export declare const ParquetWasmLoader: {
-    parse: typeof parseParquetWasm;
-    name: string;
-    id: string;
-    module: string;
-    version: any;
-    worker: boolean;
-    category: string;
-    extensions: string[];
-    mimeTypes: string[];
-    binary: boolean;
-    tests: string[];
-    options: import("./parquet-wasm-loader").ParquetLoaderOptions;
-};
+export declare const ParquetColumnarLoader: LoaderWithParser<ColumnarTable, ColumnarTableBatch, ParquetLoaderOptions>;
+export declare const ParquetWasmLoader: LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions>;
 export { ParquetWriter as _ParquetWriter } from './parquet-writer';
 export { ParquetWasmWriter } from './parquet-wasm-writer';
 export { preloadCompressions } from './parquetjs/compression';
 export { ParquetSchema } from './parquetjs/schema/schema';
 export { ParquetReader } from './parquetjs/parser/parquet-reader';
 export { ParquetEncoder } from './parquetjs/encoder/parquet-encoder';
-export { convertSchemaFromParquet, convertSchemaFromParquet as convertParquetToArrowSchema } from './lib/arrow/convert-schema-from-parquet';
+export { convertParquetSchema, convertParquetSchema as convertParquetToArrowSchema } from './lib/arrow/convert-schema-from-parquet';
 export declare const _typecheckParquetLoader: LoaderWithParser;
 export { default as geoJSONSchema } from './lib/geo/geoparquet-schema';
 export type { GeoMetadata } from './lib/geo/decode-geo-metadata';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAI/D,OAAO,EAAC,iBAAiB,IAAI,uBAAuB,EAAC,MAAM,uBAAuB,CAAC;AACnF,OAAO,EAAC,aAAa,IAAI,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAC,YAAY,EAAE,yBAAyB,EAAC,MAAM,qCAAqC,CAAC;AAC5F,OAAO,EACL,qBAAqB,EACrB,iCAAiC,EAClC,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAAC,YAAY,IAAI,gBAAgB,EAAC,MAAM,+BAA+B,CAAC;AAE/E,OAAO,EAAC,mBAAmB,EAAE,uBAAuB,EAAC,CAAC;AAEtD,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;;;oBAM1B,6BAA6B;;;;;;;;;CAF5B,CAAC;AAEF,6BAA6B;AAC7B,eAAO,MAAM,qBAAqB;;;;;oBADlC,6BAA6B;;;;;;;;;CAK5B,CAAC;AAEF,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;CAG7B,CAAC;AAIF,OAAO,EAAC,aAAa,IAAI,cAAc,EAAC,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAC,iBAAiB,EAAC,MAAM,uBAAuB,CAAC;AAIxD,OAAO,EAAC,mBAAmB,EAAC,MAAM,yBAAyB,CAAC;AAE5D,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,mCAAmC,CAAC;AAChE,OAAO,EAAC,cAAc,EAAC,MAAM,qCAAqC,CAAC;AAEnE,OAAO,EACL,wBAAwB,EACxB,wBAAwB,IAAI,2BAA2B,EACxD,MAAM,yCAAyC,CAAC;AAGjD,eAAO,MAAM,uBAAuB,EAAE,gBAAgC,CAAC;AAGvE,OAAO,EAAC,OAAO,IAAI,aAAa,EAAC,MAAM,6BAA6B,CAAC;AAErE,YAAY,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAC,cAAc,EAAE,cAAc,EAAE,iBAAiB,EAAC,MAAM,+BAA+B,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EACV,cAAc,EACd,mBAAmB,EACnB,aAAa,EACb,kBAAkB,EACnB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,KAAK,EAAC,KAAK,IAAI,UAAU,EAAC,MAAM,cAAc,CAAC;AAItD,OAAO,EACL,aAAa,IAAI,mBAAmB,EAEpC,oBAAoB,EACrB,MAAM,kBAAkB,CAAC;AAO1B,OAAO,EAAmB,wBAAwB,EAAC,MAAM,+BAA+B,CAAC;AACzF,OAAO,EAAC,iBAAiB,IAAI,uBAAuB,EAAC,MAAM,uBAAuB,CAAC;AAEnF,OAAO,EAAC,mBAAmB,EAAE,uBAAuB,EAAC,CAAC;AAEtD,6BAA6B;AAC7B,eAAO,MAAM,aAAa,EAAE,gBAAgB,CAC1C,cAAc,EACd,mBAAmB,EACnB,oBAAoB,CAKrB,CAAC;AAEF,6BAA6B;AAE7B,eAAO,MAAM,qBAAqB,EAAE,gBAAgB,CAClD,aAAa,EACb,kBAAkB,EAClB,oBAAoB,CAKrB,CAAC;AAEF,eAAO,MAAM,iBAAiB,EAAE,gBAAgB,CAAC,UAAU,EAAE,KAAK,EAAE,wBAAwB,CAG3F,CAAC;AAIF,OAAO,EAAC,aAAa,IAAI,cAAc,EAAC,MAAM,kBAAkB,CAAC;AACjE,OAAO,EAAC,iBAAiB,EAAC,MAAM,uBAAuB,CAAC;AAIxD,OAAO,EAAC,mBAAmB,EAAC,MAAM,yBAAyB,CAAC;AAE5D,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,mCAAmC,CAAC;AAChE,OAAO,EAAC,cAAc,EAAC,MAAM,qCAAqC,CAAC;AAEnE,OAAO,EACL,oBAAoB,EACpB,oBAAoB,IAAI,2BAA2B,EACpD,MAAM,yCAAyC,CAAC;AAGjD,eAAO,MAAM,uBAAuB,EAAE,gBAAgC,CAAC;AAGvE,OAAO,EAAC,OAAO,IAAI,aAAa,EAAC,MAAM,6BAA6B,CAAC;AAErE,YAAY,EAAC,WAAW,EAAC,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAC,cAAc,EAAE,cAAc,EAAE,iBAAiB,EAAC,MAAM,+BAA+B,CAAC"}

package/dist/index.js

@@ -1,17 +1,18 @@
 "use strict";
+// loaders.gl, MIT license
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.unpackGeoMetadata = exports.setGeoMetadata = exports.getGeoMetadata = exports.geoJSONSchema = exports._typecheckParquetLoader = exports.convertParquetToArrowSchema = exports.convertSchemaFromParquet = exports.ParquetEncoder = exports.ParquetReader = exports.ParquetSchema = exports.preloadCompressions = exports.ParquetWasmWriter = exports._ParquetWriter = exports.ParquetWasmLoader = exports.ParquetColumnarLoader = exports.ParquetLoader = exports.ParquetWasmWorkerLoader = exports.ParquetWorkerLoader = void 0;
+exports.unpackGeoMetadata = exports.setGeoMetadata = exports.getGeoMetadata = exports.geoJSONSchema = exports._typecheckParquetLoader = exports.convertParquetToArrowSchema = exports.convertParquetSchema = exports.ParquetEncoder = exports.ParquetReader = exports.ParquetSchema = exports.preloadCompressions = exports.ParquetWasmWriter = exports._ParquetWriter = exports.ParquetWasmLoader = exports.ParquetColumnarLoader = exports.ParquetLoader = exports.ParquetWasmWorkerLoader = exports.ParquetWorkerLoader = void 0;
 // ParquetLoader
-const parquet_wasm_loader_1 = require("./parquet-wasm-loader");
-Object.defineProperty(exports, "ParquetWasmWorkerLoader", { enumerable: true, get: function () { return parquet_wasm_loader_1.ParquetWasmLoader; } });
 const parquet_loader_1 = require("./parquet-loader");
 Object.defineProperty(exports, "ParquetWorkerLoader", { enumerable: true, get: function () { return parquet_loader_1.ParquetLoader; } });
 const parse_parquet_to_rows_1 = require("./lib/parsers/parse-parquet-to-rows");
 const parse_parquet_to_columns_1 = require("./lib/parsers/parse-parquet-to-columns");
 const parse_parquet_wasm_1 = require("./lib/wasm/parse-parquet-wasm");
+const parquet_wasm_loader_1 = require("./parquet-wasm-loader");
+Object.defineProperty(exports, "ParquetWasmWorkerLoader", { enumerable: true, get: function () { return parquet_wasm_loader_1.ParquetWasmLoader; } });
 /** ParquetJS table loader */
 exports.ParquetLoader = {
     ...parquet_loader_1.ParquetLoader,
@@ -19,6 +20,7 @@ exports.ParquetLoader = {
     parseFileInBatches: parse_parquet_to_rows_1.parseParquetFileInBatches
 };
 /** ParquetJS table loader */
+// @ts-expect-error
 exports.ParquetColumnarLoader = {
     ...parquet_loader_1.ParquetLoader,
     parse: parse_parquet_to_columns_1.parseParquetInColumns,
@@ -26,7 +28,7 @@ exports.ParquetColumnarLoader = {
 };
 exports.ParquetWasmLoader = {
     ...parquet_wasm_loader_1.ParquetWasmLoader,
-    parse: parse_parquet_wasm_1.parseParquet
+    parse: parse_parquet_wasm_1.parseParquetWasm
 };
 // ParquetWriter
 var parquet_writer_1 = require("./parquet-writer");
@@ -43,8 +45,8 @@ Object.defineProperty(exports, "ParquetReader", { enumerable: true, get: functio
 var parquet_encoder_1 = require("./parquetjs/encoder/parquet-encoder");
 Object.defineProperty(exports, "ParquetEncoder", { enumerable: true, get: function () { return parquet_encoder_1.ParquetEncoder; } });
 var convert_schema_from_parquet_1 = require("./lib/arrow/convert-schema-from-parquet");
-Object.defineProperty(exports, "convertSchemaFromParquet", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertSchemaFromParquet; } });
-Object.defineProperty(exports, "convertParquetToArrowSchema", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertSchemaFromParquet; } });
+Object.defineProperty(exports, "convertParquetSchema", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertParquetSchema; } });
+Object.defineProperty(exports, "convertParquetToArrowSchema", { enumerable: true, get: function () { return convert_schema_from_parquet_1.convertParquetSchema; } });
 // TESTS
 exports._typecheckParquetLoader = exports.ParquetLoader;
 // Geo Metadata

package/dist/lib/arrow/convert-row-group-to-columns.d.ts

@@ -1,4 +1,4 @@
 import { Schema } from '@loaders.gl/schema';
-import { ParquetBuffer } from '@loaders.gl/parquet/parquetjs/schema/declare';
-export declare function convertParquetRowGroupToColumns(schema: Schema, rowGroup: ParquetBuffer): Record<string, any[]>;
+import { ParquetRowGroup } from '@loaders.gl/parquet/parquetjs/schema/declare';
+export declare function convertParquetRowGroupToColumns(schema: Schema, rowGroup: ParquetRowGroup): Record<string, any[]>;
 //# sourceMappingURL=convert-row-group-to-columns.d.ts.map

package/dist/lib/arrow/convert-row-group-to-columns.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"convert-row-group-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-row-group-to-columns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAC,aAAa,EAAC,MAAM,8CAA8C,CAAC;AAE3E,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,aAAa,GACtB,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAMvB"}
+{"version":3,"file":"convert-row-group-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-row-group-to-columns.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAC,eAAe,EAAC,MAAM,8CAA8C,CAAC;AAE7E,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,eAAe,GACxB,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAMvB"}

package/dist/lib/arrow/convert-schema-from-parquet.d.ts

@@ -1,9 +1,9 @@
+import { Schema, DataType } from '@loaders.gl/schema';
 import type { ParquetSchema } from '../../parquetjs/schema/schema';
 import type { ParquetType } from '../../parquetjs/schema/declare';
-import { FileMetaData } from '@loaders.gl/parquet/parquetjs/parquet-thrift';
-import { Schema, DataType } from '@loaders.gl/schema';
+import { FileMetaData } from '../../parquetjs/parquet-thrift';
 export declare const PARQUET_TYPE_MAPPING: {
-    [type in ParquetType]: typeof DataType;
+    [type in ParquetType]: DataType;
 };
-export declare function convertSchemaFromParquet(parquetSchema: ParquetSchema, parquetMetadata?: FileMetaData): Schema;
+export declare function convertParquetSchema(parquetSchema: ParquetSchema, parquetMetadata: FileMetaData | null): Schema;
 //# sourceMappingURL=convert-schema-from-parquet.d.ts.map

package/dist/lib/arrow/convert-schema-from-parquet.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"convert-schema-from-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,+BAA+B,CAAC;AACjE,OAAO,KAAK,EAAgC,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAC/F,OAAO,EAAC,YAAY,EAAC,MAAM,8CAA8C,CAAC;AAE1E,OAAO,EACL,MAAM,EAGN,QAAQ,EAaT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,OAAO,QAAQ;CA+BzE,CAAC;AAEF,wBAAgB,wBAAwB,CACtC,aAAa,EAAE,aAAa,EAC5B,eAAe,CAAC,EAAE,YAAY,GAC7B,MAAM,CAIR"}
+{"version":3,"file":"convert-schema-from-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAS,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAE3D,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,+BAA+B,CAAC;AACjE,OAAO,KAAK,EAAgC,WAAW,EAAC,MAAM,gCAAgC,CAAC;AAC/F,OAAO,EAAC,YAAY,EAAC,MAAM,gCAAgC,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CA+BlE,CAAC;AAEF,wBAAgB,oBAAoB,CAClC,aAAa,EAAE,aAAa,EAC5B,eAAe,EAAE,YAAY,GAAG,IAAI,GACnC,MAAM,CAUR"}

package/dist/lib/arrow/convert-schema-from-parquet.js

@@ -1,81 +1,85 @@
 "use strict";
 // loaders.gl, MIT license
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.convertSchemaFromParquet = exports.PARQUET_TYPE_MAPPING = void 0;
-const schema_1 = require("@loaders.gl/schema");
+exports.convertParquetSchema = exports.PARQUET_TYPE_MAPPING = void 0;
 exports.PARQUET_TYPE_MAPPING = {
-    BOOLEAN: schema_1.Bool,
-    INT32: schema_1.Int32,
-    INT64: schema_1.Float64,
-    INT96: schema_1.Float64,
-    FLOAT: schema_1.Float32,
-    DOUBLE: schema_1.Float64,
-    BYTE_ARRAY: schema_1.Binary,
-    FIXED_LEN_BYTE_ARRAY: schema_1.Binary,
-    UTF8: schema_1.Utf8,
-    DATE: schema_1.Int32,
-    TIME_MILLIS: schema_1.Int64,
-    TIME_MICROS: schema_1.Int64,
-    TIMESTAMP_MILLIS: schema_1.Int64,
-    TIMESTAMP_MICROS: schema_1.Int64,
-    UINT_8: schema_1.Int32,
-    UINT_16: schema_1.Uint16,
-    UINT_32: schema_1.Uint32,
-    UINT_64: schema_1.Uint64,
-    INT_8: schema_1.Int8,
-    INT_16: schema_1.Int16,
-    INT_32: schema_1.Int32,
-    INT_64: schema_1.Int64,
-    JSON: schema_1.Binary,
-    BSON: schema_1.Binary,
-    // TODO check interval type
-    INTERVAL: schema_1.Binary,
-    DECIMAL_INT32: schema_1.Float32,
-    DECIMAL_INT64: schema_1.Float64,
-    DECIMAL_BYTE_ARRAY: schema_1.Float64,
-    DECIMAL_FIXED_LEN_BYTE_ARRAY: schema_1.Float64
+    BOOLEAN: 'bool',
+    INT32: 'int32',
+    INT64: 'float64',
+    INT96: 'float64',
+    FLOAT: 'float32',
+    DOUBLE: 'float64',
+    BYTE_ARRAY: 'binary',
+    FIXED_LEN_BYTE_ARRAY: 'binary',
+    UTF8: 'utf8',
+    DATE: 'int32',
+    TIME_MILLIS: 'int64',
+    TIME_MICROS: 'int64',
+    TIMESTAMP_MILLIS: 'int64',
+    TIMESTAMP_MICROS: 'int64',
+    UINT_8: 'int32',
+    UINT_16: 'uint16',
+    UINT_32: 'uint32',
+    UINT_64: 'uint64',
+    INT_8: 'int8',
+    INT_16: 'int16',
+    INT_32: 'int32',
+    INT_64: 'int64',
+    JSON: 'binary',
+    BSON: 'binary',
+    // TODO check interal type
+    INTERVAL: 'binary',
+    DECIMAL_INT32: 'float32',
+    DECIMAL_INT64: 'float64',
+    DECIMAL_BYTE_ARRAY: 'float64',
+    DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
 };
-function convertSchemaFromParquet(parquetSchema, parquetMetadata) {
+function convertParquetSchema(parquetSchema, parquetMetadata) {
     const fields = getFields(parquetSchema.schema);
     const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
-    return new schema_1.Schema(fields, metadata);
+    const schema = {
+        fields,
+        metadata: metadata || {}
+    };
+    return schema;
 }
-exports.convertSchemaFromParquet = convertSchemaFromParquet;
+exports.convertParquetSchema = convertParquetSchema;
 function getFields(schema) {
     const fields = [];
     for (const name in schema) {
         const field = schema[name];
         if (field.fields) {
-            const childFields = getFields(field.fields);
-            const nestedField = new schema_1.Field(name, new schema_1.Struct(childFields), field.optional);
-            fields.push(nestedField);
+            const children = getFields(field.fields);
+            fields.push({ name, type: { type: 'struct', children }, nullable: field.optional });
         }
         else {
-            const FieldType = exports.PARQUET_TYPE_MAPPING[field.type];
+            const type = exports.PARQUET_TYPE_MAPPING[field.type];
             const metadata = getFieldMetadata(field);
-            const arrowField = new schema_1.Field(name, new FieldType(), field.optional, metadata);
+            const arrowField = { name, type, nullable: field.optional, metadata };
             fields.push(arrowField);
         }
     }
     return fields;
 }
 function getFieldMetadata(field) {
-    const metadata = new Map();
+    let metadata;
     for (const key in field) {
         if (key !== 'name') {
             let value = field[key] || '';
             value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
-            metadata.set(key, value);
+            metadata = metadata || {};
+            metadata[key] = value;
         }
     }
     return metadata;
 }
 function getSchemaMetadata(parquetMetadata) {
-    const metadata = new Map();
+    let metadata;
     const keyValueList = parquetMetadata.key_value_metadata || [];
     for (const { key, value } of keyValueList) {
         if (typeof value === 'string') {
-            metadata.set(key, value);
+            metadata = metadata || {};
+            metadata[key] = value;
         }
     }
     return metadata;

package/dist/lib/arrow/convert-schema-to-parquet.d.ts

@@ -1,7 +1,7 @@
 import type { ParquetType } from '../../parquetjs/schema/declare';
 import { Schema, DataType } from '@loaders.gl/schema';
 export declare const PARQUET_TYPE_MAPPING: {
-    [type in ParquetType]: typeof DataType;
+    [type in ParquetType]: DataType;
 };
 export declare function convertToParquetSchema(schema: Schema): Schema;
 //# sourceMappingURL=convert-schema-to-parquet.d.ts.map

package/dist/lib/arrow/convert-schema-to-parquet.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"convert-schema-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,gCAAgC,CAAC;AAExC,OAAO,EACL,MAAM,EAGN,QAAQ,EAaT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,OAAO,QAAQ;CA+BzE,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAK7D"}
+{"version":3,"file":"convert-schema-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAEV,WAAW,EACZ,MAAM,gCAAgC,CAAC;AAExC,OAAO,EACL,MAAM,EAGN,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,QAAQ;CA+BlE,CAAC;AAEF,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAK7D"}