@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.6

package/src/parquetjs/schema/shred.ts

@@ -1,13 +1,14 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 
-import {ParquetBuffer, ParquetData, ParquetField, ParquetRecord} from './declare';
+import {ArrayType} from '@loaders.gl/schema';
+import {ParquetRowGroup, ParquetColumnChunk, ParquetField, ParquetRow} from './declare';
 import {ParquetSchema} from './schema';
 import * as Types from './types';
 
-export {ParquetBuffer};
+export {ParquetRowGroup};
 
-export function shredBuffer(schema: ParquetSchema): ParquetBuffer {
-  const columnData: Record<string, ParquetData> = {};
+export function shredBuffer(schema: ParquetSchema): ParquetRowGroup {
+  const columnData: Record<string, ParquetColumnChunk> = {};
   for (const field of schema.fieldList) {
     columnData[field.key] = {
       dlevels: [],
@@ -24,14 +25,14 @@ export function shredBuffer(schema: ParquetSchema): ParquetBuffer {
  * 'Shred' a record into a list of <value, repetition_level, definition_level>
  * tuples per column using the Google Dremel Algorithm..
  *
- * The buffer argument must point to an object into which the shredded record
- * will be returned. You may re-use the buffer for repeated calls to this function
- * to append to an existing buffer, as long as the schema is unchanged.
+ * The rowGroup argument must point to an object into which the shredded record
+ * will be returned. You may re-use the rowGroup for repeated calls to this function
+ * to append to an existing rowGroup, as long as the schema is unchanged.
  *
- * The format in which the shredded records will be stored in the buffer is as
+ * The format in which the shredded records will be stored in the rowGroup is as
  * follows:
  *
- *   buffer = {
+ *   rowGroup = {
  *     columnData: [
  *       'my_col': {
  *          dlevels: [d1, d2, .. dN],
@@ -42,32 +43,36 @@ export function shredBuffer(schema: ParquetSchema): ParquetBuffer {
  *      rowCount: X,
  *   }
  */
-export function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void {
+export function shredRecord(
+  schema: ParquetSchema,
+  record: ParquetRow,
+  rowGroup: ParquetRowGroup
+): void {
   /* shred the record, this may raise an exception */
   const data = shredBuffer(schema).columnData;
 
   shredRecordFields(schema.fields, record, data, 0, 0);
 
-  /* if no error during shredding, add the shredded record to the buffer */
-  if (buffer.rowCount === 0) {
-    buffer.rowCount = 1;
-    buffer.columnData = data;
+  /* if no error during shredding, add the shredded record to the rowGroup */
+  if (rowGroup.rowCount === 0) {
+    rowGroup.rowCount = 1;
+    rowGroup.columnData = data;
     return;
   }
-  buffer.rowCount += 1;
+  rowGroup.rowCount += 1;
   for (const field of schema.fieldList) {
-    Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);
-    Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);
-    Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);
-    buffer.columnData[field.key].count += data[field.key].count;
+    Array.prototype.push.apply(rowGroup.columnData[field.key].rlevels, data[field.key].rlevels);
+    Array.prototype.push.apply(rowGroup.columnData[field.key].dlevels, data[field.key].dlevels);
+    Array.prototype.push.apply(rowGroup.columnData[field.key].values, data[field.key].values);
+    rowGroup.columnData[field.key].count += data[field.key].count;
   }
 }
 
 // eslint-disable-next-line max-statements, complexity
 function shredRecordFields(
   fields: Record<string, ParquetField>,
-  record: any,
-  data: Record<string, ParquetData>,
+  record: ParquetRow,
+  data: Record<string, ParquetColumnChunk>,
   rLevel: number,
   dLevel: number
 ) {
@@ -99,7 +104,7 @@ function shredRecordFields(
     // push null
     if (values.length === 0) {
       if (field.isNested) {
-        shredRecordFields(field.fields!, null, data, rLevel, dLevel);
+        shredRecordFields(field.fields!, null!, data, rLevel, dLevel);
       } else {
         data[field.key].count += 1;
         data[field.key].rlevels.push(rLevel);
@@ -130,10 +135,10 @@ function shredRecordFields(
  * tuples back to nested records (objects/arrays) using the Google Dremel
  * Algorithm..
  *
- * The buffer argument must point to an object with the following structure (i.e.
+ * The rowGroup argument must point to an object with the following structure (i.e.
  * the same structure that is returned by shredRecords):
  *
- *   buffer = {
+ *   rowGroup = {
  *     columnData: [
  *       'my_col': {
  *          dlevels: [d1, d2, .. dN],
@@ -144,84 +149,247 @@ function shredRecordFields(
  *      rowCount: X,
  *   }
  */
-export function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[] {
-  const records: ParquetRecord[] = [];
-  for (let i = 0; i < buffer.rowCount; i++) records.push({});
-  for (const key in buffer.columnData) {
-    materializeColumn(schema, buffer, key, records);
+export function materializeRows(schema: ParquetSchema, rowGroup: ParquetRowGroup): ParquetRow[] {
+  const rows: ParquetRow[] = [];
+  // rows = new Array(rowGroup.rowCount).fill({})'
+  for (let i = 0; i < rowGroup.rowCount; i++) {
+    rows.push({});
+  }
+  for (const key in rowGroup.columnData) {
+    const columnData = rowGroup.columnData[key];
+    if (columnData.count) {
+      materializeColumnAsRows(schema, columnData, key, rows);
+    }
   }
-  return records;
+  return rows;
 }
 
+/** Populate record fields for one column */
 // eslint-disable-next-line max-statements, complexity
-function materializeColumn(
+function materializeColumnAsRows(
   schema: ParquetSchema,
-  buffer: ParquetBuffer,
+  columnData: ParquetColumnChunk,
   key: string,
-  records: ParquetRecord[]
+  rows: ParquetRow[]
+): void {
+  const field = schema.findField(key);
+  const branch = schema.findFieldBranch(key);
+
+  // tslint:disable-next-line:prefer-array-literal
+  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
+  let vIndex = 0;
+  for (let i = 0; i < columnData.count; i++) {
+    const dLevel = columnData.dlevels[i];
+    const rLevel = columnData.rlevels[i];
+    rLevels[rLevel]++;
+    rLevels.fill(0, rLevel + 1);
+
+    let rIndex = 0;
+    let record = rows[rLevels[rIndex++] - 1];
+
+    // Internal nodes - Build a nested row object
+    for (const step of branch) {
+      if (step === field || dLevel < step.dLevelMax) {
+        break;
+      }
+
+      switch (step.repetitionType) {
+        case 'REPEATED':
+          if (!(step.name in record)) {
+            // eslint-disable max-depth
+            record[step.name] = [];
+          }
+          const ix = rLevels[rIndex++];
+          while (record[step.name].length <= ix) {
+            // eslint-disable max-depth
+            record[step.name].push({});
+          }
+          record = record[step.name][ix];
+          break;
+
+        default:
+          record[step.name] = record[step.name] || {};
+          record = record[step.name];
+      }
+    }
+
+    // Leaf node - Add the value
+    if (dLevel === field.dLevelMax) {
+      const value = Types.fromPrimitive(
+        // @ts-ignore
+        field.originalType || field.primitiveType,
+        columnData.values[vIndex],
+        field
+      );
+      vIndex++;
+
+      switch (field.repetitionType) {
+        case 'REPEATED':
+          if (!(field.name in record)) {
+            // eslint-disable max-depth
+            record[field.name] = [];
+          }
+          const ix = rLevels[rIndex];
+          while (record[field.name].length <= ix) {
+            // eslint-disable max-depth
+            record[field.name].push(null);
+          }
+          record[field.name][ix] = value;
+          break;
+
+        default:
+          record[field.name] = value;
+      }
+    }
+  }
+}
+
+// Columnar export
+
+/**
+ * 'Materialize' a list of <value, repetition_level, definition_level>
+ * tuples back to nested records (objects/arrays) using the Google Dremel
+ * Algorithm..
+ *
+ * The rowGroup argument must point to an object with the following structure (i.e.
+ * the same structure that is returned by shredRecords):
+ *
+ *   rowGroup = {
+ *     columnData: [
+ *       'my_col': {
+ *          dlevels: [d1, d2, .. dN],
+ *          rlevels: [r1, r2, .. rN],
+ *          values: [v1, v2, .. vN],
+ *        }, ...
+ *      ],
+ *      rowCount: X,
+ *   }
+ */
+export function materializeColumns(
+  schema: ParquetSchema,
+  rowGroup: ParquetRowGroup
+): Record<string, ArrayType> {
+  const columns: Record<string, ArrayType> = {};
+  for (const key in rowGroup.columnData) {
+    const columnData = rowGroup.columnData[key];
+    if (columnData.count) {
+      materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);
+    }
+  }
+  return columns;
+}
+
+// eslint-disable-next-line max-statements, complexity
+function materializeColumnAsColumnarArray(
+  schema: ParquetSchema,
+  columnData: ParquetColumnChunk,
+  rowCount: number,
+  key: string,
+  columns: Record<string, ArrayType<any>>
 ) {
-  const data = buffer.columnData[key];
-  if (!data.count) return;
+  if (columnData.count <= 0) {
+    return;
+  }
 
   const field = schema.findField(key);
   const branch = schema.findFieldBranch(key);
 
+  const columnName = branch[0].name;
+
+  let column: ArrayType | undefined;
+  const {values} = columnData;
+  if (values.length === rowCount && branch[0].primitiveType) {
+    // if (branch[0].repetitionType === `REQUIRED`) {
+    //   switch (branch[0].primitiveType) {
+    //     case 'INT32': return values instanceof Int32Array ? values : new Int32Array(values);
+    //   }
+    // }
+    column = values;
+  }
+
+  if (column) {
+    columns[columnName] = column;
+    return;
+  }
+
+  column = new Array(rowCount);
+  for (let i = 0; i < rowCount; i++) {
+    column[i] = {};
+  }
+  columns[columnName] = column;
+
   // tslint:disable-next-line:prefer-array-literal
   const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
   let vIndex = 0;
-  for (let i = 0; i < data.count; i++) {
-    const dLevel = data.dlevels[i];
-    const rLevel = data.rlevels[i];
+  for (let i = 0; i < columnData.count; i++) {
+    const dLevel = columnData.dlevels[i];
+    const rLevel = columnData.rlevels[i];
     rLevels[rLevel]++;
     rLevels.fill(0, rLevel + 1);
 
     let rIndex = 0;
-    let record = records[rLevels[rIndex++] - 1];
+    let record = column[rLevels[rIndex++] - 1] as ParquetRow;
 
-    // Internal nodes
+    // Internal nodes - Build a nested row object
     for (const step of branch) {
-      if (step === field) break;
-      if (dLevel < step.dLevelMax) break;
-      if (step.repetitionType === 'REPEATED') {
-        if (!(step.name in record)) {
-          // eslint-disable max-depth
-          record[step.name] = [];
-        }
-        const ix = rLevels[rIndex++];
-        while (record[step.name].length <= ix) {
-          // eslint-disable max-depth
-          record[step.name].push({});
-        }
-        record = record[step.name][ix];
-      } else {
-        record[step.name] = record[step.name] || {};
-        record = record[step.name];
+      if (step === field || dLevel < step.dLevelMax) {
+        break;
+      }
+
+      switch (step.repetitionType) {
+        case 'REPEATED':
+          if (!(step.name in record)) {
+            // eslint-disable max-depth
+            record[step.name] = [];
+          }
+          const ix = rLevels[rIndex++];
+          while (record[step.name].length <= ix) {
+            // eslint-disable max-depth
+            record[step.name].push({});
+          }
+          record = record[step.name][ix];
+          break;
+
+        default:
+          record[step.name] = record[step.name] || {};
+          record = record[step.name];
       }
     }
 
-    // Leaf node
+    // Leaf node - Add the value
     if (dLevel === field.dLevelMax) {
       const value = Types.fromPrimitive(
         // @ts-ignore
         field.originalType || field.primitiveType,
-        data.values[vIndex],
+        columnData.values[vIndex],
         field
       );
       vIndex++;
-      if (field.repetitionType === 'REPEATED') {
-        if (!(field.name in record)) {
-          // eslint-disable max-depth
-          record[field.name] = [];
-        }
-        const ix = rLevels[rIndex];
-        while (record[field.name].length <= ix) {
-          // eslint-disable max-depth
-          record[field.name].push(null);
-        }
-        record[field.name][ix] = value;
-      } else {
-        record[field.name] = value;
+
+      switch (field.repetitionType) {
+        case 'REPEATED':
+          if (!(field.name in record)) {
+            // eslint-disable max-depth
+            record[field.name] = [];
+          }
+          const ix = rLevels[rIndex];
+          while (record[field.name].length <= ix) {
+            // eslint-disable max-depth
+            record[field.name].push(null);
+          }
+          record[field.name][ix] = value;
+          break;
+
+        default:
+          record[field.name] = value;
       }
     }
   }
+
+  // Remove one level of nesting
+  for (let i = 0; i < rowCount; ++i) {
+    if (columnName in (column[i] as object)) {
+      column[i] = (column[i] as object)[columnName];
+    }
+  }
 }

package/src/parquetjs/schema/types.ts

@@ -1,6 +1,6 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 /* eslint-disable camelcase */
-import BSON from 'bson';
+import {BSONLoader, BSONWriter} from '@loaders.gl/bson';
 import {OriginalType, ParquetField, ParquetType, PrimitiveType} from './declare';
 
 export interface ParquetTypeKit {
@@ -168,7 +168,7 @@ export const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit> = {
  * Convert a value from it's native representation to the internal/underlying
  * primitive type
  */
-export function toPrimitive(type: ParquetType, value: any, field?: ParquetField) {
+export function toPrimitive(type: ParquetType, value: unknown, field?: ParquetField): unknown {
   if (!(type in PARQUET_LOGICAL_TYPES)) {
     throw new Error(`invalid type: ${type}`);
   }
@@ -180,7 +180,7 @@ export function toPrimitive(type: ParquetType, value: any, field?: ParquetField)
  * Convert a value from it's internal/underlying primitive representation to
  * the native representation
  */
-export function fromPrimitive(type: ParquetType, value: any, field?: ParquetField) {
+export function fromPrimitive(type: ParquetType, value: unknown, field?: ParquetField) {
   if (!(type in PARQUET_LOGICAL_TYPES)) {
     throw new Error(`invalid type: ${type}`);
   }
@@ -192,29 +192,27 @@ export function fromPrimitive(type: ParquetType, value: any, field?: ParquetFiel
   return value;
 }
 
-function toPrimitive_BOOLEAN(value: any) {
+function toPrimitive_BOOLEAN(value: unknown): boolean {
   return Boolean(value);
 }
 
-function fromPrimitive_BOOLEAN(value: any) {
+function fromPrimitive_BOOLEAN(value: any): boolean {
   return Boolean(value);
 }
 
-function toPrimitive_FLOAT(value: any) {
+function toPrimitive_FLOAT(value: any): number {
   const v = parseFloat(value);
   if (isNaN(v)) {
     throw new Error(`invalid value for FLOAT: ${value}`);
   }
-
   return v;
 }
 
-function toPrimitive_DOUBLE(value: any) {
+function toPrimitive_DOUBLE(value: any): number {
   const v = parseFloat(value);
   if (isNaN(v)) {
     throw new Error(`invalid value for DOUBLE: ${value}`);
   }
-
   return v;
 }
 
@@ -263,31 +261,28 @@ function toPrimitive_INT32(value: any) {
   return v;
 }
 
-function decimalToPrimitive_INT32(value: number, field: ParquetField) {
+function decimalToPrimitive_INT32(value: number, field: ParquetField): number {
   const primitiveValue = value * 10 ** (field.scale || 0);
   const v = Math.round(((primitiveValue * 10 ** -field.presision!) % 1) * 10 ** field.presision!);
   if (v < -0x80000000 || v > 0x7fffffff || isNaN(v)) {
     throw new Error(`invalid value for INT32: ${value}`);
   }
-
   return v;
 }
 
-function toPrimitive_UINT32(value: any) {
+function toPrimitive_UINT32(value: any): number {
   const v = parseInt(value, 10);
   if (v < 0 || v > 0xffffffffffff || isNaN(v)) {
     throw new Error(`invalid value for UINT32: ${value}`);
   }
-
   return v;
 }
 
-function toPrimitive_INT64(value: any) {
+function toPrimitive_INT64(value: any): number {
   const v = parseInt(value, 10);
   if (isNaN(v)) {
     throw new Error(`invalid value for INT64: ${value}`);
   }
-
   return v;
 }
 
@@ -319,37 +314,38 @@ function toPrimitive_INT96(value: any) {
   return v;
 }
 
-function toPrimitive_BYTE_ARRAY(value: any) {
+function toPrimitive_BYTE_ARRAY(value: any): Buffer {
   return Buffer.from(value);
 }
 
-function decimalToPrimitive_BYTE_ARRAY(value: any) {
+function decimalToPrimitive_BYTE_ARRAY(value: any): Buffer {
   // TBD
   return Buffer.from(value);
 }
 
-function toPrimitive_UTF8(value: any) {
+function toPrimitive_UTF8(value: any): Buffer {
   return Buffer.from(value, 'utf8');
 }
 
-function fromPrimitive_UTF8(value: any) {
+function fromPrimitive_UTF8(value: any): string {
   return value.toString();
 }
 
-function toPrimitive_JSON(value: any) {
+function toPrimitive_JSON(value: any): Buffer {
   return Buffer.from(JSON.stringify(value));
 }
 
-function fromPrimitive_JSON(value: any) {
+function fromPrimitive_JSON(value: any): unknown {
   return JSON.parse(value);
 }
 
-function toPrimitive_BSON(value: any) {
-  return Buffer.from(BSON.serialize(value));
+function toPrimitive_BSON(value: any): Buffer {
+  const arrayBuffer = BSONWriter.encodeSync?.(value) as ArrayBuffer;
+  return Buffer.from(arrayBuffer);
 }
 
 function fromPrimitive_BSON(value: any) {
-  return BSON.deserialize(value);
+  return BSONLoader.parseSync?.(value);
 }
 
 function toPrimitive_TIME_MILLIS(value: any) {
@@ -361,18 +357,17 @@ function toPrimitive_TIME_MILLIS(value: any) {
   return v;
 }
 
-function toPrimitive_TIME_MICROS(value: any) {
+function toPrimitive_TIME_MICROS(value: any): number {
   const v = parseInt(value, 10);
   if (v < 0 || isNaN(v)) {
     throw new Error(`invalid value for TIME_MICROS: ${value}`);
   }
-
   return v;
 }
 
 const kMillisPerDay = 86400000;
 
-function toPrimitive_DATE(value: any) {
+function toPrimitive_DATE(value: any): number {
   /* convert from date */
   if (value instanceof Date) {
     return value.getTime() / kMillisPerDay;
@@ -389,11 +384,11 @@ function toPrimitive_DATE(value: any) {
   }
 }
 
-function fromPrimitive_DATE(value: any) {
+function fromPrimitive_DATE(value: any): Date {
   return new Date(value * kMillisPerDay);
 }
 
-function toPrimitive_TIMESTAMP_MILLIS(value: any) {
+function toPrimitive_TIMESTAMP_MILLIS(value: any): number {
   /* convert from date */
   if (value instanceof Date) {
     return value.getTime();
@@ -410,7 +405,7 @@ function toPrimitive_TIMESTAMP_MILLIS(value: any) {
   }
 }
 
-function fromPrimitive_TIMESTAMP_MILLIS(value: any) {
+function fromPrimitive_TIMESTAMP_MILLIS(value: any): Date {
   return new Date(value);
 }
 

package/src/parquetjs/utils/file-utils.ts

@@ -1,6 +1,5 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-import fs from 'fs';
-import {Writable} from 'stream';
+import {fs, stream} from '@loaders.gl/loader-utils';
 
 export function load(name: string): any {
   return (module || (global as any)).require(name);
@@ -14,7 +13,7 @@ export interface WriteStreamOptions {
   start?: number;
 }
 
-export function oswrite(os: Writable, buf: Buffer): Promise<void> {
+export function oswrite(os: stream.Writable, buf: Buffer): Promise<void> {
   return new Promise((resolve, reject) => {
     os.write(buf, (err) => {
       if (err) {
@@ -26,7 +25,7 @@ export function oswrite(os: Writable, buf: Buffer): Promise<void> {
   });
 }
 
-export function osclose(os: Writable): Promise<void> {
+export function osclose(os: stream.Writable): Promise<void> {
   return new Promise((resolve, reject) => {
     (os as any).close((err: any) => {
       if (err) {

package/dist/bundle.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}

package/dist/constants.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/constants.ts"],"names":["PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING"],"mappings":"AAIA,OAAO,MAAMA,aAAa,GAAG,MAAtB;AACP,OAAO,MAAMC,uBAAuB,GAAG,MAAhC;AAKP,OAAO,MAAMC,eAAe,GAAG,CAAxB;AAKP,OAAO,MAAMC,kBAAkB,GAAG,OAA3B;AACP,OAAO,MAAMC,sBAAsB,GAAG,KAA/B","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/**\n * Parquet File Magic String\n */\nexport const PARQUET_MAGIC = 'PAR1';\nexport const PARQUET_MAGIC_ENCRYPTED = 'PARE';\n\n/**\n * Parquet File Format Version\n */\nexport const PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nexport const PARQUET_RDLVL_TYPE = 'INT32';\nexport const PARQUET_RDLVL_ENCODING = 'RLE';\n"],"file":"constants.js"}

package/dist/index.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/index.ts"],"names":["ParquetLoader","ParquetWorkerLoader","parseParquet","parseParquetFileInBatches","parse","parseFileInBatches","ParquetWriter","_ParquetWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEnvelopeReader","convertParquetToArrowSchema","_typecheckParquetLoader"],"mappings":"AAIA,SAAQA,aAAa,IAAIC,mBAAzB,QAAmD,kBAAnD;AACA,SAAQC,YAAR,EAAsBC,yBAAtB,QAAsD,qBAAtD;AAEA,SAAQF,mBAAR;AAGA,OAAO,MAAMD,aAAa,GAAG,EAC3B,GAAGC,mBADwB;AAE3BG,EAAAA,KAAK,EAAEF,YAFoB;AAG3BG,EAAAA,kBAAkB,EAAEF;AAHO,CAAtB;AAQP,SAAQG,aAAa,IAAIC,cAAzB,QAA8C,kBAA9C;AAIA,SAAQC,mBAAR,QAAkC,yBAAlC;AAEA,SAAQC,aAAR,QAA4B,2BAA5B;AACA,SAAQC,aAAR,QAA4B,mCAA5B;AACA,SAAQC,qBAAR,QAAoC,4CAApC;AAEA,SAAQC,2BAAR,QAA0C,sBAA1C;AAGA,OAAO,MAAMC,uBAAyC,GAAGb,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\n\nexport {ParquetWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}

package/dist/lib/convert-schema.d.ts

@@ -1,8 +0,0 @@
-import type { ParquetSchema } from '../parquetjs/schema/schema';
-import type { ParquetType } from '../parquetjs/schema/declare';
-import { Schema, DataType } from '@loaders.gl/schema';
-export declare const PARQUET_TYPE_MAPPING: {
-    [type in ParquetType]: typeof DataType;
-};
-export declare function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema;
-//# sourceMappingURL=convert-schema.d.ts.map

package/dist/lib/convert-schema.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-schema.d.ts","sourceRoot":"","sources":["../../src/lib/convert-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,4BAA4B,CAAC;AAC9D,OAAO,KAAK,EAAgC,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAE5F,OAAO,EACL,MAAM,EAGN,QAAQ,EAaT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,OAAO,QAAQ;CA+BzE,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,aAAa,EAAE,aAAa,GAAG,MAAM,CAKhF"}