@loaders.gl/parquet 3.4.6 → 4.0.0-alpha.10

package/dist/parquet-writer.d.ts

@@ -1,4 +1,5 @@
 import type { Writer } from '@loaders.gl/loader-utils';
+import { Table, TableBatch } from '@loaders.gl/schema';
 export type ParquetWriterOptions = {};
-export declare const ParquetWriter: Writer;
+export declare const ParquetWriter: Writer<Table, TableBatch, ParquetWriterOptions>;
 //# sourceMappingURL=parquet-writer.d.ts.map

package/dist/parquet-writer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAMrD,MAAM,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAItC,eAAO,MAAM,aAAa,EAAE,MAU3B,CAAC"}
+{"version":3,"file":"parquet-writer.d.ts","sourceRoot":"","sources":["../src/parquet-writer.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAC,KAAK,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAMrD,MAAM,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAItC,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,oBAAoB,CAUzE,CAAC"}

package/dist/parquet-writer.js

@@ -1,4 +1,5 @@
 "use strict";
+// loaders.gl, MIT license
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ParquetWriter = void 0;
 // __VERSION__ is injected by babel-plugin-version-inline

package/dist/parquetjs/encoder/parquet-encoder.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="node" />
 /// <reference types="node" />
 import { stream } from '@loaders.gl/loader-utils';
-import { ParquetBuffer } from '../schema/declare';
+import { ParquetRowGroup, ParquetRow } from '../schema/declare';
 import { ParquetSchema } from '../schema/schema';
 import { RowGroup } from '../parquet-thrift';
 export interface ParquetEncoderOptions {
@@ -34,7 +34,7 @@ export declare class ParquetEncoder<T> {
     static openStream<T>(schema: ParquetSchema, outputStream: stream.Writable, opts?: ParquetEncoderOptions): Promise<ParquetEncoder<T>>;
     schema: ParquetSchema;
     envelopeWriter: ParquetEnvelopeWriter;
-    rowBuffer: ParquetBuffer;
+    rowBuffer: ParquetRowGroup;
     rowGroupSize: number;
     closed: boolean;
     userMetadata: Record<string, string>;
@@ -47,7 +47,7 @@ export declare class ParquetEncoder<T> {
      * Append a single row to the parquet file. Rows are buffered in memory until
      * rowGroupSize rows are in the buffer or close() is called
      */
-    appendRow<T>(row: T): Promise<void>;
+    appendRow<T extends ParquetRow>(row: T): Promise<void>;
     /**
      * Finish writing the parquet file and commit the footer to disk. This method
      * MUST be called after you are finished adding rows. You must not call this
@@ -101,7 +101,7 @@ export declare class ParquetEnvelopeWriter {
      * Encode a parquet row group. The records object should be created using the
      * shredRecord method
      */
-    writeRowGroup(records: ParquetBuffer): Promise<void>;
+    writeRowGroup(records: ParquetRowGroup): Promise<void>;
     /**
      * Write the parquet file footer
      */

package/dist/parquetjs/encoder/parquet-encoder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-encoder.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/parquet-encoder.ts"],"names":[],"mappings":";;AAEA,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAGhD,OAAO,EACL,aAAa,EAKd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAaL,QAAQ,EAGT,MAAM,mBAAmB,CAAC;AA2B3B,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AAEH,qBAAa,cAAc,CAAC,CAAC;IAC3B;;;OAGG;WACU,QAAQ,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,qBAAqB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAK7B;;;OAGG;WACU,UAAU,CAAC,CAAC,EACvB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,GAAE,qBAA0B,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAKtB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,EAAE,qBAAqB,CAAC;IACtC,SAAS,EAAE,aAAa,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5C;;OAEG;gBAED,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,qBAAqB,EACrC,IAAI,EAAE,qBAAqB;IAcvB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC;;;OAGG;IACG,SAAS,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzC;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7C;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIlC;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC;;OAEG;WACU,UAAU,CACrB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,qBAAqB,CAAC;IAM1B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EACvC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC5B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,qBAAqB;IAY7B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1D;;OAEG;IACH,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B"}
+{"version":3,"file":"parquet-encoder.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/parquet-encoder.ts"],"names":[],"mappings":";;AAEA,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAGhD,OAAO,EACL,eAAe,EAKf,UAAU,EACX,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAaL,QAAQ,EAGT,MAAM,mBAAmB,CAAC;AA2B3B,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AAEH,qBAAa,cAAc,CAAC,CAAC;IAC3B;;;OAGG;WACU,QAAQ,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,qBAAqB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAK7B;;;OAGG;WACU,UAAU,CAAC,CAAC,EACvB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,GAAE,qBAA0B,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAKtB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,EAAE,qBAAqB,CAAC;IACtC,SAAS,EAAE,eAAe,CAAC;IAC3B,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5C;;OAEG;gBAED,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,qBAAqB,EACrC,IAAI,EAAE,qBAAqB;IAcvB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC;;;OAGG;IACG,SAAS,CAAC,CAAC,SAAS,UAAU,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAW5D;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7C;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIlC;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC;;OAEG;WACU,UAAU,CACrB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,qBAAqB,CAAC;IAM1B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EACvC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC5B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,qBAAqB;IAY7B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAY5D;;OAEG;IACH,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B"}

package/dist/parquetjs/parser/decoders.d.ts

@@ -1,5 +1,5 @@
 /// <reference types="node" />
-import { ParquetData, ParquetOptions, ParquetPageData, SchemaDefinition } from '../schema/declare';
+import { ParquetColumnChunk, ParquetOptions, ParquetPageData, SchemaDefinition } from '../schema/declare';
 import { CursorBuffer } from '../codecs';
 import { SchemaElement } from '../parquet-thrift';
 /**
@@ -9,7 +9,7 @@ import { SchemaElement } from '../parquet-thrift';
  * @param compression - compression type
  * @returns parquet data page data
  */
-export declare function decodeDataPages(buffer: Buffer, options: ParquetOptions): Promise<ParquetData>;
+export declare function decodeDataPages(buffer: Buffer, options: ParquetOptions): Promise<ParquetColumnChunk>;
 /**
  * Decode parquet page based on page type
  * @param cursor

package/dist/parquetjs/parser/decoders.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":";AACA,OAAO,EAEL,WAAW,EACX,cAAc,EACd,eAAe,EAGf,gBAAgB,EACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,YAAY,EAAsC,MAAM,WAAW,CAAC;AAC5E,OAAO,EAML,aAAa,EAEd,MAAM,mBAAmB,CAAC;AAK3B;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,WAAW,CAAC,CAmDtB;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,eAAe,CAAC,CAyB1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,aAAa,EAAE,EAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CA4DA"}
+{"version":3,"file":"decoders.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/decoders.ts"],"names":[],"mappings":";AACA,OAAO,EAEL,kBAAkB,EAClB,cAAc,EACd,eAAe,EAGf,gBAAgB,EACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,YAAY,EAAsC,MAAM,WAAW,CAAC;AAC5E,OAAO,EAML,aAAa,EAEd,MAAM,mBAAmB,CAAC;AAK3B;;;;;;GAMG;AACH,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,kBAAkB,CAAC,CAmD7B;AAED;;;;GAIG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,YAAY,EACpB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,eAAe,CAAC,CAyB1B;AAED;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAC1B,cAAc,EAAE,aAAa,EAAE,EAC/B,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,GACV;IACD,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CA4DA"}

package/dist/parquetjs/parser/parquet-reader.d.ts

@@ -1,7 +1,7 @@
 import type { ReadableFile } from '@loaders.gl/loader-utils';
 import { ParquetSchema } from '../schema/schema';
 import { ColumnChunk, FileMetaData, RowGroup } from '../parquet-thrift';
-import { ParquetBuffer, ParquetData, ParquetOptions } from '../schema/declare';
+import { ParquetRowGroup, ParquetColumnChunk, ParquetOptions } from '../schema/declare';
 export type ParquetReaderProps = {
     defaultDictionarySize?: number;
 };
@@ -23,11 +23,11 @@ export declare class ParquetReader {
     constructor(file: ReadableFile, props?: ParquetReaderProps);
     close(): void;
     /** Yield one row at a time */
-    rowIterator(props?: ParquetIterationProps): AsyncGenerator<import("../schema/declare").ParquetRecord, void, unknown>;
+    rowIterator(props?: ParquetIterationProps): AsyncGenerator<import("../schema/declare").ParquetRow, void, unknown>;
     /** Yield one batch of rows at a time */
-    rowBatchIterator(props?: ParquetIterationProps): AsyncGenerator<import("../schema/declare").ParquetRecord[], void, unknown>;
+    rowBatchIterator(props?: ParquetIterationProps): AsyncGenerator<import("../schema/declare").ParquetRow[], void, unknown>;
     /** Iterate over the raw row groups */
-    rowGroupIterator(props?: ParquetIterationProps): AsyncGenerator<ParquetBuffer, void, unknown>;
+    rowGroupIterator(props?: ParquetIterationProps): AsyncGenerator<ParquetRowGroup, void, unknown>;
     getRowCount(): Promise<number>;
     getSchema(): Promise<ParquetSchema>;
     /**
@@ -41,11 +41,11 @@ export declare class ParquetReader {
     /** Metadata is stored in the footer */
     readFooter(): Promise<FileMetaData>;
     /** Data is stored in row groups (similar to Apache Arrow record batches) */
-    readRowGroup(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][]): Promise<ParquetBuffer>;
+    readRowGroup(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][]): Promise<ParquetRowGroup>;
     /**
      * Each row group contains column chunks for all the columns.
      */
-    readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData>;
+    readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetColumnChunk>;
     /**
      * Getting dictionary for allows to flatten values by indices.
      * @param dictionaryPageOffset

package/dist/parquetjs/parser/parquet-reader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAK/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,aAAa,EAEb,WAAW,EAEX,cAAc,EACf,MAAM,mBAAmB,CAAC;AAI3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,0DAA0D;AAC1D,MAAM,MAAM,qBAAqB,GAAG;IAClC,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;CACpC,CAAC;AAMF;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,KAAK,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IACpC,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAQ;gBAElC,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,kBAAkB;IAK1D,KAAK,IAAI,IAAI;IAOb,8BAA8B;IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAShD,wCAAwC;IACjC,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAOrD,sCAAsC;IAC/B,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAqB/C,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAK9B,SAAS,IAAI,OAAO,CAAC,aAAa,CAAC;IAQzC;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASpD,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC;IAU9C,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAajC,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,YAAY,CAAC;IAsBzC,4EAA4E;IACtE,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,GACrB,OAAO,CAAC,aAAa,CAAC;IAgBzB;;OAEG;IACG,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAoDzF;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,EAAE,CAAC;CAuBrB"}
+{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAK/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,eAAe,EAEf,kBAAkB,EAElB,cAAc,EACf,MAAM,mBAAmB,CAAC;AAI3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,0DAA0D;AAC1D,MAAM,MAAM,qBAAqB,GAAG;IAClC,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;CACpC,CAAC;AAMF;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,KAAK,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IACpC,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAQ;gBAElC,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,kBAAkB;IAK1D,KAAK,IAAI,IAAI;IAOb,8BAA8B;IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAShD,wCAAwC;IACjC,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAOrD,sCAAsC;IAC/B,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAqB/C,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAK9B,SAAS,IAAI,OAAO,CAAC,aAAa,CAAC;IAQzC;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASpD,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC;IAU9C,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAajC,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,YAAY,CAAC;IAsBzC,4EAA4E;IACtE,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,GACrB,OAAO,CAAC,eAAe,CAAC;IAgB3B;;OAEG;IACG,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAoDhG;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,EAAE,CAAC;CAuBrB"}

package/dist/parquetjs/parser/parquet-reader.js

@@ -41,7 +41,7 @@ class ParquetReader {
     async *rowBatchIterator(props) {
         const schema = await this.getSchema();
         for await (const rowGroup of this.rowGroupIterator(props)) {
-            yield (0, shred_1.materializeRecords)(schema, rowGroup);
+            yield (0, shred_1.materializeRows)(schema, rowGroup);
         }
     }
     /** Iterate over the raw row groups */

package/dist/parquetjs/schema/declare.d.ts

@@ -65,19 +65,20 @@ export interface ParquetPageData {
     /** The "raw" page header from the file */
     pageHeader: PageHeader;
 }
-export interface ParquetRecord {
+export interface ParquetRow {
     [key: string]: any;
 }
 /** @
  * Holds data for one row group (column chunks) */
-export declare class ParquetBuffer {
+export declare class ParquetRowGroup {
     /** Number of rows in this page */
     rowCount: number;
-    columnData: Record<string, ParquetData>;
-    constructor(rowCount?: number, columnData?: Record<string, ParquetData>);
+    /** Map of Column chunks */
+    columnData: Record<string, ParquetColumnChunk>;
+    constructor(rowCount?: number, columnData?: Record<string, ParquetColumnChunk>);
 }
 /** Holds the data for one column chunk */
-export interface ParquetData {
+export interface ParquetColumnChunk {
     dlevels: number[];
     rlevels: number[];
     values: any[];

package/dist/parquetjs/schema/declare.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,KAAK,GAAG,kBAAkB,CAAC;AAChE,MAAM,MAAM,kBAAkB,GAC1B,cAAc,GACd,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,KAAK,GACL,SAAS,GACT,MAAM,CAAC;AACX,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAEvD;;GAEG;AACH,MAAM,MAAM,aAAa,GAErB,SAAS,GACT,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,sBAAsB,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,YAAY,GAEpB,MAAM,GAMN,eAAe,GACf,eAAe,GACf,oBAAoB,GACpB,8BAA8B,GAC9B,MAAM,GACN,aAAa,GACb,aAAa,GACb,kBAAkB,GAClB,kBAAkB,GAClB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,CAAC;AAEf,MAAM,MAAM,iBAAiB,GAAG,MAAM,EAAE,CAAC;AAEzC,MAAM,WAAW,gBAAgB;IAC/B,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,EAAE,cAAc,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,mDAAmD;AACnD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,2BAA2B;IAC3B,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,0CAA0C;IAC1C,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED;kDACkD;AAClD,qBAAa,aAAa;IACxB,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IAEjB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBAC5B,QAAQ,GAAE,MAAU,EAAE,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAM;CAI/E;AAED,0CAA0C;AAC1C,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B"}
+{"version":3,"file":"declare.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/declare.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,MAAM,YAAY,CAAC;AAC/B,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,KAAK,GAAG,kBAAkB,CAAC;AAChE,MAAM,MAAM,kBAAkB,GAC1B,cAAc,GACd,MAAM,GACN,QAAQ,GACR,KAAK,GACL,QAAQ,GACR,KAAK,GACL,SAAS,GACT,MAAM,CAAC;AACX,MAAM,MAAM,cAAc,GAAG,UAAU,GAAG,UAAU,GAAG,UAAU,CAAC;AAClE,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAEvD;;GAEG;AACH,MAAM,MAAM,aAAa,GAErB,SAAS,GACT,OAAO,GACP,OAAO,GACP,OAAO,GACP,OAAO,GACP,QAAQ,GACR,YAAY,GACZ,sBAAsB,CAAC;AAE3B;;GAEG;AACH,MAAM,MAAM,YAAY,GAEpB,MAAM,GAMN,eAAe,GACf,eAAe,GACf,oBAAoB,GACpB,8BAA8B,GAC9B,MAAM,GACN,aAAa,GACb,aAAa,GACb,kBAAkB,GAClB,kBAAkB,GAClB,QAAQ,GACR,SAAS,GACT,SAAS,GACT,SAAS,GACT,OAAO,GACP,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,MAAM,GACN,UAAU,CAAC;AAEf,MAAM,MAAM,iBAAiB,GAAG,MAAM,EAAE,CAAC;AAEzC,MAAM,WAAW,gBAAgB;IAC/B,CAAC,MAAM,EAAE,MAAM,GAAG,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,gBAAgB,CAAC;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,cAAc,EAAE,cAAc,CAAC;IAC/B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,WAAW,CAAC,EAAE,kBAAkB,CAAC;IACjC,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;CACvC;AAED,mDAAmD;AACnD,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,WAAW,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,kBAAkB,CAAC;IAChC,MAAM,EAAE,YAAY,CAAC;IACrB,SAAS,CAAC,EAAE,KAAK,CAAC;IAClB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,2BAA2B;IAC3B,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,iBAAiB,CAAC;IAC/B,0CAA0C;IAC1C,UAAU,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,UAAU;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAC;CACpB;AAED;kDACkD;AAClD,qBAAa,eAAe;IAC1B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,2BAA2B;IAC3B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;gBAEnC,QAAQ,GAAE,MAAU,EAAE,UAAU,GAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAM;CAItF;AAED,0CAA0C;AAC1C,MAAM,WAAW,kBAAkB;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,GAAG,EAAE,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,UAAU,EAAE,CAAC;CAC3B"}

package/dist/parquetjs/schema/declare.js

@@ -1,12 +1,12 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetBuffer = void 0;
+exports.ParquetRowGroup = void 0;
 /** @
  * Holds data for one row group (column chunks) */
-class ParquetBuffer {
+class ParquetRowGroup {
     constructor(rowCount = 0, columnData = {}) {
         this.rowCount = rowCount;
         this.columnData = columnData;
     }
 }
-exports.ParquetBuffer = ParquetBuffer;
+exports.ParquetRowGroup = ParquetRowGroup;

package/dist/parquetjs/schema/schema.d.ts

@@ -1,4 +1,4 @@
-import { FieldDefinition, ParquetBuffer, ParquetCompression, ParquetField, ParquetRecord, SchemaDefinition } from './declare';
+import { FieldDefinition, ParquetRowGroup, ParquetCompression, ParquetField, ParquetRow, SchemaDefinition } from './declare';
 /**
  * A parquet file schema
  */
@@ -18,9 +18,9 @@ export declare class ParquetSchema {
      * Retrieve a field definition and all the field's ancestors
      */
     findFieldBranch(path: string | string[]): ParquetField[];
-    shredRecord(record: ParquetRecord, buffer: ParquetBuffer): void;
-    materializeRecords(buffer: ParquetBuffer): ParquetRecord[];
+    shredRecord(row: ParquetRow, rowGroup: ParquetRowGroup): void;
+    materializeRows(rowGroup: ParquetRowGroup): ParquetRow[];
     compress(type: ParquetCompression): this;
-    buffer(): ParquetBuffer;
+    rowGroup(): ParquetRowGroup;
 }
 //# sourceMappingURL=schema.d.ts.map

package/dist/parquetjs/schema/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/schema.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,YAAY,EACZ,aAAa,EAEb,gBAAgB,EACjB,MAAM,WAAW,CAAC;AAInB;;GAEG;AACH,qBAAa,aAAa;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACrC,SAAS,EAAE,YAAY,EAAE,CAAC;IAEjC;;OAEG;gBACS,MAAM,EAAE,gBAAgB;IAMpC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,YAAY;IAiBhD;;OAEG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,YAAY,EAAE;IAgBxD,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI;IAI/D,kBAAkB,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,EAAE;IAI1D,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI;IAMxC,MAAM,IAAI,aAAa;CAGxB"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/schema.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,eAAe,EACf,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,UAAU,EAEV,gBAAgB,EACjB,MAAM,WAAW,CAAC;AAInB;;GAEG;AACH,qBAAa,aAAa;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACxC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACrC,SAAS,EAAE,YAAY,EAAE,CAAC;IAEjC;;OAEG;gBACS,MAAM,EAAE,gBAAgB;IAMpC;;OAEG;IACH,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,YAAY;IAiBhD;;OAEG;IACH,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,YAAY,EAAE;IAgBxD,WAAW,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,GAAG,IAAI;IAI7D,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,UAAU,EAAE;IAIxD,QAAQ,CAAC,IAAI,EAAE,kBAAkB,GAAG,IAAI;IAMxC,QAAQ,IAAI,eAAe;CAG5B"}

package/dist/parquetjs/schema/schema.js

@@ -54,18 +54,18 @@ class ParquetSchema {
         }
         return branch;
     }
-    shredRecord(record, buffer) {
-        (0, shred_1.shredRecord)(this, record, buffer);
+    shredRecord(row, rowGroup) {
+        (0, shred_1.shredRecord)(this, row, rowGroup);
     }
-    materializeRecords(buffer) {
-        return (0, shred_1.materializeRecords)(this, buffer);
+    materializeRows(rowGroup) {
+        return (0, shred_1.materializeRows)(this, rowGroup);
     }
     compress(type) {
         setCompress(this.schema, type);
         setCompress(this.fields, type);
         return this;
     }
-    buffer() {
+    rowGroup() {
         return (0, shred_1.shredBuffer)(this);
     }
 }

package/dist/parquetjs/schema/shred.d.ts

@@ -1,19 +1,20 @@
-import { ParquetBuffer, ParquetRecord } from './declare';
+import { ArrayType } from '@loaders.gl/schema';
+import { ParquetRowGroup, ParquetRow } from './declare';
 import { ParquetSchema } from './schema';
-export { ParquetBuffer };
-export declare function shredBuffer(schema: ParquetSchema): ParquetBuffer;
+export { ParquetRowGroup };
+export declare function shredBuffer(schema: ParquetSchema): ParquetRowGroup;
 /**
  * 'Shred' a record into a list of <value, repetition_level, definition_level>
  * tuples per column using the Google Dremel Algorithm..
  *
- * The buffer argument must point to an object into which the shredded record
- * will be returned. You may re-use the buffer for repeated calls to this function
- * to append to an existing buffer, as long as the schema is unchanged.
+ * The rowGroup argument must point to an object into which the shredded record
+ * will be returned. You may re-use the rowGroup for repeated calls to this function
+ * to append to an existing rowGroup, as long as the schema is unchanged.
  *
- * The format in which the shredded records will be stored in the buffer is as
+ * The format in which the shredded records will be stored in the rowGroup is as
  * follows:
  *
- *   buffer = {
+ *   rowGroup = {
  *     columnData: [
  *       'my_col': {
  *          dlevels: [d1, d2, .. dN],
@@ -24,16 +25,16 @@ export declare function shredBuffer(schema: ParquetSchema): ParquetBuffer;
  *      rowCount: X,
  *   }
  */
-export declare function shredRecord(schema: ParquetSchema, record: any, buffer: ParquetBuffer): void;
+export declare function shredRecord(schema: ParquetSchema, record: ParquetRow, rowGroup: ParquetRowGroup): void;
 /**
  * 'Materialize' a list of <value, repetition_level, definition_level>
  * tuples back to nested records (objects/arrays) using the Google Dremel
  * Algorithm..
  *
- * The buffer argument must point to an object with the following structure (i.e.
+ * The rowGroup argument must point to an object with the following structure (i.e.
  * the same structure that is returned by shredRecords):
  *
- *   buffer = {
+ *   rowGroup = {
  *     columnData: [
  *       'my_col': {
  *          dlevels: [d1, d2, .. dN],
@@ -44,16 +45,16 @@ export declare function shredRecord(schema: ParquetSchema, record: any, buffer:
  *      rowCount: X,
  *   }
  */
-export declare function materializeRecords(schema: ParquetSchema, buffer: ParquetBuffer): ParquetRecord[];
+export declare function materializeRows(schema: ParquetSchema, rowGroup: ParquetRowGroup): ParquetRow[];
 /**
  * 'Materialize' a list of <value, repetition_level, definition_level>
  * tuples back to nested records (objects/arrays) using the Google Dremel
  * Algorithm..
  *
- * The buffer argument must point to an object with the following structure (i.e.
+ * The rowGroup argument must point to an object with the following structure (i.e.
  * the same structure that is returned by shredRecords):
  *
- *   buffer = {
+ *   rowGroup = {
  *     columnData: [
  *       'my_col': {
  *          dlevels: [d1, d2, .. dN],
@@ -63,101 +64,6 @@ export declare function materializeRecords(schema: ParquetSchema, buffer: Parque
  *      ],
  *      rowCount: X,
  *   }
- *
-export function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {
-  const columns: ParquetRecord = {};
-  for (const key in buffer.columnData) {
-    const columnData = buffer.columnData[key];
-    if (columnData.count) {
-      extractColumn(schema, columnData, key, columns);
-    }
-  }
-  return columns;
-}
-
-// eslint-disable-next-line max-statements, complexity
-function extractColumn(
-  schema: ParquetSchema,
-  columnData: ParquetData,
-  key: string,
-  columns: Record<string, unknown>
-) {
-  if (columnData.count <= 0) {
-    return;
-  }
-
-  const record = columns;
-
-  const field = schema.findField(key);
-  const branch = schema.findFieldBranch(key);
-
-  // tslint:disable-next-line:prefer-array-literal
-  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
-  let vIndex = 0;
-
-  let i = 0;
-  const dLevel = columnData.dlevels[i];
-  const rLevel = columnData.rlevels[i];
-  rLevels[rLevel]++;
-  rLevels.fill(0, rLevel + 1);
-
-  let rIndex = 0;
-  let record = records[rLevels[rIndex++] - 1];
-
-  // Internal nodes
-  for (const step of branch) {
-    if (step === field || dLevel < step.dLevelMax) {
-      break;
-    }
-
-    switch (step.repetitionType) {
-      case 'REPEATED':
-        if (!(step.name in record)) {
-          // eslint-disable max-depth
-          record[step.name] = [];
-        }
-        const ix = rLevels[rIndex++];
-        while (record[step.name].length <= ix) {
-          // eslint-disable max-depth
-          record[step.name].push({});
-        }
-        record = record[step.name][ix];
-        break;
-
-      default:
-        record[step.name] = record[step.name] || {};
-        record = record[step.name];
-    }
-  }
-
-  // Leaf node
-  if (dLevel === field.dLevelMax) {
-    const value = Types.fromPrimitive(
-      // @ts-ignore
-      field.originalType || field.primitiveType,
-      columnData.values[vIndex],
-      field
-    );
-    vIndex++;
-
-    switch (field.repetitionType) {
-      case 'REPEATED':
-        if (!(field.name in record)) {
-          // eslint-disable max-depth
-          record[field.name] = [];
-        }
-        const ix = rLevels[rIndex];
-        while (record[field.name].length <= ix) {
-          // eslint-disable max-depth
-          record[field.name].push(null);
-        }
-        record[field.name][ix] = value;
-        break;
-
-      default:
-        record[field.name] = value;
-    }
-  }
-}
-*/
+ */
+export declare function materializeColumns(schema: ParquetSchema, rowGroup: ParquetRowGroup): Record<string, ArrayType>;
 //# sourceMappingURL=shred.d.ts.map

package/dist/parquetjs/schema/shred.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,aAAa,EAA6B,aAAa,EAAC,MAAM,WAAW,CAAC;AAClF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,aAAa,EAAC,CAAC;AAEvB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,aAAa,CAYhE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI,CAmB3F;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,GAAG,aAAa,EAAE,CAYhG;AAmFD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAkHE"}
+{"version":3,"file":"shred.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/shred.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAC7C,OAAO,EAAC,eAAe,EAAoC,UAAU,EAAC,MAAM,WAAW,CAAC;AACxF,OAAO,EAAC,aAAa,EAAC,MAAM,UAAU,CAAC;AAGvC,OAAO,EAAC,eAAe,EAAC,CAAC;AAEzB,wBAAgB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,eAAe,CAYlE;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,UAAU,EAClB,QAAQ,EAAE,eAAe,GACxB,IAAI,CAmBN;AAgED;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,eAAe,GAAG,UAAU,EAAE,CAa9F;AAoFD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,eAAe,GACxB,MAAM,CAAC,MAAM,EAAE,SAAS,CAAC,CAS3B"}