@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.3

package/dist/parquetjs/compression.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AA0BpD,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAU/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAGlF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}
+{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AAuCpD,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAU/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAGlF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}

package/dist/parquetjs/compression.js

@@ -8,7 +8,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.inflate = exports.decompress = exports.deflate = exports.preloadCompressions = exports.PARQUET_COMPRESSION_METHODS = void 0;
 const compression_1 = require("@loaders.gl/compression");
-const buffer_utils_1 = require("./utils/buffer-utils");
+/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */
+function toBuffer(arrayBuffer) {
+    return Buffer.from(arrayBuffer);
+}
+function toArrayBuffer(buffer) {
+    // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+    if (Buffer.isBuffer(buffer)) {
+        const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
+        return typedArray.slice().buffer;
+    }
+    return buffer;
+}
 // TODO switch to worker compression to avoid bundling...
 // import brotli from 'brotli'; - brotli has problems with decompress in browsers
 // import brotliDecompress from 'brotli/decompress';
@@ -57,9 +68,9 @@ async function deflate(method, value) {
     if (!compression) {
         throw new Error(`parquet: invalid compression method: ${method}`);
     }
-    const inputArrayBuffer = (0, buffer_utils_1.toArrayBuffer)(value);
+    const inputArrayBuffer = toArrayBuffer(value);
     const compressedArrayBuffer = await compression.compress(inputArrayBuffer);
-    return (0, buffer_utils_1.toBuffer)(compressedArrayBuffer);
+    return toBuffer(compressedArrayBuffer);
 }
 exports.deflate = deflate;
 /**
@@ -70,9 +81,9 @@ async function decompress(method, value, size) {
     if (!compression) {
         throw new Error(`parquet: invalid compression method: ${method}`);
     }
-    const inputArrayBuffer = (0, buffer_utils_1.toArrayBuffer)(value);
+    const inputArrayBuffer = toArrayBuffer(value);
     const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);
-    return (0, buffer_utils_1.toBuffer)(compressedArrayBuffer);
+    return toBuffer(compressedArrayBuffer);
 }
 exports.decompress = decompress;
 /*

package/dist/parquetjs/encoder/{writer.d.ts → parquet-encoder.d.ts}

@@ -1,10 +1,10 @@
 /// <reference types="node" />
 /// <reference types="node" />
-import { Transform, Writable } from 'stream';
+import { stream } from '@loaders.gl/loader-utils';
 import { ParquetBuffer } from '../schema/declare';
 import { ParquetSchema } from '../schema/schema';
 import { RowGroup } from '../parquet-thrift';
-export interface ParquetWriterOptions {
+export interface ParquetEncoderOptions {
     baseOffset?: number;
     rowGroupSize?: number;
     pageSize?: number;
@@ -17,21 +17,21 @@ export interface ParquetWriterOptions {
     start?: number;
 }
 /**
- * Write a parquet file to an output stream. The ParquetWriter will perform
+ * Write a parquet file to an output stream. The ParquetEncoder will perform
  * buffering/batching for performance, so close() must be called after all rows
  * are written.
  */
-export declare class ParquetWriter<T> {
+export declare class ParquetEncoder<T> {
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified file
      */
-    static openFile<T>(schema: ParquetSchema, path: string, opts?: ParquetWriterOptions): Promise<ParquetWriter<T>>;
+    static openFile<T>(schema: ParquetSchema, path: string, opts?: ParquetEncoderOptions): Promise<ParquetEncoder<T>>;
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified stream
      */
-    static openStream<T>(schema: ParquetSchema, outputStream: Writable, opts?: ParquetWriterOptions): Promise<ParquetWriter<T>>;
+    static openStream<T>(schema: ParquetSchema, outputStream: stream.Writable, opts?: ParquetEncoderOptions): Promise<ParquetEncoder<T>>;
     schema: ParquetSchema;
     envelopeWriter: ParquetEnvelopeWriter;
     rowBuffer: ParquetBuffer;
@@ -41,7 +41,7 @@ export declare class ParquetWriter<T> {
     /**
      * Create a new buffered parquet writer for a given envelope writer
      */
-    constructor(schema: ParquetSchema, envelopeWriter: ParquetEnvelopeWriter, opts: ParquetWriterOptions);
+    constructor(schema: ParquetSchema, envelopeWriter: ParquetEnvelopeWriter, opts: ParquetEncoderOptions);
     writeHeader(): Promise<void>;
     /**
      * Append a single row to the parquet file. Rows are buffered in memory until
@@ -82,7 +82,7 @@ export declare class ParquetEnvelopeWriter {
     /**
      * Create a new parquet envelope writer that writes to the specified stream
      */
-    static openStream(schema: ParquetSchema, outputStream: Writable, opts: ParquetWriterOptions): Promise<ParquetEnvelopeWriter>;
+    static openStream(schema: ParquetSchema, outputStream: stream.Writable, opts: ParquetEncoderOptions): Promise<ParquetEnvelopeWriter>;
     schema: ParquetSchema;
     write: (buf: Buffer) => Promise<void>;
     close: () => Promise<void>;
@@ -91,7 +91,7 @@ export declare class ParquetEnvelopeWriter {
     rowGroups: RowGroup[];
     pageSize: number;
     useDataPageV2: boolean;
-    constructor(schema: ParquetSchema, writeFn: (buf: Buffer) => Promise<void>, closeFn: () => Promise<void>, fileOffset: number, opts: ParquetWriterOptions);
+    constructor(schema: ParquetSchema, writeFn: (buf: Buffer) => Promise<void>, closeFn: () => Promise<void>, fileOffset: number, opts: ParquetEncoderOptions);
     writeSection(buf: Buffer): Promise<void>;
     /**
      * Encode the parquet file header
@@ -112,13 +112,4 @@ export declare class ParquetEnvelopeWriter {
      */
     setPageSize(cnt: number): void;
 }
-/**
- * Create a parquet transform stream
- */
-export declare class ParquetTransformer<T> extends Transform {
-    writer: ParquetWriter<T>;
-    constructor(schema: ParquetSchema, opts?: ParquetWriterOptions);
-    _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void>;
-    _flush(callback: (val?: any) => void): Promise<void>;
-}
-//# sourceMappingURL=writer.d.ts.map
+//# sourceMappingURL=parquet-encoder.d.ts.map

package/dist/parquetjs/encoder/parquet-encoder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-encoder.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/parquet-encoder.ts"],"names":[],"mappings":";;AAEA,OAAO,EAAC,MAAM,EAAC,MAAM,0BAA0B,CAAC;AAGhD,OAAO,EACL,aAAa,EAKd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAaL,QAAQ,EAGT,MAAM,mBAAmB,CAAC;AA2B3B,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AAEH,qBAAa,cAAc,CAAC,CAAC;IAC3B;;;OAGG;WACU,QAAQ,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,qBAAqB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAK7B;;;OAGG;WACU,UAAU,CAAC,CAAC,EACvB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,GAAE,qBAA0B,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;IAKtB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,EAAE,qBAAqB,CAAC;IACtC,SAAS,EAAE,aAAa,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5C;;OAEG;gBAED,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,qBAAqB,EACrC,IAAI,EAAE,qBAAqB;IAcvB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC;;;OAGG;IACG,SAAS,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzC;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7C;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIlC;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC;;OAEG;WACU,UAAU,CACrB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,MAAM,CAAC,QAAQ,EAC7B,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,qBAAqB,CAAC;IAM1B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EACvC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC5B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,qBAAqB;IAY7B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1D;;OAEG;IACH,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B"}

package/dist/parquetjs/encoder/{writer.js → parquet-encoder.js}

@@ -26,10 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetTransformer = exports.ParquetEnvelopeWriter = exports.ParquetWriter = void 0;
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-/* eslint-disable camelcase */
-const stream_1 = require("stream");
+exports.ParquetEnvelopeWriter = exports.ParquetEncoder = void 0;
 const codecs_1 = require("../codecs");
 const Compression = __importStar(require("../compression"));
 const Shred = __importStar(require("../schema/shred"));
@@ -56,31 +53,27 @@ const PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;
 const PARQUET_RDLVL_TYPE = 'INT32';
 const PARQUET_RDLVL_ENCODING = 'RLE';
 /**
- * Write a parquet file to an output stream. The ParquetWriter will perform
+ * Write a parquet file to an output stream. The ParquetEncoder will perform
  * buffering/batching for performance, so close() must be called after all rows
  * are written.
  */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-class ParquetWriter {
+class ParquetEncoder {
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified file
      */
     static async openFile(schema, path, opts) {
         const outputStream = await (0, file_utils_1.osopen)(path, opts);
-        return ParquetWriter.openStream(schema, outputStream, opts);
+        return ParquetEncoder.openStream(schema, outputStream, opts);
     }
     /**
      * Convenience method to create a new buffered parquet writer that writes to
      * the specified stream
      */
-    static async openStream(schema, outputStream, opts) {
-        if (!opts) {
-            // tslint:disable-next-line:no-parameter-reassignment
-            opts = {};
-        }
+    static async openStream(schema, outputStream, opts = {}) {
         const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
-        return new ParquetWriter(schema, envelopeWriter, opts);
+        return new ParquetEncoder(schema, envelopeWriter, opts);
     }
     /**
      * Create a new buffered parquet writer for a given envelope writer
@@ -166,7 +159,7 @@ class ParquetWriter {
         this.envelopeWriter.setPageSize(cnt);
     }
 }
-exports.ParquetWriter = ParquetWriter;
+exports.ParquetEncoder = ParquetEncoder;
 /**
  * Create a parquet file from a schema and a number of row groups. This class
  * performs direct, unbuffered writes to the underlying output stream and is
@@ -237,31 +230,40 @@ class ParquetEnvelopeWriter {
 exports.ParquetEnvelopeWriter = ParquetEnvelopeWriter;
 /**
  * Create a parquet transform stream
- */
-class ParquetTransformer extends stream_1.Transform {
-    constructor(schema, opts = {}) {
-        super({ objectMode: true });
-        const writeProxy = (function (t) {
-            return async function (b) {
-                t.push(b);
-            };
-        })(this);
-        this.writer = new ParquetWriter(schema, new ParquetEnvelopeWriter(schema, writeProxy, async () => { }, 0, opts), opts);
-    }
-    // tslint:disable-next-line:function-name
-    _transform(row, encoding, callback) {
-        if (row) {
-            return this.writer.appendRow(row).then(callback);
-        }
-        callback();
-        return Promise.resolve();
-    }
-    // tslint:disable-next-line:function-name
-    async _flush(callback) {
-        await this.writer.close(callback);
+export class ParquetTransformer<T> extends stream.Transform {
+  public writer: ParquetEncoder<T>;
+
+  constructor(schema: ParquetSchema, opts: ParquetEncoderOptions = {}) {
+    super({objectMode: true});
+
+    const writeProxy = (function (t: ParquetTransformer<any>) {
+      return async function (b: any): Promise<void> {
+        t.push(b);
+      };
+    })(this);
+
+    this.writer = new ParquetEncoder(
+      schema,
+      new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts),
+      opts
+    );
+  }
+
+  // tslint:disable-next-line:function-name
+  _transform(row: any, encoding: string, callback: (val?: any) => void): Promise<void> {
+    if (row) {
+      return this.writer.appendRow(row).then(callback);
     }
+    callback();
+    return Promise.resolve();
+  }
+
+  // tslint:disable-next-line:function-name
+  async _flush(callback: (val?: any) => void) {
+    await this.writer.close(callback);
+  }
 }
-exports.ParquetTransformer = ParquetTransformer;
+ */
 /**
  * Encode a consecutive array of data using one of the parquet encodings
  */

package/dist/parquetjs/parser/parquet-reader.d.ts

@@ -1,68 +1,58 @@
-/// <reference types="node" />
-import { ParquetEnvelopeReader } from './parquet-envelope-reader';
-import { FileMetaData } from '../parquet-thrift';
+import type { ReadableFile } from '@loaders.gl/loader-utils';
 import { ParquetSchema } from '../schema/schema';
-import { ParquetCursor } from './parquet-cursor';
+import { ColumnChunk, FileMetaData, RowGroup } from '../parquet-thrift';
+import { ParquetBuffer, ParquetData, ParquetOptions } from '../schema/declare';
+export type ParquetReaderProps = {
+    defaultDictionarySize?: number;
+};
+/** Properties for initializing a ParquetRowGroupReader */
+export type ParquetIterationProps = {
+    /** Filter allowing some columns to be dropped */
+    columnList?: string[] | string[][];
+};
 /**
- * A parquet reader allows retrieving the rows from a parquet file in order.
- * The basic usage is to create a reader and then retrieve a cursor/iterator
- * which allows you to consume row after row until all rows have been read. It is
- * important that you call close() after you are finished reading the file to
- * avoid leaking file descriptors.
+ * The parquet envelope reader allows direct, unbuffered access to the individual
+ * sections of the parquet file, namely the header, footer and the row groups.
+ * This class is intended for advanced/internal users; if you just want to retrieve
+ * rows from a parquet file use the ParquetReader instead
  */
-export declare class ParquetReader<T> implements AsyncIterable<T> {
+export declare class ParquetReader {
+    props: Required<ParquetReaderProps>;
+    file: ReadableFile;
+    metadata: Promise<FileMetaData> | null;
+    constructor(file: ReadableFile, props?: ParquetReaderProps);
+    close(): void;
+    /** Yield one row at a time */
+    rowIterator(props?: ParquetIterationProps): AsyncGenerator<import("../schema/declare").ParquetRecord, void, unknown>;
+    /** Yield one batch of rows at a time */
+    rowBatchIterator(props?: ParquetIterationProps): AsyncGenerator<import("../schema/declare").ParquetRecord[], void, unknown>;
+    /** Iterate over the raw row groups */
+    rowGroupIterator(props?: ParquetIterationProps): AsyncGenerator<ParquetBuffer, void, unknown>;
+    getRowCount(): Promise<number>;
+    getSchema(): Promise<ParquetSchema>;
     /**
-     * return a new parquet reader initialized with a read function
-     */
-    static openBlob<T>(blob: Blob): Promise<ParquetReader<T>>;
-    /**
-     * return a new parquet reader initialized with a read function
-     */
-    static openArrayBuffer<T>(arrayBuffer: ArrayBuffer): Promise<ParquetReader<T>>;
-    static openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>>;
-    metadata: FileMetaData;
-    envelopeReader: ParquetEnvelopeReader;
-    schema: ParquetSchema;
-    /**
-     * Create a new parquet reader from the file metadata and an envelope reader.
-     * It is not recommended to call this constructor directly except for advanced
-     * and internal use cases. Consider using one of the open{File,Buffer} methods
-     * instead
-     */
-    constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader);
-    /**
-     * Close this parquet reader. You MUST call this method once you're finished
-     * reading rows
-     */
-    close(): Promise<void>;
-    /**
-     * Return a cursor to the file. You may open more than one cursor and use
-     * them concurrently. All cursors become invalid once close() is called on
-     * the reader object.
-     *
-     * The required_columns parameter controls which columns are actually read
-     * from disk. An empty array or no value implies all columns. A list of column
-     * names means that only those columns should be loaded from disk.
-     */
-    getCursor(): ParquetCursor<T>;
-    getCursor<K extends keyof T>(columnList: (K | K[])[]): ParquetCursor<Pick<T, K>>;
-    getCursor(columnList: (string | string[])[]): ParquetCursor<Partial<T>>;
-    /**
-     * Return the number of rows in this file. Note that the number of rows is
-     * not neccessarily equal to the number of rows in each column.
-     */
-    getRowCount(): number;
-    /**
-     * Returns the ParquetSchema for this file
+     * Returns the user (key/value) metadata for this file
+     * In parquet this is not stored on the schema like it is in arrow
      */
-    getSchema(): ParquetSchema;
+    getSchemaMetadata(): Promise<Record<string, string>>;
+    getFileMetadata(): Promise<FileMetaData>;
+    /** Metadata is stored in the footer */
+    readHeader(): Promise<void>;
+    /** Metadata is stored in the footer */
+    readFooter(): Promise<FileMetaData>;
+    /** Data is stored in row groups (similar to Apache Arrow record batches) */
+    readRowGroup(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][]): Promise<ParquetBuffer>;
     /**
-     * Returns the user (key/value) metadata for this file
+     * Each row group contains column chunks for all the columns.
      */
-    getMetadata(): Record<string, string>;
+    readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData>;
     /**
-     * Implement AsyncIterable
+     * Getting dictionary for allows to flatten values by indices.
+     * @param dictionaryPageOffset
+     * @param options
+     * @param pagesOffset
+     * @returns
      */
-    [Symbol.asyncIterator](): AsyncIterator<T>;
+    getDictionary(dictionaryPageOffset: number, options: ParquetOptions, pagesOffset: number): Promise<string[]>;
 }
 //# sourceMappingURL=parquet-reader.d.ts.map

package/dist/parquetjs/parser/parquet-reader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":";AACA,OAAO,EAAC,qBAAqB,EAAC,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAI/C;;;;;;GAMG;AACH,qBAAa,aAAa,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,CAAC;IACvD;;OAEG;WACU,QAAQ,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAkB/D;;OAEG;WACU,eAAe,CAAC,CAAC,EAAE,WAAW,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;WAevE,UAAU,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAY9D,QAAQ,EAAE,YAAY,CAAC;IACvB,cAAc,EAAE,qBAAqB,CAAC;IACtC,MAAM,EAAE,aAAa,CAAC;IAE7B;;;;;OAKG;gBACS,QAAQ,EAAE,YAAY,EAAE,cAAc,EAAE,qBAAqB;IAYzE;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAM5B;;;;;;;;OAQG;IACH,SAAS,IAAI,aAAa,CAAC,CAAC,CAAC;IAE7B,SAAS,CAAC,CAAC,SAAS,MAAM,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAChF,SAAS,CAAC,UAAU,EAAE,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC,EAAE,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAkBvE;;;OAGG;IACH,WAAW,IAAI,MAAM;IAIrB;;OAEG;IACH,SAAS,IAAI,aAAa;IAI1B;;OAEG;IACH,WAAW,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;IAQrC;;OAEG;IAEH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;CAG3C"}
+{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAK/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,aAAa,EAEb,WAAW,EAEX,cAAc,EACf,MAAM,mBAAmB,CAAC;AAI3B,MAAM,MAAM,kBAAkB,GAAG;IAC/B,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC,CAAC;AAEF,0DAA0D;AAC1D,MAAM,MAAM,qBAAqB,GAAG;IAClC,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;CACpC,CAAC;AAMF;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,KAAK,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IACpC,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAQ;gBAElC,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,kBAAkB;IAK1D,KAAK,IAAI,IAAI;IAOb,8BAA8B;IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAShD,wCAAwC;IACjC,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAOrD,sCAAsC;IAC/B,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAqB/C,WAAW,IAAI,OAAO,CAAC,MAAM,CAAC;IAK9B,SAAS,IAAI,OAAO,CAAC,aAAa,CAAC;IAQzC;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASpD,eAAe,IAAI,OAAO,CAAC,YAAY,CAAC;IAU9C,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAajC,uCAAuC;IACjC,UAAU,IAAI,OAAO,CAAC,YAAY,CAAC;IAsBzC,4EAA4E;IACtE,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,GACrB,OAAO,CAAC,aAAa,CAAC;IAgBzB;;OAEG;IACG,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAoDzF;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,EAAE,CAAC;CAuBrB"}