@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.2

package/dist/lib/read-array-buffer.js

@@ -1,29 +0,0 @@
-"use strict";
-// Random-Access read
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.readArrayBuffer = void 0;
-async function readArrayBuffer(file, start, length) {
-    if (file instanceof Blob) {
-        const slice = file.slice(start, start + length);
-        return await slice.arrayBuffer();
-    }
-    return await file.read(start, start + length);
-}
-exports.readArrayBuffer = readArrayBuffer;
-/**
- * Read a slice of a Blob or File, without loading the entire file into memory
- * The trick when reading File objects is to read successive "slices" of the File
- * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields
- * Actually reading from file happens in `readAsArrayBuffer`
- * @param blob to read
- export async function readBlob(blob: Blob): Promise<ArrayBuffer> {
-  return await new Promise((resolve, reject) => {
-    const fileReader = new FileReader();
-    fileReader.onload = (event: ProgressEvent<FileReader>) =>
-      resolve(event?.target?.result as ArrayBuffer);
-    // TODO - reject with a proper Error
-    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);
-    fileReader.readAsArrayBuffer(blob);
-  });
-}
-*/

package/dist/parquetjs/encoder/writer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/writer.ts"],"names":[],"mappings":";;AAEA,OAAO,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAG3C,OAAO,EACL,aAAa,EAKd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAaL,QAAQ,EAGT,MAAM,mBAAmB,CAAC;AA2B3B,MAAM,WAAW,oBAAoB;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AAEH,qBAAa,aAAa,CAAC,CAAC;IAC1B;;;OAGG;WACU,QAAQ,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,oBAAoB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAK5B;;;OAGG;WACU,UAAU,CAAC,CAAC,EACvB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,QAAQ,EACtB,IAAI,CAAC,EAAE,oBAAoB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAWrB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,EAAE,qBAAqB,CAAC;IACtC,SAAS,EAAE,aAAa,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5C;;OAEG;gBAED,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,qBAAqB,EACrC,IAAI,EAAE,oBAAoB;IActB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC;;;OAGG;IACG,SAAS,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzC;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7C;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIlC;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC;;OAEG;WACU,UAAU,CACrB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,QAAQ,EACtB,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,qBAAqB,CAAC;IAM1B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EACvC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC5B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,oBAAoB;IAY5B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1D;;OAEG;IACH,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;GAEG;AACH,qBAAa,kBAAkB,CAAC,CAAC,CAAE,SAAQ,SAAS;IAC3C,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;gBAEpB,MAAM,EAAE,aAAa,EAAE,IAAI,GAAE,oBAAyB;IAiBlE,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAS9E,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,IAAI;CAG3C"}

package/dist/parquetjs/file.d.ts

@@ -1,10 +0,0 @@
-/// <reference types="node" />
-import fs from 'fs';
-export declare function fopen(filePath: any): Promise<unknown>;
-export declare function fstat(filePath: any): Promise<fs.Stats>;
-export declare function fread(fd: any, position: any, length: any): Promise<unknown>;
-export declare function fclose(fd: any): Promise<unknown>;
-export declare function oswrite(os: any, buf: any): Promise<void>;
-export declare function osclose(os: any): Promise<void>;
-export declare function osopen(path: any, opts: any): Promise<unknown>;
-//# sourceMappingURL=file.d.ts.map

package/dist/parquetjs/file.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../src/parquetjs/file.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,wBAAgB,KAAK,CAAC,QAAQ,KAAA,oBAU7B;AAED,wBAAgB,KAAK,CAAC,QAAQ,KAAA,qBAU7B;AAED,wBAAgB,KAAK,CAAC,EAAE,KAAA,EAAE,QAAQ,KAAA,EAAE,MAAM,KAAA,oBAYzC;AAED,wBAAgB,MAAM,CAAC,EAAE,KAAA,oBAUxB;AAED,wBAAgB,OAAO,CAAC,EAAE,KAAA,EAAE,GAAG,KAAA,GAAG,OAAO,CAAC,IAAI,CAAC,CAU9C;AAED,wBAAgB,OAAO,CAAC,EAAE,KAAA,GAAG,OAAO,CAAC,IAAI,CAAC,CAUzC;AAED,wBAAgB,MAAM,CAAC,IAAI,KAAA,EAAE,IAAI,KAAA,oBAYhC"}

package/dist/parquetjs/file.js

@@ -1,99 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.osopen = exports.osclose = exports.oswrite = exports.fclose = exports.fread = exports.fstat = exports.fopen = void 0;
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-const fs_1 = __importDefault(require("fs"));
-function fopen(filePath) {
-    return new Promise((resolve, reject) => {
-        fs_1.default.open(filePath, 'r', (err, fd) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve(fd);
-            }
-        });
-    });
-}
-exports.fopen = fopen;
-function fstat(filePath) {
-    return new Promise((resolve, reject) => {
-        fs_1.default.stat(filePath, (err, stat) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve(stat);
-            }
-        });
-    });
-}
-exports.fstat = fstat;
-function fread(fd, position, length) {
-    const buffer = Buffer.alloc(length);
-    return new Promise((resolve, reject) => {
-        fs_1.default.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-            if (err || bytesRead !== length) {
-                reject(err || Error('read failed'));
-            }
-            else {
-                resolve(buf);
-            }
-        });
-    });
-}
-exports.fread = fread;
-function fclose(fd) {
-    return new Promise((resolve, reject) => {
-        fs_1.default.close(fd, (err) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve(err);
-            }
-        });
-    });
-}
-exports.fclose = fclose;
-function oswrite(os, buf) {
-    return new Promise((resolve, reject) => {
-        os.write(buf, (err) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve();
-            }
-        });
-    });
-}
-exports.oswrite = oswrite;
-function osclose(os) {
-    return new Promise((resolve, reject) => {
-        os.close((err) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve();
-            }
-        });
-    });
-}
-exports.osclose = osclose;
-function osopen(path, opts) {
-    return new Promise((resolve, reject) => {
-        const outputStream = fs_1.default.createWriteStream(path, opts);
-        outputStream.on('open', function (fd) {
-            resolve(outputStream);
-        });
-        outputStream.on('error', function (err) {
-            reject(err);
-        });
-    });
-}
-exports.osopen = osopen;

package/dist/parquetjs/parser/parquet-cursor.d.ts

@@ -1,36 +0,0 @@
-import { FileMetaData } from '../parquet-thrift';
-import { ParquetEnvelopeReader } from './parquet-envelope-reader';
-import { ParquetSchema } from '../schema/schema';
-import { ParquetRecord } from '../schema/declare';
-/**
- * A parquet cursor is used to retrieve rows from a parquet file in order
- */
-export declare class ParquetCursor<T> implements AsyncIterable<T> {
-    metadata: FileMetaData;
-    envelopeReader: ParquetEnvelopeReader;
-    schema: ParquetSchema;
-    columnList: string[][];
-    rowGroup: ParquetRecord[];
-    rowGroupIndex: number;
-    /**
-     * Create a new parquet reader from the file metadata and an envelope reader.
-     * It is usually not recommended to call this constructor directly except for
-     * advanced and internal use cases. Consider using getCursor() on the
-     * ParquetReader instead
-     */
-    constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader, schema: ParquetSchema, columnList: string[][]);
-    /**
-     * Retrieve the next row from the cursor. Returns a row or NULL if the end
-     * of the file was reached
-     */
-    next<T = any>(): Promise<T>;
-    /**
-     * Rewind the cursor the the beginning of the file
-     */
-    rewind(): void;
-    /**
-     * Implement AsyncIterable
-     */
-    [Symbol.asyncIterator](): AsyncIterator<T>;
-}
-//# sourceMappingURL=parquet-cursor.d.ts.map

package/dist/parquetjs/parser/parquet-cursor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parquet-cursor.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-cursor.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAC,qBAAqB,EAAC,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAC,aAAa,EAAC,MAAM,mBAAmB,CAAC;AAGhD;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,CAAC;IAChD,QAAQ,EAAE,YAAY,CAAC;IACvB,cAAc,EAAE,qBAAqB,CAAC;IACtC,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,EAAE,CAAC;IACvB,QAAQ,EAAE,aAAa,EAAE,CAAM;IAC/B,aAAa,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;gBAED,QAAQ,EAAE,YAAY,EACtB,cAAc,EAAE,qBAAqB,EACrC,MAAM,EAAE,aAAa,EACrB,UAAU,EAAE,MAAM,EAAE,EAAE;IASxB;;;OAGG;IACG,IAAI,CAAC,CAAC,GAAG,GAAG,KAAK,OAAO,CAAC,CAAC,CAAC;IAiBjC;;OAEG;IACH,MAAM,IAAI,IAAI;IAKd;;OAEG;IAEH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;CAuB3C"}

package/dist/parquetjs/parser/parquet-cursor.js

@@ -1,74 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetCursor = void 0;
-const shred_1 = require("../schema/shred");
-/**
- * A parquet cursor is used to retrieve rows from a parquet file in order
- */
-class ParquetCursor {
-    /**
-     * Create a new parquet reader from the file metadata and an envelope reader.
-     * It is usually not recommended to call this constructor directly except for
-     * advanced and internal use cases. Consider using getCursor() on the
-     * ParquetReader instead
-     */
-    constructor(metadata, envelopeReader, schema, columnList) {
-        this.rowGroup = [];
-        this.metadata = metadata;
-        this.envelopeReader = envelopeReader;
-        this.schema = schema;
-        this.columnList = columnList;
-        this.rowGroupIndex = 0;
-    }
-    /**
-     * Retrieve the next row from the cursor. Returns a row or NULL if the end
-     * of the file was reached
-     */
-    async next() {
-        if (this.rowGroup.length === 0) {
-            if (this.rowGroupIndex >= this.metadata.row_groups.length) {
-                // @ts-ignore
-                return null;
-            }
-            const rowBuffer = await this.envelopeReader.readRowGroup(this.schema, this.metadata.row_groups[this.rowGroupIndex], this.columnList);
-            this.rowGroup = (0, shred_1.materializeRecords)(this.schema, rowBuffer);
-            this.rowGroupIndex++;
-        }
-        return this.rowGroup.shift();
-    }
-    /**
-     * Rewind the cursor the the beginning of the file
-     */
-    rewind() {
-        this.rowGroup = [];
-        this.rowGroupIndex = 0;
-    }
-    /**
-     * Implement AsyncIterable
-     */
-    // tslint:disable-next-line:function-name
-    [Symbol.asyncIterator]() {
-        let done = false;
-        return {
-            next: async () => {
-                if (done) {
-                    return { done, value: null };
-                }
-                const value = await this.next();
-                if (value === null) {
-                    return { done: true, value };
-                }
-                return { done: false, value };
-            },
-            return: async () => {
-                done = true;
-                return { done, value: null };
-            },
-            throw: async () => {
-                done = true;
-                return { done: true, value: null };
-            }
-        };
-    }
-}
-exports.ParquetCursor = ParquetCursor;

package/dist/parquetjs/parser/parquet-envelope-reader.d.ts

@@ -1,40 +0,0 @@
-/// <reference types="node" />
-import { ParquetSchema } from '../schema/schema';
-import { ColumnChunk, FileMetaData, RowGroup } from '../parquet-thrift';
-import { ParquetBuffer, ParquetData, ParquetOptions } from '../schema/declare';
-/**
- * The parquet envelope reader allows direct, unbuffered access to the individual
- * sections of the parquet file, namely the header, footer and the row groups.
- * This class is intended for advanced/internal users; if you just want to retrieve
- * rows from a parquet file use the ParquetReader instead
- */
-export declare class ParquetEnvelopeReader {
-    read: (position: number, length: number) => Promise<Buffer>;
-    /**
-     * Close this parquet reader. You MUST call this method once you're finished
-     * reading rows
-     */
-    close: () => Promise<void>;
-    fileSize: number;
-    defaultDictionarySize: number;
-    static openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader>;
-    constructor(read: (position: number, length: number) => Promise<Buffer>, close: () => Promise<void>, fileSize: number, options?: any);
-    readHeader(): Promise<void>;
-    readRowGroup(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][]): Promise<ParquetBuffer>;
-    /**
-     * Do reading of parquet file's column chunk
-     * @param schema
-     * @param colChunk
-     */
-    readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData>;
-    /**
-     * Getting dictionary for allows to flatten values by indices.
-     * @param dictionaryPageOffset
-     * @param options
-     * @param pagesOffset
-     * @returns
-     */
-    getDictionary(dictionaryPageOffset: number, options: ParquetOptions, pagesOffset: number): Promise<string[]>;
-    readFooter(): Promise<FileMetaData>;
-}
-//# sourceMappingURL=parquet-envelope-reader.d.ts.map

package/dist/parquetjs/parser/parquet-envelope-reader.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parquet-envelope-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-envelope-reader.ts"],"names":[],"mappings":";AACA,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,aAAa,EAEb,WAAW,EAEX,cAAc,EACf,MAAM,mBAAmB,CAAC;AAM3B;;;;;GAKG;AACH,qBAAa,qBAAqB;IACzB,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACnE;;;OAGG;IACI,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,qBAAqB,EAAE,MAAM,CAAC;WAExB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;gBAQrE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,EAC3D,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,GAAG;IAQT,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAc3B,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,GACrB,OAAO,CAAC,aAAa,CAAC;IAgBzB;;;;OAIG;IACG,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAoDzF;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,EAAE,CAAC;IAwBd,UAAU,IAAI,OAAO,CAAC,YAAY,CAAC;CAqB1C"}

package/dist/parquetjs/parser/parquet-envelope-reader.js

@@ -1,136 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetEnvelopeReader = void 0;
-const constants_1 = require("../../constants");
-const parquet_thrift_1 = require("../parquet-thrift");
-const read_utils_1 = require("../utils/read-utils");
-const decoders_1 = require("./decoders");
-const DEFAULT_DICTIONARY_SIZE = 1e6;
-/**
- * The parquet envelope reader allows direct, unbuffered access to the individual
- * sections of the parquet file, namely the header, footer and the row groups.
- * This class is intended for advanced/internal users; if you just want to retrieve
- * rows from a parquet file use the ParquetReader instead
- */
-class ParquetEnvelopeReader {
-    static async openBuffer(buffer) {
-        const readFn = (position, length) => Promise.resolve(buffer.slice(position, position + length));
-        const closeFn = () => Promise.resolve();
-        return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
-    }
-    constructor(read, close, fileSize, options) {
-        this.read = read;
-        this.close = close;
-        this.fileSize = fileSize;
-        this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;
-    }
-    async readHeader() {
-        const buffer = await this.read(0, constants_1.PARQUET_MAGIC.length);
-        const magic = buffer.toString();
-        switch (magic) {
-            case constants_1.PARQUET_MAGIC:
-                break;
-            case constants_1.PARQUET_MAGIC_ENCRYPTED:
-                throw new Error('Encrypted parquet file not supported');
-            default:
-                throw new Error(`Invalid parquet file (magic=${magic})`);
-        }
-    }
-    async readRowGroup(schema, rowGroup, columnList) {
-        const buffer = {
-            rowCount: Number(rowGroup.num_rows),
-            columnData: {}
-        };
-        for (const colChunk of rowGroup.columns) {
-            const colMetadata = colChunk.meta_data;
-            const colKey = colMetadata?.path_in_schema;
-            if (columnList.length > 0 && (0, read_utils_1.fieldIndexOf)(columnList, colKey) < 0) {
-                continue; // eslint-disable-line no-continue
-            }
-            buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
-        }
-        return buffer;
-    }
-    /**
-     * Do reading of parquet file's column chunk
-     * @param schema
-     * @param colChunk
-     */
-    async readColumnChunk(schema, colChunk) {
-        if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
-            throw new Error('external references are not supported');
-        }
-        const field = schema.findField(colChunk.meta_data?.path_in_schema);
-        const type = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.Type, colChunk.meta_data?.type);
-        if (type !== field.primitiveType) {
-            throw new Error(`chunk type not matching schema: ${type}`);
-        }
-        const compression = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.CompressionCodec, colChunk.meta_data?.codec);
-        const pagesOffset = Number(colChunk.meta_data?.data_page_offset);
-        let pagesSize = Number(colChunk.meta_data?.total_compressed_size);
-        if (!colChunk.file_path) {
-            pagesSize = Math.min(this.fileSize - pagesOffset, Number(colChunk.meta_data?.total_compressed_size));
-        }
-        const options = {
-            type,
-            rLevelMax: field.rLevelMax,
-            dLevelMax: field.dLevelMax,
-            compression,
-            column: field,
-            numValues: colChunk.meta_data?.num_values,
-            dictionary: []
-        };
-        let dictionary;
-        const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;
-        if (dictionaryPageOffset) {
-            const dictionaryOffset = Number(dictionaryPageOffset);
-            // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.
-            dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
-        }
-        dictionary = options.dictionary?.length ? options.dictionary : dictionary;
-        const pagesBuf = await this.read(pagesOffset, pagesSize);
-        return await (0, decoders_1.decodeDataPages)(pagesBuf, { ...options, dictionary });
-    }
-    /**
-     * Getting dictionary for allows to flatten values by indices.
-     * @param dictionaryPageOffset
-     * @param options
-     * @param pagesOffset
-     * @returns
-     */
-    async getDictionary(dictionaryPageOffset, options, pagesOffset) {
-        if (dictionaryPageOffset === 0) {
-            // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);
-            // pagesBuf = await this.read(pagesOffset, dictionarySize);
-            // In this case we are working with parquet-mr files format. Problem is described below:
-            // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding
-            // We need to get dictionary page from column chunk if it exists.
-            // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.
-            return [];
-        }
-        const dictionarySize = Math.min(this.fileSize - dictionaryPageOffset, this.defaultDictionarySize);
-        const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);
-        const cursor = { buffer: pagesBuf, offset: 0, size: pagesBuf.length };
-        const decodedPage = await (0, decoders_1.decodePage)(cursor, options);
-        return decodedPage.dictionary;
-    }
-    async readFooter() {
-        const trailerLen = constants_1.PARQUET_MAGIC.length + 4;
-        const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);
-        const magic = trailerBuf.slice(4).toString();
-        if (magic !== constants_1.PARQUET_MAGIC) {
-            throw new Error(`Not a valid parquet file (magic="${magic})`);
-        }
-        const metadataSize = trailerBuf.readUInt32LE(0);
-        const metadataOffset = this.fileSize - metadataSize - trailerLen;
-        if (metadataOffset < constants_1.PARQUET_MAGIC.length) {
-            throw new Error(`Invalid metadata size ${metadataOffset}`);
-        }
-        const metadataBuf = await this.read(metadataOffset, metadataSize);
-        // let metadata = new parquet_thrift.FileMetaData();
-        // parquet_util.decodeThrift(metadata, metadataBuf);
-        const { metadata } = (0, read_utils_1.decodeFileMetadata)(metadataBuf);
-        return metadata;
-    }
-}
-exports.ParquetEnvelopeReader = ParquetEnvelopeReader;

package/dist/parquetjs/utils/buffer-utils.d.ts

@@ -1,10 +0,0 @@
-/// <reference types="node" />
-/**
- * Convert Buffer to ArrayBuffer
- */
-export declare function toArrayBuffer(buffer: Buffer): ArrayBuffer;
-/**
- * Convert (copy) ArrayBuffer to Buffer
- */
-export declare function toBuffer(arrayBuffer: ArrayBuffer): Buffer;
-//# sourceMappingURL=buffer-utils.d.ts.map

package/dist/parquetjs/utils/buffer-utils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"buffer-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/buffer-utils.ts"],"names":[],"mappings":";AAAA;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAOzD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,CAEzD"}

package/dist/parquetjs/utils/buffer-utils.js

@@ -1,22 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.toBuffer = exports.toArrayBuffer = void 0;
-/**
- * Convert Buffer to ArrayBuffer
- */
-function toArrayBuffer(buffer) {
-    // TODO - per docs we should just be able to call buffer.buffer, but there are issues
-    if (Buffer.isBuffer(buffer)) {
-        const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
-        return typedArray.slice().buffer;
-    }
-    return buffer;
-}
-exports.toArrayBuffer = toArrayBuffer;
-/**
- * Convert (copy) ArrayBuffer to Buffer
- */
-function toBuffer(arrayBuffer) {
-    return Buffer.from(arrayBuffer);
-}
-exports.toBuffer = toBuffer;

package/src/lib/parse-parquet.ts

@@ -1,27 +0,0 @@
-// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
-import type {ParquetLoaderOptions} from '../parquet-loader';
-
-import {ParquetReader} from '../parquetjs/parser/parquet-reader';
-
-export async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
-  const blob = new Blob([arrayBuffer]);
-  for await (const batch of parseParquetFileInBatches(blob, options)) {
-    return batch;
-  }
-  return null;
-}
-
-export async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {
-  const reader = await ParquetReader.openBlob(blob);
-  const rows: any[][] = [];
-  try {
-    const cursor = reader.getCursor();
-    let record: any[] | null;
-    while ((record = await cursor.next())) {
-      rows.push(record);
-    }
-  } finally {
-    await reader.close();
-  }
-  yield rows;
-}

package/src/lib/read-array-buffer.ts

@@ -1,31 +0,0 @@
-// Random-Access read
-
-export async function readArrayBuffer(
-  file: Blob | ArrayBuffer | any,
-  start: number,
-  length: number
-): Promise<ArrayBuffer> {
-  if (file instanceof Blob) {
-    const slice = file.slice(start, start + length);
-    return await slice.arrayBuffer();
-  }
-  return await file.read(start, start + length);
-}
-
-/**
- * Read a slice of a Blob or File, without loading the entire file into memory
- * The trick when reading File objects is to read successive "slices" of the File
- * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields
- * Actually reading from file happens in `readAsArrayBuffer`
- * @param blob to read
- export async function readBlob(blob: Blob): Promise<ArrayBuffer> {
-  return await new Promise((resolve, reject) => {
-    const fileReader = new FileReader();
-    fileReader.onload = (event: ProgressEvent<FileReader>) =>
-      resolve(event?.target?.result as ArrayBuffer);
-    // TODO - reject with a proper Error
-    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);
-    fileReader.readAsArrayBuffer(blob);
-  });
-}
-*/

package/src/parquetjs/file.ts

@@ -1,90 +0,0 @@
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-import fs from 'fs';
-
-export function fopen(filePath) {
-  return new Promise((resolve, reject) => {
-    fs.open(filePath, 'r', (err, fd) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(fd);
-      }
-    });
-  });
-}
-
-export function fstat(filePath) {
-  return new Promise<fs.Stats>((resolve, reject) => {
-    fs.stat(filePath, (err, stat) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stat);
-      }
-    });
-  });
-}
-
-export function fread(fd, position, length) {
-  const buffer = Buffer.alloc(length);
-
-  return new Promise((resolve, reject) => {
-    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-      if (err || bytesRead !== length) {
-        reject(err || Error('read failed'));
-      } else {
-        resolve(buf);
-      }
-    });
-  });
-}
-
-export function fclose(fd) {
-  return new Promise((resolve, reject) => {
-    fs.close(fd, (err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(err);
-      }
-    });
-  });
-}
-
-export function oswrite(os, buf): Promise<void> {
-  return new Promise((resolve, reject) => {
-    os.write(buf, (err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
-export function osclose(os): Promise<void> {
-  return new Promise((resolve, reject) => {
-    os.close((err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
-export function osopen(path, opts) {
-  return new Promise((resolve, reject) => {
-    const outputStream = fs.createWriteStream(path, opts);
-
-    outputStream.on('open', function (fd) {
-      resolve(outputStream);
-    });
-
-    outputStream.on('error', function (err) {
-      reject(err);
-    });
-  });
-}