@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.3

package/dist/parquetjs/parser/parquet-envelope-reader.d.ts

@@ -1,40 +0,0 @@
-/// <reference types="node" />
-import { ParquetSchema } from '../schema/schema';
-import { ColumnChunk, FileMetaData, RowGroup } from '../parquet-thrift';
-import { ParquetBuffer, ParquetData, ParquetOptions } from '../schema/declare';
-/**
- * The parquet envelope reader allows direct, unbuffered access to the individual
- * sections of the parquet file, namely the header, footer and the row groups.
- * This class is intended for advanced/internal users; if you just want to retrieve
- * rows from a parquet file use the ParquetReader instead
- */
-export declare class ParquetEnvelopeReader {
-    read: (position: number, length: number) => Promise<Buffer>;
-    /**
-     * Close this parquet reader. You MUST call this method once you're finished
-     * reading rows
-     */
-    close: () => Promise<void>;
-    fileSize: number;
-    defaultDictionarySize: number;
-    static openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader>;
-    constructor(read: (position: number, length: number) => Promise<Buffer>, close: () => Promise<void>, fileSize: number, options?: any);
-    readHeader(): Promise<void>;
-    readRowGroup(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][]): Promise<ParquetBuffer>;
-    /**
-     * Do reading of parquet file's column chunk
-     * @param schema
-     * @param colChunk
-     */
-    readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData>;
-    /**
-     * Getting dictionary for allows to flatten values by indices.
-     * @param dictionaryPageOffset
-     * @param options
-     * @param pagesOffset
-     * @returns
-     */
-    getDictionary(dictionaryPageOffset: number, options: ParquetOptions, pagesOffset: number): Promise<string[]>;
-    readFooter(): Promise<FileMetaData>;
-}
-//# sourceMappingURL=parquet-envelope-reader.d.ts.map

package/dist/parquetjs/parser/parquet-envelope-reader.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parquet-envelope-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-envelope-reader.ts"],"names":[],"mappings":";AACA,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,MAAM,mBAAmB,CAAC;AAC9F,OAAO,EACL,aAAa,EAEb,WAAW,EAEX,cAAc,EACf,MAAM,mBAAmB,CAAC;AAM3B;;;;;GAKG;AACH,qBAAa,qBAAqB;IACzB,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IACnE;;;OAGG;IACI,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,QAAQ,EAAE,MAAM,CAAC;IACjB,qBAAqB,EAAE,MAAM,CAAC;WAExB,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;gBAQrE,IAAI,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,EAC3D,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC1B,QAAQ,EAAE,MAAM,EAChB,OAAO,CAAC,EAAE,GAAG;IAQT,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAc3B,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,GACrB,OAAO,CAAC,aAAa,CAAC;IAgBzB;;;;OAIG;IACG,eAAe,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAoDzF;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,cAAc,EACvB,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,MAAM,EAAE,CAAC;IAwBd,UAAU,IAAI,OAAO,CAAC,YAAY,CAAC;CAqB1C"}

package/dist/parquetjs/parser/parquet-envelope-reader.js

@@ -1,136 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetEnvelopeReader = void 0;
-const constants_1 = require("../../constants");
-const parquet_thrift_1 = require("../parquet-thrift");
-const read_utils_1 = require("../utils/read-utils");
-const decoders_1 = require("./decoders");
-const DEFAULT_DICTIONARY_SIZE = 1e6;
-/**
- * The parquet envelope reader allows direct, unbuffered access to the individual
- * sections of the parquet file, namely the header, footer and the row groups.
- * This class is intended for advanced/internal users; if you just want to retrieve
- * rows from a parquet file use the ParquetReader instead
- */
-class ParquetEnvelopeReader {
-    static async openBuffer(buffer) {
-        const readFn = (position, length) => Promise.resolve(buffer.slice(position, position + length));
-        const closeFn = () => Promise.resolve();
-        return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
-    }
-    constructor(read, close, fileSize, options) {
-        this.read = read;
-        this.close = close;
-        this.fileSize = fileSize;
-        this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;
-    }
-    async readHeader() {
-        const buffer = await this.read(0, constants_1.PARQUET_MAGIC.length);
-        const magic = buffer.toString();
-        switch (magic) {
-            case constants_1.PARQUET_MAGIC:
-                break;
-            case constants_1.PARQUET_MAGIC_ENCRYPTED:
-                throw new Error('Encrypted parquet file not supported');
-            default:
-                throw new Error(`Invalid parquet file (magic=${magic})`);
-        }
-    }
-    async readRowGroup(schema, rowGroup, columnList) {
-        const buffer = {
-            rowCount: Number(rowGroup.num_rows),
-            columnData: {}
-        };
-        for (const colChunk of rowGroup.columns) {
-            const colMetadata = colChunk.meta_data;
-            const colKey = colMetadata?.path_in_schema;
-            if (columnList.length > 0 && (0, read_utils_1.fieldIndexOf)(columnList, colKey) < 0) {
-                continue; // eslint-disable-line no-continue
-            }
-            buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
-        }
-        return buffer;
-    }
-    /**
-     * Do reading of parquet file's column chunk
-     * @param schema
-     * @param colChunk
-     */
-    async readColumnChunk(schema, colChunk) {
-        if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
-            throw new Error('external references are not supported');
-        }
-        const field = schema.findField(colChunk.meta_data?.path_in_schema);
-        const type = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.Type, colChunk.meta_data?.type);
-        if (type !== field.primitiveType) {
-            throw new Error(`chunk type not matching schema: ${type}`);
-        }
-        const compression = (0, read_utils_1.getThriftEnum)(parquet_thrift_1.CompressionCodec, colChunk.meta_data?.codec);
-        const pagesOffset = Number(colChunk.meta_data?.data_page_offset);
-        let pagesSize = Number(colChunk.meta_data?.total_compressed_size);
-        if (!colChunk.file_path) {
-            pagesSize = Math.min(this.fileSize - pagesOffset, Number(colChunk.meta_data?.total_compressed_size));
-        }
-        const options = {
-            type,
-            rLevelMax: field.rLevelMax,
-            dLevelMax: field.dLevelMax,
-            compression,
-            column: field,
-            numValues: colChunk.meta_data?.num_values,
-            dictionary: []
-        };
-        let dictionary;
-        const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;
-        if (dictionaryPageOffset) {
-            const dictionaryOffset = Number(dictionaryPageOffset);
-            // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.
-            dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
-        }
-        dictionary = options.dictionary?.length ? options.dictionary : dictionary;
-        const pagesBuf = await this.read(pagesOffset, pagesSize);
-        return await (0, decoders_1.decodeDataPages)(pagesBuf, { ...options, dictionary });
-    }
-    /**
-     * Getting dictionary for allows to flatten values by indices.
-     * @param dictionaryPageOffset
-     * @param options
-     * @param pagesOffset
-     * @returns
-     */
-    async getDictionary(dictionaryPageOffset, options, pagesOffset) {
-        if (dictionaryPageOffset === 0) {
-            // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);
-            // pagesBuf = await this.read(pagesOffset, dictionarySize);
-            // In this case we are working with parquet-mr files format. Problem is described below:
-            // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding
-            // We need to get dictionary page from column chunk if it exists.
-            // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.
-            return [];
-        }
-        const dictionarySize = Math.min(this.fileSize - dictionaryPageOffset, this.defaultDictionarySize);
-        const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);
-        const cursor = { buffer: pagesBuf, offset: 0, size: pagesBuf.length };
-        const decodedPage = await (0, decoders_1.decodePage)(cursor, options);
-        return decodedPage.dictionary;
-    }
-    async readFooter() {
-        const trailerLen = constants_1.PARQUET_MAGIC.length + 4;
-        const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);
-        const magic = trailerBuf.slice(4).toString();
-        if (magic !== constants_1.PARQUET_MAGIC) {
-            throw new Error(`Not a valid parquet file (magic="${magic})`);
-        }
-        const metadataSize = trailerBuf.readUInt32LE(0);
-        const metadataOffset = this.fileSize - metadataSize - trailerLen;
-        if (metadataOffset < constants_1.PARQUET_MAGIC.length) {
-            throw new Error(`Invalid metadata size ${metadataOffset}`);
-        }
-        const metadataBuf = await this.read(metadataOffset, metadataSize);
-        // let metadata = new parquet_thrift.FileMetaData();
-        // parquet_util.decodeThrift(metadata, metadataBuf);
-        const { metadata } = (0, read_utils_1.decodeFileMetadata)(metadataBuf);
-        return metadata;
-    }
-}
-exports.ParquetEnvelopeReader = ParquetEnvelopeReader;

package/dist/parquetjs/utils/buffer-utils.d.ts

@@ -1,10 +0,0 @@
-/// <reference types="node" />
-/**
- * Convert Buffer to ArrayBuffer
- */
-export declare function toArrayBuffer(buffer: Buffer): ArrayBuffer;
-/**
- * Convert (copy) ArrayBuffer to Buffer
- */
-export declare function toBuffer(arrayBuffer: ArrayBuffer): Buffer;
-//# sourceMappingURL=buffer-utils.d.ts.map

package/dist/parquetjs/utils/buffer-utils.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"buffer-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/buffer-utils.ts"],"names":[],"mappings":";AAAA;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAOzD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,CAEzD"}

package/dist/parquetjs/utils/buffer-utils.js

@@ -1,22 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.toBuffer = exports.toArrayBuffer = void 0;
-/**
- * Convert Buffer to ArrayBuffer
- */
-function toArrayBuffer(buffer) {
-    // TODO - per docs we should just be able to call buffer.buffer, but there are issues
-    if (Buffer.isBuffer(buffer)) {
-        const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
-        return typedArray.slice().buffer;
-    }
-    return buffer;
-}
-exports.toArrayBuffer = toArrayBuffer;
-/**
- * Convert (copy) ArrayBuffer to Buffer
- */
-function toBuffer(arrayBuffer) {
-    return Buffer.from(arrayBuffer);
-}
-exports.toBuffer = toBuffer;

package/src/lib/parse-parquet.ts

@@ -1,27 +0,0 @@
-// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';
-import type {ParquetLoaderOptions} from '../parquet-loader';
-
-import {ParquetReader} from '../parquetjs/parser/parquet-reader';
-
-export async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
-  const blob = new Blob([arrayBuffer]);
-  for await (const batch of parseParquetFileInBatches(blob, options)) {
-    return batch;
-  }
-  return null;
-}
-
-export async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {
-  const reader = await ParquetReader.openBlob(blob);
-  const rows: any[][] = [];
-  try {
-    const cursor = reader.getCursor();
-    let record: any[] | null;
-    while ((record = await cursor.next())) {
-      rows.push(record);
-    }
-  } finally {
-    await reader.close();
-  }
-  yield rows;
-}

package/src/lib/read-array-buffer.ts

@@ -1,31 +0,0 @@
-// Random-Access read
-
-export async function readArrayBuffer(
-  file: Blob | ArrayBuffer | any,
-  start: number,
-  length: number
-): Promise<ArrayBuffer> {
-  if (file instanceof Blob) {
-    const slice = file.slice(start, start + length);
-    return await slice.arrayBuffer();
-  }
-  return await file.read(start, start + length);
-}
-
-/**
- * Read a slice of a Blob or File, without loading the entire file into memory
- * The trick when reading File objects is to read successive "slices" of the File
- * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields
- * Actually reading from file happens in `readAsArrayBuffer`
- * @param blob to read
- export async function readBlob(blob: Blob): Promise<ArrayBuffer> {
-  return await new Promise((resolve, reject) => {
-    const fileReader = new FileReader();
-    fileReader.onload = (event: ProgressEvent<FileReader>) =>
-      resolve(event?.target?.result as ArrayBuffer);
-    // TODO - reject with a proper Error
-    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);
-    fileReader.readAsArrayBuffer(blob);
-  });
-}
-*/

package/src/parquetjs/file.ts

@@ -1,90 +0,0 @@
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-import fs from 'fs';
-
-export function fopen(filePath) {
-  return new Promise((resolve, reject) => {
-    fs.open(filePath, 'r', (err, fd) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(fd);
-      }
-    });
-  });
-}
-
-export function fstat(filePath) {
-  return new Promise<fs.Stats>((resolve, reject) => {
-    fs.stat(filePath, (err, stat) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stat);
-      }
-    });
-  });
-}
-
-export function fread(fd, position, length) {
-  const buffer = Buffer.alloc(length);
-
-  return new Promise((resolve, reject) => {
-    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-      if (err || bytesRead !== length) {
-        reject(err || Error('read failed'));
-      } else {
-        resolve(buf);
-      }
-    });
-  });
-}
-
-export function fclose(fd) {
-  return new Promise((resolve, reject) => {
-    fs.close(fd, (err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(err);
-      }
-    });
-  });
-}
-
-export function oswrite(os, buf): Promise<void> {
-  return new Promise((resolve, reject) => {
-    os.write(buf, (err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
-export function osclose(os): Promise<void> {
-  return new Promise((resolve, reject) => {
-    os.close((err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
-export function osopen(path, opts) {
-  return new Promise((resolve, reject) => {
-    const outputStream = fs.createWriteStream(path, opts);
-
-    outputStream.on('open', function (fd) {
-      resolve(outputStream);
-    });
-
-    outputStream.on('error', function (err) {
-      reject(err);
-    });
-  });
-}

package/src/parquetjs/parser/parquet-cursor.ts

@@ -1,94 +0,0 @@
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-import {FileMetaData} from '../parquet-thrift';
-import {ParquetEnvelopeReader} from './parquet-envelope-reader';
-import {ParquetSchema} from '../schema/schema';
-import {ParquetRecord} from '../schema/declare';
-import {materializeRecords} from '../schema/shred';
-
-/**
- * A parquet cursor is used to retrieve rows from a parquet file in order
- */
-export class ParquetCursor<T> implements AsyncIterable<T> {
-  public metadata: FileMetaData;
-  public envelopeReader: ParquetEnvelopeReader;
-  public schema: ParquetSchema;
-  public columnList: string[][];
-  public rowGroup: ParquetRecord[] = [];
-  public rowGroupIndex: number;
-
-  /**
-   * Create a new parquet reader from the file metadata and an envelope reader.
-   * It is usually not recommended to call this constructor directly except for
-   * advanced and internal use cases. Consider using getCursor() on the
-   * ParquetReader instead
-   */
-  constructor(
-    metadata: FileMetaData,
-    envelopeReader: ParquetEnvelopeReader,
-    schema: ParquetSchema,
-    columnList: string[][]
-  ) {
-    this.metadata = metadata;
-    this.envelopeReader = envelopeReader;
-    this.schema = schema;
-    this.columnList = columnList;
-    this.rowGroupIndex = 0;
-  }
-
-  /**
-   * Retrieve the next row from the cursor. Returns a row or NULL if the end
-   * of the file was reached
-   */
-  async next<T = any>(): Promise<T> {
-    if (this.rowGroup.length === 0) {
-      if (this.rowGroupIndex >= this.metadata.row_groups.length) {
-        // @ts-ignore
-        return null;
-      }
-      const rowBuffer = await this.envelopeReader.readRowGroup(
-        this.schema,
-        this.metadata.row_groups[this.rowGroupIndex],
-        this.columnList
-      );
-      this.rowGroup = materializeRecords(this.schema, rowBuffer);
-      this.rowGroupIndex++;
-    }
-    return this.rowGroup.shift() as any;
-  }
-
-  /**
-   * Rewind the cursor the the beginning of the file
-   */
-  rewind(): void {
-    this.rowGroup = [];
-    this.rowGroupIndex = 0;
-  }
-
-  /**
-   * Implement AsyncIterable
-   */
-  // tslint:disable-next-line:function-name
-  [Symbol.asyncIterator](): AsyncIterator<T> {
-    let done = false;
-    return {
-      next: async () => {
-        if (done) {
-          return {done, value: null};
-        }
-        const value = await this.next();
-        if (value === null) {
-          return {done: true, value};
-        }
-        return {done: false, value};
-      },
-      return: async () => {
-        done = true;
-        return {done, value: null};
-      },
-      throw: async () => {
-        done = true;
-        return {done: true, value: null};
-      }
-    };
-  }
-}

package/src/parquetjs/parser/parquet-envelope-reader.ts

@@ -1,199 +0,0 @@
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-import {ParquetSchema} from '../schema/schema';
-import {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';
-import {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';
-import {
-  ParquetBuffer,
-  ParquetCompression,
-  ParquetData,
-  PrimitiveType,
-  ParquetOptions
-} from '../schema/declare';
-import {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';
-import {decodeDataPages, decodePage} from './decoders';
-
-const DEFAULT_DICTIONARY_SIZE = 1e6;
-
-/**
- * The parquet envelope reader allows direct, unbuffered access to the individual
- * sections of the parquet file, namely the header, footer and the row groups.
- * This class is intended for advanced/internal users; if you just want to retrieve
- * rows from a parquet file use the ParquetReader instead
- */
-export class ParquetEnvelopeReader {
-  public read: (position: number, length: number) => Promise<Buffer>;
-  /**
-   * Close this parquet reader. You MUST call this method once you're finished
-   * reading rows
-   */
-  public close: () => Promise<void>;
-  public fileSize: number;
-  public defaultDictionarySize: number;
-
-  static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {
-    const readFn = (position: number, length: number) =>
-      Promise.resolve(buffer.slice(position, position + length));
-    const closeFn = () => Promise.resolve();
-    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
-  }
-
-  constructor(
-    read: (position: number, length: number) => Promise<Buffer>,
-    close: () => Promise<void>,
-    fileSize: number,
-    options?: any
-  ) {
-    this.read = read;
-    this.close = close;
-    this.fileSize = fileSize;
-    this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;
-  }
-
-  async readHeader(): Promise<void> {
-    const buffer = await this.read(0, PARQUET_MAGIC.length);
-
-    const magic = buffer.toString();
-    switch (magic) {
-      case PARQUET_MAGIC:
-        break;
-      case PARQUET_MAGIC_ENCRYPTED:
-        throw new Error('Encrypted parquet file not supported');
-      default:
-        throw new Error(`Invalid parquet file (magic=${magic})`);
-    }
-  }
-
-  async readRowGroup(
-    schema: ParquetSchema,
-    rowGroup: RowGroup,
-    columnList: string[][]
-  ): Promise<ParquetBuffer> {
-    const buffer: ParquetBuffer = {
-      rowCount: Number(rowGroup.num_rows),
-      columnData: {}
-    };
-    for (const colChunk of rowGroup.columns) {
-      const colMetadata = colChunk.meta_data;
-      const colKey = colMetadata?.path_in_schema;
-      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {
-        continue; // eslint-disable-line no-continue
-      }
-      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);
-    }
-    return buffer;
-  }
-
-  /**
-   * Do reading of parquet file's column chunk
-   * @param schema
-   * @param colChunk
-   */
-  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {
-    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
-      throw new Error('external references are not supported');
-    }
-
-    const field = schema.findField(colChunk.meta_data?.path_in_schema!);
-    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;
-
-    if (type !== field.primitiveType) {
-      throw new Error(`chunk type not matching schema: ${type}`);
-    }
-
-    const compression: ParquetCompression = getThriftEnum(
-      CompressionCodec,
-      colChunk.meta_data?.codec!
-    ) as any;
-
-    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);
-    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);
-
-    if (!colChunk.file_path) {
-      pagesSize = Math.min(
-        this.fileSize - pagesOffset,
-        Number(colChunk.meta_data?.total_compressed_size)
-      );
-    }
-
-    const options: ParquetOptions = {
-      type,
-      rLevelMax: field.rLevelMax,
-      dLevelMax: field.dLevelMax,
-      compression,
-      column: field,
-      numValues: colChunk.meta_data?.num_values,
-      dictionary: []
-    };
-
-    let dictionary;
-
-    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;
-
-    if (dictionaryPageOffset) {
-      const dictionaryOffset = Number(dictionaryPageOffset);
-      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.
-      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
-    }
-
-    dictionary = options.dictionary?.length ? options.dictionary : dictionary;
-    const pagesBuf = await this.read(pagesOffset, pagesSize);
-    return await decodeDataPages(pagesBuf, {...options, dictionary});
-  }
-
-  /**
-   * Getting dictionary for allows to flatten values by indices.
-   * @param dictionaryPageOffset
-   * @param options
-   * @param pagesOffset
-   * @returns
-   */
-  async getDictionary(
-    dictionaryPageOffset: number,
-    options: ParquetOptions,
-    pagesOffset: number
-  ): Promise<string[]> {
-    if (dictionaryPageOffset === 0) {
-      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);
-      // pagesBuf = await this.read(pagesOffset, dictionarySize);
-
-      // In this case we are working with parquet-mr files format. Problem is described below:
-      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding
-      // We need to get dictionary page from column chunk if it exists.
-      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.
-      return [];
-    }
-
-    const dictionarySize = Math.min(
-      this.fileSize - dictionaryPageOffset,
-      this.defaultDictionarySize
-    );
-    const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);
-
-    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};
-    const decodedPage = await decodePage(cursor, options);
-
-    return decodedPage.dictionary!;
-  }
-
-  async readFooter(): Promise<FileMetaData> {
-    const trailerLen = PARQUET_MAGIC.length + 4;
-    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);
-
-    const magic = trailerBuf.slice(4).toString();
-    if (magic !== PARQUET_MAGIC) {
-      throw new Error(`Not a valid parquet file (magic="${magic})`);
-    }
-
-    const metadataSize = trailerBuf.readUInt32LE(0);
-    const metadataOffset = this.fileSize - metadataSize - trailerLen;
-    if (metadataOffset < PARQUET_MAGIC.length) {
-      throw new Error(`Invalid metadata size ${metadataOffset}`);
-    }
-
-    const metadataBuf = await this.read(metadataOffset, metadataSize);
-    // let metadata = new parquet_thrift.FileMetaData();
-    // parquet_util.decodeThrift(metadata, metadataBuf);
-    const {metadata} = decodeFileMetadata(metadataBuf);
-    return metadata;
-  }
-}

package/src/parquetjs/utils/buffer-utils.ts

@@ -1,18 +0,0 @@
-/**
- * Convert Buffer to ArrayBuffer
- */
-export function toArrayBuffer(buffer: Buffer): ArrayBuffer {
-  // TODO - per docs we should just be able to call buffer.buffer, but there are issues
-  if (Buffer.isBuffer(buffer)) {
-    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
-    return typedArray.slice().buffer;
-  }
-  return buffer;
-}
-
-/**
- * Convert (copy) ArrayBuffer to Buffer
- */
-export function toBuffer(arrayBuffer: ArrayBuffer): Buffer {
-  return Buffer.from(arrayBuffer);
-}