@loaders.gl/parquet 3.4.0-alpha.1 → 3.4.0-alpha.3

package/dist/esm/parquetjs/parser/parquet-cursor.js

@@ -1,78 +0,0 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-let _Symbol$asyncIterator;
-
-import { materializeRecords } from '../schema/shred';
-
-_Symbol$asyncIterator = Symbol.asyncIterator;
-export class ParquetCursor {
-  constructor(metadata, envelopeReader, schema, columnList) {
-    _defineProperty(this, "metadata", void 0);
-    _defineProperty(this, "envelopeReader", void 0);
-    _defineProperty(this, "schema", void 0);
-    _defineProperty(this, "columnList", void 0);
-    _defineProperty(this, "rowGroup", []);
-    _defineProperty(this, "rowGroupIndex", void 0);
-    this.metadata = metadata;
-    this.envelopeReader = envelopeReader;
-    this.schema = schema;
-    this.columnList = columnList;
-    this.rowGroupIndex = 0;
-  }
-
-  async next() {
-    if (this.rowGroup.length === 0) {
-      if (this.rowGroupIndex >= this.metadata.row_groups.length) {
-        return null;
-      }
-      const rowBuffer = await this.envelopeReader.readRowGroup(this.schema, this.metadata.row_groups[this.rowGroupIndex], this.columnList);
-      this.rowGroup = materializeRecords(this.schema, rowBuffer);
-      this.rowGroupIndex++;
-    }
-    return this.rowGroup.shift();
-  }
-
-  rewind() {
-    this.rowGroup = [];
-    this.rowGroupIndex = 0;
-  }
-
-  [_Symbol$asyncIterator]() {
-    let done = false;
-    return {
-      next: async () => {
-        if (done) {
-          return {
-            done,
-            value: null
-          };
-        }
-        const value = await this.next();
-        if (value === null) {
-          return {
-            done: true,
-            value
-          };
-        }
-        return {
-          done: false,
-          value
-        };
-      },
-      return: async () => {
-        done = true;
-        return {
-          done,
-          value: null
-        };
-      },
-      throw: async () => {
-        done = true;
-        return {
-          done: true,
-          value: null
-        };
-      }
-    };
-  }
-}
-//# sourceMappingURL=parquet-cursor.js.map

package/dist/esm/parquetjs/parser/parquet-cursor.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parquet-cursor.js","names":["materializeRecords","Symbol","asyncIterator","ParquetCursor","constructor","metadata","envelopeReader","schema","columnList","rowGroupIndex","next","rowGroup","length","row_groups","rowBuffer","readRowGroup","shift","rewind","done","value","return","throw"],"sources":["../../../../src/parquetjs/parser/parquet-cursor.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {FileMetaData} from '../parquet-thrift';\nimport {ParquetEnvelopeReader} from './parquet-envelope-reader';\nimport {ParquetSchema} from '../schema/schema';\nimport {ParquetRecord} from '../schema/declare';\nimport {materializeRecords} from '../schema/shred';\n\n/**\n * A parquet cursor is used to retrieve rows from a parquet file in order\n */\nexport class ParquetCursor<T> implements AsyncIterable<T> {\n  public metadata: FileMetaData;\n  public envelopeReader: ParquetEnvelopeReader;\n  public schema: ParquetSchema;\n  public columnList: string[][];\n  public rowGroup: ParquetRecord[] = [];\n  public rowGroupIndex: number;\n\n  /**\n   * Create a new parquet reader from the file metadata and an envelope reader.\n   * It is usually not recommended to call this constructor directly except for\n   * advanced and internal use cases. Consider using getCursor() on the\n   * ParquetReader instead\n   */\n  constructor(\n    metadata: FileMetaData,\n    envelopeReader: ParquetEnvelopeReader,\n    schema: ParquetSchema,\n    columnList: string[][]\n  ) {\n    this.metadata = metadata;\n    this.envelopeReader = envelopeReader;\n    this.schema = schema;\n    this.columnList = columnList;\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Retrieve the next row from the cursor. Returns a row or NULL if the end\n   * of the file was reached\n   */\n  async next<T = any>(): Promise<T> {\n    if (this.rowGroup.length === 0) {\n      if (this.rowGroupIndex >= this.metadata.row_groups.length) {\n        // @ts-ignore\n        return null;\n      }\n      const rowBuffer = await this.envelopeReader.readRowGroup(\n        this.schema,\n        this.metadata.row_groups[this.rowGroupIndex],\n        this.columnList\n      );\n      this.rowGroup = materializeRecords(this.schema, rowBuffer);\n      this.rowGroupIndex++;\n    }\n    return this.rowGroup.shift() as any;\n  }\n\n  /**\n   * Rewind the cursor the the beginning of the file\n   */\n  rewind(): void {\n    this.rowGroup = [];\n    this.rowGroupIndex = 0;\n  }\n\n  /**\n   * Implement AsyncIterable\n   */\n  // tslint:disable-next-line:function-name\n  [Symbol.asyncIterator](): AsyncIterator<T> {\n    let done = false;\n    return {\n      next: async () => {\n        if (done) {\n          return {done, value: null};\n        }\n        const value = await this.next();\n        if (value === null) {\n          return {done: true, value};\n        }\n        return {done: false, value};\n      },\n      return: async () => {\n        done = true;\n        return {done, value: null};\n      },\n      throw: async () => {\n        done = true;\n        return {done: true, value: null};\n      }\n    };\n  }\n}\n"],"mappings":";;;AAKA,SAAQA,kBAAkB,QAAO,iBAAiB;;AAAC,wBAiEhDC,MAAM,CAACC,aAAa;AA5DvB,OAAO,MAAMC,aAAa,CAAgC;EAcxDC,WAAW,CACTC,QAAsB,EACtBC,cAAqC,EACrCC,MAAqB,EACrBC,UAAsB,EACtB;IAAA;IAAA;IAAA;IAAA;IAAA,kCAdiC,EAAE;IAAA;IAenC,IAAI,CAACH,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACC,cAAc,GAAGA,cAAc;IACpC,IAAI,CAACC,MAAM,GAAGA,MAAM;IACpB,IAAI,CAACC,UAAU,GAAGA,UAAU;IAC5B,IAAI,CAACC,aAAa,GAAG,CAAC;EACxB;;EAMA,MAAMC,IAAI,GAAwB;IAChC,IAAI,IAAI,CAACC,QAAQ,CAACC,MAAM,KAAK,CAAC,EAAE;MAC9B,IAAI,IAAI,CAACH,aAAa,IAAI,IAAI,CAACJ,QAAQ,CAACQ,UAAU,CAACD,MAAM,EAAE;QAEzD,OAAO,IAAI;MACb;MACA,MAAME,SAAS,GAAG,MAAM,IAAI,CAACR,cAAc,CAACS,YAAY,CACtD,IAAI,CAACR,MAAM,EACX,IAAI,CAACF,QAAQ,CAACQ,UAAU,CAAC,IAAI,CAACJ,aAAa,CAAC,EAC5C,IAAI,CAACD,UAAU,CAChB;MACD,IAAI,CAACG,QAAQ,GAAGX,kBAAkB,CAAC,IAAI,CAACO,MAAM,EAAEO,SAAS,CAAC;MAC1D,IAAI,CAACL,aAAa,EAAE;IACtB;IACA,OAAO,IAAI,CAACE,QAAQ,CAACK,KAAK,EAAE;EAC9B;;EAKAC,MAAM,GAAS;IACb,IAAI,CAACN,QAAQ,GAAG,EAAE;IAClB,IAAI,CAACF,aAAa,GAAG,CAAC;EACxB;;EAMA,0BAA2C;IACzC,IAAIS,IAAI,GAAG,KAAK;IAChB,OAAO;MACLR,IAAI,EAAE,YAAY;QAChB,IAAIQ,IAAI,EAAE;UACR,OAAO;YAACA,IAAI;YAAEC,KAAK,EAAE;UAAI,CAAC;QAC5B;QACA,MAAMA,KAAK,GAAG,MAAM,IAAI,CAACT,IAAI,EAAE;QAC/B,IAAIS,KAAK,KAAK,IAAI,EAAE;UAClB,OAAO;YAACD,IAAI,EAAE,IAAI;YAAEC;UAAK,CAAC;QAC5B;QACA,OAAO;UAACD,IAAI,EAAE,KAAK;UAAEC;QAAK,CAAC;MAC7B,CAAC;MACDC,MAAM,EAAE,YAAY;QAClBF,IAAI,GAAG,IAAI;QACX,OAAO;UAACA,IAAI;UAAEC,KAAK,EAAE;QAAI,CAAC;MAC5B,CAAC;MACDE,KAAK,EAAE,YAAY;QACjBH,IAAI,GAAG,IAAI;QACX,OAAO;UAACA,IAAI,EAAE,IAAI;UAAEC,KAAK,EAAE;QAAI,CAAC;MAClC;IACF,CAAC;EACH;AACF"}

package/dist/esm/parquetjs/parser/parquet-envelope-reader.js

@@ -1,129 +0,0 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-
-import { PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED } from '../../constants';
-import { CompressionCodec, Type } from '../parquet-thrift';
-import { decodeFileMetadata, getThriftEnum, fieldIndexOf } from '../utils/read-utils';
-import { decodeDataPages, decodePage } from './decoders';
-const DEFAULT_DICTIONARY_SIZE = 1e6;
-
-export class ParquetEnvelopeReader {
-
-  static async openBuffer(buffer) {
-    const readFn = (position, length) => Promise.resolve(buffer.slice(position, position + length));
-    const closeFn = () => Promise.resolve();
-    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);
-  }
-  constructor(read, close, fileSize, options) {
-    _defineProperty(this, "read", void 0);
-    _defineProperty(this, "close", void 0);
-    _defineProperty(this, "fileSize", void 0);
-    _defineProperty(this, "defaultDictionarySize", void 0);
-    this.read = read;
-    this.close = close;
-    this.fileSize = fileSize;
-    this.defaultDictionarySize = (options === null || options === void 0 ? void 0 : options.defaultDictionarySize) || DEFAULT_DICTIONARY_SIZE;
-  }
-  async readHeader() {
-    const buffer = await this.read(0, PARQUET_MAGIC.length);
-    const magic = buffer.toString();
-    switch (magic) {
-      case PARQUET_MAGIC:
-        break;
-      case PARQUET_MAGIC_ENCRYPTED:
-        throw new Error('Encrypted parquet file not supported');
-      default:
-        throw new Error("Invalid parquet file (magic=".concat(magic, ")"));
-    }
-  }
-  async readRowGroup(schema, rowGroup, columnList) {
-    const buffer = {
-      rowCount: Number(rowGroup.num_rows),
-      columnData: {}
-    };
-    for (const colChunk of rowGroup.columns) {
-      const colMetadata = colChunk.meta_data;
-      const colKey = colMetadata === null || colMetadata === void 0 ? void 0 : colMetadata.path_in_schema;
-      if (columnList.length > 0 && fieldIndexOf(columnList, colKey) < 0) {
-        continue;
-      }
-
-      buffer.columnData[colKey.join()] = await this.readColumnChunk(schema, colChunk);
-    }
-    return buffer;
-  }
-
-  async readColumnChunk(schema, colChunk) {
-    var _colChunk$meta_data, _colChunk$meta_data2, _colChunk$meta_data3, _colChunk$meta_data4, _colChunk$meta_data5, _colChunk$meta_data7, _colChunk$meta_data8, _options$dictionary;
-    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
-      throw new Error('external references are not supported');
-    }
-    const field = schema.findField((_colChunk$meta_data = colChunk.meta_data) === null || _colChunk$meta_data === void 0 ? void 0 : _colChunk$meta_data.path_in_schema);
-    const type = getThriftEnum(Type, (_colChunk$meta_data2 = colChunk.meta_data) === null || _colChunk$meta_data2 === void 0 ? void 0 : _colChunk$meta_data2.type);
-    if (type !== field.primitiveType) {
-      throw new Error("chunk type not matching schema: ".concat(type));
-    }
-    const compression = getThriftEnum(CompressionCodec, (_colChunk$meta_data3 = colChunk.meta_data) === null || _colChunk$meta_data3 === void 0 ? void 0 : _colChunk$meta_data3.codec);
-    const pagesOffset = Number((_colChunk$meta_data4 = colChunk.meta_data) === null || _colChunk$meta_data4 === void 0 ? void 0 : _colChunk$meta_data4.data_page_offset);
-    let pagesSize = Number((_colChunk$meta_data5 = colChunk.meta_data) === null || _colChunk$meta_data5 === void 0 ? void 0 : _colChunk$meta_data5.total_compressed_size);
-    if (!colChunk.file_path) {
-      var _colChunk$meta_data6;
-      pagesSize = Math.min(this.fileSize - pagesOffset, Number((_colChunk$meta_data6 = colChunk.meta_data) === null || _colChunk$meta_data6 === void 0 ? void 0 : _colChunk$meta_data6.total_compressed_size));
-    }
-    const options = {
-      type,
-      rLevelMax: field.rLevelMax,
-      dLevelMax: field.dLevelMax,
-      compression,
-      column: field,
-      numValues: (_colChunk$meta_data7 = colChunk.meta_data) === null || _colChunk$meta_data7 === void 0 ? void 0 : _colChunk$meta_data7.num_values,
-      dictionary: []
-    };
-    let dictionary;
-    const dictionaryPageOffset = colChunk === null || colChunk === void 0 ? void 0 : (_colChunk$meta_data8 = colChunk.meta_data) === null || _colChunk$meta_data8 === void 0 ? void 0 : _colChunk$meta_data8.dictionary_page_offset;
-    if (dictionaryPageOffset) {
-      const dictionaryOffset = Number(dictionaryPageOffset);
-      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);
-    }
-    dictionary = (_options$dictionary = options.dictionary) !== null && _options$dictionary !== void 0 && _options$dictionary.length ? options.dictionary : dictionary;
-    const pagesBuf = await this.read(pagesOffset, pagesSize);
-    return await decodeDataPages(pagesBuf, {
-      ...options,
-      dictionary
-    });
-  }
-
-  async getDictionary(dictionaryPageOffset, options, pagesOffset) {
-    if (dictionaryPageOffset === 0) {
-
-      return [];
-    }
-    const dictionarySize = Math.min(this.fileSize - dictionaryPageOffset, this.defaultDictionarySize);
-    const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);
-    const cursor = {
-      buffer: pagesBuf,
-      offset: 0,
-      size: pagesBuf.length
-    };
-    const decodedPage = await decodePage(cursor, options);
-    return decodedPage.dictionary;
-  }
-  async readFooter() {
-    const trailerLen = PARQUET_MAGIC.length + 4;
-    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);
-    const magic = trailerBuf.slice(4).toString();
-    if (magic !== PARQUET_MAGIC) {
-      throw new Error("Not a valid parquet file (magic=\"".concat(magic, ")"));
-    }
-    const metadataSize = trailerBuf.readUInt32LE(0);
-    const metadataOffset = this.fileSize - metadataSize - trailerLen;
-    if (metadataOffset < PARQUET_MAGIC.length) {
-      throw new Error("Invalid metadata size ".concat(metadataOffset));
-    }
-    const metadataBuf = await this.read(metadataOffset, metadataSize);
-    const {
-      metadata
-    } = decodeFileMetadata(metadataBuf);
-    return metadata;
-  }
-}
-//# sourceMappingURL=parquet-envelope-reader.js.map

package/dist/esm/parquetjs/parser/parquet-envelope-reader.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parquet-envelope-reader.js","names":["PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","CompressionCodec","Type","decodeFileMetadata","getThriftEnum","fieldIndexOf","decodeDataPages","decodePage","DEFAULT_DICTIONARY_SIZE","ParquetEnvelopeReader","openBuffer","buffer","readFn","position","length","Promise","resolve","slice","closeFn","constructor","read","close","fileSize","options","defaultDictionarySize","readHeader","magic","toString","Error","readRowGroup","schema","rowGroup","columnList","rowCount","Number","num_rows","columnData","colChunk","columns","colMetadata","meta_data","colKey","path_in_schema","join","readColumnChunk","file_path","undefined","field","findField","type","primitiveType","compression","codec","pagesOffset","data_page_offset","pagesSize","total_compressed_size","Math","min","rLevelMax","dLevelMax","column","numValues","num_values","dictionary","dictionaryPageOffset","dictionary_page_offset","dictionaryOffset","getDictionary","pagesBuf","dictionarySize","cursor","offset","size","decodedPage","readFooter","trailerLen","trailerBuf","metadataSize","readUInt32LE","metadataOffset","metadataBuf","metadata"],"sources":["../../../../src/parquetjs/parser/parquet-envelope-reader.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport {ParquetSchema} from '../schema/schema';\nimport {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../constants';\nimport {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';\nimport {\n  ParquetBuffer,\n  ParquetCompression,\n  ParquetData,\n  PrimitiveType,\n  ParquetOptions\n} from '../schema/declare';\nimport {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';\nimport {decodeDataPages, decodePage} from './decoders';\n\nconst DEFAULT_DICTIONARY_SIZE = 1e6;\n\n/**\n * The parquet envelope reader allows direct, unbuffered access to the individual\n * sections of the parquet file, namely the header, footer and the row groups.\n * This class is intended for advanced/internal users; if you just want to retrieve\n * rows from a parquet file use the ParquetReader instead\n */\nexport class ParquetEnvelopeReader {\n  public read: (position: number, length: number) => Promise<Buffer>;\n  /**\n   * Close this parquet reader. You MUST call this method once you're finished\n   * reading rows\n   */\n  public close: () => Promise<void>;\n  public fileSize: number;\n  public defaultDictionarySize: number;\n\n  static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {\n    const readFn = (position: number, length: number) =>\n      Promise.resolve(buffer.slice(position, position + length));\n    const closeFn = () => Promise.resolve();\n    return new ParquetEnvelopeReader(readFn, closeFn, buffer.length);\n  }\n\n  constructor(\n    read: (position: number, length: number) => Promise<Buffer>,\n    close: () => Promise<void>,\n    fileSize: number,\n    options?: any\n  ) {\n    this.read = read;\n    this.close = close;\n    this.fileSize = fileSize;\n    this.defaultDictionarySize = options?.defaultDictionarySize || DEFAULT_DICTIONARY_SIZE;\n  }\n\n  async readHeader(): Promise<void> {\n    const buffer = await this.read(0, PARQUET_MAGIC.length);\n\n    const magic = buffer.toString();\n    switch (magic) {\n      case PARQUET_MAGIC:\n        break;\n      case PARQUET_MAGIC_ENCRYPTED:\n        throw new Error('Encrypted parquet file not supported');\n      default:\n        throw new Error(`Invalid parquet file (magic=${magic})`);\n    }\n  }\n\n  async readRowGroup(\n    schema: ParquetSchema,\n    rowGroup: RowGroup,\n    columnList: string[][]\n  ): Promise<ParquetBuffer> {\n    const buffer: ParquetBuffer = {\n      rowCount: Number(rowGroup.num_rows),\n      columnData: {}\n    };\n    for (const colChunk of rowGroup.columns) {\n      const colMetadata = colChunk.meta_data;\n      const colKey = colMetadata?.path_in_schema;\n      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {\n        continue; // eslint-disable-line no-continue\n      }\n      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);\n    }\n    return buffer;\n  }\n\n  /**\n   * Do reading of parquet file's column chunk\n   * @param schema\n   * @param colChunk\n   */\n  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetData> {\n    if (colChunk.file_path !== undefined && colChunk.file_path !== null) {\n      throw new Error('external references are not supported');\n    }\n\n    const field = schema.findField(colChunk.meta_data?.path_in_schema!);\n    const type: PrimitiveType = getThriftEnum(Type, colChunk.meta_data?.type!) as any;\n\n    if (type !== field.primitiveType) {\n      throw new Error(`chunk type not matching schema: ${type}`);\n    }\n\n    const compression: ParquetCompression = getThriftEnum(\n      CompressionCodec,\n      colChunk.meta_data?.codec!\n    ) as any;\n\n    const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);\n    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);\n\n    if (!colChunk.file_path) {\n      pagesSize = Math.min(\n        this.fileSize - pagesOffset,\n        Number(colChunk.meta_data?.total_compressed_size)\n      );\n    }\n\n    const options: ParquetOptions = {\n      type,\n      rLevelMax: field.rLevelMax,\n      dLevelMax: field.dLevelMax,\n      compression,\n      column: field,\n      numValues: colChunk.meta_data?.num_values,\n      dictionary: []\n    };\n\n    let dictionary;\n\n    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;\n\n    if (dictionaryPageOffset) {\n      const dictionaryOffset = Number(dictionaryPageOffset);\n      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.\n      dictionary = await this.getDictionary(dictionaryOffset, options, pagesOffset);\n    }\n\n    dictionary = options.dictionary?.length ? options.dictionary : dictionary;\n    const pagesBuf = await this.read(pagesOffset, pagesSize);\n    return await decodeDataPages(pagesBuf, {...options, dictionary});\n  }\n\n  /**\n   * Getting dictionary for allows to flatten values by indices.\n   * @param dictionaryPageOffset\n   * @param options\n   * @param pagesOffset\n   * @returns\n   */\n  async getDictionary(\n    dictionaryPageOffset: number,\n    options: ParquetOptions,\n    pagesOffset: number\n  ): Promise<string[]> {\n    if (dictionaryPageOffset === 0) {\n      // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);\n      // pagesBuf = await this.read(pagesOffset, dictionarySize);\n\n      // In this case we are working with parquet-mr files format. Problem is described below:\n      // https://stackoverflow.com/questions/55225108/why-is-dictionary-page-offset-0-for-plain-dictionary-encoding\n      // We need to get dictionary page from column chunk if it exists.\n      // Now if we use code commented above we don't get DICTIONARY_PAGE we get DATA_PAGE instead.\n      return [];\n    }\n\n    const dictionarySize = Math.min(\n      this.fileSize - dictionaryPageOffset,\n      this.defaultDictionarySize\n    );\n    const pagesBuf = await this.read(dictionaryPageOffset, dictionarySize);\n\n    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};\n    const decodedPage = await decodePage(cursor, options);\n\n    return decodedPage.dictionary!;\n  }\n\n  async readFooter(): Promise<FileMetaData> {\n    const trailerLen = PARQUET_MAGIC.length + 4;\n    const trailerBuf = await this.read(this.fileSize - trailerLen, trailerLen);\n\n    const magic = trailerBuf.slice(4).toString();\n    if (magic !== PARQUET_MAGIC) {\n      throw new Error(`Not a valid parquet file (magic=\"${magic})`);\n    }\n\n    const metadataSize = trailerBuf.readUInt32LE(0);\n    const metadataOffset = this.fileSize - metadataSize - trailerLen;\n    if (metadataOffset < PARQUET_MAGIC.length) {\n      throw new Error(`Invalid metadata size ${metadataOffset}`);\n    }\n\n    const metadataBuf = await this.read(metadataOffset, metadataSize);\n    // let metadata = new parquet_thrift.FileMetaData();\n    // parquet_util.decodeThrift(metadata, metadataBuf);\n    const {metadata} = decodeFileMetadata(metadataBuf);\n    return metadata;\n  }\n}\n"],"mappings":";;AAEA,SAAQA,aAAa,EAAEC,uBAAuB,QAAO,iBAAiB;AACtE,SAAqBC,gBAAgB,EAA0BC,IAAI,QAAO,mBAAmB;AAQ7F,SAAQC,kBAAkB,EAAEC,aAAa,EAAEC,YAAY,QAAO,qBAAqB;AACnF,SAAQC,eAAe,EAAEC,UAAU,QAAO,YAAY;AAEtD,MAAMC,uBAAuB,GAAG,GAAG;;AAQnC,OAAO,MAAMC,qBAAqB,CAAC;;EAUjC,aAAaC,UAAU,CAACC,MAAc,EAAkC;IACtE,MAAMC,MAAM,GAAG,CAACC,QAAgB,EAAEC,MAAc,KAC9CC,OAAO,CAACC,OAAO,CAACL,MAAM,CAACM,KAAK,CAACJ,QAAQ,EAAEA,QAAQ,GAAGC,MAAM,CAAC,CAAC;IAC5D,MAAMI,OAAO,GAAG,MAAMH,OAAO,CAACC,OAAO,EAAE;IACvC,OAAO,IAAIP,qBAAqB,CAACG,MAAM,EAAEM,OAAO,EAAEP,MAAM,CAACG,MAAM,CAAC;EAClE;EAEAK,WAAW,CACTC,IAA2D,EAC3DC,KAA0B,EAC1BC,QAAgB,EAChBC,OAAa,EACb;IAAA;IAAA;IAAA;IAAA;IACA,IAAI,CAACH,IAAI,GAAGA,IAAI;IAChB,IAAI,CAACC,KAAK,GAAGA,KAAK;IAClB,IAAI,CAACC,QAAQ,GAAGA,QAAQ;IACxB,IAAI,CAACE,qBAAqB,GAAG,CAAAD,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEC,qBAAqB,KAAIhB,uBAAuB;EACxF;EAEA,MAAMiB,UAAU,GAAkB;IAChC,MAAMd,MAAM,GAAG,MAAM,IAAI,CAACS,IAAI,CAAC,CAAC,EAAErB,aAAa,CAACe,MAAM,CAAC;IAEvD,MAAMY,KAAK,GAAGf,MAAM,CAACgB,QAAQ,EAAE;IAC/B,QAAQD,KAAK;MACX,KAAK3B,aAAa;QAChB;MACF,KAAKC,uBAAuB;QAC1B,MAAM,IAAI4B,KAAK,CAAC,sCAAsC,CAAC;MACzD;QACE,MAAM,IAAIA,KAAK,uCAAgCF,KAAK,OAAI;IAAC;EAE/D;EAEA,MAAMG,YAAY,CAChBC,MAAqB,EACrBC,QAAkB,EAClBC,UAAsB,EACE;IACxB,MAAMrB,MAAqB,GAAG;MAC5BsB,QAAQ,EAAEC,MAAM,CAACH,QAAQ,CAACI,QAAQ,CAAC;MACnCC,UAAU,EAAE,CAAC;IACf,CAAC;IACD,KAAK,MAAMC,QAAQ,IAAIN,QAAQ,CAACO,OAAO,EAAE;MACvC,MAAMC,WAAW,GAAGF,QAAQ,CAACG,SAAS;MACtC,MAAMC,MAAM,GAAGF,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEG,cAAc;MAC1C,IAAIV,UAAU,CAAClB,MAAM,GAAG,CAAC,IAAIT,YAAY,CAAC2B,UAAU,EAAES,MAAM,CAAE,GAAG,CAAC,EAAE;QAClE;MACF;;MACA9B,MAAM,CAACyB,UAAU,CAACK,MAAM,CAAEE,IAAI,EAAE,CAAC,GAAG,MAAM,IAAI,CAACC,eAAe,CAACd,MAAM,EAAEO,QAAQ,CAAC;IAClF;IACA,OAAO1B,MAAM;EACf;;EAOA,MAAMiC,eAAe,CAACd,MAAqB,EAAEO,QAAqB,EAAwB;IAAA;IACxF,IAAIA,QAAQ,CAACQ,SAAS,KAAKC,SAAS,IAAIT,QAAQ,CAACQ,SAAS,KAAK,IAAI,EAAE;MACnE,MAAM,IAAIjB,KAAK,CAAC,uCAAuC,CAAC;IAC1D;IAEA,MAAMmB,KAAK,GAAGjB,MAAM,CAACkB,SAAS,wBAACX,QAAQ,CAACG,SAAS,wDAAlB,oBAAoBE,cAAc,CAAE;IACnE,MAAMO,IAAmB,GAAG7C,aAAa,CAACF,IAAI,0BAAEmC,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBS,IAAI,CAAS;IAEjF,IAAIA,IAAI,KAAKF,KAAK,CAACG,aAAa,EAAE;MAChC,MAAM,IAAItB,KAAK,2CAAoCqB,IAAI,EAAG;IAC5D;IAEA,MAAME,WAA+B,GAAG/C,aAAa,CACnDH,gBAAgB,0BAChBoC,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBY,KAAK,CACnB;IAER,MAAMC,WAAW,GAAGnB,MAAM,yBAACG,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBc,gBAAgB,CAAE;IACjE,IAAIC,SAAS,GAAGrB,MAAM,yBAACG,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBgB,qBAAqB,CAAE;IAElE,IAAI,CAACnB,QAAQ,CAACQ,SAAS,EAAE;MAAA;MACvBU,SAAS,GAAGE,IAAI,CAACC,GAAG,CAClB,IAAI,CAACpC,QAAQ,GAAG+B,WAAW,EAC3BnB,MAAM,yBAACG,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBgB,qBAAqB,CAAC,CAClD;IACH;IAEA,MAAMjC,OAAuB,GAAG;MAC9B0B,IAAI;MACJU,SAAS,EAAEZ,KAAK,CAACY,SAAS;MAC1BC,SAAS,EAAEb,KAAK,CAACa,SAAS;MAC1BT,WAAW;MACXU,MAAM,EAAEd,KAAK;MACbe,SAAS,0BAAEzB,QAAQ,CAACG,SAAS,yDAAlB,qBAAoBuB,UAAU;MACzCC,UAAU,EAAE;IACd,CAAC;IAED,IAAIA,UAAU;IAEd,MAAMC,oBAAoB,GAAG5B,QAAQ,aAARA,QAAQ,+CAARA,QAAQ,CAAEG,SAAS,yDAAnB,qBAAqB0B,sBAAsB;IAExE,IAAID,oBAAoB,EAAE;MACxB,MAAME,gBAAgB,GAAGjC,MAAM,CAAC+B,oBAAoB,CAAC;MAErDD,UAAU,GAAG,MAAM,IAAI,CAACI,aAAa,CAACD,gBAAgB,EAAE5C,OAAO,EAAE8B,WAAW,CAAC;IAC/E;IAEAW,UAAU,GAAG,uBAAAzC,OAAO,CAACyC,UAAU,gDAAlB,oBAAoBlD,MAAM,GAAGS,OAAO,CAACyC,UAAU,GAAGA,UAAU;IACzE,MAAMK,QAAQ,GAAG,MAAM,IAAI,CAACjD,IAAI,CAACiC,WAAW,EAAEE,SAAS,CAAC;IACxD,OAAO,MAAMjD,eAAe,CAAC+D,QAAQ,EAAE;MAAC,GAAG9C,OAAO;MAAEyC;IAAU,CAAC,CAAC;EAClE;;EASA,MAAMI,aAAa,CACjBH,oBAA4B,EAC5B1C,OAAuB,EACvB8B,WAAmB,EACA;IACnB,IAAIY,oBAAoB,KAAK,CAAC,EAAE;;MAQ9B,OAAO,EAAE;IACX;IAEA,MAAMK,cAAc,GAAGb,IAAI,CAACC,GAAG,CAC7B,IAAI,CAACpC,QAAQ,GAAG2C,oBAAoB,EACpC,IAAI,CAACzC,qBAAqB,CAC3B;IACD,MAAM6C,QAAQ,GAAG,MAAM,IAAI,CAACjD,IAAI,CAAC6C,oBAAoB,EAAEK,cAAc,CAAC;IAEtE,MAAMC,MAAM,GAAG;MAAC5D,MAAM,EAAE0D,QAAQ;MAAEG,MAAM,EAAE,CAAC;MAAEC,IAAI,EAAEJ,QAAQ,CAACvD;IAAM,CAAC;IACnE,MAAM4D,WAAW,GAAG,MAAMnE,UAAU,CAACgE,MAAM,EAAEhD,OAAO,CAAC;IAErD,OAAOmD,WAAW,CAACV,UAAU;EAC/B;EAEA,MAAMW,UAAU,GAA0B;IACxC,MAAMC,UAAU,GAAG7E,aAAa,CAACe,MAAM,GAAG,CAAC;IAC3C,MAAM+D,UAAU,GAAG,MAAM,IAAI,CAACzD,IAAI,CAAC,IAAI,CAACE,QAAQ,GAAGsD,UAAU,EAAEA,UAAU,CAAC;IAE1E,MAAMlD,KAAK,GAAGmD,UAAU,CAAC5D,KAAK,CAAC,CAAC,CAAC,CAACU,QAAQ,EAAE;IAC5C,IAAID,KAAK,KAAK3B,aAAa,EAAE;MAC3B,MAAM,IAAI6B,KAAK,6CAAqCF,KAAK,OAAI;IAC/D;IAEA,MAAMoD,YAAY,GAAGD,UAAU,CAACE,YAAY,CAAC,CAAC,CAAC;IAC/C,MAAMC,cAAc,GAAG,IAAI,CAAC1D,QAAQ,GAAGwD,YAAY,GAAGF,UAAU;IAChE,IAAII,cAAc,GAAGjF,aAAa,CAACe,MAAM,EAAE;MACzC,MAAM,IAAIc,KAAK,iCAA0BoD,cAAc,EAAG;IAC5D;IAEA,MAAMC,WAAW,GAAG,MAAM,IAAI,CAAC7D,IAAI,CAAC4D,cAAc,EAAEF,YAAY,CAAC;IAGjE,MAAM;MAACI;IAAQ,CAAC,GAAG/E,kBAAkB,CAAC8E,WAAW,CAAC;IAClD,OAAOC,QAAQ;EACjB;AACF"}

package/dist/esm/parquetjs/utils/buffer-utils.js

@@ -1,13 +0,0 @@
-
-export function toArrayBuffer(buffer) {
-  if (Buffer.isBuffer(buffer)) {
-    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
-    return typedArray.slice().buffer;
-  }
-  return buffer;
-}
-
-export function toBuffer(arrayBuffer) {
-  return Buffer.from(arrayBuffer);
-}
-//# sourceMappingURL=buffer-utils.js.map

package/dist/esm/parquetjs/utils/buffer-utils.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"buffer-utils.js","names":["toArrayBuffer","buffer","Buffer","isBuffer","typedArray","Uint8Array","byteOffset","length","slice","toBuffer","arrayBuffer","from"],"sources":["../../../../src/parquetjs/utils/buffer-utils.ts"],"sourcesContent":["/**\n * Convert Buffer to ArrayBuffer\n */\nexport function toArrayBuffer(buffer: Buffer): ArrayBuffer {\n  // TODO - per docs we should just be able to call buffer.buffer, but there are issues\n  if (Buffer.isBuffer(buffer)) {\n    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);\n    return typedArray.slice().buffer;\n  }\n  return buffer;\n}\n\n/**\n * Convert (copy) ArrayBuffer to Buffer\n */\nexport function toBuffer(arrayBuffer: ArrayBuffer): Buffer {\n  return Buffer.from(arrayBuffer);\n}\n"],"mappings":";AAGA,OAAO,SAASA,aAAa,CAACC,MAAc,EAAe;EAEzD,IAAIC,MAAM,CAACC,QAAQ,CAACF,MAAM,CAAC,EAAE;IAC3B,MAAMG,UAAU,GAAG,IAAIC,UAAU,CAACJ,MAAM,CAACA,MAAM,EAAEA,MAAM,CAACK,UAAU,EAAEL,MAAM,CAACM,MAAM,CAAC;IAClF,OAAOH,UAAU,CAACI,KAAK,EAAE,CAACP,MAAM;EAClC;EACA,OAAOA,MAAM;AACf;;AAKA,OAAO,SAASQ,QAAQ,CAACC,WAAwB,EAAU;EACzD,OAAOR,MAAM,CAACS,IAAI,CAACD,WAAW,CAAC;AACjC"}

package/dist/lib/convert-schema.d.ts

@@ -1,8 +0,0 @@
-import type { ParquetSchema } from '../parquetjs/schema/schema';
-import type { ParquetType } from '../parquetjs/schema/declare';
-import { Schema, DataType } from '@loaders.gl/schema';
-export declare const PARQUET_TYPE_MAPPING: {
-    [type in ParquetType]: typeof DataType;
-};
-export declare function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema;
-//# sourceMappingURL=convert-schema.d.ts.map

package/dist/lib/convert-schema.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-schema.d.ts","sourceRoot":"","sources":["../../src/lib/convert-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,4BAA4B,CAAC;AAC9D,OAAO,KAAK,EAAgC,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAE5F,OAAO,EACL,MAAM,EAGN,QAAQ,EAaT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,OAAO,QAAQ;CA+BzE,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,aAAa,EAAE,aAAa,GAAG,MAAM,CAKhF"}

package/dist/lib/parse-parquet.d.ts

@@ -1,4 +0,0 @@
-import type { ParquetLoaderOptions } from '../parquet-loader';
-export declare function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions): Promise<any[][] | null>;
-export declare function parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions): AsyncGenerator<any[][], void, unknown>;
-//# sourceMappingURL=parse-parquet.d.ts.map

package/dist/lib/parse-parquet.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-parquet.d.ts","sourceRoot":"","sources":["../../src/lib/parse-parquet.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,mBAAmB,CAAC;AAI5D,wBAAsB,YAAY,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,oBAAoB,2BAM1F;AAED,wBAAuB,yBAAyB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,oBAAoB,0CAa1F"}

package/dist/lib/parse-parquet.js

@@ -1,28 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseParquetFileInBatches = exports.parseParquet = void 0;
-const parquet_reader_1 = require("../parquetjs/parser/parquet-reader");
-async function parseParquet(arrayBuffer, options) {
-    const blob = new Blob([arrayBuffer]);
-    for await (const batch of parseParquetFileInBatches(blob, options)) {
-        return batch;
-    }
-    return null;
-}
-exports.parseParquet = parseParquet;
-async function* parseParquetFileInBatches(blob, options) {
-    const reader = await parquet_reader_1.ParquetReader.openBlob(blob);
-    const rows = [];
-    try {
-        const cursor = reader.getCursor();
-        let record;
-        while ((record = await cursor.next())) {
-            rows.push(record);
-        }
-    }
-    finally {
-        await reader.close();
-    }
-    yield rows;
-}
-exports.parseParquetFileInBatches = parseParquetFileInBatches;

package/dist/lib/read-array-buffer.d.ts

@@ -1,19 +0,0 @@
-export declare function readArrayBuffer(file: Blob | ArrayBuffer | any, start: number, length: number): Promise<ArrayBuffer>;
-/**
- * Read a slice of a Blob or File, without loading the entire file into memory
- * The trick when reading File objects is to read successive "slices" of the File
- * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields
- * Actually reading from file happens in `readAsArrayBuffer`
- * @param blob to read
- export async function readBlob(blob: Blob): Promise<ArrayBuffer> {
-  return await new Promise((resolve, reject) => {
-    const fileReader = new FileReader();
-    fileReader.onload = (event: ProgressEvent<FileReader>) =>
-      resolve(event?.target?.result as ArrayBuffer);
-    // TODO - reject with a proper Error
-    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);
-    fileReader.readAsArrayBuffer(blob);
-  });
-}
-*/
-//# sourceMappingURL=read-array-buffer.d.ts.map

package/dist/lib/read-array-buffer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"read-array-buffer.d.ts","sourceRoot":"","sources":["../../src/lib/read-array-buffer.ts"],"names":[],"mappings":"AAEA,wBAAsB,eAAe,CACnC,IAAI,EAAE,IAAI,GAAG,WAAW,GAAG,GAAG,EAC9B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,WAAW,CAAC,CAMtB;AAED;;;;;;;;;;;;;;;;EAgBE"}

package/dist/lib/read-array-buffer.js

@@ -1,29 +0,0 @@
-"use strict";
-// Random-Access read
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.readArrayBuffer = void 0;
-async function readArrayBuffer(file, start, length) {
-    if (file instanceof Blob) {
-        const slice = file.slice(start, start + length);
-        return await slice.arrayBuffer();
-    }
-    return await file.read(start, start + length);
-}
-exports.readArrayBuffer = readArrayBuffer;
-/**
- * Read a slice of a Blob or File, without loading the entire file into memory
- * The trick when reading File objects is to read successive "slices" of the File
- * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields
- * Actually reading from file happens in `readAsArrayBuffer`
- * @param blob to read
- export async function readBlob(blob: Blob): Promise<ArrayBuffer> {
-  return await new Promise((resolve, reject) => {
-    const fileReader = new FileReader();
-    fileReader.onload = (event: ProgressEvent<FileReader>) =>
-      resolve(event?.target?.result as ArrayBuffer);
-    // TODO - reject with a proper Error
-    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);
-    fileReader.readAsArrayBuffer(blob);
-  });
-}
-*/

package/dist/parquetjs/encoder/writer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/encoder/writer.ts"],"names":[],"mappings":";;AAEA,OAAO,EAAC,SAAS,EAAE,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAG3C,OAAO,EACL,aAAa,EAKd,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAE/C,OAAO,EAaL,QAAQ,EAGT,MAAM,mBAAmB,CAAC;AA2B3B,MAAM,WAAW,oBAAoB;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,OAAO,CAAC;IAGxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AAEH,qBAAa,aAAa,CAAC,CAAC;IAC1B;;;OAGG;WACU,QAAQ,CAAC,CAAC,EACrB,MAAM,EAAE,aAAa,EACrB,IAAI,EAAE,MAAM,EACZ,IAAI,CAAC,EAAE,oBAAoB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAK5B;;;OAGG;WACU,UAAU,CAAC,CAAC,EACvB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,QAAQ,EACtB,IAAI,CAAC,EAAE,oBAAoB,GAC1B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAWrB,MAAM,EAAE,aAAa,CAAC;IACtB,cAAc,EAAE,qBAAqB,CAAC;IACtC,SAAS,EAAE,aAAa,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE5C;;OAEG;gBAED,MAAM,EAAE,aAAa,EACrB,cAAc,EAAE,qBAAqB,EACrC,IAAI,EAAE,oBAAoB;IActB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC;;;OAGG;IACG,SAAS,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzC;;;;;OAKG;IACG,KAAK,CAAC,QAAQ,CAAC,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAqBjD;;OAEG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAK7C;;;;;OAKG;IACH,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;IAIlC;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;;;;GAKG;AACH,qBAAa,qBAAqB;IAChC;;OAEG;WACU,UAAU,CACrB,MAAM,EAAE,aAAa,EACrB,YAAY,EAAE,QAAQ,EACtB,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,qBAAqB,CAAC;IAM1B,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACtC,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,QAAQ,EAAE,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,OAAO,CAAC;gBAG5B,MAAM,EAAE,aAAa,EACrB,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,EACvC,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,EAC5B,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,oBAAoB;IAY5B,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxC;;OAEG;IACH,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B;;;OAGG;IACG,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAY1D;;OAEG;IACH,WAAW,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAWhE;;;OAGG;IACH,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI;CAG/B;AAED;;GAEG;AACH,qBAAa,kBAAkB,CAAC,CAAC,CAAE,SAAQ,SAAS;IAC3C,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC;gBAEpB,MAAM,EAAE,aAAa,EAAE,IAAI,GAAE,oBAAyB;IAiBlE,UAAU,CAAC,GAAG,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAS9E,MAAM,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,KAAK,IAAI;CAG3C"}

package/dist/parquetjs/file.d.ts

@@ -1,10 +0,0 @@
-/// <reference types="node" />
-import fs from 'fs';
-export declare function fopen(filePath: any): Promise<unknown>;
-export declare function fstat(filePath: any): Promise<fs.Stats>;
-export declare function fread(fd: any, position: any, length: any): Promise<unknown>;
-export declare function fclose(fd: any): Promise<unknown>;
-export declare function oswrite(os: any, buf: any): Promise<void>;
-export declare function osclose(os: any): Promise<void>;
-export declare function osopen(path: any, opts: any): Promise<unknown>;
-//# sourceMappingURL=file.d.ts.map

package/dist/parquetjs/file.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../src/parquetjs/file.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,wBAAgB,KAAK,CAAC,QAAQ,KAAA,oBAU7B;AAED,wBAAgB,KAAK,CAAC,QAAQ,KAAA,qBAU7B;AAED,wBAAgB,KAAK,CAAC,EAAE,KAAA,EAAE,QAAQ,KAAA,EAAE,MAAM,KAAA,oBAYzC;AAED,wBAAgB,MAAM,CAAC,EAAE,KAAA,oBAUxB;AAED,wBAAgB,OAAO,CAAC,EAAE,KAAA,EAAE,GAAG,KAAA,GAAG,OAAO,CAAC,IAAI,CAAC,CAU9C;AAED,wBAAgB,OAAO,CAAC,EAAE,KAAA,GAAG,OAAO,CAAC,IAAI,CAAC,CAUzC;AAED,wBAAgB,MAAM,CAAC,IAAI,KAAA,EAAE,IAAI,KAAA,oBAYhC"}

package/dist/parquetjs/file.js

@@ -1,99 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.osopen = exports.osclose = exports.oswrite = exports.fclose = exports.fread = exports.fstat = exports.fopen = void 0;
-// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-const fs_1 = __importDefault(require("fs"));
-function fopen(filePath) {
-    return new Promise((resolve, reject) => {
-        fs_1.default.open(filePath, 'r', (err, fd) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve(fd);
-            }
-        });
-    });
-}
-exports.fopen = fopen;
-function fstat(filePath) {
-    return new Promise((resolve, reject) => {
-        fs_1.default.stat(filePath, (err, stat) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve(stat);
-            }
-        });
-    });
-}
-exports.fstat = fstat;
-function fread(fd, position, length) {
-    const buffer = Buffer.alloc(length);
-    return new Promise((resolve, reject) => {
-        fs_1.default.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-            if (err || bytesRead !== length) {
-                reject(err || Error('read failed'));
-            }
-            else {
-                resolve(buf);
-            }
-        });
-    });
-}
-exports.fread = fread;
-function fclose(fd) {
-    return new Promise((resolve, reject) => {
-        fs_1.default.close(fd, (err) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve(err);
-            }
-        });
-    });
-}
-exports.fclose = fclose;
-function oswrite(os, buf) {
-    return new Promise((resolve, reject) => {
-        os.write(buf, (err) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve();
-            }
-        });
-    });
-}
-exports.oswrite = oswrite;
-function osclose(os) {
-    return new Promise((resolve, reject) => {
-        os.close((err) => {
-            if (err) {
-                reject(err);
-            }
-            else {
-                resolve();
-            }
-        });
-    });
-}
-exports.osclose = osclose;
-function osopen(path, opts) {
-    return new Promise((resolve, reject) => {
-        const outputStream = fs_1.default.createWriteStream(path, opts);
-        outputStream.on('open', function (fd) {
-            resolve(outputStream);
-        });
-        outputStream.on('error', function (err) {
-            reject(err);
-        });
-    });
-}
-exports.osopen = osopen;

package/dist/parquetjs/parser/parquet-cursor.d.ts

@@ -1,36 +0,0 @@
-import { FileMetaData } from '../parquet-thrift';
-import { ParquetEnvelopeReader } from './parquet-envelope-reader';
-import { ParquetSchema } from '../schema/schema';
-import { ParquetRecord } from '../schema/declare';
-/**
- * A parquet cursor is used to retrieve rows from a parquet file in order
- */
-export declare class ParquetCursor<T> implements AsyncIterable<T> {
-    metadata: FileMetaData;
-    envelopeReader: ParquetEnvelopeReader;
-    schema: ParquetSchema;
-    columnList: string[][];
-    rowGroup: ParquetRecord[];
-    rowGroupIndex: number;
-    /**
-     * Create a new parquet reader from the file metadata and an envelope reader.
-     * It is usually not recommended to call this constructor directly except for
-     * advanced and internal use cases. Consider using getCursor() on the
-     * ParquetReader instead
-     */
-    constructor(metadata: FileMetaData, envelopeReader: ParquetEnvelopeReader, schema: ParquetSchema, columnList: string[][]);
-    /**
-     * Retrieve the next row from the cursor. Returns a row or NULL if the end
-     * of the file was reached
-     */
-    next<T = any>(): Promise<T>;
-    /**
-     * Rewind the cursor the the beginning of the file
-     */
-    rewind(): void;
-    /**
-     * Implement AsyncIterable
-     */
-    [Symbol.asyncIterator](): AsyncIterator<T>;
-}
-//# sourceMappingURL=parquet-cursor.d.ts.map

package/dist/parquetjs/parser/parquet-cursor.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parquet-cursor.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-cursor.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAC,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAC,qBAAqB,EAAC,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAC,aAAa,EAAC,MAAM,kBAAkB,CAAC;AAC/C,OAAO,EAAC,aAAa,EAAC,MAAM,mBAAmB,CAAC;AAGhD;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,CAAE,YAAW,aAAa,CAAC,CAAC,CAAC;IAChD,QAAQ,EAAE,YAAY,CAAC;IACvB,cAAc,EAAE,qBAAqB,CAAC;IACtC,MAAM,EAAE,aAAa,CAAC;IACtB,UAAU,EAAE,MAAM,EAAE,EAAE,CAAC;IACvB,QAAQ,EAAE,aAAa,EAAE,CAAM;IAC/B,aAAa,EAAE,MAAM,CAAC;IAE7B;;;;;OAKG;gBAED,QAAQ,EAAE,YAAY,EACtB,cAAc,EAAE,qBAAqB,EACrC,MAAM,EAAE,aAAa,EACrB,UAAU,EAAE,MAAM,EAAE,EAAE;IASxB;;;OAGG;IACG,IAAI,CAAC,CAAC,GAAG,GAAG,KAAK,OAAO,CAAC,CAAC,CAAC;IAiBjC;;OAEG;IACH,MAAM,IAAI,IAAI;IAKd;;OAEG;IAEH,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;CAuB3C"}

package/dist/parquetjs/parser/parquet-cursor.js

@@ -1,74 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ParquetCursor = void 0;
-const shred_1 = require("../schema/shred");
-/**
- * A parquet cursor is used to retrieve rows from a parquet file in order
- */
-class ParquetCursor {
-    /**
-     * Create a new parquet reader from the file metadata and an envelope reader.
-     * It is usually not recommended to call this constructor directly except for
-     * advanced and internal use cases. Consider using getCursor() on the
-     * ParquetReader instead
-     */
-    constructor(metadata, envelopeReader, schema, columnList) {
-        this.rowGroup = [];
-        this.metadata = metadata;
-        this.envelopeReader = envelopeReader;
-        this.schema = schema;
-        this.columnList = columnList;
-        this.rowGroupIndex = 0;
-    }
-    /**
-     * Retrieve the next row from the cursor. Returns a row or NULL if the end
-     * of the file was reached
-     */
-    async next() {
-        if (this.rowGroup.length === 0) {
-            if (this.rowGroupIndex >= this.metadata.row_groups.length) {
-                // @ts-ignore
-                return null;
-            }
-            const rowBuffer = await this.envelopeReader.readRowGroup(this.schema, this.metadata.row_groups[this.rowGroupIndex], this.columnList);
-            this.rowGroup = (0, shred_1.materializeRecords)(this.schema, rowBuffer);
-            this.rowGroupIndex++;
-        }
-        return this.rowGroup.shift();
-    }
-    /**
-     * Rewind the cursor the the beginning of the file
-     */
-    rewind() {
-        this.rowGroup = [];
-        this.rowGroupIndex = 0;
-    }
-    /**
-     * Implement AsyncIterable
-     */
-    // tslint:disable-next-line:function-name
-    [Symbol.asyncIterator]() {
-        let done = false;
-        return {
-            next: async () => {
-                if (done) {
-                    return { done, value: null };
-                }
-                const value = await this.next();
-                if (value === null) {
-                    return { done: true, value };
-                }
-                return { done: false, value };
-            },
-            return: async () => {
-                done = true;
-                return { done, value: null };
-            },
-            throw: async () => {
-                done = true;
-                return { done: true, value: null };
-            }
-        };
-    }
-}
-exports.ParquetCursor = ParquetCursor;