@loaders.gl/parquet 3.1.0-alpha.5 → 3.1.0-beta.5

package/dist/esm/parquetjs/utils/file-utils.js

@@ -2,51 +2,6 @@ import fs from 'fs';
 export function load(name) {
   return (module || global).require(name);
 }
-export function fopen(filePath) {
-  return new Promise((resolve, reject) => {
-    fs.open(filePath, 'r', (err, fd) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(fd);
-      }
-    });
-  });
-}
-export function fstat(filePath) {
-  return new Promise((resolve, reject) => {
-    fs.stat(filePath, (err, stat) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stat);
-      }
-    });
-  });
-}
-export function fread(fd, position, length) {
-  const buffer = Buffer.alloc(length);
-  return new Promise((resolve, reject) => {
-    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-      if (err || bytesRead !== length) {
-        reject(err || Error('read failed'));
-      } else {
-        resolve(buf);
-      }
-    });
-  });
-}
-export function fclose(fd) {
-  return new Promise((resolve, reject) => {
-    fs.close(fd, err => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
 export function oswrite(os, buf) {
   return new Promise((resolve, reject) => {
     os.write(buf, err => {

package/dist/esm/parquetjs/utils/file-utils.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../src/parquetjs/utils/file-utils.ts"],"names":["fs","load","name","module","global","require","fopen","filePath","Promise","resolve","reject","open","err","fd","fstat","stat","fread","position","length","buffer","Buffer","alloc","read","bytesRead","buf","Error","fclose","close","oswrite","os","write","osclose","osopen","path","opts","outputStream","createWriteStream","once"],"mappings":"AACA,OAAOA,EAAP,MAAe,IAAf;AAGA,OAAO,SAASC,IAAT,CAAcC,IAAd,EAAiC;AACtC,SAAO,CAACC,MAAM,IAAKC,MAAZ,EAA4BC,OAA5B,CAAoCH,IAApC,CAAP;AACD;AAUD,OAAO,SAASI,KAAT,CAAeC,QAAf,EAAkD;AACvD,SAAO,IAAIC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCV,IAAAA,EAAE,CAACW,IAAH,CAAQJ,QAAR,EAAkB,GAAlB,EAAuB,CAACK,GAAD,EAAMC,EAAN,KAAa;AAClC,UAAID,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO,CAACI,EAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASC,KAAT,CAAeP,QAAf,EAAoD;AACzD,SAAO,IAAIC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCV,IAAAA,EAAE,CAACe,IAAH,CAAQR,QAAR,EAAkB,CAACK,GAAD,EAAMG,IAAN,KAAe;AAC/B,UAAIH,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO,CAACM,IAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASC,KAAT,CAAeH,EAAf,EAA2BI,QAA3B,EAA6CC,MAA7C,EAA8E;AACnF,QAAMC,MAAM,GAAGC,MAAM,CAACC,KAAP,CAAaH,MAAb,CAAf;AACA,SAAO,IAAIV,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCV,IAAAA,EAAE,CAACsB,IAAH,CAAQT,EAAR,EAAYM,MAAZ,EAAoB,CAApB,EAAuBD,MAAvB,EAA+BD,QAA/B,EAAyC,CAACL,GAAD,EAAMW,SAAN,EAAiBC,GAAjB,KAAyB;AAChE,UAAIZ,GAAG,IAAIW,SAAS,KAAKL,MAAzB,EAAiC;AAC/BR,QAAAA,MAAM,CAACE,GAAG,IAAIa,KAAK,CAAC,aAAD,CAAb,CAAN;AACD,OAFD,MAEO;AACLhB,QAAAA,OAAO,CAACe,GAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASE,MAAT,CAAgBb,EAAhB,EAA2C;AAChD,SAAO,IAAIL,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCV,IAAAA,EAAE,CAAC2B,KAAH,CAASd,EAAT,EAAcD,GAAD,IAAS;AACpB,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASmB,OAAT,CAAiBC,EAAjB,EAA+BL,GAA/B,EAA2D;AAChE,SAAO,IAAIhB,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCmB,IAAAA,EAAE,CAACC,KAAH,CAASN,GAAT,EAAeZ,GAAD,IAAS;AACrB,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASsB,OAAT,CAAiBF,EAAjB,EAA8C;AACnD,SAAO,IAAIrB,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACrCmB,IAAAA,EAAD,CAAYF,KAAZ,CAAmBf,GAAD,IAAc;AAC9B,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASuB,MAAT,CAAgBC,IAAhB,EAA8BC,IAA9B,EAAkF;AACvF,SAAO,IAAI1B,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,UAAMyB,YAAY,GAAGnC,EAAE,CAACoC,iBAAH,CAAqBH,IAArB,EAA2BC,IAA3B,CAArB;AACAC,IAAAA,YAAY,CAACE,IAAb,CAAkB,MAAlB,EAA2BxB,EAAD,IAAQJ,OAAO,CAAC0B,YAAD,CAAzC;AACAA,IAAAA,YAAY,CAACE,IAAb,CAAkB,OAAlB,EAA4BzB,GAAD,IAASF,MAAM,CAACE,GAAD,CAA1C;AACD,GAJM,CAAP;AAKD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport fs from 'fs';\nimport {Writable} from 'stream';\n\nexport function load(name: string): any {\n  return (module || (global as any)).require(name);\n}\nexport interface WriteStreamOptions {\n  flags?: string;\n  encoding?: string;\n  fd?: number;\n  mode?: number;\n  autoClose?: boolean;\n  start?: number;\n}\n\nexport function fopen(filePath: string): Promise<number> {\n  return new Promise((resolve, reject) => {\n    fs.open(filePath, 'r', (err, fd) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve(fd);\n      }\n    });\n  });\n}\n\nexport function fstat(filePath: string): Promise<fs.Stats> {\n  return new Promise((resolve, reject) => {\n    fs.stat(filePath, (err, stat) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve(stat);\n      }\n    });\n  });\n}\n\nexport function fread(fd: number, position: number, length: number): Promise<Buffer> {\n  const buffer = Buffer.alloc(length);\n  return new Promise((resolve, reject) => {\n    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {\n      if (err || bytesRead !== length) {\n        reject(err || Error('read failed'));\n      } else {\n        resolve(buf);\n      }\n    });\n  });\n}\n\nexport function fclose(fd: number): Promise<void> {\n  return new Promise((resolve, reject) => {\n    fs.close(fd, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function oswrite(os: Writable, buf: Buffer): Promise<void> {\n  return new Promise((resolve, reject) => {\n    os.write(buf, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osclose(os: Writable): Promise<void> {\n  return new Promise((resolve, reject) => {\n    (os as any).close((err: any) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {\n  return new Promise((resolve, reject) => {\n    const outputStream = fs.createWriteStream(path, opts as any);\n    outputStream.once('open', (fd) => resolve(outputStream));\n    outputStream.once('error', (err) => reject(err));\n  });\n}\n"],"file":"file-utils.js"}
+{"version":3,"sources":["../../../../src/parquetjs/utils/file-utils.ts"],"names":["fs","load","name","module","global","require","oswrite","os","buf","Promise","resolve","reject","write","err","osclose","close","osopen","path","opts","outputStream","createWriteStream","once","fd"],"mappings":"AACA,OAAOA,EAAP,MAAe,IAAf;AAGA,OAAO,SAASC,IAAT,CAAcC,IAAd,EAAiC;AACtC,SAAO,CAACC,MAAM,IAAKC,MAAZ,EAA4BC,OAA5B,CAAoCH,IAApC,CAAP;AACD;AAUD,OAAO,SAASI,OAAT,CAAiBC,EAAjB,EAA+BC,GAA/B,EAA2D;AAChE,SAAO,IAAIC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCJ,IAAAA,EAAE,CAACK,KAAH,CAASJ,GAAT,EAAeK,GAAD,IAAS;AACrB,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASI,OAAT,CAAiBP,EAAjB,EAA8C;AACnD,SAAO,IAAIE,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACrCJ,IAAAA,EAAD,CAAYQ,KAAZ,CAAmBF,GAAD,IAAc;AAC9B,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASM,MAAT,CAAgBC,IAAhB,EAA8BC,IAA9B,EAAkF;AACvF,SAAO,IAAIT,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,UAAMQ,YAAY,GAAGnB,EAAE,CAACoB,iBAAH,CAAqBH,IAArB,EAA2BC,IAA3B,CAArB;AACAC,IAAAA,YAAY,CAACE,IAAb,CAAkB,MAAlB,EAA2BC,EAAD,IAAQZ,OAAO,CAACS,YAAD,CAAzC;AACAA,IAAAA,YAAY,CAACE,IAAb,CAAkB,OAAlB,EAA4BR,GAAD,IAASF,MAAM,CAACE,GAAD,CAA1C;AACD,GAJM,CAAP;AAKD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport fs from 'fs';\nimport {Writable} from 'stream';\n\nexport function load(name: string): any {\n  return (module || (global as any)).require(name);\n}\nexport interface WriteStreamOptions {\n  flags?: string;\n  encoding?: string;\n  fd?: number;\n  mode?: number;\n  autoClose?: boolean;\n  start?: number;\n}\n\nexport function oswrite(os: Writable, buf: Buffer): Promise<void> {\n  return new Promise((resolve, reject) => {\n    os.write(buf, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osclose(os: Writable): Promise<void> {\n  return new Promise((resolve, reject) => {\n    (os as any).close((err: any) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {\n  return new Promise((resolve, reject) => {\n    const outputStream = fs.createWriteStream(path, opts as any);\n    outputStream.once('open', (fd) => resolve(outputStream));\n    outputStream.once('error', (err) => reject(err));\n  });\n}\n"],"file":"file-utils.js"}

package/dist/index.d.ts

@@ -20,9 +20,9 @@ export declare const ParquetLoader: {
 };
 export { ParquetWriter as _ParquetWriter } from './parquet-writer';
 export { preloadCompressions } from './parquetjs/compression';
-export { ParquetEnvelopeReader } from './parquetjs/parser/parquet-envelope-reader';
-export { ParquetReader } from './parquetjs/parser/parquet-reader';
-export { ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer } from './parquetjs/encoder/writer';
 export { ParquetSchema } from './parquetjs/schema/schema';
+export { ParquetReader } from './parquetjs/parser/parquet-reader';
+export { ParquetEnvelopeReader } from './parquetjs/parser/parquet-envelope-reader';
 export { convertParquetToArrowSchema } from './lib/convert-schema';
 export declare const _typecheckParquetLoader: LoaderWithParser;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAI/D,OAAO,EAAC,aAAa,IAAI,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AACtE,OAAO,EAAC,YAAY,EAAE,yBAAyB,EAAC,MAAM,qBAAqB,CAAC;AAE5E,OAAO,EAAC,mBAAmB,EAAC,CAAC;AAE7B,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;;;;;;;;;;;;CAIzB,CAAC;AAIF,OAAO,EAAC,aAAa,IAAI,cAAc,EAAC,MAAM,kBAAkB,CAAC;AAIjE,OAAO,EAAC,mBAAmB,EAAC,MAAM,yBAAyB,CAAC;AAE5D,OAAO,EAAC,aAAa,EAAC,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAC,aAAa,EAAC,MAAM,mCAAmC,CAAC;AAChE,OAAO,EAAC,qBAAqB,EAAC,MAAM,4CAA4C,CAAC;AAEjF,OAAO,EAAC,2BAA2B,EAAC,MAAM,sBAAsB,CAAC;AAGjE,eAAO,MAAM,uBAAuB,EAAE,gBAAgC,CAAC"}

package/dist/index.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._typecheckParquetLoader = exports.convertParquetToArrowSchema = exports.ParquetEnvelopeReader = exports.ParquetReader = exports.ParquetSchema = exports.preloadCompressions = exports._ParquetWriter = exports.ParquetLoader = exports.ParquetWorkerLoader = void 0;
+// ParquetLoader
+const parquet_loader_1 = require("./parquet-loader");
+Object.defineProperty(exports, "ParquetWorkerLoader", { enumerable: true, get: function () { return parquet_loader_1.ParquetLoader; } });
+const parse_parquet_1 = require("./lib/parse-parquet");
+/** ParquetJS table loader */
+exports.ParquetLoader = {
+    ...parquet_loader_1.ParquetLoader,
+    parse: parse_parquet_1.parseParquet,
+    parseFileInBatches: parse_parquet_1.parseParquetFileInBatches
+};
+// ParquetWriter
+var parquet_writer_1 = require("./parquet-writer");
+Object.defineProperty(exports, "_ParquetWriter", { enumerable: true, get: function () { return parquet_writer_1.ParquetWriter; } });
+// EXPERIMENTAL - expose the internal parquetjs API
+var compression_1 = require("./parquetjs/compression");
+Object.defineProperty(exports, "preloadCompressions", { enumerable: true, get: function () { return compression_1.preloadCompressions; } });
+var schema_1 = require("./parquetjs/schema/schema");
+Object.defineProperty(exports, "ParquetSchema", { enumerable: true, get: function () { return schema_1.ParquetSchema; } });
+var parquet_reader_1 = require("./parquetjs/parser/parquet-reader");
+Object.defineProperty(exports, "ParquetReader", { enumerable: true, get: function () { return parquet_reader_1.ParquetReader; } });
+var parquet_envelope_reader_1 = require("./parquetjs/parser/parquet-envelope-reader");
+Object.defineProperty(exports, "ParquetEnvelopeReader", { enumerable: true, get: function () { return parquet_envelope_reader_1.ParquetEnvelopeReader; } });
+// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';
+var convert_schema_1 = require("./lib/convert-schema");
+Object.defineProperty(exports, "convertParquetToArrowSchema", { enumerable: true, get: function () { return convert_schema_1.convertParquetToArrowSchema; } });
+// TESTS
+exports._typecheckParquetLoader = exports.ParquetLoader;

package/dist/lib/convert-schema.d.ts

@@ -5,3 +5,4 @@ export declare const PARQUET_TYPE_MAPPING: {
     [type in ParquetType]: typeof DataType;
 };
 export declare function convertParquetToArrowSchema(parquetSchema: ParquetSchema): Schema;
+//# sourceMappingURL=convert-schema.d.ts.map

package/dist/lib/convert-schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-schema.d.ts","sourceRoot":"","sources":["../../src/lib/convert-schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAC,MAAM,4BAA4B,CAAC;AAC9D,OAAO,KAAK,EAAgC,WAAW,EAAC,MAAM,6BAA6B,CAAC;AAE5F,OAAO,EACL,MAAM,EAGN,QAAQ,EAaT,MAAM,oBAAoB,CAAC;AAE5B,eAAO,MAAM,oBAAoB,EAAE;KAAE,IAAI,IAAI,WAAW,GAAG,OAAO,QAAQ;CA+BzE,CAAC;AAEF,wBAAgB,2BAA2B,CAAC,aAAa,EAAE,aAAa,GAAG,MAAM,CAKhF"}

package/dist/lib/convert-schema.js

@@ -0,0 +1,70 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertParquetToArrowSchema = exports.PARQUET_TYPE_MAPPING = void 0;
+const schema_1 = require("@loaders.gl/schema");
+exports.PARQUET_TYPE_MAPPING = {
+    BOOLEAN: schema_1.Bool,
+    INT32: schema_1.Int32,
+    INT64: schema_1.Float64,
+    INT96: schema_1.Float64,
+    FLOAT: schema_1.Float32,
+    DOUBLE: schema_1.Float64,
+    BYTE_ARRAY: schema_1.Binary,
+    FIXED_LEN_BYTE_ARRAY: schema_1.Binary,
+    UTF8: schema_1.Utf8,
+    DATE: schema_1.Int32,
+    TIME_MILLIS: schema_1.Int64,
+    TIME_MICROS: schema_1.Int64,
+    TIMESTAMP_MILLIS: schema_1.Int64,
+    TIMESTAMP_MICROS: schema_1.Int64,
+    UINT_8: schema_1.Int32,
+    UINT_16: schema_1.Uint16,
+    UINT_32: schema_1.Uint32,
+    UINT_64: schema_1.Uint64,
+    INT_8: schema_1.Int8,
+    INT_16: schema_1.Int16,
+    INT_32: schema_1.Int32,
+    INT_64: schema_1.Int64,
+    JSON: schema_1.Binary,
+    BSON: schema_1.Binary,
+    // TODO check interal type
+    INTERVAL: schema_1.Binary,
+    DECIMAL_INT32: schema_1.Float32,
+    DECIMAL_INT64: schema_1.Float64,
+    DECIMAL_BYTE_ARRAY: schema_1.Float64,
+    DECIMAL_FIXED_LEN_BYTE_ARRAY: schema_1.Float64
+};
+function convertParquetToArrowSchema(parquetSchema) {
+    const fields = getFields(parquetSchema.schema);
+    // TODO add metadata if needed.
+    return new schema_1.Schema(fields);
+}
+exports.convertParquetToArrowSchema = convertParquetToArrowSchema;
+function getFieldMetadata(field) {
+    const metadata = new Map();
+    for (const key in field) {
+        if (key !== 'name') {
+            const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
+            metadata.set(key, value);
+        }
+    }
+    return metadata;
+}
+function getFields(schema) {
+    const fields = [];
+    for (const name in schema) {
+        const field = schema[name];
+        if (field.fields) {
+            const childField = getFields(field.fields);
+            const nestedField = new schema_1.Field(name, new schema_1.Struct(childField), field.optional);
+            fields.push(nestedField);
+        }
+        else {
+            const FieldType = exports.PARQUET_TYPE_MAPPING[field.type];
+            const metadata = getFieldMetadata(field);
+            const arrowField = new schema_1.Field(name, new FieldType(), field.optional, metadata);
+            fields.push(arrowField);
+        }
+    }
+    return fields;
+}

package/dist/lib/parse-parquet.d.ts

@@ -1,3 +1,4 @@
 import type { ParquetLoaderOptions } from '../parquet-loader';
 export declare function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions): Promise<any[][] | null>;
 export declare function parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions): AsyncGenerator<any[][], void, unknown>;
+//# sourceMappingURL=parse-parquet.d.ts.map

package/dist/lib/parse-parquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet.d.ts","sourceRoot":"","sources":["../../src/lib/parse-parquet.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,mBAAmB,CAAC;AAI5D,wBAAsB,YAAY,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,oBAAoB,2BAM1F;AAED,wBAAuB,yBAAyB,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,oBAAoB,0CAa1F"}

package/dist/lib/parse-parquet.js

@@ -0,0 +1,28 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseParquetFileInBatches = exports.parseParquet = void 0;
+const parquet_reader_1 = require("../parquetjs/parser/parquet-reader");
+async function parseParquet(arrayBuffer, options) {
+    const blob = new Blob([arrayBuffer]);
+    for await (const batch of parseParquetFileInBatches(blob, options)) {
+        return batch;
+    }
+    return null;
+}
+exports.parseParquet = parseParquet;
+async function* parseParquetFileInBatches(blob, options) {
+    const reader = await parquet_reader_1.ParquetReader.openBlob(blob);
+    const rows = [];
+    try {
+        const cursor = reader.getCursor();
+        let record;
+        while ((record = await cursor.next())) {
+            rows.push(record);
+        }
+    }
+    finally {
+        await reader.close();
+    }
+    yield rows;
+}
+exports.parseParquetFileInBatches = parseParquetFileInBatches;

package/dist/lib/read-array-buffer.d.ts

@@ -16,3 +16,4 @@ export declare function readArrayBuffer(file: Blob | ArrayBuffer | any, start: n
   });
 }
 */
+//# sourceMappingURL=read-array-buffer.d.ts.map

package/dist/lib/read-array-buffer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-array-buffer.d.ts","sourceRoot":"","sources":["../../src/lib/read-array-buffer.ts"],"names":[],"mappings":"AAEA,wBAAsB,eAAe,CACnC,IAAI,EAAE,IAAI,GAAG,WAAW,GAAG,GAAG,EAC9B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,WAAW,CAAC,CAMtB;AAED;;;;;;;;;;;;;;;;EAgBE"}

package/dist/lib/read-array-buffer.js

@@ -0,0 +1,29 @@
+"use strict";
+// Random-Access read
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readArrayBuffer = void 0;
+async function readArrayBuffer(file, start, length) {
+    if (file instanceof Blob) {
+        const slice = file.slice(start, start + length);
+        return await slice.arrayBuffer();
+    }
+    return await file.read(start, start + length);
+}
+exports.readArrayBuffer = readArrayBuffer;
+/**
+ * Read a slice of a Blob or File, without loading the entire file into memory
+ * The trick when reading File objects is to read successive "slices" of the File
+ * Per spec https://w3c.github.io/FileAPI/, slicing a File only updates the start and end fields
+ * Actually reading from file happens in `readAsArrayBuffer`
+ * @param blob to read
+ export async function readBlob(blob: Blob): Promise<ArrayBuffer> {
+  return await new Promise((resolve, reject) => {
+    const fileReader = new FileReader();
+    fileReader.onload = (event: ProgressEvent<FileReader>) =>
+      resolve(event?.target?.result as ArrayBuffer);
+    // TODO - reject with a proper Error
+    fileReader.onerror = (error: ProgressEvent<FileReader>) => reject(error);
+    fileReader.readAsArrayBuffer(blob);
+  });
+}
+*/

package/dist/parquet-loader.d.ts

@@ -20,3 +20,4 @@ export declare const ParquetLoader: {
     options: ParquetLoaderOptions;
 };
 export declare const _typecheckParquetLoader: Loader;
+//# sourceMappingURL=parquet-loader.d.ts.map

package/dist/parquet-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-loader.d.ts","sourceRoot":"","sources":["../src/parquet-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,MAAM,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAMpE,oBAAY,oBAAoB,GAAG,aAAa,GAAG;IACjD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,kBAAkB,CAAC;QAC1B,GAAG,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;CACH,CAAC;AASF,6BAA6B;AAC7B,eAAO,MAAM,aAAa;;;;;;;;;;;;CAYzB,CAAC;AAEF,eAAO,MAAM,uBAAuB,EAAE,MAAsB,CAAC"}

package/dist/parquet-loader.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._typecheckParquetLoader = exports.ParquetLoader = void 0;
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+const DEFAULT_PARQUET_LOADER_OPTIONS = {
+    parquet: {
+        type: 'object-row-table',
+        url: undefined
+    }
+};
+/** ParquetJS table loader */
+exports.ParquetLoader = {
+    name: 'Apache Parquet',
+    id: 'parquet',
+    module: 'parquet',
+    version: VERSION,
+    worker: true,
+    category: 'table',
+    extensions: ['parquet'],
+    mimeTypes: ['application/octet-stream'],
+    binary: true,
+    tests: ['PAR1', 'PARE'],
+    options: DEFAULT_PARQUET_LOADER_OPTIONS
+};
+exports._typecheckParquetLoader = exports.ParquetLoader;