@loaders.gl/parquet 4.0.0-alpha.4 → 4.0.0-alpha.5

package/dist/parquetjs/schema/types.d.ts

@@ -0,0 +1,20 @@
+import { OriginalType, ParquetField, ParquetType, PrimitiveType } from './declare';
+export interface ParquetTypeKit {
+    primitiveType: PrimitiveType;
+    originalType?: OriginalType;
+    typeLength?: number;
+    toPrimitive: Function;
+    fromPrimitive?: Function;
+}
+export declare const PARQUET_LOGICAL_TYPES: Record<ParquetType, ParquetTypeKit>;
+/**
+ * Convert a value from it's native representation to the internal/underlying
+ * primitive type
+ */
+export declare function toPrimitive(type: ParquetType, value: any, field?: ParquetField): any;
+/**
+ * Convert a value from it's internal/underlying primitive representation to
+ * the native representation
+ */
+export declare function fromPrimitive(type: ParquetType, value: any, field?: ParquetField): any;
+//# sourceMappingURL=types.d.ts.map

package/dist/parquetjs/schema/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/schema/types.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAC,MAAM,WAAW,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,aAAa,EAAE,aAAa,CAAC;IAC7B,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,QAAQ,CAAC;IACtB,aAAa,CAAC,EAAE,QAAQ,CAAC;CAC1B;AAED,eAAO,MAAM,qBAAqB,EAAE,MAAM,CAAC,WAAW,EAAE,cAAc,CAuJrE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,YAAY,OAM9E;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,CAAC,EAAE,YAAY,OAUhF"}

package/dist/parquetjs/utils/buffer-utils.d.ts

@@ -0,0 +1,10 @@
+/// <reference types="node" />
+/**
+ * Convert Buffer to ArrayBuffer
+ */
+export declare function toArrayBuffer(buffer: Buffer): ArrayBuffer;
+/**
+ * Convert (copy) ArrayBuffer to Buffer
+ */
+export declare function toBuffer(arrayBuffer: ArrayBuffer): Buffer;
+//# sourceMappingURL=buffer-utils.d.ts.map

package/dist/parquetjs/utils/buffer-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buffer-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/buffer-utils.ts"],"names":[],"mappings":";AAAA;;GAEG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAOzD;AAED;;GAEG;AACH,wBAAgB,QAAQ,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,CAEzD"}

package/dist/parquetjs/utils/file-utils.d.ts

@@ -0,0 +1,16 @@
+/// <reference types="node" />
+import fs from 'fs';
+import { Writable } from 'stream';
+export declare function load(name: string): any;
+export interface WriteStreamOptions {
+    flags?: string;
+    encoding?: string;
+    fd?: number;
+    mode?: number;
+    autoClose?: boolean;
+    start?: number;
+}
+export declare function oswrite(os: Writable, buf: Buffer): Promise<void>;
+export declare function osclose(os: Writable): Promise<void>;
+export declare function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream>;
+//# sourceMappingURL=file-utils.d.ts.map

package/dist/parquetjs/utils/file-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,MAAM,IAAI,CAAC;AACpB,OAAO,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAEhC,wBAAgB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,CAEtC;AACD,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUhE;AAED,wBAAgB,OAAO,CAAC,EAAE,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAUnD;AAED,wBAAgB,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,EAAE,CAAC,WAAW,CAAC,CAMvF"}

package/dist/parquetjs/utils/file-utils.js

@@ -2,51 +2,6 @@ import fs from 'fs';
 export function load(name) {
   return (module || global).require(name);
 }
-export function fopen(filePath) {
-  return new Promise((resolve, reject) => {
-    fs.open(filePath, 'r', (err, fd) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(fd);
-      }
-    });
-  });
-}
-export function fstat(filePath) {
-  return new Promise((resolve, reject) => {
-    fs.stat(filePath, (err, stat) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stat);
-      }
-    });
-  });
-}
-export function fread(fd, position, length) {
-  const buffer = Buffer.alloc(length);
-  return new Promise((resolve, reject) => {
-    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-      if (err || bytesRead !== length) {
-        reject(err || Error('read failed'));
-      } else {
-        resolve(buf);
-      }
-    });
-  });
-}
-export function fclose(fd) {
-  return new Promise((resolve, reject) => {
-    fs.close(fd, err => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
 export function oswrite(os, buf) {
   return new Promise((resolve, reject) => {
     os.write(buf, err => {

package/dist/parquetjs/utils/file-utils.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":["fs","load","name","module","global","require","fopen","filePath","Promise","resolve","reject","open","err","fd","fstat","stat","fread","position","length","buffer","Buffer","alloc","read","bytesRead","buf","Error","fclose","close","oswrite","os","write","osclose","osopen","path","opts","outputStream","createWriteStream","once"],"mappings":"AACA,OAAOA,EAAP,MAAe,IAAf;AAGA,OAAO,SAASC,IAAT,CAAcC,IAAd,EAAiC;AACtC,SAAO,CAACC,MAAM,IAAKC,MAAZ,EAA4BC,OAA5B,CAAoCH,IAApC,CAAP;AACD;AAUD,OAAO,SAASI,KAAT,CAAeC,QAAf,EAAkD;AACvD,SAAO,IAAIC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCV,IAAAA,EAAE,CAACW,IAAH,CAAQJ,QAAR,EAAkB,GAAlB,EAAuB,CAACK,GAAD,EAAMC,EAAN,KAAa;AAClC,UAAID,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO,CAACI,EAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASC,KAAT,CAAeP,QAAf,EAAoD;AACzD,SAAO,IAAIC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCV,IAAAA,EAAE,CAACe,IAAH,CAAQR,QAAR,EAAkB,CAACK,GAAD,EAAMG,IAAN,KAAe;AAC/B,UAAIH,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO,CAACM,IAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASC,KAAT,CAAeH,EAAf,EAA2BI,QAA3B,EAA6CC,MAA7C,EAA8E;AACnF,QAAMC,MAAM,GAAGC,MAAM,CAACC,KAAP,CAAaH,MAAb,CAAf;AACA,SAAO,IAAIV,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCV,IAAAA,EAAE,CAACsB,IAAH,CAAQT,EAAR,EAAYM,MAAZ,EAAoB,CAApB,EAAuBD,MAAvB,EAA+BD,QAA/B,EAAyC,CAACL,GAAD,EAAMW,SAAN,EAAiBC,GAAjB,KAAyB;AAChE,UAAIZ,GAAG,IAAIW,SAAS,KAAKL,MAAzB,EAAiC;AAC/BR,QAAAA,MAAM,CAACE,GAAG,IAAIa,KAAK,CAAC,aAAD,CAAb,CAAN;AACD,OAFD,MAEO;AACLhB,QAAAA,OAAO,CAACe,GAAD,CAAP;AACD;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASE,MAAT,CAAgBb,EAAhB,EAA2C;AAChD,SAAO,IAAIL,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCV,IAAAA,EAAE,CAAC2B,KAAH,CAASd,EAAT,EAAcD,GAAD,IAAS;AACpB,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASmB,OAAT,CAAiBC,EAAjB,EAA+BL,GAA/B,EAA2D;AAChE,SAAO,IAAIhB,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCmB,IAAAA,EAAE,CAACC,KAAH,CAASN,GAAT,EAAeZ,GAAD,IAAS;AACrB,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASsB,OAAT,CAAiBF,EAAjB,EAA8C;AACnD,SAAO,IAAIrB,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACrCmB,IAAAA,EAAD,CAAYF,KAAZ,CAAmBf,GAAD,IAAc;AAC9B,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASuB,MAAT,CAAgBC,IAAhB,EAA8BC,IAA9B,EAAkF;AACvF,SAAO,IAAI1B,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,UAAMyB,YAAY,GAAGnC,EAAE,CAACoC,iBAAH,CAAqBH,IAArB,EAA2BC,IAA3B,CAArB;AACAC,IAAAA,YAAY,CAACE,IAAb,CAAkB,MAAlB,EAA2BxB,EAAD,IAAQJ,OAAO,CAAC0B,YAAD,CAAzC;AACAA,IAAAA,YAAY,CAACE,IAAb,CAAkB,OAAlB,EAA4BzB,GAAD,IAASF,MAAM,CAACE,GAAD,CAA1C;AACD,GAJM,CAAP;AAKD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport fs from 'fs';\nimport {Writable} from 'stream';\n\nexport function load(name: string): any {\n  return (module || (global as any)).require(name);\n}\nexport interface WriteStreamOptions {\n  flags?: string;\n  encoding?: string;\n  fd?: number;\n  mode?: number;\n  autoClose?: boolean;\n  start?: number;\n}\n\nexport function fopen(filePath: string): Promise<number> {\n  return new Promise((resolve, reject) => {\n    fs.open(filePath, 'r', (err, fd) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve(fd);\n      }\n    });\n  });\n}\n\nexport function fstat(filePath: string): Promise<fs.Stats> {\n  return new Promise((resolve, reject) => {\n    fs.stat(filePath, (err, stat) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve(stat);\n      }\n    });\n  });\n}\n\nexport function fread(fd: number, position: number, length: number): Promise<Buffer> {\n  const buffer = Buffer.alloc(length);\n  return new Promise((resolve, reject) => {\n    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {\n      if (err || bytesRead !== length) {\n        reject(err || Error('read failed'));\n      } else {\n        resolve(buf);\n      }\n    });\n  });\n}\n\nexport function fclose(fd: number): Promise<void> {\n  return new Promise((resolve, reject) => {\n    fs.close(fd, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function oswrite(os: Writable, buf: Buffer): Promise<void> {\n  return new Promise((resolve, reject) => {\n    os.write(buf, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osclose(os: Writable): Promise<void> {\n  return new Promise((resolve, reject) => {\n    (os as any).close((err: any) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {\n  return new Promise((resolve, reject) => {\n    const outputStream = fs.createWriteStream(path, opts as any);\n    outputStream.once('open', (fd) => resolve(outputStream));\n    outputStream.once('error', (err) => reject(err));\n  });\n}\n"],"file":"file-utils.js"}
+{"version":3,"sources":["../../../src/parquetjs/utils/file-utils.ts"],"names":["fs","load","name","module","global","require","oswrite","os","buf","Promise","resolve","reject","write","err","osclose","close","osopen","path","opts","outputStream","createWriteStream","once","fd"],"mappings":"AACA,OAAOA,EAAP,MAAe,IAAf;AAGA,OAAO,SAASC,IAAT,CAAcC,IAAd,EAAiC;AACtC,SAAO,CAACC,MAAM,IAAKC,MAAZ,EAA4BC,OAA5B,CAAoCH,IAApC,CAAP;AACD;AAUD,OAAO,SAASI,OAAT,CAAiBC,EAAjB,EAA+BC,GAA/B,EAA2D;AAChE,SAAO,IAAIC,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtCJ,IAAAA,EAAE,CAACK,KAAH,CAASJ,GAAT,EAAeK,GAAD,IAAS;AACrB,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASI,OAAT,CAAiBP,EAAjB,EAA8C;AACnD,SAAO,IAAIE,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACrCJ,IAAAA,EAAD,CAAYQ,KAAZ,CAAmBF,GAAD,IAAc;AAC9B,UAAIA,GAAJ,EAAS;AACPF,QAAAA,MAAM,CAACE,GAAD,CAAN;AACD,OAFD,MAEO;AACLH,QAAAA,OAAO;AACR;AACF,KAND;AAOD,GARM,CAAP;AASD;AAED,OAAO,SAASM,MAAT,CAAgBC,IAAhB,EAA8BC,IAA9B,EAAkF;AACvF,SAAO,IAAIT,OAAJ,CAAY,CAACC,OAAD,EAAUC,MAAV,KAAqB;AACtC,UAAMQ,YAAY,GAAGnB,EAAE,CAACoB,iBAAH,CAAqBH,IAArB,EAA2BC,IAA3B,CAArB;AACAC,IAAAA,YAAY,CAACE,IAAb,CAAkB,MAAlB,EAA2BC,EAAD,IAAQZ,OAAO,CAACS,YAAD,CAAzC;AACAA,IAAAA,YAAY,CAACE,IAAb,CAAkB,OAAlB,EAA4BR,GAAD,IAASF,MAAM,CAACE,GAAD,CAA1C;AACD,GAJM,CAAP;AAKD","sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\nimport fs from 'fs';\nimport {Writable} from 'stream';\n\nexport function load(name: string): any {\n  return (module || (global as any)).require(name);\n}\nexport interface WriteStreamOptions {\n  flags?: string;\n  encoding?: string;\n  fd?: number;\n  mode?: number;\n  autoClose?: boolean;\n  start?: number;\n}\n\nexport function oswrite(os: Writable, buf: Buffer): Promise<void> {\n  return new Promise((resolve, reject) => {\n    os.write(buf, (err) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osclose(os: Writable): Promise<void> {\n  return new Promise((resolve, reject) => {\n    (os as any).close((err: any) => {\n      if (err) {\n        reject(err);\n      } else {\n        resolve();\n      }\n    });\n  });\n}\n\nexport function osopen(path: string, opts?: WriteStreamOptions): Promise<fs.WriteStream> {\n  return new Promise((resolve, reject) => {\n    const outputStream = fs.createWriteStream(path, opts as any);\n    outputStream.once('open', (fd) => resolve(outputStream));\n    outputStream.once('error', (err) => reject(err));\n  });\n}\n"],"file":"file-utils.js"}

package/dist/parquetjs/utils/read-utils.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+import { FileMetaData, PageHeader } from '../parquet-thrift';
+/**
+ * Helper function that serializes a thrift object into a buffer
+ */
+export declare function serializeThrift(obj: any): Buffer;
+export declare function decodeThrift(obj: any, buf: Buffer, offset?: number): number;
+/**
+ * FIXME not ideal that this is linear
+ */
+export declare function getThriftEnum(klass: any, value: number | string): string;
+export declare function decodeFileMetadata(buf: Buffer, offset?: number): {
+    length: number;
+    metadata: FileMetaData;
+};
+export declare function decodePageHeader(buf: Buffer, offset?: number): {
+    length: number;
+    pageHeader: PageHeader;
+};
+/**
+ * Get the number of bits required to store a given value
+ */
+export declare function getBitWidth(val: number): number;
+export declare function fieldIndexOf(arr: string[][], elem: string[]): number;
+//# sourceMappingURL=read-utils.d.ts.map

package/dist/parquetjs/utils/read-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/read-utils.ts"],"names":[],"mappings":";AACA,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAM3D;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,CAYhD;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,UAWlE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAOxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;;;EAW9D;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;;;EAW5D;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAM/C;AAKD,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAmBpE"}

package/dist/workers/parquet-worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=parquet-worker.d.ts.map

package/dist/workers/parquet-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-worker.d.ts","sourceRoot":"","sources":["../../src/workers/parquet-worker.ts"],"names":[],"mappings":""}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "4.0.0-alpha.4",
+  "version": "4.0.0-alpha.5",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -18,7 +18,7 @@
     "Parquet",
     "Apache Parquet"
   ],
-  "types": "src/index.ts",
+  "types": "dist/index.d.ts",
   "main": "dist/index.js",
   "module": "dist/index.js",
   "sideEffects": false,
@@ -29,8 +29,8 @@
   ],
   "scripts": {
     "pre-build": "npm run build-worker && npm run build-bundle",
-    "build-bundle": "webpack --display=minimal --config ../../scripts/webpack/bundle.js",
-    "build-worker": "webpack --entry ./src/workers/parquet-worker.ts --output ./dist/parquet-worker.js --config ../../scripts/webpack/worker.js"
+    "build-bundle": "esbuild src/bundle.ts --outfile=dist/dist.min.js --bundle --minify --sourcemap --external:{util,fs,path,crypto}",
+    "build-worker": "esbuild src/workers/parquet-worker.ts --outfile=dist/parquet-worker.js --bundle --minify --sourcemap --external:{util,fs,path,crypto} --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "browser": {
     "child_process": false,
@@ -39,9 +39,9 @@
     "lzo": false
   },
   "dependencies": {
-    "@loaders.gl/compression": "4.0.0-alpha.4",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.4",
-    "@loaders.gl/schema": "4.0.0-alpha.4",
+    "@loaders.gl/compression": "4.0.0-alpha.5",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.5",
+    "@loaders.gl/schema": "4.0.0-alpha.5",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "bson": "^1.0.4",
@@ -62,5 +62,5 @@
     "@types/thrift": "^0.10.8",
     "@types/varint": "^5.0.0"
   },
-  "gitHead": "53026061b3c8871f7e96d3a5826125cc6613bddc"
+  "gitHead": "7a71a54bdf1ddf985cc3af3db90b82e7fa97d025"
 }

package/src/parquetjs/compression.ts

@@ -19,23 +19,23 @@ import {toArrayBuffer, toBuffer} from './utils/buffer-utils';
 // TODO switch to worker compression to avoid bundling...
 
 // import brotli from 'brotli'; - brotli has problems with decompress in browsers
-import brotliDecompress from 'brotli/decompress';
+// import brotliDecompress from 'brotli/decompress';
 import lz4js from 'lz4js';
 import lzo from 'lzo';
-import {ZstdCodec} from 'zstd-codec';
+// import {ZstdCodec} from 'zstd-codec';
 
 // Inject large dependencies through Compression constructor options
 const modules = {
   // brotli has problems with decompress in browsers
-  brotli: {
-    decompress: brotliDecompress,
-    compress: () => {
-      throw new Error('brotli compress');
-    }
-  },
+  // brotli: {
+  //   decompress: brotliDecompress,
+  //   compress: () => {
+  //     throw new Error('brotli compress');
+  //   }
+  // },
   lz4js,
-  lzo,
-  'zstd-codec': ZstdCodec
+  lzo
+  // 'zstd-codec': ZstdCodec
 };
 
 // See https://github.com/apache/parquet-format/blob/master/Compression.md

package/src/parquetjs/parser/parquet-envelope-reader.ts

@@ -9,7 +9,6 @@ import {
   PrimitiveType,
   ParquetOptions
 } from '../schema/declare';
-import {fstat, fopen, fread, fclose} from '../utils/file-utils';
 import {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';
 import {decodeDataPages, decodePage} from './decoders';
 
@@ -31,16 +30,6 @@ export class ParquetEnvelopeReader {
   public fileSize: number;
   public defaultDictionarySize: number;
 
-  static async openFile(filePath: string): Promise<ParquetEnvelopeReader> {
-    const fileStat = await fstat(filePath);
-    const fileDescriptor = await fopen(filePath);
-
-    const readFn = fread.bind(undefined, fileDescriptor);
-    const closeFn = fclose.bind(undefined, fileDescriptor);
-
-    return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size);
-  }
-
   static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {
     const readFn = (position: number, length: number) =>
       Promise.resolve(buffer.slice(position, position + length));

package/src/parquetjs/parser/parquet-reader.ts

@@ -53,22 +53,6 @@ export class ParquetReader<T> implements AsyncIterable<T> {
     }
   }
 
-  /**
-   * Open the parquet file pointed to by the specified path and return a new
-   * parquet reader
-   */
-  static async openFile<T>(filePath: string): Promise<ParquetReader<T>> {
-    const envelopeReader = await ParquetEnvelopeReader.openFile(filePath);
-    try {
-      await envelopeReader.readHeader();
-      const metadata = await envelopeReader.readFooter();
-      return new ParquetReader<T>(metadata, envelopeReader);
-    } catch (err) {
-      await envelopeReader.close();
-      throw err;
-    }
-  }
-
   static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {
     const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);
     try {

package/src/parquetjs/utils/file-utils.ts

@@ -14,55 +14,6 @@ export interface WriteStreamOptions {
   start?: number;
 }
 
-export function fopen(filePath: string): Promise<number> {
-  return new Promise((resolve, reject) => {
-    fs.open(filePath, 'r', (err, fd) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(fd);
-      }
-    });
-  });
-}
-
-export function fstat(filePath: string): Promise<fs.Stats> {
-  return new Promise((resolve, reject) => {
-    fs.stat(filePath, (err, stat) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stat);
-      }
-    });
-  });
-}
-
-export function fread(fd: number, position: number, length: number): Promise<Buffer> {
-  const buffer = Buffer.alloc(length);
-  return new Promise((resolve, reject) => {
-    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-      if (err || bytesRead !== length) {
-        reject(err || Error('read failed'));
-      } else {
-        resolve(buf);
-      }
-    });
-  });
-}
-
-export function fclose(fd: number): Promise<void> {
-  return new Promise((resolve, reject) => {
-    fs.close(fd, (err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
 export function oswrite(os: Writable, buf: Buffer): Promise<void> {
   return new Promise((resolve, reject) => {
     os.write(buf, (err) => {