@loaders.gl/parquet 3.1.0-alpha.5 → 3.1.0-beta.5

package/dist/parquetjs/utils/file-utils.js

@@ -0,0 +1,46 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.osopen = exports.osclose = exports.oswrite = exports.load = void 0;
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+const fs_1 = __importDefault(require("fs"));
+function load(name) {
+    return (module || global).require(name);
+}
+exports.load = load;
+function oswrite(os, buf) {
+    return new Promise((resolve, reject) => {
+        os.write(buf, (err) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+}
+exports.oswrite = oswrite;
+function osclose(os) {
+    return new Promise((resolve, reject) => {
+        os.close((err) => {
+            if (err) {
+                reject(err);
+            }
+            else {
+                resolve();
+            }
+        });
+    });
+}
+exports.osclose = osclose;
+function osopen(path, opts) {
+    return new Promise((resolve, reject) => {
+        const outputStream = fs_1.default.createWriteStream(path, opts);
+        outputStream.once('open', (fd) => resolve(outputStream));
+        outputStream.once('error', (err) => reject(err));
+    });
+}
+exports.osopen = osopen;

package/dist/parquetjs/utils/read-utils.d.ts

@@ -22,3 +22,4 @@ export declare function decodePageHeader(buf: Buffer, offset?: number): {
  */
 export declare function getBitWidth(val: number): number;
 export declare function fieldIndexOf(arr: string[][], elem: string[]): number;
+//# sourceMappingURL=read-utils.d.ts.map

package/dist/parquetjs/utils/read-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"read-utils.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/utils/read-utils.ts"],"names":[],"mappings":";AACA,OAAO,EAAC,YAAY,EAAE,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAM3D;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,CAYhD;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,UAWlE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAOxE;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;;;EAW9D;AAED,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM;;;EAW5D;AAED;;GAEG;AACH,wBAAgB,WAAW,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAM/C;AAKD,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,MAAM,CAmBpE"}

package/dist/parquetjs/utils/read-utils.js

@@ -0,0 +1,109 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fieldIndexOf = exports.getBitWidth = exports.decodePageHeader = exports.decodeFileMetadata = exports.getThriftEnum = exports.decodeThrift = exports.serializeThrift = void 0;
+const thrift_1 = require("thrift");
+const parquet_thrift_1 = require("../parquet-thrift");
+class UFramedTransport extends thrift_1.TFramedTransport {
+    constructor() {
+        super(...arguments);
+        this.readPos = 0;
+    }
+}
+/**
+ * Helper function that serializes a thrift object into a buffer
+ */
+function serializeThrift(obj) {
+    const output = [];
+    const transport = new thrift_1.TBufferedTransport(undefined, (buf) => {
+        output.push(buf);
+    });
+    const protocol = new thrift_1.TCompactProtocol(transport);
+    obj.write(protocol);
+    transport.flush();
+    return Buffer.concat(output);
+}
+exports.serializeThrift = serializeThrift;
+function decodeThrift(obj, buf, offset) {
+    if (!offset) {
+        // tslint:disable-next-line:no-parameter-reassignment
+        offset = 0;
+    }
+    const transport = new UFramedTransport(buf);
+    transport.readPos = offset;
+    const protocol = new thrift_1.TCompactProtocol(transport);
+    obj.read(protocol);
+    return transport.readPos - offset;
+}
+exports.decodeThrift = decodeThrift;
+/**
+ * FIXME not ideal that this is linear
+ */
+function getThriftEnum(klass, value) {
+    for (const k in klass) {
+        if (klass[k] === value) {
+            return k;
+        }
+    }
+    throw new Error('Invalid ENUM value');
+}
+exports.getThriftEnum = getThriftEnum;
+function decodeFileMetadata(buf, offset) {
+    if (!offset) {
+        // tslint:disable-next-line:no-parameter-reassignment
+        offset = 0;
+    }
+    const transport = new UFramedTransport(buf);
+    transport.readPos = offset;
+    const protocol = new thrift_1.TCompactProtocol(transport);
+    const metadata = parquet_thrift_1.FileMetaData.read(protocol);
+    return { length: transport.readPos - offset, metadata };
+}
+exports.decodeFileMetadata = decodeFileMetadata;
+function decodePageHeader(buf, offset) {
+    if (!offset) {
+        // tslint:disable-next-line:no-parameter-reassignment
+        offset = 0;
+    }
+    const transport = new UFramedTransport(buf);
+    transport.readPos = offset;
+    const protocol = new thrift_1.TCompactProtocol(transport);
+    const pageHeader = parquet_thrift_1.PageHeader.read(protocol);
+    return { length: transport.readPos - offset, pageHeader };
+}
+exports.decodePageHeader = decodePageHeader;
+/**
+ * Get the number of bits required to store a given value
+ */
+function getBitWidth(val) {
+    if (val === 0) {
+        return 0;
+        // tslint:disable-next-line:no-else-after-return
+    }
+    return Math.ceil(Math.log2(val + 1));
+}
+exports.getBitWidth = getBitWidth;
+// Supports MQTT path wildcards
+// + all immediate children
+// # all descendents
+function fieldIndexOf(arr, elem) {
+    for (let j = 0; j < arr.length; j++) {
+        if (arr[j].length > elem.length) {
+            continue; // eslint-disable-line no-continue
+        }
+        let m = true;
+        for (let i = 0; i < elem.length; i++) {
+            if (arr[j][i] === elem[i] || arr[j][i] === '+' || arr[j][i] === '#') {
+                continue; // eslint-disable-line no-continue
+            }
+            if (i >= arr[j].length && arr[j][arr[j].length - 1] === '#') {
+                continue; // eslint-disable-line no-continue
+            }
+            m = false;
+            break;
+        }
+        if (m)
+            return j;
+    }
+    return -1;
+}
+exports.fieldIndexOf = fieldIndexOf;

package/dist/workers/parquet-worker.d.ts

@@ -1 +1,2 @@
 export {};
+//# sourceMappingURL=parquet-worker.d.ts.map

package/dist/workers/parquet-worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-worker.d.ts","sourceRoot":"","sources":["../../src/workers/parquet-worker.ts"],"names":[],"mappings":""}

package/dist/workers/parquet-worker.js

@@ -0,0 +1,5 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const index_1 = require("../index");
+(0, loader_utils_1.createLoaderWorker)(index_1.ParquetLoader);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "3.1.0-alpha.5",
+  "version": "3.1.0-beta.5",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -29,9 +29,8 @@
   ],
   "scripts": {
     "pre-build": "npm run build-worker && npm run build-bundle",
-    "post-build": "tsc",
-    "build-bundle": "webpack --display=minimal --config ../../scripts/webpack/bundle.js",
-    "build-worker": "webpack --entry ./src/workers/parquet-worker.ts --output ./dist/parquet-worker.js --config ../../scripts/webpack/worker.js"
+    "build-bundle": "esbuild src/bundle.ts --outfile=dist/dist.min.js --bundle --minify --sourcemap --external:{util,fs,path,crypto}",
+    "build-worker": "esbuild src/workers/parquet-worker.ts --outfile=dist/parquet-worker.js --bundle --minify --sourcemap --external:{util,fs,path,crypto}"
   },
   "browser": {
     "child_process": false,
@@ -40,9 +39,9 @@
     "lzo": false
   },
   "dependencies": {
-    "@loaders.gl/compression": "3.1.0-alpha.5",
-    "@loaders.gl/loader-utils": "3.1.0-alpha.5",
-    "@loaders.gl/schema": "3.1.0-alpha.5",
+    "@loaders.gl/compression": "3.1.0-beta.5",
+    "@loaders.gl/loader-utils": "3.1.0-beta.5",
+    "@loaders.gl/schema": "3.1.0-beta.5",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "bson": "^1.0.4",
@@ -63,5 +62,5 @@
     "@types/thrift": "^0.10.8",
     "@types/varint": "^5.0.0"
   },
-  "gitHead": "352241dd910a8c6307a235dadbe154ca915b885b"
+  "gitHead": "5d66468457cc878d2dbff3d7807d737ceb181c9a"
 }

package/src/index.ts

@@ -22,10 +22,10 @@ export {ParquetWriter as _ParquetWriter} from './parquet-writer';
 
 export {preloadCompressions} from './parquetjs/compression';
 
-export {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';
-export {ParquetReader} from './parquetjs/parser/parquet-reader';
-export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';
 export {ParquetSchema} from './parquetjs/schema/schema';
+export {ParquetReader} from './parquetjs/parser/parquet-reader';
+export {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';
+// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';
 export {convertParquetToArrowSchema} from './lib/convert-schema';
 
 // TESTS

package/src/parquetjs/codecs/rle.ts

@@ -63,7 +63,7 @@ export function encodeValues(
   }
 
   const envelope = Buffer.alloc(buf.length + 4);
-  envelope.writeUInt32LE(buf.length, 0);
+  envelope.writeUInt32LE(buf.length, undefined);
   buf.copy(envelope, 4);
 
   return envelope;

package/src/parquetjs/compression.ts

@@ -19,23 +19,23 @@ import {toArrayBuffer, toBuffer} from './utils/buffer-utils';
 // TODO switch to worker compression to avoid bundling...
 
 // import brotli from 'brotli'; - brotli has problems with decompress in browsers
-import brotliDecompress from 'brotli/decompress';
+// import brotliDecompress from 'brotli/decompress';
 import lz4js from 'lz4js';
 import lzo from 'lzo';
-import {ZstdCodec} from 'zstd-codec';
+// import {ZstdCodec} from 'zstd-codec';
 
 // Inject large dependencies through Compression constructor options
 const modules = {
   // brotli has problems with decompress in browsers
-  brotli: {
-    decompress: brotliDecompress,
-    compress: () => {
-      throw new Error('brotli compress');
-    }
-  },
+  // brotli: {
+  //   decompress: brotliDecompress,
+  //   compress: () => {
+  //     throw new Error('brotli compress');
+  //   }
+  // },
   lz4js,
-  lzo,
-  'zstd-codec': ZstdCodec
+  lzo
+  // 'zstd-codec': ZstdCodec
 };
 
 // See https://github.com/apache/parquet-format/blob/master/Compression.md

package/src/parquetjs/parser/decoders.ts

@@ -95,7 +95,7 @@ export async function decodePage(
   options: ParquetOptions
 ): Promise<ParquetPageData> {
   let page;
-  const {pageHeader, length} = await decodePageHeader(cursor.buffer, cursor.offset);
+  const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);
   cursor.offset += length;
 
   const pageType = getThriftEnum(PageType, pageHeader.type);

package/src/parquetjs/parser/parquet-envelope-reader.ts

@@ -9,7 +9,6 @@ import {
   PrimitiveType,
   ParquetOptions
 } from '../schema/declare';
-import {fstat, fopen, fread, fclose} from '../utils/file-utils';
 import {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';
 import {decodeDataPages, decodePage} from './decoders';
 
@@ -31,16 +30,6 @@ export class ParquetEnvelopeReader {
   public fileSize: number;
   public defaultDictionarySize: number;
 
-  static async openFile(filePath: string): Promise<ParquetEnvelopeReader> {
-    const fileStat = await fstat(filePath);
-    const fileDescriptor = await fopen(filePath);
-
-    const readFn = fread.bind(undefined, fileDescriptor);
-    const closeFn = fclose.bind(undefined, fileDescriptor);
-
-    return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size);
-  }
-
   static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {
     const readFn = (position: number, length: number) =>
       Promise.resolve(buffer.slice(position, position + length));

package/src/parquetjs/parser/parquet-reader.ts

@@ -53,22 +53,6 @@ export class ParquetReader<T> implements AsyncIterable<T> {
     }
   }
 
-  /**
-   * Open the parquet file pointed to by the specified path and return a new
-   * parquet reader
-   */
-  static async openFile<T>(filePath: string): Promise<ParquetReader<T>> {
-    const envelopeReader = await ParquetEnvelopeReader.openFile(filePath);
-    try {
-      await envelopeReader.readHeader();
-      const metadata = await envelopeReader.readFooter();
-      return new ParquetReader<T>(metadata, envelopeReader);
-    } catch (err) {
-      await envelopeReader.close();
-      throw err;
-    }
-  }
-
   static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {
     const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);
     try {

package/src/parquetjs/utils/file-utils.ts

@@ -14,55 +14,6 @@ export interface WriteStreamOptions {
   start?: number;
 }
 
-export function fopen(filePath: string): Promise<number> {
-  return new Promise((resolve, reject) => {
-    fs.open(filePath, 'r', (err, fd) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(fd);
-      }
-    });
-  });
-}
-
-export function fstat(filePath: string): Promise<fs.Stats> {
-  return new Promise((resolve, reject) => {
-    fs.stat(filePath, (err, stat) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stat);
-      }
-    });
-  });
-}
-
-export function fread(fd: number, position: number, length: number): Promise<Buffer> {
-  const buffer = Buffer.alloc(length);
-  return new Promise((resolve, reject) => {
-    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-      if (err || bytesRead !== length) {
-        reject(err || Error('read failed'));
-      } else {
-        resolve(buf);
-      }
-    });
-  });
-}
-
-export function fclose(fd: number): Promise<void> {
-  return new Promise((resolve, reject) => {
-    fs.close(fd, (err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
 export function oswrite(os: Writable, buf: Buffer): Promise<void> {
   return new Promise((resolve, reject) => {
     os.write(buf, (err) => {