@loaders.gl/parquet 3.1.0-alpha.4 → 3.1.0-beta.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "3.1.0-alpha.4",
+  "version": "3.1.0-beta.3",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -18,7 +18,7 @@
     "Parquet",
     "Apache Parquet"
   ],
-  "types": "src/index.ts",
+  "types": "dist/index.d.ts",
   "main": "dist/es5/index.js",
   "module": "dist/esm/index.js",
   "sideEffects": false,
@@ -29,8 +29,8 @@
   ],
   "scripts": {
     "pre-build": "npm run build-worker && npm run build-bundle",
-    "build-bundle": "webpack --display=minimal --config ../../scripts/webpack/bundle.js",
-    "build-worker": "webpack --entry ./src/workers/parquet-worker.ts --output ./dist/parquet-worker.js --config ../../scripts/webpack/worker.js"
+    "build-bundle": "esbuild src/bundle.ts --outfile=dist/dist.min.js --bundle --minify --sourcemap --external:{util,fs,path,crypto}",
+    "build-worker": "esbuild src/workers/parquet-worker.ts --outfile=dist/parquet-worker.js --bundle --minify --sourcemap --external:{util,fs,path,crypto}"
   },
   "browser": {
     "child_process": false,
@@ -39,9 +39,9 @@
     "lzo": false
   },
   "dependencies": {
-    "@loaders.gl/compression": "3.1.0-alpha.4",
-    "@loaders.gl/loader-utils": "3.1.0-alpha.4",
-    "@loaders.gl/schema": "3.1.0-alpha.4",
+    "@loaders.gl/compression": "3.1.0-beta.3",
+    "@loaders.gl/loader-utils": "3.1.0-beta.3",
+    "@loaders.gl/schema": "3.1.0-beta.3",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "bson": "^1.0.4",
@@ -62,5 +62,5 @@
     "@types/thrift": "^0.10.8",
     "@types/varint": "^5.0.0"
   },
-  "gitHead": "e309784af37ef9f3640c7733c7851124c72e1fa3"
+  "gitHead": "3537c382b3ea4e092b24b6f94c3edee72076039c"
 }

package/src/index.ts

@@ -22,10 +22,10 @@ export {ParquetWriter as _ParquetWriter} from './parquet-writer';
 
 export {preloadCompressions} from './parquetjs/compression';
 
-export {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';
-export {ParquetReader} from './parquetjs/parser/parquet-reader';
-export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';
 export {ParquetSchema} from './parquetjs/schema/schema';
+export {ParquetReader} from './parquetjs/parser/parquet-reader';
+export {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';
+// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';
 export {convertParquetToArrowSchema} from './lib/convert-schema';
 
 // TESTS

package/src/parquetjs/compression.ts

@@ -19,23 +19,23 @@ import {toArrayBuffer, toBuffer} from './utils/buffer-utils';
 // TODO switch to worker compression to avoid bundling...
 
 // import brotli from 'brotli'; - brotli has problems with decompress in browsers
-import brotliDecompress from 'brotli/decompress';
+// import brotliDecompress from 'brotli/decompress';
 import lz4js from 'lz4js';
 import lzo from 'lzo';
-import {ZstdCodec} from 'zstd-codec';
+// import {ZstdCodec} from 'zstd-codec';
 
 // Inject large dependencies through Compression constructor options
 const modules = {
   // brotli has problems with decompress in browsers
-  brotli: {
-    decompress: brotliDecompress,
-    compress: () => {
-      throw new Error('brotli compress');
-    }
-  },
+  // brotli: {
+  //   decompress: brotliDecompress,
+  //   compress: () => {
+  //     throw new Error('brotli compress');
+  //   }
+  // },
   lz4js,
-  lzo,
-  'zstd-codec': ZstdCodec
+  lzo
+  // 'zstd-codec': ZstdCodec
 };
 
 // See https://github.com/apache/parquet-format/blob/master/Compression.md

package/src/parquetjs/parser/decoders.ts

@@ -95,7 +95,7 @@ export async function decodePage(
   options: ParquetOptions
 ): Promise<ParquetPageData> {
   let page;
-  const {pageHeader, length} = await decodePageHeader(cursor.buffer, cursor.offset);
+  const {pageHeader, length} = decodePageHeader(cursor.buffer, cursor.offset);
   cursor.offset += length;
 
   const pageType = getThriftEnum(PageType, pageHeader.type);

package/src/parquetjs/parser/parquet-envelope-reader.ts

@@ -9,7 +9,6 @@ import {
   PrimitiveType,
   ParquetOptions
 } from '../schema/declare';
-import {fstat, fopen, fread, fclose} from '../utils/file-utils';
 import {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';
 import {decodeDataPages, decodePage} from './decoders';
 
@@ -31,16 +30,6 @@ export class ParquetEnvelopeReader {
   public fileSize: number;
   public defaultDictionarySize: number;
 
-  static async openFile(filePath: string): Promise<ParquetEnvelopeReader> {
-    const fileStat = await fstat(filePath);
-    const fileDescriptor = await fopen(filePath);
-
-    const readFn = fread.bind(undefined, fileDescriptor);
-    const closeFn = fclose.bind(undefined, fileDescriptor);
-
-    return new ParquetEnvelopeReader(readFn, closeFn, fileStat.size);
-  }
-
   static async openBuffer(buffer: Buffer): Promise<ParquetEnvelopeReader> {
     const readFn = (position: number, length: number) =>
       Promise.resolve(buffer.slice(position, position + length));

package/src/parquetjs/parser/parquet-reader.ts

@@ -53,22 +53,6 @@ export class ParquetReader<T> implements AsyncIterable<T> {
     }
   }
 
-  /**
-   * Open the parquet file pointed to by the specified path and return a new
-   * parquet reader
-   */
-  static async openFile<T>(filePath: string): Promise<ParquetReader<T>> {
-    const envelopeReader = await ParquetEnvelopeReader.openFile(filePath);
-    try {
-      await envelopeReader.readHeader();
-      const metadata = await envelopeReader.readFooter();
-      return new ParquetReader<T>(metadata, envelopeReader);
-    } catch (err) {
-      await envelopeReader.close();
-      throw err;
-    }
-  }
-
   static async openBuffer<T>(buffer: Buffer): Promise<ParquetReader<T>> {
     const envelopeReader = await ParquetEnvelopeReader.openBuffer(buffer);
     try {

package/src/parquetjs/utils/file-utils.ts

@@ -14,55 +14,6 @@ export interface WriteStreamOptions {
   start?: number;
 }
 
-export function fopen(filePath: string): Promise<number> {
-  return new Promise((resolve, reject) => {
-    fs.open(filePath, 'r', (err, fd) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(fd);
-      }
-    });
-  });
-}
-
-export function fstat(filePath: string): Promise<fs.Stats> {
-  return new Promise((resolve, reject) => {
-    fs.stat(filePath, (err, stat) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve(stat);
-      }
-    });
-  });
-}
-
-export function fread(fd: number, position: number, length: number): Promise<Buffer> {
-  const buffer = Buffer.alloc(length);
-  return new Promise((resolve, reject) => {
-    fs.read(fd, buffer, 0, length, position, (err, bytesRead, buf) => {
-      if (err || bytesRead !== length) {
-        reject(err || Error('read failed'));
-      } else {
-        resolve(buf);
-      }
-    });
-  });
-}
-
-export function fclose(fd: number): Promise<void> {
-  return new Promise((resolve, reject) => {
-    fs.close(fd, (err) => {
-      if (err) {
-        reject(err);
-      } else {
-        resolve();
-      }
-    });
-  });
-}
-
 export function oswrite(os: Writable, buf: Buffer): Promise<void> {
   return new Promise((resolve, reject) => {
     os.write(buf, (err) => {