@loaders.gl/parquet 3.4.0-alpha.2 → 3.4.0-alpha.4

package/dist/parquetjs/compression.d.ts

@@ -1,6 +1,9 @@
 /// <reference types="node" />
 import { Compression } from '@loaders.gl/compression';
 import { ParquetCompression } from './schema/declare';
+/**
+ * See https://github.com/apache/parquet-format/blob/master/Compression.md
+ */
 export declare const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression>;
 /**
  * Register compressions that have big external libraries

package/dist/parquetjs/compression.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AAuCpD,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAU/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAGlF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}
+{"version":3,"file":"compression.d.ts","sourceRoot":"","sources":["../../src/parquetjs/compression.ts"],"names":[],"mappings":";AAIA,OAAO,EACL,WAAW,EAQZ,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAC,kBAAkB,EAAC,MAAM,kBAAkB,CAAC;AAsCpD;;GAEG;AAEH,eAAO,MAAM,2BAA2B,EAAE,MAAM,CAAC,kBAAkB,EAAE,WAAW,CAW/E,CAAC;AAEF;;;GAGG;AACH,wBAAsB,mBAAmB,CAAC,OAAO,CAAC,EAAE;IAAC,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,CAAA;KAAC,CAAA;CAAC,mBAGlF;AAED;;GAEG;AACH,wBAAsB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQxF;AAED;;GAEG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,kBAAkB,EAC1B,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,MAAM,CAAC,CAQjB;AAKD,wBAAgB,OAAO,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAMvF"}

package/dist/parquetjs/compression.js

@@ -24,7 +24,7 @@ function toArrayBuffer(buffer) {
 // import brotli from 'brotli'; - brotli has problems with decompress in browsers
 // import brotliDecompress from 'brotli/decompress';
 const lz4js_1 = __importDefault(require("lz4js"));
-const lzo_1 = __importDefault(require("lzo"));
+// import lzo from 'lzo';
 // import {ZstdCodec} from 'zstd-codec';
 // Inject large dependencies through Compression constructor options
 const modules = {
@@ -35,11 +35,14 @@ const modules = {
     //     throw new Error('brotli compress');
     //   }
     // },
-    lz4js: lz4js_1.default,
-    lzo: lzo_1.default
+    lz4js: lz4js_1.default
+    // lzo
     // 'zstd-codec': ZstdCodec
 };
-// See https://github.com/apache/parquet-format/blob/master/Compression.md
+/**
+ * See https://github.com/apache/parquet-format/blob/master/Compression.md
+ */
+// @ts-expect-error
 exports.PARQUET_COMPRESSION_METHODS = {
     UNCOMPRESSED: new compression_1.NoCompression(),
     GZIP: new compression_1.GZipCompression(),
@@ -48,7 +51,8 @@ exports.PARQUET_COMPRESSION_METHODS = {
     // TODO: Understand difference between LZ4 and LZ4_RAW
     LZ4: new compression_1.LZ4Compression({ modules }),
     LZ4_RAW: new compression_1.LZ4Compression({ modules }),
-    LZO: new compression_1.LZOCompression({ modules }),
+    //
+    // LZO: new LZOCompression({modules}),
     ZSTD: new compression_1.ZstdCompression({ modules })
 };
 /**

package/dist/parquetjs/schema/types.js

@@ -1,12 +1,9 @@
 "use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.fromPrimitive = exports.toPrimitive = exports.PARQUET_LOGICAL_TYPES = void 0;
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 /* eslint-disable camelcase */
-const bson_1 = __importDefault(require("bson"));
+const bson_1 = require("@loaders.gl/bson");
 exports.PARQUET_LOGICAL_TYPES = {
     BOOLEAN: {
         primitiveType: 'BOOLEAN',
@@ -304,10 +301,11 @@ function fromPrimitive_JSON(value) {
     return JSON.parse(value);
 }
 function toPrimitive_BSON(value) {
-    return Buffer.from(bson_1.default.serialize(value));
+    const arrayBuffer = bson_1.BSONWriter.encodeSync?.(value);
+    return Buffer.from(arrayBuffer);
 }
 function fromPrimitive_BSON(value) {
-    return bson_1.default.deserialize(value);
+    return bson_1.BSONLoader.parseSync?.(value);
 }
 function toPrimitive_TIME_MILLIS(value) {
     const v = parseInt(value, 10);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "3.4.0-alpha.2",
+  "version": "3.4.0-alpha.4",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -36,21 +36,19 @@
     "child_process": false,
     "net": false,
     "tls": false,
-    "lzo": false,
     "stream": false,
     "fs": false,
     "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts"
   },
   "dependencies": {
-    "@loaders.gl/compression": "3.4.0-alpha.2",
-    "@loaders.gl/loader-utils": "3.4.0-alpha.2",
-    "@loaders.gl/schema": "3.4.0-alpha.2",
+    "@loaders.gl/bson": "3.4.0-alpha.4",
+    "@loaders.gl/compression": "3.4.0-alpha.4",
+    "@loaders.gl/loader-utils": "3.4.0-alpha.4",
+    "@loaders.gl/schema": "3.4.0-alpha.4",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
-    "bson": "^1.0.4",
     "int53": "^0.2.4",
     "lz4js": "^0.2.0",
-    "lzo": "^0.4.11",
     "node-int64": "^0.4.0",
     "object-stream": "0.0.1",
     "parquet-wasm": "^0.3.1",
@@ -63,12 +61,11 @@
     "apache-arrow": "^4.0.0"
   },
   "devDependencies": {
-    "@types/bson": "^4.0.0",
     "@types/node": "^10.14.15",
     "@types/node-int64": "^0.4.29",
     "@types/thrift": "^0.10.8",
     "@types/varint": "^5.0.0",
     "apache-arrow": "^4.0.0"
   },
-  "gitHead": "f1c00c124d8d0c41a138ff40afb0d1a00711bf2e"
+  "gitHead": "f039a523df8b908a91f26c5ba8c4cfc1924b6140"
 }

package/src/parquetjs/compression.ts

@@ -8,7 +8,7 @@ import {
   GZipCompression,
   SnappyCompression,
   BrotliCompression,
-  LZOCompression,
+  // LZOCompression,
   LZ4Compression,
   ZstdCompression
 } from '@loaders.gl/compression';
@@ -34,7 +34,7 @@ function toArrayBuffer(buffer: Buffer): ArrayBuffer {
 // import brotli from 'brotli'; - brotli has problems with decompress in browsers
 // import brotliDecompress from 'brotli/decompress';
 import lz4js from 'lz4js';
-import lzo from 'lzo';
+// import lzo from 'lzo';
 // import {ZstdCodec} from 'zstd-codec';
 
 // Inject large dependencies through Compression constructor options
@@ -46,12 +46,15 @@ const modules = {
   //     throw new Error('brotli compress');
   //   }
   // },
-  lz4js,
-  lzo
+  lz4js
+  // lzo
   // 'zstd-codec': ZstdCodec
 };
 
-// See https://github.com/apache/parquet-format/blob/master/Compression.md
+/**
+ * See https://github.com/apache/parquet-format/blob/master/Compression.md
+ */
+// @ts-expect-error
 export const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {
   UNCOMPRESSED: new NoCompression(),
   GZIP: new GZipCompression(),
@@ -60,7 +63,8 @@ export const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression
   // TODO: Understand difference between LZ4 and LZ4_RAW
   LZ4: new LZ4Compression({modules}),
   LZ4_RAW: new LZ4Compression({modules}),
-  LZO: new LZOCompression({modules}),
+  //
+  // LZO: new LZOCompression({modules}),
   ZSTD: new ZstdCompression({modules})
 };
 

package/src/parquetjs/schema/types.ts

@@ -1,6 +1,6 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 /* eslint-disable camelcase */
-import BSON from 'bson';
+import {BSONLoader, BSONWriter} from '@loaders.gl/bson';
 import {OriginalType, ParquetField, ParquetType, PrimitiveType} from './declare';
 
 export interface ParquetTypeKit {
@@ -340,11 +340,12 @@ function fromPrimitive_JSON(value: any): unknown {
 }
 
 function toPrimitive_BSON(value: any): Buffer {
-  return Buffer.from(BSON.serialize(value));
+  const arrayBuffer = BSONWriter.encodeSync?.(value) as ArrayBuffer;
+  return Buffer.from(arrayBuffer);
 }
 
 function fromPrimitive_BSON(value: any) {
-  return BSON.deserialize(value);
+  return BSONLoader.parseSync?.(value);
 }
 
 function toPrimitive_TIME_MILLIS(value: any) {