@loaders.gl/parquet 3.1.8 → 3.2.0-alpha.3

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "3.1.8",
+  "version": "3.2.0-alpha.3",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -36,12 +36,13 @@
     "child_process": false,
     "net": false,
     "tls": false,
-    "lzo": false
+    "lzo": false,
+    "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts"
   },
   "dependencies": {
-    "@loaders.gl/compression": "3.1.8",
-    "@loaders.gl/loader-utils": "3.1.8",
-    "@loaders.gl/schema": "3.1.8",
+    "@loaders.gl/compression": "3.2.0-alpha.3",
+    "@loaders.gl/loader-utils": "3.2.0-alpha.3",
+    "@loaders.gl/schema": "3.2.0-alpha.3",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "bson": "^1.0.4",
@@ -50,17 +51,22 @@
     "lzo": "^0.4.11",
     "node-int64": "^0.4.0",
     "object-stream": "0.0.1",
+    "parquet-wasm": "^0.3.1",
     "snappyjs": "^0.6.0",
     "thrift": "^0.14.2",
     "varint": "^5.0.0",
     "zstd-codec": "^0.1"
   },
+  "peerDependencies": {
+    "apache-arrow": "*"
+  },
   "devDependencies": {
     "@types/bson": "^4.0.0",
     "@types/node": "^10.14.15",
     "@types/node-int64": "^0.4.29",
     "@types/thrift": "^0.10.8",
-    "@types/varint": "^5.0.0"
+    "@types/varint": "^5.0.0",
+    "apache-arrow": "^4.0.0"
   },
-  "gitHead": "0ef07b4e9fc20f5a882224cf241c6fd1bad898d8"
+  "gitHead": "f0d4b801efeb7094283106352ee759eccfb21f10"
 }

package/src/index.ts

@@ -2,10 +2,12 @@ import type {LoaderWithParser} from '@loaders.gl/loader-utils';
 
 // ParquetLoader
 
+import {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';
 import {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';
 import {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';
+import {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';
 
-export {ParquetWorkerLoader};
+export {ParquetWorkerLoader, ParquetWasmWorkerLoader};
 
 /** ParquetJS table loader */
 export const ParquetLoader = {
@@ -14,9 +16,15 @@ export const ParquetLoader = {
   parseFileInBatches: parseParquetFileInBatches
 };
 
+export const ParquetWasmLoader = {
+  ...ParquetWasmWorkerLoader,
+  parse: parseParquetWasm
+};
+
 // ParquetWriter
 
 export {ParquetWriter as _ParquetWriter} from './parquet-writer';
+export {ParquetWasmWriter} from './parquet-wasm-writer';
 
 // EXPERIMENTAL - expose the internal parquetjs API
 

package/src/lib/wasm/encode-parquet-wasm.ts

@@ -0,0 +1,40 @@
+import type {Table} from 'apache-arrow';
+import type {WriterOptions} from '@loaders.gl/loader-utils';
+
+import {RecordBatchStreamWriter} from 'apache-arrow';
+import {loadWasm} from './load-wasm';
+
+export type ParquetWriterOptions = WriterOptions & {
+  parquet?: {
+    wasmUrl?: string;
+  };
+};
+
+/**
+ * Encode Arrow Table to Parquet buffer
+ */
+export async function encode(table: Table, options?: ParquetWriterOptions): Promise<ArrayBuffer> {
+  const wasmUrl = options?.parquet?.wasmUrl;
+  const wasm = await loadWasm(wasmUrl);
+
+  const arrowIPCBytes = tableToIPC(table);
+  // TODO: provide options for how to write table.
+  const writerProperties = new wasm.WriterPropertiesBuilder().build();
+  const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);
+  return parquetBytes.buffer.slice(
+    parquetBytes.byteOffset,
+    parquetBytes.byteLength + parquetBytes.byteOffset
+  );
+}
+
+/**
+ * Serialize a {@link Table} to the IPC format. This function is a convenience
+ * wrapper for {@link RecordBatchStreamWriter} and {@link RecordBatchFileWriter}.
+ * Opposite of {@link tableFromIPC}.
+ *
+ * @param table The Table to serialize.
+ * @param type Whether to serialize the Table as a file or a stream.
+ */
+export function tableToIPC(table: Table): Uint8Array {
+  return RecordBatchStreamWriter.writeAll(table).toUint8Array(true);
+}

package/src/lib/wasm/load-wasm/index.ts

@@ -0,0 +1 @@
+export {loadWasm} from './load-wasm-node';

package/src/lib/wasm/load-wasm/load-wasm-browser.ts

@@ -0,0 +1,15 @@
+import * as wasmEsm from 'parquet-wasm/esm2/arrow1';
+
+let cached: typeof wasmEsm | null = null;
+
+export async function loadWasm(wasmUrl?: string) {
+  if (cached !== null) {
+    return cached;
+  }
+
+  // For ESM bundles, need to await the default export, which loads the WASM
+  await wasmEsm.default(wasmUrl);
+  cached = wasmEsm;
+
+  return wasmEsm;
+}

package/src/lib/wasm/load-wasm/load-wasm-node.ts

@@ -0,0 +1,5 @@
+import * as wasmNode from 'parquet-wasm/node/arrow1';
+
+export async function loadWasm(wasmUrl?: string) {
+  return wasmNode;
+}

package/src/lib/wasm/parse-parquet-wasm.ts

@@ -0,0 +1,42 @@
+// eslint-disable
+import type {RecordBatch} from 'apache-arrow';
+import type {LoaderOptions} from '@loaders.gl/loader-utils';
+import {Table, RecordBatchStreamReader} from 'apache-arrow';
+import {loadWasm} from './load-wasm/load-wasm-node';
+
+export type ParquetLoaderOptions = LoaderOptions & {
+  parquet?: {
+    type?: 'arrow-table';
+    wasmUrl?: string;
+  };
+};
+
+export async function parseParquet(
+  arrayBuffer: ArrayBuffer,
+  options?: ParquetLoaderOptions
+): Promise<Table> {
+  const wasmUrl = options?.parquet?.wasmUrl;
+  const wasm = await loadWasm(wasmUrl);
+
+  const arr = new Uint8Array(arrayBuffer);
+  const arrowIPCUint8Arr = wasm.readParquet(arr);
+  const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(
+    arrowIPCUint8Arr.byteOffset,
+    arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset
+  );
+  const arrowTable = tableFromIPC(arrowIPCBuffer);
+  return arrowTable;
+}
+
+/**
+ * Deserialize the IPC format into a {@link Table}. This function is a
+ * convenience wrapper for {@link RecordBatchReader}. Opposite of {@link tableToIPC}.
+ */
+function tableFromIPC(input: ArrayBuffer): Table {
+  const reader = RecordBatchStreamReader.from(input);
+  const recordBatches: RecordBatch[] = [];
+  for (const recordBatch of reader) {
+    recordBatches.push(recordBatch);
+  }
+  return new Table(recordBatches);
+}

package/src/parquet-wasm-loader.ts

@@ -0,0 +1,36 @@
+import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type ParquetLoaderOptions = LoaderOptions & {
+  parquet?: {
+    type?: 'arrow-table';
+    wasmUrl?: string;
+  };
+};
+
+const DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {
+  parquet: {
+    type: 'arrow-table',
+    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
+  }
+};
+
+/** ParquetJS table loader */
+export const ParquetWasmLoader = {
+  name: 'Apache Parquet',
+  id: 'parquet-wasm',
+  module: 'parquet',
+  version: VERSION,
+  worker: false,
+  category: 'table',
+  extensions: ['parquet'],
+  mimeTypes: ['application/octet-stream'],
+  binary: true,
+  tests: ['PAR1', 'PARE'],
+  options: DEFAULT_PARQUET_LOADER_OPTIONS
+};
+
+export const _typecheckParquetLoader: Loader = ParquetWasmLoader;

package/src/parquet-wasm-writer.ts

@@ -0,0 +1,24 @@
+import type {Writer} from '@loaders.gl/loader-utils';
+import {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+const DEFAULT_PARQUET_WRITER_OPTIONS: ParquetWriterOptions = {
+  parquet: {
+    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
+  }
+};
+
+export const ParquetWasmWriter: Writer = {
+  name: 'Apache Parquet',
+  id: 'parquet-wasm',
+  module: 'parquet',
+  version: VERSION,
+  extensions: ['parquet'],
+  mimeTypes: ['application/octet-stream'],
+  encode,
+  binary: true,
+  options: DEFAULT_PARQUET_WRITER_OPTIONS
+};