@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.7

package/dist/esm/lib/parsers/parse-parquet-to-rows.js

@@ -0,0 +1,29 @@
+import { makeReadableFile } from '@loaders.gl/loader-utils';
+import { ParquetReader } from '../../parquetjs/parser/parquet-reader';
+export async function parseParquet(arrayBuffer, options) {
+  const blob = new Blob([arrayBuffer]);
+  const rows = [];
+  for await (const batch of parseParquetFileInBatches(blob, options)) {
+    for (const row of batch.data) {
+      rows.push(row);
+    }
+  }
+  return {
+    shape: 'object-row-table',
+    data: rows
+  };
+}
+export async function* parseParquetFileInBatches(blob, options) {
+  const file = makeReadableFile(blob);
+  const reader = new ParquetReader(file);
+  const rowBatches = reader.rowBatchIterator(options === null || options === void 0 ? void 0 : options.parquet);
+  for await (const rows of rowBatches) {
+    yield {
+      shape: 'object-row-table',
+      data: rows,
+      batchType: 'data',
+      length: rows.length
+    };
+  }
+}
+//# sourceMappingURL=parse-parquet-to-rows.js.map

package/dist/esm/lib/parsers/parse-parquet-to-rows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-to-rows.js","names":["makeReadableFile","ParquetReader","parseParquet","arrayBuffer","options","blob","Blob","rows","batch","parseParquetFileInBatches","row","data","push","shape","file","reader","rowBatches","rowBatchIterator","parquet","batchType","length"],"sources":["../../../../src/lib/parsers/parse-parquet-to-rows.ts"],"sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n// import {ColumnarTableBatch} from '@loaders.gl/schema';\nimport {makeReadableFile} from '@loaders.gl/loader-utils';\nimport {ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\nimport type {ParquetRow} from '../../parquetjs/schema/declare';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(\n  arrayBuffer: ArrayBuffer,\n  options?: ParquetLoaderOptions\n): Promise<ObjectRowTable> {\n  const blob = new Blob([arrayBuffer]);\n\n  const rows: ParquetRow[] = [];\n  for await (const batch of parseParquetFileInBatches(blob, options)) {\n    // we have only one input batch so return\n    for (const row of batch.data) {\n      rows.push(row);\n    }\n  }\n\n  return {\n    shape: 'object-row-table',\n    // TODO - spread can fail for very large number of batches\n    data: rows\n  };\n}\n\nexport async function* parseParquetFileInBatches(\n  blob: Blob,\n  options?: ParquetLoaderOptions\n): AsyncIterable<ObjectRowTableBatch> {\n  const file = makeReadableFile(blob);\n  const reader = new ParquetReader(file);\n  const rowBatches = reader.rowBatchIterator(options?.parquet);\n  for await (const rows of rowBatches) {\n    yield {\n      shape: 'object-row-table',\n      data: rows,\n      batchType: 'data',\n      length: rows.length\n    };\n  }\n}\n"],"mappings":"AAEA,SAAQA,gBAAgB,QAAO,0BAA0B;AAIzD,SAAQC,aAAa,QAAO,uCAAuC;AAEnE,OAAO,eAAeC,YAAYA,CAChCC,WAAwB,EACxBC,OAA8B,EACL;EACzB,MAAMC,IAAI,GAAG,IAAIC,IAAI,CAAC,CAACH,WAAW,CAAC,CAAC;EAEpC,MAAMI,IAAkB,GAAG,EAAE;EAC7B,WAAW,MAAMC,KAAK,IAAIC,yBAAyB,CAACJ,IAAI,EAAED,OAAO,CAAC,EAAE;IAElE,KAAK,MAAMM,GAAG,IAAIF,KAAK,CAACG,IAAI,EAAE;MAC5BJ,IAAI,CAACK,IAAI,CAACF,GAAG,CAAC;IAChB;EACF;EAEA,OAAO;IACLG,KAAK,EAAE,kBAAkB;IAEzBF,IAAI,EAAEJ;EACR,CAAC;AACH;AAEA,OAAO,gBAAgBE,yBAAyBA,CAC9CJ,IAAU,EACVD,OAA8B,EACM;EACpC,MAAMU,IAAI,GAAGd,gBAAgB,CAACK,IAAI,CAAC;EACnC,MAAMU,MAAM,GAAG,IAAId,aAAa,CAACa,IAAI,CAAC;EACtC,MAAME,UAAU,GAAGD,MAAM,CAACE,gBAAgB,CAACb,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEc,OAAO,CAAC;EAC5D,WAAW,MAAMX,IAAI,IAAIS,UAAU,EAAE;IACnC,MAAM;MACJH,KAAK,EAAE,kBAAkB;MACzBF,IAAI,EAAEJ,IAAI;MACVY,SAAS,EAAE,MAAM;MACjBC,MAAM,EAAEb,IAAI,CAACa;IACf,CAAC;EACH;AACF"}

package/dist/esm/lib/wasm/encode-parquet-wasm.js

@@ -0,0 +1,15 @@
+import { RecordBatchStreamWriter } from 'apache-arrow';
+import { loadWasm } from './load-wasm';
+export async function encode(table, options) {
+  var _options$parquet;
+  const wasmUrl = options === null || options === void 0 ? void 0 : (_options$parquet = options.parquet) === null || _options$parquet === void 0 ? void 0 : _options$parquet.wasmUrl;
+  const wasm = await loadWasm(wasmUrl);
+  const arrowIPCBytes = tableToIPC(table);
+  const writerProperties = new wasm.WriterPropertiesBuilder().build();
+  const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);
+  return parquetBytes.buffer.slice(parquetBytes.byteOffset, parquetBytes.byteLength + parquetBytes.byteOffset);
+}
+export function tableToIPC(table) {
+  return RecordBatchStreamWriter.writeAll(table).toUint8Array(true);
+}
+//# sourceMappingURL=encode-parquet-wasm.js.map

package/dist/esm/lib/wasm/encode-parquet-wasm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"encode-parquet-wasm.js","names":["RecordBatchStreamWriter","loadWasm","encode","table","options","_options$parquet","wasmUrl","parquet","wasm","arrowIPCBytes","tableToIPC","writerProperties","WriterPropertiesBuilder","build","parquetBytes","writeParquet","buffer","slice","byteOffset","byteLength","writeAll","toUint8Array"],"sources":["../../../../src/lib/wasm/encode-parquet-wasm.ts"],"sourcesContent":["import type {Table} from 'apache-arrow';\nimport type {WriterOptions} from '@loaders.gl/loader-utils';\n\nimport {RecordBatchStreamWriter} from 'apache-arrow';\nimport {loadWasm} from './load-wasm';\n\nexport type ParquetWriterOptions = WriterOptions & {\n  parquet?: {\n    wasmUrl?: string;\n  };\n};\n\n/**\n * Encode Arrow Table to Parquet buffer\n */\nexport async function encode(table: Table, options?: ParquetWriterOptions): Promise<ArrayBuffer> {\n  const wasmUrl = options?.parquet?.wasmUrl;\n  const wasm = await loadWasm(wasmUrl);\n\n  const arrowIPCBytes = tableToIPC(table);\n  // TODO: provide options for how to write table.\n  const writerProperties = new wasm.WriterPropertiesBuilder().build();\n  const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);\n  return parquetBytes.buffer.slice(\n    parquetBytes.byteOffset,\n    parquetBytes.byteLength + parquetBytes.byteOffset\n  );\n}\n\n/**\n * Serialize a {@link Table} to the IPC format. This function is a convenience\n * wrapper for {@link RecordBatchStreamWriter} and {@link RecordBatchFileWriter}.\n * Opposite of {@link tableFromIPC}.\n *\n * @param table The Table to serialize.\n * @param type Whether to serialize the Table as a file or a stream.\n */\nexport function tableToIPC(table: Table): Uint8Array {\n  return RecordBatchStreamWriter.writeAll(table).toUint8Array(true);\n}\n"],"mappings":"AAGA,SAAQA,uBAAuB,QAAO,cAAc;AACpD,SAAQC,QAAQ,QAAO,aAAa;AAWpC,OAAO,eAAeC,MAAMA,CAACC,KAAY,EAAEC,OAA8B,EAAwB;EAAA,IAAAC,gBAAA;EAC/F,MAAMC,OAAO,GAAGF,OAAO,aAAPA,OAAO,wBAAAC,gBAAA,GAAPD,OAAO,CAAEG,OAAO,cAAAF,gBAAA,uBAAhBA,gBAAA,CAAkBC,OAAO;EACzC,MAAME,IAAI,GAAG,MAAMP,QAAQ,CAACK,OAAO,CAAC;EAEpC,MAAMG,aAAa,GAAGC,UAAU,CAACP,KAAK,CAAC;EAEvC,MAAMQ,gBAAgB,GAAG,IAAIH,IAAI,CAACI,uBAAuB,CAAC,CAAC,CAACC,KAAK,CAAC,CAAC;EACnE,MAAMC,YAAY,GAAGN,IAAI,CAACO,YAAY,CAACN,aAAa,EAAEE,gBAAgB,CAAC;EACvE,OAAOG,YAAY,CAACE,MAAM,CAACC,KAAK,CAC9BH,YAAY,CAACI,UAAU,EACvBJ,YAAY,CAACK,UAAU,GAAGL,YAAY,CAACI,UACzC,CAAC;AACH;AAUA,OAAO,SAASR,UAAUA,CAACP,KAAY,EAAc;EACnD,OAAOH,uBAAuB,CAACoB,QAAQ,CAACjB,KAAK,CAAC,CAACkB,YAAY,CAAC,IAAI,CAAC;AACnE"}

package/dist/esm/lib/wasm/load-wasm/index.js

@@ -0,0 +1,2 @@
+export { loadWasm } from './load-wasm-node';
+//# sourceMappingURL=index.js.map

package/dist/esm/lib/wasm/load-wasm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":["loadWasm"],"sources":["../../../../../src/lib/wasm/load-wasm/index.ts"],"sourcesContent":["export {loadWasm} from './load-wasm-node';\n"],"mappings":"AAAA,SAAQA,QAAQ,QAAO,kBAAkB"}

package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js

@@ -0,0 +1,11 @@
+import * as wasmEsm from 'parquet-wasm/esm2/arrow1';
+let cached = null;
+export async function loadWasm(wasmUrl) {
+  if (cached !== null) {
+    return cached;
+  }
+  await wasmEsm.default(wasmUrl);
+  cached = wasmEsm;
+  return wasmEsm;
+}
+//# sourceMappingURL=load-wasm-browser.js.map

package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-wasm-browser.js","names":["wasmEsm","cached","loadWasm","wasmUrl","default"],"sources":["../../../../../src/lib/wasm/load-wasm/load-wasm-browser.ts"],"sourcesContent":["import * as wasmEsm from 'parquet-wasm/esm2/arrow1';\n\nlet cached: typeof wasmEsm | null = null;\n\nexport async function loadWasm(wasmUrl?: string) {\n  if (cached !== null) {\n    return cached;\n  }\n\n  // For ESM bundles, need to await the default export, which loads the WASM\n  await wasmEsm.default(wasmUrl);\n  cached = wasmEsm;\n\n  return wasmEsm;\n}\n"],"mappings":"AAAA,OAAO,KAAKA,OAAO,MAAM,0BAA0B;AAEnD,IAAIC,MAA6B,GAAG,IAAI;AAExC,OAAO,eAAeC,QAAQA,CAACC,OAAgB,EAAE;EAC/C,IAAIF,MAAM,KAAK,IAAI,EAAE;IACnB,OAAOA,MAAM;EACf;EAGA,MAAMD,OAAO,CAACI,OAAO,CAACD,OAAO,CAAC;EAC9BF,MAAM,GAAGD,OAAO;EAEhB,OAAOA,OAAO;AAChB"}

package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js

@@ -0,0 +1,5 @@
+import * as wasmNode from 'parquet-wasm/node/arrow1';
+export async function loadWasm(wasmUrl) {
+  return wasmNode;
+}
+//# sourceMappingURL=load-wasm-node.js.map

package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-wasm-node.js","names":["wasmNode","loadWasm","wasmUrl"],"sources":["../../../../../src/lib/wasm/load-wasm/load-wasm-node.ts"],"sourcesContent":["import * as wasmNode from 'parquet-wasm/node/arrow1';\n\nexport async function loadWasm(wasmUrl?: string) {\n  return wasmNode;\n}\n"],"mappings":"AAAA,OAAO,KAAKA,QAAQ,MAAM,0BAA0B;AAEpD,OAAO,eAAeC,QAAQA,CAACC,OAAgB,EAAE;EAC/C,OAAOF,QAAQ;AACjB"}

package/dist/esm/lib/wasm/parse-parquet-wasm.js

@@ -0,0 +1,21 @@
+import { Table as ArrowTable, RecordBatchStreamReader } from 'apache-arrow';
+import { loadWasm } from './load-wasm/load-wasm-node';
+export async function parseParquetWasm(arrayBuffer, options) {
+  var _options$parquet;
+  const wasmUrl = options === null || options === void 0 ? void 0 : (_options$parquet = options.parquet) === null || _options$parquet === void 0 ? void 0 : _options$parquet.wasmUrl;
+  const wasm = await loadWasm(wasmUrl);
+  const arr = new Uint8Array(arrayBuffer);
+  const arrowIPCUint8Arr = wasm.readParquet(arr);
+  const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(arrowIPCUint8Arr.byteOffset, arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset);
+  const arrowTable = tableFromIPC(arrowIPCBuffer);
+  return arrowTable;
+}
+function tableFromIPC(input) {
+  const reader = RecordBatchStreamReader.from(input);
+  const recordBatches = [];
+  for (const recordBatch of reader) {
+    recordBatches.push(recordBatch);
+  }
+  return new ArrowTable(recordBatches);
+}
+//# sourceMappingURL=parse-parquet-wasm.js.map

package/dist/esm/lib/wasm/parse-parquet-wasm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-wasm.js","names":["Table","ArrowTable","RecordBatchStreamReader","loadWasm","parseParquetWasm","arrayBuffer","options","_options$parquet","wasmUrl","parquet","wasm","arr","Uint8Array","arrowIPCUint8Arr","readParquet","arrowIPCBuffer","buffer","slice","byteOffset","byteLength","arrowTable","tableFromIPC","input","reader","from","recordBatches","recordBatch","push"],"sources":["../../../../src/lib/wasm/parse-parquet-wasm.ts"],"sourcesContent":["// eslint-disable\nimport type {RecordBatch} from 'apache-arrow';\nimport type {LoaderOptions} from '@loaders.gl/loader-utils';\nimport {Table as ArrowTable, RecordBatchStreamReader} from 'apache-arrow';\nimport {loadWasm} from './load-wasm/load-wasm-node';\n\nexport type ParquetWasmLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'arrow-table';\n    wasmUrl?: string;\n  };\n};\n\nexport async function parseParquetWasm(\n  arrayBuffer: ArrayBuffer,\n  options?: ParquetWasmLoaderOptions\n): Promise<ArrowTable> {\n  const wasmUrl = options?.parquet?.wasmUrl;\n  const wasm = await loadWasm(wasmUrl);\n\n  const arr = new Uint8Array(arrayBuffer);\n  const arrowIPCUint8Arr = wasm.readParquet(arr);\n  const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(\n    arrowIPCUint8Arr.byteOffset,\n    arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset\n  );\n  const arrowTable = tableFromIPC(arrowIPCBuffer);\n  return arrowTable;\n}\n\n/**\n * Deserialize the IPC format into a {@link Table}. This function is a\n * convenience wrapper for {@link RecordBatchReader}. Opposite of {@link tableToIPC}.\n */\nfunction tableFromIPC(input: ArrayBuffer): ArrowTable {\n  const reader = RecordBatchStreamReader.from(input);\n  const recordBatches: RecordBatch[] = [];\n  for (const recordBatch of reader) {\n    recordBatches.push(recordBatch);\n  }\n  return new ArrowTable(recordBatches);\n}\n"],"mappings":"AAGA,SAAQA,KAAK,IAAIC,UAAU,EAAEC,uBAAuB,QAAO,cAAc;AACzE,SAAQC,QAAQ,QAAO,4BAA4B;AASnD,OAAO,eAAeC,gBAAgBA,CACpCC,WAAwB,EACxBC,OAAkC,EACb;EAAA,IAAAC,gBAAA;EACrB,MAAMC,OAAO,GAAGF,OAAO,aAAPA,OAAO,wBAAAC,gBAAA,GAAPD,OAAO,CAAEG,OAAO,cAAAF,gBAAA,uBAAhBA,gBAAA,CAAkBC,OAAO;EACzC,MAAME,IAAI,GAAG,MAAMP,QAAQ,CAACK,OAAO,CAAC;EAEpC,MAAMG,GAAG,GAAG,IAAIC,UAAU,CAACP,WAAW,CAAC;EACvC,MAAMQ,gBAAgB,GAAGH,IAAI,CAACI,WAAW,CAACH,GAAG,CAAC;EAC9C,MAAMI,cAAc,GAAGF,gBAAgB,CAACG,MAAM,CAACC,KAAK,CAClDJ,gBAAgB,CAACK,UAAU,EAC3BL,gBAAgB,CAACM,UAAU,GAAGN,gBAAgB,CAACK,UACjD,CAAC;EACD,MAAME,UAAU,GAAGC,YAAY,CAACN,cAAc,CAAC;EAC/C,OAAOK,UAAU;AACnB;AAMA,SAASC,YAAYA,CAACC,KAAkB,EAAc;EACpD,MAAMC,MAAM,GAAGrB,uBAAuB,CAACsB,IAAI,CAACF,KAAK,CAAC;EAClD,MAAMG,aAA4B,GAAG,EAAE;EACvC,KAAK,MAAMC,WAAW,IAAIH,MAAM,EAAE;IAChCE,aAAa,CAACE,IAAI,CAACD,WAAW,CAAC;EACjC;EACA,OAAO,IAAIzB,UAAU,CAACwB,aAAa,CAAC;AACtC"}