@loaders.gl/parquet 3.2.0-alpha.2 → 3.2.0

package/dist/es5/index.js

@@ -5,6 +5,12 @@ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefau
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
+Object.defineProperty(exports, "ParquetWasmWorkerLoader", {
+  enumerable: true,
+  get: function get() {
+    return _parquetWasmLoader.ParquetWasmLoader;
+  }
+});
 Object.defineProperty(exports, "ParquetWorkerLoader", {
   enumerable: true,
   get: function get() {
@@ -17,6 +23,12 @@ Object.defineProperty(exports, "_ParquetWriter", {
     return _parquetWriter.ParquetWriter;
   }
 });
+Object.defineProperty(exports, "ParquetWasmWriter", {
+  enumerable: true,
+  get: function get() {
+    return _parquetWasmWriter.ParquetWasmWriter;
+  }
+});
 Object.defineProperty(exports, "preloadCompressions", {
   enumerable: true,
   get: function get() {
@@ -47,16 +59,22 @@ Object.defineProperty(exports, "convertParquetToArrowSchema", {
     return _convertSchema.convertParquetToArrowSchema;
   }
 });
-exports._typecheckParquetLoader = exports.ParquetLoader = void 0;
+exports._typecheckParquetLoader = exports.ParquetWasmLoader = exports.ParquetLoader = void 0;
 
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
 
+var _parquetWasmLoader = require("./parquet-wasm-loader");
+
 var _parquetLoader = require("./parquet-loader");
 
 var _parseParquet = require("./lib/parse-parquet");
 
+var _parseParquetWasm = require("./lib/wasm/parse-parquet-wasm");
+
 var _parquetWriter = require("./parquet-writer");
 
+var _parquetWasmWriter = require("./parquet-wasm-writer");
+
 var _compression = require("./parquetjs/compression");
 
 var _schema = require("./parquetjs/schema/schema");
@@ -77,6 +95,12 @@ var ParquetLoader = _objectSpread(_objectSpread({}, _parquetLoader.ParquetLoader
 });
 
 exports.ParquetLoader = ParquetLoader;
+
+var ParquetWasmLoader = _objectSpread(_objectSpread({}, _parquetWasmLoader.ParquetWasmLoader), {}, {
+  parse: _parseParquetWasm.parseParquet
+});
+
+exports.ParquetWasmLoader = ParquetWasmLoader;
 var _typecheckParquetLoader = ParquetLoader;
 exports._typecheckParquetLoader = _typecheckParquetLoader;
 //# sourceMappingURL=index.js.map

package/dist/es5/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts"],"names":["ParquetLoader","ParquetWorkerLoader","parse","parseParquet","parseFileInBatches","parseParquetFileInBatches","_typecheckParquetLoader"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA;;AACA;;AAaA;;AAIA;;AAEA;;AACA;;AACA;;AAEA;;;;;;AAlBO,IAAMA,aAAa,mCACrBC,4BADqB;AAExBC,EAAAA,KAAK,EAAEC,0BAFiB;AAGxBC,EAAAA,kBAAkB,EAAEC;AAHI,EAAnB;;;AAqBA,IAAMC,uBAAyC,GAAGN,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\n\nexport {ParquetWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}
+{"version":3,"sources":["../../src/index.ts"],"names":["ParquetLoader","ParquetWorkerLoader","parse","parseParquet","parseFileInBatches","parseParquetFileInBatches","ParquetWasmLoader","ParquetWasmWorkerLoader","parseParquetWasm","_typecheckParquetLoader"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA;;AACA;;AACA;;AACA;;AAkBA;;AACA;;AAIA;;AAEA;;AACA;;AACA;;AAEA;;;;;;AAxBO,IAAMA,aAAa,mCACrBC,4BADqB;AAExBC,EAAAA,KAAK,EAAEC,0BAFiB;AAGxBC,EAAAA,kBAAkB,EAAEC;AAHI,EAAnB;;;;AAMA,IAAMC,iBAAiB,mCACzBC,oCADyB;AAE5BL,EAAAA,KAAK,EAAEM;AAFqB,EAAvB;;;AAqBA,IAAMC,uBAAyC,GAAGT,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\nimport {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';\n\nexport {ParquetWorkerLoader, ParquetWasmWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\nexport const ParquetWasmLoader = {\n  ...ParquetWasmWorkerLoader,\n  parse: parseParquetWasm\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\nexport {ParquetWasmWriter} from './parquet-wasm-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}

package/dist/es5/lib/wasm/encode-parquet-wasm.js

@@ -0,0 +1,56 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.encode = encode;
+exports.tableToIPC = tableToIPC;
+
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+
+var _apacheArrow = require("apache-arrow");
+
+var _loadWasm = require("./load-wasm");
+
+function encode(_x, _x2) {
+  return _encode.apply(this, arguments);
+}
+
+function _encode() {
+  _encode = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(table, options) {
+    var _options$parquet;
+
+    var wasmUrl, wasm, arrowIPCBytes, writerProperties, parquetBytes;
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            wasmUrl = options === null || options === void 0 ? void 0 : (_options$parquet = options.parquet) === null || _options$parquet === void 0 ? void 0 : _options$parquet.wasmUrl;
+            _context.next = 3;
+            return (0, _loadWasm.loadWasm)(wasmUrl);
+
+          case 3:
+            wasm = _context.sent;
+            arrowIPCBytes = tableToIPC(table);
+            writerProperties = new wasm.WriterPropertiesBuilder().build();
+            parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);
+            return _context.abrupt("return", parquetBytes.buffer.slice(parquetBytes.byteOffset, parquetBytes.byteLength + parquetBytes.byteOffset));
+
+          case 8:
+          case "end":
+            return _context.stop();
+        }
+      }
+    }, _callee);
+  }));
+  return _encode.apply(this, arguments);
+}
+
+function tableToIPC(table) {
+  return _apacheArrow.RecordBatchStreamWriter.writeAll(table).toUint8Array(true);
+}
+//# sourceMappingURL=encode-parquet-wasm.js.map

package/dist/es5/lib/wasm/encode-parquet-wasm.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/wasm/encode-parquet-wasm.ts"],"names":["encode","table","options","wasmUrl","parquet","wasm","arrowIPCBytes","tableToIPC","writerProperties","WriterPropertiesBuilder","build","parquetBytes","writeParquet","buffer","slice","byteOffset","byteLength","RecordBatchStreamWriter","writeAll","toUint8Array"],"mappings":";;;;;;;;;;;;;;AAGA;;AACA;;SAWsBA,M;;;;;sEAAf,iBAAsBC,KAAtB,EAAoCC,OAApC;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AACCC,YAAAA,OADD,GACWD,OADX,aACWA,OADX,2CACWA,OAAO,CAAEE,OADpB,qDACW,iBAAkBD,OAD7B;AAAA;AAAA,mBAEc,wBAASA,OAAT,CAFd;;AAAA;AAECE,YAAAA,IAFD;AAICC,YAAAA,aAJD,GAIiBC,UAAU,CAACN,KAAD,CAJ3B;AAMCO,YAAAA,gBAND,GAMoB,IAAIH,IAAI,CAACI,uBAAT,GAAmCC,KAAnC,EANpB;AAOCC,YAAAA,YAPD,GAOgBN,IAAI,CAACO,YAAL,CAAkBN,aAAlB,EAAiCE,gBAAjC,CAPhB;AAAA,6CAQEG,YAAY,CAACE,MAAb,CAAoBC,KAApB,CACLH,YAAY,CAACI,UADR,EAELJ,YAAY,CAACK,UAAb,GAA0BL,YAAY,CAACI,UAFlC,CARF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;AAsBA,SAASR,UAAT,CAAoBN,KAApB,EAA8C;AACnD,SAAOgB,qCAAwBC,QAAxB,CAAiCjB,KAAjC,EAAwCkB,YAAxC,CAAqD,IAArD,CAAP;AACD","sourcesContent":["import type {Table} from 'apache-arrow';\nimport type {WriterOptions} from '@loaders.gl/loader-utils';\n\nimport {RecordBatchStreamWriter} from 'apache-arrow';\nimport {loadWasm} from './load-wasm';\n\nexport type ParquetWriterOptions = WriterOptions & {\n  parquet?: {\n    wasmUrl?: string;\n  };\n};\n\n/**\n * Encode Arrow Table to Parquet buffer\n */\nexport async function encode(table: Table, options?: ParquetWriterOptions): Promise<ArrayBuffer> {\n  const wasmUrl = options?.parquet?.wasmUrl;\n  const wasm = await loadWasm(wasmUrl);\n\n  const arrowIPCBytes = tableToIPC(table);\n  // TODO: provide options for how to write table.\n  const writerProperties = new wasm.WriterPropertiesBuilder().build();\n  const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);\n  return parquetBytes.buffer.slice(\n    parquetBytes.byteOffset,\n    parquetBytes.byteLength + parquetBytes.byteOffset\n  );\n}\n\n/**\n * Serialize a {@link Table} to the IPC format. This function is a convenience\n * wrapper for {@link RecordBatchStreamWriter} and {@link RecordBatchFileWriter}.\n * Opposite of {@link tableFromIPC}.\n *\n * @param table The Table to serialize.\n * @param type Whether to serialize the Table as a file or a stream.\n */\nexport function tableToIPC(table: Table): Uint8Array {\n  return RecordBatchStreamWriter.writeAll(table).toUint8Array(true);\n}\n"],"file":"encode-parquet-wasm.js"}

package/dist/es5/lib/wasm/load-wasm/index.js

@@ -0,0 +1,14 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+Object.defineProperty(exports, "loadWasm", {
+  enumerable: true,
+  get: function get() {
+    return _loadWasmNode.loadWasm;
+  }
+});
+
+var _loadWasmNode = require("./load-wasm-node");
+//# sourceMappingURL=index.js.map

package/dist/es5/lib/wasm/load-wasm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../src/lib/wasm/load-wasm/index.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA","sourcesContent":["export {loadWasm} from './load-wasm-node';\n"],"file":"index.js"}

package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js

@@ -0,0 +1,58 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+var _typeof = require("@babel/runtime/helpers/typeof");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.loadWasm = loadWasm;
+
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+
+var wasmEsm = _interopRequireWildcard(require("parquet-wasm/esm2/arrow1"));
+
+function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+
+function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+
+var cached = null;
+
+function loadWasm(_x) {
+  return _loadWasm.apply(this, arguments);
+}
+
+function _loadWasm() {
+  _loadWasm = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(wasmUrl) {
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            if (!(cached !== null)) {
+              _context.next = 2;
+              break;
+            }
+
+            return _context.abrupt("return", cached);
+
+          case 2:
+            _context.next = 4;
+            return wasmEsm.default(wasmUrl);
+
+          case 4:
+            cached = wasmEsm;
+            return _context.abrupt("return", wasmEsm);
+
+          case 6:
+          case "end":
+            return _context.stop();
+        }
+      }
+    }, _callee);
+  }));
+  return _loadWasm.apply(this, arguments);
+}
+//# sourceMappingURL=load-wasm-browser.js.map

package/dist/es5/lib/wasm/load-wasm/load-wasm-browser.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../src/lib/wasm/load-wasm/load-wasm-browser.ts"],"names":["cached","loadWasm","wasmUrl","wasmEsm","default"],"mappings":";;;;;;;;;;;;;;;AAAA;;;;;;AAEA,IAAIA,MAA6B,GAAG,IAApC;;SAEsBC,Q;;;;;wEAAf,iBAAwBC,OAAxB;AAAA;AAAA;AAAA;AAAA;AAAA,kBACDF,MAAM,KAAK,IADV;AAAA;AAAA;AAAA;;AAAA,6CAEIA,MAFJ;;AAAA;AAAA;AAAA,mBAMCG,OAAO,CAACC,OAAR,CAAgBF,OAAhB,CAND;;AAAA;AAOLF,YAAAA,MAAM,GAAGG,OAAT;AAPK,6CASEA,OATF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["import * as wasmEsm from 'parquet-wasm/esm2/arrow1';\n\nlet cached: typeof wasmEsm | null = null;\n\nexport async function loadWasm(wasmUrl?: string) {\n  if (cached !== null) {\n    return cached;\n  }\n\n  // For ESM bundles, need to await the default export, which loads the WASM\n  await wasmEsm.default(wasmUrl);\n  cached = wasmEsm;\n\n  return wasmEsm;\n}\n"],"file":"load-wasm-browser.js"}

package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js

@@ -0,0 +1,43 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+var _typeof = require("@babel/runtime/helpers/typeof");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.loadWasm = loadWasm;
+
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+
+var wasmNode = _interopRequireWildcard(require("parquet-wasm/node/arrow1"));
+
+function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
+
+function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || _typeof(obj) !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
+
+function loadWasm(_x) {
+  return _loadWasm.apply(this, arguments);
+}
+
+function _loadWasm() {
+  _loadWasm = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(wasmUrl) {
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            return _context.abrupt("return", wasmNode);
+
+          case 1:
+          case "end":
+            return _context.stop();
+        }
+      }
+    }, _callee);
+  }));
+  return _loadWasm.apply(this, arguments);
+}
+//# sourceMappingURL=load-wasm-node.js.map

package/dist/es5/lib/wasm/load-wasm/load-wasm-node.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../src/lib/wasm/load-wasm/load-wasm-node.ts"],"names":["loadWasm","wasmUrl","wasmNode"],"mappings":";;;;;;;;;;;;;;;AAAA;;;;;;SAEsBA,Q;;;;;wEAAf,iBAAwBC,OAAxB;AAAA;AAAA;AAAA;AAAA;AAAA,6CACEC,QADF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["import * as wasmNode from 'parquet-wasm/node/arrow1';\n\nexport async function loadWasm(wasmUrl?: string) {\n  return wasmNode;\n}\n"],"file":"load-wasm-node.js"}

package/dist/es5/lib/wasm/parse-parquet-wasm.js

@@ -0,0 +1,80 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.parseParquet = parseParquet;
+
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+
+var _apacheArrow = require("apache-arrow");
+
+var _loadWasmNode = require("./load-wasm/load-wasm-node");
+
+function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
+
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
+
+function parseParquet(_x, _x2) {
+  return _parseParquet.apply(this, arguments);
+}
+
+function _parseParquet() {
+  _parseParquet = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee(arrayBuffer, options) {
+    var _options$parquet;
+
+    var wasmUrl, wasm, arr, arrowIPCUint8Arr, arrowIPCBuffer, arrowTable;
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            wasmUrl = options === null || options === void 0 ? void 0 : (_options$parquet = options.parquet) === null || _options$parquet === void 0 ? void 0 : _options$parquet.wasmUrl;
+            _context.next = 3;
+            return (0, _loadWasmNode.loadWasm)(wasmUrl);
+
+          case 3:
+            wasm = _context.sent;
+            arr = new Uint8Array(arrayBuffer);
+            arrowIPCUint8Arr = wasm.readParquet(arr);
+            arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(arrowIPCUint8Arr.byteOffset, arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset);
+            arrowTable = tableFromIPC(arrowIPCBuffer);
+            return _context.abrupt("return", arrowTable);
+
+          case 9:
+          case "end":
+            return _context.stop();
+        }
+      }
+    }, _callee);
+  }));
+  return _parseParquet.apply(this, arguments);
+}
+
+function tableFromIPC(input) {
+  var reader = _apacheArrow.RecordBatchStreamReader.from(input);
+
+  var recordBatches = [];
+
+  var _iterator = _createForOfIteratorHelper(reader),
+      _step;
+
+  try {
+    for (_iterator.s(); !(_step = _iterator.n()).done;) {
+      var recordBatch = _step.value;
+      recordBatches.push(recordBatch);
+    }
+  } catch (err) {
+    _iterator.e(err);
+  } finally {
+    _iterator.f();
+  }
+
+  return new _apacheArrow.Table(recordBatches);
+}
+//# sourceMappingURL=parse-parquet-wasm.js.map

package/dist/es5/lib/wasm/parse-parquet-wasm.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/wasm/parse-parquet-wasm.ts"],"names":["parseParquet","arrayBuffer","options","wasmUrl","parquet","wasm","arr","Uint8Array","arrowIPCUint8Arr","readParquet","arrowIPCBuffer","buffer","slice","byteOffset","byteLength","arrowTable","tableFromIPC","input","reader","RecordBatchStreamReader","from","recordBatches","recordBatch","push","Table"],"mappings":";;;;;;;;;;;;;AAGA;;AACA;;;;;;;;SASsBA,Y;;;;;4EAAf,iBACLC,WADK,EAELC,OAFK;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;AAICC,YAAAA,OAJD,GAIWD,OAJX,aAIWA,OAJX,2CAIWA,OAAO,CAAEE,OAJpB,qDAIW,iBAAkBD,OAJ7B;AAAA;AAAA,mBAKc,4BAASA,OAAT,CALd;;AAAA;AAKCE,YAAAA,IALD;AAOCC,YAAAA,GAPD,GAOO,IAAIC,UAAJ,CAAeN,WAAf,CAPP;AAQCO,YAAAA,gBARD,GAQoBH,IAAI,CAACI,WAAL,CAAiBH,GAAjB,CARpB;AASCI,YAAAA,cATD,GASkBF,gBAAgB,CAACG,MAAjB,CAAwBC,KAAxB,CACrBJ,gBAAgB,CAACK,UADI,EAErBL,gBAAgB,CAACM,UAAjB,GAA8BN,gBAAgB,CAACK,UAF1B,CATlB;AAaCE,YAAAA,UAbD,GAacC,YAAY,CAACN,cAAD,CAb1B;AAAA,6CAcEK,UAdF;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;AAqBP,SAASC,YAAT,CAAsBC,KAAtB,EAAiD;AAC/C,MAAMC,MAAM,GAAGC,qCAAwBC,IAAxB,CAA6BH,KAA7B,CAAf;;AACA,MAAMI,aAA4B,GAAG,EAArC;;AAF+C,6CAGrBH,MAHqB;AAAA;;AAAA;AAG/C,wDAAkC;AAAA,UAAvBI,WAAuB;AAChCD,MAAAA,aAAa,CAACE,IAAd,CAAmBD,WAAnB;AACD;AAL8C;AAAA;AAAA;AAAA;AAAA;;AAM/C,SAAO,IAAIE,kBAAJ,CAAUH,aAAV,CAAP;AACD","sourcesContent":["// eslint-disable\nimport type {RecordBatch} from 'apache-arrow';\nimport type {LoaderOptions} from '@loaders.gl/loader-utils';\nimport {Table, RecordBatchStreamReader} from 'apache-arrow';\nimport {loadWasm} from './load-wasm/load-wasm-node';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'arrow-table';\n    wasmUrl?: string;\n  };\n};\n\nexport async function parseParquet(\n  arrayBuffer: ArrayBuffer,\n  options?: ParquetLoaderOptions\n): Promise<Table> {\n  const wasmUrl = options?.parquet?.wasmUrl;\n  const wasm = await loadWasm(wasmUrl);\n\n  const arr = new Uint8Array(arrayBuffer);\n  const arrowIPCUint8Arr = wasm.readParquet(arr);\n  const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(\n    arrowIPCUint8Arr.byteOffset,\n    arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset\n  );\n  const arrowTable = tableFromIPC(arrowIPCBuffer);\n  return arrowTable;\n}\n\n/**\n * Deserialize the IPC format into a {@link Table}. This function is a\n * convenience wrapper for {@link RecordBatchReader}. Opposite of {@link tableToIPC}.\n */\nfunction tableFromIPC(input: ArrayBuffer): Table {\n  const reader = RecordBatchStreamReader.from(input);\n  const recordBatches: RecordBatch[] = [];\n  for (const recordBatch of reader) {\n    recordBatches.push(recordBatch);\n  }\n  return new Table(recordBatches);\n}\n"],"file":"parse-parquet-wasm.js"}

package/dist/es5/parquet-loader.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports._typecheckParquetLoader = exports.ParquetLoader = void 0;
-var VERSION = typeof "3.2.0-alpha.2" !== 'undefined' ? "3.2.0-alpha.2" : 'latest';
+var VERSION = typeof "3.2.0" !== 'undefined' ? "3.2.0" : 'latest';
 var DEFAULT_PARQUET_LOADER_OPTIONS = {
   parquet: {
     type: 'object-row-table',

package/dist/es5/parquet-loader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AASA,IAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQO,IAAMC,aAAa,GAAG;AAC3BC,EAAAA,IAAI,EAAE,gBADqB;AAE3BC,EAAAA,EAAE,EAAE,SAFuB;AAG3BC,EAAAA,MAAM,EAAE,SAHmB;AAI3BC,EAAAA,OAAO,EAAEV,OAJkB;AAK3BW,EAAAA,MAAM,EAAE,IALmB;AAM3BC,EAAAA,QAAQ,EAAE,OANiB;AAO3BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPe;AAQ3BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARgB;AAS3BC,EAAAA,MAAM,EAAE,IATmB;AAU3BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVoB;AAW3BC,EAAAA,OAAO,EAAEhB;AAXkB,CAAtB;;AAcA,IAAMiB,uBAA+B,GAAGZ,aAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"file":"parquet-loader.js"}
+{"version":3,"sources":["../../src/parquet-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AASA,IAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,kBADC;AAEPC,IAAAA,GAAG,EAAEC;AAFE;AADkD,CAA7D;AAQO,IAAMC,aAAa,GAAG;AAC3BC,EAAAA,IAAI,EAAE,gBADqB;AAE3BC,EAAAA,EAAE,EAAE,SAFuB;AAG3BC,EAAAA,MAAM,EAAE,SAHmB;AAI3BC,EAAAA,OAAO,EAAEV,OAJkB;AAK3BW,EAAAA,MAAM,EAAE,IALmB;AAM3BC,EAAAA,QAAQ,EAAE,OANiB;AAO3BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPe;AAQ3BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARgB;AAS3BC,EAAAA,MAAM,EAAE,IATmB;AAU3BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVoB;AAW3BC,EAAAA,OAAO,EAAEhB;AAXkB,CAAtB;;AAcA,IAAMiB,uBAA+B,GAAGZ,aAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"file":"parquet-loader.js"}

package/dist/es5/parquet-wasm-loader.js

@@ -0,0 +1,30 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports._typecheckParquetLoader = exports.ParquetWasmLoader = void 0;
+var VERSION = typeof "3.2.0" !== 'undefined' ? "3.2.0" : 'latest';
+var DEFAULT_PARQUET_LOADER_OPTIONS = {
+  parquet: {
+    type: 'arrow-table',
+    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
+  }
+};
+var ParquetWasmLoader = {
+  name: 'Apache Parquet',
+  id: 'parquet-wasm',
+  module: 'parquet',
+  version: VERSION,
+  worker: false,
+  category: 'table',
+  extensions: ['parquet'],
+  mimeTypes: ['application/octet-stream'],
+  binary: true,
+  tests: ['PAR1', 'PARE'],
+  options: DEFAULT_PARQUET_LOADER_OPTIONS
+};
+exports.ParquetWasmLoader = ParquetWasmLoader;
+var _typecheckParquetLoader = ParquetWasmLoader;
+exports._typecheckParquetLoader = _typecheckParquetLoader;
+//# sourceMappingURL=parquet-wasm-loader.js.map

package/dist/es5/parquet-wasm-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/parquet-wasm-loader.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","wasmUrl","ParquetWasmLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AASA,IAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,IAAI,EAAE,aADC;AAEPC,IAAAA,OAAO,EAAE;AAFF;AADkD,CAA7D;AAQO,IAAMC,iBAAiB,GAAG;AAC/BC,EAAAA,IAAI,EAAE,gBADyB;AAE/BC,EAAAA,EAAE,EAAE,cAF2B;AAG/BC,EAAAA,MAAM,EAAE,SAHuB;AAI/BC,EAAAA,OAAO,EAAET,OAJsB;AAK/BU,EAAAA,MAAM,EAAE,KALuB;AAM/BC,EAAAA,QAAQ,EAAE,OANqB;AAO/BC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAPmB;AAQ/BC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CARoB;AAS/BC,EAAAA,MAAM,EAAE,IATuB;AAU/BC,EAAAA,KAAK,EAAE,CAAC,MAAD,EAAS,MAAT,CAVwB;AAW/BC,EAAAA,OAAO,EAAEf;AAXsB,CAA1B;;AAcA,IAAMgB,uBAA+B,GAAGZ,iBAAxC","sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'arrow-table';\n    wasmUrl?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'arrow-table',\n    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetWasmLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet-wasm',\n  module: 'parquet',\n  version: VERSION,\n  worker: false,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetWasmLoader;\n"],"file":"parquet-wasm-loader.js"}

package/dist/es5/parquet-wasm-writer.js

@@ -0,0 +1,28 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.ParquetWasmWriter = void 0;
+
+var _encodeParquetWasm = require("./lib/wasm/encode-parquet-wasm");
+
+var VERSION = typeof "3.2.0" !== 'undefined' ? "3.2.0" : 'latest';
+var DEFAULT_PARQUET_WRITER_OPTIONS = {
+  parquet: {
+    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
+  }
+};
+var ParquetWasmWriter = {
+  name: 'Apache Parquet',
+  id: 'parquet-wasm',
+  module: 'parquet',
+  version: VERSION,
+  extensions: ['parquet'],
+  mimeTypes: ['application/octet-stream'],
+  encode: _encodeParquetWasm.encode,
+  binary: true,
+  options: DEFAULT_PARQUET_WRITER_OPTIONS
+};
+exports.ParquetWasmWriter = ParquetWasmWriter;
+//# sourceMappingURL=parquet-wasm-writer.js.map

package/dist/es5/parquet-wasm-writer.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/parquet-wasm-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_WRITER_OPTIONS","parquet","wasmUrl","ParquetWasmWriter","name","id","module","version","extensions","mimeTypes","encode","binary","options"],"mappings":";;;;;;;AACA;;AAIA,IAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAEA,IAAMC,8BAAoD,GAAG;AAC3DC,EAAAA,OAAO,EAAE;AACPC,IAAAA,OAAO,EAAE;AADF;AADkD,CAA7D;AAMO,IAAMC,iBAAyB,GAAG;AACvCC,EAAAA,IAAI,EAAE,gBADiC;AAEvCC,EAAAA,EAAE,EAAE,cAFmC;AAGvCC,EAAAA,MAAM,EAAE,SAH+B;AAIvCC,EAAAA,OAAO,EAAER,OAJ8B;AAKvCS,EAAAA,UAAU,EAAE,CAAC,SAAD,CAL2B;AAMvCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CAN4B;AAOvCC,EAAAA,MAAM,EAANA,yBAPuC;AAQvCC,EAAAA,MAAM,EAAE,IAR+B;AASvCC,EAAAA,OAAO,EAAEZ;AAT8B,CAAlC","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\nimport {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nconst DEFAULT_PARQUET_WRITER_OPTIONS: ParquetWriterOptions = {\n  parquet: {\n    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n  }\n};\n\nexport const ParquetWasmWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet-wasm',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encode,\n  binary: true,\n  options: DEFAULT_PARQUET_WRITER_OPTIONS\n};\n"],"file":"parquet-wasm-writer.js"}

package/dist/es5/parquet-writer.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ParquetWriter = void 0;
-var VERSION = typeof "3.2.0-alpha.2" !== 'undefined' ? "3.2.0-alpha.2" : 'latest';
+var VERSION = typeof "3.2.0" !== 'undefined' ? "3.2.0" : 'latest';
 var DEFAULT_PARQUET_LOADER_OPTIONS = {};
 var ParquetWriter = {
   name: 'Apache Parquet',

package/dist/es5/parquet-writer.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AAIA,IAAMC,8BAA8B,GAAG,EAAvC;AAEO,IAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAAU,EAAVA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n  return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}
+{"version":3,"sources":["../../src/parquet-writer.ts"],"names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAIA,IAAMC,8BAA8B,GAAG,EAAvC;AAEO,IAAMC,aAAqB,GAAG;AACnCC,EAAAA,IAAI,EAAE,gBAD6B;AAEnCC,EAAAA,EAAE,EAAE,SAF+B;AAGnCC,EAAAA,MAAM,EAAE,SAH2B;AAInCC,EAAAA,OAAO,EAAEN,OAJ0B;AAKnCO,EAAAA,UAAU,EAAE,CAAC,SAAD,CALuB;AAMnCC,EAAAA,SAAS,EAAE,CAAC,0BAAD,CANwB;AAOnCC,EAAAA,UAAU,EAAVA,UAPmC;AAQnCC,EAAAA,MAAM,EAAE,IAR2B;AASnCC,EAAAA,OAAO,EAAEV;AAT0B,CAA9B;;;AAYP,SAASQ,UAAT,CAAoBG,IAApB,EAA0BD,OAA1B,EAA0D;AACxD,SAAO,IAAIE,WAAJ,CAAgB,CAAhB,CAAP;AACD","sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n  return new ArrayBuffer(0);\n}\n"],"file":"parquet-writer.js"}

package/dist/esm/index.js

@@ -1,11 +1,17 @@
+import { ParquetWasmLoader as ParquetWasmWorkerLoader } from './parquet-wasm-loader';
 import { ParquetLoader as ParquetWorkerLoader } from './parquet-loader';
 import { parseParquet, parseParquetFileInBatches } from './lib/parse-parquet';
-export { ParquetWorkerLoader };
+import { parseParquet as parseParquetWasm } from './lib/wasm/parse-parquet-wasm';
+export { ParquetWorkerLoader, ParquetWasmWorkerLoader };
 export const ParquetLoader = { ...ParquetWorkerLoader,
   parse: parseParquet,
   parseFileInBatches: parseParquetFileInBatches
 };
+export const ParquetWasmLoader = { ...ParquetWasmWorkerLoader,
+  parse: parseParquetWasm
+};
 export { ParquetWriter as _ParquetWriter } from './parquet-writer';
+export { ParquetWasmWriter } from './parquet-wasm-writer';
 export { preloadCompressions } from './parquetjs/compression';
 export { ParquetSchema } from './parquetjs/schema/schema';
 export { ParquetReader } from './parquetjs/parser/parquet-reader';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts"],"names":["ParquetLoader","ParquetWorkerLoader","parseParquet","parseParquetFileInBatches","parse","parseFileInBatches","ParquetWriter","_ParquetWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEnvelopeReader","convertParquetToArrowSchema","_typecheckParquetLoader"],"mappings":"AAIA,SAAQA,aAAa,IAAIC,mBAAzB,QAAmD,kBAAnD;AACA,SAAQC,YAAR,EAAsBC,yBAAtB,QAAsD,qBAAtD;AAEA,SAAQF,mBAAR;AAGA,OAAO,MAAMD,aAAa,GAAG,EAC3B,GAAGC,mBADwB;AAE3BG,EAAAA,KAAK,EAAEF,YAFoB;AAG3BG,EAAAA,kBAAkB,EAAEF;AAHO,CAAtB;AAQP,SAAQG,aAAa,IAAIC,cAAzB,QAA8C,kBAA9C;AAIA,SAAQC,mBAAR,QAAkC,yBAAlC;AAEA,SAAQC,aAAR,QAA4B,2BAA5B;AACA,SAAQC,aAAR,QAA4B,mCAA5B;AACA,SAAQC,qBAAR,QAAoC,4CAApC;AAEA,SAAQC,2BAAR,QAA0C,sBAA1C;AAGA,OAAO,MAAMC,uBAAyC,GAAGb,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\n\nexport {ParquetWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}
+{"version":3,"sources":["../../src/index.ts"],"names":["ParquetWasmLoader","ParquetWasmWorkerLoader","ParquetLoader","ParquetWorkerLoader","parseParquet","parseParquetFileInBatches","parseParquetWasm","parse","parseFileInBatches","ParquetWriter","_ParquetWriter","ParquetWasmWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEnvelopeReader","convertParquetToArrowSchema","_typecheckParquetLoader"],"mappings":"AAIA,SAAQA,iBAAiB,IAAIC,uBAA7B,QAA2D,uBAA3D;AACA,SAAQC,aAAa,IAAIC,mBAAzB,QAAmD,kBAAnD;AACA,SAAQC,YAAR,EAAsBC,yBAAtB,QAAsD,qBAAtD;AACA,SAAQD,YAAY,IAAIE,gBAAxB,QAA+C,+BAA/C;AAEA,SAAQH,mBAAR,EAA6BF,uBAA7B;AAGA,OAAO,MAAMC,aAAa,GAAG,EAC3B,GAAGC,mBADwB;AAE3BI,EAAAA,KAAK,EAAEH,YAFoB;AAG3BI,EAAAA,kBAAkB,EAAEH;AAHO,CAAtB;AAMP,OAAO,MAAML,iBAAiB,GAAG,EAC/B,GAAGC,uBAD4B;AAE/BM,EAAAA,KAAK,EAAED;AAFwB,CAA1B;AAOP,SAAQG,aAAa,IAAIC,cAAzB,QAA8C,kBAA9C;AACA,SAAQC,iBAAR,QAAgC,uBAAhC;AAIA,SAAQC,mBAAR,QAAkC,yBAAlC;AAEA,SAAQC,aAAR,QAA4B,2BAA5B;AACA,SAAQC,aAAR,QAA4B,mCAA5B;AACA,SAAQC,qBAAR,QAAoC,4CAApC;AAEA,SAAQC,2BAAR,QAA0C,sBAA1C;AAGA,OAAO,MAAMC,uBAAyC,GAAGf,aAAlD","sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\n\n// ParquetLoader\n\nimport {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';\nimport {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';\nimport {parseParquet, parseParquetFileInBatches} from './lib/parse-parquet';\nimport {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';\n\nexport {ParquetWorkerLoader, ParquetWasmWorkerLoader};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  ...ParquetWorkerLoader,\n  parse: parseParquet,\n  parseFileInBatches: parseParquetFileInBatches\n};\n\nexport const ParquetWasmLoader = {\n  ...ParquetWasmWorkerLoader,\n  parse: parseParquetWasm\n};\n\n// ParquetWriter\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\nexport {ParquetWasmWriter} from './parquet-wasm-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEnvelopeReader} from './parquetjs/parser/parquet-envelope-reader';\n// export {ParquetWriter, ParquetEnvelopeWriter, ParquetTransformer} from './parquetjs/encoder/writer';\nexport {convertParquetToArrowSchema} from './lib/convert-schema';\n\n// TESTS\nexport const _typecheckParquetLoader: LoaderWithParser = ParquetLoader;\n"],"file":"index.js"}

package/dist/esm/lib/wasm/encode-parquet-wasm.js

@@ -0,0 +1,16 @@
+import { RecordBatchStreamWriter } from 'apache-arrow';
+import { loadWasm } from './load-wasm';
+export async function encode(table, options) {
+  var _options$parquet;
+
+  const wasmUrl = options === null || options === void 0 ? void 0 : (_options$parquet = options.parquet) === null || _options$parquet === void 0 ? void 0 : _options$parquet.wasmUrl;
+  const wasm = await loadWasm(wasmUrl);
+  const arrowIPCBytes = tableToIPC(table);
+  const writerProperties = new wasm.WriterPropertiesBuilder().build();
+  const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);
+  return parquetBytes.buffer.slice(parquetBytes.byteOffset, parquetBytes.byteLength + parquetBytes.byteOffset);
+}
+export function tableToIPC(table) {
+  return RecordBatchStreamWriter.writeAll(table).toUint8Array(true);
+}
+//# sourceMappingURL=encode-parquet-wasm.js.map

package/dist/esm/lib/wasm/encode-parquet-wasm.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/wasm/encode-parquet-wasm.ts"],"names":["RecordBatchStreamWriter","loadWasm","encode","table","options","wasmUrl","parquet","wasm","arrowIPCBytes","tableToIPC","writerProperties","WriterPropertiesBuilder","build","parquetBytes","writeParquet","buffer","slice","byteOffset","byteLength","writeAll","toUint8Array"],"mappings":"AAGA,SAAQA,uBAAR,QAAsC,cAAtC;AACA,SAAQC,QAAR,QAAuB,aAAvB;AAWA,OAAO,eAAeC,MAAf,CAAsBC,KAAtB,EAAoCC,OAApC,EAA0F;AAAA;;AAC/F,QAAMC,OAAO,GAAGD,OAAH,aAAGA,OAAH,2CAAGA,OAAO,CAAEE,OAAZ,qDAAG,iBAAkBD,OAAlC;AACA,QAAME,IAAI,GAAG,MAAMN,QAAQ,CAACI,OAAD,CAA3B;AAEA,QAAMG,aAAa,GAAGC,UAAU,CAACN,KAAD,CAAhC;AAEA,QAAMO,gBAAgB,GAAG,IAAIH,IAAI,CAACI,uBAAT,GAAmCC,KAAnC,EAAzB;AACA,QAAMC,YAAY,GAAGN,IAAI,CAACO,YAAL,CAAkBN,aAAlB,EAAiCE,gBAAjC,CAArB;AACA,SAAOG,YAAY,CAACE,MAAb,CAAoBC,KAApB,CACLH,YAAY,CAACI,UADR,EAELJ,YAAY,CAACK,UAAb,GAA0BL,YAAY,CAACI,UAFlC,CAAP;AAID;AAUD,OAAO,SAASR,UAAT,CAAoBN,KAApB,EAA8C;AACnD,SAAOH,uBAAuB,CAACmB,QAAxB,CAAiChB,KAAjC,EAAwCiB,YAAxC,CAAqD,IAArD,CAAP;AACD","sourcesContent":["import type {Table} from 'apache-arrow';\nimport type {WriterOptions} from '@loaders.gl/loader-utils';\n\nimport {RecordBatchStreamWriter} from 'apache-arrow';\nimport {loadWasm} from './load-wasm';\n\nexport type ParquetWriterOptions = WriterOptions & {\n  parquet?: {\n    wasmUrl?: string;\n  };\n};\n\n/**\n * Encode Arrow Table to Parquet buffer\n */\nexport async function encode(table: Table, options?: ParquetWriterOptions): Promise<ArrayBuffer> {\n  const wasmUrl = options?.parquet?.wasmUrl;\n  const wasm = await loadWasm(wasmUrl);\n\n  const arrowIPCBytes = tableToIPC(table);\n  // TODO: provide options for how to write table.\n  const writerProperties = new wasm.WriterPropertiesBuilder().build();\n  const parquetBytes = wasm.writeParquet(arrowIPCBytes, writerProperties);\n  return parquetBytes.buffer.slice(\n    parquetBytes.byteOffset,\n    parquetBytes.byteLength + parquetBytes.byteOffset\n  );\n}\n\n/**\n * Serialize a {@link Table} to the IPC format. This function is a convenience\n * wrapper for {@link RecordBatchStreamWriter} and {@link RecordBatchFileWriter}.\n * Opposite of {@link tableFromIPC}.\n *\n * @param table The Table to serialize.\n * @param type Whether to serialize the Table as a file or a stream.\n */\nexport function tableToIPC(table: Table): Uint8Array {\n  return RecordBatchStreamWriter.writeAll(table).toUint8Array(true);\n}\n"],"file":"encode-parquet-wasm.js"}

package/dist/esm/lib/wasm/load-wasm/index.js

@@ -0,0 +1,2 @@
+export { loadWasm } from './load-wasm-node';
+//# sourceMappingURL=index.js.map

package/dist/esm/lib/wasm/load-wasm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../src/lib/wasm/load-wasm/index.ts"],"names":["loadWasm"],"mappings":"AAAA,SAAQA,QAAR,QAAuB,kBAAvB","sourcesContent":["export {loadWasm} from './load-wasm-node';\n"],"file":"index.js"}

package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js

@@ -0,0 +1,12 @@
+import * as wasmEsm from 'parquet-wasm/esm2/arrow1';
+let cached = null;
+export async function loadWasm(wasmUrl) {
+  if (cached !== null) {
+    return cached;
+  }
+
+  await wasmEsm.default(wasmUrl);
+  cached = wasmEsm;
+  return wasmEsm;
+}
+//# sourceMappingURL=load-wasm-browser.js.map

package/dist/esm/lib/wasm/load-wasm/load-wasm-browser.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../src/lib/wasm/load-wasm/load-wasm-browser.ts"],"names":["wasmEsm","cached","loadWasm","wasmUrl","default"],"mappings":"AAAA,OAAO,KAAKA,OAAZ,MAAyB,0BAAzB;AAEA,IAAIC,MAA6B,GAAG,IAApC;AAEA,OAAO,eAAeC,QAAf,CAAwBC,OAAxB,EAA0C;AAC/C,MAAIF,MAAM,KAAK,IAAf,EAAqB;AACnB,WAAOA,MAAP;AACD;;AAGD,QAAMD,OAAO,CAACI,OAAR,CAAgBD,OAAhB,CAAN;AACAF,EAAAA,MAAM,GAAGD,OAAT;AAEA,SAAOA,OAAP;AACD","sourcesContent":["import * as wasmEsm from 'parquet-wasm/esm2/arrow1';\n\nlet cached: typeof wasmEsm | null = null;\n\nexport async function loadWasm(wasmUrl?: string) {\n  if (cached !== null) {\n    return cached;\n  }\n\n  // For ESM bundles, need to await the default export, which loads the WASM\n  await wasmEsm.default(wasmUrl);\n  cached = wasmEsm;\n\n  return wasmEsm;\n}\n"],"file":"load-wasm-browser.js"}

package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js

@@ -0,0 +1,5 @@
+import * as wasmNode from 'parquet-wasm/node/arrow1';
+export async function loadWasm(wasmUrl) {
+  return wasmNode;
+}
+//# sourceMappingURL=load-wasm-node.js.map

package/dist/esm/lib/wasm/load-wasm/load-wasm-node.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../../src/lib/wasm/load-wasm/load-wasm-node.ts"],"names":["wasmNode","loadWasm","wasmUrl"],"mappings":"AAAA,OAAO,KAAKA,QAAZ,MAA0B,0BAA1B;AAEA,OAAO,eAAeC,QAAf,CAAwBC,OAAxB,EAA0C;AAC/C,SAAOF,QAAP;AACD","sourcesContent":["import * as wasmNode from 'parquet-wasm/node/arrow1';\n\nexport async function loadWasm(wasmUrl?: string) {\n  return wasmNode;\n}\n"],"file":"load-wasm-node.js"}

package/dist/esm/lib/wasm/parse-parquet-wasm.js

@@ -0,0 +1,25 @@
+import { Table, RecordBatchStreamReader } from 'apache-arrow';
+import { loadWasm } from './load-wasm/load-wasm-node';
+export async function parseParquet(arrayBuffer, options) {
+  var _options$parquet;
+
+  const wasmUrl = options === null || options === void 0 ? void 0 : (_options$parquet = options.parquet) === null || _options$parquet === void 0 ? void 0 : _options$parquet.wasmUrl;
+  const wasm = await loadWasm(wasmUrl);
+  const arr = new Uint8Array(arrayBuffer);
+  const arrowIPCUint8Arr = wasm.readParquet(arr);
+  const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(arrowIPCUint8Arr.byteOffset, arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset);
+  const arrowTable = tableFromIPC(arrowIPCBuffer);
+  return arrowTable;
+}
+
+function tableFromIPC(input) {
+  const reader = RecordBatchStreamReader.from(input);
+  const recordBatches = [];
+
+  for (const recordBatch of reader) {
+    recordBatches.push(recordBatch);
+  }
+
+  return new Table(recordBatches);
+}
+//# sourceMappingURL=parse-parquet-wasm.js.map

package/dist/esm/lib/wasm/parse-parquet-wasm.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../../../src/lib/wasm/parse-parquet-wasm.ts"],"names":["Table","RecordBatchStreamReader","loadWasm","parseParquet","arrayBuffer","options","wasmUrl","parquet","wasm","arr","Uint8Array","arrowIPCUint8Arr","readParquet","arrowIPCBuffer","buffer","slice","byteOffset","byteLength","arrowTable","tableFromIPC","input","reader","from","recordBatches","recordBatch","push"],"mappings":"AAGA,SAAQA,KAAR,EAAeC,uBAAf,QAA6C,cAA7C;AACA,SAAQC,QAAR,QAAuB,4BAAvB;AASA,OAAO,eAAeC,YAAf,CACLC,WADK,EAELC,OAFK,EAGW;AAAA;;AAChB,QAAMC,OAAO,GAAGD,OAAH,aAAGA,OAAH,2CAAGA,OAAO,CAAEE,OAAZ,qDAAG,iBAAkBD,OAAlC;AACA,QAAME,IAAI,GAAG,MAAMN,QAAQ,CAACI,OAAD,CAA3B;AAEA,QAAMG,GAAG,GAAG,IAAIC,UAAJ,CAAeN,WAAf,CAAZ;AACA,QAAMO,gBAAgB,GAAGH,IAAI,CAACI,WAAL,CAAiBH,GAAjB,CAAzB;AACA,QAAMI,cAAc,GAAGF,gBAAgB,CAACG,MAAjB,CAAwBC,KAAxB,CACrBJ,gBAAgB,CAACK,UADI,EAErBL,gBAAgB,CAACM,UAAjB,GAA8BN,gBAAgB,CAACK,UAF1B,CAAvB;AAIA,QAAME,UAAU,GAAGC,YAAY,CAACN,cAAD,CAA/B;AACA,SAAOK,UAAP;AACD;;AAMD,SAASC,YAAT,CAAsBC,KAAtB,EAAiD;AAC/C,QAAMC,MAAM,GAAGpB,uBAAuB,CAACqB,IAAxB,CAA6BF,KAA7B,CAAf;AACA,QAAMG,aAA4B,GAAG,EAArC;;AACA,OAAK,MAAMC,WAAX,IAA0BH,MAA1B,EAAkC;AAChCE,IAAAA,aAAa,CAACE,IAAd,CAAmBD,WAAnB;AACD;;AACD,SAAO,IAAIxB,KAAJ,CAAUuB,aAAV,CAAP;AACD","sourcesContent":["// eslint-disable\nimport type {RecordBatch} from 'apache-arrow';\nimport type {LoaderOptions} from '@loaders.gl/loader-utils';\nimport {Table, RecordBatchStreamReader} from 'apache-arrow';\nimport {loadWasm} from './load-wasm/load-wasm-node';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'arrow-table';\n    wasmUrl?: string;\n  };\n};\n\nexport async function parseParquet(\n  arrayBuffer: ArrayBuffer,\n  options?: ParquetLoaderOptions\n): Promise<Table> {\n  const wasmUrl = options?.parquet?.wasmUrl;\n  const wasm = await loadWasm(wasmUrl);\n\n  const arr = new Uint8Array(arrayBuffer);\n  const arrowIPCUint8Arr = wasm.readParquet(arr);\n  const arrowIPCBuffer = arrowIPCUint8Arr.buffer.slice(\n    arrowIPCUint8Arr.byteOffset,\n    arrowIPCUint8Arr.byteLength + arrowIPCUint8Arr.byteOffset\n  );\n  const arrowTable = tableFromIPC(arrowIPCBuffer);\n  return arrowTable;\n}\n\n/**\n * Deserialize the IPC format into a {@link Table}. This function is a\n * convenience wrapper for {@link RecordBatchReader}. Opposite of {@link tableToIPC}.\n */\nfunction tableFromIPC(input: ArrayBuffer): Table {\n  const reader = RecordBatchStreamReader.from(input);\n  const recordBatches: RecordBatch[] = [];\n  for (const recordBatch of reader) {\n    recordBatches.push(recordBatch);\n  }\n  return new Table(recordBatches);\n}\n"],"file":"parse-parquet-wasm.js"}

package/dist/esm/parquet-loader.js

@@ -1,4 +1,4 @@
-const VERSION = typeof "3.2.0-alpha.2" !== 'undefined' ? "3.2.0-alpha.2" : 'latest';
+const VERSION = typeof "3.2.0" !== 'undefined' ? "3.2.0" : 'latest';
 const DEFAULT_PARQUET_LOADER_OPTIONS = {
   parquet: {
     type: 'object-row-table',