@loaders.gl/parquet 3.3.0-alpha.8 → 3.3.0

package/dist/es5/lib/parse-parquet.js

@@ -10,6 +10,7 @@ var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"))
 var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
 var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
 var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
+var _loaderUtils = require("@loaders.gl/loader-utils");
 var _parquetReader = require("../parquetjs/parser/parquet-reader");
 function _asyncIterator(iterable) { var method, async, sync, retry = 2; for ("undefined" != typeof Symbol && (async = Symbol.asyncIterator, sync = Symbol.iterator); retry--;) { if (async && null != (method = iterable[async])) return method.call(iterable); if (sync && null != (method = iterable[sync])) return new AsyncFromSyncIterator(method.call(iterable)); async = "@@asyncIterator", sync = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
 function AsyncFromSyncIterator(s) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var done = r.done; return Promise.resolve(r.value).then(function (value) { return { value: value, done: done }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(s) { this.s = s, this.n = s.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, return: function _return(value) { var ret = this.s.return; return void 0 === ret ? Promise.resolve({ value: value, done: !0 }) : AsyncFromSyncIteratorContinuation(ret.apply(this.s, arguments)); }, throw: function _throw(value) { var thr = this.s.return; return void 0 === thr ? Promise.reject(value) : AsyncFromSyncIteratorContinuation(thr.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(s); }
@@ -86,44 +87,67 @@ function parseParquetFileInBatches(_x, _x2) {
 }
 function _parseParquetFileInBatches() {
   _parseParquetFileInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(blob, options) {
-    var reader, rows, cursor, record;
+    var file, reader, rowBatches, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, rows;
     return _regenerator.default.wrap(function _callee$(_context) {
       while (1) {
         switch (_context.prev = _context.next) {
           case 0:
-            _context.next = 2;
-            return (0, _awaitAsyncGenerator2.default)(_parquetReader.ParquetReader.openBlob(blob));
-          case 2:
-            reader = _context.sent;
-            rows = [];
-            _context.prev = 4;
-            cursor = reader.getCursor();
-          case 6:
-            _context.next = 8;
-            return (0, _awaitAsyncGenerator2.default)(cursor.next());
-          case 8:
-            if (!(record = _context.sent)) {
-              _context.next = 12;
+            file = (0, _loaderUtils.makeReadableFile)(blob);
+            reader = new _parquetReader.ParquetReader(file);
+            rowBatches = reader.rowBatchIterator(options === null || options === void 0 ? void 0 : options.parquet);
+            _iteratorAbruptCompletion2 = false;
+            _didIteratorError2 = false;
+            _context.prev = 5;
+            _iterator2 = _asyncIterator(rowBatches);
+          case 7:
+            _context.next = 9;
+            return (0, _awaitAsyncGenerator2.default)(_iterator2.next());
+          case 9:
+            if (!(_iteratorAbruptCompletion2 = !(_step2 = _context.sent).done)) {
+              _context.next = 16;
               break;
             }
-            rows.push(record);
-            _context.next = 6;
+            rows = _step2.value;
+            _context.next = 13;
+            return rows;
+          case 13:
+            _iteratorAbruptCompletion2 = false;
+            _context.next = 7;
             break;
-          case 12:
-            _context.prev = 12;
-            _context.next = 15;
-            return (0, _awaitAsyncGenerator2.default)(reader.close());
-          case 15:
-            return _context.finish(12);
           case 16:
-            _context.next = 18;
-            return rows;
+            _context.next = 22;
+            break;
           case 18:
+            _context.prev = 18;
+            _context.t0 = _context["catch"](5);
+            _didIteratorError2 = true;
+            _iteratorError2 = _context.t0;
+          case 22:
+            _context.prev = 22;
+            _context.prev = 23;
+            if (!(_iteratorAbruptCompletion2 && _iterator2.return != null)) {
+              _context.next = 27;
+              break;
+            }
+            _context.next = 27;
+            return (0, _awaitAsyncGenerator2.default)(_iterator2.return());
+          case 27:
+            _context.prev = 27;
+            if (!_didIteratorError2) {
+              _context.next = 30;
+              break;
+            }
+            throw _iteratorError2;
+          case 30:
+            return _context.finish(27);
+          case 31:
+            return _context.finish(22);
+          case 32:
           case "end":
             return _context.stop();
         }
       }
-    }, _callee, null, [[4,, 12, 16]]);
+    }, _callee, null, [[5, 18, 22, 32], [23,, 27, 31]]);
   }));
   return _parseParquetFileInBatches.apply(this, arguments);
 }

package/dist/es5/lib/parse-parquet.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet.js","names":["parseParquet","arrayBuffer","options","blob","Blob","parseParquetFileInBatches","batch","ParquetReader","openBlob","reader","rows","cursor","getCursor","next","record","push","close"],"sources":["../../../src/lib/parse-parquet.ts"],"sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../parquet-loader';\n\nimport {ParquetReader} from '../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInBatches(blob, options)) {\n    return batch;\n  }\n  return null;\n}\n\nexport async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {\n  const reader = await ParquetReader.openBlob(blob);\n  const rows: any[][] = [];\n  try {\n    const cursor = reader.getCursor();\n    let record: any[] | null;\n    while ((record = await cursor.next())) {\n      rows.push(record);\n    }\n  } finally {\n    await reader.close();\n  }\n  yield rows;\n}\n"],"mappings":";;;;;;;;;;;;AAGA;AAAiE;AAAA;AAAA,SAE3CA,YAAY;EAAA;AAAA;AAAA;EAAA,0EAA3B,kBAA4BC,WAAwB,EAAEC,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YACnFC,IAAI,GAAG,IAAIC,IAAI,CAAC,CAACH,WAAW,CAAC,CAAC;YAAA;YAAA;YAAA;YAAA,2BACVI,yBAAyB,CAACF,IAAI,EAAED,OAAO,CAAC;UAAA;YAAA;YAAA;UAAA;YAAA;cAAA;cAAA;YAAA;YAAjDI,KAAK;YAAA,kCACbA,KAAK;UAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;UAAA;YAAA;UAAA;YAAA;UAAA;YAAA,kCAEP,IAAI;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACZ;EAAA;AAAA;AAAA,SAEsBD,yBAAyB;EAAA;AAAA;AAAA;EAAA,yFAAzC,iBAA0CF,IAAU,EAAED,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YAAA;YAAA,0CACpEK,4BAAa,CAACC,QAAQ,CAACL,IAAI,CAAC;UAAA;YAA3CM,MAAM;YACNC,IAAa,GAAG,EAAE;YAAA;YAEhBC,MAAM,GAAGF,MAAM,CAACG,SAAS,EAAE;UAAA;YAAA;YAAA,0CAEVD,MAAM,CAACE,IAAI,EAAE;UAAA;YAAA,MAA5BC,MAAM;cAAA;cAAA;YAAA;YACZJ,IAAI,CAACK,IAAI,CAACD,MAAM,CAAC;YAAC;YAAA;UAAA;YAAA;YAAA;YAAA,0CAGdL,MAAM,CAACO,KAAK,EAAE;UAAA;YAAA;UAAA;YAAA;YAEtB,OAAMN,IAAI;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACX;EAAA;AAAA"}
+{"version":3,"file":"parse-parquet.js","names":["parseParquet","arrayBuffer","options","blob","Blob","parseParquetFileInBatches","batch","file","makeReadableFile","reader","ParquetReader","rowBatches","rowBatchIterator","parquet","rows"],"sources":["../../../src/lib/parse-parquet.ts"],"sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n// import {ColumnarTableBatch} from '@loaders.gl/schema';\nimport {makeReadableFile} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../parquet-loader';\nimport {ParquetReader} from '../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInBatches(blob, options)) {\n    return batch;\n  }\n  return null;\n}\n\nexport async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {\n  const file = makeReadableFile(blob);\n  const reader = new ParquetReader(file);\n  const rowBatches = reader.rowBatchIterator(options?.parquet);\n  for await (const rows of rowBatches) {\n    yield rows;\n  }\n}\n\n// export async function* parseParquetFileInColumnarBatches(blob: Blob, options?: {columnList?: string[][]}): AsyncIterable<ColumnarTableBatch> {\n//   const rowGroupReader = new ParquetRowGroupReader({data: blob, columnList: options?.columnList});\n//   try {\n//     for await (const rowGroup of rowGroupReader) {\n//       yield convertRowGroupToTableBatch(rowGroup);\n//     }\n//   } finally {\n//     await rowGroupReader.close();\n//   }\n// }\n\n// function convertRowGroupToTableBatch(rowGroup): ColumnarTableBatch {\n//   // @ts-expect-error\n//   return {\n//     data: rowGroup\n//   };\n// }\n"],"mappings":";;;;;;;;;;;;AAEA;AAEA;AAAiE;AAAA;AAAA,SAE3CA,YAAY;EAAA;AAAA;AAAA;EAAA,0EAA3B,kBAA4BC,WAAwB,EAAEC,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YACnFC,IAAI,GAAG,IAAIC,IAAI,CAAC,CAACH,WAAW,CAAC,CAAC;YAAA;YAAA;YAAA;YAAA,2BACVI,yBAAyB,CAACF,IAAI,EAAED,OAAO,CAAC;UAAA;YAAA;YAAA;UAAA;YAAA;cAAA;cAAA;YAAA;YAAjDI,KAAK;YAAA,kCACbA,KAAK;UAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;UAAA;YAAA;UAAA;YAAA;UAAA;YAAA,kCAEP,IAAI;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACZ;EAAA;AAAA;AAAA,SAEsBD,yBAAyB;EAAA;AAAA;AAAA;EAAA,yFAAzC,iBAA0CF,IAAU,EAAED,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YACnFK,IAAI,GAAG,IAAAC,6BAAgB,EAACL,IAAI,CAAC;YAC7BM,MAAM,GAAG,IAAIC,4BAAa,CAACH,IAAI,CAAC;YAChCI,UAAU,GAAGF,MAAM,CAACG,gBAAgB,CAACV,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEW,OAAO,CAAC;YAAA;YAAA;YAAA;YAAA,4BACnCF,UAAU;UAAA;YAAA;YAAA;UAAA;YAAA;cAAA;cAAA;YAAA;YAAlBG,IAAI;YAAA;YACnB,OAAMA,IAAI;UAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;UAAA;YAAA;UAAA;YAAA;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CAEb;EAAA;AAAA"}

package/dist/es5/parquet-loader.js

@@ -4,11 +4,12 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports._typecheckParquetLoader = exports.ParquetLoader = void 0;
-var VERSION = typeof "3.3.0-alpha.8" !== 'undefined' ? "3.3.0-alpha.8" : 'latest';
+var VERSION = typeof "3.3.0" !== 'undefined' ? "3.3.0" : 'latest';
 var DEFAULT_PARQUET_LOADER_OPTIONS = {
   parquet: {
     type: 'object-row-table',
-    url: undefined
+    url: undefined,
+    columnList: []
   }
 };
 

package/dist/es5/parquet-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-loader.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"sources":["../../src/parquet-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAS3E,IAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,IAAI,EAAE,kBAAkB;IACxBC,GAAG,EAAEC;EACP;AACF,CAAC;;AAGM,IAAMC,aAAa,GAAG;EAC3BC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,SAAS;EACbC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAEV,OAAO;EAChBW,MAAM,EAAE,IAAI;EACZC,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;EACvBC,OAAO,EAAEhB;AACX,CAAC;AAAC;AAEK,IAAMiB,uBAA+B,GAAGZ,aAAa;AAAC"}
+{"version":3,"file":"parquet-loader.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","url","undefined","columnList","ParquetLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"sources":["../../src/parquet-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'object-row-table';\n    url?: string;\n    columnList?: string[] | string[][];\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'object-row-table',\n    url: undefined,\n    columnList: []\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  worker: true,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetLoader;\n"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,cAAkB,KAAK,WAAW,aAAiB,QAAQ;AAU3E,IAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,IAAI,EAAE,kBAAkB;IACxBC,GAAG,EAAEC,SAAS;IACdC,UAAU,EAAE;EACd;AACF,CAAC;;AAGM,IAAMC,aAAa,GAAG;EAC3BC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,SAAS;EACbC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAEX,OAAO;EAChBY,MAAM,EAAE,IAAI;EACZC,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;EACvBC,OAAO,EAAEjB;AACX,CAAC;AAAC;AAEK,IAAMkB,uBAA+B,GAAGZ,aAAa;AAAC"}

package/dist/es5/parquet-wasm-loader.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports._typecheckParquetLoader = exports.ParquetWasmLoader = void 0;
-var VERSION = typeof "3.3.0-alpha.8" !== 'undefined' ? "3.3.0-alpha.8" : 'latest';
+var VERSION = typeof "3.3.0" !== 'undefined' ? "3.3.0" : 'latest';
 var DEFAULT_PARQUET_LOADER_OPTIONS = {
   parquet: {
     type: 'arrow-table',

package/dist/es5/parquet-wasm-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-wasm-loader.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","wasmUrl","ParquetWasmLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"sources":["../../src/parquet-wasm-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'arrow-table';\n    wasmUrl?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'arrow-table',\n    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetWasmLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet-wasm',\n  module: 'parquet',\n  version: VERSION,\n  worker: false,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetWasmLoader;\n"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAS3E,IAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,IAAI,EAAE,aAAa;IACnBC,OAAO,EAAE;EACX;AACF,CAAC;;AAGM,IAAMC,iBAAiB,GAAG;EAC/BC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,cAAc;EAClBC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAET,OAAO;EAChBU,MAAM,EAAE,KAAK;EACbC,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;EACvBC,OAAO,EAAEf;AACX,CAAC;AAAC;AAEK,IAAMgB,uBAA+B,GAAGZ,iBAAiB;AAAC"}
+{"version":3,"file":"parquet-wasm-loader.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","parquet","type","wasmUrl","ParquetWasmLoader","name","id","module","version","worker","category","extensions","mimeTypes","binary","tests","options","_typecheckParquetLoader"],"sources":["../../src/parquet-wasm-loader.ts"],"sourcesContent":["import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetLoaderOptions = LoaderOptions & {\n  parquet?: {\n    type?: 'arrow-table';\n    wasmUrl?: string;\n  };\n};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {\n  parquet: {\n    type: 'arrow-table',\n    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n  }\n};\n\n/** ParquetJS table loader */\nexport const ParquetWasmLoader = {\n  name: 'Apache Parquet',\n  id: 'parquet-wasm',\n  module: 'parquet',\n  version: VERSION,\n  worker: false,\n  category: 'table',\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  binary: true,\n  tests: ['PAR1', 'PARE'],\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nexport const _typecheckParquetLoader: Loader = ParquetWasmLoader;\n"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,cAAkB,KAAK,WAAW,aAAiB,QAAQ;AAS3E,IAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,IAAI,EAAE,aAAa;IACnBC,OAAO,EAAE;EACX;AACF,CAAC;;AAGM,IAAMC,iBAAiB,GAAG;EAC/BC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,cAAc;EAClBC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAET,OAAO;EAChBU,MAAM,EAAE,KAAK;EACbC,QAAQ,EAAE,OAAO;EACjBC,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,MAAM,EAAE,IAAI;EACZC,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;EACvBC,OAAO,EAAEf;AACX,CAAC;AAAC;AAEK,IAAMgB,uBAA+B,GAAGZ,iBAAiB;AAAC"}

package/dist/es5/parquet-wasm-writer.js

@@ -5,7 +5,7 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.ParquetWasmWriter = void 0;
 var _encodeParquetWasm = require("./lib/wasm/encode-parquet-wasm");
-var VERSION = typeof "3.3.0-alpha.8" !== 'undefined' ? "3.3.0-alpha.8" : 'latest';
+var VERSION = typeof "3.3.0" !== 'undefined' ? "3.3.0" : 'latest';
 var DEFAULT_PARQUET_WRITER_OPTIONS = {
   parquet: {
     wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'

package/dist/es5/parquet-wasm-writer.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-wasm-writer.js","names":["VERSION","DEFAULT_PARQUET_WRITER_OPTIONS","parquet","wasmUrl","ParquetWasmWriter","name","id","module","version","extensions","mimeTypes","encode","binary","options"],"sources":["../../src/parquet-wasm-writer.ts"],"sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\nimport {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nconst DEFAULT_PARQUET_WRITER_OPTIONS: ParquetWriterOptions = {\n  parquet: {\n    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n  }\n};\n\nexport const ParquetWasmWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet-wasm',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encode,\n  binary: true,\n  options: DEFAULT_PARQUET_WRITER_OPTIONS\n};\n"],"mappings":";;;;;;AACA;AAIA,IAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAE3E,IAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,OAAO,EAAE;EACX;AACF,CAAC;AAEM,IAAMC,iBAAyB,GAAG;EACvCC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,cAAc;EAClBC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAER,OAAO;EAChBS,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,MAAM,EAANA,yBAAM;EACNC,MAAM,EAAE,IAAI;EACZC,OAAO,EAAEZ;AACX,CAAC;AAAC"}
+{"version":3,"file":"parquet-wasm-writer.js","names":["VERSION","DEFAULT_PARQUET_WRITER_OPTIONS","parquet","wasmUrl","ParquetWasmWriter","name","id","module","version","extensions","mimeTypes","encode","binary","options"],"sources":["../../src/parquet-wasm-writer.ts"],"sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\nimport {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nconst DEFAULT_PARQUET_WRITER_OPTIONS: ParquetWriterOptions = {\n  parquet: {\n    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'\n  }\n};\n\nexport const ParquetWasmWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet-wasm',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encode,\n  binary: true,\n  options: DEFAULT_PARQUET_WRITER_OPTIONS\n};\n"],"mappings":";;;;;;AACA;AAIA,IAAMA,OAAO,GAAG,cAAkB,KAAK,WAAW,aAAiB,QAAQ;AAE3E,IAAMC,8BAAoD,GAAG;EAC3DC,OAAO,EAAE;IACPC,OAAO,EAAE;EACX;AACF,CAAC;AAEM,IAAMC,iBAAyB,GAAG;EACvCC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,cAAc;EAClBC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAER,OAAO;EAChBS,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,MAAM,EAANA,yBAAM;EACNC,MAAM,EAAE,IAAI;EACZC,OAAO,EAAEZ;AACX,CAAC;AAAC"}

package/dist/es5/parquet-writer.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ParquetWriter = void 0;
-var VERSION = typeof "3.3.0-alpha.8" !== 'undefined' ? "3.3.0-alpha.8" : 'latest';
+var VERSION = typeof "3.3.0" !== 'undefined' ? "3.3.0" : 'latest';
 var DEFAULT_PARQUET_LOADER_OPTIONS = {};
 var ParquetWriter = {
   name: 'Apache Parquet',

package/dist/es5/parquet-writer.js.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-writer.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"sources":["../../src/parquet-writer.ts"],"sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n  return new ArrayBuffer(0);\n}\n"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAI3E,IAAMC,8BAA8B,GAAG,CAAC,CAAC;AAElC,IAAMC,aAAqB,GAAG;EACnCC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,SAAS;EACbC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAEN,OAAO;EAChBO,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,UAAU,EAAVA,UAAU;EACVC,MAAM,EAAE,IAAI;EACZC,OAAO,EAAEV;AACX,CAAC;AAAC;AAEF,SAASQ,UAAU,CAACG,IAAI,EAAED,OAA8B,EAAE;EACxD,OAAO,IAAIE,WAAW,CAAC,CAAC,CAAC;AAC3B"}
+{"version":3,"file":"parquet-writer.js","names":["VERSION","DEFAULT_PARQUET_LOADER_OPTIONS","ParquetWriter","name","id","module","version","extensions","mimeTypes","encodeSync","binary","options","data","ArrayBuffer"],"sources":["../../src/parquet-writer.ts"],"sourcesContent":["import type {Writer} from '@loaders.gl/loader-utils';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type ParquetWriterOptions = {};\n\nconst DEFAULT_PARQUET_LOADER_OPTIONS = {};\n\nexport const ParquetWriter: Writer = {\n  name: 'Apache Parquet',\n  id: 'parquet',\n  module: 'parquet',\n  version: VERSION,\n  extensions: ['parquet'],\n  mimeTypes: ['application/octet-stream'],\n  encodeSync,\n  binary: true,\n  options: DEFAULT_PARQUET_LOADER_OPTIONS\n};\n\nfunction encodeSync(data, options?: ParquetWriterOptions) {\n  return new ArrayBuffer(0);\n}\n"],"mappings":";;;;;;AAIA,IAAMA,OAAO,GAAG,cAAkB,KAAK,WAAW,aAAiB,QAAQ;AAI3E,IAAMC,8BAA8B,GAAG,CAAC,CAAC;AAElC,IAAMC,aAAqB,GAAG;EACnCC,IAAI,EAAE,gBAAgB;EACtBC,EAAE,EAAE,SAAS;EACbC,MAAM,EAAE,SAAS;EACjBC,OAAO,EAAEN,OAAO;EAChBO,UAAU,EAAE,CAAC,SAAS,CAAC;EACvBC,SAAS,EAAE,CAAC,0BAA0B,CAAC;EACvCC,UAAU,EAAVA,UAAU;EACVC,MAAM,EAAE,IAAI;EACZC,OAAO,EAAEV;AACX,CAAC;AAAC;AAEF,SAASQ,UAAU,CAACG,IAAI,EAAED,OAA8B,EAAE;EACxD,OAAO,IAAIE,WAAW,CAAC,CAAC,CAAC;AAC3B"}

package/dist/es5/parquetjs/compression.js

@@ -12,10 +12,20 @@ exports.preloadCompressions = preloadCompressions;
 var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
 var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
 var _compression = require("@loaders.gl/compression");
-var _bufferUtils = require("./utils/buffer-utils");
 var _lz4js = _interopRequireDefault(require("lz4js"));
 var _lzo = _interopRequireDefault(require("lzo"));
 
+function toBuffer(arrayBuffer) {
+  return Buffer.from(arrayBuffer);
+}
+function toArrayBuffer(buffer) {
+  if (Buffer.isBuffer(buffer)) {
+    var typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
+    return typedArray.slice().buffer;
+  }
+  return buffer;
+}
+
 var modules = {
   lz4js: _lz4js.default,
   lzo: _lzo.default
@@ -86,12 +96,12 @@ function _deflate() {
             }
             throw new Error("parquet: invalid compression method: ".concat(method));
           case 3:
-            inputArrayBuffer = (0, _bufferUtils.toArrayBuffer)(value);
+            inputArrayBuffer = toArrayBuffer(value);
             _context2.next = 6;
             return compression.compress(inputArrayBuffer);
           case 6:
             compressedArrayBuffer = _context2.sent;
-            return _context2.abrupt("return", (0, _bufferUtils.toBuffer)(compressedArrayBuffer));
+            return _context2.abrupt("return", toBuffer(compressedArrayBuffer));
           case 8:
           case "end":
             return _context2.stop();
@@ -118,12 +128,12 @@ function _decompress() {
             }
             throw new Error("parquet: invalid compression method: ".concat(method));
           case 3:
-            inputArrayBuffer = (0, _bufferUtils.toArrayBuffer)(value);
+            inputArrayBuffer = toArrayBuffer(value);
             _context3.next = 6;
             return compression.decompress(inputArrayBuffer, size);
           case 6:
             compressedArrayBuffer = _context3.sent;
-            return _context3.abrupt("return", (0, _bufferUtils.toBuffer)(compressedArrayBuffer));
+            return _context3.abrupt("return", toBuffer(compressedArrayBuffer));
           case 8:
           case "end":
             return _context3.stop();

package/dist/es5/parquetjs/compression.js.map

@@ -1 +1 @@
-{"version":3,"file":"compression.js","names":["modules","lz4js","lzo","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","NoCompression","GZIP","GZipCompression","SNAPPY","SnappyCompression","BROTLI","BrotliCompression","LZ4","LZ4Compression","LZ4_RAW","LZO","LZOCompression","ZSTD","ZstdCompression","preloadCompressions","options","compressions","Object","values","Promise","all","map","compression","preload","deflate","method","value","Error","inputArrayBuffer","toArrayBuffer","compress","compressedArrayBuffer","toBuffer","decompress","size","inflate"],"sources":["../../../src/parquetjs/compression.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\nimport {toArrayBuffer, toBuffer} from './utils/buffer-utils';\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\n// import brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\nimport lzo from 'lzo';\n// import {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  // brotli: {\n  //   decompress: brotliDecompress,\n  //   compress: () => {\n  //     throw new Error('brotli compress');\n  //   }\n  // },\n  lz4js,\n  lzo\n  // 'zstd-codec': ZstdCodec\n};\n\n// See https://github.com/apache/parquet-format/blob/master/Compression.md\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"mappings":";;;;;;;;;;;;;AAIA;AAYA;AAMA;AACA;;AAIA,IAAMA,OAAO,GAAG;EAQdC,KAAK,EAALA,cAAK;EACLC,GAAG,EAAHA;AAEF,CAAC;;AAGM,IAAMC,2BAAoE,GAAG;EAClFC,YAAY,EAAE,IAAIC,0BAAa,EAAE;EACjCC,IAAI,EAAE,IAAIC,4BAAe,EAAE;EAC3BC,MAAM,EAAE,IAAIC,8BAAiB,EAAE;EAC/BC,MAAM,EAAE,IAAIC,8BAAiB,CAAC;IAACX,OAAO,EAAPA;EAAO,CAAC,CAAC;EAExCY,GAAG,EAAE,IAAIC,2BAAc,CAAC;IAACb,OAAO,EAAPA;EAAO,CAAC,CAAC;EAClCc,OAAO,EAAE,IAAID,2BAAc,CAAC;IAACb,OAAO,EAAPA;EAAO,CAAC,CAAC;EACtCe,GAAG,EAAE,IAAIC,2BAAc,CAAC;IAAChB,OAAO,EAAPA;EAAO,CAAC,CAAC;EAClCiB,IAAI,EAAE,IAAIC,4BAAe,CAAC;IAAClB,OAAO,EAAPA;EAAO,CAAC;AACrC,CAAC;;AAAC;AAAA,SAMoBmB,mBAAmB;EAAA;AAAA;AAAA;EAAA,iFAAlC,iBAAmCC,OAAyC;IAAA;IAAA;MAAA;QAAA;UAAA;YAC3EC,YAAY,GAAGC,MAAM,CAACC,MAAM,CAACpB,2BAA2B,CAAC;YAAA;YAAA,OAClDqB,OAAO,CAACC,GAAG,CAACJ,YAAY,CAACK,GAAG,CAAC,UAACC,WAAW;cAAA,OAAKA,WAAW,CAACC,OAAO,EAAE;YAAA,EAAC,CAAC;UAAA;YAAA;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACnF;EAAA;AAAA;AAAA,SAKqBC,OAAO;EAAA;AAAA;AAAA;EAAA,qEAAtB,kBAAuBC,MAA0B,EAAEC,KAAa;IAAA;IAAA;MAAA;QAAA;UAAA;YAC/DJ,WAAW,GAAGxB,2BAA2B,CAAC2B,MAAM,CAAC;YAAA,IAClDH,WAAW;cAAA;cAAA;YAAA;YAAA,MACR,IAAIK,KAAK,gDAAyCF,MAAM,EAAG;UAAA;YAE7DG,gBAAgB,GAAG,IAAAC,0BAAa,EAACH,KAAK,CAAC;YAAA;YAAA,OACTJ,WAAW,CAACQ,QAAQ,CAACF,gBAAgB,CAAC;UAAA;YAApEG,qBAAqB;YAAA,kCACpB,IAAAC,qBAAQ,EAACD,qBAAqB,CAAC;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACvC;EAAA;AAAA;AAAA,SAKqBE,UAAU;EAAA;AAAA;AAAA;EAAA,wEAAzB,kBACLR,MAA0B,EAC1BC,KAAa,EACbQ,IAAY;IAAA;IAAA;MAAA;QAAA;UAAA;YAENZ,WAAW,GAAGxB,2BAA2B,CAAC2B,MAAM,CAAC;YAAA,IAClDH,WAAW;cAAA;cAAA;YAAA;YAAA,MACR,IAAIK,KAAK,gDAAyCF,MAAM,EAAG;UAAA;YAE7DG,gBAAgB,GAAG,IAAAC,0BAAa,EAACH,KAAK,CAAC;YAAA;YAAA,OACTJ,WAAW,CAACW,UAAU,CAACL,gBAAgB,EAAEM,IAAI,CAAC;UAAA;YAA5EH,qBAAqB;YAAA,kCACpB,IAAAC,qBAAQ,EAACD,qBAAqB,CAAC;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACvC;EAAA;AAAA;AAKM,SAASI,OAAO,CAACV,MAA0B,EAAEC,KAAa,EAAEQ,IAAY,EAAU;EACvF,IAAI,EAAET,MAAM,IAAI3B,2BAA2B,CAAC,EAAE;IAC5C,MAAM,IAAI6B,KAAK,uCAAgCF,MAAM,EAAG;EAC1D;EAEA,OAAO3B,2BAA2B,CAAC2B,MAAM,CAAC,CAACU,OAAO,CAACT,KAAK,EAAEQ,IAAI,CAAC;AACjE"}
+{"version":3,"file":"compression.js","names":["toBuffer","arrayBuffer","Buffer","from","toArrayBuffer","buffer","isBuffer","typedArray","Uint8Array","byteOffset","length","slice","modules","lz4js","lzo","PARQUET_COMPRESSION_METHODS","UNCOMPRESSED","NoCompression","GZIP","GZipCompression","SNAPPY","SnappyCompression","BROTLI","BrotliCompression","LZ4","LZ4Compression","LZ4_RAW","LZO","LZOCompression","ZSTD","ZstdCompression","preloadCompressions","options","compressions","Object","values","Promise","all","map","compression","preload","deflate","method","value","Error","inputArrayBuffer","compress","compressedArrayBuffer","decompress","size","inflate"],"sources":["../../../src/parquetjs/compression.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n/* eslint-disable camelcase */\n// Forked from https://github.com/ironSource/parquetjs under MIT license\n\nimport {\n  Compression,\n  NoCompression,\n  GZipCompression,\n  SnappyCompression,\n  BrotliCompression,\n  LZOCompression,\n  LZ4Compression,\n  ZstdCompression\n} from '@loaders.gl/compression';\n\nimport {ParquetCompression} from './schema/declare';\n\n/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */\nfunction toBuffer(arrayBuffer: ArrayBuffer): Buffer {\n  return Buffer.from(arrayBuffer);\n}\n\nfunction toArrayBuffer(buffer: Buffer): ArrayBuffer {\n  // TODO - per docs we should just be able to call buffer.buffer, but there are issues\n  if (Buffer.isBuffer(buffer)) {\n    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);\n    return typedArray.slice().buffer;\n  }\n  return buffer;\n}\n\n// TODO switch to worker compression to avoid bundling...\n\n// import brotli from 'brotli'; - brotli has problems with decompress in browsers\n// import brotliDecompress from 'brotli/decompress';\nimport lz4js from 'lz4js';\nimport lzo from 'lzo';\n// import {ZstdCodec} from 'zstd-codec';\n\n// Inject large dependencies through Compression constructor options\nconst modules = {\n  // brotli has problems with decompress in browsers\n  // brotli: {\n  //   decompress: brotliDecompress,\n  //   compress: () => {\n  //     throw new Error('brotli compress');\n  //   }\n  // },\n  lz4js,\n  lzo\n  // 'zstd-codec': ZstdCodec\n};\n\n// See https://github.com/apache/parquet-format/blob/master/Compression.md\nexport const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {\n  UNCOMPRESSED: new NoCompression(),\n  GZIP: new GZipCompression(),\n  SNAPPY: new SnappyCompression(),\n  BROTLI: new BrotliCompression({modules}),\n  // TODO: Understand difference between LZ4 and LZ4_RAW\n  LZ4: new LZ4Compression({modules}),\n  LZ4_RAW: new LZ4Compression({modules}),\n  LZO: new LZOCompression({modules}),\n  ZSTD: new ZstdCompression({modules})\n};\n\n/**\n * Register compressions that have big external libraries\n * @param options.modules External library dependencies\n */\nexport async function preloadCompressions(options?: {modules: {[key: string]: any}}) {\n  const compressions = Object.values(PARQUET_COMPRESSION_METHODS);\n  return await Promise.all(compressions.map((compression) => compression.preload()));\n}\n\n/**\n * Deflate a value using compression method `method`\n */\nexport async function deflate(method: ParquetCompression, value: Buffer): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.compress(inputArrayBuffer);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/**\n * Inflate a value using compression method `method`\n */\nexport async function decompress(\n  method: ParquetCompression,\n  value: Buffer,\n  size: number\n): Promise<Buffer> {\n  const compression = PARQUET_COMPRESSION_METHODS[method];\n  if (!compression) {\n    throw new Error(`parquet: invalid compression method: ${method}`);\n  }\n  const inputArrayBuffer = toArrayBuffer(value);\n  const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);\n  return toBuffer(compressedArrayBuffer);\n}\n\n/*\n * Inflate a value using compression method `method`\n */\nexport function inflate(method: ParquetCompression, value: Buffer, size: number): Buffer {\n  if (!(method in PARQUET_COMPRESSION_METHODS)) {\n    throw new Error(`invalid compression method: ${method}`);\n  }\n  // @ts-ignore\n  return PARQUET_COMPRESSION_METHODS[method].inflate(value, size);\n}\n\n/*\nfunction deflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction deflate_gzip(value: Buffer): Buffer {\n  return zlib.gzipSync(value);\n}\n\nfunction deflate_snappy(value: Buffer): Buffer {\n  return snappyjs.compress(value);\n}\n\nfunction deflate_lzo(value: Buffer): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.compress(value);\n}\n\nfunction deflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  const result = brotli.compress(value, {\n    mode: 0,\n    quality: 8,\n    lgwin: 22\n  });\n  return result ? Buffer.from(result) : Buffer.alloc(0);\n}\n\nfunction deflate_lz4(value: Buffer): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(lz4js.encodeBound(value.length));\n    // const compressedSize = lz4.encodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, compressedSize);\n    // return result;\n    return Buffer.from(lz4js.compress(value));\n  } catch (err) {\n    throw err;\n  }\n}\nfunction inflate_identity(value: Buffer): Buffer {\n  return value;\n}\n\nfunction inflate_gzip(value: Buffer): Buffer {\n  return zlib.gunzipSync(value);\n}\n\nfunction inflate_snappy(value: Buffer): Buffer {\n  return snappyjs.uncompress(value);\n}\n\nfunction inflate_lzo(value: Buffer, size: number): Buffer {\n  lzo = lzo || Util.load('lzo');\n  return lzo.decompress(value, size);\n}\n\nfunction inflate_lz4(value: Buffer, size: number): Buffer {\n  lz4js = lz4js || Util.load('lz4js');\n  try {\n    // let result = Buffer.alloc(size);\n    // const uncompressedSize = lz4js.decodeBlock(value, result);\n    // // remove unnecessary bytes\n    // result = result.slice(0, uncompressedSize);\n    // return result;\n    return Buffer.from(lz4js.decompress(value, size));\n  } catch (err) {\n    throw err;\n  }\n}\n\nfunction inflate_brotli(value: Buffer): Buffer {\n  brotli = brotli || Util.load('brotli');\n  if (!value.length) {\n    return Buffer.alloc(0);\n  }\n  return Buffer.from(brotli.decompress(value));\n}\n*/\n"],"mappings":";;;;;;;;;;;;;AAIA;AA+BA;AACA;;AAlBA,SAASA,QAAQ,CAACC,WAAwB,EAAU;EAClD,OAAOC,MAAM,CAACC,IAAI,CAACF,WAAW,CAAC;AACjC;AAEA,SAASG,aAAa,CAACC,MAAc,EAAe;EAElD,IAAIH,MAAM,CAACI,QAAQ,CAACD,MAAM,CAAC,EAAE;IAC3B,IAAME,UAAU,GAAG,IAAIC,UAAU,CAACH,MAAM,CAACA,MAAM,EAAEA,MAAM,CAACI,UAAU,EAAEJ,MAAM,CAACK,MAAM,CAAC;IAClF,OAAOH,UAAU,CAACI,KAAK,EAAE,CAACN,MAAM;EAClC;EACA,OAAOA,MAAM;AACf;;AAWA,IAAMO,OAAO,GAAG;EAQdC,KAAK,EAALA,cAAK;EACLC,GAAG,EAAHA;AAEF,CAAC;;AAGM,IAAMC,2BAAoE,GAAG;EAClFC,YAAY,EAAE,IAAIC,0BAAa,EAAE;EACjCC,IAAI,EAAE,IAAIC,4BAAe,EAAE;EAC3BC,MAAM,EAAE,IAAIC,8BAAiB,EAAE;EAC/BC,MAAM,EAAE,IAAIC,8BAAiB,CAAC;IAACX,OAAO,EAAPA;EAAO,CAAC,CAAC;EAExCY,GAAG,EAAE,IAAIC,2BAAc,CAAC;IAACb,OAAO,EAAPA;EAAO,CAAC,CAAC;EAClCc,OAAO,EAAE,IAAID,2BAAc,CAAC;IAACb,OAAO,EAAPA;EAAO,CAAC,CAAC;EACtCe,GAAG,EAAE,IAAIC,2BAAc,CAAC;IAAChB,OAAO,EAAPA;EAAO,CAAC,CAAC;EAClCiB,IAAI,EAAE,IAAIC,4BAAe,CAAC;IAAClB,OAAO,EAAPA;EAAO,CAAC;AACrC,CAAC;;AAAC;AAAA,SAMoBmB,mBAAmB;EAAA;AAAA;AAAA;EAAA,iFAAlC,iBAAmCC,OAAyC;IAAA;IAAA;MAAA;QAAA;UAAA;YAC3EC,YAAY,GAAGC,MAAM,CAACC,MAAM,CAACpB,2BAA2B,CAAC;YAAA;YAAA,OAClDqB,OAAO,CAACC,GAAG,CAACJ,YAAY,CAACK,GAAG,CAAC,UAACC,WAAW;cAAA,OAAKA,WAAW,CAACC,OAAO,EAAE;YAAA,EAAC,CAAC;UAAA;YAAA;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACnF;EAAA;AAAA;AAAA,SAKqBC,OAAO;EAAA;AAAA;AAAA;EAAA,qEAAtB,kBAAuBC,MAA0B,EAAEC,KAAa;IAAA;IAAA;MAAA;QAAA;UAAA;YAC/DJ,WAAW,GAAGxB,2BAA2B,CAAC2B,MAAM,CAAC;YAAA,IAClDH,WAAW;cAAA;cAAA;YAAA;YAAA,MACR,IAAIK,KAAK,gDAAyCF,MAAM,EAAG;UAAA;YAE7DG,gBAAgB,GAAGzC,aAAa,CAACuC,KAAK,CAAC;YAAA;YAAA,OACTJ,WAAW,CAACO,QAAQ,CAACD,gBAAgB,CAAC;UAAA;YAApEE,qBAAqB;YAAA,kCACpB/C,QAAQ,CAAC+C,qBAAqB,CAAC;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACvC;EAAA;AAAA;AAAA,SAKqBC,UAAU;EAAA;AAAA;AAAA;EAAA,wEAAzB,kBACLN,MAA0B,EAC1BC,KAAa,EACbM,IAAY;IAAA;IAAA;MAAA;QAAA;UAAA;YAENV,WAAW,GAAGxB,2BAA2B,CAAC2B,MAAM,CAAC;YAAA,IAClDH,WAAW;cAAA;cAAA;YAAA;YAAA,MACR,IAAIK,KAAK,gDAAyCF,MAAM,EAAG;UAAA;YAE7DG,gBAAgB,GAAGzC,aAAa,CAACuC,KAAK,CAAC;YAAA;YAAA,OACTJ,WAAW,CAACS,UAAU,CAACH,gBAAgB,EAAEI,IAAI,CAAC;UAAA;YAA5EF,qBAAqB;YAAA,kCACpB/C,QAAQ,CAAC+C,qBAAqB,CAAC;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACvC;EAAA;AAAA;AAKM,SAASG,OAAO,CAACR,MAA0B,EAAEC,KAAa,EAAEM,IAAY,EAAU;EACvF,IAAI,EAAEP,MAAM,IAAI3B,2BAA2B,CAAC,EAAE;IAC5C,MAAM,IAAI6B,KAAK,uCAAgCF,MAAM,EAAG;EAC1D;EAEA,OAAO3B,2BAA2B,CAAC2B,MAAM,CAAC,CAACQ,OAAO,CAACP,KAAK,EAAEM,IAAI,CAAC;AACjE"}