@loaders.gl/parquet 3.3.2 → 3.4.0-alpha.2

package/dist/es5/lib/parsers/parse-parquet-to-columns.js

@@ -0,0 +1,177 @@
+"use strict";
+
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.parseParquetFileInColumnarBatches = parseParquetFileInColumnarBatches;
+exports.parseParquetInColumns = parseParquetInColumns;
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
+var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
+var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
+var _loaderUtils = require("@loaders.gl/loader-utils");
+var _parquetReader = require("../../parquetjs/parser/parquet-reader");
+var _convertSchemaFromParquet = require("../arrow/convert-schema-from-parquet");
+var _convertRowGroupToColumns = require("../arrow/convert-row-group-to-columns");
+var _decodeGeoMetadata = require("../geo/decode-geo-metadata");
+function _asyncIterator(iterable) { var method, async, sync, retry = 2; for ("undefined" != typeof Symbol && (async = Symbol.asyncIterator, sync = Symbol.iterator); retry--;) { if (async && null != (method = iterable[async])) return method.call(iterable); if (sync && null != (method = iterable[sync])) return new AsyncFromSyncIterator(method.call(iterable)); async = "@@asyncIterator", sync = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
+function AsyncFromSyncIterator(s) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var done = r.done; return Promise.resolve(r.value).then(function (value) { return { value: value, done: done }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(s) { this.s = s, this.n = s.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, return: function _return(value) { var ret = this.s.return; return void 0 === ret ? Promise.resolve({ value: value, done: !0 }) : AsyncFromSyncIteratorContinuation(ret.apply(this.s, arguments)); }, throw: function _throw(value) { var thr = this.s.return; return void 0 === thr ? Promise.reject(value) : AsyncFromSyncIteratorContinuation(thr.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(s); }
+function parseParquetInColumns(_x3, _x4) {
+  return _parseParquetInColumns.apply(this, arguments);
+}
+function _parseParquetInColumns() {
+  _parseParquetInColumns = (0, _asyncToGenerator2.default)(_regenerator.default.mark(function _callee2(arrayBuffer, options) {
+    var blob, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, batch;
+    return _regenerator.default.wrap(function _callee2$(_context2) {
+      while (1) {
+        switch (_context2.prev = _context2.next) {
+          case 0:
+            blob = new Blob([arrayBuffer]);
+            _iteratorAbruptCompletion = false;
+            _didIteratorError = false;
+            _context2.prev = 3;
+            _iterator = _asyncIterator(parseParquetFileInColumnarBatches(blob, options));
+          case 5:
+            _context2.next = 7;
+            return _iterator.next();
+          case 7:
+            if (!(_iteratorAbruptCompletion = !(_step = _context2.sent).done)) {
+              _context2.next = 13;
+              break;
+            }
+            batch = _step.value;
+            return _context2.abrupt("return", batch);
+          case 10:
+            _iteratorAbruptCompletion = false;
+            _context2.next = 5;
+            break;
+          case 13:
+            _context2.next = 19;
+            break;
+          case 15:
+            _context2.prev = 15;
+            _context2.t0 = _context2["catch"](3);
+            _didIteratorError = true;
+            _iteratorError = _context2.t0;
+          case 19:
+            _context2.prev = 19;
+            _context2.prev = 20;
+            if (!(_iteratorAbruptCompletion && _iterator.return != null)) {
+              _context2.next = 24;
+              break;
+            }
+            _context2.next = 24;
+            return _iterator.return();
+          case 24:
+            _context2.prev = 24;
+            if (!_didIteratorError) {
+              _context2.next = 27;
+              break;
+            }
+            throw _iteratorError;
+          case 27:
+            return _context2.finish(24);
+          case 28:
+            return _context2.finish(19);
+          case 29:
+            return _context2.abrupt("return", null);
+          case 30:
+          case "end":
+            return _context2.stop();
+        }
+      }
+    }, _callee2, null, [[3, 15, 19, 29], [20,, 24, 28]]);
+  }));
+  return _parseParquetInColumns.apply(this, arguments);
+}
+function parseParquetFileInColumnarBatches(_x, _x2) {
+  return _parseParquetFileInColumnarBatches.apply(this, arguments);
+}
+function _parseParquetFileInColumnarBatches() {
+  _parseParquetFileInColumnarBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(blob, options) {
+    var file, reader, parquetSchema, parquetMetadata, schema, rowGroups, _iteratorAbruptCompletion2, _didIteratorError2, _iteratorError2, _iterator2, _step2, rowGroup;
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            file = (0, _loaderUtils.makeReadableFile)(blob);
+            reader = new _parquetReader.ParquetReader(file);
+            _context.next = 4;
+            return (0, _awaitAsyncGenerator2.default)(reader.getSchema());
+          case 4:
+            parquetSchema = _context.sent;
+            _context.next = 7;
+            return (0, _awaitAsyncGenerator2.default)(reader.getFileMetadata());
+          case 7:
+            parquetMetadata = _context.sent;
+            schema = (0, _convertSchemaFromParquet.convertSchemaFromParquet)(parquetSchema, parquetMetadata);
+            (0, _decodeGeoMetadata.unpackGeoMetadata)(schema);
+            rowGroups = reader.rowGroupIterator(options === null || options === void 0 ? void 0 : options.parquet);
+            _iteratorAbruptCompletion2 = false;
+            _didIteratorError2 = false;
+            _context.prev = 13;
+            _iterator2 = _asyncIterator(rowGroups);
+          case 15:
+            _context.next = 17;
+            return (0, _awaitAsyncGenerator2.default)(_iterator2.next());
+          case 17:
+            if (!(_iteratorAbruptCompletion2 = !(_step2 = _context.sent).done)) {
+              _context.next = 24;
+              break;
+            }
+            rowGroup = _step2.value;
+            _context.next = 21;
+            return convertRowGroupToTableBatch(schema, rowGroup);
+          case 21:
+            _iteratorAbruptCompletion2 = false;
+            _context.next = 15;
+            break;
+          case 24:
+            _context.next = 30;
+            break;
+          case 26:
+            _context.prev = 26;
+            _context.t0 = _context["catch"](13);
+            _didIteratorError2 = true;
+            _iteratorError2 = _context.t0;
+          case 30:
+            _context.prev = 30;
+            _context.prev = 31;
+            if (!(_iteratorAbruptCompletion2 && _iterator2.return != null)) {
+              _context.next = 35;
+              break;
+            }
+            _context.next = 35;
+            return (0, _awaitAsyncGenerator2.default)(_iterator2.return());
+          case 35:
+            _context.prev = 35;
+            if (!_didIteratorError2) {
+              _context.next = 38;
+              break;
+            }
+            throw _iteratorError2;
+          case 38:
+            return _context.finish(35);
+          case 39:
+            return _context.finish(30);
+          case 40:
+          case "end":
+            return _context.stop();
+        }
+      }
+    }, _callee, null, [[13, 26, 30, 40], [31,, 35, 39]]);
+  }));
+  return _parseParquetFileInColumnarBatches.apply(this, arguments);
+}
+function convertRowGroupToTableBatch(schema, rowGroup) {
+  var data = (0, _convertRowGroupToColumns.convertParquetRowGroupToColumns)(schema, rowGroup);
+  return {
+    shape: 'columnar-table',
+    batchType: 'data',
+    schema: schema,
+    data: data,
+    length: rowGroup.rowCount
+  };
+}
+//# sourceMappingURL=parse-parquet-to-columns.js.map

package/dist/es5/lib/parsers/parse-parquet-to-columns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-to-columns.js","names":["parseParquetInColumns","arrayBuffer","options","blob","Blob","parseParquetFileInColumnarBatches","batch","file","makeReadableFile","reader","ParquetReader","getSchema","parquetSchema","getFileMetadata","parquetMetadata","schema","convertSchemaFromParquet","unpackGeoMetadata","rowGroups","rowGroupIterator","parquet","rowGroup","convertRowGroupToTableBatch","data","convertParquetRowGroupToColumns","shape","batchType","length","rowCount"],"sources":["../../../../src/lib/parsers/parse-parquet-to-columns.ts"],"sourcesContent":["// loaders.gl, MIT license\n\n// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\nimport {ColumnarTableBatch, Schema} from '@loaders.gl/schema';\nimport {makeReadableFile} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\nimport {ParquetBuffer} from '../../parquetjs/schema/declare';\nimport {convertSchemaFromParquet} from '../arrow/convert-schema-from-parquet';\nimport {convertParquetRowGroupToColumns} from '../arrow/convert-row-group-to-columns';\nimport {unpackGeoMetadata} from '../geo/decode-geo-metadata';\n\nexport async function parseParquetInColumns(\n  arrayBuffer: ArrayBuffer,\n  options?: ParquetLoaderOptions\n) {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInColumnarBatches(blob, options)) {\n    return batch;\n  }\n  return null;\n}\n\nexport async function* parseParquetFileInColumnarBatches(\n  blob: Blob,\n  options?: ParquetLoaderOptions\n): AsyncIterable<ColumnarTableBatch> {\n  const file = makeReadableFile(blob);\n  const reader = new ParquetReader(file);\n  const parquetSchema = await reader.getSchema();\n  const parquetMetadata = await reader.getFileMetadata();\n  const schema = convertSchemaFromParquet(parquetSchema, parquetMetadata);\n  unpackGeoMetadata(schema);\n  const rowGroups = reader.rowGroupIterator(options?.parquet);\n  for await (const rowGroup of rowGroups) {\n    yield convertRowGroupToTableBatch(schema, rowGroup);\n  }\n}\n\nfunction convertRowGroupToTableBatch(schema: Schema, rowGroup: ParquetBuffer): ColumnarTableBatch {\n  const data = convertParquetRowGroupToColumns(schema, rowGroup);\n  return {\n    shape: 'columnar-table',\n    batchType: 'data',\n    schema,\n    data,\n    length: rowGroup.rowCount\n  };\n}\n"],"mappings":";;;;;;;;;;;;AAIA;AAEA;AAEA;AACA;AACA;AAA6D;AAAA;AAAA,SAEvCA,qBAAqB;EAAA;AAAA;AAAA;EAAA,mFAApC,kBACLC,WAAwB,EACxBC,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YAExBC,IAAI,GAAG,IAAIC,IAAI,CAAC,CAACH,WAAW,CAAC,CAAC;YAAA;YAAA;YAAA;YAAA,2BACVI,iCAAiC,CAACF,IAAI,EAAED,OAAO,CAAC;UAAA;YAAA;YAAA;UAAA;YAAA;cAAA;cAAA;YAAA;YAAzDI,KAAK;YAAA,kCACbA,KAAK;UAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;UAAA;YAAA;UAAA;YAAA;UAAA;YAAA,kCAEP,IAAI;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACZ;EAAA;AAAA;AAAA,SAEsBD,iCAAiC;EAAA;AAAA;AAAA;EAAA,iGAAjD,iBACLF,IAAU,EACVD,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YAExBK,IAAI,GAAG,IAAAC,6BAAgB,EAACL,IAAI,CAAC;YAC7BM,MAAM,GAAG,IAAIC,4BAAa,CAACH,IAAI,CAAC;YAAA;YAAA,0CACVE,MAAM,CAACE,SAAS,EAAE;UAAA;YAAxCC,aAAa;YAAA;YAAA,0CACWH,MAAM,CAACI,eAAe,EAAE;UAAA;YAAhDC,eAAe;YACfC,MAAM,GAAG,IAAAC,kDAAwB,EAACJ,aAAa,EAAEE,eAAe,CAAC;YACvE,IAAAG,oCAAiB,EAACF,MAAM,CAAC;YACnBG,SAAS,GAAGT,MAAM,CAACU,gBAAgB,CAACjB,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEkB,OAAO,CAAC;YAAA;YAAA;YAAA;YAAA,4BAC9BF,SAAS;UAAA;YAAA;YAAA;UAAA;YAAA;cAAA;cAAA;YAAA;YAArBG,QAAQ;YAAA;YACvB,OAAMC,2BAA2B,CAACP,MAAM,EAAEM,QAAQ,CAAC;UAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;UAAA;YAAA;UAAA;YAAA;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CAEtD;EAAA;AAAA;AAED,SAASC,2BAA2B,CAACP,MAAc,EAAEM,QAAuB,EAAsB;EAChG,IAAME,IAAI,GAAG,IAAAC,yDAA+B,EAACT,MAAM,EAAEM,QAAQ,CAAC;EAC9D,OAAO;IACLI,KAAK,EAAE,gBAAgB;IACvBC,SAAS,EAAE,MAAM;IACjBX,MAAM,EAANA,MAAM;IACNQ,IAAI,EAAJA,IAAI;IACJI,MAAM,EAAEN,QAAQ,CAACO;EACnB,CAAC;AACH"}

package/dist/es5/lib/{parse-parquet.js → parsers/parse-parquet-to-rows.js}

@@ -11,7 +11,7 @@ var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/
 var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
 var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
 var _loaderUtils = require("@loaders.gl/loader-utils");
-var _parquetReader = require("../parquetjs/parser/parquet-reader");
+var _parquetReader = require("../../parquetjs/parser/parquet-reader");
 function _asyncIterator(iterable) { var method, async, sync, retry = 2; for ("undefined" != typeof Symbol && (async = Symbol.asyncIterator, sync = Symbol.iterator); retry--;) { if (async && null != (method = iterable[async])) return method.call(iterable); if (sync && null != (method = iterable[sync])) return new AsyncFromSyncIterator(method.call(iterable)); async = "@@asyncIterator", sync = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
 function AsyncFromSyncIterator(s) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var done = r.done; return Promise.resolve(r.value).then(function (value) { return { value: value, done: done }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(s) { this.s = s, this.n = s.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, return: function _return(value) { var ret = this.s.return; return void 0 === ret ? Promise.resolve({ value: value, done: !0 }) : AsyncFromSyncIteratorContinuation(ret.apply(this.s, arguments)); }, throw: function _throw(value) { var thr = this.s.return; return void 0 === thr ? Promise.reject(value) : AsyncFromSyncIteratorContinuation(thr.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(s); }
 function parseParquet(_x3, _x4) {
@@ -151,4 +151,4 @@ function _parseParquetFileInBatches() {
   }));
   return _parseParquetFileInBatches.apply(this, arguments);
 }
-//# sourceMappingURL=parse-parquet.js.map
+//# sourceMappingURL=parse-parquet-to-rows.js.map

package/dist/es5/lib/parsers/parse-parquet-to-rows.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-to-rows.js","names":["parseParquet","arrayBuffer","options","blob","Blob","parseParquetFileInBatches","batch","file","makeReadableFile","reader","ParquetReader","rowBatches","rowBatchIterator","parquet","rows"],"sources":["../../../../src/lib/parsers/parse-parquet-to-rows.ts"],"sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n// import {ColumnarTableBatch} from '@loaders.gl/schema';\nimport {makeReadableFile} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\n\nexport async function parseParquet(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {\n  const blob = new Blob([arrayBuffer]);\n  for await (const batch of parseParquetFileInBatches(blob, options)) {\n    return batch;\n  }\n  return null;\n}\n\nexport async function* parseParquetFileInBatches(blob: Blob, options?: ParquetLoaderOptions) {\n  const file = makeReadableFile(blob);\n  const reader = new ParquetReader(file);\n  const rowBatches = reader.rowBatchIterator(options?.parquet);\n  for await (const rows of rowBatches) {\n    yield rows;\n  }\n}\n\n// export async function* parseParquetFileInColumnarBatches(blob: Blob, options?: {columnList?: string[][]}): AsyncIterable<ColumnarTableBatch> {\n//   const rowGroupReader = new ParquetRowGroupReader({data: blob, columnList: options?.columnList});\n//   try {\n//     for await (const rowGroup of rowGroupReader) {\n//       yield convertRowGroupToTableBatch(rowGroup);\n//     }\n//   } finally {\n//     await rowGroupReader.close();\n//   }\n// }\n\n// function convertRowGroupToTableBatch(rowGroup): ColumnarTableBatch {\n//   // @ts-expect-error\n//   return {\n//     data: rowGroup\n//   };\n// }\n"],"mappings":";;;;;;;;;;;;AAEA;AAEA;AAAoE;AAAA;AAAA,SAE9CA,YAAY;EAAA;AAAA;AAAA;EAAA,0EAA3B,kBAA4BC,WAAwB,EAAEC,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YACnFC,IAAI,GAAG,IAAIC,IAAI,CAAC,CAACH,WAAW,CAAC,CAAC;YAAA;YAAA;YAAA;YAAA,2BACVI,yBAAyB,CAACF,IAAI,EAAED,OAAO,CAAC;UAAA;YAAA;YAAA;UAAA;YAAA;cAAA;cAAA;YAAA;YAAjDI,KAAK;YAAA,kCACbA,KAAK;UAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;UAAA;YAAA;UAAA;YAAA;UAAA;YAAA,kCAEP,IAAI;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CACZ;EAAA;AAAA;AAAA,SAEsBD,yBAAyB;EAAA;AAAA;AAAA;EAAA,yFAAzC,iBAA0CF,IAAU,EAAED,OAA8B;IAAA;IAAA;MAAA;QAAA;UAAA;YACnFK,IAAI,GAAG,IAAAC,6BAAgB,EAACL,IAAI,CAAC;YAC7BM,MAAM,GAAG,IAAIC,4BAAa,CAACH,IAAI,CAAC;YAChCI,UAAU,GAAGF,MAAM,CAACG,gBAAgB,CAACV,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEW,OAAO,CAAC;YAAA;YAAA;YAAA;YAAA,4BACnCF,UAAU;UAAA;YAAA;YAAA;UAAA;YAAA;cAAA;cAAA;YAAA;YAAlBG,IAAI;YAAA;YACnB,OAAMA,IAAI;UAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;YAAA;UAAA;YAAA;YAAA;cAAA;cAAA;YAAA;YAAA;UAAA;YAAA;UAAA;YAAA;UAAA;UAAA;YAAA;QAAA;MAAA;IAAA;EAAA,CAEb;EAAA;AAAA"}