@loaders.gl/json 3.0.13 → 3.0.14

package/dist/es5/lib/parse-json-in-batches.js

@@ -7,93 +7,252 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = parseJSONInBatches;
 
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
+
+var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
+
+var _asyncIterator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncIterator"));
+
 var _schema = require("@loaders.gl/schema");
 
 var _loaderUtils = require("@loaders.gl/loader-utils");
 
 var _streamingJsonParser = _interopRequireDefault(require("./parser/streaming-json-parser"));
 
-async function* parseJSONInBatches(binaryAsyncIterator, options) {
-  var _options$json;
-
-  const asyncIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
-  const {
-    metadata
-  } = options;
-  const {
-    jsonpaths
-  } = options.json || {};
-  let isFirstChunk = true;
-  const schema = null;
-  const shape = (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'row-table';
-  const tableBatchBuilder = new _schema.TableBatchBuilder(schema, { ...options,
-    shape
-  });
-  const parser = new _streamingJsonParser.default({
-    jsonpaths
-  });
-
-  for await (const chunk of asyncIterator) {
-    const rows = parser.write(chunk);
-    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
-
-    if (rows.length > 0 && isFirstChunk) {
-      if (metadata) {
-        const initialBatch = {
-          shape,
-          batchType: 'partial-result',
-          data: [],
-          length: 0,
-          bytesUsed: 0,
-          container: parser.getPartialResult(),
-          jsonpath
-        };
-        yield initialBatch;
-      }
+function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
+
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
+
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
+
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+
+function parseJSONInBatches(_x, _x2) {
+  return _parseJSONInBatches.apply(this, arguments);
+}
+
+function _parseJSONInBatches() {
+  _parseJSONInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(binaryAsyncIterator, options) {
+    var _options$json;
+
+    var asyncIterator, metadata, _ref, jsonpaths, isFirstChunk, schema, shape, tableBatchBuilder, parser, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _value, chunk, rows, _jsonpath, initialBatch, _iterator2, _step2, row, _batch2, _batch, jsonpath, batch, finalBatch;
+
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            asyncIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
+            metadata = options.metadata;
+            _ref = options.json || {}, jsonpaths = _ref.jsonpaths;
+            isFirstChunk = true;
+            schema = null;
+            shape = (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'row-table';
+            tableBatchBuilder = new _schema.TableBatchBuilder(schema, _objectSpread(_objectSpread({}, options), {}, {
+              shape: shape
+            }));
+            parser = new _streamingJsonParser.default({
+              jsonpaths: jsonpaths
+            });
+            _iteratorNormalCompletion = true;
+            _didIteratorError = false;
+            _context.prev = 10;
+            _iterator = (0, _asyncIterator2.default)(asyncIterator);
+
+          case 12:
+            _context.next = 14;
+            return (0, _awaitAsyncGenerator2.default)(_iterator.next());
+
+          case 14:
+            _step = _context.sent;
+            _iteratorNormalCompletion = _step.done;
+            _context.next = 18;
+            return (0, _awaitAsyncGenerator2.default)(_step.value);
+
+          case 18:
+            _value = _context.sent;
+
+            if (_iteratorNormalCompletion) {
+              _context.next = 57;
+              break;
+            }
+
+            chunk = _value;
+            rows = parser.write(chunk);
+            _jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
+
+            if (!(rows.length > 0 && isFirstChunk)) {
+              _context.next = 29;
+              break;
+            }
+
+            if (!metadata) {
+              _context.next = 28;
+              break;
+            }
+
+            initialBatch = {
+              shape: shape,
+              batchType: 'partial-result',
+              data: [],
+              length: 0,
+              bytesUsed: 0,
+              container: parser.getPartialResult(),
+              jsonpath: _jsonpath
+            };
+            _context.next = 28;
+            return initialBatch;
+
+          case 28:
+            isFirstChunk = false;
+
+          case 29:
+            _iterator2 = _createForOfIteratorHelper(rows);
+            _context.prev = 30;
+
+            _iterator2.s();
+
+          case 32:
+            if ((_step2 = _iterator2.n()).done) {
+              _context.next = 41;
+              break;
+            }
+
+            row = _step2.value;
+            tableBatchBuilder.addRow(row);
+            _batch2 = tableBatchBuilder.getFullBatch({
+              jsonpath: _jsonpath
+            });
+
+            if (!_batch2) {
+              _context.next = 39;
+              break;
+            }
+
+            _context.next = 39;
+            return _batch2;
+
+          case 39:
+            _context.next = 32;
+            break;
+
+          case 41:
+            _context.next = 46;
+            break;
+
+          case 43:
+            _context.prev = 43;
+            _context.t0 = _context["catch"](30);
+
+            _iterator2.e(_context.t0);
+
+          case 46:
+            _context.prev = 46;
+
+            _iterator2.f();
+
+            return _context.finish(46);
+
+          case 49:
+            tableBatchBuilder.chunkComplete(chunk);
+            _batch = tableBatchBuilder.getFullBatch({
+              jsonpath: _jsonpath
+            });
+
+            if (!_batch) {
+              _context.next = 54;
+              break;
+            }
+
+            _context.next = 54;
+            return _batch;
+
+          case 54:
+            _iteratorNormalCompletion = true;
+            _context.next = 12;
+            break;
+
+          case 57:
+            _context.next = 63;
+            break;
+
+          case 59:
+            _context.prev = 59;
+            _context.t1 = _context["catch"](10);
+            _didIteratorError = true;
+            _iteratorError = _context.t1;
+
+          case 63:
+            _context.prev = 63;
+            _context.prev = 64;
+
+            if (!(!_iteratorNormalCompletion && _iterator.return != null)) {
+              _context.next = 68;
+              break;
+            }
+
+            _context.next = 68;
+            return (0, _awaitAsyncGenerator2.default)(_iterator.return());
+
+          case 68:
+            _context.prev = 68;
+
+            if (!_didIteratorError) {
+              _context.next = 71;
+              break;
+            }
+
+            throw _iteratorError;
+
+          case 71:
+            return _context.finish(68);
+
+          case 72:
+            return _context.finish(63);
+
+          case 73:
+            jsonpath = parser.getStreamingJsonPathAsString();
+            batch = tableBatchBuilder.getFinalBatch({
+              jsonpath: jsonpath
+            });
+
+            if (!batch) {
+              _context.next = 78;
+              break;
+            }
+
+            _context.next = 78;
+            return batch;
 
-      isFirstChunk = false;
-    }
+          case 78:
+            if (!metadata) {
+              _context.next = 82;
+              break;
+            }
 
-    for (const row of rows) {
-      tableBatchBuilder.addRow(row);
-      const batch = tableBatchBuilder.getFullBatch({
-        jsonpath
-      });
+            finalBatch = {
+              shape: shape,
+              batchType: 'final-result',
+              container: parser.getPartialResult(),
+              jsonpath: parser.getStreamingJsonPathAsString(),
+              data: [],
+              length: 0
+            };
+            _context.next = 82;
+            return finalBatch;
 
-      if (batch) {
-        yield batch;
+          case 82:
+          case "end":
+            return _context.stop();
+        }
       }
-    }
-
-    tableBatchBuilder.chunkComplete(chunk);
-    const batch = tableBatchBuilder.getFullBatch({
-      jsonpath
-    });
-
-    if (batch) {
-      yield batch;
-    }
-  }
-
-  const jsonpath = parser.getStreamingJsonPathAsString();
-  const batch = tableBatchBuilder.getFinalBatch({
-    jsonpath
-  });
-
-  if (batch) {
-    yield batch;
-  }
-
-  if (metadata) {
-    const finalBatch = {
-      shape,
-      batchType: 'final-result',
-      container: parser.getPartialResult(),
-      jsonpath: parser.getStreamingJsonPathAsString(),
-      data: [],
-      length: 0
-    };
-    yield finalBatch;
-  }
+    }, _callee, null, [[10, 59, 63, 73], [30, 43, 46, 49], [64,, 68, 72]]);
+  }));
+  return _parseJSONInBatches.apply(this, arguments);
 }
 //# sourceMappingURL=parse-json-in-batches.js.map

package/dist/es5/lib/parse-json-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-json-in-batches.ts"],"names":["parseJSONInBatches","binaryAsyncIterator","options","asyncIterator","metadata","jsonpaths","json","isFirstChunk","schema","shape","tableBatchBuilder","TableBatchBuilder","parser","StreamingJSONParser","chunk","rows","write","jsonpath","length","getStreamingJsonPathAsString","initialBatch","batchType","data","bytesUsed","container","getPartialResult","row","addRow","batch","getFullBatch","chunkComplete","getFinalBatch","finalBatch"],"mappings":";;;;;;;;;AAEA;;AACA;;AACA;;AAIe,gBAAgBA,kBAAhB,CACbC,mBADa,EAEbC,OAFa,EAGS;AAAA;;AACtB,QAAMC,aAAa,GAAG,0CAAwBF,mBAAxB,CAAtB;AAEA,QAAM;AAACG,IAAAA;AAAD,MAAaF,OAAnB;AACA,QAAM;AAACG,IAAAA;AAAD,MAAcH,OAAO,CAACI,IAAR,IAAgB,EAApC;AAEA,MAAIC,YAAqB,GAAG,IAA5B;AAGA,QAAMC,MAAM,GAAG,IAAf;AACA,QAAMC,KAAK,GAAG,CAAAP,OAAO,SAAP,IAAAA,OAAO,WAAP,6BAAAA,OAAO,CAAEI,IAAT,gEAAeG,KAAf,KAAwB,WAAtC;AAEA,QAAMC,iBAAiB,GAAG,IAAIC,yBAAJ,CAAsBH,MAAtB,EAA8B,EACtD,GAAGN,OADmD;AAEtDO,IAAAA;AAFsD,GAA9B,CAA1B;AAKA,QAAMG,MAAM,GAAG,IAAIC,4BAAJ,CAAwB;AAACR,IAAAA;AAAD,GAAxB,CAAf;;AAEA,aAAW,MAAMS,KAAjB,IAA0BX,aAA1B,EAAyC;AACvC,UAAMY,IAAI,GAAGH,MAAM,CAACI,KAAP,CAAaF,KAAb,CAAb;AAEA,UAAMG,QAAQ,GAAGF,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBN,MAAM,CAACO,4BAAP,EAApC;;AAEA,QAAIJ,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBX,YAAvB,EAAqC;AACnC,UAAIH,QAAJ,EAAc;AACZ,cAAMgB,YAAmB,GAAG;AAE1BX,UAAAA,KAF0B;AAG1BY,UAAAA,SAAS,EAAE,gBAHe;AAI1BC,UAAAA,IAAI,EAAE,EAJoB;AAK1BJ,UAAAA,MAAM,EAAE,CALkB;AAM1BK,UAAAA,SAAS,EAAE,CANe;AAQ1BC,UAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EARe;AAS1BR,UAAAA;AAT0B,SAA5B;AAWA,cAAMG,YAAN;AACD;;AACDb,MAAAA,YAAY,GAAG,KAAf;AAED;;AAGD,SAAK,MAAMmB,GAAX,IAAkBX,IAAlB,EAAwB;AACtBL,MAAAA,iBAAiB,CAACiB,MAAlB,CAAyBD,GAAzB;AAEA,YAAME,KAAK,GAAGlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,QAAAA;AAAD,OAA/B,CAAd;;AACA,UAAIW,KAAJ,EAAW;AACT,cAAMA,KAAN;AACD;AACF;;AAEDlB,IAAAA,iBAAiB,CAACoB,aAAlB,CAAgChB,KAAhC;AACA,UAAMc,KAAK,GAAGlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,MAAAA;AAAD,KAA/B,CAAd;;AACA,QAAIW,KAAJ,EAAW;AACT,YAAMA,KAAN;AACD;AACF;;AAGD,QAAMX,QAAQ,GAAGL,MAAM,CAACO,4BAAP,EAAjB;AACA,QAAMS,KAAK,GAAGlB,iBAAiB,CAACqB,aAAlB,CAAgC;AAACd,IAAAA;AAAD,GAAhC,CAAd;;AACA,MAAIW,KAAJ,EAAW;AACT,UAAMA,KAAN;AACD;;AAED,MAAIxB,QAAJ,EAAc;AACZ,UAAM4B,UAAiB,GAAG;AACxBvB,MAAAA,KADwB;AAExBY,MAAAA,SAAS,EAAE,cAFa;AAGxBG,MAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EAHa;AAIxBR,MAAAA,QAAQ,EAAEL,MAAM,CAACO,4BAAP,EAJc;AAKxBG,MAAAA,IAAI,EAAE,EALkB;AAMxBJ,MAAAA,MAAM,EAAE;AANgB,KAA1B;AASA,UAAMc,UAAN;AACD;AACF","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n  const {metadata} = options;\n  const {jsonpaths} = options.json || {};\n\n  let isFirstChunk: boolean = true;\n\n  // TODO fix Schema deduction\n  const schema = null; // new Schema([]);\n  const shape = options?.json?.shape || 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  const parser = new StreamingJSONParser({jsonpaths});\n\n  for await (const chunk of asyncIterator) {\n    const rows = parser.write(chunk);\n\n    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n    if (rows.length > 0 && isFirstChunk) {\n      if (metadata) {\n        const initialBatch: Batch = {\n          // Common fields\n          shape,\n          batchType: 'partial-result',\n          data: [],\n          length: 0,\n          bytesUsed: 0,\n          // JSON additions\n          container: parser.getPartialResult(),\n          jsonpath\n        };\n        yield initialBatch;\n      }\n      isFirstChunk = false;\n      // schema = deduceSchema(rows);\n    }\n\n    // Add the row\n    for (const row of rows) {\n      tableBatchBuilder.addRow(row);\n      // If a batch has been completed, emit it\n      const batch = tableBatchBuilder.getFullBatch({jsonpath});\n      if (batch) {\n        yield batch;\n      }\n    }\n\n    tableBatchBuilder.chunkComplete(chunk);\n    const batch = tableBatchBuilder.getFullBatch({jsonpath});\n    if (batch) {\n      yield batch;\n    }\n  }\n\n  // yield final batch\n  const jsonpath = parser.getStreamingJsonPathAsString();\n  const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n  if (batch) {\n    yield batch;\n  }\n\n  if (metadata) {\n    const finalBatch: Batch = {\n      shape,\n      batchType: 'final-result',\n      container: parser.getPartialResult(),\n      jsonpath: parser.getStreamingJsonPathAsString(),\n      data: [],\n      length: 0\n      // schema: null\n    };\n    yield finalBatch;\n  }\n}\n"],"file":"parse-json-in-batches.js"}
+{"version":3,"sources":["../../../src/lib/parse-json-in-batches.ts"],"names":["parseJSONInBatches","binaryAsyncIterator","options","asyncIterator","metadata","json","jsonpaths","isFirstChunk","schema","shape","tableBatchBuilder","TableBatchBuilder","parser","StreamingJSONParser","chunk","rows","write","jsonpath","length","getStreamingJsonPathAsString","initialBatch","batchType","data","bytesUsed","container","getPartialResult","row","addRow","batch","getFullBatch","chunkComplete","getFinalBatch","finalBatch"],"mappings":";;;;;;;;;;;;;;;;;;;AAEA;;AACA;;AACA;;;;;;;;;;;;SAI+BA,kB;;;;;oFAAhB,iBACbC,mBADa,EAEbC,OAFa;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAIPC,YAAAA,aAJO,GAIS,0CAAwBF,mBAAxB,CAJT;AAMNG,YAAAA,QANM,GAMMF,OANN,CAMNE,QANM;AAAA,mBAOOF,OAAO,CAACG,IAAR,IAAgB,EAPvB,EAONC,SAPM,QAONA,SAPM;AASTC,YAAAA,YATS,GASe,IATf;AAYPC,YAAAA,MAZO,GAYE,IAZF;AAaPC,YAAAA,KAbO,GAaC,CAAAP,OAAO,SAAP,IAAAA,OAAO,WAAP,6BAAAA,OAAO,CAAEG,IAAT,gEAAeI,KAAf,KAAwB,WAbzB;AAePC,YAAAA,iBAfO,GAea,IAAIC,yBAAJ,CAAsBH,MAAtB,kCACrBN,OADqB;AAExBO,cAAAA,KAAK,EAALA;AAFwB,eAfb;AAoBPG,YAAAA,MApBO,GAoBE,IAAIC,4BAAJ,CAAwB;AAACP,cAAAA,SAAS,EAATA;AAAD,aAAxB,CApBF;AAAA;AAAA;AAAA;AAAA,qDAsBaH,aAtBb;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAsBIW,YAAAA,KAtBJ;AAuBLC,YAAAA,IAvBK,GAuBEH,MAAM,CAACI,KAAP,CAAaF,KAAb,CAvBF;AAyBLG,YAAAA,SAzBK,GAyBMF,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBN,MAAM,CAACO,4BAAP,EAzBzB;;AAAA,kBA2BPJ,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBX,YA3BZ;AAAA;AAAA;AAAA;;AAAA,iBA4BLH,QA5BK;AAAA;AAAA;AAAA;;AA6BDgB,YAAAA,YA7BC,GA6BqB;AAE1BX,cAAAA,KAAK,EAALA,KAF0B;AAG1BY,cAAAA,SAAS,EAAE,gBAHe;AAI1BC,cAAAA,IAAI,EAAE,EAJoB;AAK1BJ,cAAAA,MAAM,EAAE,CALkB;AAM1BK,cAAAA,SAAS,EAAE,CANe;AAQ1BC,cAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EARe;AAS1BR,cAAAA,QAAQ,EAARA;AAT0B,aA7BrB;AAAA;AAwCP,mBAAMG,YAAN;;AAxCO;AA0CTb,YAAAA,YAAY,GAAG,KAAf;;AA1CS;AAAA,oDA+COQ,IA/CP;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AA+CAW,YAAAA,GA/CA;AAgDThB,YAAAA,iBAAiB,CAACiB,MAAlB,CAAyBD,GAAzB;AAEME,YAAAA,OAlDG,GAkDKlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,cAAAA,QAAQ,EAARA;AAAD,aAA/B,CAlDL;;AAAA,iBAmDLW,OAnDK;AAAA;AAAA;AAAA;;AAAA;AAoDP,mBAAMA,OAAN;;AApDO;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;;AAAA;;AAAA;AAwDXlB,YAAAA,iBAAiB,CAACoB,aAAlB,CAAgChB,KAAhC;AACMc,YAAAA,MAzDK,GAyDGlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,cAAAA,QAAQ,EAARA;AAAD,aAA/B,CAzDH;;AAAA,iBA0DPW,MA1DO;AAAA;AAAA;AAAA;;AAAA;AA2DT,mBAAMA,MAAN;;AA3DS;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAgEPX,YAAAA,QAhEO,GAgEIL,MAAM,CAACO,4BAAP,EAhEJ;AAiEPS,YAAAA,KAjEO,GAiEClB,iBAAiB,CAACqB,aAAlB,CAAgC;AAACd,cAAAA,QAAQ,EAARA;AAAD,aAAhC,CAjED;;AAAA,iBAkETW,KAlES;AAAA;AAAA;AAAA;;AAAA;AAmEX,mBAAMA,KAAN;;AAnEW;AAAA,iBAsETxB,QAtES;AAAA;AAAA;AAAA;;AAuEL4B,YAAAA,UAvEK,GAuEe;AACxBvB,cAAAA,KAAK,EAALA,KADwB;AAExBY,cAAAA,SAAS,EAAE,cAFa;AAGxBG,cAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EAHa;AAIxBR,cAAAA,QAAQ,EAAEL,MAAM,CAACO,4BAAP,EAJc;AAKxBG,cAAAA,IAAI,EAAE,EALkB;AAMxBJ,cAAAA,MAAM,EAAE;AANgB,aAvEf;AAAA;AAgFX,mBAAMc,UAAN;;AAhFW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n  const {metadata} = options;\n  const {jsonpaths} = options.json || {};\n\n  let isFirstChunk: boolean = true;\n\n  // TODO fix Schema deduction\n  const schema = null; // new Schema([]);\n  const shape = options?.json?.shape || 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  const parser = new StreamingJSONParser({jsonpaths});\n\n  for await (const chunk of asyncIterator) {\n    const rows = parser.write(chunk);\n\n    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n    if (rows.length > 0 && isFirstChunk) {\n      if (metadata) {\n        const initialBatch: Batch = {\n          // Common fields\n          shape,\n          batchType: 'partial-result',\n          data: [],\n          length: 0,\n          bytesUsed: 0,\n          // JSON additions\n          container: parser.getPartialResult(),\n          jsonpath\n        };\n        yield initialBatch;\n      }\n      isFirstChunk = false;\n      // schema = deduceSchema(rows);\n    }\n\n    // Add the row\n    for (const row of rows) {\n      tableBatchBuilder.addRow(row);\n      // If a batch has been completed, emit it\n      const batch = tableBatchBuilder.getFullBatch({jsonpath});\n      if (batch) {\n        yield batch;\n      }\n    }\n\n    tableBatchBuilder.chunkComplete(chunk);\n    const batch = tableBatchBuilder.getFullBatch({jsonpath});\n    if (batch) {\n      yield batch;\n    }\n  }\n\n  // yield final batch\n  const jsonpath = parser.getStreamingJsonPathAsString();\n  const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n  if (batch) {\n    yield batch;\n  }\n\n  if (metadata) {\n    const finalBatch: Batch = {\n      shape,\n      batchType: 'final-result',\n      container: parser.getPartialResult(),\n      jsonpath: parser.getStreamingJsonPathAsString(),\n      data: [],\n      length: 0\n      // schema: null\n    };\n    yield finalBatch;\n  }\n}\n"],"file":"parse-json-in-batches.js"}

package/dist/es5/lib/parse-json.js

@@ -1,15 +1,19 @@
 "use strict";
 
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = parseJSONSync;
 
+var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
+
 function parseJSONSync(jsonText, options) {
   try {
     var _options$json;
 
-    const json = JSON.parse(jsonText);
+    var json = JSON.parse(jsonText);
 
     if ((_options$json = options.json) !== null && _options$json !== void 0 && _options$json.table) {
       return getFirstArray(json) || json;
@@ -26,9 +30,10 @@ function getFirstArray(json) {
     return json;
   }
 
-  if (json && typeof json === 'object') {
-    for (const value of Object.values(json)) {
-      const array = getFirstArray(value);
+  if (json && (0, _typeof2.default)(json) === 'object') {
+    for (var _i = 0, _Object$values = Object.values(json); _i < _Object$values.length; _i++) {
+      var value = _Object$values[_i];
+      var array = getFirstArray(value);
 
       if (array) {
         return array;

package/dist/es5/lib/parse-json.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-json.ts"],"names":["parseJSONSync","jsonText","options","json","JSON","parse","table","getFirstArray","error","Error","Array","isArray","value","Object","values","array"],"mappings":";;;;;;;AAEe,SAASA,aAAT,CAAuBC,QAAvB,EAAyCC,OAAzC,EAAqE;AAClF,MAAI;AAAA;;AACF,UAAMC,IAAI,GAAGC,IAAI,CAACC,KAAL,CAAWJ,QAAX,CAAb;;AACA,yBAAIC,OAAO,CAACC,IAAZ,0CAAI,cAAcG,KAAlB,EAAyB;AACvB,aAAOC,aAAa,CAACJ,IAAD,CAAb,IAAuBA,IAA9B;AACD;;AACD,WAAOA,IAAP;AACD,GAND,CAME,OAAOK,KAAP,EAAc;AACd,UAAM,IAAIC,KAAJ,CAAU,kCAAV,CAAN;AACD;AACF;;AAED,SAASF,aAAT,CAAuBJ,IAAvB,EAA6B;AAC3B,MAAIO,KAAK,CAACC,OAAN,CAAcR,IAAd,CAAJ,EAAyB;AACvB,WAAOA,IAAP;AACD;;AACD,MAAIA,IAAI,IAAI,OAAOA,IAAP,KAAgB,QAA5B,EAAsC;AACpC,SAAK,MAAMS,KAAX,IAAoBC,MAAM,CAACC,MAAP,CAAcX,IAAd,CAApB,EAAyC;AACvC,YAAMY,KAAK,GAAGR,aAAa,CAACK,KAAD,CAA3B;;AACA,UAAIG,KAAJ,EAAW;AACT,eAAOA,KAAP;AACD;AACF;AACF;;AACD,SAAO,IAAP;AACD","sourcesContent":["import type {JSONLoaderOptions} from '../json-loader';\n\nexport default function parseJSONSync(jsonText: string, options: JSONLoaderOptions) {\n  try {\n    const json = JSON.parse(jsonText);\n    if (options.json?.table) {\n      return getFirstArray(json) || json;\n    }\n    return json;\n  } catch (error) {\n    throw new Error('JSONLoader: failed to parse JSON');\n  }\n}\n\nfunction getFirstArray(json) {\n  if (Array.isArray(json)) {\n    return json;\n  }\n  if (json && typeof json === 'object') {\n    for (const value of Object.values(json)) {\n      const array = getFirstArray(value);\n      if (array) {\n        return array;\n      }\n    }\n  }\n  return null;\n}\n"],"file":"parse-json.js"}
+{"version":3,"sources":["../../../src/lib/parse-json.ts"],"names":["parseJSONSync","jsonText","options","json","JSON","parse","table","getFirstArray","error","Error","Array","isArray","Object","values","value","array"],"mappings":";;;;;;;;;;;AAEe,SAASA,aAAT,CAAuBC,QAAvB,EAAyCC,OAAzC,EAAqE;AAClF,MAAI;AAAA;;AACF,QAAMC,IAAI,GAAGC,IAAI,CAACC,KAAL,CAAWJ,QAAX,CAAb;;AACA,yBAAIC,OAAO,CAACC,IAAZ,0CAAI,cAAcG,KAAlB,EAAyB;AACvB,aAAOC,aAAa,CAACJ,IAAD,CAAb,IAAuBA,IAA9B;AACD;;AACD,WAAOA,IAAP;AACD,GAND,CAME,OAAOK,KAAP,EAAc;AACd,UAAM,IAAIC,KAAJ,CAAU,kCAAV,CAAN;AACD;AACF;;AAED,SAASF,aAAT,CAAuBJ,IAAvB,EAA6B;AAC3B,MAAIO,KAAK,CAACC,OAAN,CAAcR,IAAd,CAAJ,EAAyB;AACvB,WAAOA,IAAP;AACD;;AACD,MAAIA,IAAI,IAAI,sBAAOA,IAAP,MAAgB,QAA5B,EAAsC;AACpC,sCAAoBS,MAAM,CAACC,MAAP,CAAcV,IAAd,CAApB,oCAAyC;AAApC,UAAMW,KAAK,qBAAX;AACH,UAAMC,KAAK,GAAGR,aAAa,CAACO,KAAD,CAA3B;;AACA,UAAIC,KAAJ,EAAW;AACT,eAAOA,KAAP;AACD;AACF;AACF;;AACD,SAAO,IAAP;AACD","sourcesContent":["import type {JSONLoaderOptions} from '../json-loader';\n\nexport default function parseJSONSync(jsonText: string, options: JSONLoaderOptions) {\n  try {\n    const json = JSON.parse(jsonText);\n    if (options.json?.table) {\n      return getFirstArray(json) || json;\n    }\n    return json;\n  } catch (error) {\n    throw new Error('JSONLoader: failed to parse JSON');\n  }\n}\n\nfunction getFirstArray(json) {\n  if (Array.isArray(json)) {\n    return json;\n  }\n  if (json && typeof json === 'object') {\n    for (const value of Object.values(json)) {\n      const array = getFirstArray(value);\n      if (array) {\n        return array;\n      }\n    }\n  }\n  return null;\n}\n"],"file":"parse-json.js"}

package/dist/es5/lib/parser/json-parser.js

@@ -7,14 +7,19 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = void 0;
 
+var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
+
+var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
+
 var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
 
 var _clarinet = _interopRequireDefault(require("../clarinet/clarinet"));
 
 var _jsonpath = _interopRequireDefault(require("../jsonpath/jsonpath"));
 
-class JSONParser {
-  constructor() {
+var JSONParser = function () {
+  function JSONParser() {
+    (0, _classCallCheck2.default)(this, JSONParser);
     (0, _defineProperty2.default)(this, "jsonpath", void 0);
     (0, _defineProperty2.default)(this, "_parser", void 0);
     this.reset();
@@ -22,118 +27,134 @@ class JSONParser {
     this._initializeParser();
   }
 
-  reset() {
-    this.result = undefined;
-    this.previousStates = [];
-    this.currentState = Object.freeze({
-      container: [],
-      key: null
-    });
-    this.jsonpath = new _jsonpath.default();
-  }
-
-  write(chunk) {
-    this.parser.write(chunk);
-  }
-
-  close() {
-    this.parser.close();
-  }
-
-  _pushOrSet(value) {
-    const {
-      container,
-      key
-    } = this.currentState;
-
-    if (key !== null) {
-      container[key] = value;
-      this.currentState.key = null;
-    } else {
-      container.push(value);
+  (0, _createClass2.default)(JSONParser, [{
+    key: "reset",
+    value: function reset() {
+      this.result = undefined;
+      this.previousStates = [];
+      this.currentState = Object.freeze({
+        container: [],
+        key: null
+      });
+      this.jsonpath = new _jsonpath.default();
     }
-  }
-
-  _openArray(newContainer = []) {
-    this.jsonpath.push(null);
-
-    this._pushOrSet(newContainer);
-
-    this.previousStates.push(this.currentState);
-    this.currentState = {
-      container: newContainer,
-      isArray: true,
-      key: null
-    };
-  }
-
-  _closeArray() {
-    this.jsonpath.pop();
-    this.currentState = this.previousStates.pop();
-  }
-
-  _openObject(newContainer = {}) {
-    this.jsonpath.push(null);
-
-    this._pushOrSet(newContainer);
-
-    this.previousStates.push(this.currentState);
-    this.currentState = {
-      container: newContainer,
-      isArray: false,
-      key: null
-    };
-  }
-
-  _closeObject() {
-    this.jsonpath.pop();
-    this.currentState = this.previousStates.pop();
-  }
-
-  _initializeParser() {
-    this._parser = new _clarinet.default({
-      onready: () => {
-        this.jsonpath = new _jsonpath.default();
-        this.previousStates.length = 0;
-        this.currentState.container.length = 0;
-      },
-      onopenobject: name => {
-        this._openObject({});
-
-        if (typeof name !== 'undefined') {
-          this.parser.onkey(name);
-        }
-      },
-      onkey: name => {
-        this.jsonpath.set(name);
-        this.currentState.key = name;
-      },
-      oncloseobject: () => {
-        this._closeObject();
-      },
-      onopenarray: () => {
-        this._openArray();
-      },
-      onclosearray: () => {
-        this._closeArray();
-      },
-      onvalue: value => {
-        this._pushOrSet(value);
-      },
-      onerror: error => {
-        throw error;
-      },
-      onend: () => {
-        this.result = this.currentState.container.pop();
+  }, {
+    key: "write",
+    value: function write(chunk) {
+      this.parser.write(chunk);
+    }
+  }, {
+    key: "close",
+    value: function close() {
+      this.parser.close();
+    }
+  }, {
+    key: "_pushOrSet",
+    value: function _pushOrSet(value) {
+      var _this$currentState = this.currentState,
+          container = _this$currentState.container,
+          key = _this$currentState.key;
+
+      if (key !== null) {
+        container[key] = value;
+        this.currentState.key = null;
+      } else {
+        container.push(value);
       }
-    });
-  }
-
-  get parser() {
-    return this._parser;
-  }
-
-}
+    }
+  }, {
+    key: "_openArray",
+    value: function _openArray() {
+      var newContainer = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : [];
+      this.jsonpath.push(null);
+
+      this._pushOrSet(newContainer);
+
+      this.previousStates.push(this.currentState);
+      this.currentState = {
+        container: newContainer,
+        isArray: true,
+        key: null
+      };
+    }
+  }, {
+    key: "_closeArray",
+    value: function _closeArray() {
+      this.jsonpath.pop();
+      this.currentState = this.previousStates.pop();
+    }
+  }, {
+    key: "_openObject",
+    value: function _openObject() {
+      var newContainer = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
+      this.jsonpath.push(null);
+
+      this._pushOrSet(newContainer);
+
+      this.previousStates.push(this.currentState);
+      this.currentState = {
+        container: newContainer,
+        isArray: false,
+        key: null
+      };
+    }
+  }, {
+    key: "_closeObject",
+    value: function _closeObject() {
+      this.jsonpath.pop();
+      this.currentState = this.previousStates.pop();
+    }
+  }, {
+    key: "_initializeParser",
+    value: function _initializeParser() {
+      var _this = this;
+
+      this._parser = new _clarinet.default({
+        onready: function onready() {
+          _this.jsonpath = new _jsonpath.default();
+          _this.previousStates.length = 0;
+          _this.currentState.container.length = 0;
+        },
+        onopenobject: function onopenobject(name) {
+          _this._openObject({});
+
+          if (typeof name !== 'undefined') {
+            _this.parser.onkey(name);
+          }
+        },
+        onkey: function onkey(name) {
+          _this.jsonpath.set(name);
+
+          _this.currentState.key = name;
+        },
+        oncloseobject: function oncloseobject() {
+          _this._closeObject();
+        },
+        onopenarray: function onopenarray() {
+          _this._openArray();
+        },
+        onclosearray: function onclosearray() {
+          _this._closeArray();
+        },
+        onvalue: function onvalue(value) {
+          _this._pushOrSet(value);
+        },
+        onerror: function onerror(error) {
+          throw error;
+        },
+        onend: function onend() {
+          _this.result = _this.currentState.container.pop();
+        }
+      });
+    }
+  }, {
+    key: "parser",
+    get: function get() {
+      return this._parser;
+    }
+  }]);
+  return JSONParser;
+}();
 
 exports.default = JSONParser;
 //# sourceMappingURL=json-parser.js.map