@loaders.gl/json 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/es5/lib/parse-json-in-batches.js

@@ -7,252 +7,93 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = parseJSONInBatches;
 
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
-var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
-var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
-
-var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
-
-var _asyncIterator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncIterator"));
-
 var _schema = require("@loaders.gl/schema");
 
 var _loaderUtils = require("@loaders.gl/loader-utils");
 
 var _streamingJsonParser = _interopRequireDefault(require("./parser/streaming-json-parser"));
 
-function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
-
-function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
-
-function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
-
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
-
-function parseJSONInBatches(_x, _x2) {
-  return _parseJSONInBatches.apply(this, arguments);
-}
-
-function _parseJSONInBatches() {
-  _parseJSONInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(binaryAsyncIterator, options) {
-    var _options$json;
-
-    var asyncIterator, metadata, _ref, jsonpaths, isFirstChunk, schema, shape, tableBatchBuilder, parser, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _value, chunk, rows, _jsonpath, initialBatch, _iterator2, _step2, row, _batch2, _batch, jsonpath, batch, finalBatch;
-
-    return _regenerator.default.wrap(function _callee$(_context) {
-      while (1) {
-        switch (_context.prev = _context.next) {
-          case 0:
-            asyncIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
-            metadata = options.metadata;
-            _ref = options.json || {}, jsonpaths = _ref.jsonpaths;
-            isFirstChunk = true;
-            schema = null;
-            shape = (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'row-table';
-            tableBatchBuilder = new _schema.TableBatchBuilder(schema, _objectSpread(_objectSpread({}, options), {}, {
-              shape: shape
-            }));
-            parser = new _streamingJsonParser.default({
-              jsonpaths: jsonpaths
-            });
-            _iteratorNormalCompletion = true;
-            _didIteratorError = false;
-            _context.prev = 10;
-            _iterator = (0, _asyncIterator2.default)(asyncIterator);
-
-          case 12:
-            _context.next = 14;
-            return (0, _awaitAsyncGenerator2.default)(_iterator.next());
-
-          case 14:
-            _step = _context.sent;
-            _iteratorNormalCompletion = _step.done;
-            _context.next = 18;
-            return (0, _awaitAsyncGenerator2.default)(_step.value);
-
-          case 18:
-            _value = _context.sent;
-
-            if (_iteratorNormalCompletion) {
-              _context.next = 57;
-              break;
-            }
-
-            chunk = _value;
-            rows = parser.write(chunk);
-            _jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
-
-            if (!(rows.length > 0 && isFirstChunk)) {
-              _context.next = 29;
-              break;
-            }
-
-            if (!metadata) {
-              _context.next = 28;
-              break;
-            }
-
-            initialBatch = {
-              shape: shape,
-              batchType: 'partial-result',
-              data: [],
-              length: 0,
-              bytesUsed: 0,
-              container: parser.getPartialResult(),
-              jsonpath: _jsonpath
-            };
-            _context.next = 28;
-            return initialBatch;
-
-          case 28:
-            isFirstChunk = false;
-
-          case 29:
-            _iterator2 = _createForOfIteratorHelper(rows);
-            _context.prev = 30;
-
-            _iterator2.s();
-
-          case 32:
-            if ((_step2 = _iterator2.n()).done) {
-              _context.next = 41;
-              break;
-            }
-
-            row = _step2.value;
-            tableBatchBuilder.addRow(row);
-            _batch2 = tableBatchBuilder.getFullBatch({
-              jsonpath: _jsonpath
-            });
-
-            if (!_batch2) {
-              _context.next = 39;
-              break;
-            }
-
-            _context.next = 39;
-            return _batch2;
-
-          case 39:
-            _context.next = 32;
-            break;
-
-          case 41:
-            _context.next = 46;
-            break;
-
-          case 43:
-            _context.prev = 43;
-            _context.t0 = _context["catch"](30);
-
-            _iterator2.e(_context.t0);
-
-          case 46:
-            _context.prev = 46;
-
-            _iterator2.f();
-
-            return _context.finish(46);
-
-          case 49:
-            tableBatchBuilder.chunkComplete(chunk);
-            _batch = tableBatchBuilder.getFullBatch({
-              jsonpath: _jsonpath
-            });
-
-            if (!_batch) {
-              _context.next = 54;
-              break;
-            }
-
-            _context.next = 54;
-            return _batch;
-
-          case 54:
-            _iteratorNormalCompletion = true;
-            _context.next = 12;
-            break;
-
-          case 57:
-            _context.next = 63;
-            break;
-
-          case 59:
-            _context.prev = 59;
-            _context.t1 = _context["catch"](10);
-            _didIteratorError = true;
-            _iteratorError = _context.t1;
-
-          case 63:
-            _context.prev = 63;
-            _context.prev = 64;
-
-            if (!(!_iteratorNormalCompletion && _iterator.return != null)) {
-              _context.next = 68;
-              break;
-            }
-
-            _context.next = 68;
-            return (0, _awaitAsyncGenerator2.default)(_iterator.return());
-
-          case 68:
-            _context.prev = 68;
-
-            if (!_didIteratorError) {
-              _context.next = 71;
-              break;
-            }
-
-            throw _iteratorError;
-
-          case 71:
-            return _context.finish(68);
-
-          case 72:
-            return _context.finish(63);
-
-          case 73:
-            jsonpath = parser.getStreamingJsonPathAsString();
-            batch = tableBatchBuilder.getFinalBatch({
-              jsonpath: jsonpath
-            });
-
-            if (!batch) {
-              _context.next = 78;
-              break;
-            }
-
-            _context.next = 78;
-            return batch;
+async function* parseJSONInBatches(binaryAsyncIterator, options) {
+  var _options$json;
+
+  const asyncIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
+  const {
+    metadata
+  } = options;
+  const {
+    jsonpaths
+  } = options.json || {};
+  let isFirstChunk = true;
+  const schema = null;
+  const shape = (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'row-table';
+  const tableBatchBuilder = new _schema.TableBatchBuilder(schema, { ...options,
+    shape
+  });
+  const parser = new _streamingJsonParser.default({
+    jsonpaths
+  });
+
+  for await (const chunk of asyncIterator) {
+    const rows = parser.write(chunk);
+    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
+
+    if (rows.length > 0 && isFirstChunk) {
+      if (metadata) {
+        const initialBatch = {
+          shape,
+          batchType: 'partial-result',
+          data: [],
+          length: 0,
+          bytesUsed: 0,
+          container: parser.getPartialResult(),
+          jsonpath
+        };
+        yield initialBatch;
+      }
 
-          case 78:
-            if (!metadata) {
-              _context.next = 82;
-              break;
-            }
+      isFirstChunk = false;
+    }
 
-            finalBatch = {
-              shape: shape,
-              batchType: 'final-result',
-              container: parser.getPartialResult(),
-              jsonpath: parser.getStreamingJsonPathAsString(),
-              data: [],
-              length: 0
-            };
-            _context.next = 82;
-            return finalBatch;
+    for (const row of rows) {
+      tableBatchBuilder.addRow(row);
+      const batch = tableBatchBuilder.getFullBatch({
+        jsonpath
+      });
 
-          case 82:
-          case "end":
-            return _context.stop();
-        }
+      if (batch) {
+        yield batch;
       }
-    }, _callee, null, [[10, 59, 63, 73], [30, 43, 46, 49], [64,, 68, 72]]);
-  }));
-  return _parseJSONInBatches.apply(this, arguments);
+    }
+
+    tableBatchBuilder.chunkComplete(chunk);
+    const batch = tableBatchBuilder.getFullBatch({
+      jsonpath
+    });
+
+    if (batch) {
+      yield batch;
+    }
+  }
+
+  const jsonpath = parser.getStreamingJsonPathAsString();
+  const batch = tableBatchBuilder.getFinalBatch({
+    jsonpath
+  });
+
+  if (batch) {
+    yield batch;
+  }
+
+  if (metadata) {
+    const finalBatch = {
+      shape,
+      batchType: 'final-result',
+      container: parser.getPartialResult(),
+      jsonpath: parser.getStreamingJsonPathAsString(),
+      data: [],
+      length: 0
+    };
+    yield finalBatch;
+  }
 }
 //# sourceMappingURL=parse-json-in-batches.js.map

package/dist/es5/lib/parse-json-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-json-in-batches.ts"],"names":["parseJSONInBatches","binaryAsyncIterator","options","asyncIterator","metadata","json","jsonpaths","isFirstChunk","schema","shape","tableBatchBuilder","TableBatchBuilder","parser","StreamingJSONParser","chunk","rows","write","jsonpath","length","getStreamingJsonPathAsString","initialBatch","batchType","data","bytesUsed","container","getPartialResult","row","addRow","batch","getFullBatch","chunkComplete","getFinalBatch","finalBatch"],"mappings":";;;;;;;;;;;;;;;;;;;AAEA;;AACA;;AACA;;;;;;;;;;;;SAI+BA,kB;;;;;oFAAhB,iBACbC,mBADa,EAEbC,OAFa;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAIPC,YAAAA,aAJO,GAIS,0CAAwBF,mBAAxB,CAJT;AAMNG,YAAAA,QANM,GAMMF,OANN,CAMNE,QANM;AAAA,mBAOOF,OAAO,CAACG,IAAR,IAAgB,EAPvB,EAONC,SAPM,QAONA,SAPM;AASTC,YAAAA,YATS,GASe,IATf;AAYPC,YAAAA,MAZO,GAYE,IAZF;AAaPC,YAAAA,KAbO,GAaC,CAAAP,OAAO,SAAP,IAAAA,OAAO,WAAP,6BAAAA,OAAO,CAAEG,IAAT,gEAAeI,KAAf,KAAwB,WAbzB;AAePC,YAAAA,iBAfO,GAea,IAAIC,yBAAJ,CAAsBH,MAAtB,kCACrBN,OADqB;AAExBO,cAAAA,KAAK,EAALA;AAFwB,eAfb;AAoBPG,YAAAA,MApBO,GAoBE,IAAIC,4BAAJ,CAAwB;AAACP,cAAAA,SAAS,EAATA;AAAD,aAAxB,CApBF;AAAA;AAAA;AAAA;AAAA,qDAsBaH,aAtBb;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAsBIW,YAAAA,KAtBJ;AAuBLC,YAAAA,IAvBK,GAuBEH,MAAM,CAACI,KAAP,CAAaF,KAAb,CAvBF;AAyBLG,YAAAA,SAzBK,GAyBMF,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBN,MAAM,CAACO,4BAAP,EAzBzB;;AAAA,kBA2BPJ,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBX,YA3BZ;AAAA;AAAA;AAAA;;AAAA,iBA4BLH,QA5BK;AAAA;AAAA;AAAA;;AA6BDgB,YAAAA,YA7BC,GA6BqB;AAE1BX,cAAAA,KAAK,EAALA,KAF0B;AAG1BY,cAAAA,SAAS,EAAE,gBAHe;AAI1BC,cAAAA,IAAI,EAAE,EAJoB;AAK1BJ,cAAAA,MAAM,EAAE,CALkB;AAM1BK,cAAAA,SAAS,EAAE,CANe;AAQ1BC,cAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EARe;AAS1BR,cAAAA,QAAQ,EAARA;AAT0B,aA7BrB;AAAA;AAwCP,mBAAMG,YAAN;;AAxCO;AA0CTb,YAAAA,YAAY,GAAG,KAAf;;AA1CS;AAAA,oDA+COQ,IA/CP;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AA+CAW,YAAAA,GA/CA;AAgDThB,YAAAA,iBAAiB,CAACiB,MAAlB,CAAyBD,GAAzB;AAEME,YAAAA,OAlDG,GAkDKlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,cAAAA,QAAQ,EAARA;AAAD,aAA/B,CAlDL;;AAAA,iBAmDLW,OAnDK;AAAA;AAAA;AAAA;;AAAA;AAoDP,mBAAMA,OAAN;;AApDO;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;;AAAA;;AAAA;AAwDXlB,YAAAA,iBAAiB,CAACoB,aAAlB,CAAgChB,KAAhC;AACMc,YAAAA,MAzDK,GAyDGlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,cAAAA,QAAQ,EAARA;AAAD,aAA/B,CAzDH;;AAAA,iBA0DPW,MA1DO;AAAA;AAAA;AAAA;;AAAA;AA2DT,mBAAMA,MAAN;;AA3DS;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAgEPX,YAAAA,QAhEO,GAgEIL,MAAM,CAACO,4BAAP,EAhEJ;AAiEPS,YAAAA,KAjEO,GAiEClB,iBAAiB,CAACqB,aAAlB,CAAgC;AAACd,cAAAA,QAAQ,EAARA;AAAD,aAAhC,CAjED;;AAAA,iBAkETW,KAlES;AAAA;AAAA;AAAA;;AAAA;AAmEX,mBAAMA,KAAN;;AAnEW;AAAA,iBAsETxB,QAtES;AAAA;AAAA;AAAA;;AAuEL4B,YAAAA,UAvEK,GAuEe;AACxBvB,cAAAA,KAAK,EAALA,KADwB;AAExBY,cAAAA,SAAS,EAAE,cAFa;AAGxBG,cAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EAHa;AAIxBR,cAAAA,QAAQ,EAAEL,MAAM,CAACO,4BAAP,EAJc;AAKxBG,cAAAA,IAAI,EAAE,EALkB;AAMxBJ,cAAAA,MAAM,EAAE;AANgB,aAvEf;AAAA;AAgFX,mBAAMc,UAAN;;AAhFW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n  const {metadata} = options;\n  const {jsonpaths} = options.json || {};\n\n  let isFirstChunk: boolean = true;\n\n  // TODO fix Schema deduction\n  const schema = null; // new Schema([]);\n  const shape = options?.json?.shape || 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  const parser = new StreamingJSONParser({jsonpaths});\n\n  for await (const chunk of asyncIterator) {\n    const rows = parser.write(chunk);\n\n    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n    if (rows.length > 0 && isFirstChunk) {\n      if (metadata) {\n        const initialBatch: Batch = {\n          // Common fields\n          shape,\n          batchType: 'partial-result',\n          data: [],\n          length: 0,\n          bytesUsed: 0,\n          // JSON additions\n          container: parser.getPartialResult(),\n          jsonpath\n        };\n        yield initialBatch;\n      }\n      isFirstChunk = false;\n      // schema = deduceSchema(rows);\n    }\n\n    // Add the row\n    for (const row of rows) {\n      tableBatchBuilder.addRow(row);\n      // If a batch has been completed, emit it\n      const batch = tableBatchBuilder.getFullBatch({jsonpath});\n      if (batch) {\n        yield batch;\n      }\n    }\n\n    tableBatchBuilder.chunkComplete(chunk);\n    const batch = tableBatchBuilder.getFullBatch({jsonpath});\n    if (batch) {\n      yield batch;\n    }\n  }\n\n  // yield final batch\n  const jsonpath = parser.getStreamingJsonPathAsString();\n  const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n  if (batch) {\n    yield batch;\n  }\n\n  if (metadata) {\n    const finalBatch: Batch = {\n      shape,\n      batchType: 'final-result',\n      container: parser.getPartialResult(),\n      jsonpath: parser.getStreamingJsonPathAsString(),\n      data: [],\n      length: 0\n      // schema: null\n    };\n    yield finalBatch;\n  }\n}\n"],"file":"parse-json-in-batches.js"}
+{"version":3,"sources":["../../../src/lib/parse-json-in-batches.ts"],"names":["parseJSONInBatches","binaryAsyncIterator","options","asyncIterator","metadata","jsonpaths","json","isFirstChunk","schema","shape","tableBatchBuilder","TableBatchBuilder","parser","StreamingJSONParser","chunk","rows","write","jsonpath","length","getStreamingJsonPathAsString","initialBatch","batchType","data","bytesUsed","container","getPartialResult","row","addRow","batch","getFullBatch","chunkComplete","getFinalBatch","finalBatch"],"mappings":";;;;;;;;;AAEA;;AACA;;AACA;;AAIe,gBAAgBA,kBAAhB,CACbC,mBADa,EAEbC,OAFa,EAGS;AAAA;;AACtB,QAAMC,aAAa,GAAG,0CAAwBF,mBAAxB,CAAtB;AAEA,QAAM;AAACG,IAAAA;AAAD,MAAaF,OAAnB;AACA,QAAM;AAACG,IAAAA;AAAD,MAAcH,OAAO,CAACI,IAAR,IAAgB,EAApC;AAEA,MAAIC,YAAqB,GAAG,IAA5B;AAGA,QAAMC,MAAM,GAAG,IAAf;AACA,QAAMC,KAAK,GAAG,CAAAP,OAAO,SAAP,IAAAA,OAAO,WAAP,6BAAAA,OAAO,CAAEI,IAAT,gEAAeG,KAAf,KAAwB,WAAtC;AAEA,QAAMC,iBAAiB,GAAG,IAAIC,yBAAJ,CAAsBH,MAAtB,EAA8B,EACtD,GAAGN,OADmD;AAEtDO,IAAAA;AAFsD,GAA9B,CAA1B;AAKA,QAAMG,MAAM,GAAG,IAAIC,4BAAJ,CAAwB;AAACR,IAAAA;AAAD,GAAxB,CAAf;;AAEA,aAAW,MAAMS,KAAjB,IAA0BX,aAA1B,EAAyC;AACvC,UAAMY,IAAI,GAAGH,MAAM,CAACI,KAAP,CAAaF,KAAb,CAAb;AAEA,UAAMG,QAAQ,GAAGF,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBN,MAAM,CAACO,4BAAP,EAApC;;AAEA,QAAIJ,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBX,YAAvB,EAAqC;AACnC,UAAIH,QAAJ,EAAc;AACZ,cAAMgB,YAAmB,GAAG;AAE1BX,UAAAA,KAF0B;AAG1BY,UAAAA,SAAS,EAAE,gBAHe;AAI1BC,UAAAA,IAAI,EAAE,EAJoB;AAK1BJ,UAAAA,MAAM,EAAE,CALkB;AAM1BK,UAAAA,SAAS,EAAE,CANe;AAQ1BC,UAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EARe;AAS1BR,UAAAA;AAT0B,SAA5B;AAWA,cAAMG,YAAN;AACD;;AACDb,MAAAA,YAAY,GAAG,KAAf;AAED;;AAGD,SAAK,MAAMmB,GAAX,IAAkBX,IAAlB,EAAwB;AACtBL,MAAAA,iBAAiB,CAACiB,MAAlB,CAAyBD,GAAzB;AAEA,YAAME,KAAK,GAAGlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,QAAAA;AAAD,OAA/B,CAAd;;AACA,UAAIW,KAAJ,EAAW;AACT,cAAMA,KAAN;AACD;AACF;;AAEDlB,IAAAA,iBAAiB,CAACoB,aAAlB,CAAgChB,KAAhC;AACA,UAAMc,KAAK,GAAGlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,MAAAA;AAAD,KAA/B,CAAd;;AACA,QAAIW,KAAJ,EAAW;AACT,YAAMA,KAAN;AACD;AACF;;AAGD,QAAMX,QAAQ,GAAGL,MAAM,CAACO,4BAAP,EAAjB;AACA,QAAMS,KAAK,GAAGlB,iBAAiB,CAACqB,aAAlB,CAAgC;AAACd,IAAAA;AAAD,GAAhC,CAAd;;AACA,MAAIW,KAAJ,EAAW;AACT,UAAMA,KAAN;AACD;;AAED,MAAIxB,QAAJ,EAAc;AACZ,UAAM4B,UAAiB,GAAG;AACxBvB,MAAAA,KADwB;AAExBY,MAAAA,SAAS,EAAE,cAFa;AAGxBG,MAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EAHa;AAIxBR,MAAAA,QAAQ,EAAEL,MAAM,CAACO,4BAAP,EAJc;AAKxBG,MAAAA,IAAI,EAAE,EALkB;AAMxBJ,MAAAA,MAAM,EAAE;AANgB,KAA1B;AASA,UAAMc,UAAN;AACD;AACF","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n  const {metadata} = options;\n  const {jsonpaths} = options.json || {};\n\n  let isFirstChunk: boolean = true;\n\n  // TODO fix Schema deduction\n  const schema = null; // new Schema([]);\n  const shape = options?.json?.shape || 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  const parser = new StreamingJSONParser({jsonpaths});\n\n  for await (const chunk of asyncIterator) {\n    const rows = parser.write(chunk);\n\n    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n    if (rows.length > 0 && isFirstChunk) {\n      if (metadata) {\n        const initialBatch: Batch = {\n          // Common fields\n          shape,\n          batchType: 'partial-result',\n          data: [],\n          length: 0,\n          bytesUsed: 0,\n          // JSON additions\n          container: parser.getPartialResult(),\n          jsonpath\n        };\n        yield initialBatch;\n      }\n      isFirstChunk = false;\n      // schema = deduceSchema(rows);\n    }\n\n    // Add the row\n    for (const row of rows) {\n      tableBatchBuilder.addRow(row);\n      // If a batch has been completed, emit it\n      const batch = tableBatchBuilder.getFullBatch({jsonpath});\n      if (batch) {\n        yield batch;\n      }\n    }\n\n    tableBatchBuilder.chunkComplete(chunk);\n    const batch = tableBatchBuilder.getFullBatch({jsonpath});\n    if (batch) {\n      yield batch;\n    }\n  }\n\n  // yield final batch\n  const jsonpath = parser.getStreamingJsonPathAsString();\n  const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n  if (batch) {\n    yield batch;\n  }\n\n  if (metadata) {\n    const finalBatch: Batch = {\n      shape,\n      batchType: 'final-result',\n      container: parser.getPartialResult(),\n      jsonpath: parser.getStreamingJsonPathAsString(),\n      data: [],\n      length: 0\n      // schema: null\n    };\n    yield finalBatch;\n  }\n}\n"],"file":"parse-json-in-batches.js"}

package/dist/es5/lib/parse-json.js

@@ -1,19 +1,15 @@
 "use strict";
 
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = parseJSONSync;
 
-var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
-
 function parseJSONSync(jsonText, options) {
   try {
     var _options$json;
 
-    var json = JSON.parse(jsonText);
+    const json = JSON.parse(jsonText);
 
     if ((_options$json = options.json) !== null && _options$json !== void 0 && _options$json.table) {
       return getFirstArray(json) || json;
@@ -30,10 +26,9 @@ function getFirstArray(json) {
     return json;
   }
 
-  if (json && (0, _typeof2.default)(json) === 'object') {
-    for (var _i = 0, _Object$values = Object.values(json); _i < _Object$values.length; _i++) {
-      var value = _Object$values[_i];
-      var array = getFirstArray(value);
+  if (json && typeof json === 'object') {
+    for (const value of Object.values(json)) {
+      const array = getFirstArray(value);
 
       if (array) {
         return array;

package/dist/es5/lib/parse-json.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-json.ts"],"names":["parseJSONSync","jsonText","options","json","JSON","parse","table","getFirstArray","error","Error","Array","isArray","Object","values","value","array"],"mappings":";;;;;;;;;;;AAEe,SAASA,aAAT,CAAuBC,QAAvB,EAAyCC,OAAzC,EAAqE;AAClF,MAAI;AAAA;;AACF,QAAMC,IAAI,GAAGC,IAAI,CAACC,KAAL,CAAWJ,QAAX,CAAb;;AACA,yBAAIC,OAAO,CAACC,IAAZ,0CAAI,cAAcG,KAAlB,EAAyB;AACvB,aAAOC,aAAa,CAACJ,IAAD,CAAb,IAAuBA,IAA9B;AACD;;AACD,WAAOA,IAAP;AACD,GAND,CAME,OAAOK,KAAP,EAAc;AACd,UAAM,IAAIC,KAAJ,CAAU,kCAAV,CAAN;AACD;AACF;;AAED,SAASF,aAAT,CAAuBJ,IAAvB,EAA6B;AAC3B,MAAIO,KAAK,CAACC,OAAN,CAAcR,IAAd,CAAJ,EAAyB;AACvB,WAAOA,IAAP;AACD;;AACD,MAAIA,IAAI,IAAI,sBAAOA,IAAP,MAAgB,QAA5B,EAAsC;AACpC,sCAAoBS,MAAM,CAACC,MAAP,CAAcV,IAAd,CAApB,oCAAyC;AAApC,UAAMW,KAAK,qBAAX;AACH,UAAMC,KAAK,GAAGR,aAAa,CAACO,KAAD,CAA3B;;AACA,UAAIC,KAAJ,EAAW;AACT,eAAOA,KAAP;AACD;AACF;AACF;;AACD,SAAO,IAAP;AACD","sourcesContent":["import type {JSONLoaderOptions} from '../json-loader';\n\nexport default function parseJSONSync(jsonText: string, options: JSONLoaderOptions) {\n  try {\n    const json = JSON.parse(jsonText);\n    if (options.json?.table) {\n      return getFirstArray(json) || json;\n    }\n    return json;\n  } catch (error) {\n    throw new Error('JSONLoader: failed to parse JSON');\n  }\n}\n\nfunction getFirstArray(json) {\n  if (Array.isArray(json)) {\n    return json;\n  }\n  if (json && typeof json === 'object') {\n    for (const value of Object.values(json)) {\n      const array = getFirstArray(value);\n      if (array) {\n        return array;\n      }\n    }\n  }\n  return null;\n}\n"],"file":"parse-json.js"}
+{"version":3,"sources":["../../../src/lib/parse-json.ts"],"names":["parseJSONSync","jsonText","options","json","JSON","parse","table","getFirstArray","error","Error","Array","isArray","value","Object","values","array"],"mappings":";;;;;;;AAEe,SAASA,aAAT,CAAuBC,QAAvB,EAAyCC,OAAzC,EAAqE;AAClF,MAAI;AAAA;;AACF,UAAMC,IAAI,GAAGC,IAAI,CAACC,KAAL,CAAWJ,QAAX,CAAb;;AACA,yBAAIC,OAAO,CAACC,IAAZ,0CAAI,cAAcG,KAAlB,EAAyB;AACvB,aAAOC,aAAa,CAACJ,IAAD,CAAb,IAAuBA,IAA9B;AACD;;AACD,WAAOA,IAAP;AACD,GAND,CAME,OAAOK,KAAP,EAAc;AACd,UAAM,IAAIC,KAAJ,CAAU,kCAAV,CAAN;AACD;AACF;;AAED,SAASF,aAAT,CAAuBJ,IAAvB,EAA6B;AAC3B,MAAIO,KAAK,CAACC,OAAN,CAAcR,IAAd,CAAJ,EAAyB;AACvB,WAAOA,IAAP;AACD;;AACD,MAAIA,IAAI,IAAI,OAAOA,IAAP,KAAgB,QAA5B,EAAsC;AACpC,SAAK,MAAMS,KAAX,IAAoBC,MAAM,CAACC,MAAP,CAAcX,IAAd,CAApB,EAAyC;AACvC,YAAMY,KAAK,GAAGR,aAAa,CAACK,KAAD,CAA3B;;AACA,UAAIG,KAAJ,EAAW;AACT,eAAOA,KAAP;AACD;AACF;AACF;;AACD,SAAO,IAAP;AACD","sourcesContent":["import type {JSONLoaderOptions} from '../json-loader';\n\nexport default function parseJSONSync(jsonText: string, options: JSONLoaderOptions) {\n  try {\n    const json = JSON.parse(jsonText);\n    if (options.json?.table) {\n      return getFirstArray(json) || json;\n    }\n    return json;\n  } catch (error) {\n    throw new Error('JSONLoader: failed to parse JSON');\n  }\n}\n\nfunction getFirstArray(json) {\n  if (Array.isArray(json)) {\n    return json;\n  }\n  if (json && typeof json === 'object') {\n    for (const value of Object.values(json)) {\n      const array = getFirstArray(value);\n      if (array) {\n        return array;\n      }\n    }\n  }\n  return null;\n}\n"],"file":"parse-json.js"}

package/dist/es5/lib/parse-ndjson-in-batches.js

@@ -1,158 +1,46 @@
 "use strict";
 
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
-
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = parseNDJSONInBatches;
 
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-
-var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-
-var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
-
-var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
-
-var _asyncIterator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncIterator"));
-
 var _schema = require("@loaders.gl/schema");
 
 var _loaderUtils = require("@loaders.gl/loader-utils");
 
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
-
-function parseNDJSONInBatches(_x, _x2) {
-  return _parseNDJSONInBatches.apply(this, arguments);
-}
-
-function _parseNDJSONInBatches() {
-  _parseNDJSONInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(binaryAsyncIterator, options) {
-    var textIterator, lineIterator, numberedLineIterator, schema, shape, tableBatchBuilder, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _value, _value2, counter, line, row, _batch, batch;
-
-    return _regenerator.default.wrap(function _callee$(_context) {
-      while (1) {
-        switch (_context.prev = _context.next) {
-          case 0:
-            textIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
-            lineIterator = (0, _loaderUtils.makeLineIterator)(textIterator);
-            numberedLineIterator = (0, _loaderUtils.makeNumberedLineIterator)(lineIterator);
-            schema = null;
-            shape = 'row-table';
-            tableBatchBuilder = new _schema.TableBatchBuilder(schema, _objectSpread(_objectSpread({}, options), {}, {
-              shape: shape
-            }));
-            _iteratorNormalCompletion = true;
-            _didIteratorError = false;
-            _context.prev = 8;
-            _iterator = (0, _asyncIterator2.default)(numberedLineIterator);
-
-          case 10:
-            _context.next = 12;
-            return (0, _awaitAsyncGenerator2.default)(_iterator.next());
-
-          case 12:
-            _step = _context.sent;
-            _iteratorNormalCompletion = _step.done;
-            _context.next = 16;
-            return (0, _awaitAsyncGenerator2.default)(_step.value);
-
-          case 16:
-            _value = _context.sent;
-
-            if (_iteratorNormalCompletion) {
-              _context.next = 35;
-              break;
-            }
-
-            _value2 = _value, counter = _value2.counter, line = _value2.line;
-            _context.prev = 19;
-            row = JSON.parse(line);
-            tableBatchBuilder.addRow(row);
-            tableBatchBuilder.chunkComplete(line);
-            _batch = tableBatchBuilder.getFullBatch();
-
-            if (!_batch) {
-              _context.next = 27;
-              break;
-            }
-
-            _context.next = 27;
-            return _batch;
-
-          case 27:
-            _context.next = 32;
-            break;
-
-          case 29:
-            _context.prev = 29;
-            _context.t0 = _context["catch"](19);
-            throw new Error("NDJSONLoader: failed to parse JSON on line ".concat(counter));
-
-          case 32:
-            _iteratorNormalCompletion = true;
-            _context.next = 10;
-            break;
-
-          case 35:
-            _context.next = 41;
-            break;
-
-          case 37:
-            _context.prev = 37;
-            _context.t1 = _context["catch"](8);
-            _didIteratorError = true;
-            _iteratorError = _context.t1;
-
-          case 41:
-            _context.prev = 41;
-            _context.prev = 42;
-
-            if (!(!_iteratorNormalCompletion && _iterator.return != null)) {
-              _context.next = 46;
-              break;
-            }
-
-            _context.next = 46;
-            return (0, _awaitAsyncGenerator2.default)(_iterator.return());
-
-          case 46:
-            _context.prev = 46;
-
-            if (!_didIteratorError) {
-              _context.next = 49;
-              break;
-            }
-
-            throw _iteratorError;
-
-          case 49:
-            return _context.finish(46);
-
-          case 50:
-            return _context.finish(41);
-
-          case 51:
-            batch = tableBatchBuilder.getFinalBatch();
-
-            if (!batch) {
-              _context.next = 55;
-              break;
-            }
+async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
+  const textIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
+  const lineIterator = (0, _loaderUtils.makeLineIterator)(textIterator);
+  const numberedLineIterator = (0, _loaderUtils.makeNumberedLineIterator)(lineIterator);
+  const schema = null;
+  const shape = 'row-table';
+  const tableBatchBuilder = new _schema.TableBatchBuilder(schema, { ...options,
+    shape
+  });
+
+  for await (const {
+    counter,
+    line
+  } of numberedLineIterator) {
+    try {
+      const row = JSON.parse(line);
+      tableBatchBuilder.addRow(row);
+      tableBatchBuilder.chunkComplete(line);
+      const batch = tableBatchBuilder.getFullBatch();
+
+      if (batch) {
+        yield batch;
+      }
+    } catch (error) {
+      throw new Error("NDJSONLoader: failed to parse JSON on line ".concat(counter));
+    }
+  }
 
-            _context.next = 55;
-            return batch;
+  const batch = tableBatchBuilder.getFinalBatch();
 
-          case 55:
-          case "end":
-            return _context.stop();
-        }
-      }
-    }, _callee, null, [[8, 37, 41, 51], [19, 29], [42,, 46, 50]]);
-  }));
-  return _parseNDJSONInBatches.apply(this, arguments);
+  if (batch) {
+    yield batch;
+  }
 }
 //# sourceMappingURL=parse-ndjson-in-batches.js.map

package/dist/es5/lib/parse-ndjson-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-ndjson-in-batches.ts"],"names":["parseNDJSONInBatches","binaryAsyncIterator","options","textIterator","lineIterator","numberedLineIterator","schema","shape","tableBatchBuilder","TableBatchBuilder","counter","line","row","JSON","parse","addRow","chunkComplete","batch","getFullBatch","Error","getFinalBatch"],"mappings":";;;;;;;;;;;;;;;;;;;AACA;;AACA;;;;;;SAO+BA,oB;;;;;sFAAhB,iBACbC,mBADa,EAEbC,OAFa;AAAA;;AAAA;AAAA;AAAA;AAAA;AAIPC,YAAAA,YAJO,GAIQ,0CAAwBF,mBAAxB,CAJR;AAKPG,YAAAA,YALO,GAKQ,mCAAiBD,YAAjB,CALR;AAMPE,YAAAA,oBANO,GAMgB,2CAAyBD,YAAzB,CANhB;AAQPE,YAAAA,MARO,GAQE,IARF;AASPC,YAAAA,KATO,GASC,WATD;AAWPC,YAAAA,iBAXO,GAWa,IAAIC,yBAAJ,CAAsBH,MAAtB,kCACrBJ,OADqB;AAExBK,cAAAA,KAAK,EAALA;AAFwB,eAXb;AAAA;AAAA;AAAA;AAAA,qDAgBuBF,oBAhBvB;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA,8BAgBKK,OAhBL,WAgBKA,OAhBL,EAgBcC,IAhBd,WAgBcA,IAhBd;AAAA;AAkBHC,YAAAA,GAlBG,GAkBGC,IAAI,CAACC,KAAL,CAAWH,IAAX,CAlBH;AAmBTH,YAAAA,iBAAiB,CAACO,MAAlB,CAAyBH,GAAzB;AACAJ,YAAAA,iBAAiB,CAACQ,aAAlB,CAAgCL,IAAhC;AACMM,YAAAA,MArBG,GAqBKT,iBAAiB,CAACU,YAAlB,EArBL;;AAAA,iBAsBLD,MAtBK;AAAA;AAAA;AAAA;;AAAA;AAuBP,mBAAMA,MAAN;;AAvBO;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA,kBA0BH,IAAIE,KAAJ,sDAAwDT,OAAxD,EA1BG;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AA8BPO,YAAAA,KA9BO,GA8BCT,iBAAiB,CAACY,aAAlB,EA9BD;;AAAA,iBA+BTH,KA/BS;AAAA;AAAA;AAAA;;AAAA;AAgCX,mBAAMA,KAAN;;AAhCW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {\n  LoaderOptions,\n  makeLineIterator,\n  makeNumberedLineIterator,\n  makeTextDecoderIterator\n} from '@loaders.gl/loader-utils';\n\nexport default async function* parseNDJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: LoaderOptions\n): AsyncIterable<Batch> {\n  const textIterator = makeTextDecoderIterator(binaryAsyncIterator);\n  const lineIterator = makeLineIterator(textIterator);\n  const numberedLineIterator = makeNumberedLineIterator(lineIterator);\n\n  const schema = null;\n  const shape = 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  for await (const {counter, line} of numberedLineIterator) {\n    try {\n      const row = JSON.parse(line);\n      tableBatchBuilder.addRow(row);\n      tableBatchBuilder.chunkComplete(line);\n      const batch = tableBatchBuilder.getFullBatch();\n      if (batch) {\n        yield batch;\n      }\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);\n    }\n  }\n\n  const batch = tableBatchBuilder.getFinalBatch();\n  if (batch) {\n    yield batch;\n  }\n}\n"],"file":"parse-ndjson-in-batches.js"}
+{"version":3,"sources":["../../../src/lib/parse-ndjson-in-batches.ts"],"names":["parseNDJSONInBatches","binaryAsyncIterator","options","textIterator","lineIterator","numberedLineIterator","schema","shape","tableBatchBuilder","TableBatchBuilder","counter","line","row","JSON","parse","addRow","chunkComplete","batch","getFullBatch","error","Error","getFinalBatch"],"mappings":";;;;;;;AACA;;AACA;;AAOe,gBAAgBA,oBAAhB,CACbC,mBADa,EAEbC,OAFa,EAGS;AACtB,QAAMC,YAAY,GAAG,0CAAwBF,mBAAxB,CAArB;AACA,QAAMG,YAAY,GAAG,mCAAiBD,YAAjB,CAArB;AACA,QAAME,oBAAoB,GAAG,2CAAyBD,YAAzB,CAA7B;AAEA,QAAME,MAAM,GAAG,IAAf;AACA,QAAMC,KAAK,GAAG,WAAd;AAEA,QAAMC,iBAAiB,GAAG,IAAIC,yBAAJ,CAAsBH,MAAtB,EAA8B,EACtD,GAAGJ,OADmD;AAEtDK,IAAAA;AAFsD,GAA9B,CAA1B;;AAKA,aAAW,MAAM;AAACG,IAAAA,OAAD;AAAUC,IAAAA;AAAV,GAAjB,IAAoCN,oBAApC,EAA0D;AACxD,QAAI;AACF,YAAMO,GAAG,GAAGC,IAAI,CAACC,KAAL,CAAWH,IAAX,CAAZ;AACAH,MAAAA,iBAAiB,CAACO,MAAlB,CAAyBH,GAAzB;AACAJ,MAAAA,iBAAiB,CAACQ,aAAlB,CAAgCL,IAAhC;AACA,YAAMM,KAAK,GAAGT,iBAAiB,CAACU,YAAlB,EAAd;;AACA,UAAID,KAAJ,EAAW;AACT,cAAMA,KAAN;AACD;AACF,KARD,CAQE,OAAOE,KAAP,EAAc;AACd,YAAM,IAAIC,KAAJ,sDAAwDV,OAAxD,EAAN;AACD;AACF;;AAED,QAAMO,KAAK,GAAGT,iBAAiB,CAACa,aAAlB,EAAd;;AACA,MAAIJ,KAAJ,EAAW;AACT,UAAMA,KAAN;AACD;AACF","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {\n  LoaderOptions,\n  makeLineIterator,\n  makeNumberedLineIterator,\n  makeTextDecoderIterator\n} from '@loaders.gl/loader-utils';\n\nexport default async function* parseNDJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: LoaderOptions\n): AsyncIterable<Batch> {\n  const textIterator = makeTextDecoderIterator(binaryAsyncIterator);\n  const lineIterator = makeLineIterator(textIterator);\n  const numberedLineIterator = makeNumberedLineIterator(lineIterator);\n\n  const schema = null;\n  const shape = 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  for await (const {counter, line} of numberedLineIterator) {\n    try {\n      const row = JSON.parse(line);\n      tableBatchBuilder.addRow(row);\n      tableBatchBuilder.chunkComplete(line);\n      const batch = tableBatchBuilder.getFullBatch();\n      if (batch) {\n        yield batch;\n      }\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);\n    }\n  }\n\n  const batch = tableBatchBuilder.getFinalBatch();\n  if (batch) {\n    yield batch;\n  }\n}\n"],"file":"parse-ndjson-in-batches.js"}

package/dist/es5/lib/parse-ndjson.js

@@ -6,8 +6,8 @@ Object.defineProperty(exports, "__esModule", {
 exports.default = parseNDJSONSync;
 
 function parseNDJSONSync(ndjsonText) {
-  var lines = ndjsonText.trim().split('\n');
-  return lines.map(function (line, counter) {
+  const lines = ndjsonText.trim().split('\n');
+  return lines.map((line, counter) => {
     try {
       return JSON.parse(line);
     } catch (error) {

package/dist/es5/lib/parse-ndjson.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-ndjson.ts"],"names":["parseNDJSONSync","ndjsonText","lines","trim","split","map","line","counter","JSON","parse","error","Error"],"mappings":";;;;;;;AAAe,SAASA,eAAT,CAAyBC,UAAzB,EAA6C;AAC1D,MAAMC,KAAK,GAAGD,UAAU,CAACE,IAAX,GAAkBC,KAAlB,CAAwB,IAAxB,CAAd;AACA,SAAOF,KAAK,CAACG,GAAN,CAAU,UAACC,IAAD,EAAOC,OAAP,EAAmB;AAClC,QAAI;AACF,aAAOC,IAAI,CAACC,KAAL,CAAWH,IAAX,CAAP;AACD,KAFD,CAEE,OAAOI,KAAP,EAAc;AACd,YAAM,IAAIC,KAAJ,sDAAwDJ,OAAO,GAAG,CAAlE,EAAN;AACD;AACF,GANM,CAAP;AAOD","sourcesContent":["export default function parseNDJSONSync(ndjsonText: string) {\n  const lines = ndjsonText.trim().split('\\n');\n  return lines.map((line, counter) => {\n    try {\n      return JSON.parse(line);\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);\n    }\n  });\n}\n"],"file":"parse-ndjson.js"}
+{"version":3,"sources":["../../../src/lib/parse-ndjson.ts"],"names":["parseNDJSONSync","ndjsonText","lines","trim","split","map","line","counter","JSON","parse","error","Error"],"mappings":";;;;;;;AAAe,SAASA,eAAT,CAAyBC,UAAzB,EAA6C;AAC1D,QAAMC,KAAK,GAAGD,UAAU,CAACE,IAAX,GAAkBC,KAAlB,CAAwB,IAAxB,CAAd;AACA,SAAOF,KAAK,CAACG,GAAN,CAAU,CAACC,IAAD,EAAOC,OAAP,KAAmB;AAClC,QAAI;AACF,aAAOC,IAAI,CAACC,KAAL,CAAWH,IAAX,CAAP;AACD,KAFD,CAEE,OAAOI,KAAP,EAAc;AACd,YAAM,IAAIC,KAAJ,sDAAwDJ,OAAO,GAAG,CAAlE,EAAN;AACD;AACF,GANM,CAAP;AAOD","sourcesContent":["export default function parseNDJSONSync(ndjsonText: string) {\n  const lines = ndjsonText.trim().split('\\n');\n  return lines.map((line, counter) => {\n    try {\n      return JSON.parse(line);\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);\n    }\n  });\n}\n"],"file":"parse-ndjson.js"}