@loaders.gl/json 3.1.0-beta.3 → 3.1.1

package/dist/es5/lib/parse-json-in-batches.js

@@ -6,6 +6,17 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = parseJSONInBatches;
+exports.rebuildJsonObject = rebuildJsonObject;
+
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
+
+var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
+
+var _asyncIterator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncIterator"));
 
 var _schema = require("@loaders.gl/schema");
 
@@ -13,87 +24,255 @@ var _loaderUtils = require("@loaders.gl/loader-utils");
 
 var _streamingJsonParser = _interopRequireDefault(require("./parser/streaming-json-parser"));
 
-async function* parseJSONInBatches(binaryAsyncIterator, options) {
-  var _options$json;
-
-  const asyncIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
-  const {
-    metadata
-  } = options;
-  const {
-    jsonpaths
-  } = options.json || {};
-  let isFirstChunk = true;
-  const schema = null;
-  const shape = (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'row-table';
-  const tableBatchBuilder = new _schema.TableBatchBuilder(schema, { ...options,
-    shape
-  });
-  const parser = new _streamingJsonParser.default({
-    jsonpaths
-  });
-
-  for await (const chunk of asyncIterator) {
-    const rows = parser.write(chunk);
-    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
-
-    if (rows.length > 0 && isFirstChunk) {
-      if (metadata) {
-        const initialBatch = {
-          shape,
-          batchType: 'partial-result',
-          data: [],
-          length: 0,
-          bytesUsed: 0,
-          container: parser.getPartialResult(),
-          jsonpath
-        };
-        yield initialBatch;
-      }
+var _jsonpath2 = _interopRequireDefault(require("./jsonpath/jsonpath"));
 
-      isFirstChunk = false;
-    }
+function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
 
-    for (const row of rows) {
-      tableBatchBuilder.addRow(row);
-      const batch = tableBatchBuilder.getFullBatch({
-        jsonpath
-      });
+function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
 
-      if (batch) {
-        yield batch;
-      }
-    }
+function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
 
-    tableBatchBuilder.chunkComplete(chunk);
-    const batch = tableBatchBuilder.getFullBatch({
-      jsonpath
-    });
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
 
-    if (batch) {
-      yield batch;
-    }
-  }
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+
+function parseJSONInBatches(_x, _x2) {
+  return _parseJSONInBatches.apply(this, arguments);
+}
+
+function _parseJSONInBatches() {
+  _parseJSONInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(binaryAsyncIterator, options) {
+    var _options$json;
+
+    var asyncIterator, metadata, _ref, jsonpaths, isFirstChunk, schema, shape, tableBatchBuilder, parser, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _value, chunk, rows, _jsonpath, initialBatch, _iterator2, _step2, row, _batch2, _batch, jsonpath, batch, finalBatch;
+
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            asyncIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
+            metadata = options.metadata;
+            _ref = options.json || {}, jsonpaths = _ref.jsonpaths;
+            isFirstChunk = true;
+            schema = null;
+            shape = (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'row-table';
+            tableBatchBuilder = new _schema.TableBatchBuilder(schema, _objectSpread(_objectSpread({}, options), {}, {
+              shape: shape
+            }));
+            parser = new _streamingJsonParser.default({
+              jsonpaths: jsonpaths
+            });
+            _iteratorNormalCompletion = true;
+            _didIteratorError = false;
+            _context.prev = 10;
+            _iterator = (0, _asyncIterator2.default)(asyncIterator);
+
+          case 12:
+            _context.next = 14;
+            return (0, _awaitAsyncGenerator2.default)(_iterator.next());
+
+          case 14:
+            _step = _context.sent;
+            _iteratorNormalCompletion = _step.done;
+            _context.next = 18;
+            return (0, _awaitAsyncGenerator2.default)(_step.value);
+
+          case 18:
+            _value = _context.sent;
+
+            if (_iteratorNormalCompletion) {
+              _context.next = 57;
+              break;
+            }
+
+            chunk = _value;
+            rows = parser.write(chunk);
+            _jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
+
+            if (!(rows.length > 0 && isFirstChunk)) {
+              _context.next = 29;
+              break;
+            }
+
+            if (!metadata) {
+              _context.next = 28;
+              break;
+            }
+
+            initialBatch = {
+              shape: shape,
+              batchType: 'partial-result',
+              data: [],
+              length: 0,
+              bytesUsed: 0,
+              container: parser.getPartialResult(),
+              jsonpath: _jsonpath
+            };
+            _context.next = 28;
+            return initialBatch;
+
+          case 28:
+            isFirstChunk = false;
+
+          case 29:
+            _iterator2 = _createForOfIteratorHelper(rows);
+            _context.prev = 30;
+
+            _iterator2.s();
+
+          case 32:
+            if ((_step2 = _iterator2.n()).done) {
+              _context.next = 41;
+              break;
+            }
+
+            row = _step2.value;
+            tableBatchBuilder.addRow(row);
+            _batch2 = tableBatchBuilder.getFullBatch({
+              jsonpath: _jsonpath
+            });
+
+            if (!_batch2) {
+              _context.next = 39;
+              break;
+            }
+
+            _context.next = 39;
+            return _batch2;
+
+          case 39:
+            _context.next = 32;
+            break;
 
-  const jsonpath = parser.getStreamingJsonPathAsString();
-  const batch = tableBatchBuilder.getFinalBatch({
-    jsonpath
-  });
+          case 41:
+            _context.next = 46;
+            break;
 
-  if (batch) {
-    yield batch;
+          case 43:
+            _context.prev = 43;
+            _context.t0 = _context["catch"](30);
+
+            _iterator2.e(_context.t0);
+
+          case 46:
+            _context.prev = 46;
+
+            _iterator2.f();
+
+            return _context.finish(46);
+
+          case 49:
+            tableBatchBuilder.chunkComplete(chunk);
+            _batch = tableBatchBuilder.getFullBatch({
+              jsonpath: _jsonpath
+            });
+
+            if (!_batch) {
+              _context.next = 54;
+              break;
+            }
+
+            _context.next = 54;
+            return _batch;
+
+          case 54:
+            _iteratorNormalCompletion = true;
+            _context.next = 12;
+            break;
+
+          case 57:
+            _context.next = 63;
+            break;
+
+          case 59:
+            _context.prev = 59;
+            _context.t1 = _context["catch"](10);
+            _didIteratorError = true;
+            _iteratorError = _context.t1;
+
+          case 63:
+            _context.prev = 63;
+            _context.prev = 64;
+
+            if (!(!_iteratorNormalCompletion && _iterator.return != null)) {
+              _context.next = 68;
+              break;
+            }
+
+            _context.next = 68;
+            return (0, _awaitAsyncGenerator2.default)(_iterator.return());
+
+          case 68:
+            _context.prev = 68;
+
+            if (!_didIteratorError) {
+              _context.next = 71;
+              break;
+            }
+
+            throw _iteratorError;
+
+          case 71:
+            return _context.finish(68);
+
+          case 72:
+            return _context.finish(63);
+
+          case 73:
+            jsonpath = parser.getStreamingJsonPathAsString();
+            batch = tableBatchBuilder.getFinalBatch({
+              jsonpath: jsonpath
+            });
+
+            if (!batch) {
+              _context.next = 78;
+              break;
+            }
+
+            _context.next = 78;
+            return batch;
+
+          case 78:
+            if (!metadata) {
+              _context.next = 82;
+              break;
+            }
+
+            finalBatch = {
+              shape: shape,
+              batchType: 'final-result',
+              container: parser.getPartialResult(),
+              jsonpath: parser.getStreamingJsonPathAsString(),
+              data: [],
+              length: 0
+            };
+            _context.next = 82;
+            return finalBatch;
+
+          case 82:
+          case "end":
+            return _context.stop();
+        }
+      }
+    }, _callee, null, [[10, 59, 63, 73], [30, 43, 46, 49], [64,, 68, 72]]);
+  }));
+  return _parseJSONInBatches.apply(this, arguments);
+}
+
+function rebuildJsonObject(batch, data) {
+  (0, _loaderUtils.assert)(batch.batchType === 'final-result');
+
+  if (batch.jsonpath === '$') {
+    return data;
   }
 
-  if (metadata) {
-    const finalBatch = {
-      shape,
-      batchType: 'final-result',
-      container: parser.getPartialResult(),
-      jsonpath: parser.getStreamingJsonPathAsString(),
-      data: [],
-      length: 0
-    };
-    yield finalBatch;
+  if (batch.jsonpath && batch.jsonpath.length > 1) {
+    var topLevelObject = batch.container;
+    var streamingPath = new _jsonpath2.default(batch.jsonpath);
+    streamingPath.setFieldAtPath(topLevelObject, data);
+    return topLevelObject;
   }
+
+  return batch.container;
 }
 //# sourceMappingURL=parse-json-in-batches.js.map

package/dist/es5/lib/parse-json-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-json-in-batches.ts"],"names":["parseJSONInBatches","binaryAsyncIterator","options","asyncIterator","metadata","jsonpaths","json","isFirstChunk","schema","shape","tableBatchBuilder","TableBatchBuilder","parser","StreamingJSONParser","chunk","rows","write","jsonpath","length","getStreamingJsonPathAsString","initialBatch","batchType","data","bytesUsed","container","getPartialResult","row","addRow","batch","getFullBatch","chunkComplete","getFinalBatch","finalBatch"],"mappings":";;;;;;;;;AAEA;;AACA;;AACA;;AAIe,gBAAgBA,kBAAhB,CACbC,mBADa,EAEbC,OAFa,EAGS;AAAA;;AACtB,QAAMC,aAAa,GAAG,0CAAwBF,mBAAxB,CAAtB;AAEA,QAAM;AAACG,IAAAA;AAAD,MAAaF,OAAnB;AACA,QAAM;AAACG,IAAAA;AAAD,MAAcH,OAAO,CAACI,IAAR,IAAgB,EAApC;AAEA,MAAIC,YAAqB,GAAG,IAA5B;AAGA,QAAMC,MAAM,GAAG,IAAf;AACA,QAAMC,KAAK,GAAG,CAAAP,OAAO,SAAP,IAAAA,OAAO,WAAP,6BAAAA,OAAO,CAAEI,IAAT,gEAAeG,KAAf,KAAwB,WAAtC;AAEA,QAAMC,iBAAiB,GAAG,IAAIC,yBAAJ,CAAsBH,MAAtB,EAA8B,EACtD,GAAGN,OADmD;AAEtDO,IAAAA;AAFsD,GAA9B,CAA1B;AAKA,QAAMG,MAAM,GAAG,IAAIC,4BAAJ,CAAwB;AAACR,IAAAA;AAAD,GAAxB,CAAf;;AAEA,aAAW,MAAMS,KAAjB,IAA0BX,aAA1B,EAAyC;AACvC,UAAMY,IAAI,GAAGH,MAAM,CAACI,KAAP,CAAaF,KAAb,CAAb;AAEA,UAAMG,QAAQ,GAAGF,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBN,MAAM,CAACO,4BAAP,EAApC;;AAEA,QAAIJ,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBX,YAAvB,EAAqC;AACnC,UAAIH,QAAJ,EAAc;AACZ,cAAMgB,YAAmB,GAAG;AAE1BX,UAAAA,KAF0B;AAG1BY,UAAAA,SAAS,EAAE,gBAHe;AAI1BC,UAAAA,IAAI,EAAE,EAJoB;AAK1BJ,UAAAA,MAAM,EAAE,CALkB;AAM1BK,UAAAA,SAAS,EAAE,CANe;AAQ1BC,UAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EARe;AAS1BR,UAAAA;AAT0B,SAA5B;AAWA,cAAMG,YAAN;AACD;;AACDb,MAAAA,YAAY,GAAG,KAAf;AAED;;AAGD,SAAK,MAAMmB,GAAX,IAAkBX,IAAlB,EAAwB;AACtBL,MAAAA,iBAAiB,CAACiB,MAAlB,CAAyBD,GAAzB;AAEA,YAAME,KAAK,GAAGlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,QAAAA;AAAD,OAA/B,CAAd;;AACA,UAAIW,KAAJ,EAAW;AACT,cAAMA,KAAN;AACD;AACF;;AAEDlB,IAAAA,iBAAiB,CAACoB,aAAlB,CAAgChB,KAAhC;AACA,UAAMc,KAAK,GAAGlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,MAAAA;AAAD,KAA/B,CAAd;;AACA,QAAIW,KAAJ,EAAW;AACT,YAAMA,KAAN;AACD;AACF;;AAGD,QAAMX,QAAQ,GAAGL,MAAM,CAACO,4BAAP,EAAjB;AACA,QAAMS,KAAK,GAAGlB,iBAAiB,CAACqB,aAAlB,CAAgC;AAACd,IAAAA;AAAD,GAAhC,CAAd;;AACA,MAAIW,KAAJ,EAAW;AACT,UAAMA,KAAN;AACD;;AAED,MAAIxB,QAAJ,EAAc;AACZ,UAAM4B,UAAiB,GAAG;AACxBvB,MAAAA,KADwB;AAExBY,MAAAA,SAAS,EAAE,cAFa;AAGxBG,MAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EAHa;AAIxBR,MAAAA,QAAQ,EAAEL,MAAM,CAACO,4BAAP,EAJc;AAKxBG,MAAAA,IAAI,EAAE,EALkB;AAMxBJ,MAAAA,MAAM,EAAE;AANgB,KAA1B;AASA,UAAMc,UAAN;AACD;AACF","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n  const {metadata} = options;\n  const {jsonpaths} = options.json || {};\n\n  let isFirstChunk: boolean = true;\n\n  // TODO fix Schema deduction\n  const schema = null; // new Schema([]);\n  const shape = options?.json?.shape || 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  const parser = new StreamingJSONParser({jsonpaths});\n\n  for await (const chunk of asyncIterator) {\n    const rows = parser.write(chunk);\n\n    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n    if (rows.length > 0 && isFirstChunk) {\n      if (metadata) {\n        const initialBatch: Batch = {\n          // Common fields\n          shape,\n          batchType: 'partial-result',\n          data: [],\n          length: 0,\n          bytesUsed: 0,\n          // JSON additions\n          container: parser.getPartialResult(),\n          jsonpath\n        };\n        yield initialBatch;\n      }\n      isFirstChunk = false;\n      // schema = deduceSchema(rows);\n    }\n\n    // Add the row\n    for (const row of rows) {\n      tableBatchBuilder.addRow(row);\n      // If a batch has been completed, emit it\n      const batch = tableBatchBuilder.getFullBatch({jsonpath});\n      if (batch) {\n        yield batch;\n      }\n    }\n\n    tableBatchBuilder.chunkComplete(chunk);\n    const batch = tableBatchBuilder.getFullBatch({jsonpath});\n    if (batch) {\n      yield batch;\n    }\n  }\n\n  // yield final batch\n  const jsonpath = parser.getStreamingJsonPathAsString();\n  const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n  if (batch) {\n    yield batch;\n  }\n\n  if (metadata) {\n    const finalBatch: Batch = {\n      shape,\n      batchType: 'final-result',\n      container: parser.getPartialResult(),\n      jsonpath: parser.getStreamingJsonPathAsString(),\n      data: [],\n      length: 0\n      // schema: null\n    };\n    yield finalBatch;\n  }\n}\n"],"file":"parse-json-in-batches.js"}
+{"version":3,"sources":["../../../src/lib/parse-json-in-batches.ts"],"names":["parseJSONInBatches","binaryAsyncIterator","options","asyncIterator","metadata","json","jsonpaths","isFirstChunk","schema","shape","tableBatchBuilder","TableBatchBuilder","parser","StreamingJSONParser","chunk","rows","write","jsonpath","length","getStreamingJsonPathAsString","initialBatch","batchType","data","bytesUsed","container","getPartialResult","row","addRow","batch","getFullBatch","chunkComplete","getFinalBatch","finalBatch","rebuildJsonObject","topLevelObject","streamingPath","JSONPath","setFieldAtPath"],"mappings":";;;;;;;;;;;;;;;;;;;;AAEA;;AACA;;AACA;;AACA;;;;;;;;;;;;SAI+BA,kB;;;;;oFAAhB,iBACbC,mBADa,EAEbC,OAFa;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAIPC,YAAAA,aAJO,GAIS,0CAAwBF,mBAAxB,CAJT;AAMNG,YAAAA,QANM,GAMMF,OANN,CAMNE,QANM;AAAA,mBAOOF,OAAO,CAACG,IAAR,IAAgB,EAPvB,EAONC,SAPM,QAONA,SAPM;AASTC,YAAAA,YATS,GASe,IATf;AAYPC,YAAAA,MAZO,GAYE,IAZF;AAaPC,YAAAA,KAbO,GAaC,CAAAP,OAAO,SAAP,IAAAA,OAAO,WAAP,6BAAAA,OAAO,CAAEG,IAAT,gEAAeI,KAAf,KAAwB,WAbzB;AAePC,YAAAA,iBAfO,GAea,IAAIC,yBAAJ,CAAsBH,MAAtB,kCACrBN,OADqB;AAExBO,cAAAA,KAAK,EAALA;AAFwB,eAfb;AAoBPG,YAAAA,MApBO,GAoBE,IAAIC,4BAAJ,CAAwB;AAACP,cAAAA,SAAS,EAATA;AAAD,aAAxB,CApBF;AAAA;AAAA;AAAA;AAAA,qDAsBaH,aAtBb;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAsBIW,YAAAA,KAtBJ;AAuBLC,YAAAA,IAvBK,GAuBEH,MAAM,CAACI,KAAP,CAAaF,KAAb,CAvBF;AAyBLG,YAAAA,SAzBK,GAyBMF,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBN,MAAM,CAACO,4BAAP,EAzBzB;;AAAA,kBA2BPJ,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBX,YA3BZ;AAAA;AAAA;AAAA;;AAAA,iBA4BLH,QA5BK;AAAA;AAAA;AAAA;;AA6BDgB,YAAAA,YA7BC,GA6BqB;AAE1BX,cAAAA,KAAK,EAALA,KAF0B;AAG1BY,cAAAA,SAAS,EAAE,gBAHe;AAI1BC,cAAAA,IAAI,EAAE,EAJoB;AAK1BJ,cAAAA,MAAM,EAAE,CALkB;AAM1BK,cAAAA,SAAS,EAAE,CANe;AAQ1BC,cAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EARe;AAS1BR,cAAAA,QAAQ,EAARA;AAT0B,aA7BrB;AAAA;AAwCP,mBAAMG,YAAN;;AAxCO;AA0CTb,YAAAA,YAAY,GAAG,KAAf;;AA1CS;AAAA,oDA+COQ,IA/CP;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AA+CAW,YAAAA,GA/CA;AAgDThB,YAAAA,iBAAiB,CAACiB,MAAlB,CAAyBD,GAAzB;AAEME,YAAAA,OAlDG,GAkDKlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,cAAAA,QAAQ,EAARA;AAAD,aAA/B,CAlDL;;AAAA,iBAmDLW,OAnDK;AAAA;AAAA;AAAA;;AAAA;AAoDP,mBAAMA,OAAN;;AApDO;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;;AAAA;;AAAA;AAwDXlB,YAAAA,iBAAiB,CAACoB,aAAlB,CAAgChB,KAAhC;AACMc,YAAAA,MAzDK,GAyDGlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,cAAAA,QAAQ,EAARA;AAAD,aAA/B,CAzDH;;AAAA,iBA0DPW,MA1DO;AAAA;AAAA;AAAA;;AAAA;AA2DT,mBAAMA,MAAN;;AA3DS;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAgEPX,YAAAA,QAhEO,GAgEIL,MAAM,CAACO,4BAAP,EAhEJ;AAiEPS,YAAAA,KAjEO,GAiEClB,iBAAiB,CAACqB,aAAlB,CAAgC;AAACd,cAAAA,QAAQ,EAARA;AAAD,aAAhC,CAjED;;AAAA,iBAkETW,KAlES;AAAA;AAAA;AAAA;;AAAA;AAmEX,mBAAMA,KAAN;;AAnEW;AAAA,iBAsETxB,QAtES;AAAA;AAAA;AAAA;;AAuEL4B,YAAAA,UAvEK,GAuEe;AACxBvB,cAAAA,KAAK,EAALA,KADwB;AAExBY,cAAAA,SAAS,EAAE,cAFa;AAGxBG,cAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EAHa;AAIxBR,cAAAA,QAAQ,EAAEL,MAAM,CAACO,4BAAP,EAJc;AAKxBG,cAAAA,IAAI,EAAE,EALkB;AAMxBJ,cAAAA,MAAM,EAAE;AANgB,aAvEf;AAAA;AAgFX,mBAAMc,UAAN;;AAhFW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;AAoFR,SAASC,iBAAT,CAA2BL,KAA3B,EAAkCN,IAAlC,EAAwC;AAE7C,2BAAOM,KAAK,CAACP,SAAN,KAAoB,cAA3B;;AAGA,MAAIO,KAAK,CAACX,QAAN,KAAmB,GAAvB,EAA4B;AAC1B,WAAOK,IAAP;AACD;;AAGD,MAAIM,KAAK,CAACX,QAAN,IAAkBW,KAAK,CAACX,QAAN,CAAeC,MAAf,GAAwB,CAA9C,EAAiD;AAC/C,QAAMgB,cAAc,GAAGN,KAAK,CAACJ,SAA7B;AACA,QAAMW,aAAa,GAAG,IAAIC,kBAAJ,CAAaR,KAAK,CAACX,QAAnB,CAAtB;AACAkB,IAAAA,aAAa,CAACE,cAAd,CAA6BH,cAA7B,EAA6CZ,IAA7C;AACA,WAAOY,cAAP;AACD;;AAGD,SAAON,KAAK,CAACJ,SAAb;AACD","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {assert, makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\nimport JSONPath from './jsonpath/jsonpath';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n  const {metadata} = options;\n  const {jsonpaths} = options.json || {};\n\n  let isFirstChunk: boolean = true;\n\n  // TODO fix Schema deduction\n  const schema = null; // new Schema([]);\n  const shape = options?.json?.shape || 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  const parser = new StreamingJSONParser({jsonpaths});\n\n  for await (const chunk of asyncIterator) {\n    const rows = parser.write(chunk);\n\n    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n    if (rows.length > 0 && isFirstChunk) {\n      if (metadata) {\n        const initialBatch: Batch = {\n          // Common fields\n          shape,\n          batchType: 'partial-result',\n          data: [],\n          length: 0,\n          bytesUsed: 0,\n          // JSON additions\n          container: parser.getPartialResult(),\n          jsonpath\n        };\n        yield initialBatch;\n      }\n      isFirstChunk = false;\n      // schema = deduceSchema(rows);\n    }\n\n    // Add the row\n    for (const row of rows) {\n      tableBatchBuilder.addRow(row);\n      // If a batch has been completed, emit it\n      const batch = tableBatchBuilder.getFullBatch({jsonpath});\n      if (batch) {\n        yield batch;\n      }\n    }\n\n    tableBatchBuilder.chunkComplete(chunk);\n    const batch = tableBatchBuilder.getFullBatch({jsonpath});\n    if (batch) {\n      yield batch;\n    }\n  }\n\n  // yield final batch\n  const jsonpath = parser.getStreamingJsonPathAsString();\n  const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n  if (batch) {\n    yield batch;\n  }\n\n  if (metadata) {\n    const finalBatch: Batch = {\n      shape,\n      batchType: 'final-result',\n      container: parser.getPartialResult(),\n      jsonpath: parser.getStreamingJsonPathAsString(),\n      data: [],\n      length: 0\n      // schema: null\n    };\n    yield finalBatch;\n  }\n}\n\nexport function rebuildJsonObject(batch, data) {\n  // Last batch will have this special type and will provide all the root object of the parsed file\n  assert(batch.batchType === 'final-result');\n\n  // The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects\n  if (batch.jsonpath === '$') {\n    return data;\n  }\n\n  // (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object\n  if (batch.jsonpath && batch.jsonpath.length > 1) {\n    const topLevelObject = batch.container;\n    const streamingPath = new JSONPath(batch.jsonpath);\n    streamingPath.setFieldAtPath(topLevelObject, data);\n    return topLevelObject;\n  }\n\n  // No jsonpath, in this case nothing was streamed.\n  return batch.container;\n}\n"],"file":"parse-json-in-batches.js"}

package/dist/es5/lib/parse-json.js

@@ -1,15 +1,19 @@
 "use strict";
 
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = parseJSONSync;
 
+var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
+
 function parseJSONSync(jsonText, options) {
   try {
     var _options$json;
 
-    const json = JSON.parse(jsonText);
+    var json = JSON.parse(jsonText);
 
     if ((_options$json = options.json) !== null && _options$json !== void 0 && _options$json.table) {
       return getFirstArray(json) || json;
@@ -26,9 +30,10 @@ function getFirstArray(json) {
     return json;
   }
 
-  if (json && typeof json === 'object') {
-    for (const value of Object.values(json)) {
-      const array = getFirstArray(value);
+  if (json && (0, _typeof2.default)(json) === 'object') {
+    for (var _i = 0, _Object$values = Object.values(json); _i < _Object$values.length; _i++) {
+      var value = _Object$values[_i];
+      var array = getFirstArray(value);
 
       if (array) {
         return array;

package/dist/es5/lib/parse-json.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-json.ts"],"names":["parseJSONSync","jsonText","options","json","JSON","parse","table","getFirstArray","error","Error","Array","isArray","value","Object","values","array"],"mappings":";;;;;;;AAEe,SAASA,aAAT,CAAuBC,QAAvB,EAAyCC,OAAzC,EAAqE;AAClF,MAAI;AAAA;;AACF,UAAMC,IAAI,GAAGC,IAAI,CAACC,KAAL,CAAWJ,QAAX,CAAb;;AACA,yBAAIC,OAAO,CAACC,IAAZ,0CAAI,cAAcG,KAAlB,EAAyB;AACvB,aAAOC,aAAa,CAACJ,IAAD,CAAb,IAAuBA,IAA9B;AACD;;AACD,WAAOA,IAAP;AACD,GAND,CAME,OAAOK,KAAP,EAAc;AACd,UAAM,IAAIC,KAAJ,CAAU,kCAAV,CAAN;AACD;AACF;;AAED,SAASF,aAAT,CAAuBJ,IAAvB,EAA6B;AAC3B,MAAIO,KAAK,CAACC,OAAN,CAAcR,IAAd,CAAJ,EAAyB;AACvB,WAAOA,IAAP;AACD;;AACD,MAAIA,IAAI,IAAI,OAAOA,IAAP,KAAgB,QAA5B,EAAsC;AACpC,SAAK,MAAMS,KAAX,IAAoBC,MAAM,CAACC,MAAP,CAAcX,IAAd,CAApB,EAAyC;AACvC,YAAMY,KAAK,GAAGR,aAAa,CAACK,KAAD,CAA3B;;AACA,UAAIG,KAAJ,EAAW;AACT,eAAOA,KAAP;AACD;AACF;AACF;;AACD,SAAO,IAAP;AACD","sourcesContent":["import type {JSONLoaderOptions} from '../json-loader';\n\nexport default function parseJSONSync(jsonText: string, options: JSONLoaderOptions) {\n  try {\n    const json = JSON.parse(jsonText);\n    if (options.json?.table) {\n      return getFirstArray(json) || json;\n    }\n    return json;\n  } catch (error) {\n    throw new Error('JSONLoader: failed to parse JSON');\n  }\n}\n\nfunction getFirstArray(json) {\n  if (Array.isArray(json)) {\n    return json;\n  }\n  if (json && typeof json === 'object') {\n    for (const value of Object.values(json)) {\n      const array = getFirstArray(value);\n      if (array) {\n        return array;\n      }\n    }\n  }\n  return null;\n}\n"],"file":"parse-json.js"}
+{"version":3,"sources":["../../../src/lib/parse-json.ts"],"names":["parseJSONSync","jsonText","options","json","JSON","parse","table","getFirstArray","error","Error","Array","isArray","Object","values","value","array"],"mappings":";;;;;;;;;;;AAEe,SAASA,aAAT,CAAuBC,QAAvB,EAAyCC,OAAzC,EAAqE;AAClF,MAAI;AAAA;;AACF,QAAMC,IAAI,GAAGC,IAAI,CAACC,KAAL,CAAWJ,QAAX,CAAb;;AACA,yBAAIC,OAAO,CAACC,IAAZ,0CAAI,cAAcG,KAAlB,EAAyB;AACvB,aAAOC,aAAa,CAACJ,IAAD,CAAb,IAAuBA,IAA9B;AACD;;AACD,WAAOA,IAAP;AACD,GAND,CAME,OAAOK,KAAP,EAAc;AACd,UAAM,IAAIC,KAAJ,CAAU,kCAAV,CAAN;AACD;AACF;;AAED,SAASF,aAAT,CAAuBJ,IAAvB,EAA6B;AAC3B,MAAIO,KAAK,CAACC,OAAN,CAAcR,IAAd,CAAJ,EAAyB;AACvB,WAAOA,IAAP;AACD;;AACD,MAAIA,IAAI,IAAI,sBAAOA,IAAP,MAAgB,QAA5B,EAAsC;AACpC,sCAAoBS,MAAM,CAACC,MAAP,CAAcV,IAAd,CAApB,oCAAyC;AAApC,UAAMW,KAAK,qBAAX;AACH,UAAMC,KAAK,GAAGR,aAAa,CAACO,KAAD,CAA3B;;AACA,UAAIC,KAAJ,EAAW;AACT,eAAOA,KAAP;AACD;AACF;AACF;;AACD,SAAO,IAAP;AACD","sourcesContent":["import type {JSONLoaderOptions} from '../json-loader';\n\nexport default function parseJSONSync(jsonText: string, options: JSONLoaderOptions) {\n  try {\n    const json = JSON.parse(jsonText);\n    if (options.json?.table) {\n      return getFirstArray(json) || json;\n    }\n    return json;\n  } catch (error) {\n    throw new Error('JSONLoader: failed to parse JSON');\n  }\n}\n\nfunction getFirstArray(json) {\n  if (Array.isArray(json)) {\n    return json;\n  }\n  if (json && typeof json === 'object') {\n    for (const value of Object.values(json)) {\n      const array = getFirstArray(value);\n      if (array) {\n        return array;\n      }\n    }\n  }\n  return null;\n}\n"],"file":"parse-json.js"}

package/dist/es5/lib/parse-ndjson-in-batches.js

@@ -1,46 +1,158 @@
 "use strict";
 
+var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
+
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = parseNDJSONInBatches;
 
+var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
+
+var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
+
+var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
+
+var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
+
+var _asyncIterator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncIterator"));
+
 var _schema = require("@loaders.gl/schema");
 
 var _loaderUtils = require("@loaders.gl/loader-utils");
 
-async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
-  const textIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
-  const lineIterator = (0, _loaderUtils.makeLineIterator)(textIterator);
-  const numberedLineIterator = (0, _loaderUtils.makeNumberedLineIterator)(lineIterator);
-  const schema = null;
-  const shape = 'row-table';
-  const tableBatchBuilder = new _schema.TableBatchBuilder(schema, { ...options,
-    shape
-  });
-
-  for await (const {
-    counter,
-    line
-  } of numberedLineIterator) {
-    try {
-      const row = JSON.parse(line);
-      tableBatchBuilder.addRow(row);
-      tableBatchBuilder.chunkComplete(line);
-      const batch = tableBatchBuilder.getFullBatch();
-
-      if (batch) {
-        yield batch;
-      }
-    } catch (error) {
-      throw new Error("NDJSONLoader: failed to parse JSON on line ".concat(counter));
-    }
-  }
+function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
+
+function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
+
+function parseNDJSONInBatches(_x, _x2) {
+  return _parseNDJSONInBatches.apply(this, arguments);
+}
+
+function _parseNDJSONInBatches() {
+  _parseNDJSONInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(binaryAsyncIterator, options) {
+    var textIterator, lineIterator, numberedLineIterator, schema, shape, tableBatchBuilder, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _value, _value2, counter, line, row, _batch, batch;
+
+    return _regenerator.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            textIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
+            lineIterator = (0, _loaderUtils.makeLineIterator)(textIterator);
+            numberedLineIterator = (0, _loaderUtils.makeNumberedLineIterator)(lineIterator);
+            schema = null;
+            shape = 'row-table';
+            tableBatchBuilder = new _schema.TableBatchBuilder(schema, _objectSpread(_objectSpread({}, options), {}, {
+              shape: shape
+            }));
+            _iteratorNormalCompletion = true;
+            _didIteratorError = false;
+            _context.prev = 8;
+            _iterator = (0, _asyncIterator2.default)(numberedLineIterator);
+
+          case 10:
+            _context.next = 12;
+            return (0, _awaitAsyncGenerator2.default)(_iterator.next());
+
+          case 12:
+            _step = _context.sent;
+            _iteratorNormalCompletion = _step.done;
+            _context.next = 16;
+            return (0, _awaitAsyncGenerator2.default)(_step.value);
+
+          case 16:
+            _value = _context.sent;
+
+            if (_iteratorNormalCompletion) {
+              _context.next = 35;
+              break;
+            }
+
+            _value2 = _value, counter = _value2.counter, line = _value2.line;
+            _context.prev = 19;
+            row = JSON.parse(line);
+            tableBatchBuilder.addRow(row);
+            tableBatchBuilder.chunkComplete(line);
+            _batch = tableBatchBuilder.getFullBatch();
+
+            if (!_batch) {
+              _context.next = 27;
+              break;
+            }
 
-  const batch = tableBatchBuilder.getFinalBatch();
+            _context.next = 27;
+            return _batch;
 
-  if (batch) {
-    yield batch;
-  }
+          case 27:
+            _context.next = 32;
+            break;
+
+          case 29:
+            _context.prev = 29;
+            _context.t0 = _context["catch"](19);
+            throw new Error("NDJSONLoader: failed to parse JSON on line ".concat(counter));
+
+          case 32:
+            _iteratorNormalCompletion = true;
+            _context.next = 10;
+            break;
+
+          case 35:
+            _context.next = 41;
+            break;
+
+          case 37:
+            _context.prev = 37;
+            _context.t1 = _context["catch"](8);
+            _didIteratorError = true;
+            _iteratorError = _context.t1;
+
+          case 41:
+            _context.prev = 41;
+            _context.prev = 42;
+
+            if (!(!_iteratorNormalCompletion && _iterator.return != null)) {
+              _context.next = 46;
+              break;
+            }
+
+            _context.next = 46;
+            return (0, _awaitAsyncGenerator2.default)(_iterator.return());
+
+          case 46:
+            _context.prev = 46;
+
+            if (!_didIteratorError) {
+              _context.next = 49;
+              break;
+            }
+
+            throw _iteratorError;
+
+          case 49:
+            return _context.finish(46);
+
+          case 50:
+            return _context.finish(41);
+
+          case 51:
+            batch = tableBatchBuilder.getFinalBatch();
+
+            if (!batch) {
+              _context.next = 55;
+              break;
+            }
+
+            _context.next = 55;
+            return batch;
+
+          case 55:
+          case "end":
+            return _context.stop();
+        }
+      }
+    }, _callee, null, [[8, 37, 41, 51], [19, 29], [42,, 46, 50]]);
+  }));
+  return _parseNDJSONInBatches.apply(this, arguments);
 }
 //# sourceMappingURL=parse-ndjson-in-batches.js.map

package/dist/es5/lib/parse-ndjson-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-ndjson-in-batches.ts"],"names":["parseNDJSONInBatches","binaryAsyncIterator","options","textIterator","lineIterator","numberedLineIterator","schema","shape","tableBatchBuilder","TableBatchBuilder","counter","line","row","JSON","parse","addRow","chunkComplete","batch","getFullBatch","error","Error","getFinalBatch"],"mappings":";;;;;;;AACA;;AACA;;AAOe,gBAAgBA,oBAAhB,CACbC,mBADa,EAEbC,OAFa,EAGS;AACtB,QAAMC,YAAY,GAAG,0CAAwBF,mBAAxB,CAArB;AACA,QAAMG,YAAY,GAAG,mCAAiBD,YAAjB,CAArB;AACA,QAAME,oBAAoB,GAAG,2CAAyBD,YAAzB,CAA7B;AAEA,QAAME,MAAM,GAAG,IAAf;AACA,QAAMC,KAAK,GAAG,WAAd;AAEA,QAAMC,iBAAiB,GAAG,IAAIC,yBAAJ,CAAsBH,MAAtB,EAA8B,EACtD,GAAGJ,OADmD;AAEtDK,IAAAA;AAFsD,GAA9B,CAA1B;;AAKA,aAAW,MAAM;AAACG,IAAAA,OAAD;AAAUC,IAAAA;AAAV,GAAjB,IAAoCN,oBAApC,EAA0D;AACxD,QAAI;AACF,YAAMO,GAAG,GAAGC,IAAI,CAACC,KAAL,CAAWH,IAAX,CAAZ;AACAH,MAAAA,iBAAiB,CAACO,MAAlB,CAAyBH,GAAzB;AACAJ,MAAAA,iBAAiB,CAACQ,aAAlB,CAAgCL,IAAhC;AACA,YAAMM,KAAK,GAAGT,iBAAiB,CAACU,YAAlB,EAAd;;AACA,UAAID,KAAJ,EAAW;AACT,cAAMA,KAAN;AACD;AACF,KARD,CAQE,OAAOE,KAAP,EAAc;AACd,YAAM,IAAIC,KAAJ,sDAAwDV,OAAxD,EAAN;AACD;AACF;;AAED,QAAMO,KAAK,GAAGT,iBAAiB,CAACa,aAAlB,EAAd;;AACA,MAAIJ,KAAJ,EAAW;AACT,UAAMA,KAAN;AACD;AACF","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {\n  LoaderOptions,\n  makeLineIterator,\n  makeNumberedLineIterator,\n  makeTextDecoderIterator\n} from '@loaders.gl/loader-utils';\n\nexport default async function* parseNDJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: LoaderOptions\n): AsyncIterable<Batch> {\n  const textIterator = makeTextDecoderIterator(binaryAsyncIterator);\n  const lineIterator = makeLineIterator(textIterator);\n  const numberedLineIterator = makeNumberedLineIterator(lineIterator);\n\n  const schema = null;\n  const shape = 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  for await (const {counter, line} of numberedLineIterator) {\n    try {\n      const row = JSON.parse(line);\n      tableBatchBuilder.addRow(row);\n      tableBatchBuilder.chunkComplete(line);\n      const batch = tableBatchBuilder.getFullBatch();\n      if (batch) {\n        yield batch;\n      }\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);\n    }\n  }\n\n  const batch = tableBatchBuilder.getFinalBatch();\n  if (batch) {\n    yield batch;\n  }\n}\n"],"file":"parse-ndjson-in-batches.js"}
+{"version":3,"sources":["../../../src/lib/parse-ndjson-in-batches.ts"],"names":["parseNDJSONInBatches","binaryAsyncIterator","options","textIterator","lineIterator","numberedLineIterator","schema","shape","tableBatchBuilder","TableBatchBuilder","counter","line","row","JSON","parse","addRow","chunkComplete","batch","getFullBatch","Error","getFinalBatch"],"mappings":";;;;;;;;;;;;;;;;;;;AACA;;AACA;;;;;;SAO+BA,oB;;;;;sFAAhB,iBACbC,mBADa,EAEbC,OAFa;AAAA;;AAAA;AAAA;AAAA;AAAA;AAIPC,YAAAA,YAJO,GAIQ,0CAAwBF,mBAAxB,CAJR;AAKPG,YAAAA,YALO,GAKQ,mCAAiBD,YAAjB,CALR;AAMPE,YAAAA,oBANO,GAMgB,2CAAyBD,YAAzB,CANhB;AAQPE,YAAAA,MARO,GAQE,IARF;AASPC,YAAAA,KATO,GASC,WATD;AAWPC,YAAAA,iBAXO,GAWa,IAAIC,yBAAJ,CAAsBH,MAAtB,kCACrBJ,OADqB;AAExBK,cAAAA,KAAK,EAALA;AAFwB,eAXb;AAAA;AAAA;AAAA;AAAA,qDAgBuBF,oBAhBvB;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA,8BAgBKK,OAhBL,WAgBKA,OAhBL,EAgBcC,IAhBd,WAgBcA,IAhBd;AAAA;AAkBHC,YAAAA,GAlBG,GAkBGC,IAAI,CAACC,KAAL,CAAWH,IAAX,CAlBH;AAmBTH,YAAAA,iBAAiB,CAACO,MAAlB,CAAyBH,GAAzB;AACAJ,YAAAA,iBAAiB,CAACQ,aAAlB,CAAgCL,IAAhC;AACMM,YAAAA,MArBG,GAqBKT,iBAAiB,CAACU,YAAlB,EArBL;;AAAA,iBAsBLD,MAtBK;AAAA;AAAA;AAAA;;AAAA;AAuBP,mBAAMA,MAAN;;AAvBO;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA,kBA0BH,IAAIE,KAAJ,sDAAwDT,OAAxD,EA1BG;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AA8BPO,YAAAA,KA9BO,GA8BCT,iBAAiB,CAACY,aAAlB,EA9BD;;AAAA,iBA+BTH,KA/BS;AAAA;AAAA;AAAA;;AAAA;AAgCX,mBAAMA,KAAN;;AAhCW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {\n  LoaderOptions,\n  makeLineIterator,\n  makeNumberedLineIterator,\n  makeTextDecoderIterator\n} from '@loaders.gl/loader-utils';\n\nexport default async function* parseNDJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: LoaderOptions\n): AsyncIterable<Batch> {\n  const textIterator = makeTextDecoderIterator(binaryAsyncIterator);\n  const lineIterator = makeLineIterator(textIterator);\n  const numberedLineIterator = makeNumberedLineIterator(lineIterator);\n\n  const schema = null;\n  const shape = 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  for await (const {counter, line} of numberedLineIterator) {\n    try {\n      const row = JSON.parse(line);\n      tableBatchBuilder.addRow(row);\n      tableBatchBuilder.chunkComplete(line);\n      const batch = tableBatchBuilder.getFullBatch();\n      if (batch) {\n        yield batch;\n      }\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);\n    }\n  }\n\n  const batch = tableBatchBuilder.getFinalBatch();\n  if (batch) {\n    yield batch;\n  }\n}\n"],"file":"parse-ndjson-in-batches.js"}

package/dist/es5/lib/parse-ndjson.js

@@ -6,8 +6,8 @@ Object.defineProperty(exports, "__esModule", {
 exports.default = parseNDJSONSync;
 
 function parseNDJSONSync(ndjsonText) {
-  const lines = ndjsonText.trim().split('\n');
-  return lines.map((line, counter) => {
+  var lines = ndjsonText.trim().split('\n');
+  return lines.map(function (line, counter) {
     try {
       return JSON.parse(line);
     } catch (error) {

package/dist/es5/lib/parse-ndjson.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-ndjson.ts"],"names":["parseNDJSONSync","ndjsonText","lines","trim","split","map","line","counter","JSON","parse","error","Error"],"mappings":";;;;;;;AAAe,SAASA,eAAT,CAAyBC,UAAzB,EAA6C;AAC1D,QAAMC,KAAK,GAAGD,UAAU,CAACE,IAAX,GAAkBC,KAAlB,CAAwB,IAAxB,CAAd;AACA,SAAOF,KAAK,CAACG,GAAN,CAAU,CAACC,IAAD,EAAOC,OAAP,KAAmB;AAClC,QAAI;AACF,aAAOC,IAAI,CAACC,KAAL,CAAWH,IAAX,CAAP;AACD,KAFD,CAEE,OAAOI,KAAP,EAAc;AACd,YAAM,IAAIC,KAAJ,sDAAwDJ,OAAO,GAAG,CAAlE,EAAN;AACD;AACF,GANM,CAAP;AAOD","sourcesContent":["export default function parseNDJSONSync(ndjsonText: string) {\n  const lines = ndjsonText.trim().split('\\n');\n  return lines.map((line, counter) => {\n    try {\n      return JSON.parse(line);\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);\n    }\n  });\n}\n"],"file":"parse-ndjson.js"}
+{"version":3,"sources":["../../../src/lib/parse-ndjson.ts"],"names":["parseNDJSONSync","ndjsonText","lines","trim","split","map","line","counter","JSON","parse","error","Error"],"mappings":";;;;;;;AAAe,SAASA,eAAT,CAAyBC,UAAzB,EAA6C;AAC1D,MAAMC,KAAK,GAAGD,UAAU,CAACE,IAAX,GAAkBC,KAAlB,CAAwB,IAAxB,CAAd;AACA,SAAOF,KAAK,CAACG,GAAN,CAAU,UAACC,IAAD,EAAOC,OAAP,EAAmB;AAClC,QAAI;AACF,aAAOC,IAAI,CAACC,KAAL,CAAWH,IAAX,CAAP;AACD,KAFD,CAEE,OAAOI,KAAP,EAAc;AACd,YAAM,IAAIC,KAAJ,sDAAwDJ,OAAO,GAAG,CAAlE,EAAN;AACD;AACF,GANM,CAAP;AAOD","sourcesContent":["export default function parseNDJSONSync(ndjsonText: string) {\n  const lines = ndjsonText.trim().split('\\n');\n  return lines.map((line, counter) => {\n    try {\n      return JSON.parse(line);\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);\n    }\n  });\n}\n"],"file":"parse-ndjson.js"}