@loaders.gl/json 3.4.13 → 3.4.15

package/dist/es5/lib/parse-json-in-batches.js

@@ -6,189 +6,82 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = parseJSONInBatches;
 exports.rebuildJsonObject = rebuildJsonObject;
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
-var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
 var _schema = require("@loaders.gl/schema");
 var _loaderUtils = require("@loaders.gl/loader-utils");
 var _streamingJsonParser = _interopRequireDefault(require("./parser/streaming-json-parser"));
-var _jsonpath2 = _interopRequireDefault(require("./jsonpath/jsonpath"));
-function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
-function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
-function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) arr2[i] = arr[i]; return arr2; }
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
-function _asyncIterator(iterable) { var method, async, sync, retry = 2; for ("undefined" != typeof Symbol && (async = Symbol.asyncIterator, sync = Symbol.iterator); retry--;) { if (async && null != (method = iterable[async])) return method.call(iterable); if (sync && null != (method = iterable[sync])) return new AsyncFromSyncIterator(method.call(iterable)); async = "@@asyncIterator", sync = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
-function AsyncFromSyncIterator(s) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var done = r.done; return Promise.resolve(r.value).then(function (value) { return { value: value, done: done }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(s) { this.s = s, this.n = s.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, return: function _return(value) { var ret = this.s.return; return void 0 === ret ? Promise.resolve({ value: value, done: !0 }) : AsyncFromSyncIteratorContinuation(ret.apply(this.s, arguments)); }, throw: function _throw(value) { var thr = this.s.return; return void 0 === thr ? Promise.reject(value) : AsyncFromSyncIteratorContinuation(thr.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(s); }
-function parseJSONInBatches(_x, _x2) {
-  return _parseJSONInBatches.apply(this, arguments);
-}
-function _parseJSONInBatches() {
-  _parseJSONInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(binaryAsyncIterator, options) {
-    var _options$json;
-    var asyncIterator, metadata, _ref, jsonpaths, isFirstChunk, schema, shape, tableBatchBuilder, parser, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, chunk, rows, _jsonpath, initialBatch, _iterator2, _step2, row, _batch2, _batch, jsonpath, batch, finalBatch;
-    return _regenerator.default.wrap(function _callee$(_context) {
-      while (1) switch (_context.prev = _context.next) {
-        case 0:
-          asyncIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
-          metadata = options.metadata;
-          _ref = options.json || {}, jsonpaths = _ref.jsonpaths;
-          isFirstChunk = true;
-          schema = null;
-          shape = (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'row-table';
-          tableBatchBuilder = new _schema.TableBatchBuilder(schema, _objectSpread(_objectSpread({}, options), {}, {
-            shape: shape
-          }));
-          parser = new _streamingJsonParser.default({
-            jsonpaths: jsonpaths
-          });
-          _iteratorAbruptCompletion = false;
-          _didIteratorError = false;
-          _context.prev = 10;
-          _iterator = _asyncIterator(asyncIterator);
-        case 12:
-          _context.next = 14;
-          return (0, _awaitAsyncGenerator2.default)(_iterator.next());
-        case 14:
-          if (!(_iteratorAbruptCompletion = !(_step = _context.sent).done)) {
-            _context.next = 52;
-            break;
-          }
-          chunk = _step.value;
-          rows = parser.write(chunk);
-          _jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
-          if (!(rows.length > 0 && isFirstChunk)) {
-            _context.next = 24;
-            break;
-          }
-          if (!metadata) {
-            _context.next = 23;
-            break;
-          }
-          initialBatch = {
-            shape: shape,
-            batchType: 'partial-result',
-            data: [],
-            length: 0,
-            bytesUsed: 0,
-            container: parser.getPartialResult(),
-            jsonpath: _jsonpath
-          };
-          _context.next = 23;
-          return initialBatch;
-        case 23:
-          isFirstChunk = false;
-        case 24:
-          _iterator2 = _createForOfIteratorHelper(rows);
-          _context.prev = 25;
-          _iterator2.s();
-        case 27:
-          if ((_step2 = _iterator2.n()).done) {
-            _context.next = 36;
-            break;
-          }
-          row = _step2.value;
-          tableBatchBuilder.addRow(row);
-          _batch2 = tableBatchBuilder.getFullBatch({
-            jsonpath: _jsonpath
-          });
-          if (!_batch2) {
-            _context.next = 34;
-            break;
-          }
-          _context.next = 34;
-          return _batch2;
-        case 34:
-          _context.next = 27;
-          break;
-        case 36:
-          _context.next = 41;
-          break;
-        case 38:
-          _context.prev = 38;
-          _context.t0 = _context["catch"](25);
-          _iterator2.e(_context.t0);
-        case 41:
-          _context.prev = 41;
-          _iterator2.f();
-          return _context.finish(41);
-        case 44:
-          tableBatchBuilder.chunkComplete(chunk);
-          _batch = tableBatchBuilder.getFullBatch({
-            jsonpath: _jsonpath
-          });
-          if (!_batch) {
-            _context.next = 49;
-            break;
-          }
-          _context.next = 49;
-          return _batch;
-        case 49:
-          _iteratorAbruptCompletion = false;
-          _context.next = 12;
-          break;
-        case 52:
-          _context.next = 58;
-          break;
-        case 54:
-          _context.prev = 54;
-          _context.t1 = _context["catch"](10);
-          _didIteratorError = true;
-          _iteratorError = _context.t1;
-        case 58:
-          _context.prev = 58;
-          _context.prev = 59;
-          if (!(_iteratorAbruptCompletion && _iterator.return != null)) {
-            _context.next = 63;
-            break;
-          }
-          _context.next = 63;
-          return (0, _awaitAsyncGenerator2.default)(_iterator.return());
-        case 63:
-          _context.prev = 63;
-          if (!_didIteratorError) {
-            _context.next = 66;
-            break;
-          }
-          throw _iteratorError;
-        case 66:
-          return _context.finish(63);
-        case 67:
-          return _context.finish(58);
-        case 68:
-          jsonpath = parser.getStreamingJsonPathAsString();
-          batch = tableBatchBuilder.getFinalBatch({
-            jsonpath: jsonpath
-          });
-          if (!batch) {
-            _context.next = 73;
-            break;
-          }
-          _context.next = 73;
-          return batch;
-        case 73:
-          if (!metadata) {
-            _context.next = 77;
-            break;
-          }
-          finalBatch = {
-            shape: shape,
-            batchType: 'final-result',
-            container: parser.getPartialResult(),
-            jsonpath: parser.getStreamingJsonPathAsString(),
-            data: [],
-            length: 0
-          };
-          _context.next = 77;
-          return finalBatch;
-        case 77:
-        case "end":
-          return _context.stop();
+var _jsonpath = _interopRequireDefault(require("./jsonpath/jsonpath"));
+async function* parseJSONInBatches(binaryAsyncIterator, options) {
+  var _options$json;
+  const asyncIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
+  const {
+    metadata
+  } = options;
+  const {
+    jsonpaths
+  } = options.json || {};
+  let isFirstChunk = true;
+  const schema = null;
+  const shape = (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'row-table';
+  const tableBatchBuilder = new _schema.TableBatchBuilder(schema, {
+    ...options,
+    shape
+  });
+  const parser = new _streamingJsonParser.default({
+    jsonpaths
+  });
+  for await (const chunk of asyncIterator) {
+    const rows = parser.write(chunk);
+    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
+    if (rows.length > 0 && isFirstChunk) {
+      if (metadata) {
+        const initialBatch = {
+          shape,
+          batchType: 'partial-result',
+          data: [],
+          length: 0,
+          bytesUsed: 0,
+          container: parser.getPartialResult(),
+          jsonpath
+        };
+        yield initialBatch;
+      }
+      isFirstChunk = false;
+    }
+    for (const row of rows) {
+      tableBatchBuilder.addRow(row);
+      const batch = tableBatchBuilder.getFullBatch({
+        jsonpath
+      });
+      if (batch) {
+        yield batch;
       }
-    }, _callee, null, [[10, 54, 58, 68], [25, 38, 41, 44], [59,, 63, 67]]);
-  }));
-  return _parseJSONInBatches.apply(this, arguments);
+    }
+    tableBatchBuilder.chunkComplete(chunk);
+    const batch = tableBatchBuilder.getFullBatch({
+      jsonpath
+    });
+    if (batch) {
+      yield batch;
+    }
+  }
+  const jsonpath = parser.getStreamingJsonPathAsString();
+  const batch = tableBatchBuilder.getFinalBatch({
+    jsonpath
+  });
+  if (batch) {
+    yield batch;
+  }
+  if (metadata) {
+    const finalBatch = {
+      shape,
+      batchType: 'final-result',
+      container: parser.getPartialResult(),
+      jsonpath: parser.getStreamingJsonPathAsString(),
+      data: [],
+      length: 0
+    };
+    yield finalBatch;
+  }
 }
 function rebuildJsonObject(batch, data) {
   (0, _loaderUtils.assert)(batch.batchType === 'final-result');
@@ -196,8 +89,8 @@ function rebuildJsonObject(batch, data) {
     return data;
   }
   if (batch.jsonpath && batch.jsonpath.length > 1) {
-    var topLevelObject = batch.container;
-    var streamingPath = new _jsonpath2.default(batch.jsonpath);
+    const topLevelObject = batch.container;
+    const streamingPath = new _jsonpath.default(batch.jsonpath);
     streamingPath.setFieldAtPath(topLevelObject, data);
     return topLevelObject;
   }

package/dist/es5/lib/parse-json-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-json-in-batches.js","names":["_schema","require","_loaderUtils","_streamingJsonParser","_interopRequireDefault","_jsonpath2","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","call","step","next","_e2","return","minLen","_arrayLikeToArray","Object","prototype","toString","slice","constructor","name","from","test","arr","len","arr2","ownKeys","object","enumerableOnly","keys","getOwnPropertySymbols","symbols","filter","sym","getOwnPropertyDescriptor","enumerable","push","apply","_objectSpread","target","arguments","source","forEach","key","_defineProperty2","default","getOwnPropertyDescriptors","defineProperties","defineProperty","_asyncIterator","iterable","method","async","sync","retry","asyncIterator","AsyncFromSyncIterator","AsyncFromSyncIteratorContinuation","r","Promise","reject","resolve","then","_return","ret","throw","_throw","thr","parseJSONInBatches","_x","_x2","_parseJSONInBatches","_wrapAsyncGenerator2","_regenerator","mark","_callee","binaryAsyncIterator","options","_options$json","metadata","_ref","jsonpaths","isFirstChunk","schema","shape","tableBatchBuilder","parser","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","chunk","rows","_jsonpath","initialBatch","_iterator2","_step2","row","_batch2","_batch","jsonpath","batch","finalBatch","wrap","_callee$","_context","prev","makeTextDecoderIterator","json","TableBatchBuilder","StreamingJSONParser","_awaitAsyncGenerator2","sent","write","getStreamingJsonPathAsString","batchType","data","bytesUsed","container","getPartialResult","addRow","getFullBatch","t0","finish","chunkComplete","t1","getFinalBatch","stop","rebuildJsonObject","assert","topLevelObject","streamingPath","JSONPath","setFieldAtPath"],"sources":["../../../src/lib/parse-json-in-batches.ts"],"sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {assert, makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\nimport JSONPath from './jsonpath/jsonpath';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n  const {metadata} = options;\n  const {jsonpaths} = options.json || {};\n\n  let isFirstChunk: boolean = true;\n\n  // TODO fix Schema deduction\n  const schema = null; // new Schema([]);\n  const shape = options?.json?.shape || 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  const parser = new StreamingJSONParser({jsonpaths});\n\n  for await (const chunk of asyncIterator) {\n    const rows = parser.write(chunk);\n\n    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n    if (rows.length > 0 && isFirstChunk) {\n      if (metadata) {\n        const initialBatch: Batch = {\n          // Common fields\n          shape,\n          batchType: 'partial-result',\n          data: [],\n          length: 0,\n          bytesUsed: 0,\n          // JSON additions\n          container: parser.getPartialResult(),\n          jsonpath\n        };\n        yield initialBatch;\n      }\n      isFirstChunk = false;\n      // schema = deduceSchema(rows);\n    }\n\n    // Add the row\n    for (const row of rows) {\n      tableBatchBuilder.addRow(row);\n      // If a batch has been completed, emit it\n      const batch = tableBatchBuilder.getFullBatch({jsonpath});\n      if (batch) {\n        yield batch;\n      }\n    }\n\n    tableBatchBuilder.chunkComplete(chunk);\n    const batch = tableBatchBuilder.getFullBatch({jsonpath});\n    if (batch) {\n      yield batch;\n    }\n  }\n\n  // yield final batch\n  const jsonpath = parser.getStreamingJsonPathAsString();\n  const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n  if (batch) {\n    yield batch;\n  }\n\n  if (metadata) {\n    const finalBatch: Batch = {\n      shape,\n      batchType: 'final-result',\n      container: parser.getPartialResult(),\n      jsonpath: parser.getStreamingJsonPathAsString(),\n      data: [],\n      length: 0\n      // schema: null\n    };\n    yield finalBatch;\n  }\n}\n\nexport function rebuildJsonObject(batch, data) {\n  // Last batch will have this special type and will provide all the root object of the parsed file\n  assert(batch.batchType === 'final-result');\n\n  // The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects\n  if (batch.jsonpath === '$') {\n    return data;\n  }\n\n  // (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object\n  if (batch.jsonpath && batch.jsonpath.length > 1) {\n    const topLevelObject = batch.container;\n    const streamingPath = new JSONPath(batch.jsonpath);\n    streamingPath.setFieldAtPath(topLevelObject, data);\n    return topLevelObject;\n  }\n\n  // No jsonpath, in this case nothing was streamed.\n  return batch.container;\n}\n"],"mappings":";;;;;;;;;;;;AAEA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AACA,IAAAE,oBAAA,GAAAC,sBAAA,CAAAH,OAAA;AACA,IAAAI,UAAA,GAAAD,sBAAA,CAAAH,OAAA;AAA2C,SAAAK,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAoB,IAAA,CAAAtB,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAW,IAAA,GAAArB,EAAA,CAAAsB,IAAA,IAAAL,gBAAA,GAAAI,IAAA,CAAAV,IAAA,SAAAU,IAAA,KAAAR,CAAA,WAAAA,EAAAU,GAAA,IAAAL,MAAA,SAAAC,GAAA,GAAAI,GAAA,KAAAR,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAwB,MAAA,UAAAxB,EAAA,CAAAwB,MAAA,oBAAAN,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA2B,MAAA,SAAA3B,CAAA,qBAAAA,CAAA,sBAAA4B,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA,OAAAf,CAAA,GAAAiB,MAAA,CAAAC,SAAA,CAAAC,QAAA,CAAAT,IAAA,CAAAtB,CAAA,EAAAgC,KAAA,aAAApB,CAAA,iBAAAZ,CAAA,CAAAiC,WAAA,EAAArB,CAAA,GAAAZ,CAAA,CAAAiC,WAAA,CAAAC,IAAA,MAAAtB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA8B,IAAA,CAAAnC,CAAA,OAAAY,CAAA,+DAAAwB,IAAA,CAAAxB,CAAA,UAAAgB,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA;AAAA,SAAAC,kBAAAS,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA7B,MAAA,EAAA8B,GAAA,GAAAD,GAAA,CAAA7B,MAAA,WAAAC,CAAA,MAAA8B,IAAA,OAAAlC,KAAA,CAAAiC,GAAA,GAAA7B,CAAA,GAAA6B,GAAA,EAAA7B,CAAA,IAAA8B,IAAA,CAAA9B,CAAA,IAAA4B,GAAA,CAAA5B,CAAA,UAAA8B,IAAA;AAAA,SAAAC,QAAAC,MAAA,EAAAC,cAAA,QAAAC,IAAA,GAAAd,MAAA,CAAAc,IAAA,CAAAF,MAAA,OAAAZ,MAAA,CAAAe,qBAAA,QAAAC,OAAA,GAAAhB,MAAA,CAAAe,qBAAA,CAAAH,MAAA,GAAAC,cAAA,KAAAG,OAAA,GAAAA,OAAA,CAAAC,MAAA,WAAAC,GAAA,WAAAlB,MAAA,CAAAmB,wBAAA,CAAAP,MAAA,EAAAM,GAAA,EAAAE,UAAA,OAAAN,IAAA,CAAAO,IAAA,CAAAC,KAAA,CAAAR,IAAA,EAAAE,OAAA,YAAAF,IAAA;AAAA,SAAAS,cAAAC,MAAA,aAAA5C,CAAA,MAAAA,CAAA,GAAA6C,SAAA,CAAA9C,MAAA,EAAAC,CAAA,UAAA8C,MAAA,WAAAD,SAAA,CAAA7C,CAAA,IAAA6C,SAAA,CAAA7C,CAAA,QAAAA,CAAA,OAAA+B,OAAA,CAAAX,MAAA,CAAA0B,MAAA,OAAAC,OAAA,WAAAC,GAAA,QAAAC,gBAAA,CAAAC,OAAA,EAAAN,MAAA,EAAAI,GAAA,EAAAF,MAAA,CAAAE,GAAA,SAAA5B,MAAA,CAAA+B,yBAAA,GAAA/B,MAAA,CAAAgC,gBAAA,CAAAR,MAAA,EAAAxB,MAAA,CAAA+B,yBAAA,CAAAL,MAAA,KAAAf,OAAA,CAAAX,MAAA,CAAA0B,MAAA,GAAAC,OAAA,WAAAC,GAAA,IAAA5B,MAAA,CAAAiC,cAAA,CAAAT,MAAA,EAAAI,GAAA,EAAA5B,MAAA,CAAAmB,wBAAA,CAAAO,MAAA,EAAAE,GAAA,iBAAAJ,MAAA;AAAA,SAAAU,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAjE,MAAA,KAAA+D,KAAA,GAAA/D,MAAA,CAAAkE,aAAA,EAAAF,IAAA,GAAAhE,MAAA,CAAAC,QAAA,GAAAgE,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAA3C,IAAA,CAAA0C,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAG,qBAAA,CAAAL,MAAA,CAAA3C,IAAA,CAAA0C,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAjD,SAAA;AAAA,SAAAoD,sBAAA3D,CAAA,aAAA4D,kCAAAC,CAAA,QAAA3C,MAAA,CAAA2C,CAAA,MAAAA,CAAA,SAAAC,OAAA,CAAAC,MAAA,KAAAxD,SAAA,CAAAsD,CAAA,+BAAA3D,IAAA,GAAA2D,CAAA,CAAA3D,IAAA,SAAA4D,OAAA,CAAAE,OAAA,CAAAH,CAAA,CAAA1D,KAAA,EAAA8D,IAAA,WAAA9D,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAD,IAAA,EAAAA,IAAA,iBAAAyD,qBAAA,YAAAA,sBAAA3D,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAC,CAAA,GAAAD,CAAA,CAAAa,IAAA,KAAA8C,qBAAA,CAAAxC,SAAA,KAAAnB,CAAA,QAAAC,CAAA,QAAAY,IAAA,WAAAA,KAAA,WAAA+C,iCAAA,MAAA3D,CAAA,CAAAuC,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,OAAA5B,MAAA,WAAAmD,QAAA/D,KAAA,QAAAgE,GAAA,QAAAnE,CAAA,CAAAe,MAAA,oBAAAoD,GAAA,GAAAL,OAAA,CAAAE,OAAA,GAAA7D,KAAA,EAAAA,KAAA,EAAAD,IAAA,UAAA0D,iCAAA,CAAAO,GAAA,CAAA3B,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,OAAAyB,KAAA,WAAAC,OAAAlE,KAAA,QAAAmE,GAAA,QAAAtE,CAAA,CAAAe,MAAA,oBAAAuD,GAAA,GAAAR,OAAA,CAAAC,MAAA,CAAA5D,KAAA,IAAAyD,iCAAA,CAAAU,GAAA,CAAA9B,KAAA,MAAAxC,CAAA,EAAA2C,SAAA,aAAAgB,qBAAA,CAAA3D,CAAA;AAAA,SAIZuE,kBAAkBA,CAAAC,EAAA,EAAAC,GAAA;EAAA,OAAAC,mBAAA,CAAAlC,KAAA,OAAAG,SAAA;AAAA;AAAA,SAAA+B,oBAAA;EAAAA,mBAAA,OAAAC,oBAAA,CAAA3B,OAAA,EAAA4B,YAAA,CAAA5B,OAAA,CAAA6B,IAAA,CAAlC,SAAAC,QACbC,mBAAuE,EACvEC,OAA0B;IAAA,IAAAC,aAAA;IAAA,IAAAvB,aAAA,EAAAwB,QAAA,EAAAC,IAAA,EAAAC,SAAA,EAAAC,YAAA,EAAAC,MAAA,EAAAC,KAAA,EAAAC,iBAAA,EAAAC,MAAA,EAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,SAAA,EAAAC,YAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,GAAA,EAAAC,OAAA,EAAAC,MAAA,EAAAC,QAAA,EAAAC,KAAA,EAAAC,UAAA;IAAA,OAAA9B,YAAA,CAAA5B,OAAA,CAAA2D,IAAA,UAAAC,SAAAC,QAAA;MAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAhG,IAAA;QAAA;UAEpB6C,aAAa,GAAG,IAAAqD,oCAAuB,EAAChC,mBAAmB,CAAC;UAE3DG,QAAQ,GAAIF,OAAO,CAAnBE,QAAQ;UAAAC,IAAA,GACKH,OAAO,CAACgC,IAAI,IAAI,CAAC,CAAC,EAA/B5B,SAAS,GAAAD,IAAA,CAATC,SAAS;UAEZC,YAAqB,GAAG,IAAI;UAG1BC,MAAM,GAAG,IAAI;UACbC,KAAK,GAAG,CAAAP,OAAO,aAAPA,OAAO,wBAAAC,aAAA,GAAPD,OAAO,CAAEgC,IAAI,cAAA/B,aAAA,uBAAbA,aAAA,CAAeM,KAAK,KAAI,WAAW;UAE3CC,iBAAiB,GAAG,IAAIyB,yBAAiB,CAAC3B,MAAM,EAAA7C,aAAA,CAAAA,aAAA,KACjDuC,OAAO;YACVO,KAAK,EAALA;UAAK,EACN,CAAC;UAEIE,MAAM,GAAG,IAAIyB,4BAAmB,CAAC;YAAC9B,SAAS,EAATA;UAAS,CAAC,CAAC;UAAAM,yBAAA;UAAAC,iBAAA;UAAAkB,QAAA,CAAAC,IAAA;UAAAjB,SAAA,GAAAzC,cAAA,CAEzBM,aAAa;QAAA;UAAAmD,QAAA,CAAAhG,IAAA;UAAA,WAAAsG,qBAAA,CAAAnE,OAAA,EAAA6C,SAAA,CAAAhF,IAAA;QAAA;UAAA,MAAA6E,yBAAA,KAAAI,KAAA,GAAAe,QAAA,CAAAO,IAAA,EAAAlH,IAAA;YAAA2G,QAAA,CAAAhG,IAAA;YAAA;UAAA;UAAtBkF,KAAK,GAAAD,KAAA,CAAA3F,KAAA;UACd6F,IAAI,GAAGP,MAAM,CAAC4B,KAAK,CAACtB,KAAK,CAAC;UAE1BS,SAAQ,GAAGR,IAAI,CAACnG,MAAM,GAAG,CAAC,IAAI4F,MAAM,CAAC6B,4BAA4B,CAAC,CAAC;UAAA,MAErEtB,IAAI,CAACnG,MAAM,GAAG,CAAC,IAAIwF,YAAY;YAAAwB,QAAA,CAAAhG,IAAA;YAAA;UAAA;UAAA,KAC7BqE,QAAQ;YAAA2B,QAAA,CAAAhG,IAAA;YAAA;UAAA;UACJqF,YAAmB,GAAG;YAE1BX,KAAK,EAALA,KAAK;YACLgC,SAAS,EAAE,gBAAgB;YAC3BC,IAAI,EAAE,EAAE;YACR3H,MAAM,EAAE,CAAC;YACT4H,SAAS,EAAE,CAAC;YAEZC,SAAS,EAAEjC,MAAM,CAACkC,gBAAgB,CAAC,CAAC;YACpCnB,QAAQ,EAARA;UACF,CAAC;UAAAK,QAAA,CAAAhG,IAAA;UACD,OAAMqF,YAAY;QAAA;UAEpBb,YAAY,GAAG,KAAK;QAAC;UAAAc,UAAA,GAAA/G,0BAAA,CAKL4G,IAAI;UAAAa,QAAA,CAAAC,IAAA;UAAAX,UAAA,CAAAnG,CAAA;QAAA;UAAA,KAAAoG,MAAA,GAAAD,UAAA,CAAAlG,CAAA,IAAAC,IAAA;YAAA2G,QAAA,CAAAhG,IAAA;YAAA;UAAA;UAAXwF,GAAG,GAAAD,MAAA,CAAAjG,KAAA;UACZqF,iBAAiB,CAACoC,MAAM,CAACvB,GAAG,CAAC;UAEvBI,OAAK,GAAGjB,iBAAiB,CAACqC,YAAY,CAAC;YAACrB,QAAQ,EAARA;UAAQ,CAAC,CAAC;UAAA,KACpDC,OAAK;YAAAI,QAAA,CAAAhG,IAAA;YAAA;UAAA;UAAAgG,QAAA,CAAAhG,IAAA;UACP,OAAM4F,OAAK;QAAA;UAAAI,QAAA,CAAAhG,IAAA;UAAA;QAAA;UAAAgG,QAAA,CAAAhG,IAAA;UAAA;QAAA;UAAAgG,QAAA,CAAAC,IAAA;UAAAD,QAAA,CAAAiB,EAAA,GAAAjB,QAAA;UAAAV,UAAA,CAAA/F,CAAA,CAAAyG,QAAA,CAAAiB,EAAA;QAAA;UAAAjB,QAAA,CAAAC,IAAA;UAAAX,UAAA,CAAA7F,CAAA;UAAA,OAAAuG,QAAA,CAAAkB,MAAA;QAAA;UAIfvC,iBAAiB,CAACwC,aAAa,CAACjC,KAAK,CAAC;UAChCU,MAAK,GAAGjB,iBAAiB,CAACqC,YAAY,CAAC;YAACrB,QAAQ,EAARA;UAAQ,CAAC,CAAC;UAAA,KACpDC,MAAK;YAAAI,QAAA,CAAAhG,IAAA;YAAA;UAAA;UAAAgG,QAAA,CAAAhG,IAAA;UACP,OAAM4F,MAAK;QAAA;UAAAf,yBAAA;UAAAmB,QAAA,CAAAhG,IAAA;UAAA;QAAA;UAAAgG,QAAA,CAAAhG,IAAA;UAAA;QAAA;UAAAgG,QAAA,CAAAC,IAAA;UAAAD,QAAA,CAAAoB,EAAA,GAAApB,QAAA;UAAAlB,iBAAA;UAAAC,cAAA,GAAAiB,QAAA,CAAAoB,EAAA;QAAA;UAAApB,QAAA,CAAAC,IAAA;UAAAD,QAAA,CAAAC,IAAA;UAAA,MAAApB,yBAAA,IAAAG,SAAA,CAAA9E,MAAA;YAAA8F,QAAA,CAAAhG,IAAA;YAAA;UAAA;UAAAgG,QAAA,CAAAhG,IAAA;UAAA,WAAAsG,qBAAA,CAAAnE,OAAA,EAAA6C,SAAA,CAAA9E,MAAA;QAAA;UAAA8F,QAAA,CAAAC,IAAA;UAAA,KAAAnB,iBAAA;YAAAkB,QAAA,CAAAhG,IAAA;YAAA;UAAA;UAAA,MAAA+E,cAAA;QAAA;UAAA,OAAAiB,QAAA,CAAAkB,MAAA;QAAA;UAAA,OAAAlB,QAAA,CAAAkB,MAAA;QAAA;UAKTvB,QAAQ,GAAGf,MAAM,CAAC6B,4BAA4B,CAAC,CAAC;UAChDb,KAAK,GAAGjB,iBAAiB,CAAC0C,aAAa,CAAC;YAAC1B,QAAQ,EAARA;UAAQ,CAAC,CAAC;UAAA,KACrDC,KAAK;YAAAI,QAAA,CAAAhG,IAAA;YAAA;UAAA;UAAAgG,QAAA,CAAAhG,IAAA;UACP,OAAM4F,KAAK;QAAA;UAAA,KAGTvB,QAAQ;YAAA2B,QAAA,CAAAhG,IAAA;YAAA;UAAA;UACJ6F,UAAiB,GAAG;YACxBnB,KAAK,EAALA,KAAK;YACLgC,SAAS,EAAE,cAAc;YACzBG,SAAS,EAAEjC,MAAM,CAACkC,gBAAgB,CAAC,CAAC;YACpCnB,QAAQ,EAAEf,MAAM,CAAC6B,4BAA4B,CAAC,CAAC;YAC/CE,IAAI,EAAE,EAAE;YACR3H,MAAM,EAAE;UAEV,CAAC;UAAAgH,QAAA,CAAAhG,IAAA;UACD,OAAM6F,UAAU;QAAA;QAAA;UAAA,OAAAG,QAAA,CAAAsB,IAAA;MAAA;IAAA,GAAArD,OAAA;EAAA,CAEnB;EAAA,OAAAJ,mBAAA,CAAAlC,KAAA,OAAAG,SAAA;AAAA;AAEM,SAASyF,iBAAiBA,CAAC3B,KAAK,EAAEe,IAAI,EAAE;EAE7C,IAAAa,mBAAM,EAAC5B,KAAK,CAACc,SAAS,KAAK,cAAc,CAAC;EAG1C,IAAId,KAAK,CAACD,QAAQ,KAAK,GAAG,EAAE;IAC1B,OAAOgB,IAAI;EACb;EAGA,IAAIf,KAAK,CAACD,QAAQ,IAAIC,KAAK,CAACD,QAAQ,CAAC3G,MAAM,GAAG,CAAC,EAAE;IAC/C,IAAMyI,cAAc,GAAG7B,KAAK,CAACiB,SAAS;IACtC,IAAMa,aAAa,GAAG,IAAIC,kBAAQ,CAAC/B,KAAK,CAACD,QAAQ,CAAC;IAClD+B,aAAa,CAACE,cAAc,CAACH,cAAc,EAAEd,IAAI,CAAC;IAClD,OAAOc,cAAc;EACvB;EAGA,OAAO7B,KAAK,CAACiB,SAAS;AACxB"}
+{"version":3,"file":"parse-json-in-batches.js","names":["_schema","require","_loaderUtils","_streamingJsonParser","_interopRequireDefault","_jsonpath","parseJSONInBatches","binaryAsyncIterator","options","_options$json","asyncIterator","makeTextDecoderIterator","metadata","jsonpaths","json","isFirstChunk","schema","shape","tableBatchBuilder","TableBatchBuilder","parser","StreamingJSONParser","chunk","rows","write","jsonpath","length","getStreamingJsonPathAsString","initialBatch","batchType","data","bytesUsed","container","getPartialResult","row","addRow","batch","getFullBatch","chunkComplete","getFinalBatch","finalBatch","rebuildJsonObject","assert","topLevelObject","streamingPath","JSONPath","setFieldAtPath"],"sources":["../../../src/lib/parse-json-in-batches.ts"],"sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {assert, makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\nimport JSONPath from './jsonpath/jsonpath';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n  const {metadata} = options;\n  const {jsonpaths} = options.json || {};\n\n  let isFirstChunk: boolean = true;\n\n  // TODO fix Schema deduction\n  const schema = null; // new Schema([]);\n  const shape = options?.json?.shape || 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  const parser = new StreamingJSONParser({jsonpaths});\n\n  for await (const chunk of asyncIterator) {\n    const rows = parser.write(chunk);\n\n    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n    if (rows.length > 0 && isFirstChunk) {\n      if (metadata) {\n        const initialBatch: Batch = {\n          // Common fields\n          shape,\n          batchType: 'partial-result',\n          data: [],\n          length: 0,\n          bytesUsed: 0,\n          // JSON additions\n          container: parser.getPartialResult(),\n          jsonpath\n        };\n        yield initialBatch;\n      }\n      isFirstChunk = false;\n      // schema = deduceSchema(rows);\n    }\n\n    // Add the row\n    for (const row of rows) {\n      tableBatchBuilder.addRow(row);\n      // If a batch has been completed, emit it\n      const batch = tableBatchBuilder.getFullBatch({jsonpath});\n      if (batch) {\n        yield batch;\n      }\n    }\n\n    tableBatchBuilder.chunkComplete(chunk);\n    const batch = tableBatchBuilder.getFullBatch({jsonpath});\n    if (batch) {\n      yield batch;\n    }\n  }\n\n  // yield final batch\n  const jsonpath = parser.getStreamingJsonPathAsString();\n  const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n  if (batch) {\n    yield batch;\n  }\n\n  if (metadata) {\n    const finalBatch: Batch = {\n      shape,\n      batchType: 'final-result',\n      container: parser.getPartialResult(),\n      jsonpath: parser.getStreamingJsonPathAsString(),\n      data: [],\n      length: 0\n      // schema: null\n    };\n    yield finalBatch;\n  }\n}\n\nexport function rebuildJsonObject(batch, data) {\n  // Last batch will have this special type and will provide all the root object of the parsed file\n  assert(batch.batchType === 'final-result');\n\n  // The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects\n  if (batch.jsonpath === '$') {\n    return data;\n  }\n\n  // (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object\n  if (batch.jsonpath && batch.jsonpath.length > 1) {\n    const topLevelObject = batch.container;\n    const streamingPath = new JSONPath(batch.jsonpath);\n    streamingPath.setFieldAtPath(topLevelObject, data);\n    return topLevelObject;\n  }\n\n  // No jsonpath, in this case nothing was streamed.\n  return batch.container;\n}\n"],"mappings":";;;;;;;;AAEA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AACA,IAAAE,oBAAA,GAAAC,sBAAA,CAAAH,OAAA;AACA,IAAAI,SAAA,GAAAD,sBAAA,CAAAH,OAAA;AAIe,gBAAgBK,kBAAkBA,CAC/CC,mBAAuE,EACvEC,OAA0B,EACJ;EAAA,IAAAC,aAAA;EACtB,MAAMC,aAAa,GAAG,IAAAC,oCAAuB,EAACJ,mBAAmB,CAAC;EAElE,MAAM;IAACK;EAAQ,CAAC,GAAGJ,OAAO;EAC1B,MAAM;IAACK;EAAS,CAAC,GAAGL,OAAO,CAACM,IAAI,IAAI,CAAC,CAAC;EAEtC,IAAIC,YAAqB,GAAG,IAAI;EAGhC,MAAMC,MAAM,GAAG,IAAI;EACnB,MAAMC,KAAK,GAAG,CAAAT,OAAO,aAAPA,OAAO,wBAAAC,aAAA,GAAPD,OAAO,CAAEM,IAAI,cAAAL,aAAA,uBAAbA,aAAA,CAAeQ,KAAK,KAAI,WAAW;EAEjD,MAAMC,iBAAiB,GAAG,IAAIC,yBAAiB,CAACH,MAAM,EAAE;IACtD,GAAGR,OAAO;IACVS;EACF,CAAC,CAAC;EAEF,MAAMG,MAAM,GAAG,IAAIC,4BAAmB,CAAC;IAACR;EAAS,CAAC,CAAC;EAEnD,WAAW,MAAMS,KAAK,IAAIZ,aAAa,EAAE;IACvC,MAAMa,IAAI,GAAGH,MAAM,CAACI,KAAK,CAACF,KAAK,CAAC;IAEhC,MAAMG,QAAQ,GAAGF,IAAI,CAACG,MAAM,GAAG,CAAC,IAAIN,MAAM,CAACO,4BAA4B,CAAC,CAAC;IAEzE,IAAIJ,IAAI,CAACG,MAAM,GAAG,CAAC,IAAIX,YAAY,EAAE;MACnC,IAAIH,QAAQ,EAAE;QACZ,MAAMgB,YAAmB,GAAG;UAE1BX,KAAK;UACLY,SAAS,EAAE,gBAAgB;UAC3BC,IAAI,EAAE,EAAE;UACRJ,MAAM,EAAE,CAAC;UACTK,SAAS,EAAE,CAAC;UAEZC,SAAS,EAAEZ,MAAM,CAACa,gBAAgB,CAAC,CAAC;UACpCR;QACF,CAAC;QACD,MAAMG,YAAY;MACpB;MACAb,YAAY,GAAG,KAAK;IAEtB;IAGA,KAAK,MAAMmB,GAAG,IAAIX,IAAI,EAAE;MACtBL,iBAAiB,CAACiB,MAAM,CAACD,GAAG,CAAC;MAE7B,MAAME,KAAK,GAAGlB,iBAAiB,CAACmB,YAAY,CAAC;QAACZ;MAAQ,CAAC,CAAC;MACxD,IAAIW,KAAK,EAAE;QACT,MAAMA,KAAK;MACb;IACF;IAEAlB,iBAAiB,CAACoB,aAAa,CAAChB,KAAK,CAAC;IACtC,MAAMc,KAAK,GAAGlB,iBAAiB,CAACmB,YAAY,CAAC;MAACZ;IAAQ,CAAC,CAAC;IACxD,IAAIW,KAAK,EAAE;MACT,MAAMA,KAAK;IACb;EACF;EAGA,MAAMX,QAAQ,GAAGL,MAAM,CAACO,4BAA4B,CAAC,CAAC;EACtD,MAAMS,KAAK,GAAGlB,iBAAiB,CAACqB,aAAa,CAAC;IAACd;EAAQ,CAAC,CAAC;EACzD,IAAIW,KAAK,EAAE;IACT,MAAMA,KAAK;EACb;EAEA,IAAIxB,QAAQ,EAAE;IACZ,MAAM4B,UAAiB,GAAG;MACxBvB,KAAK;MACLY,SAAS,EAAE,cAAc;MACzBG,SAAS,EAAEZ,MAAM,CAACa,gBAAgB,CAAC,CAAC;MACpCR,QAAQ,EAAEL,MAAM,CAACO,4BAA4B,CAAC,CAAC;MAC/CG,IAAI,EAAE,EAAE;MACRJ,MAAM,EAAE;IAEV,CAAC;IACD,MAAMc,UAAU;EAClB;AACF;AAEO,SAASC,iBAAiBA,CAACL,KAAK,EAAEN,IAAI,EAAE;EAE7C,IAAAY,mBAAM,EAACN,KAAK,CAACP,SAAS,KAAK,cAAc,CAAC;EAG1C,IAAIO,KAAK,CAACX,QAAQ,KAAK,GAAG,EAAE;IAC1B,OAAOK,IAAI;EACb;EAGA,IAAIM,KAAK,CAACX,QAAQ,IAAIW,KAAK,CAACX,QAAQ,CAACC,MAAM,GAAG,CAAC,EAAE;IAC/C,MAAMiB,cAAc,GAAGP,KAAK,CAACJ,SAAS;IACtC,MAAMY,aAAa,GAAG,IAAIC,iBAAQ,CAACT,KAAK,CAACX,QAAQ,CAAC;IAClDmB,aAAa,CAACE,cAAc,CAACH,cAAc,EAAEb,IAAI,CAAC;IAClD,OAAOa,cAAc;EACvB;EAGA,OAAOP,KAAK,CAACJ,SAAS;AACxB"}

package/dist/es5/lib/parse-json.js

@@ -1,15 +1,13 @@
 "use strict";
 
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = parseJSONSync;
-var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
 function parseJSONSync(jsonText, options) {
   try {
     var _options$json;
-    var json = JSON.parse(jsonText);
+    const json = JSON.parse(jsonText);
     if ((_options$json = options.json) !== null && _options$json !== void 0 && _options$json.table) {
       return getFirstArray(json) || json;
     }
@@ -22,10 +20,9 @@ function getFirstArray(json) {
   if (Array.isArray(json)) {
     return json;
   }
-  if (json && (0, _typeof2.default)(json) === 'object') {
-    for (var _i = 0, _Object$values = Object.values(json); _i < _Object$values.length; _i++) {
-      var value = _Object$values[_i];
-      var array = getFirstArray(value);
+  if (json && typeof json === 'object') {
+    for (const value of Object.values(json)) {
+      const array = getFirstArray(value);
       if (array) {
         return array;
       }

package/dist/es5/lib/parse-json.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-json.js","names":["parseJSONSync","jsonText","options","_options$json","json","JSON","parse","table","getFirstArray","error","Error","Array","isArray","_typeof2","default","_i","_Object$values","Object","values","length","value","array"],"sources":["../../../src/lib/parse-json.ts"],"sourcesContent":["import type {JSONLoaderOptions} from '../json-loader';\n\nexport default function parseJSONSync(jsonText: string, options: JSONLoaderOptions) {\n  try {\n    const json = JSON.parse(jsonText);\n    if (options.json?.table) {\n      return getFirstArray(json) || json;\n    }\n    return json;\n  } catch (error) {\n    throw new Error('JSONLoader: failed to parse JSON');\n  }\n}\n\nfunction getFirstArray(json) {\n  if (Array.isArray(json)) {\n    return json;\n  }\n  if (json && typeof json === 'object') {\n    for (const value of Object.values(json)) {\n      const array = getFirstArray(value);\n      if (array) {\n        return array;\n      }\n    }\n  }\n  return null;\n}\n"],"mappings":";;;;;;;;AAEe,SAASA,aAAaA,CAACC,QAAgB,EAAEC,OAA0B,EAAE;EAClF,IAAI;IAAA,IAAAC,aAAA;IACF,IAAMC,IAAI,GAAGC,IAAI,CAACC,KAAK,CAACL,QAAQ,CAAC;IACjC,KAAAE,aAAA,GAAID,OAAO,CAACE,IAAI,cAAAD,aAAA,eAAZA,aAAA,CAAcI,KAAK,EAAE;MACvB,OAAOC,aAAa,CAACJ,IAAI,CAAC,IAAIA,IAAI;IACpC;IACA,OAAOA,IAAI;EACb,CAAC,CAAC,OAAOK,KAAK,EAAE;IACd,MAAM,IAAIC,KAAK,CAAC,kCAAkC,CAAC;EACrD;AACF;AAEA,SAASF,aAAaA,CAACJ,IAAI,EAAE;EAC3B,IAAIO,KAAK,CAACC,OAAO,CAACR,IAAI,CAAC,EAAE;IACvB,OAAOA,IAAI;EACb;EACA,IAAIA,IAAI,IAAI,IAAAS,QAAA,CAAAC,OAAA,EAAOV,IAAI,MAAK,QAAQ,EAAE;IACpC,SAAAW,EAAA,MAAAC,cAAA,GAAoBC,MAAM,CAACC,MAAM,CAACd,IAAI,CAAC,EAAAW,EAAA,GAAAC,cAAA,CAAAG,MAAA,EAAAJ,EAAA,IAAE;MAApC,IAAMK,KAAK,GAAAJ,cAAA,CAAAD,EAAA;MACd,IAAMM,KAAK,GAAGb,aAAa,CAACY,KAAK,CAAC;MAClC,IAAIC,KAAK,EAAE;QACT,OAAOA,KAAK;MACd;IACF;EACF;EACA,OAAO,IAAI;AACb"}
+{"version":3,"file":"parse-json.js","names":["parseJSONSync","jsonText","options","_options$json","json","JSON","parse","table","getFirstArray","error","Error","Array","isArray","value","Object","values","array"],"sources":["../../../src/lib/parse-json.ts"],"sourcesContent":["import type {JSONLoaderOptions} from '../json-loader';\n\nexport default function parseJSONSync(jsonText: string, options: JSONLoaderOptions) {\n  try {\n    const json = JSON.parse(jsonText);\n    if (options.json?.table) {\n      return getFirstArray(json) || json;\n    }\n    return json;\n  } catch (error) {\n    throw new Error('JSONLoader: failed to parse JSON');\n  }\n}\n\nfunction getFirstArray(json) {\n  if (Array.isArray(json)) {\n    return json;\n  }\n  if (json && typeof json === 'object') {\n    for (const value of Object.values(json)) {\n      const array = getFirstArray(value);\n      if (array) {\n        return array;\n      }\n    }\n  }\n  return null;\n}\n"],"mappings":";;;;;;AAEe,SAASA,aAAaA,CAACC,QAAgB,EAAEC,OAA0B,EAAE;EAClF,IAAI;IAAA,IAAAC,aAAA;IACF,MAAMC,IAAI,GAAGC,IAAI,CAACC,KAAK,CAACL,QAAQ,CAAC;IACjC,KAAAE,aAAA,GAAID,OAAO,CAACE,IAAI,cAAAD,aAAA,eAAZA,aAAA,CAAcI,KAAK,EAAE;MACvB,OAAOC,aAAa,CAACJ,IAAI,CAAC,IAAIA,IAAI;IACpC;IACA,OAAOA,IAAI;EACb,CAAC,CAAC,OAAOK,KAAK,EAAE;IACd,MAAM,IAAIC,KAAK,CAAC,kCAAkC,CAAC;EACrD;AACF;AAEA,SAASF,aAAaA,CAACJ,IAAI,EAAE;EAC3B,IAAIO,KAAK,CAACC,OAAO,CAACR,IAAI,CAAC,EAAE;IACvB,OAAOA,IAAI;EACb;EACA,IAAIA,IAAI,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;IACpC,KAAK,MAAMS,KAAK,IAAIC,MAAM,CAACC,MAAM,CAACX,IAAI,CAAC,EAAE;MACvC,MAAMY,KAAK,GAAGR,aAAa,CAACK,KAAK,CAAC;MAClC,IAAIG,KAAK,EAAE;QACT,OAAOA,KAAK;MACd;IACF;EACF;EACA,OAAO,IAAI;AACb"}

package/dist/es5/lib/parse-ndjson-in-batches.js

@@ -1,114 +1,40 @@
 "use strict";
 
-var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.default = parseNDJSONInBatches;
-var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
-var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
-var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
-var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
 var _schema = require("@loaders.gl/schema");
 var _loaderUtils = require("@loaders.gl/loader-utils");
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
-function _asyncIterator(iterable) { var method, async, sync, retry = 2; for ("undefined" != typeof Symbol && (async = Symbol.asyncIterator, sync = Symbol.iterator); retry--;) { if (async && null != (method = iterable[async])) return method.call(iterable); if (sync && null != (method = iterable[sync])) return new AsyncFromSyncIterator(method.call(iterable)); async = "@@asyncIterator", sync = "@@iterator"; } throw new TypeError("Object is not async iterable"); }
-function AsyncFromSyncIterator(s) { function AsyncFromSyncIteratorContinuation(r) { if (Object(r) !== r) return Promise.reject(new TypeError(r + " is not an object.")); var done = r.done; return Promise.resolve(r.value).then(function (value) { return { value: value, done: done }; }); } return AsyncFromSyncIterator = function AsyncFromSyncIterator(s) { this.s = s, this.n = s.next; }, AsyncFromSyncIterator.prototype = { s: null, n: null, next: function next() { return AsyncFromSyncIteratorContinuation(this.n.apply(this.s, arguments)); }, return: function _return(value) { var ret = this.s.return; return void 0 === ret ? Promise.resolve({ value: value, done: !0 }) : AsyncFromSyncIteratorContinuation(ret.apply(this.s, arguments)); }, throw: function _throw(value) { var thr = this.s.return; return void 0 === thr ? Promise.reject(value) : AsyncFromSyncIteratorContinuation(thr.apply(this.s, arguments)); } }, new AsyncFromSyncIterator(s); }
-function parseNDJSONInBatches(_x, _x2) {
-  return _parseNDJSONInBatches.apply(this, arguments);
-}
-function _parseNDJSONInBatches() {
-  _parseNDJSONInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(binaryAsyncIterator, options) {
-    var textIterator, lineIterator, numberedLineIterator, schema, shape, tableBatchBuilder, _iteratorAbruptCompletion, _didIteratorError, _iteratorError, _iterator, _step, _step$value, counter, line, row, _batch, batch;
-    return _regenerator.default.wrap(function _callee$(_context) {
-      while (1) switch (_context.prev = _context.next) {
-        case 0:
-          textIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
-          lineIterator = (0, _loaderUtils.makeLineIterator)(textIterator);
-          numberedLineIterator = (0, _loaderUtils.makeNumberedLineIterator)(lineIterator);
-          schema = null;
-          shape = 'row-table';
-          tableBatchBuilder = new _schema.TableBatchBuilder(schema, _objectSpread(_objectSpread({}, options), {}, {
-            shape: shape
-          }));
-          _iteratorAbruptCompletion = false;
-          _didIteratorError = false;
-          _context.prev = 8;
-          _iterator = _asyncIterator(numberedLineIterator);
-        case 10:
-          _context.next = 12;
-          return (0, _awaitAsyncGenerator2.default)(_iterator.next());
-        case 12:
-          if (!(_iteratorAbruptCompletion = !(_step = _context.sent).done)) {
-            _context.next = 30;
-            break;
-          }
-          _step$value = _step.value, counter = _step$value.counter, line = _step$value.line;
-          _context.prev = 14;
-          row = JSON.parse(line);
-          tableBatchBuilder.addRow(row);
-          tableBatchBuilder.chunkComplete(line);
-          _batch = tableBatchBuilder.getFullBatch();
-          if (!_batch) {
-            _context.next = 22;
-            break;
-          }
-          _context.next = 22;
-          return _batch;
-        case 22:
-          _context.next = 27;
-          break;
-        case 24:
-          _context.prev = 24;
-          _context.t0 = _context["catch"](14);
-          throw new Error("NDJSONLoader: failed to parse JSON on line ".concat(counter));
-        case 27:
-          _iteratorAbruptCompletion = false;
-          _context.next = 10;
-          break;
-        case 30:
-          _context.next = 36;
-          break;
-        case 32:
-          _context.prev = 32;
-          _context.t1 = _context["catch"](8);
-          _didIteratorError = true;
-          _iteratorError = _context.t1;
-        case 36:
-          _context.prev = 36;
-          _context.prev = 37;
-          if (!(_iteratorAbruptCompletion && _iterator.return != null)) {
-            _context.next = 41;
-            break;
-          }
-          _context.next = 41;
-          return (0, _awaitAsyncGenerator2.default)(_iterator.return());
-        case 41:
-          _context.prev = 41;
-          if (!_didIteratorError) {
-            _context.next = 44;
-            break;
-          }
-          throw _iteratorError;
-        case 44:
-          return _context.finish(41);
-        case 45:
-          return _context.finish(36);
-        case 46:
-          batch = tableBatchBuilder.getFinalBatch();
-          if (!batch) {
-            _context.next = 50;
-            break;
-          }
-          _context.next = 50;
-          return batch;
-        case 50:
-        case "end":
-          return _context.stop();
+async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
+  const textIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
+  const lineIterator = (0, _loaderUtils.makeLineIterator)(textIterator);
+  const numberedLineIterator = (0, _loaderUtils.makeNumberedLineIterator)(lineIterator);
+  const schema = null;
+  const shape = 'row-table';
+  const tableBatchBuilder = new _schema.TableBatchBuilder(schema, {
+    ...options,
+    shape
+  });
+  for await (const {
+    counter,
+    line
+  } of numberedLineIterator) {
+    try {
+      const row = JSON.parse(line);
+      tableBatchBuilder.addRow(row);
+      tableBatchBuilder.chunkComplete(line);
+      const batch = tableBatchBuilder.getFullBatch();
+      if (batch) {
+        yield batch;
       }
-    }, _callee, null, [[8, 32, 36, 46], [14, 24], [37,, 41, 45]]);
-  }));
-  return _parseNDJSONInBatches.apply(this, arguments);
+    } catch (error) {
+      throw new Error("NDJSONLoader: failed to parse JSON on line ".concat(counter));
+    }
+  }
+  const batch = tableBatchBuilder.getFinalBatch();
+  if (batch) {
+    yield batch;
+  }
 }
 //# sourceMappingURL=parse-ndjson-in-batches.js.map

package/dist/es5/lib/parse-ndjson-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-ndjson-in-batches.js","names":["_schema","require","_loaderUtils","ownKeys","object","enumerableOnly","keys","Object","getOwnPropertySymbols","symbols","filter","sym","getOwnPropertyDescriptor","enumerable","push","apply","_objectSpread","target","i","arguments","length","source","forEach","key","_defineProperty2","default","getOwnPropertyDescriptors","defineProperties","defineProperty","_asyncIterator","iterable","method","async","sync","retry","Symbol","asyncIterator","iterator","call","AsyncFromSyncIterator","TypeError","s","AsyncFromSyncIteratorContinuation","r","Promise","reject","done","resolve","value","then","n","next","prototype","return","_return","ret","throw","_throw","thr","parseNDJSONInBatches","_x","_x2","_parseNDJSONInBatches","_wrapAsyncGenerator2","_regenerator","mark","_callee","binaryAsyncIterator","options","textIterator","lineIterator","numberedLineIterator","schema","shape","tableBatchBuilder","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","_step$value","counter","line","row","_batch","batch","wrap","_callee$","_context","prev","makeTextDecoderIterator","makeLineIterator","makeNumberedLineIterator","TableBatchBuilder","_awaitAsyncGenerator2","sent","JSON","parse","addRow","chunkComplete","getFullBatch","t0","Error","concat","t1","finish","getFinalBatch","stop"],"sources":["../../../src/lib/parse-ndjson-in-batches.ts"],"sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {\n  LoaderOptions,\n  makeLineIterator,\n  makeNumberedLineIterator,\n  makeTextDecoderIterator\n} from '@loaders.gl/loader-utils';\n\nexport default async function* parseNDJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: LoaderOptions\n): AsyncIterable<Batch> {\n  const textIterator = makeTextDecoderIterator(binaryAsyncIterator);\n  const lineIterator = makeLineIterator(textIterator);\n  const numberedLineIterator = makeNumberedLineIterator(lineIterator);\n\n  const schema = null;\n  const shape = 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  for await (const {counter, line} of numberedLineIterator) {\n    try {\n      const row = JSON.parse(line);\n      tableBatchBuilder.addRow(row);\n      tableBatchBuilder.chunkComplete(line);\n      const batch = tableBatchBuilder.getFullBatch();\n      if (batch) {\n        yield batch;\n      }\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);\n    }\n  }\n\n  const batch = tableBatchBuilder.getFinalBatch();\n  if (batch) {\n    yield batch;\n  }\n}\n"],"mappings":";;;;;;;;;;;AACA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AAKkC,SAAAE,QAAAC,MAAA,EAAAC,cAAA,QAAAC,IAAA,GAAAC,MAAA,CAAAD,IAAA,CAAAF,MAAA,OAAAG,MAAA,CAAAC,qBAAA,QAAAC,OAAA,GAAAF,MAAA,CAAAC,qBAAA,CAAAJ,MAAA,GAAAC,cAAA,KAAAI,OAAA,GAAAA,OAAA,CAAAC,MAAA,WAAAC,GAAA,WAAAJ,MAAA,CAAAK,wBAAA,CAAAR,MAAA,EAAAO,GAAA,EAAAE,UAAA,OAAAP,IAAA,CAAAQ,IAAA,CAAAC,KAAA,CAAAT,IAAA,EAAAG,OAAA,YAAAH,IAAA;AAAA,SAAAU,cAAAC,MAAA,aAAAC,CAAA,MAAAA,CAAA,GAAAC,SAAA,CAAAC,MAAA,EAAAF,CAAA,UAAAG,MAAA,WAAAF,SAAA,CAAAD,CAAA,IAAAC,SAAA,CAAAD,CAAA,QAAAA,CAAA,OAAAf,OAAA,CAAAI,MAAA,CAAAc,MAAA,OAAAC,OAAA,WAAAC,GAAA,QAAAC,gBAAA,CAAAC,OAAA,EAAAR,MAAA,EAAAM,GAAA,EAAAF,MAAA,CAAAE,GAAA,SAAAhB,MAAA,CAAAmB,yBAAA,GAAAnB,MAAA,CAAAoB,gBAAA,CAAAV,MAAA,EAAAV,MAAA,CAAAmB,yBAAA,CAAAL,MAAA,KAAAlB,OAAA,CAAAI,MAAA,CAAAc,MAAA,GAAAC,OAAA,WAAAC,GAAA,IAAAhB,MAAA,CAAAqB,cAAA,CAAAX,MAAA,EAAAM,GAAA,EAAAhB,MAAA,CAAAK,wBAAA,CAAAS,MAAA,EAAAE,GAAA,iBAAAN,MAAA;AAAA,SAAAY,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAC,MAAA,KAAAH,KAAA,GAAAG,MAAA,CAAAC,aAAA,EAAAH,IAAA,GAAAE,MAAA,CAAAE,QAAA,GAAAH,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAAO,IAAA,CAAAR,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAM,qBAAA,CAAAR,MAAA,CAAAO,IAAA,CAAAR,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAO,SAAA;AAAA,SAAAD,sBAAAE,CAAA,aAAAC,kCAAAC,CAAA,QAAApC,MAAA,CAAAoC,CAAA,MAAAA,CAAA,SAAAC,OAAA,CAAAC,MAAA,KAAAL,SAAA,CAAAG,CAAA,+BAAAG,IAAA,GAAAH,CAAA,CAAAG,IAAA,SAAAF,OAAA,CAAAG,OAAA,CAAAJ,CAAA,CAAAK,KAAA,EAAAC,IAAA,WAAAD,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAF,IAAA,EAAAA,IAAA,iBAAAP,qBAAA,YAAAA,sBAAAE,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAS,CAAA,GAAAT,CAAA,CAAAU,IAAA,KAAAZ,qBAAA,CAAAa,SAAA,KAAAX,CAAA,QAAAS,CAAA,QAAAC,IAAA,WAAAA,KAAA,WAAAT,iCAAA,MAAAQ,CAAA,CAAAnC,KAAA,MAAA0B,CAAA,EAAAtB,SAAA,OAAAkC,MAAA,WAAAC,QAAAN,KAAA,QAAAO,GAAA,QAAAd,CAAA,CAAAY,MAAA,oBAAAE,GAAA,GAAAX,OAAA,CAAAG,OAAA,GAAAC,KAAA,EAAAA,KAAA,EAAAF,IAAA,UAAAJ,iCAAA,CAAAa,GAAA,CAAAxC,KAAA,MAAA0B,CAAA,EAAAtB,SAAA,OAAAqC,KAAA,WAAAC,OAAAT,KAAA,QAAAU,GAAA,QAAAjB,CAAA,CAAAY,MAAA,oBAAAK,GAAA,GAAAd,OAAA,CAAAC,MAAA,CAAAG,KAAA,IAAAN,iCAAA,CAAAgB,GAAA,CAAA3C,KAAA,MAAA0B,CAAA,EAAAtB,SAAA,aAAAoB,qBAAA,CAAAE,CAAA;AAAA,SAEHkB,oBAAoBA,CAAAC,EAAA,EAAAC,GAAA;EAAA,OAAAC,qBAAA,CAAA/C,KAAA,OAAAI,SAAA;AAAA;AAAA,SAAA2C,sBAAA;EAAAA,qBAAA,OAAAC,oBAAA,CAAAtC,OAAA,EAAAuC,YAAA,CAAAvC,OAAA,CAAAwC,IAAA,CAApC,SAAAC,QACbC,mBAAuE,EACvEC,OAAuB;IAAA,IAAAC,YAAA,EAAAC,YAAA,EAAAC,oBAAA,EAAAC,MAAA,EAAAC,KAAA,EAAAC,iBAAA,EAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,WAAA,EAAAC,OAAA,EAAAC,IAAA,EAAAC,GAAA,EAAAC,MAAA,EAAAC,KAAA;IAAA,OAAArB,YAAA,CAAAvC,OAAA,CAAA6D,IAAA,UAAAC,SAAAC,QAAA;MAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAArC,IAAA;QAAA;UAEjBkB,YAAY,GAAG,IAAAqB,oCAAuB,EAACvB,mBAAmB,CAAC;UAC3DG,YAAY,GAAG,IAAAqB,6BAAgB,EAACtB,YAAY,CAAC;UAC7CE,oBAAoB,GAAG,IAAAqB,qCAAwB,EAACtB,YAAY,CAAC;UAE7DE,MAAM,GAAG,IAAI;UACbC,KAAK,GAAG,WAAW;UAEnBC,iBAAiB,GAAG,IAAImB,yBAAiB,CAACrB,MAAM,EAAAxD,aAAA,CAAAA,aAAA,KACjDoD,OAAO;YACVK,KAAK,EAALA;UAAK,EACN,CAAC;UAAAE,yBAAA;UAAAC,iBAAA;UAAAY,QAAA,CAAAC,IAAA;UAAAX,SAAA,GAAAjD,cAAA,CAEkC0C,oBAAoB;QAAA;UAAAiB,QAAA,CAAArC,IAAA;UAAA,WAAA2C,qBAAA,CAAArE,OAAA,EAAAqD,SAAA,CAAA3B,IAAA;QAAA;UAAA,MAAAwB,yBAAA,KAAAI,KAAA,GAAAS,QAAA,CAAAO,IAAA,EAAAjD,IAAA;YAAA0C,QAAA,CAAArC,IAAA;YAAA;UAAA;UAAA6B,WAAA,GAAAD,KAAA,CAAA/B,KAAA,EAAtCiC,OAAO,GAAAD,WAAA,CAAPC,OAAO,EAAEC,IAAI,GAAAF,WAAA,CAAJE,IAAI;UAAAM,QAAA,CAAAC,IAAA;UAErBN,GAAG,GAAGa,IAAI,CAACC,KAAK,CAACf,IAAI,CAAC;UAC5BR,iBAAiB,CAACwB,MAAM,CAACf,GAAG,CAAC;UAC7BT,iBAAiB,CAACyB,aAAa,CAACjB,IAAI,CAAC;UAC/BG,MAAK,GAAGX,iBAAiB,CAAC0B,YAAY,CAAC,CAAC;UAAA,KAC1Cf,MAAK;YAAAG,QAAA,CAAArC,IAAA;YAAA;UAAA;UAAAqC,QAAA,CAAArC,IAAA;UACP,OAAMkC,MAAK;QAAA;UAAAG,QAAA,CAAArC,IAAA;UAAA;QAAA;UAAAqC,QAAA,CAAAC,IAAA;UAAAD,QAAA,CAAAa,EAAA,GAAAb,QAAA;UAAA,MAGP,IAAIc,KAAK,+CAAAC,MAAA,CAA+CtB,OAAO,CAAE,CAAC;QAAA;UAAAN,yBAAA;UAAAa,QAAA,CAAArC,IAAA;UAAA;QAAA;UAAAqC,QAAA,CAAArC,IAAA;UAAA;QAAA;UAAAqC,QAAA,CAAAC,IAAA;UAAAD,QAAA,CAAAgB,EAAA,GAAAhB,QAAA;UAAAZ,iBAAA;UAAAC,cAAA,GAAAW,QAAA,CAAAgB,EAAA;QAAA;UAAAhB,QAAA,CAAAC,IAAA;UAAAD,QAAA,CAAAC,IAAA;UAAA,MAAAd,yBAAA,IAAAG,SAAA,CAAAzB,MAAA;YAAAmC,QAAA,CAAArC,IAAA;YAAA;UAAA;UAAAqC,QAAA,CAAArC,IAAA;UAAA,WAAA2C,qBAAA,CAAArE,OAAA,EAAAqD,SAAA,CAAAzB,MAAA;QAAA;UAAAmC,QAAA,CAAAC,IAAA;UAAA,KAAAb,iBAAA;YAAAY,QAAA,CAAArC,IAAA;YAAA;UAAA;UAAA,MAAA0B,cAAA;QAAA;UAAA,OAAAW,QAAA,CAAAiB,MAAA;QAAA;UAAA,OAAAjB,QAAA,CAAAiB,MAAA;QAAA;UAItEpB,KAAK,GAAGX,iBAAiB,CAACgC,aAAa,CAAC,CAAC;UAAA,KAC3CrB,KAAK;YAAAG,QAAA,CAAArC,IAAA;YAAA;UAAA;UAAAqC,QAAA,CAAArC,IAAA;UACP,OAAMkC,KAAK;QAAA;QAAA;UAAA,OAAAG,QAAA,CAAAmB,IAAA;MAAA;IAAA,GAAAzC,OAAA;EAAA,CAEd;EAAA,OAAAJ,qBAAA,CAAA/C,KAAA,OAAAI,SAAA;AAAA"}
+{"version":3,"file":"parse-ndjson-in-batches.js","names":["_schema","require","_loaderUtils","parseNDJSONInBatches","binaryAsyncIterator","options","textIterator","makeTextDecoderIterator","lineIterator","makeLineIterator","numberedLineIterator","makeNumberedLineIterator","schema","shape","tableBatchBuilder","TableBatchBuilder","counter","line","row","JSON","parse","addRow","chunkComplete","batch","getFullBatch","error","Error","concat","getFinalBatch"],"sources":["../../../src/lib/parse-ndjson-in-batches.ts"],"sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {\n  LoaderOptions,\n  makeLineIterator,\n  makeNumberedLineIterator,\n  makeTextDecoderIterator\n} from '@loaders.gl/loader-utils';\n\nexport default async function* parseNDJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: LoaderOptions\n): AsyncIterable<Batch> {\n  const textIterator = makeTextDecoderIterator(binaryAsyncIterator);\n  const lineIterator = makeLineIterator(textIterator);\n  const numberedLineIterator = makeNumberedLineIterator(lineIterator);\n\n  const schema = null;\n  const shape = 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  for await (const {counter, line} of numberedLineIterator) {\n    try {\n      const row = JSON.parse(line);\n      tableBatchBuilder.addRow(row);\n      tableBatchBuilder.chunkComplete(line);\n      const batch = tableBatchBuilder.getFullBatch();\n      if (batch) {\n        yield batch;\n      }\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);\n    }\n  }\n\n  const batch = tableBatchBuilder.getFinalBatch();\n  if (batch) {\n    yield batch;\n  }\n}\n"],"mappings":";;;;;;AACA,IAAAA,OAAA,GAAAC,OAAA;AACA,IAAAC,YAAA,GAAAD,OAAA;AAOe,gBAAgBE,oBAAoBA,CACjDC,mBAAuE,EACvEC,OAAuB,EACD;EACtB,MAAMC,YAAY,GAAG,IAAAC,oCAAuB,EAACH,mBAAmB,CAAC;EACjE,MAAMI,YAAY,GAAG,IAAAC,6BAAgB,EAACH,YAAY,CAAC;EACnD,MAAMI,oBAAoB,GAAG,IAAAC,qCAAwB,EAACH,YAAY,CAAC;EAEnE,MAAMI,MAAM,GAAG,IAAI;EACnB,MAAMC,KAAK,GAAG,WAAW;EAEzB,MAAMC,iBAAiB,GAAG,IAAIC,yBAAiB,CAACH,MAAM,EAAE;IACtD,GAAGP,OAAO;IACVQ;EACF,CAAC,CAAC;EAEF,WAAW,MAAM;IAACG,OAAO;IAAEC;EAAI,CAAC,IAAIP,oBAAoB,EAAE;IACxD,IAAI;MACF,MAAMQ,GAAG,GAAGC,IAAI,CAACC,KAAK,CAACH,IAAI,CAAC;MAC5BH,iBAAiB,CAACO,MAAM,CAACH,GAAG,CAAC;MAC7BJ,iBAAiB,CAACQ,aAAa,CAACL,IAAI,CAAC;MACrC,MAAMM,KAAK,GAAGT,iBAAiB,CAACU,YAAY,CAAC,CAAC;MAC9C,IAAID,KAAK,EAAE;QACT,MAAMA,KAAK;MACb;IACF,CAAC,CAAC,OAAOE,KAAK,EAAE;MACd,MAAM,IAAIC,KAAK,+CAAAC,MAAA,CAA+CX,OAAO,CAAE,CAAC;IAC1E;EACF;EAEA,MAAMO,KAAK,GAAGT,iBAAiB,CAACc,aAAa,CAAC,CAAC;EAC/C,IAAIL,KAAK,EAAE;IACT,MAAMA,KAAK;EACb;AACF"}

package/dist/es5/lib/parse-ndjson.js

@@ -5,8 +5,8 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.default = parseNDJSONSync;
 function parseNDJSONSync(ndjsonText) {
-  var lines = ndjsonText.trim().split('\n');
-  return lines.map(function (line, counter) {
+  const lines = ndjsonText.trim().split('\n');
+  return lines.map((line, counter) => {
     try {
       return JSON.parse(line);
     } catch (error) {

package/dist/es5/lib/parse-ndjson.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-ndjson.js","names":["parseNDJSONSync","ndjsonText","lines","trim","split","map","line","counter","JSON","parse","error","Error","concat"],"sources":["../../../src/lib/parse-ndjson.ts"],"sourcesContent":["export default function parseNDJSONSync(ndjsonText: string) {\n  const lines = ndjsonText.trim().split('\\n');\n  return lines.map((line, counter) => {\n    try {\n      return JSON.parse(line);\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);\n    }\n  });\n}\n"],"mappings":";;;;;;AAAe,SAASA,eAAeA,CAACC,UAAkB,EAAE;EAC1D,IAAMC,KAAK,GAAGD,UAAU,CAACE,IAAI,CAAC,CAAC,CAACC,KAAK,CAAC,IAAI,CAAC;EAC3C,OAAOF,KAAK,CAACG,GAAG,CAAC,UAACC,IAAI,EAAEC,OAAO,EAAK;IAClC,IAAI;MACF,OAAOC,IAAI,CAACC,KAAK,CAACH,IAAI,CAAC;IACzB,CAAC,CAAC,OAAOI,KAAK,EAAE;MACd,MAAM,IAAIC,KAAK,+CAAAC,MAAA,CAA+CL,OAAO,GAAG,CAAC,CAAE,CAAC;IAC9E;EACF,CAAC,CAAC;AACJ"}
+{"version":3,"file":"parse-ndjson.js","names":["parseNDJSONSync","ndjsonText","lines","trim","split","map","line","counter","JSON","parse","error","Error","concat"],"sources":["../../../src/lib/parse-ndjson.ts"],"sourcesContent":["export default function parseNDJSONSync(ndjsonText: string) {\n  const lines = ndjsonText.trim().split('\\n');\n  return lines.map((line, counter) => {\n    try {\n      return JSON.parse(line);\n    } catch (error) {\n      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);\n    }\n  });\n}\n"],"mappings":";;;;;;AAAe,SAASA,eAAeA,CAACC,UAAkB,EAAE;EAC1D,MAAMC,KAAK,GAAGD,UAAU,CAACE,IAAI,CAAC,CAAC,CAACC,KAAK,CAAC,IAAI,CAAC;EAC3C,OAAOF,KAAK,CAACG,GAAG,CAAC,CAACC,IAAI,EAAEC,OAAO,KAAK;IAClC,IAAI;MACF,OAAOC,IAAI,CAACC,KAAK,CAACH,IAAI,CAAC;IACzB,CAAC,CAAC,OAAOI,KAAK,EAAE;MACd,MAAM,IAAIC,KAAK,+CAAAC,MAAA,CAA+CL,OAAO,GAAG,CAAC,CAAE,CAAC;IAC9E;EACF,CAAC,CAAC;AACJ"}