@loaders.gl/json 3.1.0-beta.3 → 3.1.1

package/dist/esm/geojson-loader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/geojson-loader.ts"],"names":["geojsonToBinary","parseJSONSync","parseJSONInBatches","VERSION","DEFAULT_GEOJSON_LOADER_OPTIONS","geojson","shape","json","jsonpaths","gis","format","GeoJSONWorkerLoader","name","id","module","version","worker","extensions","mimeTypes","category","text","options","GeoJSONLoader","parse","parseTextSync","parseInBatches","arrayBuffer","TextDecoder","decode","asyncIterator","geojsonIterator","makeBinaryGeometryIterator","batch","data"],"mappings":"AAEA,SAAQA,eAAR,QAA8B,iBAA9B;AACA,OAAOC,aAAP,MAA0B,kBAA1B;AACA,OAAOC,kBAAP,MAA+B,6BAA/B;AAIA,MAAMC,OAAO,GAAG,0BAAuB,WAAvB,oBAAmD,QAAnE;AAWA,MAAMC,8BAA8B,GAAG;AACrCC,EAAAA,OAAO,EAAE;AACPC,IAAAA,KAAK,EAAE;AADA,GAD4B;AAIrCC,EAAAA,IAAI,EAAE;AACJC,IAAAA,SAAS,EAAE,CAAC,GAAD,EAAM,YAAN;AADP,GAJ+B;AAOrCC,EAAAA,GAAG,EAAE;AACHC,IAAAA,MAAM,EAAE;AADL;AAPgC,CAAvC;AAeA,OAAO,MAAMC,mBAA2B,GAAG;AACzCC,EAAAA,IAAI,EAAE,SADmC;AAEzCC,EAAAA,EAAE,EAAE,SAFqC;AAGzCC,EAAAA,MAAM,EAAE,SAHiC;AAIzCC,EAAAA,OAAO,EAAEZ,OAJgC;AAKzCa,EAAAA,MAAM,EAAE,IALiC;AAMzCC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAN6B;AAOzCC,EAAAA,SAAS,EAAE,CAAC,sBAAD,CAP8B;AAQzCC,EAAAA,QAAQ,EAAE,UAR+B;AASzCC,EAAAA,IAAI,EAAE,IATmC;AAUzCC,EAAAA,OAAO,EAAEjB;AAVgC,CAApC;AAaP,OAAO,MAAMkB,aAA+B,GAAG,EAC7C,GAAGX,mBAD0C;AAE7CY,EAAAA,KAF6C;AAG7CC,EAAAA,aAH6C;AAI7CC,EAAAA;AAJ6C,CAAxC;;AAOP,eAAeF,KAAf,CAAqBG,WAArB,EAAkCL,OAAlC,EAA2C;AACzC,SAAOG,aAAa,CAAC,IAAIG,WAAJ,GAAkBC,MAAlB,CAAyBF,WAAzB,CAAD,EAAwCL,OAAxC,CAApB;AACD;;AAED,SAASG,aAAT,CAAuBJ,IAAvB,EAA6BC,OAA7B,EAAsC;AAEpCA,EAAAA,OAAO,GAAG,EAAC,GAAGjB,8BAAJ;AAAoC,OAAGiB;AAAvC,GAAV;AACAA,EAAAA,OAAO,CAACd,IAAR,GAAe,EAAC,GAAGH,8BAA8B,CAACC,OAAnC;AAA4C,OAAGgB,OAAO,CAAChB;AAAvD,GAAf;AACAgB,EAAAA,OAAO,CAACZ,GAAR,GAAcY,OAAO,CAACZ,GAAR,IAAe,EAA7B;AACA,QAAMF,IAAI,GAAGN,aAAa,CAACmB,IAAD,EAAOC,OAAP,CAA1B;;AACA,UAAQA,OAAO,CAACZ,GAAR,CAAYC,MAApB;AACE,SAAK,QAAL;AACE,aAAOV,eAAe,CAACO,IAAD,CAAtB;;AACF;AACE,aAAOA,IAAP;AAJJ;AAMD;;AAED,SAASkB,cAAT,CAAwBI,aAAxB,EAAuCR,OAAvC,EAAoE;AAElEA,EAAAA,OAAO,GAAG,EAAC,GAAGjB,8BAAJ;AAAoC,OAAGiB;AAAvC,GAAV;AACAA,EAAAA,OAAO,CAACd,IAAR,GAAe,EAAC,GAAGH,8BAA8B,CAACC,OAAnC;AAA4C,OAAGgB,OAAO,CAAChB;AAAvD,GAAf;AAEA,QAAMyB,eAAe,GAAG5B,kBAAkB,CAAC2B,aAAD,EAAgBR,OAAhB,CAA1C;;AAEA,UAAQA,OAAO,CAACZ,GAAR,CAAYC,MAApB;AACE,SAAK,QAAL;AACE,aAAOqB,0BAA0B,CAACD,eAAD,CAAjC;;AACF;AACE,aAAOA,eAAP;AAJJ;AAMD;;AAED,gBAAgBC,0BAAhB,CAA2CD,eAA3C,EAA4D;AAC1D,aAAW,MAAME,KAAjB,IAA0BF,eAA1B,EAA2C;AACzCE,IAAAA,KAAK,CAACC,IAAN,GAAajC,eAAe,CAACgC,KAAK,CAACC,IAAP,CAA5B;AACA,UAAMD,KAAN;AACD;AACF","sourcesContent":["import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {JSONLoaderOptions} from './json-loader';\nimport {geojsonToBinary} from '@loaders.gl/gis';\nimport parseJSONSync from './lib/parse-json';\nimport parseJSONInBatches from './lib/parse-json-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type GeoJSONLoaderOptions = JSONLoaderOptions & {\n  geojson?: {\n    shape?: 'object-row-table';\n  };\n  gis?: {\n    format: 'geojson';\n  };\n};\n\nconst DEFAULT_GEOJSON_LOADER_OPTIONS = {\n  geojson: {\n    shape: 'object-row-table'\n  },\n  json: {\n    jsonpaths: ['$', '$.features']\n  },\n  gis: {\n    format: 'geojson'\n  }\n};\n\n/**\n * GeoJSON loader\n */\nexport const GeoJSONWorkerLoader: Loader = {\n  name: 'GeoJSON',\n  id: 'geojson',\n  module: 'geojson',\n  version: VERSION,\n  worker: true,\n  extensions: ['geojson'],\n  mimeTypes: ['application/geo+json'],\n  category: 'geometry',\n  text: true,\n  options: DEFAULT_GEOJSON_LOADER_OPTIONS\n};\n\nexport const GeoJSONLoader: LoaderWithParser = {\n  ...GeoJSONWorkerLoader,\n  parse,\n  parseTextSync,\n  parseInBatches\n};\n\nasync function parse(arrayBuffer, options) {\n  return parseTextSync(new TextDecoder().decode(arrayBuffer), options);\n}\n\nfunction parseTextSync(text, options) {\n  // Apps can call the parse method directly, we so apply default options here\n  options = {...DEFAULT_GEOJSON_LOADER_OPTIONS, ...options};\n  options.json = {...DEFAULT_GEOJSON_LOADER_OPTIONS.geojson, ...options.geojson};\n  options.gis = options.gis || {};\n  const json = parseJSONSync(text, options);\n  switch (options.gis.format) {\n    case 'binary':\n      return geojsonToBinary(json);\n    default:\n      return json;\n  }\n}\n\nfunction parseInBatches(asyncIterator, options): AsyncIterable<any> {\n  // Apps can call the parse method directly, we so apply default options here\n  options = {...DEFAULT_GEOJSON_LOADER_OPTIONS, ...options};\n  options.json = {...DEFAULT_GEOJSON_LOADER_OPTIONS.geojson, ...options.geojson};\n\n  const geojsonIterator = parseJSONInBatches(asyncIterator, options);\n\n  switch (options.gis.format) {\n    case 'binary':\n      return makeBinaryGeometryIterator(geojsonIterator);\n    default:\n      return geojsonIterator;\n  }\n}\n\nasync function* makeBinaryGeometryIterator(geojsonIterator) {\n  for await (const batch of geojsonIterator) {\n    batch.data = geojsonToBinary(batch.data);\n    yield batch;\n  }\n}\n"],"file":"geojson-loader.js"}
+{"version":3,"sources":["../../src/geojson-loader.ts"],"names":["geojsonToBinary","parseJSONSync","parseJSONInBatches","VERSION","DEFAULT_GEOJSON_LOADER_OPTIONS","geojson","shape","json","jsonpaths","gis","format","GeoJSONWorkerLoader","name","id","module","version","worker","extensions","mimeTypes","category","text","options","GeoJSONLoader","parse","parseTextSync","parseInBatches","arrayBuffer","TextDecoder","decode","asyncIterator","geojsonIterator","makeBinaryGeometryIterator","batch","data"],"mappings":"AAEA,SAAQA,eAAR,QAA8B,iBAA9B;AACA,OAAOC,aAAP,MAA0B,kBAA1B;AACA,OAAOC,kBAAP,MAA+B,6BAA/B;AAIA,MAAMC,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAWA,MAAMC,8BAA8B,GAAG;AACrCC,EAAAA,OAAO,EAAE;AACPC,IAAAA,KAAK,EAAE;AADA,GAD4B;AAIrCC,EAAAA,IAAI,EAAE;AACJC,IAAAA,SAAS,EAAE,CAAC,GAAD,EAAM,YAAN;AADP,GAJ+B;AAOrCC,EAAAA,GAAG,EAAE;AACHC,IAAAA,MAAM,EAAE;AADL;AAPgC,CAAvC;AAeA,OAAO,MAAMC,mBAA2B,GAAG;AACzCC,EAAAA,IAAI,EAAE,SADmC;AAEzCC,EAAAA,EAAE,EAAE,SAFqC;AAGzCC,EAAAA,MAAM,EAAE,SAHiC;AAIzCC,EAAAA,OAAO,EAAEZ,OAJgC;AAKzCa,EAAAA,MAAM,EAAE,IALiC;AAMzCC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAN6B;AAOzCC,EAAAA,SAAS,EAAE,CAAC,sBAAD,CAP8B;AAQzCC,EAAAA,QAAQ,EAAE,UAR+B;AASzCC,EAAAA,IAAI,EAAE,IATmC;AAUzCC,EAAAA,OAAO,EAAEjB;AAVgC,CAApC;AAaP,OAAO,MAAMkB,aAA+B,GAAG,EAC7C,GAAGX,mBAD0C;AAE7CY,EAAAA,KAF6C;AAG7CC,EAAAA,aAH6C;AAI7CC,EAAAA;AAJ6C,CAAxC;;AAOP,eAAeF,KAAf,CAAqBG,WAArB,EAAkCL,OAAlC,EAA2C;AACzC,SAAOG,aAAa,CAAC,IAAIG,WAAJ,GAAkBC,MAAlB,CAAyBF,WAAzB,CAAD,EAAwCL,OAAxC,CAApB;AACD;;AAED,SAASG,aAAT,CAAuBJ,IAAvB,EAA6BC,OAA7B,EAAsC;AAEpCA,EAAAA,OAAO,GAAG,EAAC,GAAGjB,8BAAJ;AAAoC,OAAGiB;AAAvC,GAAV;AACAA,EAAAA,OAAO,CAACd,IAAR,GAAe,EAAC,GAAGH,8BAA8B,CAACC,OAAnC;AAA4C,OAAGgB,OAAO,CAAChB;AAAvD,GAAf;AACAgB,EAAAA,OAAO,CAACZ,GAAR,GAAcY,OAAO,CAACZ,GAAR,IAAe,EAA7B;AACA,QAAMF,IAAI,GAAGN,aAAa,CAACmB,IAAD,EAAOC,OAAP,CAA1B;;AACA,UAAQA,OAAO,CAACZ,GAAR,CAAYC,MAApB;AACE,SAAK,QAAL;AACE,aAAOV,eAAe,CAACO,IAAD,CAAtB;;AACF;AACE,aAAOA,IAAP;AAJJ;AAMD;;AAED,SAASkB,cAAT,CAAwBI,aAAxB,EAAuCR,OAAvC,EAAoE;AAElEA,EAAAA,OAAO,GAAG,EAAC,GAAGjB,8BAAJ;AAAoC,OAAGiB;AAAvC,GAAV;AACAA,EAAAA,OAAO,CAACd,IAAR,GAAe,EAAC,GAAGH,8BAA8B,CAACC,OAAnC;AAA4C,OAAGgB,OAAO,CAAChB;AAAvD,GAAf;AAEA,QAAMyB,eAAe,GAAG5B,kBAAkB,CAAC2B,aAAD,EAAgBR,OAAhB,CAA1C;;AAEA,UAAQA,OAAO,CAACZ,GAAR,CAAYC,MAApB;AACE,SAAK,QAAL;AACE,aAAOqB,0BAA0B,CAACD,eAAD,CAAjC;;AACF;AACE,aAAOA,eAAP;AAJJ;AAMD;;AAED,gBAAgBC,0BAAhB,CAA2CD,eAA3C,EAA4D;AAC1D,aAAW,MAAME,KAAjB,IAA0BF,eAA1B,EAA2C;AACzCE,IAAAA,KAAK,CAACC,IAAN,GAAajC,eAAe,CAACgC,KAAK,CAACC,IAAP,CAA5B;AACA,UAAMD,KAAN;AACD;AACF","sourcesContent":["import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {JSONLoaderOptions} from './json-loader';\nimport {geojsonToBinary} from '@loaders.gl/gis';\nimport parseJSONSync from './lib/parse-json';\nimport parseJSONInBatches from './lib/parse-json-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type GeoJSONLoaderOptions = JSONLoaderOptions & {\n  geojson?: {\n    shape?: 'object-row-table';\n  };\n  gis?: {\n    format: 'geojson';\n  };\n};\n\nconst DEFAULT_GEOJSON_LOADER_OPTIONS = {\n  geojson: {\n    shape: 'object-row-table'\n  },\n  json: {\n    jsonpaths: ['$', '$.features']\n  },\n  gis: {\n    format: 'geojson'\n  }\n};\n\n/**\n * GeoJSON loader\n */\nexport const GeoJSONWorkerLoader: Loader = {\n  name: 'GeoJSON',\n  id: 'geojson',\n  module: 'geojson',\n  version: VERSION,\n  worker: true,\n  extensions: ['geojson'],\n  mimeTypes: ['application/geo+json'],\n  category: 'geometry',\n  text: true,\n  options: DEFAULT_GEOJSON_LOADER_OPTIONS\n};\n\nexport const GeoJSONLoader: LoaderWithParser = {\n  ...GeoJSONWorkerLoader,\n  parse,\n  parseTextSync,\n  parseInBatches\n};\n\nasync function parse(arrayBuffer, options) {\n  return parseTextSync(new TextDecoder().decode(arrayBuffer), options);\n}\n\nfunction parseTextSync(text, options) {\n  // Apps can call the parse method directly, we so apply default options here\n  options = {...DEFAULT_GEOJSON_LOADER_OPTIONS, ...options};\n  options.json = {...DEFAULT_GEOJSON_LOADER_OPTIONS.geojson, ...options.geojson};\n  options.gis = options.gis || {};\n  const json = parseJSONSync(text, options);\n  switch (options.gis.format) {\n    case 'binary':\n      return geojsonToBinary(json);\n    default:\n      return json;\n  }\n}\n\nfunction parseInBatches(asyncIterator, options): AsyncIterable<any> {\n  // Apps can call the parse method directly, we so apply default options here\n  options = {...DEFAULT_GEOJSON_LOADER_OPTIONS, ...options};\n  options.json = {...DEFAULT_GEOJSON_LOADER_OPTIONS.geojson, ...options.geojson};\n\n  const geojsonIterator = parseJSONInBatches(asyncIterator, options);\n\n  switch (options.gis.format) {\n    case 'binary':\n      return makeBinaryGeometryIterator(geojsonIterator);\n    default:\n      return geojsonIterator;\n  }\n}\n\nasync function* makeBinaryGeometryIterator(geojsonIterator) {\n  for await (const batch of geojsonIterator) {\n    batch.data = geojsonToBinary(batch.data);\n    yield batch;\n  }\n}\n"],"file":"geojson-loader.js"}

package/dist/esm/index.js

@@ -3,4 +3,5 @@ export { NDJSONLoader } from './ndjson-loader';
 export { GeoJSONLoader as _GeoJSONLoader, GeoJSONWorkerLoader as _GeoJSONWorkerLoader } from './geojson-loader';
 export { default as _JSONPath } from './lib/jsonpath/jsonpath';
 export { default as _ClarinetParser } from './lib/clarinet/clarinet';
+export { rebuildJsonObject as _rebuildJsonObject } from './lib/parse-json-in-batches';
 //# sourceMappingURL=index.js.map

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/index.ts"],"names":["JSONLoader","NDJSONLoader","GeoJSONLoader","_GeoJSONLoader","GeoJSONWorkerLoader","_GeoJSONWorkerLoader","default","_JSONPath","_ClarinetParser"],"mappings":"AACA,SAAQA,UAAR,QAAyB,eAAzB;AACA,SAAQC,YAAR,QAA2B,iBAA3B;AAIA,SACEC,aAAa,IAAIC,cADnB,EAEEC,mBAAmB,IAAIC,oBAFzB,QAGO,kBAHP;AAKA,SAAQC,OAAO,IAAIC,SAAnB,QAAmC,yBAAnC;AACA,SAAQD,OAAO,IAAIE,eAAnB,QAAyC,yBAAzC","sourcesContent":["export type {JSONLoaderOptions} from './json-loader';\nexport {JSONLoader} from './json-loader';\nexport {NDJSONLoader} from './ndjson-loader';\n\n// EXPERIMENTAL EXPORTS - WARNING: MAY BE REMOVED WIHTOUT NOTICE IN FUTURE RELEASES\nexport type {GeoJSONLoaderOptions as _GeoJSONLoaderOptions} from './geojson-loader';\nexport {\n  GeoJSONLoader as _GeoJSONLoader,\n  GeoJSONWorkerLoader as _GeoJSONWorkerLoader\n} from './geojson-loader';\n\nexport {default as _JSONPath} from './lib/jsonpath/jsonpath';\nexport {default as _ClarinetParser} from './lib/clarinet/clarinet';\n"],"file":"index.js"}
+{"version":3,"sources":["../../src/index.ts"],"names":["JSONLoader","NDJSONLoader","GeoJSONLoader","_GeoJSONLoader","GeoJSONWorkerLoader","_GeoJSONWorkerLoader","default","_JSONPath","_ClarinetParser","rebuildJsonObject","_rebuildJsonObject"],"mappings":"AACA,SAAQA,UAAR,QAAyB,eAAzB;AACA,SAAQC,YAAR,QAA2B,iBAA3B;AAIA,SACEC,aAAa,IAAIC,cADnB,EAEEC,mBAAmB,IAAIC,oBAFzB,QAGO,kBAHP;AAKA,SAAQC,OAAO,IAAIC,SAAnB,QAAmC,yBAAnC;AACA,SAAQD,OAAO,IAAIE,eAAnB,QAAyC,yBAAzC;AAEA,SAAQC,iBAAiB,IAAIC,kBAA7B,QAAsD,6BAAtD","sourcesContent":["export type {JSONLoaderOptions} from './json-loader';\nexport {JSONLoader} from './json-loader';\nexport {NDJSONLoader} from './ndjson-loader';\n\n// EXPERIMENTAL EXPORTS - WARNING: MAY BE REMOVED WIHTOUT NOTICE IN FUTURE RELEASES\nexport type {GeoJSONLoaderOptions as _GeoJSONLoaderOptions} from './geojson-loader';\nexport {\n  GeoJSONLoader as _GeoJSONLoader,\n  GeoJSONWorkerLoader as _GeoJSONWorkerLoader\n} from './geojson-loader';\n\nexport {default as _JSONPath} from './lib/jsonpath/jsonpath';\nexport {default as _ClarinetParser} from './lib/clarinet/clarinet';\n\nexport {rebuildJsonObject as _rebuildJsonObject} from './lib/parse-json-in-batches';\n"],"file":"index.js"}

package/dist/esm/json-loader.js

@@ -1,6 +1,6 @@
 import parseJSONSync from './lib/parse-json';
 import parseJSONInBatches from './lib/parse-json-in-batches';
-const VERSION = typeof "3.1.0-beta.3" !== 'undefined' ? "3.1.0-beta.3" : 'latest';
+const VERSION = typeof "3.1.1" !== 'undefined' ? "3.1.1" : 'latest';
 const DEFAULT_JSON_LOADER_OPTIONS = {
   json: {
     shape: 'row-table',

package/dist/esm/json-loader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/json-loader.ts"],"names":["parseJSONSync","parseJSONInBatches","VERSION","DEFAULT_JSON_LOADER_OPTIONS","json","shape","table","jsonpaths","JSONLoader","name","id","module","version","extensions","mimeTypes","category","text","parse","parseTextSync","parseInBatches","options","arrayBuffer","TextDecoder","decode","jsonOptions","asyncIterator"],"mappings":"AAEA,OAAOA,aAAP,MAA0B,kBAA1B;AACA,OAAOC,kBAAP,MAA+B,6BAA/B;AAIA,MAAMC,OAAO,GAAG,0BAAuB,WAAvB,oBAAmD,QAAnE;AAcA,MAAMC,2BAA2B,GAAG;AAClCC,EAAAA,IAAI,EAAE;AACJC,IAAAA,KAAK,EAAE,WADH;AAEJC,IAAAA,KAAK,EAAE,KAFH;AAGJC,IAAAA,SAAS,EAAE;AAHP;AAD4B,CAApC;AASA,OAAO,MAAMC,UAA4B,GAAG;AAC1CC,EAAAA,IAAI,EAAE,MADoC;AAE1CC,EAAAA,EAAE,EAAE,MAFsC;AAG1CC,EAAAA,MAAM,EAAE,MAHkC;AAI1CC,EAAAA,OAAO,EAAEV,OAJiC;AAK1CW,EAAAA,UAAU,EAAE,CAAC,MAAD,EAAS,SAAT,CAL8B;AAM1CC,EAAAA,SAAS,EAAE,CAAC,kBAAD,CAN+B;AAoB1CC,EAAAA,QAAQ,EAAE,OApBgC;AAqB1CC,EAAAA,IAAI,EAAE,IArBoC;AAsB1CC,EAAAA,KAtB0C;AAuB1CC,EAAAA,aAvB0C;AAwB1CC,EAAAA,cAxB0C;AAyB1CC,EAAAA,OAAO,EAAEjB;AAzBiC,CAArC;;AA4BP,eAAec,KAAf,CAAqBI,WAArB,EAA+CD,OAA/C,EAA4E;AAC1E,SAAOF,aAAa,CAAC,IAAII,WAAJ,GAAkBC,MAAlB,CAAyBF,WAAzB,CAAD,EAAwCD,OAAxC,CAApB;AACD;;AAED,SAASF,aAAT,CAAuBF,IAAvB,EAAqCI,OAArC,EAAkE;AAChE,QAAMI,WAAW,GAAG,EAAC,GAAGJ,OAAJ;AAAahB,IAAAA,IAAI,EAAE,EAAC,GAAGD,2BAA2B,CAACC,IAAhC;AAAsC,UAAGgB,OAAH,aAAGA,OAAH,uBAAGA,OAAO,CAAEhB,IAAZ;AAAtC;AAAnB,GAApB;AACA,SAAOJ,aAAa,CAACgB,IAAD,EAAOQ,WAAP,CAApB;AACD;;AAED,SAASL,cAAT,CACEM,aADF,EAEEL,OAFF,EAGwB;AACtB,QAAMI,WAAW,GAAG,EAAC,GAAGJ,OAAJ;AAAahB,IAAAA,IAAI,EAAE,EAAC,GAAGD,2BAA2B,CAACC,IAAhC;AAAsC,UAAGgB,OAAH,aAAGA,OAAH,uBAAGA,OAAO,CAAEhB,IAAZ;AAAtC;AAAnB,GAApB;AACA,SAAOH,kBAAkB,CAACwB,aAAD,EAAgBD,WAAhB,CAAzB;AACD","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport parseJSONSync from './lib/parse-json';\nimport parseJSONInBatches from './lib/parse-json-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\n/**\n * @param table -\n * @param jsonpaths -\n */\nexport type JSONLoaderOptions = LoaderOptions & {\n  json?: {\n    shape?: 'row-table';\n    table?: false;\n    jsonpaths?: string[];\n  };\n};\n\nconst DEFAULT_JSON_LOADER_OPTIONS = {\n  json: {\n    shape: 'row-table',\n    table: false,\n    jsonpaths: []\n    // batchSize: 'auto'\n  }\n};\n\nexport const JSONLoader: LoaderWithParser = {\n  name: 'JSON',\n  id: 'json',\n  module: 'json',\n  version: VERSION,\n  extensions: ['json', 'geojson'],\n  mimeTypes: ['application/json'],\n  // TODO - support various line based JSON formats\n  /*\n  extensions: {\n    json: null,\n    jsonl: {stream: true},\n    ndjson: {stream: true}\n  },\n  mimeTypes: {\n    'application/json': null,\n    'application/json-seq': {stream: true},\n    'application/x-ndjson': {stream: true}\n  },\n  */\n  category: 'table',\n  text: true,\n  parse,\n  parseTextSync,\n  parseInBatches,\n  options: DEFAULT_JSON_LOADER_OPTIONS\n};\n\nasync function parse(arrayBuffer: ArrayBuffer, options?: JSONLoaderOptions) {\n  return parseTextSync(new TextDecoder().decode(arrayBuffer), options);\n}\n\nfunction parseTextSync(text: string, options?: JSONLoaderOptions) {\n  const jsonOptions = {...options, json: {...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json}};\n  return parseJSONSync(text, jsonOptions as JSONLoaderOptions);\n}\n\nfunction parseInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const jsonOptions = {...options, json: {...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json}};\n  return parseJSONInBatches(asyncIterator, jsonOptions as JSONLoaderOptions);\n}\n"],"file":"json-loader.js"}
+{"version":3,"sources":["../../src/json-loader.ts"],"names":["parseJSONSync","parseJSONInBatches","VERSION","DEFAULT_JSON_LOADER_OPTIONS","json","shape","table","jsonpaths","JSONLoader","name","id","module","version","extensions","mimeTypes","category","text","parse","parseTextSync","parseInBatches","options","arrayBuffer","TextDecoder","decode","jsonOptions","asyncIterator"],"mappings":"AAEA,OAAOA,aAAP,MAA0B,kBAA1B;AACA,OAAOC,kBAAP,MAA+B,6BAA/B;AAIA,MAAMC,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAcA,MAAMC,2BAA2B,GAAG;AAClCC,EAAAA,IAAI,EAAE;AACJC,IAAAA,KAAK,EAAE,WADH;AAEJC,IAAAA,KAAK,EAAE,KAFH;AAGJC,IAAAA,SAAS,EAAE;AAHP;AAD4B,CAApC;AASA,OAAO,MAAMC,UAA4B,GAAG;AAC1CC,EAAAA,IAAI,EAAE,MADoC;AAE1CC,EAAAA,EAAE,EAAE,MAFsC;AAG1CC,EAAAA,MAAM,EAAE,MAHkC;AAI1CC,EAAAA,OAAO,EAAEV,OAJiC;AAK1CW,EAAAA,UAAU,EAAE,CAAC,MAAD,EAAS,SAAT,CAL8B;AAM1CC,EAAAA,SAAS,EAAE,CAAC,kBAAD,CAN+B;AAoB1CC,EAAAA,QAAQ,EAAE,OApBgC;AAqB1CC,EAAAA,IAAI,EAAE,IArBoC;AAsB1CC,EAAAA,KAtB0C;AAuB1CC,EAAAA,aAvB0C;AAwB1CC,EAAAA,cAxB0C;AAyB1CC,EAAAA,OAAO,EAAEjB;AAzBiC,CAArC;;AA4BP,eAAec,KAAf,CAAqBI,WAArB,EAA+CD,OAA/C,EAA4E;AAC1E,SAAOF,aAAa,CAAC,IAAII,WAAJ,GAAkBC,MAAlB,CAAyBF,WAAzB,CAAD,EAAwCD,OAAxC,CAApB;AACD;;AAED,SAASF,aAAT,CAAuBF,IAAvB,EAAqCI,OAArC,EAAkE;AAChE,QAAMI,WAAW,GAAG,EAAC,GAAGJ,OAAJ;AAAahB,IAAAA,IAAI,EAAE,EAAC,GAAGD,2BAA2B,CAACC,IAAhC;AAAsC,UAAGgB,OAAH,aAAGA,OAAH,uBAAGA,OAAO,CAAEhB,IAAZ;AAAtC;AAAnB,GAApB;AACA,SAAOJ,aAAa,CAACgB,IAAD,EAAOQ,WAAP,CAApB;AACD;;AAED,SAASL,cAAT,CACEM,aADF,EAEEL,OAFF,EAGwB;AACtB,QAAMI,WAAW,GAAG,EAAC,GAAGJ,OAAJ;AAAahB,IAAAA,IAAI,EAAE,EAAC,GAAGD,2BAA2B,CAACC,IAAhC;AAAsC,UAAGgB,OAAH,aAAGA,OAAH,uBAAGA,OAAO,CAAEhB,IAAZ;AAAtC;AAAnB,GAApB;AACA,SAAOH,kBAAkB,CAACwB,aAAD,EAAgBD,WAAhB,CAAzB;AACD","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport parseJSONSync from './lib/parse-json';\nimport parseJSONInBatches from './lib/parse-json-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\n/**\n * @param table -\n * @param jsonpaths -\n */\nexport type JSONLoaderOptions = LoaderOptions & {\n  json?: {\n    shape?: 'row-table';\n    table?: false;\n    jsonpaths?: string[];\n  };\n};\n\nconst DEFAULT_JSON_LOADER_OPTIONS = {\n  json: {\n    shape: 'row-table',\n    table: false,\n    jsonpaths: []\n    // batchSize: 'auto'\n  }\n};\n\nexport const JSONLoader: LoaderWithParser = {\n  name: 'JSON',\n  id: 'json',\n  module: 'json',\n  version: VERSION,\n  extensions: ['json', 'geojson'],\n  mimeTypes: ['application/json'],\n  // TODO - support various line based JSON formats\n  /*\n  extensions: {\n    json: null,\n    jsonl: {stream: true},\n    ndjson: {stream: true}\n  },\n  mimeTypes: {\n    'application/json': null,\n    'application/json-seq': {stream: true},\n    'application/x-ndjson': {stream: true}\n  },\n  */\n  category: 'table',\n  text: true,\n  parse,\n  parseTextSync,\n  parseInBatches,\n  options: DEFAULT_JSON_LOADER_OPTIONS\n};\n\nasync function parse(arrayBuffer: ArrayBuffer, options?: JSONLoaderOptions) {\n  return parseTextSync(new TextDecoder().decode(arrayBuffer), options);\n}\n\nfunction parseTextSync(text: string, options?: JSONLoaderOptions) {\n  const jsonOptions = {...options, json: {...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json}};\n  return parseJSONSync(text, jsonOptions as JSONLoaderOptions);\n}\n\nfunction parseInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const jsonOptions = {...options, json: {...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json}};\n  return parseJSONInBatches(asyncIterator, jsonOptions as JSONLoaderOptions);\n}\n"],"file":"json-loader.js"}

package/dist/esm/lib/parse-json-in-batches.js

@@ -1,6 +1,7 @@
 import { TableBatchBuilder } from '@loaders.gl/schema';
-import { makeTextDecoderIterator } from '@loaders.gl/loader-utils';
+import { assert, makeTextDecoderIterator } from '@loaders.gl/loader-utils';
 import StreamingJSONParser from './parser/streaming-json-parser';
+import JSONPath from './jsonpath/jsonpath';
 export default async function* parseJSONInBatches(binaryAsyncIterator, options) {
   var _options$json;
 
@@ -84,4 +85,20 @@ export default async function* parseJSONInBatches(binaryAsyncIterator, options)
     yield finalBatch;
   }
 }
+export function rebuildJsonObject(batch, data) {
+  assert(batch.batchType === 'final-result');
+
+  if (batch.jsonpath === '$') {
+    return data;
+  }
+
+  if (batch.jsonpath && batch.jsonpath.length > 1) {
+    const topLevelObject = batch.container;
+    const streamingPath = new JSONPath(batch.jsonpath);
+    streamingPath.setFieldAtPath(topLevelObject, data);
+    return topLevelObject;
+  }
+
+  return batch.container;
+}
 //# sourceMappingURL=parse-json-in-batches.js.map

package/dist/esm/lib/parse-json-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../src/lib/parse-json-in-batches.ts"],"names":["TableBatchBuilder","makeTextDecoderIterator","StreamingJSONParser","parseJSONInBatches","binaryAsyncIterator","options","asyncIterator","metadata","jsonpaths","json","isFirstChunk","schema","shape","tableBatchBuilder","parser","chunk","rows","write","jsonpath","length","getStreamingJsonPathAsString","initialBatch","batchType","data","bytesUsed","container","getPartialResult","row","addRow","batch","getFullBatch","chunkComplete","getFinalBatch","finalBatch"],"mappings":"AAEA,SAAQA,iBAAR,QAAgC,oBAAhC;AACA,SAAQC,uBAAR,QAAsC,0BAAtC;AACA,OAAOC,mBAAP,MAAgC,gCAAhC;AAIA,eAAe,gBAAgBC,kBAAhB,CACbC,mBADa,EAEbC,OAFa,EAGS;AAAA;;AACtB,QAAMC,aAAa,GAAGL,uBAAuB,CAACG,mBAAD,CAA7C;AAEA,QAAM;AAACG,IAAAA;AAAD,MAAaF,OAAnB;AACA,QAAM;AAACG,IAAAA;AAAD,MAAcH,OAAO,CAACI,IAAR,IAAgB,EAApC;AAEA,MAAIC,YAAqB,GAAG,IAA5B;AAGA,QAAMC,MAAM,GAAG,IAAf;AACA,QAAMC,KAAK,GAAG,CAAAP,OAAO,SAAP,IAAAA,OAAO,WAAP,6BAAAA,OAAO,CAAEI,IAAT,gEAAeG,KAAf,KAAwB,WAAtC;AAEA,QAAMC,iBAAiB,GAAG,IAAIb,iBAAJ,CAAsBW,MAAtB,EAA8B,EACtD,GAAGN,OADmD;AAEtDO,IAAAA;AAFsD,GAA9B,CAA1B;AAKA,QAAME,MAAM,GAAG,IAAIZ,mBAAJ,CAAwB;AAACM,IAAAA;AAAD,GAAxB,CAAf;;AAEA,aAAW,MAAMO,KAAjB,IAA0BT,aAA1B,EAAyC;AACvC,UAAMU,IAAI,GAAGF,MAAM,CAACG,KAAP,CAAaF,KAAb,CAAb;AAEA,UAAMG,QAAQ,GAAGF,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBL,MAAM,CAACM,4BAAP,EAApC;;AAEA,QAAIJ,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBT,YAAvB,EAAqC;AACnC,UAAIH,QAAJ,EAAc;AACZ,cAAMc,YAAmB,GAAG;AAE1BT,UAAAA,KAF0B;AAG1BU,UAAAA,SAAS,EAAE,gBAHe;AAI1BC,UAAAA,IAAI,EAAE,EAJoB;AAK1BJ,UAAAA,MAAM,EAAE,CALkB;AAM1BK,UAAAA,SAAS,EAAE,CANe;AAQ1BC,UAAAA,SAAS,EAAEX,MAAM,CAACY,gBAAP,EARe;AAS1BR,UAAAA;AAT0B,SAA5B;AAWA,cAAMG,YAAN;AACD;;AACDX,MAAAA,YAAY,GAAG,KAAf;AAED;;AAGD,SAAK,MAAMiB,GAAX,IAAkBX,IAAlB,EAAwB;AACtBH,MAAAA,iBAAiB,CAACe,MAAlB,CAAyBD,GAAzB;AAEA,YAAME,KAAK,GAAGhB,iBAAiB,CAACiB,YAAlB,CAA+B;AAACZ,QAAAA;AAAD,OAA/B,CAAd;;AACA,UAAIW,KAAJ,EAAW;AACT,cAAMA,KAAN;AACD;AACF;;AAEDhB,IAAAA,iBAAiB,CAACkB,aAAlB,CAAgChB,KAAhC;AACA,UAAMc,KAAK,GAAGhB,iBAAiB,CAACiB,YAAlB,CAA+B;AAACZ,MAAAA;AAAD,KAA/B,CAAd;;AACA,QAAIW,KAAJ,EAAW;AACT,YAAMA,KAAN;AACD;AACF;;AAGD,QAAMX,QAAQ,GAAGJ,MAAM,CAACM,4BAAP,EAAjB;AACA,QAAMS,KAAK,GAAGhB,iBAAiB,CAACmB,aAAlB,CAAgC;AAACd,IAAAA;AAAD,GAAhC,CAAd;;AACA,MAAIW,KAAJ,EAAW;AACT,UAAMA,KAAN;AACD;;AAED,MAAItB,QAAJ,EAAc;AACZ,UAAM0B,UAAiB,GAAG;AACxBrB,MAAAA,KADwB;AAExBU,MAAAA,SAAS,EAAE,cAFa;AAGxBG,MAAAA,SAAS,EAAEX,MAAM,CAACY,gBAAP,EAHa;AAIxBR,MAAAA,QAAQ,EAAEJ,MAAM,CAACM,4BAAP,EAJc;AAKxBG,MAAAA,IAAI,EAAE,EALkB;AAMxBJ,MAAAA,MAAM,EAAE;AANgB,KAA1B;AASA,UAAMc,UAAN;AACD;AACF","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n  const {metadata} = options;\n  const {jsonpaths} = options.json || {};\n\n  let isFirstChunk: boolean = true;\n\n  // TODO fix Schema deduction\n  const schema = null; // new Schema([]);\n  const shape = options?.json?.shape || 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  const parser = new StreamingJSONParser({jsonpaths});\n\n  for await (const chunk of asyncIterator) {\n    const rows = parser.write(chunk);\n\n    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n    if (rows.length > 0 && isFirstChunk) {\n      if (metadata) {\n        const initialBatch: Batch = {\n          // Common fields\n          shape,\n          batchType: 'partial-result',\n          data: [],\n          length: 0,\n          bytesUsed: 0,\n          // JSON additions\n          container: parser.getPartialResult(),\n          jsonpath\n        };\n        yield initialBatch;\n      }\n      isFirstChunk = false;\n      // schema = deduceSchema(rows);\n    }\n\n    // Add the row\n    for (const row of rows) {\n      tableBatchBuilder.addRow(row);\n      // If a batch has been completed, emit it\n      const batch = tableBatchBuilder.getFullBatch({jsonpath});\n      if (batch) {\n        yield batch;\n      }\n    }\n\n    tableBatchBuilder.chunkComplete(chunk);\n    const batch = tableBatchBuilder.getFullBatch({jsonpath});\n    if (batch) {\n      yield batch;\n    }\n  }\n\n  // yield final batch\n  const jsonpath = parser.getStreamingJsonPathAsString();\n  const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n  if (batch) {\n    yield batch;\n  }\n\n  if (metadata) {\n    const finalBatch: Batch = {\n      shape,\n      batchType: 'final-result',\n      container: parser.getPartialResult(),\n      jsonpath: parser.getStreamingJsonPathAsString(),\n      data: [],\n      length: 0\n      // schema: null\n    };\n    yield finalBatch;\n  }\n}\n"],"file":"parse-json-in-batches.js"}
+{"version":3,"sources":["../../../src/lib/parse-json-in-batches.ts"],"names":["TableBatchBuilder","assert","makeTextDecoderIterator","StreamingJSONParser","JSONPath","parseJSONInBatches","binaryAsyncIterator","options","asyncIterator","metadata","jsonpaths","json","isFirstChunk","schema","shape","tableBatchBuilder","parser","chunk","rows","write","jsonpath","length","getStreamingJsonPathAsString","initialBatch","batchType","data","bytesUsed","container","getPartialResult","row","addRow","batch","getFullBatch","chunkComplete","getFinalBatch","finalBatch","rebuildJsonObject","topLevelObject","streamingPath","setFieldAtPath"],"mappings":"AAEA,SAAQA,iBAAR,QAAgC,oBAAhC;AACA,SAAQC,MAAR,EAAgBC,uBAAhB,QAA8C,0BAA9C;AACA,OAAOC,mBAAP,MAAgC,gCAAhC;AACA,OAAOC,QAAP,MAAqB,qBAArB;AAIA,eAAe,gBAAgBC,kBAAhB,CACbC,mBADa,EAEbC,OAFa,EAGS;AAAA;;AACtB,QAAMC,aAAa,GAAGN,uBAAuB,CAACI,mBAAD,CAA7C;AAEA,QAAM;AAACG,IAAAA;AAAD,MAAaF,OAAnB;AACA,QAAM;AAACG,IAAAA;AAAD,MAAcH,OAAO,CAACI,IAAR,IAAgB,EAApC;AAEA,MAAIC,YAAqB,GAAG,IAA5B;AAGA,QAAMC,MAAM,GAAG,IAAf;AACA,QAAMC,KAAK,GAAG,CAAAP,OAAO,SAAP,IAAAA,OAAO,WAAP,6BAAAA,OAAO,CAAEI,IAAT,gEAAeG,KAAf,KAAwB,WAAtC;AAEA,QAAMC,iBAAiB,GAAG,IAAIf,iBAAJ,CAAsBa,MAAtB,EAA8B,EACtD,GAAGN,OADmD;AAEtDO,IAAAA;AAFsD,GAA9B,CAA1B;AAKA,QAAME,MAAM,GAAG,IAAIb,mBAAJ,CAAwB;AAACO,IAAAA;AAAD,GAAxB,CAAf;;AAEA,aAAW,MAAMO,KAAjB,IAA0BT,aAA1B,EAAyC;AACvC,UAAMU,IAAI,GAAGF,MAAM,CAACG,KAAP,CAAaF,KAAb,CAAb;AAEA,UAAMG,QAAQ,GAAGF,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBL,MAAM,CAACM,4BAAP,EAApC;;AAEA,QAAIJ,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBT,YAAvB,EAAqC;AACnC,UAAIH,QAAJ,EAAc;AACZ,cAAMc,YAAmB,GAAG;AAE1BT,UAAAA,KAF0B;AAG1BU,UAAAA,SAAS,EAAE,gBAHe;AAI1BC,UAAAA,IAAI,EAAE,EAJoB;AAK1BJ,UAAAA,MAAM,EAAE,CALkB;AAM1BK,UAAAA,SAAS,EAAE,CANe;AAQ1BC,UAAAA,SAAS,EAAEX,MAAM,CAACY,gBAAP,EARe;AAS1BR,UAAAA;AAT0B,SAA5B;AAWA,cAAMG,YAAN;AACD;;AACDX,MAAAA,YAAY,GAAG,KAAf;AAED;;AAGD,SAAK,MAAMiB,GAAX,IAAkBX,IAAlB,EAAwB;AACtBH,MAAAA,iBAAiB,CAACe,MAAlB,CAAyBD,GAAzB;AAEA,YAAME,KAAK,GAAGhB,iBAAiB,CAACiB,YAAlB,CAA+B;AAACZ,QAAAA;AAAD,OAA/B,CAAd;;AACA,UAAIW,KAAJ,EAAW;AACT,cAAMA,KAAN;AACD;AACF;;AAEDhB,IAAAA,iBAAiB,CAACkB,aAAlB,CAAgChB,KAAhC;AACA,UAAMc,KAAK,GAAGhB,iBAAiB,CAACiB,YAAlB,CAA+B;AAACZ,MAAAA;AAAD,KAA/B,CAAd;;AACA,QAAIW,KAAJ,EAAW;AACT,YAAMA,KAAN;AACD;AACF;;AAGD,QAAMX,QAAQ,GAAGJ,MAAM,CAACM,4BAAP,EAAjB;AACA,QAAMS,KAAK,GAAGhB,iBAAiB,CAACmB,aAAlB,CAAgC;AAACd,IAAAA;AAAD,GAAhC,CAAd;;AACA,MAAIW,KAAJ,EAAW;AACT,UAAMA,KAAN;AACD;;AAED,MAAItB,QAAJ,EAAc;AACZ,UAAM0B,UAAiB,GAAG;AACxBrB,MAAAA,KADwB;AAExBU,MAAAA,SAAS,EAAE,cAFa;AAGxBG,MAAAA,SAAS,EAAEX,MAAM,CAACY,gBAAP,EAHa;AAIxBR,MAAAA,QAAQ,EAAEJ,MAAM,CAACM,4BAAP,EAJc;AAKxBG,MAAAA,IAAI,EAAE,EALkB;AAMxBJ,MAAAA,MAAM,EAAE;AANgB,KAA1B;AASA,UAAMc,UAAN;AACD;AACF;AAED,OAAO,SAASC,iBAAT,CAA2BL,KAA3B,EAAkCN,IAAlC,EAAwC;AAE7CxB,EAAAA,MAAM,CAAC8B,KAAK,CAACP,SAAN,KAAoB,cAArB,CAAN;;AAGA,MAAIO,KAAK,CAACX,QAAN,KAAmB,GAAvB,EAA4B;AAC1B,WAAOK,IAAP;AACD;;AAGD,MAAIM,KAAK,CAACX,QAAN,IAAkBW,KAAK,CAACX,QAAN,CAAeC,MAAf,GAAwB,CAA9C,EAAiD;AAC/C,UAAMgB,cAAc,GAAGN,KAAK,CAACJ,SAA7B;AACA,UAAMW,aAAa,GAAG,IAAIlC,QAAJ,CAAa2B,KAAK,CAACX,QAAnB,CAAtB;AACAkB,IAAAA,aAAa,CAACC,cAAd,CAA6BF,cAA7B,EAA6CZ,IAA7C;AACA,WAAOY,cAAP;AACD;;AAGD,SAAON,KAAK,CAACJ,SAAb;AACD","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {assert, makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\nimport JSONPath from './jsonpath/jsonpath';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n  const {metadata} = options;\n  const {jsonpaths} = options.json || {};\n\n  let isFirstChunk: boolean = true;\n\n  // TODO fix Schema deduction\n  const schema = null; // new Schema([]);\n  const shape = options?.json?.shape || 'row-table';\n  // @ts-ignore\n  const tableBatchBuilder = new TableBatchBuilder(schema, {\n    ...options,\n    shape\n  });\n\n  const parser = new StreamingJSONParser({jsonpaths});\n\n  for await (const chunk of asyncIterator) {\n    const rows = parser.write(chunk);\n\n    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n    if (rows.length > 0 && isFirstChunk) {\n      if (metadata) {\n        const initialBatch: Batch = {\n          // Common fields\n          shape,\n          batchType: 'partial-result',\n          data: [],\n          length: 0,\n          bytesUsed: 0,\n          // JSON additions\n          container: parser.getPartialResult(),\n          jsonpath\n        };\n        yield initialBatch;\n      }\n      isFirstChunk = false;\n      // schema = deduceSchema(rows);\n    }\n\n    // Add the row\n    for (const row of rows) {\n      tableBatchBuilder.addRow(row);\n      // If a batch has been completed, emit it\n      const batch = tableBatchBuilder.getFullBatch({jsonpath});\n      if (batch) {\n        yield batch;\n      }\n    }\n\n    tableBatchBuilder.chunkComplete(chunk);\n    const batch = tableBatchBuilder.getFullBatch({jsonpath});\n    if (batch) {\n      yield batch;\n    }\n  }\n\n  // yield final batch\n  const jsonpath = parser.getStreamingJsonPathAsString();\n  const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n  if (batch) {\n    yield batch;\n  }\n\n  if (metadata) {\n    const finalBatch: Batch = {\n      shape,\n      batchType: 'final-result',\n      container: parser.getPartialResult(),\n      jsonpath: parser.getStreamingJsonPathAsString(),\n      data: [],\n      length: 0\n      // schema: null\n    };\n    yield finalBatch;\n  }\n}\n\nexport function rebuildJsonObject(batch, data) {\n  // Last batch will have this special type and will provide all the root object of the parsed file\n  assert(batch.batchType === 'final-result');\n\n  // The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects\n  if (batch.jsonpath === '$') {\n    return data;\n  }\n\n  // (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object\n  if (batch.jsonpath && batch.jsonpath.length > 1) {\n    const topLevelObject = batch.container;\n    const streamingPath = new JSONPath(batch.jsonpath);\n    streamingPath.setFieldAtPath(topLevelObject, data);\n    return topLevelObject;\n  }\n\n  // No jsonpath, in this case nothing was streamed.\n  return batch.container;\n}\n"],"file":"parse-json-in-batches.js"}

package/dist/esm/ndjson-loader.js

@@ -1,6 +1,6 @@
 import parseNDJSONSync from './lib/parse-ndjson';
 import parseNDJSONInBatches from './lib/parse-ndjson-in-batches';
-const VERSION = typeof "3.1.0-beta.3" !== 'undefined' ? "3.1.0-beta.3" : 'latest';
+const VERSION = typeof "3.1.1" !== 'undefined' ? "3.1.1" : 'latest';
 export const NDJSONLoader = {
   name: 'NDJSON',
   id: 'ndjson',

package/dist/esm/ndjson-loader.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/ndjson-loader.ts"],"names":["parseNDJSONSync","parseNDJSONInBatches","VERSION","NDJSONLoader","name","id","module","version","extensions","mimeTypes","category","text","parse","parseTextSync","parseInBatches","options","arrayBuffer","TextDecoder","decode","asyncIterator","_typecheckNDJSONLoader"],"mappings":"AAEA,OAAOA,eAAP,MAA4B,oBAA5B;AACA,OAAOC,oBAAP,MAAiC,+BAAjC;AAIA,MAAMC,OAAO,GAAG,0BAAuB,WAAvB,oBAAmD,QAAnE;AAEA,OAAO,MAAMC,YAAY,GAAG;AAC1BC,EAAAA,IAAI,EAAE,QADoB;AAE1BC,EAAAA,EAAE,EAAE,QAFsB;AAG1BC,EAAAA,MAAM,EAAE,MAHkB;AAI1BC,EAAAA,OAAO,EAAEL,OAJiB;AAK1BM,EAAAA,UAAU,EAAE,CAAC,QAAD,CALc;AAM1BC,EAAAA,SAAS,EAAE,CAAC,sBAAD,CANe;AAO1BC,EAAAA,QAAQ,EAAE,OAPgB;AAQ1BC,EAAAA,IAAI,EAAE,IARoB;AAS1BC,EAAAA,KAT0B;AAU1BC,EAAAA,aAV0B;AAW1BC,EAAAA,cAX0B;AAY1BC,EAAAA,OAAO,EAAE;AAZiB,CAArB;;AAeP,eAAeH,KAAf,CAAqBI,WAArB,EAA+C;AAC7C,SAAOH,aAAa,CAAC,IAAII,WAAJ,GAAkBC,MAAlB,CAAyBF,WAAzB,CAAD,CAApB;AACD;;AAED,SAASH,aAAT,CAAuBF,IAAvB,EAAqC;AACnC,SAAOX,eAAe,CAACW,IAAD,CAAtB;AACD;;AAED,SAASG,cAAT,CACEK,aADF,EAEEJ,OAFF,EAGwB;AACtB,SAAOd,oBAAoB,CAACkB,aAAD,EAAgBJ,OAAhB,CAA3B;AACD;;AAED,OAAO,MAAMK,sBAAwC,GAAGjB,YAAjD","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport parseNDJSONSync from './lib/parse-ndjson';\nimport parseNDJSONInBatches from './lib/parse-ndjson-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport const NDJSONLoader = {\n  name: 'NDJSON',\n  id: 'ndjson',\n  module: 'json',\n  version: VERSION,\n  extensions: ['ndjson'],\n  mimeTypes: ['application/x-ndjson'],\n  category: 'table',\n  text: true,\n  parse,\n  parseTextSync,\n  parseInBatches,\n  options: {}\n};\n\nasync function parse(arrayBuffer: ArrayBuffer) {\n  return parseTextSync(new TextDecoder().decode(arrayBuffer));\n}\n\nfunction parseTextSync(text: string) {\n  return parseNDJSONSync(text);\n}\n\nfunction parseInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: LoaderOptions\n): AsyncIterable<Batch> {\n  return parseNDJSONInBatches(asyncIterator, options);\n}\n\nexport const _typecheckNDJSONLoader: LoaderWithParser = NDJSONLoader;\n"],"file":"ndjson-loader.js"}
+{"version":3,"sources":["../../src/ndjson-loader.ts"],"names":["parseNDJSONSync","parseNDJSONInBatches","VERSION","NDJSONLoader","name","id","module","version","extensions","mimeTypes","category","text","parse","parseTextSync","parseInBatches","options","arrayBuffer","TextDecoder","decode","asyncIterator","_typecheckNDJSONLoader"],"mappings":"AAEA,OAAOA,eAAP,MAA4B,oBAA5B;AACA,OAAOC,oBAAP,MAAiC,+BAAjC;AAIA,MAAMC,OAAO,GAAG,mBAAuB,WAAvB,aAAmD,QAAnE;AAEA,OAAO,MAAMC,YAAY,GAAG;AAC1BC,EAAAA,IAAI,EAAE,QADoB;AAE1BC,EAAAA,EAAE,EAAE,QAFsB;AAG1BC,EAAAA,MAAM,EAAE,MAHkB;AAI1BC,EAAAA,OAAO,EAAEL,OAJiB;AAK1BM,EAAAA,UAAU,EAAE,CAAC,QAAD,CALc;AAM1BC,EAAAA,SAAS,EAAE,CAAC,sBAAD,CANe;AAO1BC,EAAAA,QAAQ,EAAE,OAPgB;AAQ1BC,EAAAA,IAAI,EAAE,IARoB;AAS1BC,EAAAA,KAT0B;AAU1BC,EAAAA,aAV0B;AAW1BC,EAAAA,cAX0B;AAY1BC,EAAAA,OAAO,EAAE;AAZiB,CAArB;;AAeP,eAAeH,KAAf,CAAqBI,WAArB,EAA+C;AAC7C,SAAOH,aAAa,CAAC,IAAII,WAAJ,GAAkBC,MAAlB,CAAyBF,WAAzB,CAAD,CAApB;AACD;;AAED,SAASH,aAAT,CAAuBF,IAAvB,EAAqC;AACnC,SAAOX,eAAe,CAACW,IAAD,CAAtB;AACD;;AAED,SAASG,cAAT,CACEK,aADF,EAEEJ,OAFF,EAGwB;AACtB,SAAOd,oBAAoB,CAACkB,aAAD,EAAgBJ,OAAhB,CAA3B;AACD;;AAED,OAAO,MAAMK,sBAAwC,GAAGjB,YAAjD","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport parseNDJSONSync from './lib/parse-ndjson';\nimport parseNDJSONInBatches from './lib/parse-ndjson-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport const NDJSONLoader = {\n  name: 'NDJSON',\n  id: 'ndjson',\n  module: 'json',\n  version: VERSION,\n  extensions: ['ndjson'],\n  mimeTypes: ['application/x-ndjson'],\n  category: 'table',\n  text: true,\n  parse,\n  parseTextSync,\n  parseInBatches,\n  options: {}\n};\n\nasync function parse(arrayBuffer: ArrayBuffer) {\n  return parseTextSync(new TextDecoder().decode(arrayBuffer));\n}\n\nfunction parseTextSync(text: string) {\n  return parseNDJSONSync(text);\n}\n\nfunction parseInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: LoaderOptions\n): AsyncIterable<Batch> {\n  return parseNDJSONInBatches(asyncIterator, options);\n}\n\nexport const _typecheckNDJSONLoader: LoaderWithParser = NDJSONLoader;\n"],"file":"ndjson-loader.js"}