@loaders.gl/json 3.1.3 → 4.0.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.js +2 -2
- package/dist/bundle.js.map +1 -0
- package/dist/geojson-loader.js +69 -66
- package/dist/geojson-loader.js.map +1 -0
- package/dist/geojson-worker.js +1 -1
- package/dist/index.js +7 -19
- package/dist/index.js.map +1 -0
- package/dist/json-loader.js +38 -48
- package/dist/json-loader.js.map +1 -0
- package/dist/jsonl-loader.js +1 -53
- package/dist/{es5/jsonl-loader.js.map → jsonl-loader.js.map} +0 -0
- package/dist/{es5/lib → lib}/clarinet/LICENSE +0 -0
- package/dist/lib/clarinet/clarinet.js +493 -510
- package/dist/lib/clarinet/clarinet.js.map +1 -0
- package/dist/lib/jsonpath/jsonpath.js +78 -81
- package/dist/lib/jsonpath/jsonpath.js.map +1 -0
- package/dist/lib/parse-json-in-batches.js +98 -94
- package/dist/lib/parse-json-in-batches.js.map +1 -0
- package/dist/lib/parse-json.js +29 -24
- package/dist/lib/parse-json.js.map +1 -0
- package/dist/lib/parse-ndjson-in-batches.js +34 -32
- package/dist/lib/parse-ndjson-in-batches.js.map +1 -0
- package/dist/lib/parse-ndjson.js +10 -13
- package/dist/lib/parse-ndjson.js.map +1 -0
- package/dist/lib/parser/json-parser.js +125 -95
- package/dist/lib/parser/json-parser.js.map +1 -0
- package/dist/lib/parser/streaming-json-parser.js +88 -95
- package/dist/lib/parser/streaming-json-parser.js.map +1 -0
- package/dist/ndjson-loader.js +25 -28
- package/dist/ndjson-loader.js.map +1 -0
- package/dist/workers/geojson-worker.js +4 -5
- package/dist/workers/geojson-worker.js.map +1 -0
- package/package.json +8 -8
- package/dist/es5/bundle.js +0 -7
- package/dist/es5/bundle.js.map +0 -1
- package/dist/es5/geojson-loader.js +0 -209
- package/dist/es5/geojson-loader.js.map +0 -1
- package/dist/es5/index.js +0 -62
- package/dist/es5/index.js.map +0 -1
- package/dist/es5/json-loader.js +0 -85
- package/dist/es5/json-loader.js.map +0 -1
- package/dist/es5/jsonl-loader.js +0 -2
- package/dist/es5/lib/clarinet/clarinet.js +0 -535
- package/dist/es5/lib/clarinet/clarinet.js.map +0 -1
- package/dist/es5/lib/jsonpath/jsonpath.js +0 -144
- package/dist/es5/lib/jsonpath/jsonpath.js.map +0 -1
- package/dist/es5/lib/parse-json-in-batches.js +0 -278
- package/dist/es5/lib/parse-json-in-batches.js.map +0 -1
- package/dist/es5/lib/parse-json.js +0 -46
- package/dist/es5/lib/parse-json.js.map +0 -1
- package/dist/es5/lib/parse-ndjson-in-batches.js +0 -158
- package/dist/es5/lib/parse-ndjson-in-batches.js.map +0 -1
- package/dist/es5/lib/parse-ndjson.js +0 -18
- package/dist/es5/lib/parse-ndjson.js.map +0 -1
- package/dist/es5/lib/parser/json-parser.js +0 -159
- package/dist/es5/lib/parser/json-parser.js.map +0 -1
- package/dist/es5/lib/parser/streaming-json-parser.js +0 -158
- package/dist/es5/lib/parser/streaming-json-parser.js.map +0 -1
- package/dist/es5/ndjson-loader.js +0 -67
- package/dist/es5/ndjson-loader.js.map +0 -1
- package/dist/es5/workers/geojson-worker.js +0 -8
- package/dist/es5/workers/geojson-worker.js.map +0 -1
- package/dist/esm/bundle.js +0 -5
- package/dist/esm/bundle.js.map +0 -1
- package/dist/esm/geojson-loader.js +0 -81
- package/dist/esm/geojson-loader.js.map +0 -1
- package/dist/esm/index.js +0 -7
- package/dist/esm/index.js.map +0 -1
- package/dist/esm/json-loader.js +0 -47
- package/dist/esm/json-loader.js.map +0 -1
- package/dist/esm/jsonl-loader.js +0 -2
- package/dist/esm/jsonl-loader.js.map +0 -1
- package/dist/esm/lib/clarinet/LICENSE +0 -28
- package/dist/esm/lib/clarinet/clarinet.js +0 -518
- package/dist/esm/lib/clarinet/clarinet.js.map +0 -1
- package/dist/esm/lib/jsonpath/jsonpath.js +0 -86
- package/dist/esm/lib/jsonpath/jsonpath.js.map +0 -1
- package/dist/esm/lib/parse-json-in-batches.js +0 -104
- package/dist/esm/lib/parse-json-in-batches.js.map +0 -1
- package/dist/esm/lib/parse-json.js +0 -34
- package/dist/esm/lib/parse-json.js.map +0 -1
- package/dist/esm/lib/parse-ndjson-in-batches.js +0 -37
- package/dist/esm/lib/parse-ndjson-in-batches.js.map +0 -1
- package/dist/esm/lib/parse-ndjson.js +0 -11
- package/dist/esm/lib/parse-ndjson.js.map +0 -1
- package/dist/esm/lib/parser/json-parser.js +0 -128
- package/dist/esm/lib/parser/json-parser.js.map +0 -1
- package/dist/esm/lib/parser/streaming-json-parser.js +0 -93
- package/dist/esm/lib/parser/streaming-json-parser.js.map +0 -1
- package/dist/esm/ndjson-loader.js +0 -32
- package/dist/esm/ndjson-loader.js.map +0 -1
- package/dist/esm/workers/geojson-worker.js +0 -4
- package/dist/esm/workers/geojson-worker.js.map +0 -1
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
|
-
Object.defineProperty(exports, "__esModule", {
|
|
6
|
-
value: true
|
|
7
|
-
});
|
|
8
|
-
exports.default = void 0;
|
|
9
|
-
|
|
10
|
-
var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
|
|
11
|
-
|
|
12
|
-
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
|
|
13
|
-
|
|
14
|
-
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
|
|
15
|
-
|
|
16
|
-
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
17
|
-
|
|
18
|
-
function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
|
|
19
|
-
|
|
20
|
-
function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
|
|
21
|
-
|
|
22
|
-
function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
|
|
23
|
-
|
|
24
|
-
var JSONPath = function () {
|
|
25
|
-
function JSONPath() {
|
|
26
|
-
var path = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : null;
|
|
27
|
-
(0, _classCallCheck2.default)(this, JSONPath);
|
|
28
|
-
(0, _defineProperty2.default)(this, "path", void 0);
|
|
29
|
-
this.path = ['$'];
|
|
30
|
-
|
|
31
|
-
if (path instanceof JSONPath) {
|
|
32
|
-
this.path = (0, _toConsumableArray2.default)(path.path);
|
|
33
|
-
return;
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
if (Array.isArray(path)) {
|
|
37
|
-
var _this$path;
|
|
38
|
-
|
|
39
|
-
(_this$path = this.path).push.apply(_this$path, (0, _toConsumableArray2.default)(path));
|
|
40
|
-
|
|
41
|
-
return;
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
if (typeof path === 'string') {
|
|
45
|
-
this.path = path.split('.');
|
|
46
|
-
|
|
47
|
-
if (this.path[0] !== '$') {
|
|
48
|
-
throw new Error('JSONPaths must start with $');
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
(0, _createClass2.default)(JSONPath, [{
|
|
54
|
-
key: "clone",
|
|
55
|
-
value: function clone() {
|
|
56
|
-
return new JSONPath(this);
|
|
57
|
-
}
|
|
58
|
-
}, {
|
|
59
|
-
key: "toString",
|
|
60
|
-
value: function toString() {
|
|
61
|
-
return this.path.join('.');
|
|
62
|
-
}
|
|
63
|
-
}, {
|
|
64
|
-
key: "push",
|
|
65
|
-
value: function push(name) {
|
|
66
|
-
this.path.push(name);
|
|
67
|
-
}
|
|
68
|
-
}, {
|
|
69
|
-
key: "pop",
|
|
70
|
-
value: function pop() {
|
|
71
|
-
return this.path.pop();
|
|
72
|
-
}
|
|
73
|
-
}, {
|
|
74
|
-
key: "set",
|
|
75
|
-
value: function set(name) {
|
|
76
|
-
this.path[this.path.length - 1] = name;
|
|
77
|
-
}
|
|
78
|
-
}, {
|
|
79
|
-
key: "equals",
|
|
80
|
-
value: function equals(other) {
|
|
81
|
-
if (!this || !other || this.path.length !== other.path.length) {
|
|
82
|
-
return false;
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
for (var i = 0; i < this.path.length; ++i) {
|
|
86
|
-
if (this.path[i] !== other.path[i]) {
|
|
87
|
-
return false;
|
|
88
|
-
}
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
return true;
|
|
92
|
-
}
|
|
93
|
-
}, {
|
|
94
|
-
key: "setFieldAtPath",
|
|
95
|
-
value: function setFieldAtPath(object, value) {
|
|
96
|
-
var path = (0, _toConsumableArray2.default)(this.path);
|
|
97
|
-
path.shift();
|
|
98
|
-
var field = path.pop();
|
|
99
|
-
|
|
100
|
-
var _iterator = _createForOfIteratorHelper(path),
|
|
101
|
-
_step;
|
|
102
|
-
|
|
103
|
-
try {
|
|
104
|
-
for (_iterator.s(); !(_step = _iterator.n()).done;) {
|
|
105
|
-
var component = _step.value;
|
|
106
|
-
object = object[component];
|
|
107
|
-
}
|
|
108
|
-
} catch (err) {
|
|
109
|
-
_iterator.e(err);
|
|
110
|
-
} finally {
|
|
111
|
-
_iterator.f();
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
object[field] = value;
|
|
115
|
-
}
|
|
116
|
-
}, {
|
|
117
|
-
key: "getFieldAtPath",
|
|
118
|
-
value: function getFieldAtPath(object) {
|
|
119
|
-
var path = (0, _toConsumableArray2.default)(this.path);
|
|
120
|
-
path.shift();
|
|
121
|
-
var field = path.pop();
|
|
122
|
-
|
|
123
|
-
var _iterator2 = _createForOfIteratorHelper(path),
|
|
124
|
-
_step2;
|
|
125
|
-
|
|
126
|
-
try {
|
|
127
|
-
for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
|
|
128
|
-
var component = _step2.value;
|
|
129
|
-
object = object[component];
|
|
130
|
-
}
|
|
131
|
-
} catch (err) {
|
|
132
|
-
_iterator2.e(err);
|
|
133
|
-
} finally {
|
|
134
|
-
_iterator2.f();
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
return object[field];
|
|
138
|
-
}
|
|
139
|
-
}]);
|
|
140
|
-
return JSONPath;
|
|
141
|
-
}();
|
|
142
|
-
|
|
143
|
-
exports.default = JSONPath;
|
|
144
|
-
//# sourceMappingURL=jsonpath.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../../src/lib/jsonpath/jsonpath.ts"],"names":["JSONPath","path","Array","isArray","push","split","Error","join","name","pop","length","other","i","object","value","shift","field","component"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;IAQqBA,Q;AAGnB,sBAA8D;AAAA,QAAlDC,IAAkD,uEAAN,IAAM;AAAA;AAAA;AAC5D,SAAKA,IAAL,GAAY,CAAC,GAAD,CAAZ;;AAEA,QAAIA,IAAI,YAAYD,QAApB,EAA8B;AAE5B,WAAKC,IAAL,oCAAgBA,IAAI,CAACA,IAArB;AACA;AACD;;AAED,QAAIC,KAAK,CAACC,OAAN,CAAcF,IAAd,CAAJ,EAAyB;AAAA;;AACvB,yBAAKA,IAAL,EAAUG,IAAV,oDAAkBH,IAAlB;;AACA;AACD;;AAGD,QAAI,OAAOA,IAAP,KAAgB,QAApB,EAA8B;AAC5B,WAAKA,IAAL,GAAYA,IAAI,CAACI,KAAL,CAAW,GAAX,CAAZ;;AACA,UAAI,KAAKJ,IAAL,CAAU,CAAV,MAAiB,GAArB,EAA0B;AACxB,cAAM,IAAIK,KAAJ,CAAU,6BAAV,CAAN;AACD;AACF;AACF;;;;WAED,iBAAkB;AAChB,aAAO,IAAIN,QAAJ,CAAa,IAAb,CAAP;AACD;;;WAED,oBAAmB;AACjB,aAAO,KAAKC,IAAL,CAAUM,IAAV,CAAe,GAAf,CAAP;AACD;;;WAED,cAAKC,IAAL,EAAyB;AACvB,WAAKP,IAAL,CAAUG,IAAV,CAAeI,IAAf;AACD;;;WAED,eAAM;AACJ,aAAO,KAAKP,IAAL,CAAUQ,GAAV,EAAP;AACD;;;WAED,aAAID,IAAJ,EAAwB;AACtB,WAAKP,IAAL,CAAU,KAAKA,IAAL,CAAUS,MAAV,GAAmB,CAA7B,IAAkCF,IAAlC;AACD;;;WAED,gBAAOG,KAAP,EAAiC;AAC/B,UAAI,CAAC,IAAD,IAAS,CAACA,KAAV,IAAmB,KAAKV,IAAL,CAAUS,MAAV,KAAqBC,KAAK,CAACV,IAAN,CAAWS,MAAvD,EAA+D;AAC7D,eAAO,KAAP;AACD;;AAED,WAAK,IAAIE,CAAC,GAAG,CAAb,EAAgBA,CAAC,GAAG,KAAKX,IAAL,CAAUS,MAA9B,EAAsC,EAAEE,CAAxC,EAA2C;AACzC,YAAI,KAAKX,IAAL,CAAUW,CAAV,MAAiBD,KAAK,CAACV,IAAN,CAAWW,CAAX,CAArB,EAAoC;AAClC,iBAAO,KAAP;AACD;AACF;;AAED,aAAO,IAAP;AACD;;;WAQD,wBAAeC,MAAf,EAAuBC,KAAvB,EAA8B;AAC5B,UAAMb,IAAI,oCAAO,KAAKA,IAAZ,CAAV;AACAA,MAAAA,IAAI,CAACc,KAAL;AACA,UAAMC,KAAK,GAAGf,IAAI,CAACQ,GAAL,EAAd;;AAH4B,iDAIJR,IAJI;AAAA;;AAAA;AAI5B,4DAA8B;AAAA,cAAnBgB,SAAmB;AAC5BJ,UAAAA,MAAM,GAAGA,MAAM,CAACI,SAAD,CAAf;AACD;AAN2B;AAAA;AAAA;AAAA;AAAA;;AAQ5BJ,MAAAA,MAAM,CAACG,KAAD,CAAN,GAAgBF,KAAhB;AACD;;;WAOD,wBAAeD,MAAf,EAAuB;AACrB,UAAMZ,IAAI,oCAAO,KAAKA,IAAZ,CAAV;AACAA,MAAAA,IAAI,CAACc,KAAL;AACA,UAAMC,KAAK,GAAGf,IAAI,CAACQ,GAAL,EAAd;;AAHqB,kDAIGR,IAJH;AAAA;;AAAA;AAIrB,+DAA8B;AAAA,cAAnBgB,SAAmB;AAC5BJ,UAAAA,MAAM,GAAGA,MAAM,CAACI,SAAD,CAAf;AACD;AANoB;AAAA;AAAA;AAAA;AAAA;;AAQrB,aAAOJ,MAAM,CAACG,KAAD,CAAb;AACD","sourcesContent":["/**\n * A parser for a minimal subset of the jsonpath standard\n * Full JSON path parsers for JS exist but are quite large (bundle size)\n *\n * Supports\n *\n * `$.component.component.component`\n */\nexport default class JSONPath {\n path: string[];\n\n constructor(path: JSONPath | string[] | string | null = null) {\n this.path = ['$'];\n\n if (path instanceof JSONPath) {\n // @ts-ignore\n this.path = [...path.path];\n return;\n }\n\n if (Array.isArray(path)) {\n this.path.push(...path);\n return;\n }\n\n // Parse a string as a JSONPath\n if (typeof path === 'string') {\n this.path = path.split('.');\n if (this.path[0] !== '$') {\n throw new Error('JSONPaths must start with $');\n }\n }\n }\n\n clone(): JSONPath {\n return new JSONPath(this);\n }\n\n toString(): string {\n return this.path.join('.');\n }\n\n push(name: string): void {\n this.path.push(name);\n }\n\n pop() {\n return this.path.pop();\n }\n\n set(name: string): void {\n this.path[this.path.length - 1] = name;\n }\n\n equals(other: JSONPath): boolean {\n if (!this || !other || this.path.length !== other.path.length) {\n return false;\n }\n\n for (let i = 0; i < this.path.length; ++i) {\n if (this.path[i] !== other.path[i]) {\n return false;\n }\n }\n\n return true;\n }\n\n /**\n * Sets the value pointed at by path\n * TODO - handle root path\n * @param object\n * @param value\n */\n setFieldAtPath(object, value) {\n const path = [...this.path];\n path.shift();\n const field = path.pop();\n for (const component of path) {\n object = object[component];\n }\n // @ts-ignore\n object[field] = value;\n }\n\n /**\n * Gets the value pointed at by path\n * TODO - handle root path\n * @param object\n */\n getFieldAtPath(object) {\n const path = [...this.path];\n path.shift();\n const field = path.pop();\n for (const component of path) {\n object = object[component];\n }\n // @ts-ignore\n return object[field];\n }\n}\n"],"file":"jsonpath.js"}
|
|
@@ -1,278 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
|
-
Object.defineProperty(exports, "__esModule", {
|
|
6
|
-
value: true
|
|
7
|
-
});
|
|
8
|
-
exports.default = parseJSONInBatches;
|
|
9
|
-
exports.rebuildJsonObject = rebuildJsonObject;
|
|
10
|
-
|
|
11
|
-
var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
|
|
12
|
-
|
|
13
|
-
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
14
|
-
|
|
15
|
-
var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
|
|
16
|
-
|
|
17
|
-
var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
|
|
18
|
-
|
|
19
|
-
var _asyncIterator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncIterator"));
|
|
20
|
-
|
|
21
|
-
var _schema = require("@loaders.gl/schema");
|
|
22
|
-
|
|
23
|
-
var _loaderUtils = require("@loaders.gl/loader-utils");
|
|
24
|
-
|
|
25
|
-
var _streamingJsonParser = _interopRequireDefault(require("./parser/streaming-json-parser"));
|
|
26
|
-
|
|
27
|
-
var _jsonpath2 = _interopRequireDefault(require("./jsonpath/jsonpath"));
|
|
28
|
-
|
|
29
|
-
function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
|
|
30
|
-
|
|
31
|
-
function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
|
|
32
|
-
|
|
33
|
-
function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
|
|
34
|
-
|
|
35
|
-
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
|
|
36
|
-
|
|
37
|
-
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
|
|
38
|
-
|
|
39
|
-
function parseJSONInBatches(_x, _x2) {
|
|
40
|
-
return _parseJSONInBatches.apply(this, arguments);
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
function _parseJSONInBatches() {
|
|
44
|
-
_parseJSONInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(binaryAsyncIterator, options) {
|
|
45
|
-
var _options$json;
|
|
46
|
-
|
|
47
|
-
var asyncIterator, metadata, _ref, jsonpaths, isFirstChunk, schema, shape, tableBatchBuilder, parser, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _value, chunk, rows, _jsonpath, initialBatch, _iterator2, _step2, row, _batch2, _batch, jsonpath, batch, finalBatch;
|
|
48
|
-
|
|
49
|
-
return _regenerator.default.wrap(function _callee$(_context) {
|
|
50
|
-
while (1) {
|
|
51
|
-
switch (_context.prev = _context.next) {
|
|
52
|
-
case 0:
|
|
53
|
-
asyncIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
|
|
54
|
-
metadata = options.metadata;
|
|
55
|
-
_ref = options.json || {}, jsonpaths = _ref.jsonpaths;
|
|
56
|
-
isFirstChunk = true;
|
|
57
|
-
schema = null;
|
|
58
|
-
shape = (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'row-table';
|
|
59
|
-
tableBatchBuilder = new _schema.TableBatchBuilder(schema, _objectSpread(_objectSpread({}, options), {}, {
|
|
60
|
-
shape: shape
|
|
61
|
-
}));
|
|
62
|
-
parser = new _streamingJsonParser.default({
|
|
63
|
-
jsonpaths: jsonpaths
|
|
64
|
-
});
|
|
65
|
-
_iteratorNormalCompletion = true;
|
|
66
|
-
_didIteratorError = false;
|
|
67
|
-
_context.prev = 10;
|
|
68
|
-
_iterator = (0, _asyncIterator2.default)(asyncIterator);
|
|
69
|
-
|
|
70
|
-
case 12:
|
|
71
|
-
_context.next = 14;
|
|
72
|
-
return (0, _awaitAsyncGenerator2.default)(_iterator.next());
|
|
73
|
-
|
|
74
|
-
case 14:
|
|
75
|
-
_step = _context.sent;
|
|
76
|
-
_iteratorNormalCompletion = _step.done;
|
|
77
|
-
_context.next = 18;
|
|
78
|
-
return (0, _awaitAsyncGenerator2.default)(_step.value);
|
|
79
|
-
|
|
80
|
-
case 18:
|
|
81
|
-
_value = _context.sent;
|
|
82
|
-
|
|
83
|
-
if (_iteratorNormalCompletion) {
|
|
84
|
-
_context.next = 57;
|
|
85
|
-
break;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
chunk = _value;
|
|
89
|
-
rows = parser.write(chunk);
|
|
90
|
-
_jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
|
|
91
|
-
|
|
92
|
-
if (!(rows.length > 0 && isFirstChunk)) {
|
|
93
|
-
_context.next = 29;
|
|
94
|
-
break;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
if (!metadata) {
|
|
98
|
-
_context.next = 28;
|
|
99
|
-
break;
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
initialBatch = {
|
|
103
|
-
shape: shape,
|
|
104
|
-
batchType: 'partial-result',
|
|
105
|
-
data: [],
|
|
106
|
-
length: 0,
|
|
107
|
-
bytesUsed: 0,
|
|
108
|
-
container: parser.getPartialResult(),
|
|
109
|
-
jsonpath: _jsonpath
|
|
110
|
-
};
|
|
111
|
-
_context.next = 28;
|
|
112
|
-
return initialBatch;
|
|
113
|
-
|
|
114
|
-
case 28:
|
|
115
|
-
isFirstChunk = false;
|
|
116
|
-
|
|
117
|
-
case 29:
|
|
118
|
-
_iterator2 = _createForOfIteratorHelper(rows);
|
|
119
|
-
_context.prev = 30;
|
|
120
|
-
|
|
121
|
-
_iterator2.s();
|
|
122
|
-
|
|
123
|
-
case 32:
|
|
124
|
-
if ((_step2 = _iterator2.n()).done) {
|
|
125
|
-
_context.next = 41;
|
|
126
|
-
break;
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
row = _step2.value;
|
|
130
|
-
tableBatchBuilder.addRow(row);
|
|
131
|
-
_batch2 = tableBatchBuilder.getFullBatch({
|
|
132
|
-
jsonpath: _jsonpath
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
if (!_batch2) {
|
|
136
|
-
_context.next = 39;
|
|
137
|
-
break;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
_context.next = 39;
|
|
141
|
-
return _batch2;
|
|
142
|
-
|
|
143
|
-
case 39:
|
|
144
|
-
_context.next = 32;
|
|
145
|
-
break;
|
|
146
|
-
|
|
147
|
-
case 41:
|
|
148
|
-
_context.next = 46;
|
|
149
|
-
break;
|
|
150
|
-
|
|
151
|
-
case 43:
|
|
152
|
-
_context.prev = 43;
|
|
153
|
-
_context.t0 = _context["catch"](30);
|
|
154
|
-
|
|
155
|
-
_iterator2.e(_context.t0);
|
|
156
|
-
|
|
157
|
-
case 46:
|
|
158
|
-
_context.prev = 46;
|
|
159
|
-
|
|
160
|
-
_iterator2.f();
|
|
161
|
-
|
|
162
|
-
return _context.finish(46);
|
|
163
|
-
|
|
164
|
-
case 49:
|
|
165
|
-
tableBatchBuilder.chunkComplete(chunk);
|
|
166
|
-
_batch = tableBatchBuilder.getFullBatch({
|
|
167
|
-
jsonpath: _jsonpath
|
|
168
|
-
});
|
|
169
|
-
|
|
170
|
-
if (!_batch) {
|
|
171
|
-
_context.next = 54;
|
|
172
|
-
break;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
_context.next = 54;
|
|
176
|
-
return _batch;
|
|
177
|
-
|
|
178
|
-
case 54:
|
|
179
|
-
_iteratorNormalCompletion = true;
|
|
180
|
-
_context.next = 12;
|
|
181
|
-
break;
|
|
182
|
-
|
|
183
|
-
case 57:
|
|
184
|
-
_context.next = 63;
|
|
185
|
-
break;
|
|
186
|
-
|
|
187
|
-
case 59:
|
|
188
|
-
_context.prev = 59;
|
|
189
|
-
_context.t1 = _context["catch"](10);
|
|
190
|
-
_didIteratorError = true;
|
|
191
|
-
_iteratorError = _context.t1;
|
|
192
|
-
|
|
193
|
-
case 63:
|
|
194
|
-
_context.prev = 63;
|
|
195
|
-
_context.prev = 64;
|
|
196
|
-
|
|
197
|
-
if (!(!_iteratorNormalCompletion && _iterator.return != null)) {
|
|
198
|
-
_context.next = 68;
|
|
199
|
-
break;
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
_context.next = 68;
|
|
203
|
-
return (0, _awaitAsyncGenerator2.default)(_iterator.return());
|
|
204
|
-
|
|
205
|
-
case 68:
|
|
206
|
-
_context.prev = 68;
|
|
207
|
-
|
|
208
|
-
if (!_didIteratorError) {
|
|
209
|
-
_context.next = 71;
|
|
210
|
-
break;
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
throw _iteratorError;
|
|
214
|
-
|
|
215
|
-
case 71:
|
|
216
|
-
return _context.finish(68);
|
|
217
|
-
|
|
218
|
-
case 72:
|
|
219
|
-
return _context.finish(63);
|
|
220
|
-
|
|
221
|
-
case 73:
|
|
222
|
-
jsonpath = parser.getStreamingJsonPathAsString();
|
|
223
|
-
batch = tableBatchBuilder.getFinalBatch({
|
|
224
|
-
jsonpath: jsonpath
|
|
225
|
-
});
|
|
226
|
-
|
|
227
|
-
if (!batch) {
|
|
228
|
-
_context.next = 78;
|
|
229
|
-
break;
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
_context.next = 78;
|
|
233
|
-
return batch;
|
|
234
|
-
|
|
235
|
-
case 78:
|
|
236
|
-
if (!metadata) {
|
|
237
|
-
_context.next = 82;
|
|
238
|
-
break;
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
finalBatch = {
|
|
242
|
-
shape: shape,
|
|
243
|
-
batchType: 'final-result',
|
|
244
|
-
container: parser.getPartialResult(),
|
|
245
|
-
jsonpath: parser.getStreamingJsonPathAsString(),
|
|
246
|
-
data: [],
|
|
247
|
-
length: 0
|
|
248
|
-
};
|
|
249
|
-
_context.next = 82;
|
|
250
|
-
return finalBatch;
|
|
251
|
-
|
|
252
|
-
case 82:
|
|
253
|
-
case "end":
|
|
254
|
-
return _context.stop();
|
|
255
|
-
}
|
|
256
|
-
}
|
|
257
|
-
}, _callee, null, [[10, 59, 63, 73], [30, 43, 46, 49], [64,, 68, 72]]);
|
|
258
|
-
}));
|
|
259
|
-
return _parseJSONInBatches.apply(this, arguments);
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
function rebuildJsonObject(batch, data) {
|
|
263
|
-
(0, _loaderUtils.assert)(batch.batchType === 'final-result');
|
|
264
|
-
|
|
265
|
-
if (batch.jsonpath === '$') {
|
|
266
|
-
return data;
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
if (batch.jsonpath && batch.jsonpath.length > 1) {
|
|
270
|
-
var topLevelObject = batch.container;
|
|
271
|
-
var streamingPath = new _jsonpath2.default(batch.jsonpath);
|
|
272
|
-
streamingPath.setFieldAtPath(topLevelObject, data);
|
|
273
|
-
return topLevelObject;
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
return batch.container;
|
|
277
|
-
}
|
|
278
|
-
//# sourceMappingURL=parse-json-in-batches.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/lib/parse-json-in-batches.ts"],"names":["parseJSONInBatches","binaryAsyncIterator","options","asyncIterator","metadata","json","jsonpaths","isFirstChunk","schema","shape","tableBatchBuilder","TableBatchBuilder","parser","StreamingJSONParser","chunk","rows","write","jsonpath","length","getStreamingJsonPathAsString","initialBatch","batchType","data","bytesUsed","container","getPartialResult","row","addRow","batch","getFullBatch","chunkComplete","getFinalBatch","finalBatch","rebuildJsonObject","topLevelObject","streamingPath","JSONPath","setFieldAtPath"],"mappings":";;;;;;;;;;;;;;;;;;;;AAEA;;AACA;;AACA;;AACA;;;;;;;;;;;;SAI+BA,kB;;;;;oFAAhB,iBACbC,mBADa,EAEbC,OAFa;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAIPC,YAAAA,aAJO,GAIS,0CAAwBF,mBAAxB,CAJT;AAMNG,YAAAA,QANM,GAMMF,OANN,CAMNE,QANM;AAAA,mBAOOF,OAAO,CAACG,IAAR,IAAgB,EAPvB,EAONC,SAPM,QAONA,SAPM;AASTC,YAAAA,YATS,GASe,IATf;AAYPC,YAAAA,MAZO,GAYE,IAZF;AAaPC,YAAAA,KAbO,GAaC,CAAAP,OAAO,SAAP,IAAAA,OAAO,WAAP,6BAAAA,OAAO,CAAEG,IAAT,gEAAeI,KAAf,KAAwB,WAbzB;AAePC,YAAAA,iBAfO,GAea,IAAIC,yBAAJ,CAAsBH,MAAtB,kCACrBN,OADqB;AAExBO,cAAAA,KAAK,EAALA;AAFwB,eAfb;AAoBPG,YAAAA,MApBO,GAoBE,IAAIC,4BAAJ,CAAwB;AAACP,cAAAA,SAAS,EAATA;AAAD,aAAxB,CApBF;AAAA;AAAA;AAAA;AAAA,qDAsBaH,aAtBb;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAsBIW,YAAAA,KAtBJ;AAuBLC,YAAAA,IAvBK,GAuBEH,MAAM,CAACI,KAAP,CAAaF,KAAb,CAvBF;AAyBLG,YAAAA,SAzBK,GAyBMF,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBN,MAAM,CAACO,4BAAP,EAzBzB;;AAAA,kBA2BPJ,IAAI,CAACG,MAAL,GAAc,CAAd,IAAmBX,YA3BZ;AAAA;AAAA;AAAA;;AAAA,iBA4BLH,QA5BK;AAAA;AAAA;AAAA;;AA6BDgB,YAAAA,YA7BC,GA6BqB;AAE1BX,cAAAA,KAAK,EAALA,KAF0B;AAG1BY,cAAAA,SAAS,EAAE,gBAHe;AAI1BC,cAAAA,IAAI,EAAE,EAJoB;AAK1BJ,cAAAA,MAAM,EAAE,CALkB;AAM1BK,cAAAA,SAAS,EAAE,CANe;AAQ1BC,cAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EARe;AAS1BR,cAAAA,QAAQ,EAARA;AAT0B,aA7BrB;AAAA;AAwCP,mBAAMG,YAAN;;AAxCO;AA0CTb,YAAAA,YAAY,GAAG,KAAf;;AA1CS;AAAA,oDA+COQ,IA/CP;AAAA;;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AA+CAW,YAAAA,GA/CA;AAgDThB,YAAAA,iBAAiB,CAACiB,MAAlB,CAAyBD,GAAzB;AAEME,YAAAA,OAlDG,GAkDKlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,cAAAA,QAAQ,EAARA;AAAD,aAA/B,CAlDL;;AAAA,iBAmDLW,OAnDK;AAAA;AAAA;AAAA;;AAAA;AAoDP,mBAAMA,OAAN;;AApDO;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;;AAAA;;AAAA;AAwDXlB,YAAAA,iBAAiB,CAACoB,aAAlB,CAAgChB,KAAhC;AACMc,YAAAA,MAzDK,GAyDGlB,iBAAiB,CAACmB,YAAlB,CAA+B;AAACZ,cAAAA,QAAQ,EAARA;AAAD,aAA/B,CAzDH;;AAAA,iBA0DPW,MA1DO;AAAA;AAAA;AAAA;;AAAA;AA2DT,mBAAMA,MAAN;;AA3DS;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAgEPX,YAAAA,QAhEO,GAgEIL,MAAM,CAACO,4BAAP,EAhEJ;AAiEPS,YAAAA,KAjEO,GAiEClB,iBAAiB,CAACqB,aAAlB,CAAgC;AAACd,cAAAA,QAAQ,EAARA;AAAD,aAAhC,CAjED;;AAAA,iBAkETW,KAlES;AAAA;AAAA;AAAA;;AAAA;AAmEX,mBAAMA,KAAN;;AAnEW;AAAA,iBAsETxB,QAtES;AAAA;AAAA;AAAA;;AAuEL4B,YAAAA,UAvEK,GAuEe;AACxBvB,cAAAA,KAAK,EAALA,KADwB;AAExBY,cAAAA,SAAS,EAAE,cAFa;AAGxBG,cAAAA,SAAS,EAAEZ,MAAM,CAACa,gBAAP,EAHa;AAIxBR,cAAAA,QAAQ,EAAEL,MAAM,CAACO,4BAAP,EAJc;AAKxBG,cAAAA,IAAI,EAAE,EALkB;AAMxBJ,cAAAA,MAAM,EAAE;AANgB,aAvEf;AAAA;AAgFX,mBAAMc,UAAN;;AAhFW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G;;;;AAoFR,SAASC,iBAAT,CAA2BL,KAA3B,EAAkCN,IAAlC,EAAwC;AAE7C,2BAAOM,KAAK,CAACP,SAAN,KAAoB,cAA3B;;AAGA,MAAIO,KAAK,CAACX,QAAN,KAAmB,GAAvB,EAA4B;AAC1B,WAAOK,IAAP;AACD;;AAGD,MAAIM,KAAK,CAACX,QAAN,IAAkBW,KAAK,CAACX,QAAN,CAAeC,MAAf,GAAwB,CAA9C,EAAiD;AAC/C,QAAMgB,cAAc,GAAGN,KAAK,CAACJ,SAA7B;AACA,QAAMW,aAAa,GAAG,IAAIC,kBAAJ,CAAaR,KAAK,CAACX,QAAnB,CAAtB;AACAkB,IAAAA,aAAa,CAACE,cAAd,CAA6BH,cAA7B,EAA6CZ,IAA7C;AACA,WAAOY,cAAP;AACD;;AAGD,SAAON,KAAK,CAACJ,SAAb;AACD","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {JSONLoaderOptions} from '../json-loader';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {assert, makeTextDecoderIterator} from '@loaders.gl/loader-utils';\nimport StreamingJSONParser from './parser/streaming-json-parser';\nimport JSONPath from './jsonpath/jsonpath';\n\n// TODO - support batch size 0 = no batching/single batch?\n// eslint-disable-next-line max-statements, complexity\nexport default async function* parseJSONInBatches(\n binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n options: JSONLoaderOptions\n): AsyncIterable<Batch> {\n const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);\n\n const {metadata} = options;\n const {jsonpaths} = options.json || {};\n\n let isFirstChunk: boolean = true;\n\n // TODO fix Schema deduction\n const schema = null; // new Schema([]);\n const shape = options?.json?.shape || 'row-table';\n // @ts-ignore\n const tableBatchBuilder = new TableBatchBuilder(schema, {\n ...options,\n shape\n });\n\n const parser = new StreamingJSONParser({jsonpaths});\n\n for await (const chunk of asyncIterator) {\n const rows = parser.write(chunk);\n\n const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();\n\n if (rows.length > 0 && isFirstChunk) {\n if (metadata) {\n const initialBatch: Batch = {\n // Common fields\n shape,\n batchType: 'partial-result',\n data: [],\n length: 0,\n bytesUsed: 0,\n // JSON additions\n container: parser.getPartialResult(),\n jsonpath\n };\n yield initialBatch;\n }\n isFirstChunk = false;\n // schema = deduceSchema(rows);\n }\n\n // Add the row\n for (const row of rows) {\n tableBatchBuilder.addRow(row);\n // If a batch has been completed, emit it\n const batch = tableBatchBuilder.getFullBatch({jsonpath});\n if (batch) {\n yield batch;\n }\n }\n\n tableBatchBuilder.chunkComplete(chunk);\n const batch = tableBatchBuilder.getFullBatch({jsonpath});\n if (batch) {\n yield batch;\n }\n }\n\n // yield final batch\n const jsonpath = parser.getStreamingJsonPathAsString();\n const batch = tableBatchBuilder.getFinalBatch({jsonpath});\n if (batch) {\n yield batch;\n }\n\n if (metadata) {\n const finalBatch: Batch = {\n shape,\n batchType: 'final-result',\n container: parser.getPartialResult(),\n jsonpath: parser.getStreamingJsonPathAsString(),\n data: [],\n length: 0\n // schema: null\n };\n yield finalBatch;\n }\n}\n\nexport function rebuildJsonObject(batch, data) {\n // Last batch will have this special type and will provide all the root object of the parsed file\n assert(batch.batchType === 'final-result');\n\n // The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects\n if (batch.jsonpath === '$') {\n return data;\n }\n\n // (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object\n if (batch.jsonpath && batch.jsonpath.length > 1) {\n const topLevelObject = batch.container;\n const streamingPath = new JSONPath(batch.jsonpath);\n streamingPath.setFieldAtPath(topLevelObject, data);\n return topLevelObject;\n }\n\n // No jsonpath, in this case nothing was streamed.\n return batch.container;\n}\n"],"file":"parse-json-in-batches.js"}
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
|
-
Object.defineProperty(exports, "__esModule", {
|
|
6
|
-
value: true
|
|
7
|
-
});
|
|
8
|
-
exports.default = parseJSONSync;
|
|
9
|
-
|
|
10
|
-
var _typeof2 = _interopRequireDefault(require("@babel/runtime/helpers/typeof"));
|
|
11
|
-
|
|
12
|
-
function parseJSONSync(jsonText, options) {
|
|
13
|
-
try {
|
|
14
|
-
var _options$json;
|
|
15
|
-
|
|
16
|
-
var json = JSON.parse(jsonText);
|
|
17
|
-
|
|
18
|
-
if ((_options$json = options.json) !== null && _options$json !== void 0 && _options$json.table) {
|
|
19
|
-
return getFirstArray(json) || json;
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
return json;
|
|
23
|
-
} catch (error) {
|
|
24
|
-
throw new Error('JSONLoader: failed to parse JSON');
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
function getFirstArray(json) {
|
|
29
|
-
if (Array.isArray(json)) {
|
|
30
|
-
return json;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
if (json && (0, _typeof2.default)(json) === 'object') {
|
|
34
|
-
for (var _i = 0, _Object$values = Object.values(json); _i < _Object$values.length; _i++) {
|
|
35
|
-
var value = _Object$values[_i];
|
|
36
|
-
var array = getFirstArray(value);
|
|
37
|
-
|
|
38
|
-
if (array) {
|
|
39
|
-
return array;
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
return null;
|
|
45
|
-
}
|
|
46
|
-
//# sourceMappingURL=parse-json.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/lib/parse-json.ts"],"names":["parseJSONSync","jsonText","options","json","JSON","parse","table","getFirstArray","error","Error","Array","isArray","Object","values","value","array"],"mappings":";;;;;;;;;;;AAEe,SAASA,aAAT,CAAuBC,QAAvB,EAAyCC,OAAzC,EAAqE;AAClF,MAAI;AAAA;;AACF,QAAMC,IAAI,GAAGC,IAAI,CAACC,KAAL,CAAWJ,QAAX,CAAb;;AACA,yBAAIC,OAAO,CAACC,IAAZ,0CAAI,cAAcG,KAAlB,EAAyB;AACvB,aAAOC,aAAa,CAACJ,IAAD,CAAb,IAAuBA,IAA9B;AACD;;AACD,WAAOA,IAAP;AACD,GAND,CAME,OAAOK,KAAP,EAAc;AACd,UAAM,IAAIC,KAAJ,CAAU,kCAAV,CAAN;AACD;AACF;;AAED,SAASF,aAAT,CAAuBJ,IAAvB,EAA6B;AAC3B,MAAIO,KAAK,CAACC,OAAN,CAAcR,IAAd,CAAJ,EAAyB;AACvB,WAAOA,IAAP;AACD;;AACD,MAAIA,IAAI,IAAI,sBAAOA,IAAP,MAAgB,QAA5B,EAAsC;AACpC,sCAAoBS,MAAM,CAACC,MAAP,CAAcV,IAAd,CAApB,oCAAyC;AAApC,UAAMW,KAAK,qBAAX;AACH,UAAMC,KAAK,GAAGR,aAAa,CAACO,KAAD,CAA3B;;AACA,UAAIC,KAAJ,EAAW;AACT,eAAOA,KAAP;AACD;AACF;AACF;;AACD,SAAO,IAAP;AACD","sourcesContent":["import type {JSONLoaderOptions} from '../json-loader';\n\nexport default function parseJSONSync(jsonText: string, options: JSONLoaderOptions) {\n try {\n const json = JSON.parse(jsonText);\n if (options.json?.table) {\n return getFirstArray(json) || json;\n }\n return json;\n } catch (error) {\n throw new Error('JSONLoader: failed to parse JSON');\n }\n}\n\nfunction getFirstArray(json) {\n if (Array.isArray(json)) {\n return json;\n }\n if (json && typeof json === 'object') {\n for (const value of Object.values(json)) {\n const array = getFirstArray(value);\n if (array) {\n return array;\n }\n }\n }\n return null;\n}\n"],"file":"parse-json.js"}
|
|
@@ -1,158 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
|
|
3
|
-
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
|
|
4
|
-
|
|
5
|
-
Object.defineProperty(exports, "__esModule", {
|
|
6
|
-
value: true
|
|
7
|
-
});
|
|
8
|
-
exports.default = parseNDJSONInBatches;
|
|
9
|
-
|
|
10
|
-
var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
|
|
11
|
-
|
|
12
|
-
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
|
|
13
|
-
|
|
14
|
-
var _awaitAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/awaitAsyncGenerator"));
|
|
15
|
-
|
|
16
|
-
var _wrapAsyncGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/wrapAsyncGenerator"));
|
|
17
|
-
|
|
18
|
-
var _asyncIterator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncIterator"));
|
|
19
|
-
|
|
20
|
-
var _schema = require("@loaders.gl/schema");
|
|
21
|
-
|
|
22
|
-
var _loaderUtils = require("@loaders.gl/loader-utils");
|
|
23
|
-
|
|
24
|
-
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }
|
|
25
|
-
|
|
26
|
-
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
|
|
27
|
-
|
|
28
|
-
function parseNDJSONInBatches(_x, _x2) {
|
|
29
|
-
return _parseNDJSONInBatches.apply(this, arguments);
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
function _parseNDJSONInBatches() {
|
|
33
|
-
_parseNDJSONInBatches = (0, _wrapAsyncGenerator2.default)(_regenerator.default.mark(function _callee(binaryAsyncIterator, options) {
|
|
34
|
-
var textIterator, lineIterator, numberedLineIterator, schema, shape, tableBatchBuilder, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, _value, _value2, counter, line, row, _batch, batch;
|
|
35
|
-
|
|
36
|
-
return _regenerator.default.wrap(function _callee$(_context) {
|
|
37
|
-
while (1) {
|
|
38
|
-
switch (_context.prev = _context.next) {
|
|
39
|
-
case 0:
|
|
40
|
-
textIterator = (0, _loaderUtils.makeTextDecoderIterator)(binaryAsyncIterator);
|
|
41
|
-
lineIterator = (0, _loaderUtils.makeLineIterator)(textIterator);
|
|
42
|
-
numberedLineIterator = (0, _loaderUtils.makeNumberedLineIterator)(lineIterator);
|
|
43
|
-
schema = null;
|
|
44
|
-
shape = 'row-table';
|
|
45
|
-
tableBatchBuilder = new _schema.TableBatchBuilder(schema, _objectSpread(_objectSpread({}, options), {}, {
|
|
46
|
-
shape: shape
|
|
47
|
-
}));
|
|
48
|
-
_iteratorNormalCompletion = true;
|
|
49
|
-
_didIteratorError = false;
|
|
50
|
-
_context.prev = 8;
|
|
51
|
-
_iterator = (0, _asyncIterator2.default)(numberedLineIterator);
|
|
52
|
-
|
|
53
|
-
case 10:
|
|
54
|
-
_context.next = 12;
|
|
55
|
-
return (0, _awaitAsyncGenerator2.default)(_iterator.next());
|
|
56
|
-
|
|
57
|
-
case 12:
|
|
58
|
-
_step = _context.sent;
|
|
59
|
-
_iteratorNormalCompletion = _step.done;
|
|
60
|
-
_context.next = 16;
|
|
61
|
-
return (0, _awaitAsyncGenerator2.default)(_step.value);
|
|
62
|
-
|
|
63
|
-
case 16:
|
|
64
|
-
_value = _context.sent;
|
|
65
|
-
|
|
66
|
-
if (_iteratorNormalCompletion) {
|
|
67
|
-
_context.next = 35;
|
|
68
|
-
break;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
_value2 = _value, counter = _value2.counter, line = _value2.line;
|
|
72
|
-
_context.prev = 19;
|
|
73
|
-
row = JSON.parse(line);
|
|
74
|
-
tableBatchBuilder.addRow(row);
|
|
75
|
-
tableBatchBuilder.chunkComplete(line);
|
|
76
|
-
_batch = tableBatchBuilder.getFullBatch();
|
|
77
|
-
|
|
78
|
-
if (!_batch) {
|
|
79
|
-
_context.next = 27;
|
|
80
|
-
break;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
_context.next = 27;
|
|
84
|
-
return _batch;
|
|
85
|
-
|
|
86
|
-
case 27:
|
|
87
|
-
_context.next = 32;
|
|
88
|
-
break;
|
|
89
|
-
|
|
90
|
-
case 29:
|
|
91
|
-
_context.prev = 29;
|
|
92
|
-
_context.t0 = _context["catch"](19);
|
|
93
|
-
throw new Error("NDJSONLoader: failed to parse JSON on line ".concat(counter));
|
|
94
|
-
|
|
95
|
-
case 32:
|
|
96
|
-
_iteratorNormalCompletion = true;
|
|
97
|
-
_context.next = 10;
|
|
98
|
-
break;
|
|
99
|
-
|
|
100
|
-
case 35:
|
|
101
|
-
_context.next = 41;
|
|
102
|
-
break;
|
|
103
|
-
|
|
104
|
-
case 37:
|
|
105
|
-
_context.prev = 37;
|
|
106
|
-
_context.t1 = _context["catch"](8);
|
|
107
|
-
_didIteratorError = true;
|
|
108
|
-
_iteratorError = _context.t1;
|
|
109
|
-
|
|
110
|
-
case 41:
|
|
111
|
-
_context.prev = 41;
|
|
112
|
-
_context.prev = 42;
|
|
113
|
-
|
|
114
|
-
if (!(!_iteratorNormalCompletion && _iterator.return != null)) {
|
|
115
|
-
_context.next = 46;
|
|
116
|
-
break;
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
_context.next = 46;
|
|
120
|
-
return (0, _awaitAsyncGenerator2.default)(_iterator.return());
|
|
121
|
-
|
|
122
|
-
case 46:
|
|
123
|
-
_context.prev = 46;
|
|
124
|
-
|
|
125
|
-
if (!_didIteratorError) {
|
|
126
|
-
_context.next = 49;
|
|
127
|
-
break;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
throw _iteratorError;
|
|
131
|
-
|
|
132
|
-
case 49:
|
|
133
|
-
return _context.finish(46);
|
|
134
|
-
|
|
135
|
-
case 50:
|
|
136
|
-
return _context.finish(41);
|
|
137
|
-
|
|
138
|
-
case 51:
|
|
139
|
-
batch = tableBatchBuilder.getFinalBatch();
|
|
140
|
-
|
|
141
|
-
if (!batch) {
|
|
142
|
-
_context.next = 55;
|
|
143
|
-
break;
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
_context.next = 55;
|
|
147
|
-
return batch;
|
|
148
|
-
|
|
149
|
-
case 55:
|
|
150
|
-
case "end":
|
|
151
|
-
return _context.stop();
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
}, _callee, null, [[8, 37, 41, 51], [19, 29], [42,, 46, 50]]);
|
|
155
|
-
}));
|
|
156
|
-
return _parseNDJSONInBatches.apply(this, arguments);
|
|
157
|
-
}
|
|
158
|
-
//# sourceMappingURL=parse-ndjson-in-batches.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../../../src/lib/parse-ndjson-in-batches.ts"],"names":["parseNDJSONInBatches","binaryAsyncIterator","options","textIterator","lineIterator","numberedLineIterator","schema","shape","tableBatchBuilder","TableBatchBuilder","counter","line","row","JSON","parse","addRow","chunkComplete","batch","getFullBatch","Error","getFinalBatch"],"mappings":";;;;;;;;;;;;;;;;;;;AACA;;AACA;;;;;;SAO+BA,oB;;;;;sFAAhB,iBACbC,mBADa,EAEbC,OAFa;AAAA;;AAAA;AAAA;AAAA;AAAA;AAIPC,YAAAA,YAJO,GAIQ,0CAAwBF,mBAAxB,CAJR;AAKPG,YAAAA,YALO,GAKQ,mCAAiBD,YAAjB,CALR;AAMPE,YAAAA,oBANO,GAMgB,2CAAyBD,YAAzB,CANhB;AAQPE,YAAAA,MARO,GAQE,IARF;AASPC,YAAAA,KATO,GASC,WATD;AAWPC,YAAAA,iBAXO,GAWa,IAAIC,yBAAJ,CAAsBH,MAAtB,kCACrBJ,OADqB;AAExBK,cAAAA,KAAK,EAALA;AAFwB,eAXb;AAAA;AAAA;AAAA;AAAA,qDAgBuBF,oBAhBvB;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA,8BAgBKK,OAhBL,WAgBKA,OAhBL,EAgBcC,IAhBd,WAgBcA,IAhBd;AAAA;AAkBHC,YAAAA,GAlBG,GAkBGC,IAAI,CAACC,KAAL,CAAWH,IAAX,CAlBH;AAmBTH,YAAAA,iBAAiB,CAACO,MAAlB,CAAyBH,GAAzB;AACAJ,YAAAA,iBAAiB,CAACQ,aAAlB,CAAgCL,IAAhC;AACMM,YAAAA,MArBG,GAqBKT,iBAAiB,CAACU,YAAlB,EArBL;;AAAA,iBAsBLD,MAtBK;AAAA;AAAA;AAAA;;AAAA;AAuBP,mBAAMA,MAAN;;AAvBO;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA,kBA0BH,IAAIE,KAAJ,sDAAwDT,OAAxD,EA1BG;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AAAA;AAAA;AAAA;;AAAA;;AAAA;AAAA;;AAAA;AAAA;;AAAA;AA8BPO,YAAAA,KA9BO,GA8BCT,iBAAiB,CAACY,aAAlB,EA9BD;;AAAA,iBA+BTH,KA/BS;AAAA;AAAA;AAAA;;AAAA;AAgCX,mBAAMA,KAAN;;AAhCW;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,G","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport {\n LoaderOptions,\n makeLineIterator,\n makeNumberedLineIterator,\n makeTextDecoderIterator\n} from '@loaders.gl/loader-utils';\n\nexport default async function* parseNDJSONInBatches(\n binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n options?: LoaderOptions\n): AsyncIterable<Batch> {\n const textIterator = makeTextDecoderIterator(binaryAsyncIterator);\n const lineIterator = makeLineIterator(textIterator);\n const numberedLineIterator = makeNumberedLineIterator(lineIterator);\n\n const schema = null;\n const shape = 'row-table';\n // @ts-ignore\n const tableBatchBuilder = new TableBatchBuilder(schema, {\n ...options,\n shape\n });\n\n for await (const {counter, line} of numberedLineIterator) {\n try {\n const row = JSON.parse(line);\n tableBatchBuilder.addRow(row);\n tableBatchBuilder.chunkComplete(line);\n const batch = tableBatchBuilder.getFullBatch();\n if (batch) {\n yield batch;\n }\n } catch (error) {\n throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);\n }\n }\n\n const batch = tableBatchBuilder.getFinalBatch();\n if (batch) {\n yield batch;\n }\n}\n"],"file":"parse-ndjson-in-batches.js"}
|