@loaders.gl/json 3.1.0-alpha.3 → 3.1.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle.d.ts +2 -0
- package/dist/bundle.d.ts.map +1 -0
- package/dist/bundle.js +1838 -4
- package/dist/es5/bundle.js +7 -0
- package/dist/es5/bundle.js.map +1 -0
- package/dist/es5/geojson-loader.js +95 -0
- package/dist/es5/geojson-loader.js.map +1 -0
- package/dist/es5/index.js +54 -0
- package/dist/es5/index.js.map +1 -0
- package/dist/es5/json-loader.js +59 -0
- package/dist/es5/json-loader.js.map +1 -0
- package/dist/es5/jsonl-loader.js +2 -0
- package/dist/{jsonl-loader.js.map → es5/jsonl-loader.js.map} +0 -0
- package/dist/{lib → es5/lib}/clarinet/LICENSE +0 -0
- package/dist/es5/lib/clarinet/clarinet.js +511 -0
- package/dist/es5/lib/clarinet/clarinet.js.map +1 -0
- package/dist/es5/lib/jsonpath/jsonpath.js +97 -0
- package/dist/es5/lib/jsonpath/jsonpath.js.map +1 -0
- package/dist/es5/lib/parse-json-in-batches.js +99 -0
- package/dist/es5/lib/parse-json-in-batches.js.map +1 -0
- package/dist/es5/lib/parse-json.js +41 -0
- package/dist/es5/lib/parse-json.js.map +1 -0
- package/dist/es5/lib/parse-ndjson-in-batches.js +46 -0
- package/dist/es5/lib/parse-ndjson-in-batches.js.map +1 -0
- package/dist/es5/lib/parse-ndjson.js +18 -0
- package/dist/es5/lib/parse-ndjson.js.map +1 -0
- package/dist/es5/lib/parser/json-parser.js +137 -0
- package/dist/es5/lib/parser/json-parser.js.map +1 -0
- package/dist/es5/lib/parser/streaming-json-parser.js +102 -0
- package/dist/es5/lib/parser/streaming-json-parser.js.map +1 -0
- package/dist/es5/ndjson-loader.js +45 -0
- package/dist/es5/ndjson-loader.js.map +1 -0
- package/dist/es5/workers/geojson-worker.js +8 -0
- package/dist/es5/workers/geojson-worker.js.map +1 -0
- package/dist/esm/bundle.js +5 -0
- package/dist/esm/bundle.js.map +1 -0
- package/dist/esm/geojson-loader.js +81 -0
- package/dist/esm/geojson-loader.js.map +1 -0
- package/dist/esm/index.js +6 -0
- package/dist/esm/index.js.map +1 -0
- package/dist/esm/json-loader.js +47 -0
- package/dist/esm/json-loader.js.map +1 -0
- package/dist/esm/jsonl-loader.js +2 -0
- package/dist/esm/jsonl-loader.js.map +1 -0
- package/dist/esm/lib/clarinet/LICENSE +28 -0
- package/dist/esm/lib/clarinet/clarinet.js +518 -0
- package/dist/esm/lib/clarinet/clarinet.js.map +1 -0
- package/dist/esm/lib/jsonpath/jsonpath.js +86 -0
- package/dist/esm/lib/jsonpath/jsonpath.js.map +1 -0
- package/dist/esm/lib/parse-json-in-batches.js +87 -0
- package/dist/esm/lib/parse-json-in-batches.js.map +1 -0
- package/dist/esm/lib/parse-json.js +34 -0
- package/dist/esm/lib/parse-json.js.map +1 -0
- package/dist/esm/lib/parse-ndjson-in-batches.js +37 -0
- package/dist/esm/lib/parse-ndjson-in-batches.js.map +1 -0
- package/dist/esm/lib/parse-ndjson.js +11 -0
- package/dist/esm/lib/parse-ndjson.js.map +1 -0
- package/dist/esm/lib/parser/json-parser.js +128 -0
- package/dist/esm/lib/parser/json-parser.js.map +1 -0
- package/dist/esm/lib/parser/streaming-json-parser.js +93 -0
- package/dist/esm/lib/parser/streaming-json-parser.js.map +1 -0
- package/dist/esm/ndjson-loader.js +32 -0
- package/dist/esm/ndjson-loader.js.map +1 -0
- package/dist/esm/workers/geojson-worker.js +4 -0
- package/dist/esm/workers/geojson-worker.js.map +1 -0
- package/dist/geojson-loader.d.ts +16 -0
- package/dist/geojson-loader.d.ts.map +1 -0
- package/dist/geojson-loader.js +66 -69
- package/dist/geojson-worker.js +1702 -2
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +17 -6
- package/dist/json-loader.d.ts +14 -0
- package/dist/json-loader.d.ts.map +1 -0
- package/dist/json-loader.js +48 -38
- package/dist/jsonl-loader.d.ts +1 -0
- package/dist/jsonl-loader.d.ts.map +1 -0
- package/dist/jsonl-loader.js +53 -1
- package/dist/lib/clarinet/clarinet.d.ts +74 -0
- package/dist/lib/clarinet/clarinet.d.ts.map +1 -0
- package/dist/lib/clarinet/clarinet.js +517 -538
- package/dist/lib/jsonpath/jsonpath.d.ts +32 -0
- package/dist/lib/jsonpath/jsonpath.d.ts.map +1 -0
- package/dist/lib/jsonpath/jsonpath.js +81 -78
- package/dist/lib/parse-json-in-batches.d.ts +4 -0
- package/dist/lib/parse-json-in-batches.d.ts.map +1 -0
- package/dist/lib/parse-json-in-batches.js +75 -82
- package/dist/lib/parse-json.d.ts +3 -0
- package/dist/lib/parse-json.d.ts.map +1 -0
- package/dist/lib/parse-json.js +24 -29
- package/dist/lib/parse-ndjson-in-batches.d.ts +4 -0
- package/dist/lib/parse-ndjson-in-batches.d.ts.map +1 -0
- package/dist/lib/parse-ndjson-in-batches.js +32 -34
- package/dist/lib/parse-ndjson.d.ts +2 -0
- package/dist/lib/parse-ndjson.d.ts.map +1 -0
- package/dist/lib/parse-ndjson.js +13 -10
- package/dist/lib/parser/json-parser.d.ts +22 -0
- package/dist/lib/parser/json-parser.d.ts.map +1 -0
- package/dist/lib/parser/json-parser.js +95 -124
- package/dist/lib/parser/streaming-json-parser.d.ts +37 -0
- package/dist/lib/parser/streaming-json-parser.d.ts.map +1 -0
- package/dist/lib/parser/streaming-json-parser.js +95 -93
- package/dist/ndjson-loader.d.ts +22 -0
- package/dist/ndjson-loader.d.ts.map +1 -0
- package/dist/ndjson-loader.js +28 -23
- package/dist/workers/geojson-worker.d.ts +2 -0
- package/dist/workers/geojson-worker.d.ts.map +1 -0
- package/dist/workers/geojson-worker.js +5 -4
- package/package.json +10 -10
- package/src/lib/clarinet/clarinet.ts +539 -0
- package/src/lib/parser/json-parser.ts +52 -55
- package/src/lib/parser/streaming-json-parser.ts +28 -32
- package/src/ndjson-loader.ts +3 -1
- package/dist/bundle.js.map +0 -1
- package/dist/dist.min.js +0 -2
- package/dist/dist.min.js.map +0 -1
- package/dist/geojson-loader.js.map +0 -1
- package/dist/geojson-worker.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/json-loader.js.map +0 -1
- package/dist/lib/clarinet/clarinet.js.map +0 -1
- package/dist/lib/jsonpath/jsonpath.js.map +0 -1
- package/dist/lib/parse-json-in-batches.js.map +0 -1
- package/dist/lib/parse-json.js.map +0 -1
- package/dist/lib/parse-ndjson-in-batches.js.map +0 -1
- package/dist/lib/parse-ndjson.js.map +0 -1
- package/dist/lib/parser/json-parser.js.map +0 -1
- package/dist/lib/parser/streaming-json-parser.js.map +0 -1
- package/dist/ndjson-loader.js.map +0 -1
- package/dist/workers/geojson-worker.js.map +0 -1
- package/src/lib/clarinet/clarinet.js +0 -578
package/dist/bundle.js
CHANGED
|
@@ -1,5 +1,1839 @@
|
|
|
1
|
-
|
|
1
|
+
(() => {
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __markAsModule = (target) => __defProp(target, "__esModule", { value: true });
|
|
4
|
+
var __esm = (fn, res) => function __init() {
|
|
5
|
+
return fn && (res = (0, fn[Object.keys(fn)[0]])(fn = 0)), res;
|
|
6
|
+
};
|
|
7
|
+
var __commonJS = (cb, mod) => function __require() {
|
|
8
|
+
return mod || (0, cb[Object.keys(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
|
|
9
|
+
};
|
|
10
|
+
var __export = (target, all) => {
|
|
11
|
+
__markAsModule(target);
|
|
12
|
+
for (var name in all)
|
|
13
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
14
|
+
};
|
|
2
15
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
16
|
+
// src/lib/parse-json.ts
|
|
17
|
+
function parseJSONSync(jsonText, options) {
|
|
18
|
+
try {
|
|
19
|
+
const json = JSON.parse(jsonText);
|
|
20
|
+
if (options.json?.table) {
|
|
21
|
+
return getFirstArray(json) || json;
|
|
22
|
+
}
|
|
23
|
+
return json;
|
|
24
|
+
} catch (error) {
|
|
25
|
+
throw new Error("JSONLoader: failed to parse JSON");
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
function getFirstArray(json) {
|
|
29
|
+
if (Array.isArray(json)) {
|
|
30
|
+
return json;
|
|
31
|
+
}
|
|
32
|
+
if (json && typeof json === "object") {
|
|
33
|
+
for (const value of Object.values(json)) {
|
|
34
|
+
const array = getFirstArray(value);
|
|
35
|
+
if (array) {
|
|
36
|
+
return array;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
var init_parse_json = __esm({
|
|
43
|
+
"src/lib/parse-json.ts"() {
|
|
44
|
+
}
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
// ../schema/src/lib/batches/base-table-batch-aggregator.ts
|
|
48
|
+
var DEFAULT_ROW_COUNT, RowTableBatchAggregator;
|
|
49
|
+
var init_base_table_batch_aggregator = __esm({
|
|
50
|
+
"../schema/src/lib/batches/base-table-batch-aggregator.ts"() {
|
|
51
|
+
DEFAULT_ROW_COUNT = 100;
|
|
52
|
+
RowTableBatchAggregator = class {
|
|
53
|
+
constructor(schema, options) {
|
|
54
|
+
this.length = 0;
|
|
55
|
+
this.rows = null;
|
|
56
|
+
this.cursor = 0;
|
|
57
|
+
this._headers = [];
|
|
58
|
+
this.options = options;
|
|
59
|
+
this.schema = schema;
|
|
60
|
+
if (!Array.isArray(schema)) {
|
|
61
|
+
this._headers = [];
|
|
62
|
+
for (const key in schema) {
|
|
63
|
+
this._headers[schema[key].index] = schema[key].name;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
rowCount() {
|
|
68
|
+
return this.length;
|
|
69
|
+
}
|
|
70
|
+
addArrayRow(row, cursor) {
|
|
71
|
+
if (Number.isFinite(cursor)) {
|
|
72
|
+
this.cursor = cursor;
|
|
73
|
+
}
|
|
74
|
+
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
75
|
+
this.rows[this.length] = row;
|
|
76
|
+
this.length++;
|
|
77
|
+
}
|
|
78
|
+
addObjectRow(row, cursor) {
|
|
79
|
+
if (Number.isFinite(cursor)) {
|
|
80
|
+
this.cursor = cursor;
|
|
81
|
+
}
|
|
82
|
+
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
|
|
83
|
+
this.rows[this.length] = row;
|
|
84
|
+
this.length++;
|
|
85
|
+
}
|
|
86
|
+
getBatch() {
|
|
87
|
+
let rows = this.rows;
|
|
88
|
+
if (!rows) {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
rows = rows.slice(0, this.length);
|
|
92
|
+
this.rows = null;
|
|
93
|
+
const batch = {
|
|
94
|
+
shape: this.options.shape,
|
|
95
|
+
batchType: "data",
|
|
96
|
+
data: rows,
|
|
97
|
+
length: this.length,
|
|
98
|
+
schema: this.schema,
|
|
99
|
+
cursor: this.cursor
|
|
100
|
+
};
|
|
101
|
+
return batch;
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
// ../schema/src/lib/utils/row-utils.ts
|
|
108
|
+
function convertToObjectRow(arrayRow, headers) {
|
|
109
|
+
if (!arrayRow) {
|
|
110
|
+
throw new Error("null row");
|
|
111
|
+
}
|
|
112
|
+
if (!headers) {
|
|
113
|
+
throw new Error("no headers");
|
|
114
|
+
}
|
|
115
|
+
const objectRow = {};
|
|
116
|
+
for (let i = 0; i < headers.length; i++) {
|
|
117
|
+
objectRow[headers[i]] = arrayRow[i];
|
|
118
|
+
}
|
|
119
|
+
return objectRow;
|
|
120
|
+
}
|
|
121
|
+
function convertToArrayRow(objectRow, headers) {
|
|
122
|
+
if (!objectRow) {
|
|
123
|
+
throw new Error("null row");
|
|
124
|
+
}
|
|
125
|
+
if (!headers) {
|
|
126
|
+
throw new Error("no headers");
|
|
127
|
+
}
|
|
128
|
+
const arrayRow = new Array(headers.length);
|
|
129
|
+
for (let i = 0; i < headers.length; i++) {
|
|
130
|
+
arrayRow[i] = objectRow[headers[i]];
|
|
131
|
+
}
|
|
132
|
+
return arrayRow;
|
|
133
|
+
}
|
|
134
|
+
var init_row_utils = __esm({
|
|
135
|
+
"../schema/src/lib/utils/row-utils.ts"() {
|
|
136
|
+
}
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
// ../schema/src/lib/batches/row-table-batch-aggregator.ts
|
|
140
|
+
var DEFAULT_ROW_COUNT2, RowTableBatchAggregator2;
|
|
141
|
+
var init_row_table_batch_aggregator = __esm({
|
|
142
|
+
"../schema/src/lib/batches/row-table-batch-aggregator.ts"() {
|
|
143
|
+
init_row_utils();
|
|
144
|
+
DEFAULT_ROW_COUNT2 = 100;
|
|
145
|
+
RowTableBatchAggregator2 = class {
|
|
146
|
+
constructor(schema, options) {
|
|
147
|
+
this.length = 0;
|
|
148
|
+
this.objectRows = null;
|
|
149
|
+
this.arrayRows = null;
|
|
150
|
+
this.cursor = 0;
|
|
151
|
+
this._headers = [];
|
|
152
|
+
this.options = options;
|
|
153
|
+
this.schema = schema;
|
|
154
|
+
if (!Array.isArray(schema)) {
|
|
155
|
+
this._headers = [];
|
|
156
|
+
for (const key in schema) {
|
|
157
|
+
this._headers[schema[key].index] = schema[key].name;
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
rowCount() {
|
|
162
|
+
return this.length;
|
|
163
|
+
}
|
|
164
|
+
addArrayRow(row, cursor) {
|
|
165
|
+
if (Number.isFinite(cursor)) {
|
|
166
|
+
this.cursor = cursor;
|
|
167
|
+
}
|
|
168
|
+
switch (this.options.shape) {
|
|
169
|
+
case "object-row-table":
|
|
170
|
+
const rowObject = convertToObjectRow(row, this._headers);
|
|
171
|
+
this.addObjectRow(rowObject, cursor);
|
|
172
|
+
break;
|
|
173
|
+
case "array-row-table":
|
|
174
|
+
this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT2);
|
|
175
|
+
this.arrayRows[this.length] = row;
|
|
176
|
+
this.length++;
|
|
177
|
+
break;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
addObjectRow(row, cursor) {
|
|
181
|
+
if (Number.isFinite(cursor)) {
|
|
182
|
+
this.cursor = cursor;
|
|
183
|
+
}
|
|
184
|
+
switch (this.options.shape) {
|
|
185
|
+
case "array-row-table":
|
|
186
|
+
const rowArray = convertToArrayRow(row, this._headers);
|
|
187
|
+
this.addArrayRow(rowArray, cursor);
|
|
188
|
+
break;
|
|
189
|
+
case "object-row-table":
|
|
190
|
+
this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT2);
|
|
191
|
+
this.objectRows[this.length] = row;
|
|
192
|
+
this.length++;
|
|
193
|
+
break;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
getBatch() {
|
|
197
|
+
let rows = this.arrayRows || this.objectRows;
|
|
198
|
+
if (!rows) {
|
|
199
|
+
return null;
|
|
200
|
+
}
|
|
201
|
+
rows = rows.slice(0, this.length);
|
|
202
|
+
this.arrayRows = null;
|
|
203
|
+
this.objectRows = null;
|
|
204
|
+
return {
|
|
205
|
+
shape: this.options.shape,
|
|
206
|
+
batchType: "data",
|
|
207
|
+
data: rows,
|
|
208
|
+
length: this.length,
|
|
209
|
+
schema: this.schema,
|
|
210
|
+
cursor: this.cursor
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
|
|
217
|
+
// ../schema/src/lib/batches/columnar-table-batch-aggregator.ts
|
|
218
|
+
var DEFAULT_ROW_COUNT3, ColumnarTableBatchAggregator;
|
|
219
|
+
var init_columnar_table_batch_aggregator = __esm({
|
|
220
|
+
"../schema/src/lib/batches/columnar-table-batch-aggregator.ts"() {
|
|
221
|
+
DEFAULT_ROW_COUNT3 = 100;
|
|
222
|
+
ColumnarTableBatchAggregator = class {
|
|
223
|
+
constructor(schema, options) {
|
|
224
|
+
this.length = 0;
|
|
225
|
+
this.allocated = 0;
|
|
226
|
+
this.columns = {};
|
|
227
|
+
this.schema = schema;
|
|
228
|
+
this._reallocateColumns();
|
|
229
|
+
}
|
|
230
|
+
rowCount() {
|
|
231
|
+
return this.length;
|
|
232
|
+
}
|
|
233
|
+
addArrayRow(row) {
|
|
234
|
+
this._reallocateColumns();
|
|
235
|
+
let i = 0;
|
|
236
|
+
for (const fieldName in this.columns) {
|
|
237
|
+
this.columns[fieldName][this.length] = row[i++];
|
|
238
|
+
}
|
|
239
|
+
this.length++;
|
|
240
|
+
}
|
|
241
|
+
addObjectRow(row) {
|
|
242
|
+
this._reallocateColumns();
|
|
243
|
+
for (const fieldName in row) {
|
|
244
|
+
this.columns[fieldName][this.length] = row[fieldName];
|
|
245
|
+
}
|
|
246
|
+
this.length++;
|
|
247
|
+
}
|
|
248
|
+
getBatch() {
|
|
249
|
+
this._pruneColumns();
|
|
250
|
+
const columns = Array.isArray(this.schema) ? this.columns : {};
|
|
251
|
+
if (!Array.isArray(this.schema)) {
|
|
252
|
+
for (const fieldName in this.schema) {
|
|
253
|
+
const field = this.schema[fieldName];
|
|
254
|
+
columns[field.name] = this.columns[field.index];
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
this.columns = {};
|
|
258
|
+
const batch = {
|
|
259
|
+
shape: "columnar-table",
|
|
260
|
+
batchType: "data",
|
|
261
|
+
data: columns,
|
|
262
|
+
schema: this.schema,
|
|
263
|
+
length: this.length
|
|
264
|
+
};
|
|
265
|
+
return batch;
|
|
266
|
+
}
|
|
267
|
+
_reallocateColumns() {
|
|
268
|
+
if (this.length < this.allocated) {
|
|
269
|
+
return;
|
|
270
|
+
}
|
|
271
|
+
this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT3;
|
|
272
|
+
this.columns = {};
|
|
273
|
+
for (const fieldName in this.schema) {
|
|
274
|
+
const field = this.schema[fieldName];
|
|
275
|
+
const ArrayType = field.type || Float32Array;
|
|
276
|
+
const oldColumn = this.columns[field.index];
|
|
277
|
+
if (oldColumn && ArrayBuffer.isView(oldColumn)) {
|
|
278
|
+
const typedArray = new ArrayType(this.allocated);
|
|
279
|
+
typedArray.set(oldColumn);
|
|
280
|
+
this.columns[field.index] = typedArray;
|
|
281
|
+
} else if (oldColumn) {
|
|
282
|
+
oldColumn.length = this.allocated;
|
|
283
|
+
this.columns[field.index] = oldColumn;
|
|
284
|
+
} else {
|
|
285
|
+
this.columns[field.index] = new ArrayType(this.allocated);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
_pruneColumns() {
|
|
290
|
+
for (const [columnName, column] of Object.entries(this.columns)) {
|
|
291
|
+
this.columns[columnName] = column.slice(0, this.length);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
// ../schema/src/lib/batches/table-batch-builder.ts
|
|
299
|
+
var DEFAULT_OPTIONS, ERR_MESSAGE, TableBatchBuilder;
|
|
300
|
+
var init_table_batch_builder = __esm({
|
|
301
|
+
"../schema/src/lib/batches/table-batch-builder.ts"() {
|
|
302
|
+
init_base_table_batch_aggregator();
|
|
303
|
+
init_row_table_batch_aggregator();
|
|
304
|
+
init_columnar_table_batch_aggregator();
|
|
305
|
+
DEFAULT_OPTIONS = {
|
|
306
|
+
shape: "array-row-table",
|
|
307
|
+
batchSize: "auto",
|
|
308
|
+
batchDebounceMs: 0,
|
|
309
|
+
limit: 0,
|
|
310
|
+
_limitMB: 0
|
|
311
|
+
};
|
|
312
|
+
ERR_MESSAGE = "TableBatchBuilder";
|
|
313
|
+
TableBatchBuilder = class {
|
|
314
|
+
constructor(schema, options) {
|
|
315
|
+
this.aggregator = null;
|
|
316
|
+
this.batchCount = 0;
|
|
317
|
+
this.bytesUsed = 0;
|
|
318
|
+
this.isChunkComplete = false;
|
|
319
|
+
this.lastBatchEmittedMs = Date.now();
|
|
320
|
+
this.totalLength = 0;
|
|
321
|
+
this.totalBytes = 0;
|
|
322
|
+
this.rowBytes = 0;
|
|
323
|
+
this.schema = schema;
|
|
324
|
+
this.options = { ...DEFAULT_OPTIONS, ...options };
|
|
325
|
+
}
|
|
326
|
+
limitReached() {
|
|
327
|
+
if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
|
|
328
|
+
return true;
|
|
329
|
+
}
|
|
330
|
+
if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
|
|
331
|
+
return true;
|
|
332
|
+
}
|
|
333
|
+
return false;
|
|
334
|
+
}
|
|
335
|
+
addRow(row) {
|
|
336
|
+
if (this.limitReached()) {
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
this.totalLength++;
|
|
340
|
+
this.rowBytes = this.rowBytes || this._estimateRowMB(row);
|
|
341
|
+
this.totalBytes += this.rowBytes;
|
|
342
|
+
if (Array.isArray(row)) {
|
|
343
|
+
this.addArrayRow(row);
|
|
344
|
+
} else {
|
|
345
|
+
this.addObjectRow(row);
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
addArrayRow(row) {
|
|
349
|
+
if (!this.aggregator) {
|
|
350
|
+
const TableBatchType = this._getTableBatchType();
|
|
351
|
+
this.aggregator = new TableBatchType(this.schema, this.options);
|
|
352
|
+
}
|
|
353
|
+
this.aggregator.addArrayRow(row);
|
|
354
|
+
}
|
|
355
|
+
addObjectRow(row) {
|
|
356
|
+
if (!this.aggregator) {
|
|
357
|
+
const TableBatchType = this._getTableBatchType();
|
|
358
|
+
this.aggregator = new TableBatchType(this.schema, this.options);
|
|
359
|
+
}
|
|
360
|
+
this.aggregator.addObjectRow(row);
|
|
361
|
+
}
|
|
362
|
+
chunkComplete(chunk) {
|
|
363
|
+
if (chunk instanceof ArrayBuffer) {
|
|
364
|
+
this.bytesUsed += chunk.byteLength;
|
|
365
|
+
}
|
|
366
|
+
if (typeof chunk === "string") {
|
|
367
|
+
this.bytesUsed += chunk.length;
|
|
368
|
+
}
|
|
369
|
+
this.isChunkComplete = true;
|
|
370
|
+
}
|
|
371
|
+
getFullBatch(options) {
|
|
372
|
+
return this._isFull() ? this._getBatch(options) : null;
|
|
373
|
+
}
|
|
374
|
+
getFinalBatch(options) {
|
|
375
|
+
return this._getBatch(options);
|
|
376
|
+
}
|
|
377
|
+
_estimateRowMB(row) {
|
|
378
|
+
return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
|
|
379
|
+
}
|
|
380
|
+
_isFull() {
|
|
381
|
+
if (!this.aggregator || this.aggregator.rowCount() === 0) {
|
|
382
|
+
return false;
|
|
383
|
+
}
|
|
384
|
+
if (this.options.batchSize === "auto") {
|
|
385
|
+
if (!this.isChunkComplete) {
|
|
386
|
+
return false;
|
|
387
|
+
}
|
|
388
|
+
} else if (this.options.batchSize > this.aggregator.rowCount()) {
|
|
389
|
+
return false;
|
|
390
|
+
}
|
|
391
|
+
if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
|
|
392
|
+
return false;
|
|
393
|
+
}
|
|
394
|
+
this.isChunkComplete = false;
|
|
395
|
+
this.lastBatchEmittedMs = Date.now();
|
|
396
|
+
return true;
|
|
397
|
+
}
|
|
398
|
+
_getBatch(options) {
|
|
399
|
+
if (!this.aggregator) {
|
|
400
|
+
return null;
|
|
401
|
+
}
|
|
402
|
+
if (options?.bytesUsed) {
|
|
403
|
+
this.bytesUsed = options.bytesUsed;
|
|
404
|
+
}
|
|
405
|
+
const normalizedBatch = this.aggregator.getBatch();
|
|
406
|
+
normalizedBatch.count = this.batchCount;
|
|
407
|
+
normalizedBatch.bytesUsed = this.bytesUsed;
|
|
408
|
+
Object.assign(normalizedBatch, options);
|
|
409
|
+
this.batchCount++;
|
|
410
|
+
this.aggregator = null;
|
|
411
|
+
return normalizedBatch;
|
|
412
|
+
}
|
|
413
|
+
_getTableBatchType() {
|
|
414
|
+
switch (this.options.shape) {
|
|
415
|
+
case "row-table":
|
|
416
|
+
return RowTableBatchAggregator;
|
|
417
|
+
case "array-row-table":
|
|
418
|
+
case "object-row-table":
|
|
419
|
+
return RowTableBatchAggregator2;
|
|
420
|
+
case "columnar-table":
|
|
421
|
+
return ColumnarTableBatchAggregator;
|
|
422
|
+
case "arrow-table":
|
|
423
|
+
if (!TableBatchBuilder.ArrowBatch) {
|
|
424
|
+
throw new Error(ERR_MESSAGE);
|
|
425
|
+
}
|
|
426
|
+
return TableBatchBuilder.ArrowBatch;
|
|
427
|
+
default:
|
|
428
|
+
throw new Error(ERR_MESSAGE);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
});
|
|
434
|
+
|
|
435
|
+
// ../schema/src/index.ts
|
|
436
|
+
var init_src = __esm({
|
|
437
|
+
"../schema/src/index.ts"() {
|
|
438
|
+
init_table_batch_builder();
|
|
439
|
+
}
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
// ../loader-utils/src/lib/iterators/text-iterators.ts
|
|
443
|
+
async function* makeTextDecoderIterator(arrayBufferIterator, options = {}) {
|
|
444
|
+
const textDecoder = new TextDecoder(void 0, options);
|
|
445
|
+
for await (const arrayBuffer of arrayBufferIterator) {
|
|
446
|
+
yield typeof arrayBuffer === "string" ? arrayBuffer : textDecoder.decode(arrayBuffer, { stream: true });
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
async function* makeLineIterator(textIterator) {
|
|
450
|
+
let previous = "";
|
|
451
|
+
for await (const textChunk of textIterator) {
|
|
452
|
+
previous += textChunk;
|
|
453
|
+
let eolIndex;
|
|
454
|
+
while ((eolIndex = previous.indexOf("\n")) >= 0) {
|
|
455
|
+
const line = previous.slice(0, eolIndex + 1);
|
|
456
|
+
previous = previous.slice(eolIndex + 1);
|
|
457
|
+
yield line;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
if (previous.length > 0) {
|
|
461
|
+
yield previous;
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
async function* makeNumberedLineIterator(lineIterator) {
|
|
465
|
+
let counter = 1;
|
|
466
|
+
for await (const line of lineIterator) {
|
|
467
|
+
yield { counter, line };
|
|
468
|
+
counter++;
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
var init_text_iterators = __esm({
|
|
472
|
+
"../loader-utils/src/lib/iterators/text-iterators.ts"() {
|
|
473
|
+
}
|
|
474
|
+
});
|
|
475
|
+
|
|
476
|
+
// ../loader-utils/src/index.ts
|
|
477
|
+
var init_src2 = __esm({
|
|
478
|
+
"../loader-utils/src/index.ts"() {
|
|
479
|
+
init_text_iterators();
|
|
480
|
+
}
|
|
481
|
+
});
|
|
482
|
+
|
|
483
|
+
// src/lib/clarinet/clarinet.ts
|
|
484
|
+
function isWhitespace(c) {
|
|
485
|
+
return c === Char.carriageReturn || c === Char.lineFeed || c === Char.space || c === Char.tab;
|
|
486
|
+
}
|
|
487
|
+
function checkBufferLength(parser) {
|
|
488
|
+
const maxAllowed = Math.max(MAX_BUFFER_LENGTH, 10);
|
|
489
|
+
let maxActual = 0;
|
|
490
|
+
for (const buffer of ["textNode", "numberNode"]) {
|
|
491
|
+
const len = parser[buffer] === void 0 ? 0 : parser[buffer].length;
|
|
492
|
+
if (len > maxAllowed) {
|
|
493
|
+
switch (buffer) {
|
|
494
|
+
case "text":
|
|
495
|
+
break;
|
|
496
|
+
default:
|
|
497
|
+
parser._error(`Max buffer length exceeded: ${buffer}`);
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
maxActual = Math.max(maxActual, len);
|
|
501
|
+
}
|
|
502
|
+
parser.bufferCheckPosition = MAX_BUFFER_LENGTH - maxActual + parser.position;
|
|
503
|
+
}
|
|
504
|
+
var MAX_BUFFER_LENGTH, STATE, Char, stringTokenPattern, DEFAULT_OPTIONS2, ClarinetParser;
|
|
505
|
+
var init_clarinet = __esm({
|
|
506
|
+
"src/lib/clarinet/clarinet.ts"() {
|
|
507
|
+
MAX_BUFFER_LENGTH = Number.MAX_SAFE_INTEGER;
|
|
508
|
+
(function(STATE2) {
|
|
509
|
+
STATE2[STATE2["BEGIN"] = 0] = "BEGIN";
|
|
510
|
+
STATE2[STATE2["VALUE"] = 1] = "VALUE";
|
|
511
|
+
STATE2[STATE2["OPEN_OBJECT"] = 2] = "OPEN_OBJECT";
|
|
512
|
+
STATE2[STATE2["CLOSE_OBJECT"] = 3] = "CLOSE_OBJECT";
|
|
513
|
+
STATE2[STATE2["OPEN_ARRAY"] = 4] = "OPEN_ARRAY";
|
|
514
|
+
STATE2[STATE2["CLOSE_ARRAY"] = 5] = "CLOSE_ARRAY";
|
|
515
|
+
STATE2[STATE2["TEXT_ESCAPE"] = 6] = "TEXT_ESCAPE";
|
|
516
|
+
STATE2[STATE2["STRING"] = 7] = "STRING";
|
|
517
|
+
STATE2[STATE2["BACKSLASH"] = 8] = "BACKSLASH";
|
|
518
|
+
STATE2[STATE2["END"] = 9] = "END";
|
|
519
|
+
STATE2[STATE2["OPEN_KEY"] = 10] = "OPEN_KEY";
|
|
520
|
+
STATE2[STATE2["CLOSE_KEY"] = 11] = "CLOSE_KEY";
|
|
521
|
+
STATE2[STATE2["TRUE"] = 12] = "TRUE";
|
|
522
|
+
STATE2[STATE2["TRUE2"] = 13] = "TRUE2";
|
|
523
|
+
STATE2[STATE2["TRUE3"] = 14] = "TRUE3";
|
|
524
|
+
STATE2[STATE2["FALSE"] = 15] = "FALSE";
|
|
525
|
+
STATE2[STATE2["FALSE2"] = 16] = "FALSE2";
|
|
526
|
+
STATE2[STATE2["FALSE3"] = 17] = "FALSE3";
|
|
527
|
+
STATE2[STATE2["FALSE4"] = 18] = "FALSE4";
|
|
528
|
+
STATE2[STATE2["NULL"] = 19] = "NULL";
|
|
529
|
+
STATE2[STATE2["NULL2"] = 20] = "NULL2";
|
|
530
|
+
STATE2[STATE2["NULL3"] = 21] = "NULL3";
|
|
531
|
+
STATE2[STATE2["NUMBER_DECIMAL_POINT"] = 22] = "NUMBER_DECIMAL_POINT";
|
|
532
|
+
STATE2[STATE2["NUMBER_DIGIT"] = 23] = "NUMBER_DIGIT";
|
|
533
|
+
})(STATE || (STATE = {}));
|
|
534
|
+
Char = {
|
|
535
|
+
tab: 9,
|
|
536
|
+
lineFeed: 10,
|
|
537
|
+
carriageReturn: 13,
|
|
538
|
+
space: 32,
|
|
539
|
+
doubleQuote: 34,
|
|
540
|
+
plus: 43,
|
|
541
|
+
comma: 44,
|
|
542
|
+
minus: 45,
|
|
543
|
+
period: 46,
|
|
544
|
+
_0: 48,
|
|
545
|
+
_9: 57,
|
|
546
|
+
colon: 58,
|
|
547
|
+
E: 69,
|
|
548
|
+
openBracket: 91,
|
|
549
|
+
backslash: 92,
|
|
550
|
+
closeBracket: 93,
|
|
551
|
+
a: 97,
|
|
552
|
+
b: 98,
|
|
553
|
+
e: 101,
|
|
554
|
+
f: 102,
|
|
555
|
+
l: 108,
|
|
556
|
+
n: 110,
|
|
557
|
+
r: 114,
|
|
558
|
+
s: 115,
|
|
559
|
+
t: 116,
|
|
560
|
+
u: 117,
|
|
561
|
+
openBrace: 123,
|
|
562
|
+
closeBrace: 125
|
|
563
|
+
};
|
|
564
|
+
stringTokenPattern = /[\\"\n]/g;
|
|
565
|
+
DEFAULT_OPTIONS2 = {
|
|
566
|
+
onready: () => {
|
|
567
|
+
},
|
|
568
|
+
onopenobject: () => {
|
|
569
|
+
},
|
|
570
|
+
onkey: () => {
|
|
571
|
+
},
|
|
572
|
+
oncloseobject: () => {
|
|
573
|
+
},
|
|
574
|
+
onopenarray: () => {
|
|
575
|
+
},
|
|
576
|
+
onclosearray: () => {
|
|
577
|
+
},
|
|
578
|
+
onvalue: () => {
|
|
579
|
+
},
|
|
580
|
+
onerror: () => {
|
|
581
|
+
},
|
|
582
|
+
onend: () => {
|
|
583
|
+
},
|
|
584
|
+
onchunkparsed: () => {
|
|
585
|
+
}
|
|
586
|
+
};
|
|
587
|
+
ClarinetParser = class {
|
|
588
|
+
constructor(options = {}) {
|
|
589
|
+
this.options = DEFAULT_OPTIONS2;
|
|
590
|
+
this.bufferCheckPosition = MAX_BUFFER_LENGTH;
|
|
591
|
+
this.q = "";
|
|
592
|
+
this.c = "";
|
|
593
|
+
this.p = "";
|
|
594
|
+
this.closed = false;
|
|
595
|
+
this.closedRoot = false;
|
|
596
|
+
this.sawRoot = false;
|
|
597
|
+
this.error = null;
|
|
598
|
+
this.state = 0;
|
|
599
|
+
this.stack = [];
|
|
600
|
+
this.position = 0;
|
|
601
|
+
this.column = 0;
|
|
602
|
+
this.line = 1;
|
|
603
|
+
this.slashed = false;
|
|
604
|
+
this.unicodeI = 0;
|
|
605
|
+
this.unicodeS = null;
|
|
606
|
+
this.depth = 0;
|
|
607
|
+
this.options = { ...DEFAULT_OPTIONS2, ...options };
|
|
608
|
+
this.textNode = void 0;
|
|
609
|
+
this.numberNode = "";
|
|
610
|
+
this.emit("onready");
|
|
611
|
+
}
|
|
612
|
+
end() {
|
|
613
|
+
if (this.state !== 1 || this.depth !== 0)
|
|
614
|
+
this._error("Unexpected end");
|
|
615
|
+
this._closeValue();
|
|
616
|
+
this.c = "";
|
|
617
|
+
this.closed = true;
|
|
618
|
+
this.emit("onend");
|
|
619
|
+
return this;
|
|
620
|
+
}
|
|
621
|
+
resume() {
|
|
622
|
+
this.error = null;
|
|
623
|
+
return this;
|
|
624
|
+
}
|
|
625
|
+
close() {
|
|
626
|
+
return this.write(null);
|
|
627
|
+
}
|
|
628
|
+
emit(event, data) {
|
|
629
|
+
this.options[event]?.(data, this);
|
|
630
|
+
}
|
|
631
|
+
emitNode(event, data) {
|
|
632
|
+
this._closeValue();
|
|
633
|
+
this.emit(event, data);
|
|
634
|
+
}
|
|
635
|
+
write(chunk) {
|
|
636
|
+
if (this.error) {
|
|
637
|
+
throw this.error;
|
|
638
|
+
}
|
|
639
|
+
if (this.closed) {
|
|
640
|
+
return this._error("Cannot write after close. Assign an onready handler.");
|
|
641
|
+
}
|
|
642
|
+
if (chunk === null) {
|
|
643
|
+
return this.end();
|
|
644
|
+
}
|
|
645
|
+
let i = 0;
|
|
646
|
+
let c = chunk.charCodeAt(0);
|
|
647
|
+
let p = this.p;
|
|
648
|
+
while (c) {
|
|
649
|
+
p = c;
|
|
650
|
+
this.c = c = chunk.charCodeAt(i++);
|
|
651
|
+
if (p !== c) {
|
|
652
|
+
this.p = p;
|
|
653
|
+
} else {
|
|
654
|
+
p = this.p;
|
|
655
|
+
}
|
|
656
|
+
if (!c)
|
|
657
|
+
break;
|
|
658
|
+
this.position++;
|
|
659
|
+
if (c === Char.lineFeed) {
|
|
660
|
+
this.line++;
|
|
661
|
+
this.column = 0;
|
|
662
|
+
} else
|
|
663
|
+
this.column++;
|
|
664
|
+
switch (this.state) {
|
|
665
|
+
case 0:
|
|
666
|
+
if (c === Char.openBrace)
|
|
667
|
+
this.state = 2;
|
|
668
|
+
else if (c === Char.openBracket)
|
|
669
|
+
this.state = 4;
|
|
670
|
+
else if (!isWhitespace(c)) {
|
|
671
|
+
this._error("Non-whitespace before {[.");
|
|
672
|
+
}
|
|
673
|
+
continue;
|
|
674
|
+
case 10:
|
|
675
|
+
case 2:
|
|
676
|
+
if (isWhitespace(c))
|
|
677
|
+
continue;
|
|
678
|
+
if (this.state === 10)
|
|
679
|
+
this.stack.push(11);
|
|
680
|
+
else if (c === Char.closeBrace) {
|
|
681
|
+
this.emit("onopenobject");
|
|
682
|
+
this.depth++;
|
|
683
|
+
this.emit("oncloseobject");
|
|
684
|
+
this.depth--;
|
|
685
|
+
this.state = this.stack.pop() || 1;
|
|
686
|
+
continue;
|
|
687
|
+
} else
|
|
688
|
+
this.stack.push(3);
|
|
689
|
+
if (c === Char.doubleQuote)
|
|
690
|
+
this.state = 7;
|
|
691
|
+
else
|
|
692
|
+
this._error('Malformed object key should start with "');
|
|
693
|
+
continue;
|
|
694
|
+
case 11:
|
|
695
|
+
case 3:
|
|
696
|
+
if (isWhitespace(c))
|
|
697
|
+
continue;
|
|
698
|
+
if (c === Char.colon) {
|
|
699
|
+
if (this.state === 3) {
|
|
700
|
+
this.stack.push(3);
|
|
701
|
+
this._closeValue("onopenobject");
|
|
702
|
+
this.depth++;
|
|
703
|
+
} else
|
|
704
|
+
this._closeValue("onkey");
|
|
705
|
+
this.state = 1;
|
|
706
|
+
} else if (c === Char.closeBrace) {
|
|
707
|
+
this.emitNode("oncloseobject");
|
|
708
|
+
this.depth--;
|
|
709
|
+
this.state = this.stack.pop() || 1;
|
|
710
|
+
} else if (c === Char.comma) {
|
|
711
|
+
if (this.state === 3)
|
|
712
|
+
this.stack.push(3);
|
|
713
|
+
this._closeValue();
|
|
714
|
+
this.state = 10;
|
|
715
|
+
} else
|
|
716
|
+
this._error("Bad object");
|
|
717
|
+
continue;
|
|
718
|
+
case 4:
|
|
719
|
+
case 1:
|
|
720
|
+
if (isWhitespace(c))
|
|
721
|
+
continue;
|
|
722
|
+
if (this.state === 4) {
|
|
723
|
+
this.emit("onopenarray");
|
|
724
|
+
this.depth++;
|
|
725
|
+
this.state = 1;
|
|
726
|
+
if (c === Char.closeBracket) {
|
|
727
|
+
this.emit("onclosearray");
|
|
728
|
+
this.depth--;
|
|
729
|
+
this.state = this.stack.pop() || 1;
|
|
730
|
+
continue;
|
|
731
|
+
} else {
|
|
732
|
+
this.stack.push(5);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
if (c === Char.doubleQuote)
|
|
736
|
+
this.state = 7;
|
|
737
|
+
else if (c === Char.openBrace)
|
|
738
|
+
this.state = 2;
|
|
739
|
+
else if (c === Char.openBracket)
|
|
740
|
+
this.state = 4;
|
|
741
|
+
else if (c === Char.t)
|
|
742
|
+
this.state = 12;
|
|
743
|
+
else if (c === Char.f)
|
|
744
|
+
this.state = 15;
|
|
745
|
+
else if (c === Char.n)
|
|
746
|
+
this.state = 19;
|
|
747
|
+
else if (c === Char.minus) {
|
|
748
|
+
this.numberNode += "-";
|
|
749
|
+
} else if (Char._0 <= c && c <= Char._9) {
|
|
750
|
+
this.numberNode += String.fromCharCode(c);
|
|
751
|
+
this.state = 23;
|
|
752
|
+
} else
|
|
753
|
+
this._error("Bad value");
|
|
754
|
+
continue;
|
|
755
|
+
case 5:
|
|
756
|
+
if (c === Char.comma) {
|
|
757
|
+
this.stack.push(5);
|
|
758
|
+
this._closeValue("onvalue");
|
|
759
|
+
this.state = 1;
|
|
760
|
+
} else if (c === Char.closeBracket) {
|
|
761
|
+
this.emitNode("onclosearray");
|
|
762
|
+
this.depth--;
|
|
763
|
+
this.state = this.stack.pop() || 1;
|
|
764
|
+
} else if (isWhitespace(c))
|
|
765
|
+
continue;
|
|
766
|
+
else
|
|
767
|
+
this._error("Bad array");
|
|
768
|
+
continue;
|
|
769
|
+
case 7:
|
|
770
|
+
if (this.textNode === void 0) {
|
|
771
|
+
this.textNode = "";
|
|
772
|
+
}
|
|
773
|
+
let starti = i - 1;
|
|
774
|
+
let slashed = this.slashed;
|
|
775
|
+
let unicodeI = this.unicodeI;
|
|
776
|
+
STRING_BIGLOOP:
|
|
777
|
+
while (true) {
|
|
778
|
+
while (unicodeI > 0) {
|
|
779
|
+
this.unicodeS += String.fromCharCode(c);
|
|
780
|
+
c = chunk.charCodeAt(i++);
|
|
781
|
+
this.position++;
|
|
782
|
+
if (unicodeI === 4) {
|
|
783
|
+
this.textNode += String.fromCharCode(parseInt(this.unicodeS, 16));
|
|
784
|
+
unicodeI = 0;
|
|
785
|
+
starti = i - 1;
|
|
786
|
+
} else {
|
|
787
|
+
unicodeI++;
|
|
788
|
+
}
|
|
789
|
+
if (!c)
|
|
790
|
+
break STRING_BIGLOOP;
|
|
791
|
+
}
|
|
792
|
+
if (c === Char.doubleQuote && !slashed) {
|
|
793
|
+
this.state = this.stack.pop() || 1;
|
|
794
|
+
this.textNode += chunk.substring(starti, i - 1);
|
|
795
|
+
this.position += i - 1 - starti;
|
|
796
|
+
break;
|
|
797
|
+
}
|
|
798
|
+
if (c === Char.backslash && !slashed) {
|
|
799
|
+
slashed = true;
|
|
800
|
+
this.textNode += chunk.substring(starti, i - 1);
|
|
801
|
+
this.position += i - 1 - starti;
|
|
802
|
+
c = chunk.charCodeAt(i++);
|
|
803
|
+
this.position++;
|
|
804
|
+
if (!c)
|
|
805
|
+
break;
|
|
806
|
+
}
|
|
807
|
+
if (slashed) {
|
|
808
|
+
slashed = false;
|
|
809
|
+
if (c === Char.n) {
|
|
810
|
+
this.textNode += "\n";
|
|
811
|
+
} else if (c === Char.r) {
|
|
812
|
+
this.textNode += "\r";
|
|
813
|
+
} else if (c === Char.t) {
|
|
814
|
+
this.textNode += " ";
|
|
815
|
+
} else if (c === Char.f) {
|
|
816
|
+
this.textNode += "\f";
|
|
817
|
+
} else if (c === Char.b) {
|
|
818
|
+
this.textNode += "\b";
|
|
819
|
+
} else if (c === Char.u) {
|
|
820
|
+
unicodeI = 1;
|
|
821
|
+
this.unicodeS = "";
|
|
822
|
+
} else {
|
|
823
|
+
this.textNode += String.fromCharCode(c);
|
|
824
|
+
}
|
|
825
|
+
c = chunk.charCodeAt(i++);
|
|
826
|
+
this.position++;
|
|
827
|
+
starti = i - 1;
|
|
828
|
+
if (!c)
|
|
829
|
+
break;
|
|
830
|
+
else
|
|
831
|
+
continue;
|
|
832
|
+
}
|
|
833
|
+
stringTokenPattern.lastIndex = i;
|
|
834
|
+
const reResult = stringTokenPattern.exec(chunk);
|
|
835
|
+
if (reResult === null) {
|
|
836
|
+
i = chunk.length + 1;
|
|
837
|
+
this.textNode += chunk.substring(starti, i - 1);
|
|
838
|
+
this.position += i - 1 - starti;
|
|
839
|
+
break;
|
|
840
|
+
}
|
|
841
|
+
i = reResult.index + 1;
|
|
842
|
+
c = chunk.charCodeAt(reResult.index);
|
|
843
|
+
if (!c) {
|
|
844
|
+
this.textNode += chunk.substring(starti, i - 1);
|
|
845
|
+
this.position += i - 1 - starti;
|
|
846
|
+
break;
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
this.slashed = slashed;
|
|
850
|
+
this.unicodeI = unicodeI;
|
|
851
|
+
continue;
|
|
852
|
+
case 12:
|
|
853
|
+
if (c === Char.r)
|
|
854
|
+
this.state = 13;
|
|
855
|
+
else
|
|
856
|
+
this._error(`Invalid true started with t${c}`);
|
|
857
|
+
continue;
|
|
858
|
+
case 13:
|
|
859
|
+
if (c === Char.u)
|
|
860
|
+
this.state = 14;
|
|
861
|
+
else
|
|
862
|
+
this._error(`Invalid true started with tr${c}`);
|
|
863
|
+
continue;
|
|
864
|
+
case 14:
|
|
865
|
+
if (c === Char.e) {
|
|
866
|
+
this.emit("onvalue", true);
|
|
867
|
+
this.state = this.stack.pop() || 1;
|
|
868
|
+
} else
|
|
869
|
+
this._error(`Invalid true started with tru${c}`);
|
|
870
|
+
continue;
|
|
871
|
+
case 15:
|
|
872
|
+
if (c === Char.a)
|
|
873
|
+
this.state = 16;
|
|
874
|
+
else
|
|
875
|
+
this._error(`Invalid false started with f${c}`);
|
|
876
|
+
continue;
|
|
877
|
+
case 16:
|
|
878
|
+
if (c === Char.l)
|
|
879
|
+
this.state = 17;
|
|
880
|
+
else
|
|
881
|
+
this._error(`Invalid false started with fa${c}`);
|
|
882
|
+
continue;
|
|
883
|
+
case 17:
|
|
884
|
+
if (c === Char.s)
|
|
885
|
+
this.state = 18;
|
|
886
|
+
else
|
|
887
|
+
this._error(`Invalid false started with fal${c}`);
|
|
888
|
+
continue;
|
|
889
|
+
case 18:
|
|
890
|
+
if (c === Char.e) {
|
|
891
|
+
this.emit("onvalue", false);
|
|
892
|
+
this.state = this.stack.pop() || 1;
|
|
893
|
+
} else
|
|
894
|
+
this._error(`Invalid false started with fals${c}`);
|
|
895
|
+
continue;
|
|
896
|
+
case 19:
|
|
897
|
+
if (c === Char.u)
|
|
898
|
+
this.state = 20;
|
|
899
|
+
else
|
|
900
|
+
this._error(`Invalid null started with n${c}`);
|
|
901
|
+
continue;
|
|
902
|
+
case 20:
|
|
903
|
+
if (c === Char.l)
|
|
904
|
+
this.state = 21;
|
|
905
|
+
else
|
|
906
|
+
this._error(`Invalid null started with nu${c}`);
|
|
907
|
+
continue;
|
|
908
|
+
case 21:
|
|
909
|
+
if (c === Char.l) {
|
|
910
|
+
this.emit("onvalue", null);
|
|
911
|
+
this.state = this.stack.pop() || 1;
|
|
912
|
+
} else
|
|
913
|
+
this._error(`Invalid null started with nul${c}`);
|
|
914
|
+
continue;
|
|
915
|
+
case 22:
|
|
916
|
+
if (c === Char.period) {
|
|
917
|
+
this.numberNode += ".";
|
|
918
|
+
this.state = 23;
|
|
919
|
+
} else
|
|
920
|
+
this._error("Leading zero not followed by .");
|
|
921
|
+
continue;
|
|
922
|
+
case 23:
|
|
923
|
+
if (Char._0 <= c && c <= Char._9)
|
|
924
|
+
this.numberNode += String.fromCharCode(c);
|
|
925
|
+
else if (c === Char.period) {
|
|
926
|
+
if (this.numberNode.indexOf(".") !== -1)
|
|
927
|
+
this._error("Invalid number has two dots");
|
|
928
|
+
this.numberNode += ".";
|
|
929
|
+
} else if (c === Char.e || c === Char.E) {
|
|
930
|
+
if (this.numberNode.indexOf("e") !== -1 || this.numberNode.indexOf("E") !== -1)
|
|
931
|
+
this._error("Invalid number has two exponential");
|
|
932
|
+
this.numberNode += "e";
|
|
933
|
+
} else if (c === Char.plus || c === Char.minus) {
|
|
934
|
+
if (!(p === Char.e || p === Char.E))
|
|
935
|
+
this._error("Invalid symbol in number");
|
|
936
|
+
this.numberNode += String.fromCharCode(c);
|
|
937
|
+
} else {
|
|
938
|
+
this._closeNumber();
|
|
939
|
+
i--;
|
|
940
|
+
this.state = this.stack.pop() || 1;
|
|
941
|
+
}
|
|
942
|
+
continue;
|
|
943
|
+
default:
|
|
944
|
+
this._error(`Unknown state: ${this.state}`);
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
if (this.position >= this.bufferCheckPosition) {
|
|
948
|
+
checkBufferLength(this);
|
|
949
|
+
}
|
|
950
|
+
this.emit("onchunkparsed");
|
|
951
|
+
return this;
|
|
952
|
+
}
|
|
953
|
+
_closeValue(event = "onvalue") {
|
|
954
|
+
if (this.textNode !== void 0) {
|
|
955
|
+
this.emit(event, this.textNode);
|
|
956
|
+
}
|
|
957
|
+
this.textNode = void 0;
|
|
958
|
+
}
|
|
959
|
+
_closeNumber() {
|
|
960
|
+
if (this.numberNode)
|
|
961
|
+
this.emit("onvalue", parseFloat(this.numberNode));
|
|
962
|
+
this.numberNode = "";
|
|
963
|
+
}
|
|
964
|
+
_error(message = "") {
|
|
965
|
+
this._closeValue();
|
|
966
|
+
message += `
|
|
967
|
+
Line: ${this.line}
|
|
968
|
+
Column: ${this.column}
|
|
969
|
+
Char: ${this.c}`;
|
|
970
|
+
const error = new Error(message);
|
|
971
|
+
this.error = error;
|
|
972
|
+
this.emit("onerror", error);
|
|
973
|
+
}
|
|
974
|
+
};
|
|
975
|
+
}
|
|
976
|
+
});
|
|
977
|
+
|
|
978
|
+
// src/lib/jsonpath/jsonpath.ts
|
|
979
|
+
var JSONPath;
|
|
980
|
+
var init_jsonpath = __esm({
|
|
981
|
+
"src/lib/jsonpath/jsonpath.ts"() {
|
|
982
|
+
JSONPath = class {
|
|
983
|
+
constructor(path = null) {
|
|
984
|
+
this.path = ["$"];
|
|
985
|
+
if (path instanceof JSONPath) {
|
|
986
|
+
this.path = [...path.path];
|
|
987
|
+
return;
|
|
988
|
+
}
|
|
989
|
+
if (Array.isArray(path)) {
|
|
990
|
+
this.path.push(...path);
|
|
991
|
+
return;
|
|
992
|
+
}
|
|
993
|
+
if (typeof path === "string") {
|
|
994
|
+
this.path = path.split(".");
|
|
995
|
+
if (this.path[0] !== "$") {
|
|
996
|
+
throw new Error("JSONPaths must start with $");
|
|
997
|
+
}
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
clone() {
|
|
1001
|
+
return new JSONPath(this);
|
|
1002
|
+
}
|
|
1003
|
+
toString() {
|
|
1004
|
+
return this.path.join(".");
|
|
1005
|
+
}
|
|
1006
|
+
push(name) {
|
|
1007
|
+
this.path.push(name);
|
|
1008
|
+
}
|
|
1009
|
+
pop() {
|
|
1010
|
+
return this.path.pop();
|
|
1011
|
+
}
|
|
1012
|
+
set(name) {
|
|
1013
|
+
this.path[this.path.length - 1] = name;
|
|
1014
|
+
}
|
|
1015
|
+
equals(other) {
|
|
1016
|
+
if (!this || !other || this.path.length !== other.path.length) {
|
|
1017
|
+
return false;
|
|
1018
|
+
}
|
|
1019
|
+
for (let i = 0; i < this.path.length; ++i) {
|
|
1020
|
+
if (this.path[i] !== other.path[i]) {
|
|
1021
|
+
return false;
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
return true;
|
|
1025
|
+
}
|
|
1026
|
+
setFieldAtPath(object, value) {
|
|
1027
|
+
const path = [...this.path];
|
|
1028
|
+
path.shift();
|
|
1029
|
+
const field = path.pop();
|
|
1030
|
+
for (const component of path) {
|
|
1031
|
+
object = object[component];
|
|
1032
|
+
}
|
|
1033
|
+
object[field] = value;
|
|
1034
|
+
}
|
|
1035
|
+
getFieldAtPath(object) {
|
|
1036
|
+
const path = [...this.path];
|
|
1037
|
+
path.shift();
|
|
1038
|
+
const field = path.pop();
|
|
1039
|
+
for (const component of path) {
|
|
1040
|
+
object = object[component];
|
|
1041
|
+
}
|
|
1042
|
+
return object[field];
|
|
1043
|
+
}
|
|
1044
|
+
};
|
|
1045
|
+
}
|
|
1046
|
+
});
|
|
1047
|
+
|
|
1048
|
+
// src/lib/parser/json-parser.ts
|
|
1049
|
+
var JSONParser;
|
|
1050
|
+
var init_json_parser = __esm({
|
|
1051
|
+
"src/lib/parser/json-parser.ts"() {
|
|
1052
|
+
init_clarinet();
|
|
1053
|
+
init_jsonpath();
|
|
1054
|
+
JSONParser = class {
|
|
1055
|
+
constructor(options) {
|
|
1056
|
+
this.result = void 0;
|
|
1057
|
+
this.previousStates = [];
|
|
1058
|
+
this.currentState = Object.freeze({ container: [], key: null });
|
|
1059
|
+
this.jsonpath = new JSONPath();
|
|
1060
|
+
this.reset();
|
|
1061
|
+
this.parser = new ClarinetParser({
|
|
1062
|
+
onready: () => {
|
|
1063
|
+
this.jsonpath = new JSONPath();
|
|
1064
|
+
this.previousStates.length = 0;
|
|
1065
|
+
this.currentState.container.length = 0;
|
|
1066
|
+
},
|
|
1067
|
+
onopenobject: (name) => {
|
|
1068
|
+
this._openObject({});
|
|
1069
|
+
if (typeof name !== "undefined") {
|
|
1070
|
+
this.parser.emit("onkey", name);
|
|
1071
|
+
}
|
|
1072
|
+
},
|
|
1073
|
+
onkey: (name) => {
|
|
1074
|
+
this.jsonpath.set(name);
|
|
1075
|
+
this.currentState.key = name;
|
|
1076
|
+
},
|
|
1077
|
+
oncloseobject: () => {
|
|
1078
|
+
this._closeObject();
|
|
1079
|
+
},
|
|
1080
|
+
onopenarray: () => {
|
|
1081
|
+
this._openArray();
|
|
1082
|
+
},
|
|
1083
|
+
onclosearray: () => {
|
|
1084
|
+
this._closeArray();
|
|
1085
|
+
},
|
|
1086
|
+
onvalue: (value) => {
|
|
1087
|
+
this._pushOrSet(value);
|
|
1088
|
+
},
|
|
1089
|
+
onerror: (error) => {
|
|
1090
|
+
throw error;
|
|
1091
|
+
},
|
|
1092
|
+
onend: () => {
|
|
1093
|
+
this.result = this.currentState.container.pop();
|
|
1094
|
+
},
|
|
1095
|
+
...options
|
|
1096
|
+
});
|
|
1097
|
+
}
|
|
1098
|
+
reset() {
|
|
1099
|
+
this.result = void 0;
|
|
1100
|
+
this.previousStates = [];
|
|
1101
|
+
this.currentState = Object.freeze({ container: [], key: null });
|
|
1102
|
+
this.jsonpath = new JSONPath();
|
|
1103
|
+
}
|
|
1104
|
+
write(chunk) {
|
|
1105
|
+
this.parser.write(chunk);
|
|
1106
|
+
}
|
|
1107
|
+
close() {
|
|
1108
|
+
this.parser.close();
|
|
1109
|
+
}
|
|
1110
|
+
_pushOrSet(value) {
|
|
1111
|
+
const { container, key } = this.currentState;
|
|
1112
|
+
if (key !== null) {
|
|
1113
|
+
container[key] = value;
|
|
1114
|
+
this.currentState.key = null;
|
|
1115
|
+
} else {
|
|
1116
|
+
container.push(value);
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
_openArray(newContainer = []) {
|
|
1120
|
+
this.jsonpath.push(null);
|
|
1121
|
+
this._pushOrSet(newContainer);
|
|
1122
|
+
this.previousStates.push(this.currentState);
|
|
1123
|
+
this.currentState = { container: newContainer, isArray: true, key: null };
|
|
1124
|
+
}
|
|
1125
|
+
_closeArray() {
|
|
1126
|
+
this.jsonpath.pop();
|
|
1127
|
+
this.currentState = this.previousStates.pop();
|
|
1128
|
+
}
|
|
1129
|
+
_openObject(newContainer = {}) {
|
|
1130
|
+
this.jsonpath.push(null);
|
|
1131
|
+
this._pushOrSet(newContainer);
|
|
1132
|
+
this.previousStates.push(this.currentState);
|
|
1133
|
+
this.currentState = { container: newContainer, isArray: false, key: null };
|
|
1134
|
+
}
|
|
1135
|
+
_closeObject() {
|
|
1136
|
+
this.jsonpath.pop();
|
|
1137
|
+
this.currentState = this.previousStates.pop();
|
|
1138
|
+
}
|
|
1139
|
+
};
|
|
1140
|
+
}
|
|
1141
|
+
});
|
|
1142
|
+
|
|
1143
|
+
// src/lib/parser/streaming-json-parser.ts
|
|
1144
|
+
var StreamingJSONParser;
|
|
1145
|
+
var init_streaming_json_parser = __esm({
|
|
1146
|
+
"src/lib/parser/streaming-json-parser.ts"() {
|
|
1147
|
+
init_json_parser();
|
|
1148
|
+
init_jsonpath();
|
|
1149
|
+
StreamingJSONParser = class extends JSONParser {
|
|
1150
|
+
constructor(options = {}) {
|
|
1151
|
+
super({
|
|
1152
|
+
onopenarray: () => {
|
|
1153
|
+
if (!this.streamingArray) {
|
|
1154
|
+
if (this._matchJSONPath()) {
|
|
1155
|
+
this.streamingJsonPath = this.getJsonPath().clone();
|
|
1156
|
+
this.streamingArray = [];
|
|
1157
|
+
this._openArray(this.streamingArray);
|
|
1158
|
+
return;
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
this._openArray();
|
|
1162
|
+
},
|
|
1163
|
+
onopenobject: (name) => {
|
|
1164
|
+
if (!this.topLevelObject) {
|
|
1165
|
+
this.topLevelObject = {};
|
|
1166
|
+
this._openObject(this.topLevelObject);
|
|
1167
|
+
} else {
|
|
1168
|
+
this._openObject({});
|
|
1169
|
+
}
|
|
1170
|
+
if (typeof name !== "undefined") {
|
|
1171
|
+
this.parser.emit("onkey", name);
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
});
|
|
1175
|
+
this.streamingJsonPath = null;
|
|
1176
|
+
this.streamingArray = null;
|
|
1177
|
+
this.topLevelObject = null;
|
|
1178
|
+
const jsonpaths = options.jsonpaths || [];
|
|
1179
|
+
this.jsonPaths = jsonpaths.map((jsonpath) => new JSONPath(jsonpath));
|
|
1180
|
+
}
|
|
1181
|
+
write(chunk) {
|
|
1182
|
+
super.write(chunk);
|
|
1183
|
+
let array = [];
|
|
1184
|
+
if (this.streamingArray) {
|
|
1185
|
+
array = [...this.streamingArray];
|
|
1186
|
+
this.streamingArray.length = 0;
|
|
1187
|
+
}
|
|
1188
|
+
return array;
|
|
1189
|
+
}
|
|
1190
|
+
getPartialResult() {
|
|
1191
|
+
return this.topLevelObject;
|
|
1192
|
+
}
|
|
1193
|
+
getStreamingJsonPath() {
|
|
1194
|
+
return this.streamingJsonPath;
|
|
1195
|
+
}
|
|
1196
|
+
getStreamingJsonPathAsString() {
|
|
1197
|
+
return this.streamingJsonPath && this.streamingJsonPath.toString();
|
|
1198
|
+
}
|
|
1199
|
+
getJsonPath() {
|
|
1200
|
+
return this.jsonpath;
|
|
1201
|
+
}
|
|
1202
|
+
_matchJSONPath() {
|
|
1203
|
+
const currentPath = this.getJsonPath();
|
|
1204
|
+
if (this.jsonPaths.length === 0) {
|
|
1205
|
+
return true;
|
|
1206
|
+
}
|
|
1207
|
+
for (const jsonPath of this.jsonPaths) {
|
|
1208
|
+
if (jsonPath.equals(currentPath)) {
|
|
1209
|
+
return true;
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
return false;
|
|
1213
|
+
}
|
|
1214
|
+
};
|
|
1215
|
+
}
|
|
1216
|
+
});
|
|
1217
|
+
|
|
1218
|
+
// src/lib/parse-json-in-batches.ts
|
|
1219
|
+
async function* parseJSONInBatches(binaryAsyncIterator, options) {
|
|
1220
|
+
const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);
|
|
1221
|
+
const { metadata } = options;
|
|
1222
|
+
const { jsonpaths } = options.json || {};
|
|
1223
|
+
let isFirstChunk = true;
|
|
1224
|
+
const schema = null;
|
|
1225
|
+
const shape = options?.json?.shape || "row-table";
|
|
1226
|
+
const tableBatchBuilder = new TableBatchBuilder(schema, {
|
|
1227
|
+
...options,
|
|
1228
|
+
shape
|
|
1229
|
+
});
|
|
1230
|
+
const parser = new StreamingJSONParser({ jsonpaths });
|
|
1231
|
+
for await (const chunk of asyncIterator) {
|
|
1232
|
+
const rows = parser.write(chunk);
|
|
1233
|
+
const jsonpath2 = rows.length > 0 && parser.getStreamingJsonPathAsString();
|
|
1234
|
+
if (rows.length > 0 && isFirstChunk) {
|
|
1235
|
+
if (metadata) {
|
|
1236
|
+
const initialBatch = {
|
|
1237
|
+
shape,
|
|
1238
|
+
batchType: "partial-result",
|
|
1239
|
+
data: [],
|
|
1240
|
+
length: 0,
|
|
1241
|
+
bytesUsed: 0,
|
|
1242
|
+
container: parser.getPartialResult(),
|
|
1243
|
+
jsonpath: jsonpath2
|
|
1244
|
+
};
|
|
1245
|
+
yield initialBatch;
|
|
1246
|
+
}
|
|
1247
|
+
isFirstChunk = false;
|
|
1248
|
+
}
|
|
1249
|
+
for (const row of rows) {
|
|
1250
|
+
tableBatchBuilder.addRow(row);
|
|
1251
|
+
const batch3 = tableBatchBuilder.getFullBatch({ jsonpath: jsonpath2 });
|
|
1252
|
+
if (batch3) {
|
|
1253
|
+
yield batch3;
|
|
1254
|
+
}
|
|
1255
|
+
}
|
|
1256
|
+
tableBatchBuilder.chunkComplete(chunk);
|
|
1257
|
+
const batch2 = tableBatchBuilder.getFullBatch({ jsonpath: jsonpath2 });
|
|
1258
|
+
if (batch2) {
|
|
1259
|
+
yield batch2;
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
const jsonpath = parser.getStreamingJsonPathAsString();
|
|
1263
|
+
const batch = tableBatchBuilder.getFinalBatch({ jsonpath });
|
|
1264
|
+
if (batch) {
|
|
1265
|
+
yield batch;
|
|
1266
|
+
}
|
|
1267
|
+
if (metadata) {
|
|
1268
|
+
const finalBatch = {
|
|
1269
|
+
shape,
|
|
1270
|
+
batchType: "final-result",
|
|
1271
|
+
container: parser.getPartialResult(),
|
|
1272
|
+
jsonpath: parser.getStreamingJsonPathAsString(),
|
|
1273
|
+
data: [],
|
|
1274
|
+
length: 0
|
|
1275
|
+
};
|
|
1276
|
+
yield finalBatch;
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
var init_parse_json_in_batches = __esm({
|
|
1280
|
+
"src/lib/parse-json-in-batches.ts"() {
|
|
1281
|
+
init_src();
|
|
1282
|
+
init_src2();
|
|
1283
|
+
init_streaming_json_parser();
|
|
1284
|
+
}
|
|
1285
|
+
});
|
|
1286
|
+
|
|
1287
|
+
// src/json-loader.ts
|
|
1288
|
+
async function parse(arrayBuffer, options) {
|
|
1289
|
+
return parseTextSync(new TextDecoder().decode(arrayBuffer), options);
|
|
1290
|
+
}
|
|
1291
|
+
function parseTextSync(text, options) {
|
|
1292
|
+
const jsonOptions = { ...options, json: { ...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json } };
|
|
1293
|
+
return parseJSONSync(text, jsonOptions);
|
|
1294
|
+
}
|
|
1295
|
+
function parseInBatches(asyncIterator, options) {
|
|
1296
|
+
const jsonOptions = { ...options, json: { ...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json } };
|
|
1297
|
+
return parseJSONInBatches(asyncIterator, jsonOptions);
|
|
1298
|
+
}
|
|
1299
|
+
var VERSION, DEFAULT_JSON_LOADER_OPTIONS, JSONLoader;
|
|
1300
|
+
var init_json_loader = __esm({
|
|
1301
|
+
"src/json-loader.ts"() {
|
|
1302
|
+
init_parse_json();
|
|
1303
|
+
init_parse_json_in_batches();
|
|
1304
|
+
VERSION = typeof __VERSION__ !== "undefined" ? __VERSION__ : "latest";
|
|
1305
|
+
DEFAULT_JSON_LOADER_OPTIONS = {
|
|
1306
|
+
json: {
|
|
1307
|
+
shape: "row-table",
|
|
1308
|
+
table: false,
|
|
1309
|
+
jsonpaths: []
|
|
1310
|
+
}
|
|
1311
|
+
};
|
|
1312
|
+
JSONLoader = {
|
|
1313
|
+
name: "JSON",
|
|
1314
|
+
id: "json",
|
|
1315
|
+
module: "json",
|
|
1316
|
+
version: VERSION,
|
|
1317
|
+
extensions: ["json", "geojson"],
|
|
1318
|
+
mimeTypes: ["application/json"],
|
|
1319
|
+
category: "table",
|
|
1320
|
+
text: true,
|
|
1321
|
+
parse,
|
|
1322
|
+
parseTextSync,
|
|
1323
|
+
parseInBatches,
|
|
1324
|
+
options: DEFAULT_JSON_LOADER_OPTIONS
|
|
1325
|
+
};
|
|
1326
|
+
}
|
|
1327
|
+
});
|
|
1328
|
+
|
|
1329
|
+
// src/lib/parse-ndjson.ts
|
|
1330
|
+
function parseNDJSONSync(ndjsonText) {
|
|
1331
|
+
const lines = ndjsonText.trim().split("\n");
|
|
1332
|
+
return lines.map((line, counter) => {
|
|
1333
|
+
try {
|
|
1334
|
+
return JSON.parse(line);
|
|
1335
|
+
} catch (error) {
|
|
1336
|
+
throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
|
|
1337
|
+
}
|
|
1338
|
+
});
|
|
1339
|
+
}
|
|
1340
|
+
var init_parse_ndjson = __esm({
|
|
1341
|
+
"src/lib/parse-ndjson.ts"() {
|
|
1342
|
+
}
|
|
1343
|
+
});
|
|
1344
|
+
|
|
1345
|
+
// src/lib/parse-ndjson-in-batches.ts
|
|
1346
|
+
async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
|
|
1347
|
+
const textIterator = makeTextDecoderIterator(binaryAsyncIterator);
|
|
1348
|
+
const lineIterator = makeLineIterator(textIterator);
|
|
1349
|
+
const numberedLineIterator = makeNumberedLineIterator(lineIterator);
|
|
1350
|
+
const schema = null;
|
|
1351
|
+
const shape = "row-table";
|
|
1352
|
+
const tableBatchBuilder = new TableBatchBuilder(schema, {
|
|
1353
|
+
...options,
|
|
1354
|
+
shape
|
|
1355
|
+
});
|
|
1356
|
+
for await (const { counter, line } of numberedLineIterator) {
|
|
1357
|
+
try {
|
|
1358
|
+
const row = JSON.parse(line);
|
|
1359
|
+
tableBatchBuilder.addRow(row);
|
|
1360
|
+
tableBatchBuilder.chunkComplete(line);
|
|
1361
|
+
const batch2 = tableBatchBuilder.getFullBatch();
|
|
1362
|
+
if (batch2) {
|
|
1363
|
+
yield batch2;
|
|
1364
|
+
}
|
|
1365
|
+
} catch (error) {
|
|
1366
|
+
throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
const batch = tableBatchBuilder.getFinalBatch();
|
|
1370
|
+
if (batch) {
|
|
1371
|
+
yield batch;
|
|
1372
|
+
}
|
|
1373
|
+
}
|
|
1374
|
+
var init_parse_ndjson_in_batches = __esm({
|
|
1375
|
+
"src/lib/parse-ndjson-in-batches.ts"() {
|
|
1376
|
+
init_src();
|
|
1377
|
+
init_src2();
|
|
1378
|
+
}
|
|
1379
|
+
});
|
|
1380
|
+
|
|
1381
|
+
// src/ndjson-loader.ts
|
|
1382
|
+
async function parse2(arrayBuffer) {
|
|
1383
|
+
return parseTextSync2(new TextDecoder().decode(arrayBuffer));
|
|
1384
|
+
}
|
|
1385
|
+
function parseTextSync2(text) {
|
|
1386
|
+
return parseNDJSONSync(text);
|
|
1387
|
+
}
|
|
1388
|
+
function parseInBatches2(asyncIterator, options) {
|
|
1389
|
+
return parseNDJSONInBatches(asyncIterator, options);
|
|
1390
|
+
}
|
|
1391
|
+
var VERSION2, NDJSONLoader;
|
|
1392
|
+
var init_ndjson_loader = __esm({
|
|
1393
|
+
"src/ndjson-loader.ts"() {
|
|
1394
|
+
init_parse_ndjson();
|
|
1395
|
+
init_parse_ndjson_in_batches();
|
|
1396
|
+
VERSION2 = typeof __VERSION__ !== "undefined" ? __VERSION__ : "latest";
|
|
1397
|
+
NDJSONLoader = {
|
|
1398
|
+
name: "NDJSON",
|
|
1399
|
+
id: "ndjson",
|
|
1400
|
+
module: "json",
|
|
1401
|
+
version: VERSION2,
|
|
1402
|
+
extensions: ["ndjson"],
|
|
1403
|
+
mimeTypes: ["application/x-ndjson"],
|
|
1404
|
+
category: "table",
|
|
1405
|
+
text: true,
|
|
1406
|
+
parse: parse2,
|
|
1407
|
+
parseTextSync: parseTextSync2,
|
|
1408
|
+
parseInBatches: parseInBatches2,
|
|
1409
|
+
options: {}
|
|
1410
|
+
};
|
|
1411
|
+
}
|
|
1412
|
+
});
|
|
1413
|
+
|
|
1414
|
+
// ../gis/src/lib/geojson-to-binary.ts
|
|
1415
|
+
function geojsonToBinary(features, options = {}) {
|
|
1416
|
+
const firstPassData = firstPass(features);
|
|
1417
|
+
return secondPass(features, firstPassData, {
|
|
1418
|
+
coordLength: options.coordLength || firstPassData.coordLength,
|
|
1419
|
+
numericPropKeys: options.numericPropKeys || firstPassData.numericPropKeys,
|
|
1420
|
+
PositionDataType: options.PositionDataType || Float32Array
|
|
1421
|
+
});
|
|
1422
|
+
}
|
|
1423
|
+
function firstPass(features) {
|
|
1424
|
+
let pointPositionsCount = 0;
|
|
1425
|
+
let pointFeaturesCount = 0;
|
|
1426
|
+
let linePositionsCount = 0;
|
|
1427
|
+
let linePathsCount = 0;
|
|
1428
|
+
let lineFeaturesCount = 0;
|
|
1429
|
+
let polygonPositionsCount = 0;
|
|
1430
|
+
let polygonObjectsCount = 0;
|
|
1431
|
+
let polygonRingsCount = 0;
|
|
1432
|
+
let polygonFeaturesCount = 0;
|
|
1433
|
+
const coordLengths = new Set();
|
|
1434
|
+
const numericPropKeys = {};
|
|
1435
|
+
for (const feature of features) {
|
|
1436
|
+
const geometry = feature.geometry;
|
|
1437
|
+
switch (geometry.type) {
|
|
1438
|
+
case "Point":
|
|
1439
|
+
pointFeaturesCount++;
|
|
1440
|
+
pointPositionsCount++;
|
|
1441
|
+
coordLengths.add(geometry.coordinates.length);
|
|
1442
|
+
break;
|
|
1443
|
+
case "MultiPoint":
|
|
1444
|
+
pointFeaturesCount++;
|
|
1445
|
+
pointPositionsCount += geometry.coordinates.length;
|
|
1446
|
+
for (const point of geometry.coordinates) {
|
|
1447
|
+
coordLengths.add(point.length);
|
|
1448
|
+
}
|
|
1449
|
+
break;
|
|
1450
|
+
case "LineString":
|
|
1451
|
+
lineFeaturesCount++;
|
|
1452
|
+
linePositionsCount += geometry.coordinates.length;
|
|
1453
|
+
linePathsCount++;
|
|
1454
|
+
for (const coord of geometry.coordinates) {
|
|
1455
|
+
coordLengths.add(coord.length);
|
|
1456
|
+
}
|
|
1457
|
+
break;
|
|
1458
|
+
case "MultiLineString":
|
|
1459
|
+
lineFeaturesCount++;
|
|
1460
|
+
for (const line of geometry.coordinates) {
|
|
1461
|
+
linePositionsCount += line.length;
|
|
1462
|
+
linePathsCount++;
|
|
1463
|
+
for (const coord of line) {
|
|
1464
|
+
coordLengths.add(coord.length);
|
|
1465
|
+
}
|
|
1466
|
+
}
|
|
1467
|
+
break;
|
|
1468
|
+
case "Polygon":
|
|
1469
|
+
polygonFeaturesCount++;
|
|
1470
|
+
polygonObjectsCount++;
|
|
1471
|
+
polygonRingsCount += geometry.coordinates.length;
|
|
1472
|
+
polygonPositionsCount += flatten(geometry.coordinates).length;
|
|
1473
|
+
for (const coord of flatten(geometry.coordinates)) {
|
|
1474
|
+
coordLengths.add(coord.length);
|
|
1475
|
+
}
|
|
1476
|
+
break;
|
|
1477
|
+
case "MultiPolygon":
|
|
1478
|
+
polygonFeaturesCount++;
|
|
1479
|
+
for (const polygon of geometry.coordinates) {
|
|
1480
|
+
polygonObjectsCount++;
|
|
1481
|
+
polygonRingsCount += polygon.length;
|
|
1482
|
+
polygonPositionsCount += flatten(polygon).length;
|
|
1483
|
+
for (const coord of flatten(polygon)) {
|
|
1484
|
+
coordLengths.add(coord.length);
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
break;
|
|
1488
|
+
default:
|
|
1489
|
+
throw new Error(`Unsupported geometry type: ${geometry.type}`);
|
|
1490
|
+
}
|
|
1491
|
+
if (feature.properties) {
|
|
1492
|
+
for (const key in feature.properties) {
|
|
1493
|
+
const val = feature.properties[key];
|
|
1494
|
+
numericPropKeys[key] = numericPropKeys[key] || numericPropKeys[key] === void 0 ? isNumeric(val) : numericPropKeys[key];
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
}
|
|
1498
|
+
return {
|
|
1499
|
+
coordLength: coordLengths.size > 0 ? Math.max(...coordLengths) : 2,
|
|
1500
|
+
pointPositionsCount,
|
|
1501
|
+
pointFeaturesCount,
|
|
1502
|
+
linePositionsCount,
|
|
1503
|
+
linePathsCount,
|
|
1504
|
+
lineFeaturesCount,
|
|
1505
|
+
polygonPositionsCount,
|
|
1506
|
+
polygonObjectsCount,
|
|
1507
|
+
polygonRingsCount,
|
|
1508
|
+
polygonFeaturesCount,
|
|
1509
|
+
numericPropKeys: Object.keys(numericPropKeys).filter((k) => numericPropKeys[k])
|
|
1510
|
+
};
|
|
1511
|
+
}
|
|
1512
|
+
function secondPass(features, firstPassData, options) {
|
|
1513
|
+
const {
|
|
1514
|
+
pointPositionsCount,
|
|
1515
|
+
pointFeaturesCount,
|
|
1516
|
+
linePositionsCount,
|
|
1517
|
+
linePathsCount,
|
|
1518
|
+
lineFeaturesCount,
|
|
1519
|
+
polygonPositionsCount,
|
|
1520
|
+
polygonObjectsCount,
|
|
1521
|
+
polygonRingsCount,
|
|
1522
|
+
polygonFeaturesCount
|
|
1523
|
+
} = firstPassData;
|
|
1524
|
+
const { coordLength, numericPropKeys, PositionDataType = Float32Array } = options;
|
|
1525
|
+
const GlobalFeatureIdsDataType = features.length > 65535 ? Uint32Array : Uint16Array;
|
|
1526
|
+
const points = {
|
|
1527
|
+
positions: new PositionDataType(pointPositionsCount * coordLength),
|
|
1528
|
+
globalFeatureIds: new GlobalFeatureIdsDataType(pointPositionsCount),
|
|
1529
|
+
featureIds: pointFeaturesCount > 65535 ? new Uint32Array(pointPositionsCount) : new Uint16Array(pointPositionsCount),
|
|
1530
|
+
numericProps: {},
|
|
1531
|
+
properties: Array(),
|
|
1532
|
+
fields: Array()
|
|
1533
|
+
};
|
|
1534
|
+
const lines = {
|
|
1535
|
+
positions: new PositionDataType(linePositionsCount * coordLength),
|
|
1536
|
+
pathIndices: linePositionsCount > 65535 ? new Uint32Array(linePathsCount + 1) : new Uint16Array(linePathsCount + 1),
|
|
1537
|
+
globalFeatureIds: new GlobalFeatureIdsDataType(linePositionsCount),
|
|
1538
|
+
featureIds: lineFeaturesCount > 65535 ? new Uint32Array(linePositionsCount) : new Uint16Array(linePositionsCount),
|
|
1539
|
+
numericProps: {},
|
|
1540
|
+
properties: Array(),
|
|
1541
|
+
fields: Array()
|
|
1542
|
+
};
|
|
1543
|
+
const polygons = {
|
|
1544
|
+
positions: new PositionDataType(polygonPositionsCount * coordLength),
|
|
1545
|
+
polygonIndices: polygonPositionsCount > 65535 ? new Uint32Array(polygonObjectsCount + 1) : new Uint16Array(polygonObjectsCount + 1),
|
|
1546
|
+
primitivePolygonIndices: polygonPositionsCount > 65535 ? new Uint32Array(polygonRingsCount + 1) : new Uint16Array(polygonRingsCount + 1),
|
|
1547
|
+
globalFeatureIds: new GlobalFeatureIdsDataType(polygonPositionsCount),
|
|
1548
|
+
featureIds: polygonFeaturesCount > 65535 ? new Uint32Array(polygonPositionsCount) : new Uint16Array(polygonPositionsCount),
|
|
1549
|
+
numericProps: {},
|
|
1550
|
+
properties: Array(),
|
|
1551
|
+
fields: Array()
|
|
1552
|
+
};
|
|
1553
|
+
for (const object of [points, lines, polygons]) {
|
|
1554
|
+
for (const propName of numericPropKeys || []) {
|
|
1555
|
+
object.numericProps[propName] = new Float32Array(object.positions.length / coordLength);
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
lines.pathIndices[linePathsCount] = linePositionsCount;
|
|
1559
|
+
polygons.polygonIndices[polygonObjectsCount] = polygonPositionsCount;
|
|
1560
|
+
polygons.primitivePolygonIndices[polygonRingsCount] = polygonPositionsCount;
|
|
1561
|
+
const indexMap = {
|
|
1562
|
+
pointPosition: 0,
|
|
1563
|
+
pointFeature: 0,
|
|
1564
|
+
linePosition: 0,
|
|
1565
|
+
linePath: 0,
|
|
1566
|
+
lineFeature: 0,
|
|
1567
|
+
polygonPosition: 0,
|
|
1568
|
+
polygonObject: 0,
|
|
1569
|
+
polygonRing: 0,
|
|
1570
|
+
polygonFeature: 0,
|
|
1571
|
+
feature: 0
|
|
1572
|
+
};
|
|
1573
|
+
for (const feature of features) {
|
|
1574
|
+
const geometry = feature.geometry;
|
|
1575
|
+
const properties = feature.properties || {};
|
|
1576
|
+
switch (geometry.type) {
|
|
1577
|
+
case "Point":
|
|
1578
|
+
handlePoint(geometry.coordinates, points, indexMap, coordLength, properties);
|
|
1579
|
+
points.properties.push(keepStringProperties(properties, numericPropKeys));
|
|
1580
|
+
indexMap.pointFeature++;
|
|
1581
|
+
break;
|
|
1582
|
+
case "MultiPoint":
|
|
1583
|
+
handleMultiPoint(geometry.coordinates, points, indexMap, coordLength, properties);
|
|
1584
|
+
points.properties.push(keepStringProperties(properties, numericPropKeys));
|
|
1585
|
+
indexMap.pointFeature++;
|
|
1586
|
+
break;
|
|
1587
|
+
case "LineString":
|
|
1588
|
+
handleLineString(geometry.coordinates, lines, indexMap, coordLength, properties);
|
|
1589
|
+
lines.properties.push(keepStringProperties(properties, numericPropKeys));
|
|
1590
|
+
indexMap.lineFeature++;
|
|
1591
|
+
break;
|
|
1592
|
+
case "MultiLineString":
|
|
1593
|
+
handleMultiLineString(geometry.coordinates, lines, indexMap, coordLength, properties);
|
|
1594
|
+
lines.properties.push(keepStringProperties(properties, numericPropKeys));
|
|
1595
|
+
indexMap.lineFeature++;
|
|
1596
|
+
break;
|
|
1597
|
+
case "Polygon":
|
|
1598
|
+
handlePolygon(geometry.coordinates, polygons, indexMap, coordLength, properties);
|
|
1599
|
+
polygons.properties.push(keepStringProperties(properties, numericPropKeys));
|
|
1600
|
+
indexMap.polygonFeature++;
|
|
1601
|
+
break;
|
|
1602
|
+
case "MultiPolygon":
|
|
1603
|
+
handleMultiPolygon(geometry.coordinates, polygons, indexMap, coordLength, properties);
|
|
1604
|
+
polygons.properties.push(keepStringProperties(properties, numericPropKeys));
|
|
1605
|
+
indexMap.polygonFeature++;
|
|
1606
|
+
break;
|
|
1607
|
+
default:
|
|
1608
|
+
throw new Error("Invalid geometry type");
|
|
1609
|
+
}
|
|
1610
|
+
indexMap.feature++;
|
|
1611
|
+
}
|
|
1612
|
+
return makeAccessorObjects(points, lines, polygons, coordLength);
|
|
1613
|
+
}
|
|
1614
|
+
function handlePoint(coords, points, indexMap, coordLength, properties) {
|
|
1615
|
+
points.positions.set(coords, indexMap.pointPosition * coordLength);
|
|
1616
|
+
points.globalFeatureIds[indexMap.pointPosition] = indexMap.feature;
|
|
1617
|
+
points.featureIds[indexMap.pointPosition] = indexMap.pointFeature;
|
|
1618
|
+
fillNumericProperties(points, properties, indexMap.pointPosition, 1);
|
|
1619
|
+
indexMap.pointPosition++;
|
|
1620
|
+
}
|
|
1621
|
+
function handleMultiPoint(coords, points, indexMap, coordLength, properties) {
|
|
1622
|
+
for (const point of coords) {
|
|
1623
|
+
handlePoint(point, points, indexMap, coordLength, properties);
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
function handleLineString(coords, lines, indexMap, coordLength, properties) {
|
|
1627
|
+
lines.pathIndices[indexMap.linePath] = indexMap.linePosition;
|
|
1628
|
+
indexMap.linePath++;
|
|
1629
|
+
fillCoords(lines.positions, coords, indexMap.linePosition, coordLength);
|
|
1630
|
+
const nPositions = coords.length;
|
|
1631
|
+
fillNumericProperties(lines, properties, indexMap.linePosition, nPositions);
|
|
1632
|
+
lines.globalFeatureIds.set(new Uint32Array(nPositions).fill(indexMap.feature), indexMap.linePosition);
|
|
1633
|
+
lines.featureIds.set(new Uint32Array(nPositions).fill(indexMap.lineFeature), indexMap.linePosition);
|
|
1634
|
+
indexMap.linePosition += nPositions;
|
|
1635
|
+
}
|
|
1636
|
+
function handleMultiLineString(coords, lines, indexMap, coordLength, properties) {
|
|
1637
|
+
for (const line of coords) {
|
|
1638
|
+
handleLineString(line, lines, indexMap, coordLength, properties);
|
|
1639
|
+
}
|
|
1640
|
+
}
|
|
1641
|
+
function handlePolygon(coords, polygons, indexMap, coordLength, properties) {
|
|
1642
|
+
polygons.polygonIndices[indexMap.polygonObject] = indexMap.polygonPosition;
|
|
1643
|
+
indexMap.polygonObject++;
|
|
1644
|
+
for (const ring of coords) {
|
|
1645
|
+
polygons.primitivePolygonIndices[indexMap.polygonRing] = indexMap.polygonPosition;
|
|
1646
|
+
indexMap.polygonRing++;
|
|
1647
|
+
fillCoords(polygons.positions, ring, indexMap.polygonPosition, coordLength);
|
|
1648
|
+
const nPositions = ring.length;
|
|
1649
|
+
fillNumericProperties(polygons, properties, indexMap.polygonPosition, nPositions);
|
|
1650
|
+
polygons.globalFeatureIds.set(new Uint32Array(nPositions).fill(indexMap.feature), indexMap.polygonPosition);
|
|
1651
|
+
polygons.featureIds.set(new Uint32Array(nPositions).fill(indexMap.polygonFeature), indexMap.polygonPosition);
|
|
1652
|
+
indexMap.polygonPosition += nPositions;
|
|
1653
|
+
}
|
|
1654
|
+
}
|
|
1655
|
+
function handleMultiPolygon(coords, polygons, indexMap, coordLength, properties) {
|
|
1656
|
+
for (const polygon of coords) {
|
|
1657
|
+
handlePolygon(polygon, polygons, indexMap, coordLength, properties);
|
|
1658
|
+
}
|
|
1659
|
+
}
|
|
1660
|
+
function makeAccessorObjects(points, lines, polygons, coordLength) {
|
|
1661
|
+
const returnObj = {
|
|
1662
|
+
points: {
|
|
1663
|
+
...points,
|
|
1664
|
+
positions: { value: points.positions, size: coordLength },
|
|
1665
|
+
globalFeatureIds: { value: points.globalFeatureIds, size: 1 },
|
|
1666
|
+
featureIds: { value: points.featureIds, size: 1 },
|
|
1667
|
+
type: "Point"
|
|
1668
|
+
},
|
|
1669
|
+
lines: {
|
|
1670
|
+
...lines,
|
|
1671
|
+
pathIndices: { value: lines.pathIndices, size: 1 },
|
|
1672
|
+
positions: { value: lines.positions, size: coordLength },
|
|
1673
|
+
globalFeatureIds: { value: lines.globalFeatureIds, size: 1 },
|
|
1674
|
+
featureIds: { value: lines.featureIds, size: 1 },
|
|
1675
|
+
type: "LineString"
|
|
1676
|
+
},
|
|
1677
|
+
polygons: {
|
|
1678
|
+
...polygons,
|
|
1679
|
+
polygonIndices: { value: polygons.polygonIndices, size: 1 },
|
|
1680
|
+
primitivePolygonIndices: { value: polygons.primitivePolygonIndices, size: 1 },
|
|
1681
|
+
positions: { value: polygons.positions, size: coordLength },
|
|
1682
|
+
globalFeatureIds: { value: polygons.globalFeatureIds, size: 1 },
|
|
1683
|
+
featureIds: { value: polygons.featureIds, size: 1 },
|
|
1684
|
+
type: "Polygon"
|
|
1685
|
+
}
|
|
1686
|
+
};
|
|
1687
|
+
for (const geomType in returnObj) {
|
|
1688
|
+
for (const numericProp in returnObj[geomType].numericProps) {
|
|
1689
|
+
returnObj[geomType].numericProps[numericProp] = {
|
|
1690
|
+
value: returnObj[geomType].numericProps[numericProp],
|
|
1691
|
+
size: 1
|
|
1692
|
+
};
|
|
1693
|
+
}
|
|
1694
|
+
}
|
|
1695
|
+
return returnObj;
|
|
1696
|
+
}
|
|
1697
|
+
function fillNumericProperties(object, properties, index, length) {
|
|
1698
|
+
for (const numericPropName in object.numericProps) {
|
|
1699
|
+
if (numericPropName in properties) {
|
|
1700
|
+
object.numericProps[numericPropName].set(new Array(length).fill(properties[numericPropName]), index);
|
|
1701
|
+
}
|
|
1702
|
+
}
|
|
1703
|
+
}
|
|
1704
|
+
function keepStringProperties(properties, numericKeys) {
|
|
1705
|
+
const props = {};
|
|
1706
|
+
for (const key in properties) {
|
|
1707
|
+
if (!numericKeys.includes(key)) {
|
|
1708
|
+
props[key] = properties[key];
|
|
1709
|
+
}
|
|
1710
|
+
}
|
|
1711
|
+
return props;
|
|
1712
|
+
}
|
|
1713
|
+
function fillCoords(array, coords, startVertex, coordLength) {
|
|
1714
|
+
let index = startVertex * coordLength;
|
|
1715
|
+
for (const coord of coords) {
|
|
1716
|
+
array.set(coord, index);
|
|
1717
|
+
index += coordLength;
|
|
1718
|
+
}
|
|
1719
|
+
}
|
|
1720
|
+
function flatten(arrays) {
|
|
1721
|
+
return [].concat(...arrays);
|
|
1722
|
+
}
|
|
1723
|
+
function isNumeric(x) {
|
|
1724
|
+
return Number.isFinite(x);
|
|
1725
|
+
}
|
|
1726
|
+
var init_geojson_to_binary = __esm({
|
|
1727
|
+
"../gis/src/lib/geojson-to-binary.ts"() {
|
|
1728
|
+
}
|
|
1729
|
+
});
|
|
1730
|
+
|
|
1731
|
+
// ../gis/src/index.ts
|
|
1732
|
+
var init_src3 = __esm({
|
|
1733
|
+
"../gis/src/index.ts"() {
|
|
1734
|
+
init_geojson_to_binary();
|
|
1735
|
+
}
|
|
1736
|
+
});
|
|
1737
|
+
|
|
1738
|
+
// src/geojson-loader.ts
|
|
1739
|
+
async function parse3(arrayBuffer, options) {
|
|
1740
|
+
return parseTextSync3(new TextDecoder().decode(arrayBuffer), options);
|
|
1741
|
+
}
|
|
1742
|
+
function parseTextSync3(text, options) {
|
|
1743
|
+
options = { ...DEFAULT_GEOJSON_LOADER_OPTIONS, ...options };
|
|
1744
|
+
options.json = { ...DEFAULT_GEOJSON_LOADER_OPTIONS.geojson, ...options.geojson };
|
|
1745
|
+
options.gis = options.gis || {};
|
|
1746
|
+
const json = parseJSONSync(text, options);
|
|
1747
|
+
switch (options.gis.format) {
|
|
1748
|
+
case "binary":
|
|
1749
|
+
return geojsonToBinary(json);
|
|
1750
|
+
default:
|
|
1751
|
+
return json;
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
function parseInBatches3(asyncIterator, options) {
|
|
1755
|
+
options = { ...DEFAULT_GEOJSON_LOADER_OPTIONS, ...options };
|
|
1756
|
+
options.json = { ...DEFAULT_GEOJSON_LOADER_OPTIONS.geojson, ...options.geojson };
|
|
1757
|
+
const geojsonIterator = parseJSONInBatches(asyncIterator, options);
|
|
1758
|
+
switch (options.gis.format) {
|
|
1759
|
+
case "binary":
|
|
1760
|
+
return makeBinaryGeometryIterator(geojsonIterator);
|
|
1761
|
+
default:
|
|
1762
|
+
return geojsonIterator;
|
|
1763
|
+
}
|
|
1764
|
+
}
|
|
1765
|
+
async function* makeBinaryGeometryIterator(geojsonIterator) {
|
|
1766
|
+
for await (const batch of geojsonIterator) {
|
|
1767
|
+
batch.data = geojsonToBinary(batch.data);
|
|
1768
|
+
yield batch;
|
|
1769
|
+
}
|
|
1770
|
+
}
|
|
1771
|
+
var VERSION3, DEFAULT_GEOJSON_LOADER_OPTIONS, GeoJSONWorkerLoader, GeoJSONLoader;
|
|
1772
|
+
var init_geojson_loader = __esm({
|
|
1773
|
+
"src/geojson-loader.ts"() {
|
|
1774
|
+
init_src3();
|
|
1775
|
+
init_parse_json();
|
|
1776
|
+
init_parse_json_in_batches();
|
|
1777
|
+
VERSION3 = typeof __VERSION__ !== "undefined" ? __VERSION__ : "latest";
|
|
1778
|
+
DEFAULT_GEOJSON_LOADER_OPTIONS = {
|
|
1779
|
+
geojson: {
|
|
1780
|
+
shape: "object-row-table"
|
|
1781
|
+
},
|
|
1782
|
+
json: {
|
|
1783
|
+
jsonpaths: ["$", "$.features"]
|
|
1784
|
+
},
|
|
1785
|
+
gis: {
|
|
1786
|
+
format: "geojson"
|
|
1787
|
+
}
|
|
1788
|
+
};
|
|
1789
|
+
GeoJSONWorkerLoader = {
|
|
1790
|
+
name: "GeoJSON",
|
|
1791
|
+
id: "geojson",
|
|
1792
|
+
module: "geojson",
|
|
1793
|
+
version: VERSION3,
|
|
1794
|
+
worker: true,
|
|
1795
|
+
extensions: ["geojson"],
|
|
1796
|
+
mimeTypes: ["application/geo+json"],
|
|
1797
|
+
category: "geometry",
|
|
1798
|
+
text: true,
|
|
1799
|
+
options: DEFAULT_GEOJSON_LOADER_OPTIONS
|
|
1800
|
+
};
|
|
1801
|
+
GeoJSONLoader = {
|
|
1802
|
+
...GeoJSONWorkerLoader,
|
|
1803
|
+
parse: parse3,
|
|
1804
|
+
parseTextSync: parseTextSync3,
|
|
1805
|
+
parseInBatches: parseInBatches3
|
|
1806
|
+
};
|
|
1807
|
+
}
|
|
1808
|
+
});
|
|
1809
|
+
|
|
1810
|
+
// src/index.ts
|
|
1811
|
+
var src_exports = {};
|
|
1812
|
+
__export(src_exports, {
|
|
1813
|
+
JSONLoader: () => JSONLoader,
|
|
1814
|
+
NDJSONLoader: () => NDJSONLoader,
|
|
1815
|
+
_ClarinetParser: () => ClarinetParser,
|
|
1816
|
+
_GeoJSONLoader: () => GeoJSONLoader,
|
|
1817
|
+
_GeoJSONWorkerLoader: () => GeoJSONWorkerLoader,
|
|
1818
|
+
_JSONPath: () => JSONPath
|
|
1819
|
+
});
|
|
1820
|
+
var init_src4 = __esm({
|
|
1821
|
+
"src/index.ts"() {
|
|
1822
|
+
init_json_loader();
|
|
1823
|
+
init_ndjson_loader();
|
|
1824
|
+
init_geojson_loader();
|
|
1825
|
+
init_jsonpath();
|
|
1826
|
+
init_clarinet();
|
|
1827
|
+
}
|
|
1828
|
+
});
|
|
1829
|
+
|
|
1830
|
+
// src/bundle.ts
|
|
1831
|
+
var require_bundle = __commonJS({
|
|
1832
|
+
"src/bundle.ts"(exports, module) {
|
|
1833
|
+
var moduleExports = (init_src4(), src_exports);
|
|
1834
|
+
globalThis.loaders = globalThis.loaders || {};
|
|
1835
|
+
module.exports = Object.assign(globalThis.loaders, moduleExports);
|
|
1836
|
+
}
|
|
1837
|
+
});
|
|
1838
|
+
require_bundle();
|
|
1839
|
+
})();
|