@loaders.gl/json 4.0.0-alpha.9 → 4.0.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +132 -82
- package/dist/es5/geojson-loader.js +30 -21
- package/dist/es5/geojson-loader.js.map +1 -1
- package/dist/es5/geojson-writer.js.map +1 -1
- package/dist/es5/json-loader.js +2 -2
- package/dist/es5/json-loader.js.map +1 -1
- package/dist/es5/json-writer.js +1 -1
- package/dist/es5/json-writer.js.map +1 -1
- package/dist/es5/lib/encoder-utils/encode-table-row.js +54 -0
- package/dist/es5/lib/encoder-utils/encode-table-row.js.map +1 -0
- package/dist/es5/lib/encoder-utils/encode-utils.js.map +1 -0
- package/dist/es5/lib/encoder-utils/utf8-encoder.js.map +1 -0
- package/dist/es5/lib/encoders/geojson-encoder.js +47 -84
- package/dist/es5/lib/encoders/geojson-encoder.js.map +1 -1
- package/dist/es5/lib/encoders/json-encoder.js +3 -3
- package/dist/es5/lib/encoders/json-encoder.js.map +1 -1
- package/dist/es5/lib/parsers/parse-json-in-batches.js +1 -1
- package/dist/es5/lib/parsers/parse-json-in-batches.js.map +1 -1
- package/dist/es5/lib/parsers/parse-ndjson-in-batches.js.map +1 -1
- package/dist/es5/ndgeoson-loader.js +10 -13
- package/dist/es5/ndgeoson-loader.js.map +1 -1
- package/dist/es5/ndjson-loader.js +1 -1
- package/dist/es5/ndjson-loader.js.map +1 -1
- package/dist/esm/geojson-loader.js +31 -22
- package/dist/esm/geojson-loader.js.map +1 -1
- package/dist/esm/geojson-writer.js.map +1 -1
- package/dist/esm/json-loader.js +2 -2
- package/dist/esm/json-loader.js.map +1 -1
- package/dist/esm/json-writer.js +1 -1
- package/dist/esm/json-writer.js.map +1 -1
- package/dist/esm/lib/encoder-utils/encode-table-row.js +44 -0
- package/dist/esm/lib/encoder-utils/encode-table-row.js.map +1 -0
- package/dist/esm/lib/encoder-utils/encode-utils.js.map +1 -0
- package/dist/esm/lib/encoder-utils/utf8-encoder.js.map +1 -0
- package/dist/esm/lib/encoders/geojson-encoder.js +9 -52
- package/dist/esm/lib/encoders/geojson-encoder.js.map +1 -1
- package/dist/esm/lib/encoders/json-encoder.js +3 -3
- package/dist/esm/lib/encoders/json-encoder.js.map +1 -1
- package/dist/esm/lib/parsers/parse-json-in-batches.js +1 -1
- package/dist/esm/lib/parsers/parse-json-in-batches.js.map +1 -1
- package/dist/esm/lib/parsers/parse-ndjson-in-batches.js.map +1 -1
- package/dist/esm/ndgeoson-loader.js +9 -11
- package/dist/esm/ndgeoson-loader.js.map +1 -1
- package/dist/esm/ndjson-loader.js +1 -1
- package/dist/esm/ndjson-loader.js.map +1 -1
- package/dist/geojson-loader.d.ts +3 -2
- package/dist/geojson-loader.d.ts.map +1 -1
- package/dist/geojson-worker.js +55 -209
- package/dist/geojson-writer.d.ts +8 -3
- package/dist/geojson-writer.d.ts.map +1 -1
- package/dist/json-loader.d.ts +1 -1
- package/dist/json-loader.d.ts.map +1 -1
- package/dist/json-writer.d.ts +13 -3
- package/dist/json-writer.d.ts.map +1 -1
- package/dist/lib/encoder-utils/encode-table-row.d.ts +7 -0
- package/dist/lib/encoder-utils/encode-table-row.d.ts.map +1 -0
- package/dist/lib/encoder-utils/encode-utils.d.ts.map +1 -0
- package/dist/lib/encoder-utils/utf8-encoder.d.ts.map +1 -0
- package/dist/lib/encoders/geojson-encoder.d.ts +2 -8
- package/dist/lib/encoders/geojson-encoder.d.ts.map +1 -1
- package/dist/lib/encoders/json-encoder.d.ts +1 -10
- package/dist/lib/encoders/json-encoder.d.ts.map +1 -1
- package/dist/lib/parsers/parse-json-in-batches.d.ts.map +1 -1
- package/dist/lib/parsers/parse-ndjson-in-batches.d.ts +2 -2
- package/dist/lib/parsers/parse-ndjson-in-batches.d.ts.map +1 -1
- package/dist/ndgeoson-loader.d.ts +4 -24
- package/dist/ndgeoson-loader.d.ts.map +1 -1
- package/package.json +5 -5
- package/src/geojson-loader.ts +42 -27
- package/src/geojson-writer.ts +8 -3
- package/src/json-loader.ts +2 -2
- package/src/json-writer.ts +12 -4
- package/src/lib/encoder-utils/encode-table-row.ts +69 -0
- package/src/lib/encoders/geojson-encoder.ts +15 -78
- package/src/lib/encoders/json-encoder.ts +3 -11
- package/src/lib/parsers/parse-json-in-batches.ts +2 -1
- package/src/lib/parsers/parse-ndjson-in-batches.ts +2 -2
- package/src/ndgeoson-loader.ts +16 -13
- package/dist/bundle.js +0 -5
- package/dist/es5/lib/encoders/encode-utils.js.map +0 -1
- package/dist/es5/lib/encoders/utf8-encoder.js.map +0 -1
- package/dist/esm/lib/encoders/encode-utils.js.map +0 -1
- package/dist/esm/lib/encoders/utf8-encoder.js.map +0 -1
- package/dist/geojson-loader.js +0 -77
- package/dist/geojson-writer.js +0 -22
- package/dist/index.js +0 -24
- package/dist/json-loader.js +0 -42
- package/dist/json-writer.js +0 -18
- package/dist/lib/clarinet/clarinet.js +0 -535
- package/dist/lib/encoders/encode-utils.d.ts.map +0 -1
- package/dist/lib/encoders/encode-utils.js +0 -47
- package/dist/lib/encoders/geojson-encoder.js +0 -104
- package/dist/lib/encoders/json-encoder.js +0 -22
- package/dist/lib/encoders/utf8-encoder.d.ts.map +0 -1
- package/dist/lib/encoders/utf8-encoder.js +0 -32
- package/dist/lib/json-parser/json-parser.js +0 -98
- package/dist/lib/json-parser/streaming-json-parser.js +0 -100
- package/dist/lib/jsonpath/jsonpath.js +0 -89
- package/dist/lib/parsers/parse-json-in-batches.js +0 -100
- package/dist/lib/parsers/parse-json.js +0 -32
- package/dist/lib/parsers/parse-ndjson-in-batches.js +0 -36
- package/dist/lib/parsers/parse-ndjson.js +0 -17
- package/dist/ndgeoson-loader.js +0 -37
- package/dist/ndjson-loader.js +0 -27
- package/dist/workers/geojson-worker.js +0 -5
- /package/dist/es5/lib/{encoders → encoder-utils}/encode-utils.js +0 -0
- /package/dist/es5/lib/{encoders → encoder-utils}/utf8-encoder.js +0 -0
- /package/dist/esm/lib/{encoders → encoder-utils}/encode-utils.js +0 -0
- /package/dist/esm/lib/{encoders → encoder-utils}/utf8-encoder.js +0 -0
- /package/dist/lib/{encoders → encoder-utils}/encode-utils.d.ts +0 -0
- /package/dist/lib/{encoders → encoder-utils}/utf8-encoder.d.ts +0 -0
- /package/src/lib/{encoders → encoder-utils}/encode-utils.ts +0 -0
- /package/src/lib/{encoders → encoder-utils}/utf8-encoder.ts +0 -0
|
@@ -1,104 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT license
|
|
3
|
-
// Copyright 2022 Foursquare Labs, Inc.
|
|
4
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.encodeTableAsGeojsonInBatches = void 0;
|
|
6
|
-
const schema_1 = require("@loaders.gl/schema");
|
|
7
|
-
const schema_2 = require("@loaders.gl/schema");
|
|
8
|
-
const encode_utils_1 = require("./encode-utils");
|
|
9
|
-
const utf8_encoder_1 = require("./utf8-encoder");
|
|
10
|
-
/**
|
|
11
|
-
* Encode a table as GeoJSON
|
|
12
|
-
*/
|
|
13
|
-
// eslint-disable-next-line max-statements
|
|
14
|
-
async function* encodeTableAsGeojsonInBatches(batchIterator, // | Iterable<TableBatch>,
|
|
15
|
-
inputOpts = {}) {
|
|
16
|
-
const options = { geojson: {}, chunkSize: 10000, ...inputOpts };
|
|
17
|
-
const utf8Encoder = new utf8_encoder_1.Utf8ArrayBufferEncoder(options.chunkSize);
|
|
18
|
-
if (!options.geojson.featureArray) {
|
|
19
|
-
utf8Encoder.push('{\n', '"type": "FeatureCollection",\n', '"features":\n');
|
|
20
|
-
}
|
|
21
|
-
utf8Encoder.push('['); // Note no newline
|
|
22
|
-
let geometryColumn = options.geojson.geometryColumn;
|
|
23
|
-
let isFirstLine = true;
|
|
24
|
-
for await (const batch of batchIterator) {
|
|
25
|
-
const { table, start, end = (0, schema_1.getTableLength)(batch.table) - start } = batch;
|
|
26
|
-
// Deduce geometry column if not already done
|
|
27
|
-
if (!geometryColumn) {
|
|
28
|
-
geometryColumn = geometryColumn || (0, encode_utils_1.detectGeometryColumnIndex)(table);
|
|
29
|
-
}
|
|
30
|
-
for (let rowIndex = start; rowIndex < end; ++rowIndex) {
|
|
31
|
-
// Add a comma except on final feature
|
|
32
|
-
if (!isFirstLine) {
|
|
33
|
-
utf8Encoder.push(',');
|
|
34
|
-
}
|
|
35
|
-
utf8Encoder.push('\n');
|
|
36
|
-
isFirstLine = false;
|
|
37
|
-
encodeRow(table, rowIndex, geometryColumn, utf8Encoder);
|
|
38
|
-
// eslint-disable-next-line max-depth
|
|
39
|
-
if (utf8Encoder.isFull()) {
|
|
40
|
-
yield utf8Encoder.getArrayBufferBatch();
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
const arrayBufferBatch = utf8Encoder.getArrayBufferBatch();
|
|
44
|
-
if (arrayBufferBatch.byteLength > 0) {
|
|
45
|
-
yield arrayBufferBatch;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
utf8Encoder.push('\n');
|
|
49
|
-
// Add completing rows and emit final batch
|
|
50
|
-
utf8Encoder.push(']\n');
|
|
51
|
-
if (!options.geojson.featureArray) {
|
|
52
|
-
utf8Encoder.push('}');
|
|
53
|
-
}
|
|
54
|
-
// Note: Since we pushed a few final lines, the last batch will always exist, no need to check first
|
|
55
|
-
yield utf8Encoder.getArrayBufferBatch();
|
|
56
|
-
}
|
|
57
|
-
exports.encodeTableAsGeojsonInBatches = encodeTableAsGeojsonInBatches;
|
|
58
|
-
// Helpers
|
|
59
|
-
/**
|
|
60
|
-
* Encode a row. Currently this ignores properties in the geometry column.
|
|
61
|
-
*/
|
|
62
|
-
function encodeRow(table, rowIndex, geometryColumnIndex, utf8Encoder) {
|
|
63
|
-
const row = (0, schema_2.getTableRowAsObject)(table, rowIndex);
|
|
64
|
-
if (!row)
|
|
65
|
-
return;
|
|
66
|
-
const featureWithProperties = getFeatureFromRow(table, row, geometryColumnIndex);
|
|
67
|
-
const featureString = JSON.stringify(featureWithProperties);
|
|
68
|
-
utf8Encoder.push(featureString);
|
|
69
|
-
}
|
|
70
|
-
/**
|
|
71
|
-
* Encode a row as a Feature. Currently this ignores properties objects in the geometry column.
|
|
72
|
-
*/
|
|
73
|
-
function getFeatureFromRow(table, row, geometryColumnIndex) {
|
|
74
|
-
// Extract non-feature/geometry properties
|
|
75
|
-
const properties = (0, encode_utils_1.getRowPropertyObject)(table, row, [geometryColumnIndex]);
|
|
76
|
-
// Extract geometry feature
|
|
77
|
-
const columnName = table.schema?.fields[geometryColumnIndex].name;
|
|
78
|
-
let featureOrGeometry = columnName && row[columnName];
|
|
79
|
-
// GeoJSON support null geometries
|
|
80
|
-
if (!featureOrGeometry) {
|
|
81
|
-
// @ts-ignore Feature type does not support null geometries
|
|
82
|
-
return { type: 'Feature', geometry: null, properties };
|
|
83
|
-
}
|
|
84
|
-
// Support string geometries?
|
|
85
|
-
// TODO: This assumes GeoJSON strings, which may not be the correct format
|
|
86
|
-
// (could be WKT, encoded WKB...)
|
|
87
|
-
if (typeof featureOrGeometry === 'string') {
|
|
88
|
-
try {
|
|
89
|
-
featureOrGeometry = JSON.parse(featureOrGeometry);
|
|
90
|
-
}
|
|
91
|
-
catch (err) {
|
|
92
|
-
throw new Error('Invalid string geometry');
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
if (typeof featureOrGeometry !== 'object' || typeof featureOrGeometry?.type !== 'string') {
|
|
96
|
-
throw new Error('invalid geometry column value');
|
|
97
|
-
}
|
|
98
|
-
if (featureOrGeometry?.type === 'Feature') {
|
|
99
|
-
// @ts-ignore Feature type does not support null geometries
|
|
100
|
-
return { ...featureOrGeometry, properties };
|
|
101
|
-
}
|
|
102
|
-
// @ts-ignore Feature type does not support null geometries
|
|
103
|
-
return { type: 'Feature', geometry: featureOrGeometry, properties };
|
|
104
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT license
|
|
3
|
-
// Copyright 2022 Foursquare Labs, Inc.
|
|
4
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.encodeTableAsJSON = void 0;
|
|
6
|
-
const schema_1 = require("@loaders.gl/schema");
|
|
7
|
-
/**
|
|
8
|
-
* Encode a table as a JSON string
|
|
9
|
-
*/
|
|
10
|
-
function encodeTableAsJSON(table, options = {}) {
|
|
11
|
-
const shape = options.shape || 'object-row-table';
|
|
12
|
-
const strings = [];
|
|
13
|
-
const rowIterator = (0, schema_1.makeRowIterator)(table, shape);
|
|
14
|
-
for (const row of rowIterator) {
|
|
15
|
-
// Round elements etc
|
|
16
|
-
// processRow(wrappedRow, table.schema);
|
|
17
|
-
// const wrappedRow = options.wrapper ? options.wrapper(row) : row;
|
|
18
|
-
strings.push(JSON.stringify(row));
|
|
19
|
-
}
|
|
20
|
-
return `[${strings.join(',')}]`;
|
|
21
|
-
}
|
|
22
|
-
exports.encodeTableAsJSON = encodeTableAsJSON;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"utf8-encoder.d.ts","sourceRoot":"","sources":["../../../src/lib/encoders/utf8-encoder.ts"],"names":[],"mappings":"AAGA,qBAAa,sBAAsB;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,OAAO,CAAgB;IAC/B,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,WAAW,CAAkC;gBAEzC,SAAS,EAAE,MAAM;IAI7B,IAAI,CAAC,GAAG,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI;IAOhC,MAAM,IAAI,OAAO;IAIjB,mBAAmB,IAAI,eAAe;IAItC,cAAc,IAAI,MAAM;CAMzB"}
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// loaders.gl, MIT License
|
|
3
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
|
-
exports.Utf8ArrayBufferEncoder = void 0;
|
|
5
|
-
/* global TextEncoder */
|
|
6
|
-
class Utf8ArrayBufferEncoder {
|
|
7
|
-
constructor(chunkSize) {
|
|
8
|
-
this.strings = [];
|
|
9
|
-
this.totalLength = 0;
|
|
10
|
-
this.textEncoder = new TextEncoder();
|
|
11
|
-
this.chunkSize = chunkSize;
|
|
12
|
-
}
|
|
13
|
-
push(...strings) {
|
|
14
|
-
for (const string of strings) {
|
|
15
|
-
this.strings.push(string);
|
|
16
|
-
this.totalLength += string.length;
|
|
17
|
-
}
|
|
18
|
-
}
|
|
19
|
-
isFull() {
|
|
20
|
-
return this.totalLength >= this.chunkSize;
|
|
21
|
-
}
|
|
22
|
-
getArrayBufferBatch() {
|
|
23
|
-
return this.textEncoder.encode(this.getStringBatch()).buffer;
|
|
24
|
-
}
|
|
25
|
-
getStringBatch() {
|
|
26
|
-
const stringChunk = this.strings.join('');
|
|
27
|
-
this.strings = [];
|
|
28
|
-
this.totalLength = 0;
|
|
29
|
-
return stringChunk;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
exports.Utf8ArrayBufferEncoder = Utf8ArrayBufferEncoder;
|
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
// @ts-nocheck
|
|
3
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
4
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
5
|
-
};
|
|
6
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
-
const clarinet_1 = __importDefault(require("../clarinet/clarinet"));
|
|
8
|
-
const jsonpath_1 = __importDefault(require("../jsonpath/jsonpath"));
|
|
9
|
-
// JSONParser builds a JSON object using the events emitted by the Clarinet parser
|
|
10
|
-
class JSONParser {
|
|
11
|
-
constructor(options) {
|
|
12
|
-
this.result = undefined;
|
|
13
|
-
this.previousStates = [];
|
|
14
|
-
this.currentState = Object.freeze({ container: [], key: null });
|
|
15
|
-
this.jsonpath = new jsonpath_1.default();
|
|
16
|
-
this.reset();
|
|
17
|
-
this.parser = new clarinet_1.default({
|
|
18
|
-
onready: () => {
|
|
19
|
-
this.jsonpath = new jsonpath_1.default();
|
|
20
|
-
this.previousStates.length = 0;
|
|
21
|
-
this.currentState.container.length = 0;
|
|
22
|
-
},
|
|
23
|
-
onopenobject: (name) => {
|
|
24
|
-
this._openObject({});
|
|
25
|
-
if (typeof name !== 'undefined') {
|
|
26
|
-
this.parser.emit('onkey', name);
|
|
27
|
-
}
|
|
28
|
-
},
|
|
29
|
-
onkey: (name) => {
|
|
30
|
-
this.jsonpath.set(name);
|
|
31
|
-
this.currentState.key = name;
|
|
32
|
-
},
|
|
33
|
-
oncloseobject: () => {
|
|
34
|
-
this._closeObject();
|
|
35
|
-
},
|
|
36
|
-
onopenarray: () => {
|
|
37
|
-
this._openArray();
|
|
38
|
-
},
|
|
39
|
-
onclosearray: () => {
|
|
40
|
-
this._closeArray();
|
|
41
|
-
},
|
|
42
|
-
onvalue: (value) => {
|
|
43
|
-
this._pushOrSet(value);
|
|
44
|
-
},
|
|
45
|
-
onerror: (error) => {
|
|
46
|
-
throw error;
|
|
47
|
-
},
|
|
48
|
-
onend: () => {
|
|
49
|
-
this.result = this.currentState.container.pop();
|
|
50
|
-
},
|
|
51
|
-
...options
|
|
52
|
-
});
|
|
53
|
-
}
|
|
54
|
-
reset() {
|
|
55
|
-
this.result = undefined;
|
|
56
|
-
this.previousStates = [];
|
|
57
|
-
this.currentState = Object.freeze({ container: [], key: null });
|
|
58
|
-
this.jsonpath = new jsonpath_1.default();
|
|
59
|
-
}
|
|
60
|
-
write(chunk) {
|
|
61
|
-
this.parser.write(chunk);
|
|
62
|
-
}
|
|
63
|
-
close() {
|
|
64
|
-
this.parser.close();
|
|
65
|
-
}
|
|
66
|
-
// PRIVATE METHODS
|
|
67
|
-
_pushOrSet(value) {
|
|
68
|
-
const { container, key } = this.currentState;
|
|
69
|
-
if (key !== null) {
|
|
70
|
-
container[key] = value;
|
|
71
|
-
this.currentState.key = null;
|
|
72
|
-
}
|
|
73
|
-
else {
|
|
74
|
-
container.push(value);
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
_openArray(newContainer = []) {
|
|
78
|
-
this.jsonpath.push(null);
|
|
79
|
-
this._pushOrSet(newContainer);
|
|
80
|
-
this.previousStates.push(this.currentState);
|
|
81
|
-
this.currentState = { container: newContainer, isArray: true, key: null };
|
|
82
|
-
}
|
|
83
|
-
_closeArray() {
|
|
84
|
-
this.jsonpath.pop();
|
|
85
|
-
this.currentState = this.previousStates.pop();
|
|
86
|
-
}
|
|
87
|
-
_openObject(newContainer = {}) {
|
|
88
|
-
this.jsonpath.push(null);
|
|
89
|
-
this._pushOrSet(newContainer);
|
|
90
|
-
this.previousStates.push(this.currentState);
|
|
91
|
-
this.currentState = { container: newContainer, isArray: false, key: null };
|
|
92
|
-
}
|
|
93
|
-
_closeObject() {
|
|
94
|
-
this.jsonpath.pop();
|
|
95
|
-
this.currentState = this.previousStates.pop();
|
|
96
|
-
}
|
|
97
|
-
}
|
|
98
|
-
exports.default = JSONParser;
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
const json_parser_1 = __importDefault(require("./json-parser"));
|
|
7
|
-
const jsonpath_1 = __importDefault(require("../jsonpath/jsonpath"));
|
|
8
|
-
/**
|
|
9
|
-
* The `StreamingJSONParser` looks for the first array in the JSON structure.
|
|
10
|
-
* and emits an array of chunks
|
|
11
|
-
*/
|
|
12
|
-
class StreamingJSONParser extends json_parser_1.default {
|
|
13
|
-
constructor(options = {}) {
|
|
14
|
-
super({
|
|
15
|
-
onopenarray: () => {
|
|
16
|
-
if (!this.streamingArray) {
|
|
17
|
-
if (this._matchJSONPath()) {
|
|
18
|
-
// @ts-ignore
|
|
19
|
-
this.streamingJsonPath = this.getJsonPath().clone();
|
|
20
|
-
this.streamingArray = [];
|
|
21
|
-
this._openArray(this.streamingArray);
|
|
22
|
-
return;
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
this._openArray();
|
|
26
|
-
},
|
|
27
|
-
// Redefine onopenarray to inject value for top-level object
|
|
28
|
-
onopenobject: (name) => {
|
|
29
|
-
if (!this.topLevelObject) {
|
|
30
|
-
this.topLevelObject = {};
|
|
31
|
-
this._openObject(this.topLevelObject);
|
|
32
|
-
}
|
|
33
|
-
else {
|
|
34
|
-
this._openObject({});
|
|
35
|
-
}
|
|
36
|
-
if (typeof name !== 'undefined') {
|
|
37
|
-
this.parser.emit('onkey', name);
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
});
|
|
41
|
-
this.streamingJsonPath = null;
|
|
42
|
-
this.streamingArray = null;
|
|
43
|
-
this.topLevelObject = null;
|
|
44
|
-
const jsonpaths = options.jsonpaths || [];
|
|
45
|
-
this.jsonPaths = jsonpaths.map((jsonpath) => new jsonpath_1.default(jsonpath));
|
|
46
|
-
}
|
|
47
|
-
/**
|
|
48
|
-
* write REDEFINITION
|
|
49
|
-
* - super.write() chunk to parser
|
|
50
|
-
* - get the contents (so far) of "topmost-level" array as batch of rows
|
|
51
|
-
* - clear top-level array
|
|
52
|
-
* - return the batch of rows\
|
|
53
|
-
*/
|
|
54
|
-
write(chunk) {
|
|
55
|
-
super.write(chunk);
|
|
56
|
-
let array = [];
|
|
57
|
-
if (this.streamingArray) {
|
|
58
|
-
array = [...this.streamingArray];
|
|
59
|
-
this.streamingArray.length = 0;
|
|
60
|
-
}
|
|
61
|
-
return array;
|
|
62
|
-
}
|
|
63
|
-
/**
|
|
64
|
-
* Returns a partially formed result object
|
|
65
|
-
* Useful for returning the "wrapper" object when array is not top level
|
|
66
|
-
* e.g. GeoJSON
|
|
67
|
-
*/
|
|
68
|
-
getPartialResult() {
|
|
69
|
-
return this.topLevelObject;
|
|
70
|
-
}
|
|
71
|
-
getStreamingJsonPath() {
|
|
72
|
-
return this.streamingJsonPath;
|
|
73
|
-
}
|
|
74
|
-
getStreamingJsonPathAsString() {
|
|
75
|
-
return this.streamingJsonPath && this.streamingJsonPath.toString();
|
|
76
|
-
}
|
|
77
|
-
getJsonPath() {
|
|
78
|
-
return this.jsonpath;
|
|
79
|
-
}
|
|
80
|
-
// PRIVATE METHODS
|
|
81
|
-
/**
|
|
82
|
-
* Checks is this.getJsonPath matches the jsonpaths provided in options
|
|
83
|
-
*/
|
|
84
|
-
_matchJSONPath() {
|
|
85
|
-
const currentPath = this.getJsonPath();
|
|
86
|
-
// console.debug(`Testing JSONPath`, currentPath);
|
|
87
|
-
// Backwards compatibility, match any array
|
|
88
|
-
// TODO implement using wildcard once that is supported
|
|
89
|
-
if (this.jsonPaths.length === 0) {
|
|
90
|
-
return true;
|
|
91
|
-
}
|
|
92
|
-
for (const jsonPath of this.jsonPaths) {
|
|
93
|
-
if (jsonPath.equals(currentPath)) {
|
|
94
|
-
return true;
|
|
95
|
-
}
|
|
96
|
-
}
|
|
97
|
-
return false;
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
exports.default = StreamingJSONParser;
|
|
@@ -1,89 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
/**
|
|
4
|
-
* A parser for a minimal subset of the jsonpath standard
|
|
5
|
-
* Full JSON path parsers for JS exist but are quite large (bundle size)
|
|
6
|
-
*
|
|
7
|
-
* Supports
|
|
8
|
-
*
|
|
9
|
-
* `$.component.component.component`
|
|
10
|
-
*/
|
|
11
|
-
class JSONPath {
|
|
12
|
-
constructor(path = null) {
|
|
13
|
-
this.path = ['$'];
|
|
14
|
-
if (path instanceof JSONPath) {
|
|
15
|
-
// @ts-ignore
|
|
16
|
-
this.path = [...path.path];
|
|
17
|
-
return;
|
|
18
|
-
}
|
|
19
|
-
if (Array.isArray(path)) {
|
|
20
|
-
this.path.push(...path);
|
|
21
|
-
return;
|
|
22
|
-
}
|
|
23
|
-
// Parse a string as a JSONPath
|
|
24
|
-
if (typeof path === 'string') {
|
|
25
|
-
this.path = path.split('.');
|
|
26
|
-
if (this.path[0] !== '$') {
|
|
27
|
-
throw new Error('JSONPaths must start with $');
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
clone() {
|
|
32
|
-
return new JSONPath(this);
|
|
33
|
-
}
|
|
34
|
-
toString() {
|
|
35
|
-
return this.path.join('.');
|
|
36
|
-
}
|
|
37
|
-
push(name) {
|
|
38
|
-
this.path.push(name);
|
|
39
|
-
}
|
|
40
|
-
pop() {
|
|
41
|
-
return this.path.pop();
|
|
42
|
-
}
|
|
43
|
-
set(name) {
|
|
44
|
-
this.path[this.path.length - 1] = name;
|
|
45
|
-
}
|
|
46
|
-
equals(other) {
|
|
47
|
-
if (!this || !other || this.path.length !== other.path.length) {
|
|
48
|
-
return false;
|
|
49
|
-
}
|
|
50
|
-
for (let i = 0; i < this.path.length; ++i) {
|
|
51
|
-
if (this.path[i] !== other.path[i]) {
|
|
52
|
-
return false;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
return true;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Sets the value pointed at by path
|
|
59
|
-
* TODO - handle root path
|
|
60
|
-
* @param object
|
|
61
|
-
* @param value
|
|
62
|
-
*/
|
|
63
|
-
setFieldAtPath(object, value) {
|
|
64
|
-
const path = [...this.path];
|
|
65
|
-
path.shift();
|
|
66
|
-
const field = path.pop();
|
|
67
|
-
for (const component of path) {
|
|
68
|
-
object = object[component];
|
|
69
|
-
}
|
|
70
|
-
// @ts-ignore
|
|
71
|
-
object[field] = value;
|
|
72
|
-
}
|
|
73
|
-
/**
|
|
74
|
-
* Gets the value pointed at by path
|
|
75
|
-
* TODO - handle root path
|
|
76
|
-
* @param object
|
|
77
|
-
*/
|
|
78
|
-
getFieldAtPath(object) {
|
|
79
|
-
const path = [...this.path];
|
|
80
|
-
path.shift();
|
|
81
|
-
const field = path.pop();
|
|
82
|
-
for (const component of path) {
|
|
83
|
-
object = object[component];
|
|
84
|
-
}
|
|
85
|
-
// @ts-ignore
|
|
86
|
-
return object[field];
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
exports.default = JSONPath;
|
|
@@ -1,100 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.rebuildJsonObject = exports.parseJSONInBatches = void 0;
|
|
7
|
-
const schema_1 = require("@loaders.gl/schema");
|
|
8
|
-
const loader_utils_1 = require("@loaders.gl/loader-utils");
|
|
9
|
-
const streaming_json_parser_1 = __importDefault(require("../json-parser/streaming-json-parser"));
|
|
10
|
-
const jsonpath_1 = __importDefault(require("../jsonpath/jsonpath"));
|
|
11
|
-
// TODO - support batch size 0 = no batching/single batch?
|
|
12
|
-
// eslint-disable-next-line max-statements, complexity
|
|
13
|
-
async function* parseJSONInBatches(binaryAsyncIterator, options) {
|
|
14
|
-
const asyncIterator = (0, loader_utils_1.makeTextDecoderIterator)(binaryAsyncIterator);
|
|
15
|
-
const { metadata } = options;
|
|
16
|
-
const { jsonpaths } = options.json || {};
|
|
17
|
-
let isFirstChunk = true;
|
|
18
|
-
// TODO fix Schema deduction
|
|
19
|
-
const schema = null; // new Schema([]);
|
|
20
|
-
const shape = options?.json?.shape || 'row-table';
|
|
21
|
-
// @ts-ignore
|
|
22
|
-
const tableBatchBuilder = new schema_1.TableBatchBuilder(schema, {
|
|
23
|
-
...options,
|
|
24
|
-
shape
|
|
25
|
-
});
|
|
26
|
-
const parser = new streaming_json_parser_1.default({ jsonpaths });
|
|
27
|
-
for await (const chunk of asyncIterator) {
|
|
28
|
-
const rows = parser.write(chunk);
|
|
29
|
-
const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
|
|
30
|
-
if (rows.length > 0 && isFirstChunk) {
|
|
31
|
-
if (metadata) {
|
|
32
|
-
const initialBatch = {
|
|
33
|
-
// Common fields
|
|
34
|
-
shape,
|
|
35
|
-
batchType: 'partial-result',
|
|
36
|
-
data: [],
|
|
37
|
-
length: 0,
|
|
38
|
-
bytesUsed: 0,
|
|
39
|
-
// JSON additions
|
|
40
|
-
container: parser.getPartialResult(),
|
|
41
|
-
jsonpath
|
|
42
|
-
};
|
|
43
|
-
yield initialBatch;
|
|
44
|
-
}
|
|
45
|
-
isFirstChunk = false;
|
|
46
|
-
// schema = deduceSchema(rows);
|
|
47
|
-
}
|
|
48
|
-
// Add the row
|
|
49
|
-
for (const row of rows) {
|
|
50
|
-
tableBatchBuilder.addRow(row);
|
|
51
|
-
// If a batch has been completed, emit it
|
|
52
|
-
const batch = tableBatchBuilder.getFullBatch({ jsonpath });
|
|
53
|
-
if (batch) {
|
|
54
|
-
yield batch;
|
|
55
|
-
}
|
|
56
|
-
}
|
|
57
|
-
tableBatchBuilder.chunkComplete(chunk);
|
|
58
|
-
const batch = tableBatchBuilder.getFullBatch({ jsonpath });
|
|
59
|
-
if (batch) {
|
|
60
|
-
yield batch;
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
// yield final batch
|
|
64
|
-
const jsonpath = parser.getStreamingJsonPathAsString();
|
|
65
|
-
const batch = tableBatchBuilder.getFinalBatch({ jsonpath });
|
|
66
|
-
if (batch) {
|
|
67
|
-
yield batch;
|
|
68
|
-
}
|
|
69
|
-
if (metadata) {
|
|
70
|
-
const finalBatch = {
|
|
71
|
-
shape,
|
|
72
|
-
batchType: 'final-result',
|
|
73
|
-
container: parser.getPartialResult(),
|
|
74
|
-
jsonpath: parser.getStreamingJsonPathAsString(),
|
|
75
|
-
data: [],
|
|
76
|
-
length: 0
|
|
77
|
-
// schema: null
|
|
78
|
-
};
|
|
79
|
-
yield finalBatch;
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
exports.parseJSONInBatches = parseJSONInBatches;
|
|
83
|
-
function rebuildJsonObject(batch, data) {
|
|
84
|
-
// Last batch will have this special type and will provide all the root object of the parsed file
|
|
85
|
-
(0, loader_utils_1.assert)(batch.batchType === 'final-result');
|
|
86
|
-
// The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects
|
|
87
|
-
if (batch.jsonpath === '$') {
|
|
88
|
-
return data;
|
|
89
|
-
}
|
|
90
|
-
// (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object
|
|
91
|
-
if (batch.jsonpath && batch.jsonpath.length > 1) {
|
|
92
|
-
const topLevelObject = batch.container;
|
|
93
|
-
const streamingPath = new jsonpath_1.default(batch.jsonpath);
|
|
94
|
-
streamingPath.setFieldAtPath(topLevelObject, data);
|
|
95
|
-
return topLevelObject;
|
|
96
|
-
}
|
|
97
|
-
// No jsonpath, in this case nothing was streamed.
|
|
98
|
-
return batch.container;
|
|
99
|
-
}
|
|
100
|
-
exports.rebuildJsonObject = rebuildJsonObject;
|
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.parseJSONSync = void 0;
|
|
4
|
-
const schema_1 = require("@loaders.gl/schema");
|
|
5
|
-
function parseJSONSync(jsonText, options) {
|
|
6
|
-
try {
|
|
7
|
-
const json = JSON.parse(jsonText);
|
|
8
|
-
if (options.json?.table) {
|
|
9
|
-
const data = getFirstArray(json) || json;
|
|
10
|
-
return (0, schema_1.makeTableFromData)(data);
|
|
11
|
-
}
|
|
12
|
-
return json;
|
|
13
|
-
}
|
|
14
|
-
catch (error) {
|
|
15
|
-
throw new Error('JSONLoader: failed to parse JSON');
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
exports.parseJSONSync = parseJSONSync;
|
|
19
|
-
function getFirstArray(json) {
|
|
20
|
-
if (Array.isArray(json)) {
|
|
21
|
-
return json;
|
|
22
|
-
}
|
|
23
|
-
if (json && typeof json === 'object') {
|
|
24
|
-
for (const value of Object.values(json)) {
|
|
25
|
-
const array = getFirstArray(value);
|
|
26
|
-
if (array) {
|
|
27
|
-
return array;
|
|
28
|
-
}
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
return null;
|
|
32
|
-
}
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.parseNDJSONInBatches = void 0;
|
|
4
|
-
const schema_1 = require("@loaders.gl/schema");
|
|
5
|
-
const loader_utils_1 = require("@loaders.gl/loader-utils");
|
|
6
|
-
async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
|
|
7
|
-
const textIterator = (0, loader_utils_1.makeTextDecoderIterator)(binaryAsyncIterator);
|
|
8
|
-
const lineIterator = (0, loader_utils_1.makeLineIterator)(textIterator);
|
|
9
|
-
const numberedLineIterator = (0, loader_utils_1.makeNumberedLineIterator)(lineIterator);
|
|
10
|
-
const schema = null;
|
|
11
|
-
const shape = 'row-table';
|
|
12
|
-
// @ts-ignore
|
|
13
|
-
const tableBatchBuilder = new schema_1.TableBatchBuilder(schema, {
|
|
14
|
-
...options,
|
|
15
|
-
shape
|
|
16
|
-
});
|
|
17
|
-
for await (const { counter, line } of numberedLineIterator) {
|
|
18
|
-
try {
|
|
19
|
-
const row = JSON.parse(line);
|
|
20
|
-
tableBatchBuilder.addRow(row);
|
|
21
|
-
tableBatchBuilder.chunkComplete(line);
|
|
22
|
-
const batch = tableBatchBuilder.getFullBatch();
|
|
23
|
-
if (batch) {
|
|
24
|
-
yield batch;
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
catch (error) {
|
|
28
|
-
throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
|
|
29
|
-
}
|
|
30
|
-
}
|
|
31
|
-
const batch = tableBatchBuilder.getFinalBatch();
|
|
32
|
-
if (batch) {
|
|
33
|
-
yield batch;
|
|
34
|
-
}
|
|
35
|
-
}
|
|
36
|
-
exports.parseNDJSONInBatches = parseNDJSONInBatches;
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.parseNDJSONSync = void 0;
|
|
4
|
-
const schema_1 = require("@loaders.gl/schema");
|
|
5
|
-
function parseNDJSONSync(ndjsonText) {
|
|
6
|
-
const lines = ndjsonText.trim().split('\n');
|
|
7
|
-
const parsedLines = lines.map((line, counter) => {
|
|
8
|
-
try {
|
|
9
|
-
return JSON.parse(line);
|
|
10
|
-
}
|
|
11
|
-
catch (error) {
|
|
12
|
-
throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
|
|
13
|
-
}
|
|
14
|
-
});
|
|
15
|
-
return (0, schema_1.makeTableFromData)(parsedLines);
|
|
16
|
-
}
|
|
17
|
-
exports.parseNDJSONSync = parseNDJSONSync;
|
package/dist/ndgeoson-loader.js
DELETED
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports._typecheckNDJSONLoader = exports.NDJSONLoader = void 0;
|
|
4
|
-
const parse_ndjson_1 = require("./lib/parsers/parse-ndjson");
|
|
5
|
-
const parse_ndjson_in_batches_1 = require("./lib/parsers/parse-ndjson-in-batches");
|
|
6
|
-
// __VERSION__ is injected by babel-plugin-version-inline
|
|
7
|
-
// @ts-ignore TS2304: Cannot find name '__VERSION__'.
|
|
8
|
-
const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
|
|
9
|
-
const DEFAULT_NDGEOJSON_LOADER_OPTIONS = {
|
|
10
|
-
geojson: {
|
|
11
|
-
shape: 'object-row-table'
|
|
12
|
-
},
|
|
13
|
-
gis: {
|
|
14
|
-
format: 'geojson'
|
|
15
|
-
}
|
|
16
|
-
};
|
|
17
|
-
exports.NDJSONLoader = {
|
|
18
|
-
name: 'NDJSON',
|
|
19
|
-
id: 'ndjson',
|
|
20
|
-
module: 'json',
|
|
21
|
-
version: VERSION,
|
|
22
|
-
extensions: ['ndjson', 'ndgeojson'],
|
|
23
|
-
mimeTypes: [
|
|
24
|
-
'application/geo+x-ndjson',
|
|
25
|
-
'application/geo+x-ldjson',
|
|
26
|
-
'application/jsonlines',
|
|
27
|
-
'application/geo+json-seq',
|
|
28
|
-
'application/x-ndjson'
|
|
29
|
-
],
|
|
30
|
-
category: 'table',
|
|
31
|
-
text: true,
|
|
32
|
-
parse: async (arrayBuffer) => (0, parse_ndjson_1.parseNDJSONSync)(new TextDecoder().decode(arrayBuffer)),
|
|
33
|
-
parseTextSync: parse_ndjson_1.parseNDJSONSync,
|
|
34
|
-
parseInBatches: parse_ndjson_in_batches_1.parseNDJSONInBatches,
|
|
35
|
-
options: DEFAULT_NDGEOJSON_LOADER_OPTIONS
|
|
36
|
-
};
|
|
37
|
-
exports._typecheckNDJSONLoader = exports.NDJSONLoader;
|