@loaders.gl/json 4.0.0-alpha.4 → 4.0.0-alpha.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (188) hide show
  1. package/dist/bundle.d.ts +2 -0
  2. package/dist/bundle.d.ts.map +1 -0
  3. package/dist/bundle.js +2 -2
  4. package/dist/dist.min.js +3094 -0
  5. package/dist/es5/bundle.js +6 -0
  6. package/dist/es5/bundle.js.map +1 -0
  7. package/dist/es5/geojson-loader.js +157 -0
  8. package/dist/es5/geojson-loader.js.map +1 -0
  9. package/dist/es5/geojson-writer.js +27 -0
  10. package/dist/es5/geojson-writer.js.map +1 -0
  11. package/dist/es5/index.js +69 -0
  12. package/dist/es5/index.js.map +1 -0
  13. package/dist/es5/json-loader.js +67 -0
  14. package/dist/es5/json-loader.js.map +1 -0
  15. package/dist/es5/json-writer.js +42 -0
  16. package/dist/es5/json-writer.js.map +1 -0
  17. package/dist/es5/lib/clarinet/clarinet.js +446 -0
  18. package/dist/es5/lib/clarinet/clarinet.js.map +1 -0
  19. package/dist/es5/lib/encoders/encode-utils.js +42 -0
  20. package/dist/es5/lib/encoders/encode-utils.js.map +1 -0
  21. package/dist/es5/lib/encoders/geojson-encoder.js +178 -0
  22. package/dist/es5/lib/encoders/geojson-encoder.js.map +1 -0
  23. package/dist/es5/lib/encoders/json-encoder.js +30 -0
  24. package/dist/es5/lib/encoders/json-encoder.js.map +1 -0
  25. package/dist/es5/lib/encoders/utf8-encoder.js +54 -0
  26. package/dist/es5/lib/encoders/utf8-encoder.js.map +1 -0
  27. package/dist/es5/lib/json-parser/json-parser.js +140 -0
  28. package/dist/es5/lib/json-parser/json-parser.js.map +1 -0
  29. package/dist/es5/lib/json-parser/streaming-json-parser.js +123 -0
  30. package/dist/es5/lib/json-parser/streaming-json-parser.js.map +1 -0
  31. package/dist/es5/lib/jsonpath/jsonpath.js +119 -0
  32. package/dist/es5/lib/jsonpath/jsonpath.js.map +1 -0
  33. package/dist/es5/lib/parsers/parse-json-in-batches.js +206 -0
  34. package/dist/es5/lib/parsers/parse-json-in-batches.js.map +1 -0
  35. package/dist/es5/lib/parsers/parse-json.js +38 -0
  36. package/dist/es5/lib/parsers/parse-json.js.map +1 -0
  37. package/dist/es5/lib/parsers/parse-ndjson-in-batches.js +114 -0
  38. package/dist/es5/lib/parsers/parse-ndjson-in-batches.js.map +1 -0
  39. package/dist/es5/lib/parsers/parse-ndjson.js +19 -0
  40. package/dist/es5/lib/parsers/parse-ndjson.js.map +1 -0
  41. package/dist/es5/ndgeoson-loader.js +54 -0
  42. package/dist/es5/ndgeoson-loader.js.map +1 -0
  43. package/dist/es5/ndjson-loader.js +44 -0
  44. package/dist/es5/ndjson-loader.js.map +1 -0
  45. package/dist/es5/workers/geojson-worker.js +6 -0
  46. package/dist/es5/workers/geojson-worker.js.map +1 -0
  47. package/dist/esm/bundle.js +4 -0
  48. package/dist/esm/bundle.js.map +1 -0
  49. package/dist/esm/geojson-loader.js +79 -0
  50. package/dist/esm/geojson-loader.js.map +1 -0
  51. package/dist/esm/geojson-writer.js +18 -0
  52. package/dist/esm/geojson-writer.js.map +1 -0
  53. package/dist/esm/index.js +9 -0
  54. package/dist/esm/index.js.map +1 -0
  55. package/dist/esm/json-loader.js +48 -0
  56. package/dist/esm/json-loader.js.map +1 -0
  57. package/dist/esm/json-writer.js +14 -0
  58. package/dist/esm/json-writer.js.map +1 -0
  59. package/dist/esm/lib/clarinet/LICENSE +28 -0
  60. package/dist/esm/lib/clarinet/clarinet.js +415 -0
  61. package/dist/esm/lib/clarinet/clarinet.js.map +1 -0
  62. package/dist/esm/lib/encoders/encode-utils.js +31 -0
  63. package/dist/esm/lib/encoders/encode-utils.js.map +1 -0
  64. package/dist/esm/lib/encoders/geojson-encoder.js +98 -0
  65. package/dist/esm/lib/encoders/geojson-encoder.js.map +1 -0
  66. package/dist/esm/lib/encoders/json-encoder.js +12 -0
  67. package/dist/esm/lib/encoders/json-encoder.js.map +1 -0
  68. package/dist/esm/lib/encoders/utf8-encoder.js +32 -0
  69. package/dist/esm/lib/encoders/utf8-encoder.js.map +1 -0
  70. package/dist/{lib/parser → esm/lib/json-parser}/json-parser.js +4 -22
  71. package/dist/esm/lib/json-parser/json-parser.js.map +1 -0
  72. package/dist/{lib/parser → esm/lib/json-parser}/streaming-json-parser.js +2 -23
  73. package/dist/esm/lib/json-parser/streaming-json-parser.js.map +1 -0
  74. package/dist/esm/lib/jsonpath/jsonpath.js +67 -0
  75. package/dist/esm/lib/jsonpath/jsonpath.js.map +1 -0
  76. package/dist/{lib → esm/lib/parsers}/parse-json-in-batches.js +19 -15
  77. package/dist/esm/lib/parsers/parse-json-in-batches.js.map +1 -0
  78. package/dist/{lib → esm/lib/parsers}/parse-json.js +4 -9
  79. package/dist/esm/lib/parsers/parse-json.js.map +1 -0
  80. package/dist/{lib → esm/lib/parsers}/parse-ndjson-in-batches.js +3 -6
  81. package/dist/esm/lib/parsers/parse-ndjson-in-batches.js.map +1 -0
  82. package/dist/esm/lib/parsers/parse-ndjson.js +13 -0
  83. package/dist/esm/lib/parsers/parse-ndjson.js.map +1 -0
  84. package/dist/esm/ndgeoson-loader.js +27 -0
  85. package/dist/esm/ndgeoson-loader.js.map +1 -0
  86. package/dist/esm/ndjson-loader.js +18 -0
  87. package/dist/esm/ndjson-loader.js.map +1 -0
  88. package/dist/esm/workers/geojson-worker.js +4 -0
  89. package/dist/esm/workers/geojson-worker.js.map +1 -0
  90. package/dist/geojson-loader.d.ts +16 -0
  91. package/dist/geojson-loader.d.ts.map +1 -0
  92. package/dist/geojson-loader.js +65 -69
  93. package/dist/geojson-worker.js +1016 -232
  94. package/dist/geojson-writer.d.ts +6 -0
  95. package/dist/geojson-writer.d.ts.map +1 -0
  96. package/dist/geojson-writer.js +22 -0
  97. package/dist/index.d.ts +13 -0
  98. package/dist/index.d.ts.map +1 -0
  99. package/dist/index.js +24 -6
  100. package/dist/json-loader.d.ts +17 -0
  101. package/dist/json-loader.d.ts.map +1 -0
  102. package/dist/json-loader.js +33 -38
  103. package/dist/json-writer.d.ts +6 -0
  104. package/dist/json-writer.d.ts.map +1 -0
  105. package/dist/json-writer.js +18 -0
  106. package/dist/lib/clarinet/clarinet.d.ts +74 -0
  107. package/dist/lib/clarinet/clarinet.d.ts.map +1 -0
  108. package/dist/lib/clarinet/clarinet.js +510 -493
  109. package/dist/lib/encoders/encode-utils.d.ts +19 -0
  110. package/dist/lib/encoders/encode-utils.d.ts.map +1 -0
  111. package/dist/lib/encoders/encode-utils.js +47 -0
  112. package/dist/lib/encoders/geojson-encoder.d.ts +14 -0
  113. package/dist/lib/encoders/geojson-encoder.d.ts.map +1 -0
  114. package/dist/lib/encoders/geojson-encoder.js +104 -0
  115. package/dist/lib/encoders/json-encoder.d.ts +16 -0
  116. package/dist/lib/encoders/json-encoder.d.ts.map +1 -0
  117. package/dist/lib/encoders/json-encoder.js +22 -0
  118. package/dist/lib/encoders/utf8-encoder.d.ts +12 -0
  119. package/dist/lib/encoders/utf8-encoder.d.ts.map +1 -0
  120. package/dist/lib/encoders/utf8-encoder.js +32 -0
  121. package/dist/lib/json-parser/json-parser.d.ts +22 -0
  122. package/dist/lib/json-parser/json-parser.d.ts.map +1 -0
  123. package/dist/lib/json-parser/json-parser.js +98 -0
  124. package/dist/lib/json-parser/streaming-json-parser.d.ts +37 -0
  125. package/dist/lib/json-parser/streaming-json-parser.d.ts.map +1 -0
  126. package/dist/lib/json-parser/streaming-json-parser.js +100 -0
  127. package/dist/lib/jsonpath/jsonpath.d.ts +32 -0
  128. package/dist/lib/jsonpath/jsonpath.d.ts.map +1 -0
  129. package/dist/lib/jsonpath/jsonpath.js +81 -78
  130. package/dist/lib/parsers/parse-json-in-batches.d.ts +5 -0
  131. package/dist/lib/parsers/parse-json-in-batches.d.ts.map +1 -0
  132. package/dist/lib/parsers/parse-json-in-batches.js +100 -0
  133. package/dist/lib/parsers/parse-json.d.ts +4 -0
  134. package/dist/lib/parsers/parse-json.d.ts.map +1 -0
  135. package/dist/lib/parsers/parse-json.js +32 -0
  136. package/dist/lib/parsers/parse-ndjson-in-batches.d.ts +4 -0
  137. package/dist/lib/parsers/parse-ndjson-in-batches.d.ts.map +1 -0
  138. package/dist/lib/parsers/parse-ndjson-in-batches.js +36 -0
  139. package/dist/lib/parsers/parse-ndjson.d.ts +3 -0
  140. package/dist/lib/parsers/parse-ndjson.d.ts.map +1 -0
  141. package/dist/lib/parsers/parse-ndjson.js +17 -0
  142. package/dist/ndgeoson-loader.d.ts +34 -0
  143. package/dist/ndgeoson-loader.d.ts.map +1 -0
  144. package/dist/ndgeoson-loader.js +37 -0
  145. package/dist/ndjson-loader.d.ts +4 -0
  146. package/dist/ndjson-loader.d.ts.map +1 -0
  147. package/dist/ndjson-loader.js +26 -31
  148. package/dist/workers/geojson-worker.d.ts +2 -0
  149. package/dist/workers/geojson-worker.d.ts.map +1 -0
  150. package/dist/workers/geojson-worker.js +5 -4
  151. package/package.json +10 -10
  152. package/src/geojson-loader.ts +10 -6
  153. package/src/geojson-writer.ts +27 -0
  154. package/src/index.ts +10 -0
  155. package/src/json-loader.ts +15 -24
  156. package/src/json-writer.ts +24 -0
  157. package/src/lib/encoders/encode-utils.ts +54 -0
  158. package/src/lib/encoders/geojson-encoder.ts +139 -0
  159. package/src/lib/encoders/json-encoder.ts +30 -0
  160. package/src/lib/encoders/utf8-encoder.ts +35 -0
  161. package/src/lib/{parse-json-in-batches.ts → parsers/parse-json-in-batches.ts} +30 -8
  162. package/src/lib/{parse-json.ts → parsers/parse-json.ts} +7 -3
  163. package/src/lib/{parse-ndjson-in-batches.ts → parsers/parse-ndjson-in-batches.ts} +1 -1
  164. package/src/lib/parsers/parse-ndjson.ts +15 -0
  165. package/src/ndgeoson-loader.ts +48 -0
  166. package/src/ndjson-loader.ts +20 -27
  167. package/dist/bundle.js.map +0 -1
  168. package/dist/geojson-loader.js.map +0 -1
  169. package/dist/index.js.map +0 -1
  170. package/dist/json-loader.js.map +0 -1
  171. package/dist/jsonl-loader.js +0 -2
  172. package/dist/jsonl-loader.js.map +0 -1
  173. package/dist/lib/clarinet/clarinet.js.map +0 -1
  174. package/dist/lib/jsonpath/jsonpath.js.map +0 -1
  175. package/dist/lib/parse-json-in-batches.js.map +0 -1
  176. package/dist/lib/parse-json.js.map +0 -1
  177. package/dist/lib/parse-ndjson-in-batches.js.map +0 -1
  178. package/dist/lib/parse-ndjson.js +0 -11
  179. package/dist/lib/parse-ndjson.js.map +0 -1
  180. package/dist/lib/parser/json-parser.js.map +0 -1
  181. package/dist/lib/parser/streaming-json-parser.js.map +0 -1
  182. package/dist/ndjson-loader.js.map +0 -1
  183. package/dist/workers/geojson-worker.js.map +0 -1
  184. package/src/jsonl-loader.ts +0 -53
  185. package/src/lib/parse-ndjson.ts +0 -10
  186. /package/dist/{lib → es5/lib}/clarinet/LICENSE +0 -0
  187. /package/src/lib/{parser → json-parser}/json-parser.ts +0 -0
  188. /package/src/lib/{parser → json-parser}/streaming-json-parser.ts +0 -0
@@ -1,86 +1,89 @@
1
- import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
2
- export default class JSONPath {
3
- constructor(path = null) {
4
- _defineProperty(this, "path", void 0);
5
-
6
- this.path = ['$'];
7
-
8
- if (path instanceof JSONPath) {
9
- this.path = [...path.path];
10
- return;
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ /**
4
+ * A parser for a minimal subset of the jsonpath standard
5
+ * Full JSON path parsers for JS exist but are quite large (bundle size)
6
+ *
7
+ * Supports
8
+ *
9
+ * `$.component.component.component`
10
+ */
11
+ class JSONPath {
12
+ constructor(path = null) {
13
+ this.path = ['$'];
14
+ if (path instanceof JSONPath) {
15
+ // @ts-ignore
16
+ this.path = [...path.path];
17
+ return;
18
+ }
19
+ if (Array.isArray(path)) {
20
+ this.path.push(...path);
21
+ return;
22
+ }
23
+ // Parse a string as a JSONPath
24
+ if (typeof path === 'string') {
25
+ this.path = path.split('.');
26
+ if (this.path[0] !== '$') {
27
+ throw new Error('JSONPaths must start with $');
28
+ }
29
+ }
11
30
  }
12
-
13
- if (Array.isArray(path)) {
14
- this.path.push(...path);
15
- return;
31
+ clone() {
32
+ return new JSONPath(this);
16
33
  }
17
-
18
- if (typeof path === 'string') {
19
- this.path = path.split('.');
20
-
21
- if (this.path[0] !== '$') {
22
- throw new Error('JSONPaths must start with $');
23
- }
34
+ toString() {
35
+ return this.path.join('.');
24
36
  }
25
- }
26
-
27
- clone() {
28
- return new JSONPath(this);
29
- }
30
-
31
- toString() {
32
- return this.path.join('.');
33
- }
34
-
35
- push(name) {
36
- this.path.push(name);
37
- }
38
-
39
- pop() {
40
- return this.path.pop();
41
- }
42
-
43
- set(name) {
44
- this.path[this.path.length - 1] = name;
45
- }
46
-
47
- equals(other) {
48
- if (!this || !other || this.path.length !== other.path.length) {
49
- return false;
37
+ push(name) {
38
+ this.path.push(name);
50
39
  }
51
-
52
- for (let i = 0; i < this.path.length; ++i) {
53
- if (this.path[i] !== other.path[i]) {
54
- return false;
55
- }
40
+ pop() {
41
+ return this.path.pop();
56
42
  }
57
-
58
- return true;
59
- }
60
-
61
- setFieldAtPath(object, value) {
62
- const path = [...this.path];
63
- path.shift();
64
- const field = path.pop();
65
-
66
- for (const component of path) {
67
- object = object[component];
43
+ set(name) {
44
+ this.path[this.path.length - 1] = name;
68
45
  }
69
-
70
- object[field] = value;
71
- }
72
-
73
- getFieldAtPath(object) {
74
- const path = [...this.path];
75
- path.shift();
76
- const field = path.pop();
77
-
78
- for (const component of path) {
79
- object = object[component];
46
+ equals(other) {
47
+ if (!this || !other || this.path.length !== other.path.length) {
48
+ return false;
49
+ }
50
+ for (let i = 0; i < this.path.length; ++i) {
51
+ if (this.path[i] !== other.path[i]) {
52
+ return false;
53
+ }
54
+ }
55
+ return true;
56
+ }
57
+ /**
58
+ * Sets the value pointed at by path
59
+ * TODO - handle root path
60
+ * @param object
61
+ * @param value
62
+ */
63
+ setFieldAtPath(object, value) {
64
+ const path = [...this.path];
65
+ path.shift();
66
+ const field = path.pop();
67
+ for (const component of path) {
68
+ object = object[component];
69
+ }
70
+ // @ts-ignore
71
+ object[field] = value;
72
+ }
73
+ /**
74
+ * Gets the value pointed at by path
75
+ * TODO - handle root path
76
+ * @param object
77
+ */
78
+ getFieldAtPath(object) {
79
+ const path = [...this.path];
80
+ path.shift();
81
+ const field = path.pop();
82
+ for (const component of path) {
83
+ object = object[component];
84
+ }
85
+ // @ts-ignore
86
+ return object[field];
80
87
  }
81
-
82
- return object[field];
83
- }
84
-
85
88
  }
86
- //# sourceMappingURL=jsonpath.js.map
89
+ exports.default = JSONPath;
@@ -0,0 +1,5 @@
1
+ import type { TableBatch } from '@loaders.gl/schema';
2
+ import type { JSONLoaderOptions } from '../../json-loader';
3
+ export declare function parseJSONInBatches(binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options: JSONLoaderOptions): AsyncIterable<TableBatch>;
4
+ export declare function rebuildJsonObject(batch: any, data: any): any;
5
+ //# sourceMappingURL=parse-json-in-batches.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parse-json-in-batches.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-json-in-batches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,oBAAoB,CAAC;AACnD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAQzD,wBAAuB,kBAAkB,CACvC,mBAAmB,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACvE,OAAO,EAAE,iBAAiB,GACzB,aAAa,CAAC,UAAU,CAAC,CA+E3B;AAED,wBAAgB,iBAAiB,CAAC,KAAK,KAAA,EAAE,IAAI,KAAA,OAmB5C"}
@@ -0,0 +1,100 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.rebuildJsonObject = exports.parseJSONInBatches = void 0;
7
+ const schema_1 = require("@loaders.gl/schema");
8
+ const loader_utils_1 = require("@loaders.gl/loader-utils");
9
+ const streaming_json_parser_1 = __importDefault(require("../json-parser/streaming-json-parser"));
10
+ const jsonpath_1 = __importDefault(require("../jsonpath/jsonpath"));
11
+ // TODO - support batch size 0 = no batching/single batch?
12
+ // eslint-disable-next-line max-statements, complexity
13
+ async function* parseJSONInBatches(binaryAsyncIterator, options) {
14
+ const asyncIterator = (0, loader_utils_1.makeTextDecoderIterator)(binaryAsyncIterator);
15
+ const { metadata } = options;
16
+ const { jsonpaths } = options.json || {};
17
+ let isFirstChunk = true;
18
+ // TODO fix Schema deduction
19
+ const schema = null; // new Schema([]);
20
+ const shape = options?.json?.shape || 'row-table';
21
+ // @ts-ignore
22
+ const tableBatchBuilder = new schema_1.TableBatchBuilder(schema, {
23
+ ...options,
24
+ shape
25
+ });
26
+ const parser = new streaming_json_parser_1.default({ jsonpaths });
27
+ for await (const chunk of asyncIterator) {
28
+ const rows = parser.write(chunk);
29
+ const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
30
+ if (rows.length > 0 && isFirstChunk) {
31
+ if (metadata) {
32
+ const initialBatch = {
33
+ // Common fields
34
+ shape,
35
+ batchType: 'partial-result',
36
+ data: [],
37
+ length: 0,
38
+ bytesUsed: 0,
39
+ // JSON additions
40
+ container: parser.getPartialResult(),
41
+ jsonpath
42
+ };
43
+ yield initialBatch;
44
+ }
45
+ isFirstChunk = false;
46
+ // schema = deduceSchema(rows);
47
+ }
48
+ // Add the row
49
+ for (const row of rows) {
50
+ tableBatchBuilder.addRow(row);
51
+ // If a batch has been completed, emit it
52
+ const batch = tableBatchBuilder.getFullBatch({ jsonpath });
53
+ if (batch) {
54
+ yield batch;
55
+ }
56
+ }
57
+ tableBatchBuilder.chunkComplete(chunk);
58
+ const batch = tableBatchBuilder.getFullBatch({ jsonpath });
59
+ if (batch) {
60
+ yield batch;
61
+ }
62
+ }
63
+ // yield final batch
64
+ const jsonpath = parser.getStreamingJsonPathAsString();
65
+ const batch = tableBatchBuilder.getFinalBatch({ jsonpath });
66
+ if (batch) {
67
+ yield batch;
68
+ }
69
+ if (metadata) {
70
+ const finalBatch = {
71
+ shape,
72
+ batchType: 'final-result',
73
+ container: parser.getPartialResult(),
74
+ jsonpath: parser.getStreamingJsonPathAsString(),
75
+ data: [],
76
+ length: 0
77
+ // schema: null
78
+ };
79
+ yield finalBatch;
80
+ }
81
+ }
82
+ exports.parseJSONInBatches = parseJSONInBatches;
83
+ function rebuildJsonObject(batch, data) {
84
+ // Last batch will have this special type and will provide all the root object of the parsed file
85
+ (0, loader_utils_1.assert)(batch.batchType === 'final-result');
86
+ // The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects
87
+ if (batch.jsonpath === '$') {
88
+ return data;
89
+ }
90
+ // (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object
91
+ if (batch.jsonpath && batch.jsonpath.length > 1) {
92
+ const topLevelObject = batch.container;
93
+ const streamingPath = new jsonpath_1.default(batch.jsonpath);
94
+ streamingPath.setFieldAtPath(topLevelObject, data);
95
+ return topLevelObject;
96
+ }
97
+ // No jsonpath, in this case nothing was streamed.
98
+ return batch.container;
99
+ }
100
+ exports.rebuildJsonObject = rebuildJsonObject;
@@ -0,0 +1,4 @@
1
+ import type { RowTable } from '@loaders.gl/schema';
2
+ import type { JSONLoaderOptions } from '../../json-loader';
3
+ export declare function parseJSONSync(jsonText: string, options: JSONLoaderOptions): RowTable;
4
+ //# sourceMappingURL=parse-json.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parse-json.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-json.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAEzD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,QAAQ,CAWpF"}
@@ -0,0 +1,32 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.parseJSONSync = void 0;
4
+ const schema_1 = require("@loaders.gl/schema");
5
+ function parseJSONSync(jsonText, options) {
6
+ try {
7
+ const json = JSON.parse(jsonText);
8
+ if (options.json?.table) {
9
+ const data = getFirstArray(json) || json;
10
+ return (0, schema_1.makeTableFromData)(data);
11
+ }
12
+ return json;
13
+ }
14
+ catch (error) {
15
+ throw new Error('JSONLoader: failed to parse JSON');
16
+ }
17
+ }
18
+ exports.parseJSONSync = parseJSONSync;
19
+ function getFirstArray(json) {
20
+ if (Array.isArray(json)) {
21
+ return json;
22
+ }
23
+ if (json && typeof json === 'object') {
24
+ for (const value of Object.values(json)) {
25
+ const array = getFirstArray(value);
26
+ if (array) {
27
+ return array;
28
+ }
29
+ }
30
+ }
31
+ return null;
32
+ }
@@ -0,0 +1,4 @@
1
+ import type { Batch } from '@loaders.gl/schema';
2
+ import { LoaderOptions } from '@loaders.gl/loader-utils';
3
+ export declare function parseNDJSONInBatches(binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: LoaderOptions): AsyncIterable<Batch>;
4
+ //# sourceMappingURL=parse-ndjson-in-batches.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parse-ndjson-in-batches.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-ndjson-in-batches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EACL,aAAa,EAId,MAAM,0BAA0B,CAAC;AAElC,wBAAuB,oBAAoB,CACzC,mBAAmB,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACvE,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,KAAK,CAAC,CA+BtB"}
@@ -0,0 +1,36 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.parseNDJSONInBatches = void 0;
4
+ const schema_1 = require("@loaders.gl/schema");
5
+ const loader_utils_1 = require("@loaders.gl/loader-utils");
6
+ async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
7
+ const textIterator = (0, loader_utils_1.makeTextDecoderIterator)(binaryAsyncIterator);
8
+ const lineIterator = (0, loader_utils_1.makeLineIterator)(textIterator);
9
+ const numberedLineIterator = (0, loader_utils_1.makeNumberedLineIterator)(lineIterator);
10
+ const schema = null;
11
+ const shape = 'row-table';
12
+ // @ts-ignore
13
+ const tableBatchBuilder = new schema_1.TableBatchBuilder(schema, {
14
+ ...options,
15
+ shape
16
+ });
17
+ for await (const { counter, line } of numberedLineIterator) {
18
+ try {
19
+ const row = JSON.parse(line);
20
+ tableBatchBuilder.addRow(row);
21
+ tableBatchBuilder.chunkComplete(line);
22
+ const batch = tableBatchBuilder.getFullBatch();
23
+ if (batch) {
24
+ yield batch;
25
+ }
26
+ }
27
+ catch (error) {
28
+ throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
29
+ }
30
+ }
31
+ const batch = tableBatchBuilder.getFinalBatch();
32
+ if (batch) {
33
+ yield batch;
34
+ }
35
+ }
36
+ exports.parseNDJSONInBatches = parseNDJSONInBatches;
@@ -0,0 +1,3 @@
1
+ import type { ArrayRowTable, ObjectRowTable } from '@loaders.gl/schema';
2
+ export declare function parseNDJSONSync(ndjsonText: string): ArrayRowTable | ObjectRowTable;
3
+ //# sourceMappingURL=parse-ndjson.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"parse-ndjson.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-ndjson.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAGtE,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,cAAc,CAWlF"}
@@ -0,0 +1,17 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.parseNDJSONSync = void 0;
4
+ const schema_1 = require("@loaders.gl/schema");
5
+ function parseNDJSONSync(ndjsonText) {
6
+ const lines = ndjsonText.trim().split('\n');
7
+ const parsedLines = lines.map((line, counter) => {
8
+ try {
9
+ return JSON.parse(line);
10
+ }
11
+ catch (error) {
12
+ throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
13
+ }
14
+ });
15
+ return (0, schema_1.makeTableFromData)(parsedLines);
16
+ }
17
+ exports.parseNDJSONSync = parseNDJSONSync;
@@ -0,0 +1,34 @@
1
+ import type { LoaderWithParser, LoaderOptions } from '@loaders.gl/loader-utils';
2
+ import { parseNDJSONSync } from './lib/parsers/parse-ndjson';
3
+ import { parseNDJSONInBatches } from './lib/parsers/parse-ndjson-in-batches';
4
+ export type NDGeoJSONLoaderOptions = LoaderOptions & {
5
+ geojson?: {
6
+ shape?: 'object-row-table';
7
+ };
8
+ gis?: {
9
+ format: 'geojson';
10
+ };
11
+ };
12
+ export declare const NDJSONLoader: {
13
+ name: string;
14
+ id: string;
15
+ module: string;
16
+ version: any;
17
+ extensions: string[];
18
+ mimeTypes: string[];
19
+ category: string;
20
+ text: boolean;
21
+ parse: (arrayBuffer: ArrayBuffer) => Promise<import("@loaders.gl/schema").ArrayRowTable | import("@loaders.gl/schema").ObjectRowTable>;
22
+ parseTextSync: typeof parseNDJSONSync;
23
+ parseInBatches: typeof parseNDJSONInBatches;
24
+ options: {
25
+ geojson: {
26
+ shape: string;
27
+ };
28
+ gis: {
29
+ format: string;
30
+ };
31
+ };
32
+ };
33
+ export declare const _typecheckNDJSONLoader: LoaderWithParser;
34
+ //# sourceMappingURL=ndgeoson-loader.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ndgeoson-loader.d.ts","sourceRoot":"","sources":["../src/ndgeoson-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,EAAC,eAAe,EAAC,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAC,oBAAoB,EAAC,MAAM,uCAAuC,CAAC;AAM3E,MAAM,MAAM,sBAAsB,GAAG,aAAa,GAAG;IACnD,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,kBAAkB,CAAC;KAC5B,CAAC;IACF,GAAG,CAAC,EAAE;QACJ,MAAM,EAAE,SAAS,CAAC;KACnB,CAAC;CACH,CAAC;AAWF,eAAO,MAAM,YAAY;;;;;;;;;yBAeI,WAAW;;;;;;;;;;;CAIvC,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,gBAA+B,CAAC"}
@@ -0,0 +1,37 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports._typecheckNDJSONLoader = exports.NDJSONLoader = void 0;
4
+ const parse_ndjson_1 = require("./lib/parsers/parse-ndjson");
5
+ const parse_ndjson_in_batches_1 = require("./lib/parsers/parse-ndjson-in-batches");
6
+ // __VERSION__ is injected by babel-plugin-version-inline
7
+ // @ts-ignore TS2304: Cannot find name '__VERSION__'.
8
+ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
9
+ const DEFAULT_NDGEOJSON_LOADER_OPTIONS = {
10
+ geojson: {
11
+ shape: 'object-row-table'
12
+ },
13
+ gis: {
14
+ format: 'geojson'
15
+ }
16
+ };
17
+ exports.NDJSONLoader = {
18
+ name: 'NDJSON',
19
+ id: 'ndjson',
20
+ module: 'json',
21
+ version: VERSION,
22
+ extensions: ['ndjson', 'ndgeojson'],
23
+ mimeTypes: [
24
+ 'application/geo+x-ndjson',
25
+ 'application/geo+x-ldjson',
26
+ 'application/jsonlines',
27
+ 'application/geo+json-seq',
28
+ 'application/x-ndjson'
29
+ ],
30
+ category: 'table',
31
+ text: true,
32
+ parse: async (arrayBuffer) => (0, parse_ndjson_1.parseNDJSONSync)(new TextDecoder().decode(arrayBuffer)),
33
+ parseTextSync: parse_ndjson_1.parseNDJSONSync,
34
+ parseInBatches: parse_ndjson_in_batches_1.parseNDJSONInBatches,
35
+ options: DEFAULT_NDGEOJSON_LOADER_OPTIONS
36
+ };
37
+ exports._typecheckNDJSONLoader = exports.NDJSONLoader;
@@ -0,0 +1,4 @@
1
+ import { LoaderWithParser, LoaderOptions } from '@loaders.gl/loader-utils';
2
+ import { ObjectRowTable, ArrayRowTable, TableBatch } from '@loaders.gl/schema';
3
+ export declare const NDJSONLoader: LoaderWithParser<ObjectRowTable | ArrayRowTable, TableBatch, LoaderOptions>;
4
+ //# sourceMappingURL=ndjson-loader.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ndjson-loader.d.ts","sourceRoot":"","sources":["../src/ndjson-loader.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACzE,OAAO,EAAC,cAAc,EAAE,aAAa,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAQ7E,eAAO,MAAM,YAAY,EAAE,gBAAgB,CACzC,cAAc,GAAG,aAAa,EAC9B,UAAU,EACV,aAAa,CAkBd,CAAC"}
@@ -1,32 +1,27 @@
1
- import parseNDJSONSync from './lib/parse-ndjson';
2
- import parseNDJSONInBatches from './lib/parse-ndjson-in-batches';
3
- const VERSION = typeof "4.0.0-alpha.4" !== 'undefined' ? "4.0.0-alpha.4" : 'latest';
4
- export const NDJSONLoader = {
5
- name: 'NDJSON',
6
- id: 'ndjson',
7
- module: 'json',
8
- version: VERSION,
9
- extensions: ['ndjson'],
10
- mimeTypes: ['application/x-ndjson'],
11
- category: 'table',
12
- text: true,
13
- parse,
14
- parseTextSync,
15
- parseInBatches,
16
- options: {}
1
+ "use strict";
2
+ // loaders.gl, MIT license
3
+ Object.defineProperty(exports, "__esModule", { value: true });
4
+ exports.NDJSONLoader = void 0;
5
+ const parse_ndjson_1 = require("./lib/parsers/parse-ndjson");
6
+ const parse_ndjson_in_batches_1 = require("./lib/parsers/parse-ndjson-in-batches");
7
+ // __VERSION__ is injected by babel-plugin-version-inline
8
+ // @ts-ignore TS2304: Cannot find name '__VERSION__'.
9
+ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
10
+ exports.NDJSONLoader = {
11
+ name: 'NDJSON',
12
+ id: 'ndjson',
13
+ module: 'json',
14
+ version: VERSION,
15
+ extensions: ['ndjson', 'jsonl'],
16
+ mimeTypes: [
17
+ 'application/x-ndjson',
18
+ 'application/jsonlines',
19
+ 'application/json-seq'
20
+ ],
21
+ category: 'table',
22
+ text: true,
23
+ parse: async (arrayBuffer) => (0, parse_ndjson_1.parseNDJSONSync)(new TextDecoder().decode(arrayBuffer)),
24
+ parseTextSync: parse_ndjson_1.parseNDJSONSync,
25
+ parseInBatches: parse_ndjson_in_batches_1.parseNDJSONInBatches,
26
+ options: {}
17
27
  };
18
-
19
- async function parse(arrayBuffer) {
20
- return parseTextSync(new TextDecoder().decode(arrayBuffer));
21
- }
22
-
23
- function parseTextSync(text) {
24
- return parseNDJSONSync(text);
25
- }
26
-
27
- function parseInBatches(asyncIterator, options) {
28
- return parseNDJSONInBatches(asyncIterator, options);
29
- }
30
-
31
- export const _typecheckNDJSONLoader = NDJSONLoader;
32
- //# sourceMappingURL=ndjson-loader.js.map
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=geojson-worker.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"geojson-worker.d.ts","sourceRoot":"","sources":["../../src/workers/geojson-worker.ts"],"names":[],"mappings":""}
@@ -1,4 +1,5 @@
1
- import { createLoaderWorker } from '@loaders.gl/loader-utils';
2
- import { GeoJSONLoader } from '../geojson-loader';
3
- createLoaderWorker(GeoJSONLoader);
4
- //# sourceMappingURL=geojson-worker.js.map
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const loader_utils_1 = require("@loaders.gl/loader-utils");
4
+ const geojson_loader_1 = require("../geojson-loader");
5
+ (0, loader_utils_1.createLoaderWorker)(geojson_loader_1.GeoJSONLoader);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@loaders.gl/json",
3
- "version": "4.0.0-alpha.4",
3
+ "version": "4.0.0-alpha.6",
4
4
  "description": "Framework-independent loader for JSON and streaming JSON formats",
5
5
  "license": "MIT",
6
6
  "publishConfig": {
@@ -20,9 +20,9 @@
20
20
  "JSON stream",
21
21
  "JSON async iterator"
22
22
  ],
23
- "types": "src/index.ts",
24
- "main": "dist/index.js",
25
- "module": "dist/index.js",
23
+ "types": "dist/index.d.ts",
24
+ "main": "dist/es5/index.js",
25
+ "module": "dist/esm/index.js",
26
26
  "sideEffects": false,
27
27
  "files": [
28
28
  "src",
@@ -31,13 +31,13 @@
31
31
  ],
32
32
  "scripts": {
33
33
  "pre-build": "npm run build-bundle && npm run build-worker",
34
- "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/bundle.js",
35
- "build-worker": "esbuild src/workers/geojson-worker.ts --bundle --outfile=dist/geojson-worker.js"
34
+ "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/dist.min.js",
35
+ "build-worker": "esbuild src/workers/geojson-worker.ts --bundle --outfile=dist/geojson-worker.js --define:__VERSION__=\\\"$npm_package_version\\\""
36
36
  },
37
37
  "dependencies": {
38
- "@loaders.gl/gis": "4.0.0-alpha.4",
39
- "@loaders.gl/loader-utils": "4.0.0-alpha.4",
40
- "@loaders.gl/schema": "4.0.0-alpha.4"
38
+ "@loaders.gl/gis": "4.0.0-alpha.6",
39
+ "@loaders.gl/loader-utils": "4.0.0-alpha.6",
40
+ "@loaders.gl/schema": "4.0.0-alpha.6"
41
41
  },
42
- "gitHead": "53026061b3c8871f7e96d3a5826125cc6613bddc"
42
+ "gitHead": "acc1985050dfaa0f1f0c066f8da5bce7454a046c"
43
43
  }
@@ -1,8 +1,11 @@
1
+ // loaders.gl, MIT license
2
+
1
3
  import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils';
2
4
  import type {JSONLoaderOptions} from './json-loader';
3
5
  import {geojsonToBinary} from '@loaders.gl/gis';
4
- import parseJSONSync from './lib/parse-json';
5
- import parseJSONInBatches from './lib/parse-json-in-batches';
6
+ import {parseJSONSync} from './lib/parsers/parse-json';
7
+ import {parseJSONInBatches} from './lib/parsers/parse-json-in-batches';
8
+ import {GeoJSONRowTable} from '@loaders.gl/schema';
6
9
 
7
10
  // __VERSION__ is injected by babel-plugin-version-inline
8
11
  // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -13,7 +16,7 @@ export type GeoJSONLoaderOptions = JSONLoaderOptions & {
13
16
  shape?: 'object-row-table';
14
17
  };
15
18
  gis?: {
16
- format: 'geojson';
19
+ format?: 'geojson' | 'binary';
17
20
  };
18
21
  };
19
22
 
@@ -61,12 +64,13 @@ function parseTextSync(text, options) {
61
64
  options = {...DEFAULT_GEOJSON_LOADER_OPTIONS, ...options};
62
65
  options.json = {...DEFAULT_GEOJSON_LOADER_OPTIONS.geojson, ...options.geojson};
63
66
  options.gis = options.gis || {};
64
- const json = parseJSONSync(text, options);
67
+ const table = parseJSONSync(text, options) as GeoJSONRowTable;
68
+ table.shape = 'geojson-row-table';
65
69
  switch (options.gis.format) {
66
70
  case 'binary':
67
- return geojsonToBinary(json);
71
+ return geojsonToBinary(table.data);
68
72
  default:
69
- return json;
73
+ return table;
70
74
  }
71
75
  }
72
76