@loaders.gl/core 4.0.0-alpha.20 → 4.0.0-alpha.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +34 -39
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/iterators/batch-iterators/timed-batch-iterator.js.map +1 -1
- package/dist/es5/iterators/make-stream/make-dom-stream.js.map +1 -1
- package/dist/es5/iterators/make-stream/make-node-stream.js.map +1 -1
- package/dist/es5/javascript-utils/is-type.js +1 -1
- package/dist/es5/javascript-utils/is-type.js.map +1 -1
- package/dist/es5/lib/api/encode-table.js.map +1 -1
- package/dist/es5/lib/api/encode.js.map +1 -1
- package/dist/es5/lib/api/load-in-batches.js +6 -3
- package/dist/es5/lib/api/load-in-batches.js.map +1 -1
- package/dist/es5/lib/api/load.js.map +1 -1
- package/dist/es5/lib/api/parse-in-batches.js +11 -13
- package/dist/es5/lib/api/parse-in-batches.js.map +1 -1
- package/dist/es5/lib/api/parse-sync.js +2 -6
- package/dist/es5/lib/api/parse-sync.js.map +1 -1
- package/dist/es5/lib/api/parse.js +20 -22
- package/dist/es5/lib/api/parse.js.map +1 -1
- package/dist/es5/lib/api/save.js.map +1 -1
- package/dist/es5/lib/api/select-loader.js +5 -3
- package/dist/es5/lib/api/select-loader.js.map +1 -1
- package/dist/es5/lib/filesystems/browser-filesystem.js.map +1 -1
- package/dist/es5/lib/filesystems/filesystem.js.map +1 -1
- package/dist/es5/lib/init.js +1 -1
- package/dist/es5/lib/init.js.map +1 -1
- package/dist/es5/lib/loader-utils/get-data.js.map +1 -1
- package/dist/es5/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/es5/lib/loader-utils/loader-context.js +2 -2
- package/dist/es5/lib/loader-utils/loader-context.js.map +1 -1
- package/dist/es5/lib/loader-utils/loggers.js.map +1 -1
- package/dist/es5/lib/loader-utils/option-utils.js +3 -4
- package/dist/es5/lib/loader-utils/option-utils.js.map +1 -1
- package/dist/es5/lib/progress/fetch-progress.js.map +1 -1
- package/dist/es5/null-loader.js +6 -14
- package/dist/es5/null-loader.js.map +1 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/iterators/batch-iterators/timed-batch-iterator.js.map +1 -1
- package/dist/esm/iterators/make-stream/make-dom-stream.js.map +1 -1
- package/dist/esm/iterators/make-stream/make-node-stream.js.map +1 -1
- package/dist/esm/javascript-utils/is-type.js +1 -1
- package/dist/esm/javascript-utils/is-type.js.map +1 -1
- package/dist/esm/lib/api/encode-table.js.map +1 -1
- package/dist/esm/lib/api/encode.js.map +1 -1
- package/dist/esm/lib/api/load-in-batches.js +6 -3
- package/dist/esm/lib/api/load-in-batches.js.map +1 -1
- package/dist/esm/lib/api/load.js.map +1 -1
- package/dist/esm/lib/api/parse-in-batches.js +3 -4
- package/dist/esm/lib/api/parse-in-batches.js.map +1 -1
- package/dist/esm/lib/api/parse-sync.js +2 -4
- package/dist/esm/lib/api/parse-sync.js.map +1 -1
- package/dist/esm/lib/api/parse.js +10 -11
- package/dist/esm/lib/api/parse.js.map +1 -1
- package/dist/esm/lib/api/save.js.map +1 -1
- package/dist/esm/lib/api/select-loader.js +5 -3
- package/dist/esm/lib/api/select-loader.js.map +1 -1
- package/dist/esm/lib/filesystems/browser-filesystem.js.map +1 -1
- package/dist/esm/lib/filesystems/filesystem.js.map +1 -1
- package/dist/esm/lib/init.js +1 -1
- package/dist/esm/lib/init.js.map +1 -1
- package/dist/esm/lib/loader-utils/get-data.js.map +1 -1
- package/dist/esm/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/esm/lib/loader-utils/loader-context.js +2 -2
- package/dist/esm/lib/loader-utils/loader-context.js.map +1 -1
- package/dist/esm/lib/loader-utils/loggers.js.map +1 -1
- package/dist/esm/lib/loader-utils/option-utils.js +2 -2
- package/dist/esm/lib/loader-utils/option-utils.js.map +1 -1
- package/dist/esm/lib/progress/fetch-progress.js.map +1 -1
- package/dist/esm/null-loader.js +6 -14
- package/dist/esm/null-loader.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/iterators/batch-iterators/timed-batch-iterator.d.ts +1 -1
- package/dist/iterators/batch-iterators/timed-batch-iterator.d.ts.map +1 -1
- package/dist/iterators/make-stream/make-dom-stream.d.ts.map +1 -1
- package/dist/iterators/make-stream/make-dom-stream.js +1 -0
- package/dist/iterators/make-stream/make-node-stream.d.ts.map +1 -1
- package/dist/iterators/make-stream/make-node-stream.js +5 -2
- package/dist/javascript-utils/is-type.d.ts +5 -5
- package/dist/javascript-utils/is-type.d.ts.map +1 -1
- package/dist/javascript-utils/is-type.js +1 -1
- package/dist/lib/api/encode.d.ts +5 -5
- package/dist/lib/api/encode.d.ts.map +1 -1
- package/dist/lib/api/load-in-batches.d.ts +5 -0
- package/dist/lib/api/load-in-batches.d.ts.map +1 -1
- package/dist/lib/api/load-in-batches.js +10 -3
- package/dist/lib/api/load.d.ts +1 -2
- package/dist/lib/api/load.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.d.ts +13 -7
- package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.js +2 -2
- package/dist/lib/api/parse-sync.d.ts +13 -7
- package/dist/lib/api/parse-sync.d.ts.map +1 -1
- package/dist/lib/api/parse-sync.js +2 -7
- package/dist/lib/api/parse.d.ts +11 -1
- package/dist/lib/api/parse.d.ts.map +1 -1
- package/dist/lib/api/parse.js +14 -11
- package/dist/lib/api/save.d.ts +2 -2
- package/dist/lib/api/save.d.ts.map +1 -1
- package/dist/lib/api/select-loader.d.ts.map +1 -1
- package/dist/lib/api/select-loader.js +8 -3
- package/dist/lib/filesystems/browser-filesystem.d.ts +2 -2
- package/dist/lib/filesystems/browser-filesystem.d.ts.map +1 -1
- package/dist/lib/filesystems/browser-filesystem.js +1 -1
- package/dist/lib/filesystems/filesystem.d.ts +3 -3
- package/dist/lib/filesystems/filesystem.d.ts.map +1 -1
- package/dist/lib/filesystems/filesystem.js +1 -0
- package/dist/lib/init.d.ts.map +1 -1
- package/dist/lib/init.js +1 -0
- package/dist/lib/loader-utils/get-data.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-data.js +1 -0
- package/dist/lib/loader-utils/get-fetch-function.d.ts +2 -2
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts +5 -2
- package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
- package/dist/lib/loader-utils/loader-context.js +3 -3
- package/dist/lib/loader-utils/loggers.d.ts +10 -8
- package/dist/lib/loader-utils/loggers.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.d.ts +1 -1
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +7 -5
- package/dist/null-loader.d.ts +6 -3
- package/dist/null-loader.d.ts.map +1 -1
- package/dist/null-loader.js +9 -14
- package/dist/null-worker-node.js +14 -14
- package/dist/null-worker.js +14 -14
- package/package.json +4 -4
- package/src/index.ts +1 -0
- package/src/iterators/batch-iterators/timed-batch-iterator.ts +4 -1
- package/src/iterators/make-stream/make-dom-stream.ts +2 -1
- package/src/iterators/make-stream/make-node-stream.ts +5 -2
- package/src/javascript-utils/is-type.ts +12 -8
- package/src/lib/api/encode-table.ts +1 -1
- package/src/lib/api/encode.ts +10 -10
- package/src/lib/api/load-in-batches.ts +45 -7
- package/src/lib/api/load.ts +2 -5
- package/src/lib/api/parse-in-batches.ts +44 -15
- package/src/lib/api/parse-sync.ts +38 -17
- package/src/lib/api/parse.ts +36 -15
- package/src/lib/api/save.ts +2 -2
- package/src/lib/api/select-loader.ts +24 -13
- package/src/lib/filesystems/browser-filesystem.ts +3 -3
- package/src/lib/filesystems/filesystem.ts +5 -3
- package/src/lib/init.ts +1 -0
- package/src/lib/loader-utils/get-data.ts +1 -0
- package/src/lib/loader-utils/get-fetch-function.ts +2 -2
- package/src/lib/loader-utils/loader-context.ts +8 -5
- package/src/lib/loader-utils/loggers.ts +10 -8
- package/src/lib/loader-utils/option-utils.ts +23 -14
- package/src/lib/progress/fetch-progress.ts +9 -1
- package/src/null-loader.ts +23 -17
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"null-loader.d.ts","sourceRoot":"","sources":["../src/null-loader.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,MAAM,EAAE,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"null-loader.d.ts","sourceRoot":"","sources":["../src/null-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAGtF,MAAM,MAAM,iBAAiB,GAAG,aAAa,GAAG;IAC9C,IAAI,CAAC,EAAE,EAAE,CAAC;CACX,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,iBAAiB,CAYnE,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,UAAU,EAAE,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,iBAAiB,CAmBtE,CAAC"}
|
package/dist/null-loader.js
CHANGED
|
@@ -20,16 +20,6 @@ exports.NullWorkerLoader = {
|
|
|
20
20
|
null: {}
|
|
21
21
|
}
|
|
22
22
|
};
|
|
23
|
-
/**
|
|
24
|
-
* Returns arguments passed to the parse API in a format that can be transfered to a
|
|
25
|
-
* web worker. The `context` parameter is stripped using JSON.stringify & parse.
|
|
26
|
-
*/
|
|
27
|
-
function parseSync(arrayBuffer, options, context) {
|
|
28
|
-
if (!options.null.echoParameters)
|
|
29
|
-
return null;
|
|
30
|
-
context = context && JSON.parse(JSON.stringify(context));
|
|
31
|
-
return { arrayBuffer, options, context };
|
|
32
|
-
}
|
|
33
23
|
/**
|
|
34
24
|
* Loads any data and returns null (or optionally passes through data unparsed)
|
|
35
25
|
*/
|
|
@@ -40,7 +30,7 @@ exports.NullLoader = {
|
|
|
40
30
|
version: VERSION,
|
|
41
31
|
mimeTypes: ['application/x.empty'],
|
|
42
32
|
extensions: ['null'],
|
|
43
|
-
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options, context),
|
|
33
|
+
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
|
|
44
34
|
parseSync,
|
|
45
35
|
parseInBatches: async function* generator(asyncIterator, options, context) {
|
|
46
36
|
for await (const batch of asyncIterator) {
|
|
@@ -49,8 +39,13 @@ exports.NullLoader = {
|
|
|
49
39
|
},
|
|
50
40
|
tests: [() => false],
|
|
51
41
|
options: {
|
|
52
|
-
null: {
|
|
53
|
-
echoParameters: false
|
|
54
|
-
}
|
|
42
|
+
null: {}
|
|
55
43
|
}
|
|
56
44
|
};
|
|
45
|
+
/**
|
|
46
|
+
* Returns arguments passed to the parse API in a format that can be transferred to a
|
|
47
|
+
* web worker. The `context` parameter is stripped using JSON.stringify & parse.
|
|
48
|
+
*/
|
|
49
|
+
function parseSync(arrayBuffer, options, context) {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
package/dist/null-worker-node.js
CHANGED
|
@@ -143,7 +143,7 @@ function createLoaderWorker(loader) {
|
|
|
143
143
|
arrayBuffer: input,
|
|
144
144
|
options,
|
|
145
145
|
context: __spreadProps(__spreadValues({}, context), {
|
|
146
|
-
|
|
146
|
+
_parse: parseOnMainThread
|
|
147
147
|
})
|
|
148
148
|
});
|
|
149
149
|
WorkerBody.postMessage("done", { result });
|
|
@@ -156,7 +156,7 @@ function createLoaderWorker(loader) {
|
|
|
156
156
|
}
|
|
157
157
|
};
|
|
158
158
|
}
|
|
159
|
-
function parseOnMainThread(arrayBuffer, options) {
|
|
159
|
+
function parseOnMainThread(arrayBuffer, loader, options, context) {
|
|
160
160
|
return new Promise((resolve, reject) => {
|
|
161
161
|
const id = requestId++;
|
|
162
162
|
const onMessage = (type, payload2) => {
|
|
@@ -180,7 +180,12 @@ function parseOnMainThread(arrayBuffer, options) {
|
|
|
180
180
|
WorkerBody.postMessage("process", payload);
|
|
181
181
|
});
|
|
182
182
|
}
|
|
183
|
-
async function parseData({
|
|
183
|
+
async function parseData({
|
|
184
|
+
loader,
|
|
185
|
+
arrayBuffer,
|
|
186
|
+
options,
|
|
187
|
+
context
|
|
188
|
+
}) {
|
|
184
189
|
let data;
|
|
185
190
|
let parser;
|
|
186
191
|
if (loader.parseSync || loader.parse) {
|
|
@@ -201,13 +206,7 @@ async function parseData({ loader, arrayBuffer, options, context }) {
|
|
|
201
206
|
}
|
|
202
207
|
|
|
203
208
|
// src/null-loader.ts
|
|
204
|
-
var VERSION = true ? "4.0.0-alpha.
|
|
205
|
-
function parseSync(arrayBuffer, options, context) {
|
|
206
|
-
if (!options.null.echoParameters)
|
|
207
|
-
return null;
|
|
208
|
-
context = context && JSON.parse(JSON.stringify(context));
|
|
209
|
-
return { arrayBuffer, options, context };
|
|
210
|
-
}
|
|
209
|
+
var VERSION = true ? "4.0.0-alpha.22" : "latest";
|
|
211
210
|
var NullLoader = {
|
|
212
211
|
name: "Null loader",
|
|
213
212
|
id: "null",
|
|
@@ -215,7 +214,7 @@ var NullLoader = {
|
|
|
215
214
|
version: VERSION,
|
|
216
215
|
mimeTypes: ["application/x.empty"],
|
|
217
216
|
extensions: ["null"],
|
|
218
|
-
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options, context),
|
|
217
|
+
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
|
|
219
218
|
parseSync,
|
|
220
219
|
parseInBatches: async function* generator(asyncIterator, options, context) {
|
|
221
220
|
for await (const batch of asyncIterator) {
|
|
@@ -224,11 +223,12 @@ var NullLoader = {
|
|
|
224
223
|
},
|
|
225
224
|
tests: [() => false],
|
|
226
225
|
options: {
|
|
227
|
-
null: {
|
|
228
|
-
echoParameters: false
|
|
229
|
-
}
|
|
226
|
+
null: {}
|
|
230
227
|
}
|
|
231
228
|
};
|
|
229
|
+
function parseSync(arrayBuffer, options, context) {
|
|
230
|
+
return null;
|
|
231
|
+
}
|
|
232
232
|
|
|
233
233
|
// src/workers/null-worker.ts
|
|
234
234
|
createLoaderWorker(NullLoader);
|
package/dist/null-worker.js
CHANGED
|
@@ -125,7 +125,7 @@
|
|
|
125
125
|
options,
|
|
126
126
|
context: {
|
|
127
127
|
...context,
|
|
128
|
-
|
|
128
|
+
_parse: parseOnMainThread
|
|
129
129
|
}
|
|
130
130
|
});
|
|
131
131
|
WorkerBody.postMessage("done", { result });
|
|
@@ -138,7 +138,7 @@
|
|
|
138
138
|
}
|
|
139
139
|
};
|
|
140
140
|
}
|
|
141
|
-
function parseOnMainThread(arrayBuffer, options) {
|
|
141
|
+
function parseOnMainThread(arrayBuffer, loader, options, context) {
|
|
142
142
|
return new Promise((resolve, reject) => {
|
|
143
143
|
const id = requestId++;
|
|
144
144
|
const onMessage = (type, payload2) => {
|
|
@@ -162,7 +162,12 @@
|
|
|
162
162
|
WorkerBody.postMessage("process", payload);
|
|
163
163
|
});
|
|
164
164
|
}
|
|
165
|
-
async function parseData({
|
|
165
|
+
async function parseData({
|
|
166
|
+
loader,
|
|
167
|
+
arrayBuffer,
|
|
168
|
+
options,
|
|
169
|
+
context
|
|
170
|
+
}) {
|
|
166
171
|
let data;
|
|
167
172
|
let parser;
|
|
168
173
|
if (loader.parseSync || loader.parse) {
|
|
@@ -184,13 +189,7 @@
|
|
|
184
189
|
}
|
|
185
190
|
|
|
186
191
|
// src/null-loader.ts
|
|
187
|
-
var VERSION = true ? "4.0.0-alpha.
|
|
188
|
-
function parseSync(arrayBuffer, options, context) {
|
|
189
|
-
if (!options.null.echoParameters)
|
|
190
|
-
return null;
|
|
191
|
-
context = context && JSON.parse(JSON.stringify(context));
|
|
192
|
-
return { arrayBuffer, options, context };
|
|
193
|
-
}
|
|
192
|
+
var VERSION = true ? "4.0.0-alpha.22" : "latest";
|
|
194
193
|
var NullLoader = {
|
|
195
194
|
name: "Null loader",
|
|
196
195
|
id: "null",
|
|
@@ -198,7 +197,7 @@
|
|
|
198
197
|
version: VERSION,
|
|
199
198
|
mimeTypes: ["application/x.empty"],
|
|
200
199
|
extensions: ["null"],
|
|
201
|
-
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options, context),
|
|
200
|
+
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
|
|
202
201
|
parseSync,
|
|
203
202
|
parseInBatches: async function* generator(asyncIterator, options, context) {
|
|
204
203
|
for await (const batch of asyncIterator) {
|
|
@@ -207,11 +206,12 @@
|
|
|
207
206
|
},
|
|
208
207
|
tests: [() => false],
|
|
209
208
|
options: {
|
|
210
|
-
null: {
|
|
211
|
-
echoParameters: false
|
|
212
|
-
}
|
|
209
|
+
null: {}
|
|
213
210
|
}
|
|
214
211
|
};
|
|
212
|
+
function parseSync(arrayBuffer, options, context) {
|
|
213
|
+
return null;
|
|
214
|
+
}
|
|
215
215
|
|
|
216
216
|
// src/workers/null-worker.ts
|
|
217
217
|
createLoaderWorker(NullLoader);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/core",
|
|
3
|
-
"version": "4.0.0-alpha.
|
|
3
|
+
"version": "4.0.0-alpha.22",
|
|
4
4
|
"description": "The core API for working with loaders.gl loaders and writers",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
49
|
"@babel/runtime": "^7.3.1",
|
|
50
|
-
"@loaders.gl/loader-utils": "4.0.0-alpha.
|
|
51
|
-
"@loaders.gl/worker-utils": "4.0.0-alpha.
|
|
50
|
+
"@loaders.gl/loader-utils": "4.0.0-alpha.22",
|
|
51
|
+
"@loaders.gl/worker-utils": "4.0.0-alpha.22",
|
|
52
52
|
"@probe.gl/log": "^4.0.2"
|
|
53
53
|
},
|
|
54
|
-
"gitHead": "
|
|
54
|
+
"gitHead": "0da838c506d1275383f2fd3d244d9c72b25397d2"
|
|
55
55
|
}
|
package/src/index.ts
CHANGED
|
@@ -55,6 +55,7 @@ export {JSONLoader} from '@loaders.gl/loader-utils';
|
|
|
55
55
|
// EXPERIMENTAL
|
|
56
56
|
export {default as _fetchProgress} from './lib/progress/fetch-progress';
|
|
57
57
|
export {default as _BrowserFileSystem} from './lib/filesystems/browser-filesystem';
|
|
58
|
+
export type {FileSystem} from './lib/filesystems/filesystem';
|
|
58
59
|
|
|
59
60
|
// FOR TESTING
|
|
60
61
|
export {_unregisterLoaders} from './lib/api/register-loaders';
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* "Debounces" batches and returns them in groups
|
|
3
3
|
*/
|
|
4
|
-
export async function* timedBatchIterator<Batch>(
|
|
4
|
+
export async function* timedBatchIterator<Batch>(
|
|
5
|
+
batchIterator: AsyncIterable<Batch>,
|
|
6
|
+
timeout: number
|
|
7
|
+
): AsyncIterable<Batch[]> {
|
|
5
8
|
let start = Date.now();
|
|
6
9
|
let batches: Batch[] = [];
|
|
7
10
|
for await (const batch of batchIterator) {
|
|
@@ -15,7 +15,8 @@ export function makeStream<ArrayBuffer>(
|
|
|
15
15
|
source: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,
|
|
16
16
|
options?: MakeStreamOptions
|
|
17
17
|
): ReadableStream {
|
|
18
|
-
|
|
18
|
+
// TODO - add AsyncGenerator to parameter types?
|
|
19
|
+
const iterator = (source as AsyncGenerator<ArrayBuffer>)[Symbol.asyncIterator]
|
|
19
20
|
? (source as AsyncIterable<ArrayBuffer>)[Symbol.asyncIterator]()
|
|
20
21
|
: (source as Iterable<ArrayBuffer>)[Symbol.iterator]();
|
|
21
22
|
|
|
@@ -13,9 +13,12 @@ export function makeStream<ArrayBuffer>(
|
|
|
13
13
|
source: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,
|
|
14
14
|
options?: ReadableOptions
|
|
15
15
|
): ReadableType {
|
|
16
|
+
// @ts-ignore AsyncGenerator
|
|
16
17
|
const iterator = source[Symbol.asyncIterator]
|
|
17
|
-
?
|
|
18
|
-
|
|
18
|
+
? // @ts-ignore AsyncGenerator
|
|
19
|
+
source[Symbol.asyncIterator]()
|
|
20
|
+
: // @ts-ignore AsyncGenerator
|
|
21
|
+
source[Symbol.iterator]();
|
|
19
22
|
return new AsyncIterableReadable(iterator, options);
|
|
20
23
|
}
|
|
21
24
|
|
|
@@ -1,17 +1,19 @@
|
|
|
1
|
+
// luma.gl, MIT license
|
|
1
2
|
import type {Readable} from 'stream';
|
|
2
3
|
|
|
3
4
|
/** A DOM or Node readable stream */
|
|
4
5
|
export type ReadableStreamType = ReadableStream | Readable;
|
|
5
6
|
|
|
6
|
-
const isBoolean: (x:
|
|
7
|
-
const isFunction: (x:
|
|
7
|
+
const isBoolean: (x: unknown) => boolean = (x) => typeof x === 'boolean';
|
|
8
|
+
const isFunction: (x: unknown) => boolean = (x) => typeof x === 'function';
|
|
8
9
|
|
|
9
|
-
export const isObject: (x:
|
|
10
|
+
export const isObject: (x: unknown) => boolean = (x) => x !== null && typeof x === 'object';
|
|
10
11
|
export const isPureObject: (x: any) => boolean = (x) =>
|
|
11
12
|
isObject(x) && x.constructor === {}.constructor;
|
|
12
13
|
export const isPromise: (x: any) => boolean = (x) => isObject(x) && isFunction(x.then);
|
|
13
14
|
|
|
14
|
-
export const isIterable: (x: any) => boolean = (x) =>
|
|
15
|
+
export const isIterable: (x: any) => boolean = (x) =>
|
|
16
|
+
Boolean(x) && typeof x[Symbol.iterator] === 'function';
|
|
15
17
|
export const isAsyncIterable: (x: any) => boolean = (x) =>
|
|
16
18
|
x && typeof x[Symbol.asyncIterator] === 'function';
|
|
17
19
|
export const isIterator: (x: any) => boolean = (x) => x && isFunction(x.next);
|
|
@@ -20,8 +22,10 @@ export const isResponse: (x: any) => boolean = (x) =>
|
|
|
20
22
|
(typeof Response !== 'undefined' && x instanceof Response) ||
|
|
21
23
|
(x && x.arrayBuffer && x.text && x.json);
|
|
22
24
|
|
|
23
|
-
export const isFile: (x:
|
|
24
|
-
|
|
25
|
+
export const isFile: (x: unknown) => boolean = (x) =>
|
|
26
|
+
typeof File !== 'undefined' && x instanceof File;
|
|
27
|
+
export const isBlob: (x: unknown) => boolean = (x) =>
|
|
28
|
+
typeof Blob !== 'undefined' && x instanceof Blob;
|
|
25
29
|
|
|
26
30
|
/** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */
|
|
27
31
|
export const isBuffer: (x: any) => boolean = (x) => x && typeof x === 'object' && x.isBuffer;
|
|
@@ -38,7 +42,7 @@ export const isWritableNodeStream: (x: any) => boolean = (x) =>
|
|
|
38
42
|
isObject(x) && isFunction(x.end) && isFunction(x.write) && isBoolean(x.writable);
|
|
39
43
|
export const isReadableNodeStream: (x: any) => boolean = (x) =>
|
|
40
44
|
isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
|
|
41
|
-
export const isReadableStream: (x:
|
|
45
|
+
export const isReadableStream: (x: unknown) => boolean = (x) =>
|
|
42
46
|
isReadableDOMStream(x) || isReadableNodeStream(x);
|
|
43
|
-
export const isWritableStream: (x:
|
|
47
|
+
export const isWritableStream: (x: unknown) => boolean = (x) =>
|
|
44
48
|
isWritableDOMStream(x) || isWritableNodeStream(x);
|
|
@@ -65,7 +65,7 @@ export function encodeTableInBatches<WriterT extends Writer = Writer>(
|
|
|
65
65
|
throw new Error('Writer could not encode data in batches');
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
function getIterator(data) {
|
|
68
|
+
function getIterator(data: any): Iterable<{table: any; start: number; end: number}> {
|
|
69
69
|
const dataIterator = [{table: data, start: 0, end: data.length}];
|
|
70
70
|
return dataIterator;
|
|
71
71
|
}
|
package/src/lib/api/encode.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {getLoaderOptions} from './loader-options';
|
|
|
10
10
|
* Encode loaded data into a binary ArrayBuffer using the specified Writer.
|
|
11
11
|
*/
|
|
12
12
|
export async function encode(
|
|
13
|
-
data:
|
|
13
|
+
data: unknown,
|
|
14
14
|
writer: Writer,
|
|
15
15
|
options?: WriterOptions
|
|
16
16
|
): Promise<ArrayBuffer> {
|
|
@@ -40,7 +40,7 @@ export async function encode(
|
|
|
40
40
|
const batches = encodeInBatches(data, writer, options);
|
|
41
41
|
|
|
42
42
|
// Concatenate the output
|
|
43
|
-
const chunks:
|
|
43
|
+
const chunks: unknown[] = [];
|
|
44
44
|
for await (const batch of batches) {
|
|
45
45
|
chunks.push(batch);
|
|
46
46
|
}
|
|
@@ -51,7 +51,7 @@ export async function encode(
|
|
|
51
51
|
if (!isBrowser && writer.encodeURLtoURL) {
|
|
52
52
|
// TODO - how to generate filenames with correct extensions?
|
|
53
53
|
const tmpInputFilename = getTemporaryFilename('input');
|
|
54
|
-
await writeFile(tmpInputFilename, data);
|
|
54
|
+
await writeFile(tmpInputFilename, data as ArrayBuffer);
|
|
55
55
|
|
|
56
56
|
const tmpOutputFilename = getTemporaryFilename('output');
|
|
57
57
|
|
|
@@ -72,7 +72,7 @@ export async function encode(
|
|
|
72
72
|
/**
|
|
73
73
|
* Encode loaded data into a binary ArrayBuffer using the specified Writer.
|
|
74
74
|
*/
|
|
75
|
-
export function encodeSync(data:
|
|
75
|
+
export function encodeSync(data: unknown, writer: Writer, options?: WriterOptions): ArrayBuffer {
|
|
76
76
|
if (writer.encodeSync) {
|
|
77
77
|
return writer.encodeSync(data, options);
|
|
78
78
|
}
|
|
@@ -86,7 +86,7 @@ export function encodeSync(data: any, writer: Writer, options?: WriterOptions):
|
|
|
86
86
|
* @throws if the writer does not generate text output
|
|
87
87
|
*/
|
|
88
88
|
export async function encodeText(
|
|
89
|
-
data:
|
|
89
|
+
data: unknown,
|
|
90
90
|
writer: Writer,
|
|
91
91
|
options?: WriterOptions
|
|
92
92
|
): Promise<string> {
|
|
@@ -106,7 +106,7 @@ export async function encodeText(
|
|
|
106
106
|
* Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
|
|
107
107
|
*/
|
|
108
108
|
export function encodeInBatches(
|
|
109
|
-
data:
|
|
109
|
+
data: unknown,
|
|
110
110
|
writer: Writer,
|
|
111
111
|
options?: WriterOptions
|
|
112
112
|
): AsyncIterable<ArrayBuffer> {
|
|
@@ -124,10 +124,10 @@ export function encodeInBatches(
|
|
|
124
124
|
* @note Node.js only. This function enables using command-line converters as "writers".
|
|
125
125
|
*/
|
|
126
126
|
export async function encodeURLtoURL(
|
|
127
|
-
inputUrl,
|
|
128
|
-
outputUrl,
|
|
127
|
+
inputUrl: string,
|
|
128
|
+
outputUrl: string,
|
|
129
129
|
writer: Writer,
|
|
130
|
-
options
|
|
130
|
+
options?: WriterOptions
|
|
131
131
|
): Promise<string> {
|
|
132
132
|
inputUrl = resolvePath(inputUrl);
|
|
133
133
|
outputUrl = resolvePath(outputUrl);
|
|
@@ -141,7 +141,7 @@ export async function encodeURLtoURL(
|
|
|
141
141
|
/**
|
|
142
142
|
* @todo TODO - this is an unacceptable hack!!!
|
|
143
143
|
*/
|
|
144
|
-
function getIterator(data) {
|
|
144
|
+
function getIterator(data: any): Iterable<{table: any; start: number; end: number}> {
|
|
145
145
|
const dataIterator = [{table: data, start: 0, end: data.length}];
|
|
146
146
|
return dataIterator;
|
|
147
147
|
}
|
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
|
|
3
|
+
import type {
|
|
4
|
+
LoaderWithParser,
|
|
5
|
+
LoaderOptions,
|
|
6
|
+
LoaderContext,
|
|
7
|
+
FetchLike
|
|
8
|
+
} from '@loaders.gl/loader-utils';
|
|
9
|
+
import type {LoaderBatchType, LoaderOptionsType} from '@loaders.gl/loader-utils';
|
|
2
10
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
3
11
|
import {getFetchFunction} from '../loader-utils/get-fetch-function';
|
|
4
12
|
|
|
@@ -6,6 +14,19 @@ import {parseInBatches} from './parse-in-batches';
|
|
|
6
14
|
|
|
7
15
|
type FileType = string | File | Blob | Response | (string | File | Blob | Response)[] | FileList;
|
|
8
16
|
|
|
17
|
+
/**
|
|
18
|
+
* Parses `data` synchronously using a specified loader
|
|
19
|
+
*/
|
|
20
|
+
export async function loadInBatches<
|
|
21
|
+
LoaderT extends LoaderWithParser,
|
|
22
|
+
OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
|
|
23
|
+
>(
|
|
24
|
+
files: FileType,
|
|
25
|
+
loader: LoaderT,
|
|
26
|
+
options?: OptionsT,
|
|
27
|
+
context?: LoaderContext
|
|
28
|
+
): Promise<AsyncIterable<LoaderBatchType<LoaderT>>>;
|
|
29
|
+
|
|
9
30
|
/**
|
|
10
31
|
* Parses `data` using a specified loader
|
|
11
32
|
* @param data
|
|
@@ -27,12 +48,20 @@ export function loadInBatches(
|
|
|
27
48
|
context?: LoaderContext
|
|
28
49
|
): Promise<AsyncIterable<any>>;
|
|
29
50
|
|
|
30
|
-
export function loadInBatches(
|
|
51
|
+
export function loadInBatches(
|
|
52
|
+
files: FileType | FileType[] | FileList,
|
|
53
|
+
loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
|
|
54
|
+
options?: LoaderOptions,
|
|
55
|
+
context?: LoaderContext
|
|
56
|
+
) {
|
|
57
|
+
let loadersArray: LoaderWithParser | LoaderWithParser[] | undefined;
|
|
31
58
|
// Signature: load(url, options)
|
|
32
59
|
if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
33
60
|
context = undefined; // context not supported in short signature
|
|
34
|
-
options = loaders;
|
|
35
|
-
|
|
61
|
+
options = loaders as LoaderOptions;
|
|
62
|
+
loadersArray = undefined;
|
|
63
|
+
} else {
|
|
64
|
+
loadersArray = loaders as LoaderWithParser | LoaderWithParser[] | undefined;
|
|
36
65
|
}
|
|
37
66
|
|
|
38
67
|
// Select fetch function
|
|
@@ -40,21 +69,30 @@ export function loadInBatches(files, loaders, options, context) {
|
|
|
40
69
|
|
|
41
70
|
// Single url/file
|
|
42
71
|
if (!Array.isArray(files)) {
|
|
43
|
-
return loadOneFileInBatches(files,
|
|
72
|
+
return loadOneFileInBatches(files, loadersArray!, options || {}, fetch);
|
|
44
73
|
}
|
|
45
74
|
|
|
46
75
|
// Multiple URLs / files
|
|
47
|
-
const promises = files.map((file) =>
|
|
76
|
+
const promises = files.map((file) =>
|
|
77
|
+
loadOneFileInBatches(file, loadersArray!, options || {}, fetch)
|
|
78
|
+
);
|
|
48
79
|
|
|
49
80
|
// No point in waiting here for all responses before starting to stream individual streams?
|
|
50
81
|
return promises;
|
|
51
82
|
}
|
|
52
83
|
|
|
53
|
-
async function loadOneFileInBatches(
|
|
84
|
+
async function loadOneFileInBatches(
|
|
85
|
+
file: FileType,
|
|
86
|
+
loaders: LoaderWithParser | LoaderWithParser[],
|
|
87
|
+
options: LoaderOptions,
|
|
88
|
+
fetch: FetchLike
|
|
89
|
+
) {
|
|
54
90
|
if (typeof file === 'string') {
|
|
55
91
|
const url = file;
|
|
56
92
|
const response = await fetch(url);
|
|
93
|
+
// @ts-expect-error
|
|
57
94
|
return await parseInBatches(response, loaders, options);
|
|
58
95
|
}
|
|
96
|
+
// @ts-expect-error TODO
|
|
59
97
|
return await parseInBatches(file, loaders, options);
|
|
60
98
|
}
|
package/src/lib/api/load.ts
CHANGED
|
@@ -25,10 +25,7 @@ export async function load<LoaderT extends Loader>(
|
|
|
25
25
|
context?: LoaderContext
|
|
26
26
|
): Promise<LoaderReturnType<LoaderT>>;
|
|
27
27
|
|
|
28
|
-
export async function load<
|
|
29
|
-
LoaderT extends Loader, // eslint-disable-line @typescript-eslint/no-unused-vars
|
|
30
|
-
LoaderOptionsT extends LoaderOptions = LoaderOptions
|
|
31
|
-
>(
|
|
28
|
+
export async function load<LoaderOptionsT extends LoaderOptions = LoaderOptions>(
|
|
32
29
|
url: string | DataType,
|
|
33
30
|
loaders: Loader[] | LoaderOptions,
|
|
34
31
|
options?: LoaderOptionsT,
|
|
@@ -36,7 +33,7 @@ export async function load<
|
|
|
36
33
|
): Promise<any>;
|
|
37
34
|
|
|
38
35
|
// implementation signature
|
|
39
|
-
export async function load<LoaderOptionsT extends LoaderOptions>(
|
|
36
|
+
export async function load<LoaderOptionsT extends LoaderOptions = LoaderOptions>(
|
|
40
37
|
url: string | DataType,
|
|
41
38
|
loaders?: Loader[] | LoaderOptions,
|
|
42
39
|
options?: LoaderOptionsT,
|
|
@@ -1,12 +1,10 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
|
|
1
3
|
import type {Batch} from '@loaders.gl/schema';
|
|
2
|
-
import type {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
LoaderContext,
|
|
7
|
-
LoaderOptions
|
|
8
|
-
} from '@loaders.gl/loader-utils';
|
|
9
|
-
import {assert, concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
|
|
4
|
+
import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
|
|
5
|
+
import type {LoaderContext, BatchableDataType} from '@loaders.gl/loader-utils';
|
|
6
|
+
import type {LoaderBatchType, LoaderOptionsType} from '@loaders.gl/loader-utils';
|
|
7
|
+
import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
|
|
10
8
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
11
9
|
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
12
10
|
import {getLoaderContext} from '../loader-utils/loader-context';
|
|
@@ -17,6 +15,37 @@ import {selectLoader} from './select-loader';
|
|
|
17
15
|
// Ensure `parse` is available in context if loader falls back to `parse`
|
|
18
16
|
import {parse} from './parse';
|
|
19
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Parses `data` synchronously using a specified loader
|
|
20
|
+
*/
|
|
21
|
+
export async function parseInBatches<
|
|
22
|
+
LoaderT extends Loader,
|
|
23
|
+
OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
|
|
24
|
+
>(
|
|
25
|
+
data: BatchableDataType,
|
|
26
|
+
loader: LoaderT,
|
|
27
|
+
options?: OptionsT,
|
|
28
|
+
context?: LoaderContext
|
|
29
|
+
): Promise<AsyncIterable<LoaderBatchType<LoaderT>>>;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Parses `data` using one of the supplied loaders
|
|
33
|
+
*/
|
|
34
|
+
export async function parseInBatches(
|
|
35
|
+
data: BatchableDataType,
|
|
36
|
+
loaders: Loader[],
|
|
37
|
+
options?: LoaderOptions,
|
|
38
|
+
context?: LoaderContext
|
|
39
|
+
): Promise<AsyncIterable<any>>;
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Parses `data` in batches by selecting a pre-registered loader
|
|
43
|
+
*/
|
|
44
|
+
export async function parseInBatches(
|
|
45
|
+
data: BatchableDataType,
|
|
46
|
+
options?: LoaderOptions
|
|
47
|
+
): Promise<AsyncIterable<any>>;
|
|
48
|
+
|
|
20
49
|
/**
|
|
21
50
|
* Parses `data` using a specified loader
|
|
22
51
|
* @param data
|
|
@@ -29,9 +58,7 @@ export async function parseInBatches(
|
|
|
29
58
|
loaders?: Loader | Loader[] | LoaderOptions,
|
|
30
59
|
options?: LoaderOptions,
|
|
31
60
|
context?: LoaderContext
|
|
32
|
-
): Promise<AsyncIterable<
|
|
33
|
-
assert(!context || typeof context === 'object'); // parseInBatches no longer accepts final url
|
|
34
|
-
|
|
61
|
+
): Promise<AsyncIterable<unknown>> {
|
|
35
62
|
const loaderArray = Array.isArray(loaders) ? loaders : undefined;
|
|
36
63
|
|
|
37
64
|
// Signature: parseInBatches(data, options, url) - Uses registered loaders
|
|
@@ -59,7 +86,7 @@ export async function parseInBatches(
|
|
|
59
86
|
// Normalize options
|
|
60
87
|
options = normalizeOptions(options, loader, loaderArray, url);
|
|
61
88
|
context = getLoaderContext(
|
|
62
|
-
{url, parseInBatches, parse, loaders: loaderArray},
|
|
89
|
+
{url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},
|
|
63
90
|
options,
|
|
64
91
|
context || null
|
|
65
92
|
);
|
|
@@ -75,7 +102,7 @@ async function parseWithLoaderInBatches(
|
|
|
75
102
|
data: BatchableDataType,
|
|
76
103
|
options: LoaderOptions,
|
|
77
104
|
context: LoaderContext
|
|
78
|
-
): Promise<AsyncIterable<
|
|
105
|
+
): Promise<AsyncIterable<unknown>> {
|
|
79
106
|
const outputIterator = await parseToOutputIterator(loader, data, options, context);
|
|
80
107
|
|
|
81
108
|
// Generate metadata batch if requested
|
|
@@ -94,7 +121,9 @@ async function parseWithLoaderInBatches(
|
|
|
94
121
|
bytesUsed: 0
|
|
95
122
|
};
|
|
96
123
|
|
|
97
|
-
async function* makeMetadataBatchIterator(
|
|
124
|
+
async function* makeMetadataBatchIterator(
|
|
125
|
+
iterator: Iterable<unknown> | AsyncIterable<unknown>
|
|
126
|
+
): AsyncIterable<unknown> {
|
|
98
127
|
yield metadataBatch;
|
|
99
128
|
yield* iterator;
|
|
100
129
|
}
|
|
@@ -112,7 +141,7 @@ async function parseToOutputIterator(
|
|
|
112
141
|
data: BatchableDataType,
|
|
113
142
|
options: LoaderOptions,
|
|
114
143
|
context: LoaderContext
|
|
115
|
-
): Promise<AsyncIterable<
|
|
144
|
+
): Promise<AsyncIterable<unknown>> {
|
|
116
145
|
// Get an iterator from the input
|
|
117
146
|
const inputIterator = await getAsyncIterableFromData(data, options);
|
|
118
147
|
|