@loaders.gl/core 4.0.0-alpha.20 → 4.0.0-alpha.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/dist.min.js +19 -21
- package/dist/es5/index.js +7 -0
- package/dist/es5/index.js.map +1 -1
- package/dist/es5/iterators/batch-iterators/timed-batch-iterator.js.map +1 -1
- package/dist/es5/iterators/make-stream/make-dom-stream.js.map +1 -1
- package/dist/es5/javascript-utils/is-type.js +1 -1
- package/dist/es5/javascript-utils/is-type.js.map +1 -1
- package/dist/es5/lib/api/encode-table.js.map +1 -1
- package/dist/es5/lib/api/encode.js.map +1 -1
- package/dist/es5/lib/api/load-in-batches.js +6 -3
- package/dist/es5/lib/api/load-in-batches.js.map +1 -1
- package/dist/es5/lib/api/load.js.map +1 -1
- package/dist/es5/lib/api/parse-in-batches.js.map +1 -1
- package/dist/es5/lib/api/parse-sync.js.map +1 -1
- package/dist/es5/lib/api/parse.js +10 -10
- package/dist/es5/lib/api/parse.js.map +1 -1
- package/dist/es5/lib/filesystems/filesystem.js.map +1 -1
- package/dist/es5/lib/init.js +1 -1
- package/dist/es5/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/es5/null-loader.js +6 -14
- package/dist/es5/null-loader.js.map +1 -1
- package/dist/esm/index.js +1 -0
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/iterators/batch-iterators/timed-batch-iterator.js.map +1 -1
- package/dist/esm/iterators/make-stream/make-dom-stream.js.map +1 -1
- package/dist/esm/javascript-utils/is-type.js +1 -1
- package/dist/esm/javascript-utils/is-type.js.map +1 -1
- package/dist/esm/lib/api/encode-table.js.map +1 -1
- package/dist/esm/lib/api/encode.js.map +1 -1
- package/dist/esm/lib/api/load-in-batches.js +6 -3
- package/dist/esm/lib/api/load-in-batches.js.map +1 -1
- package/dist/esm/lib/api/load.js.map +1 -1
- package/dist/esm/lib/api/parse-in-batches.js.map +1 -1
- package/dist/esm/lib/api/parse-sync.js.map +1 -1
- package/dist/esm/lib/api/parse.js +9 -9
- package/dist/esm/lib/api/parse.js.map +1 -1
- package/dist/esm/lib/filesystems/filesystem.js.map +1 -1
- package/dist/esm/lib/init.js +1 -1
- package/dist/esm/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/esm/null-loader.js +6 -14
- package/dist/esm/null-loader.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/iterators/batch-iterators/timed-batch-iterator.d.ts +1 -1
- package/dist/iterators/batch-iterators/timed-batch-iterator.d.ts.map +1 -1
- package/dist/iterators/make-stream/make-dom-stream.d.ts.map +1 -1
- package/dist/iterators/make-stream/make-dom-stream.js +1 -0
- package/dist/javascript-utils/is-type.d.ts +5 -5
- package/dist/javascript-utils/is-type.d.ts.map +1 -1
- package/dist/javascript-utils/is-type.js +1 -1
- package/dist/lib/api/encode.d.ts +5 -5
- package/dist/lib/api/encode.d.ts.map +1 -1
- package/dist/lib/api/load-in-batches.d.ts +5 -0
- package/dist/lib/api/load-in-batches.d.ts.map +1 -1
- package/dist/lib/api/load-in-batches.js +10 -3
- package/dist/lib/api/load.d.ts +1 -2
- package/dist/lib/api/load.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.d.ts +13 -7
- package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.js +1 -0
- package/dist/lib/api/parse-sync.d.ts +13 -7
- package/dist/lib/api/parse-sync.d.ts.map +1 -1
- package/dist/lib/api/parse-sync.js +0 -4
- package/dist/lib/api/parse.d.ts +11 -1
- package/dist/lib/api/parse.d.ts.map +1 -1
- package/dist/lib/api/parse.js +10 -9
- package/dist/lib/filesystems/filesystem.d.ts +3 -3
- package/dist/lib/filesystems/filesystem.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.d.ts +2 -2
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
- package/dist/null-loader.d.ts +6 -3
- package/dist/null-loader.d.ts.map +1 -1
- package/dist/null-loader.js +9 -14
- package/dist/null-worker-node.js +6 -11
- package/dist/null-worker.js +6 -11
- package/package.json +4 -4
- package/src/index.ts +1 -0
- package/src/iterators/batch-iterators/timed-batch-iterator.ts +4 -1
- package/src/iterators/make-stream/make-dom-stream.ts +2 -1
- package/src/javascript-utils/is-type.ts +12 -8
- package/src/lib/api/encode-table.ts +1 -1
- package/src/lib/api/encode.ts +10 -10
- package/src/lib/api/load-in-batches.ts +45 -7
- package/src/lib/api/load.ts +2 -5
- package/src/lib/api/parse-in-batches.ts +39 -10
- package/src/lib/api/parse-sync.ts +34 -12
- package/src/lib/api/parse.ts +29 -12
- package/src/lib/filesystems/filesystem.ts +3 -3
- package/src/lib/loader-utils/get-fetch-function.ts +2 -2
- package/src/null-loader.ts +23 -17
|
@@ -35,9 +35,9 @@ export interface FileSystem {
|
|
|
35
35
|
* A random access file system
|
|
36
36
|
*/
|
|
37
37
|
export interface RandomAccessReadFileSystem extends FileSystem {
|
|
38
|
-
open(path: string, flags:
|
|
39
|
-
close(fd:
|
|
40
|
-
fstat(fd:
|
|
38
|
+
open(path: string, flags: unknown, mode?: unknown): Promise<any>;
|
|
39
|
+
close(fd: unknown): Promise<void>;
|
|
40
|
+
fstat(fd: unknown): Promise<Stat>;
|
|
41
41
|
read(fd: any, buffer: ArrayBuffer | ArrayBufferView, offset?: number, length?: number, position?: number): Promise<{
|
|
42
42
|
bytesRead: number;
|
|
43
43
|
buffer: ArrayBuffer;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"filesystem.d.ts","sourceRoot":"","sources":["../../../src/lib/filesystems/filesystem.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,WAAW,GAAG,EAAE,CAAC;AAE7B,MAAM,MAAM,IAAI,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,OAAO,CAAC;CAC5B,CAAC;AAEF;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE9E;;;;;OAKG;IACH,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAC,CAAC,CAAC;IAElE;;;;OAIG;IACH,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;CACxE;AAED;;GAEG;AACH,MAAM,WAAW,0BAA2B,SAAQ,UAAU;IAC5D,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"filesystem.d.ts","sourceRoot":"","sources":["../../../src/lib/filesystems/filesystem.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,WAAW,GAAG,EAAE,CAAC;AAE7B,MAAM,MAAM,IAAI,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,OAAO,CAAC;CAC5B,CAAC;AAEF;;;GAGG;AACH,MAAM,WAAW,UAAU;IACzB;;;OAGG;IACH,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE9E;;;;;OAKG;IACH,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAC,CAAC,CAAC;IAElE;;;;OAIG;IACH,KAAK,CAAC,QAAQ,EAAE,WAAW,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;CACxE;AAED;;GAEG;AACH,MAAM,WAAW,0BAA2B,SAAQ,UAAU;IAC5D,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACjE,KAAK,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,KAAK,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,IAAI,CACF,EAAE,EAAE,GAAG,EACP,MAAM,EAAE,WAAW,GAAG,eAAe,EACrC,MAAM,CAAC,EAAE,MAAM,EACf,MAAM,CAAC,EAAE,MAAM,EACf,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,WAAW,CAAA;KAAC,CAAC,CAAC;CACtD"}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import type { LoaderContext, LoaderOptions } from '@loaders.gl/loader-utils';
|
|
1
|
+
import type { LoaderContext, LoaderOptions, FetchLike } from '@loaders.gl/loader-utils';
|
|
2
2
|
/**
|
|
3
3
|
* Gets the current fetch function from options and context
|
|
4
4
|
* @param options
|
|
5
5
|
* @param context
|
|
6
6
|
*/
|
|
7
|
-
export declare function getFetchFunction(options?: LoaderOptions, context?: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>):
|
|
7
|
+
export declare function getFetchFunction(options?: LoaderOptions, context?: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>): FetchLike;
|
|
8
8
|
//# sourceMappingURL=get-fetch-function.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"get-fetch-function.d.ts","sourceRoot":"","sources":["../../../src/lib/loader-utils/get-fetch-function.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,aAAa,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"get-fetch-function.d.ts","sourceRoot":"","sources":["../../../src/lib/loader-utils/get-fetch-function.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,aAAa,EAAE,aAAa,EAAE,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAKtF;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,CAAC,EAAE,aAAa,EACvB,OAAO,CAAC,EAAE,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC,GAC7E,SAAS,CAsBX"}
|
package/dist/null-loader.d.ts
CHANGED
|
@@ -1,10 +1,13 @@
|
|
|
1
|
-
import { Loader, LoaderWithParser } from '@loaders.gl/loader-utils';
|
|
1
|
+
import type { Loader, LoaderWithParser, LoaderOptions } from '@loaders.gl/loader-utils';
|
|
2
|
+
export type NullLoaderOptions = LoaderOptions & {
|
|
3
|
+
null?: {};
|
|
4
|
+
};
|
|
2
5
|
/**
|
|
3
6
|
* Loads any data and returns null (or optionally passes through data unparsed)
|
|
4
7
|
*/
|
|
5
|
-
export declare const NullWorkerLoader: Loader
|
|
8
|
+
export declare const NullWorkerLoader: Loader<null, never, NullLoaderOptions>;
|
|
6
9
|
/**
|
|
7
10
|
* Loads any data and returns null (or optionally passes through data unparsed)
|
|
8
11
|
*/
|
|
9
|
-
export declare const NullLoader: LoaderWithParser
|
|
12
|
+
export declare const NullLoader: LoaderWithParser<null, null, NullLoaderOptions>;
|
|
10
13
|
//# sourceMappingURL=null-loader.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"null-loader.d.ts","sourceRoot":"","sources":["../src/null-loader.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,MAAM,EAAE,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"null-loader.d.ts","sourceRoot":"","sources":["../src/null-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,MAAM,EAAE,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAGtF,MAAM,MAAM,iBAAiB,GAAG,aAAa,GAAG;IAC9C,IAAI,CAAC,EAAE,EAAE,CAAC;CACX,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,iBAAiB,CAYnE,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,UAAU,EAAE,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,iBAAiB,CAmBtE,CAAC"}
|
package/dist/null-loader.js
CHANGED
|
@@ -20,16 +20,6 @@ exports.NullWorkerLoader = {
|
|
|
20
20
|
null: {}
|
|
21
21
|
}
|
|
22
22
|
};
|
|
23
|
-
/**
|
|
24
|
-
* Returns arguments passed to the parse API in a format that can be transfered to a
|
|
25
|
-
* web worker. The `context` parameter is stripped using JSON.stringify & parse.
|
|
26
|
-
*/
|
|
27
|
-
function parseSync(arrayBuffer, options, context) {
|
|
28
|
-
if (!options.null.echoParameters)
|
|
29
|
-
return null;
|
|
30
|
-
context = context && JSON.parse(JSON.stringify(context));
|
|
31
|
-
return { arrayBuffer, options, context };
|
|
32
|
-
}
|
|
33
23
|
/**
|
|
34
24
|
* Loads any data and returns null (or optionally passes through data unparsed)
|
|
35
25
|
*/
|
|
@@ -40,7 +30,7 @@ exports.NullLoader = {
|
|
|
40
30
|
version: VERSION,
|
|
41
31
|
mimeTypes: ['application/x.empty'],
|
|
42
32
|
extensions: ['null'],
|
|
43
|
-
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options, context),
|
|
33
|
+
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
|
|
44
34
|
parseSync,
|
|
45
35
|
parseInBatches: async function* generator(asyncIterator, options, context) {
|
|
46
36
|
for await (const batch of asyncIterator) {
|
|
@@ -49,8 +39,13 @@ exports.NullLoader = {
|
|
|
49
39
|
},
|
|
50
40
|
tests: [() => false],
|
|
51
41
|
options: {
|
|
52
|
-
null: {
|
|
53
|
-
echoParameters: false
|
|
54
|
-
}
|
|
42
|
+
null: {}
|
|
55
43
|
}
|
|
56
44
|
};
|
|
45
|
+
/**
|
|
46
|
+
* Returns arguments passed to the parse API in a format that can be transferred to a
|
|
47
|
+
* web worker. The `context` parameter is stripped using JSON.stringify & parse.
|
|
48
|
+
*/
|
|
49
|
+
function parseSync(arrayBuffer, options, context) {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
package/dist/null-worker-node.js
CHANGED
|
@@ -201,13 +201,7 @@ async function parseData({ loader, arrayBuffer, options, context }) {
|
|
|
201
201
|
}
|
|
202
202
|
|
|
203
203
|
// src/null-loader.ts
|
|
204
|
-
var VERSION = true ? "4.0.0-alpha.
|
|
205
|
-
function parseSync(arrayBuffer, options, context) {
|
|
206
|
-
if (!options.null.echoParameters)
|
|
207
|
-
return null;
|
|
208
|
-
context = context && JSON.parse(JSON.stringify(context));
|
|
209
|
-
return { arrayBuffer, options, context };
|
|
210
|
-
}
|
|
204
|
+
var VERSION = true ? "4.0.0-alpha.21" : "latest";
|
|
211
205
|
var NullLoader = {
|
|
212
206
|
name: "Null loader",
|
|
213
207
|
id: "null",
|
|
@@ -215,7 +209,7 @@ var NullLoader = {
|
|
|
215
209
|
version: VERSION,
|
|
216
210
|
mimeTypes: ["application/x.empty"],
|
|
217
211
|
extensions: ["null"],
|
|
218
|
-
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options, context),
|
|
212
|
+
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
|
|
219
213
|
parseSync,
|
|
220
214
|
parseInBatches: async function* generator(asyncIterator, options, context) {
|
|
221
215
|
for await (const batch of asyncIterator) {
|
|
@@ -224,11 +218,12 @@ var NullLoader = {
|
|
|
224
218
|
},
|
|
225
219
|
tests: [() => false],
|
|
226
220
|
options: {
|
|
227
|
-
null: {
|
|
228
|
-
echoParameters: false
|
|
229
|
-
}
|
|
221
|
+
null: {}
|
|
230
222
|
}
|
|
231
223
|
};
|
|
224
|
+
function parseSync(arrayBuffer, options, context) {
|
|
225
|
+
return null;
|
|
226
|
+
}
|
|
232
227
|
|
|
233
228
|
// src/workers/null-worker.ts
|
|
234
229
|
createLoaderWorker(NullLoader);
|
package/dist/null-worker.js
CHANGED
|
@@ -184,13 +184,7 @@
|
|
|
184
184
|
}
|
|
185
185
|
|
|
186
186
|
// src/null-loader.ts
|
|
187
|
-
var VERSION = true ? "4.0.0-alpha.
|
|
188
|
-
function parseSync(arrayBuffer, options, context) {
|
|
189
|
-
if (!options.null.echoParameters)
|
|
190
|
-
return null;
|
|
191
|
-
context = context && JSON.parse(JSON.stringify(context));
|
|
192
|
-
return { arrayBuffer, options, context };
|
|
193
|
-
}
|
|
187
|
+
var VERSION = true ? "4.0.0-alpha.21" : "latest";
|
|
194
188
|
var NullLoader = {
|
|
195
189
|
name: "Null loader",
|
|
196
190
|
id: "null",
|
|
@@ -198,7 +192,7 @@
|
|
|
198
192
|
version: VERSION,
|
|
199
193
|
mimeTypes: ["application/x.empty"],
|
|
200
194
|
extensions: ["null"],
|
|
201
|
-
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options, context),
|
|
195
|
+
parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options || {}, context),
|
|
202
196
|
parseSync,
|
|
203
197
|
parseInBatches: async function* generator(asyncIterator, options, context) {
|
|
204
198
|
for await (const batch of asyncIterator) {
|
|
@@ -207,11 +201,12 @@
|
|
|
207
201
|
},
|
|
208
202
|
tests: [() => false],
|
|
209
203
|
options: {
|
|
210
|
-
null: {
|
|
211
|
-
echoParameters: false
|
|
212
|
-
}
|
|
204
|
+
null: {}
|
|
213
205
|
}
|
|
214
206
|
};
|
|
207
|
+
function parseSync(arrayBuffer, options, context) {
|
|
208
|
+
return null;
|
|
209
|
+
}
|
|
215
210
|
|
|
216
211
|
// src/workers/null-worker.ts
|
|
217
212
|
createLoaderWorker(NullLoader);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@loaders.gl/core",
|
|
3
|
-
"version": "4.0.0-alpha.
|
|
3
|
+
"version": "4.0.0-alpha.21",
|
|
4
4
|
"description": "The core API for working with loaders.gl loaders and writers",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"publishConfig": {
|
|
@@ -47,9 +47,9 @@
|
|
|
47
47
|
},
|
|
48
48
|
"dependencies": {
|
|
49
49
|
"@babel/runtime": "^7.3.1",
|
|
50
|
-
"@loaders.gl/loader-utils": "4.0.0-alpha.
|
|
51
|
-
"@loaders.gl/worker-utils": "4.0.0-alpha.
|
|
50
|
+
"@loaders.gl/loader-utils": "4.0.0-alpha.21",
|
|
51
|
+
"@loaders.gl/worker-utils": "4.0.0-alpha.21",
|
|
52
52
|
"@probe.gl/log": "^4.0.2"
|
|
53
53
|
},
|
|
54
|
-
"gitHead": "
|
|
54
|
+
"gitHead": "df5d670b136192b26941396e944f9c46be788e83"
|
|
55
55
|
}
|
package/src/index.ts
CHANGED
|
@@ -55,6 +55,7 @@ export {JSONLoader} from '@loaders.gl/loader-utils';
|
|
|
55
55
|
// EXPERIMENTAL
|
|
56
56
|
export {default as _fetchProgress} from './lib/progress/fetch-progress';
|
|
57
57
|
export {default as _BrowserFileSystem} from './lib/filesystems/browser-filesystem';
|
|
58
|
+
export {FileSystem} from './lib/filesystems/filesystem';
|
|
58
59
|
|
|
59
60
|
// FOR TESTING
|
|
60
61
|
export {_unregisterLoaders} from './lib/api/register-loaders';
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* "Debounces" batches and returns them in groups
|
|
3
3
|
*/
|
|
4
|
-
export async function* timedBatchIterator<Batch>(
|
|
4
|
+
export async function* timedBatchIterator<Batch>(
|
|
5
|
+
batchIterator: AsyncIterable<Batch>,
|
|
6
|
+
timeout: number
|
|
7
|
+
): AsyncIterable<Batch[]> {
|
|
5
8
|
let start = Date.now();
|
|
6
9
|
let batches: Batch[] = [];
|
|
7
10
|
for await (const batch of batchIterator) {
|
|
@@ -15,7 +15,8 @@ export function makeStream<ArrayBuffer>(
|
|
|
15
15
|
source: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,
|
|
16
16
|
options?: MakeStreamOptions
|
|
17
17
|
): ReadableStream {
|
|
18
|
-
|
|
18
|
+
// TODO - add AsyncGenerator to parameter types?
|
|
19
|
+
const iterator = (source as AsyncGenerator<ArrayBuffer>)[Symbol.asyncIterator]
|
|
19
20
|
? (source as AsyncIterable<ArrayBuffer>)[Symbol.asyncIterator]()
|
|
20
21
|
: (source as Iterable<ArrayBuffer>)[Symbol.iterator]();
|
|
21
22
|
|
|
@@ -1,17 +1,19 @@
|
|
|
1
|
+
// luma.gl, MIT license
|
|
1
2
|
import type {Readable} from 'stream';
|
|
2
3
|
|
|
3
4
|
/** A DOM or Node readable stream */
|
|
4
5
|
export type ReadableStreamType = ReadableStream | Readable;
|
|
5
6
|
|
|
6
|
-
const isBoolean: (x:
|
|
7
|
-
const isFunction: (x:
|
|
7
|
+
const isBoolean: (x: unknown) => boolean = (x) => typeof x === 'boolean';
|
|
8
|
+
const isFunction: (x: unknown) => boolean = (x) => typeof x === 'function';
|
|
8
9
|
|
|
9
|
-
export const isObject: (x:
|
|
10
|
+
export const isObject: (x: unknown) => boolean = (x) => x !== null && typeof x === 'object';
|
|
10
11
|
export const isPureObject: (x: any) => boolean = (x) =>
|
|
11
12
|
isObject(x) && x.constructor === {}.constructor;
|
|
12
13
|
export const isPromise: (x: any) => boolean = (x) => isObject(x) && isFunction(x.then);
|
|
13
14
|
|
|
14
|
-
export const isIterable: (x: any) => boolean = (x) =>
|
|
15
|
+
export const isIterable: (x: any) => boolean = (x) =>
|
|
16
|
+
Boolean(x) && typeof x[Symbol.iterator] === 'function';
|
|
15
17
|
export const isAsyncIterable: (x: any) => boolean = (x) =>
|
|
16
18
|
x && typeof x[Symbol.asyncIterator] === 'function';
|
|
17
19
|
export const isIterator: (x: any) => boolean = (x) => x && isFunction(x.next);
|
|
@@ -20,8 +22,10 @@ export const isResponse: (x: any) => boolean = (x) =>
|
|
|
20
22
|
(typeof Response !== 'undefined' && x instanceof Response) ||
|
|
21
23
|
(x && x.arrayBuffer && x.text && x.json);
|
|
22
24
|
|
|
23
|
-
export const isFile: (x:
|
|
24
|
-
|
|
25
|
+
export const isFile: (x: unknown) => boolean = (x) =>
|
|
26
|
+
typeof File !== 'undefined' && x instanceof File;
|
|
27
|
+
export const isBlob: (x: unknown) => boolean = (x) =>
|
|
28
|
+
typeof Blob !== 'undefined' && x instanceof Blob;
|
|
25
29
|
|
|
26
30
|
/** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */
|
|
27
31
|
export const isBuffer: (x: any) => boolean = (x) => x && typeof x === 'object' && x.isBuffer;
|
|
@@ -38,7 +42,7 @@ export const isWritableNodeStream: (x: any) => boolean = (x) =>
|
|
|
38
42
|
isObject(x) && isFunction(x.end) && isFunction(x.write) && isBoolean(x.writable);
|
|
39
43
|
export const isReadableNodeStream: (x: any) => boolean = (x) =>
|
|
40
44
|
isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
|
|
41
|
-
export const isReadableStream: (x:
|
|
45
|
+
export const isReadableStream: (x: unknown) => boolean = (x) =>
|
|
42
46
|
isReadableDOMStream(x) || isReadableNodeStream(x);
|
|
43
|
-
export const isWritableStream: (x:
|
|
47
|
+
export const isWritableStream: (x: unknown) => boolean = (x) =>
|
|
44
48
|
isWritableDOMStream(x) || isWritableNodeStream(x);
|
|
@@ -65,7 +65,7 @@ export function encodeTableInBatches<WriterT extends Writer = Writer>(
|
|
|
65
65
|
throw new Error('Writer could not encode data in batches');
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
-
function getIterator(data) {
|
|
68
|
+
function getIterator(data: any): Iterable<{table: any; start: number; end: number}> {
|
|
69
69
|
const dataIterator = [{table: data, start: 0, end: data.length}];
|
|
70
70
|
return dataIterator;
|
|
71
71
|
}
|
package/src/lib/api/encode.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {getLoaderOptions} from './loader-options';
|
|
|
10
10
|
* Encode loaded data into a binary ArrayBuffer using the specified Writer.
|
|
11
11
|
*/
|
|
12
12
|
export async function encode(
|
|
13
|
-
data:
|
|
13
|
+
data: unknown,
|
|
14
14
|
writer: Writer,
|
|
15
15
|
options?: WriterOptions
|
|
16
16
|
): Promise<ArrayBuffer> {
|
|
@@ -40,7 +40,7 @@ export async function encode(
|
|
|
40
40
|
const batches = encodeInBatches(data, writer, options);
|
|
41
41
|
|
|
42
42
|
// Concatenate the output
|
|
43
|
-
const chunks:
|
|
43
|
+
const chunks: unknown[] = [];
|
|
44
44
|
for await (const batch of batches) {
|
|
45
45
|
chunks.push(batch);
|
|
46
46
|
}
|
|
@@ -51,7 +51,7 @@ export async function encode(
|
|
|
51
51
|
if (!isBrowser && writer.encodeURLtoURL) {
|
|
52
52
|
// TODO - how to generate filenames with correct extensions?
|
|
53
53
|
const tmpInputFilename = getTemporaryFilename('input');
|
|
54
|
-
await writeFile(tmpInputFilename, data);
|
|
54
|
+
await writeFile(tmpInputFilename, data as ArrayBuffer);
|
|
55
55
|
|
|
56
56
|
const tmpOutputFilename = getTemporaryFilename('output');
|
|
57
57
|
|
|
@@ -72,7 +72,7 @@ export async function encode(
|
|
|
72
72
|
/**
|
|
73
73
|
* Encode loaded data into a binary ArrayBuffer using the specified Writer.
|
|
74
74
|
*/
|
|
75
|
-
export function encodeSync(data:
|
|
75
|
+
export function encodeSync(data: unknown, writer: Writer, options?: WriterOptions): ArrayBuffer {
|
|
76
76
|
if (writer.encodeSync) {
|
|
77
77
|
return writer.encodeSync(data, options);
|
|
78
78
|
}
|
|
@@ -86,7 +86,7 @@ export function encodeSync(data: any, writer: Writer, options?: WriterOptions):
|
|
|
86
86
|
* @throws if the writer does not generate text output
|
|
87
87
|
*/
|
|
88
88
|
export async function encodeText(
|
|
89
|
-
data:
|
|
89
|
+
data: unknown,
|
|
90
90
|
writer: Writer,
|
|
91
91
|
options?: WriterOptions
|
|
92
92
|
): Promise<string> {
|
|
@@ -106,7 +106,7 @@ export async function encodeText(
|
|
|
106
106
|
* Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
|
|
107
107
|
*/
|
|
108
108
|
export function encodeInBatches(
|
|
109
|
-
data:
|
|
109
|
+
data: unknown,
|
|
110
110
|
writer: Writer,
|
|
111
111
|
options?: WriterOptions
|
|
112
112
|
): AsyncIterable<ArrayBuffer> {
|
|
@@ -124,10 +124,10 @@ export function encodeInBatches(
|
|
|
124
124
|
* @note Node.js only. This function enables using command-line converters as "writers".
|
|
125
125
|
*/
|
|
126
126
|
export async function encodeURLtoURL(
|
|
127
|
-
inputUrl,
|
|
128
|
-
outputUrl,
|
|
127
|
+
inputUrl: string,
|
|
128
|
+
outputUrl: string,
|
|
129
129
|
writer: Writer,
|
|
130
|
-
options
|
|
130
|
+
options?: WriterOptions
|
|
131
131
|
): Promise<string> {
|
|
132
132
|
inputUrl = resolvePath(inputUrl);
|
|
133
133
|
outputUrl = resolvePath(outputUrl);
|
|
@@ -141,7 +141,7 @@ export async function encodeURLtoURL(
|
|
|
141
141
|
/**
|
|
142
142
|
* @todo TODO - this is an unacceptable hack!!!
|
|
143
143
|
*/
|
|
144
|
-
function getIterator(data) {
|
|
144
|
+
function getIterator(data: any): Iterable<{table: any; start: number; end: number}> {
|
|
145
145
|
const dataIterator = [{table: data, start: 0, end: data.length}];
|
|
146
146
|
return dataIterator;
|
|
147
147
|
}
|
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
|
|
3
|
+
import type {
|
|
4
|
+
LoaderWithParser,
|
|
5
|
+
LoaderOptions,
|
|
6
|
+
LoaderContext,
|
|
7
|
+
FetchLike
|
|
8
|
+
} from '@loaders.gl/loader-utils';
|
|
9
|
+
import type {LoaderBatchType, LoaderOptionsType} from '@loaders.gl/loader-utils';
|
|
2
10
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
3
11
|
import {getFetchFunction} from '../loader-utils/get-fetch-function';
|
|
4
12
|
|
|
@@ -6,6 +14,19 @@ import {parseInBatches} from './parse-in-batches';
|
|
|
6
14
|
|
|
7
15
|
type FileType = string | File | Blob | Response | (string | File | Blob | Response)[] | FileList;
|
|
8
16
|
|
|
17
|
+
/**
|
|
18
|
+
* Parses `data` synchronously using a specified loader
|
|
19
|
+
*/
|
|
20
|
+
export async function loadInBatches<
|
|
21
|
+
LoaderT extends LoaderWithParser,
|
|
22
|
+
OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
|
|
23
|
+
>(
|
|
24
|
+
files: FileType,
|
|
25
|
+
loader: LoaderT,
|
|
26
|
+
options?: OptionsT,
|
|
27
|
+
context?: LoaderContext
|
|
28
|
+
): Promise<AsyncIterable<LoaderBatchType<LoaderT>>>;
|
|
29
|
+
|
|
9
30
|
/**
|
|
10
31
|
* Parses `data` using a specified loader
|
|
11
32
|
* @param data
|
|
@@ -27,12 +48,20 @@ export function loadInBatches(
|
|
|
27
48
|
context?: LoaderContext
|
|
28
49
|
): Promise<AsyncIterable<any>>;
|
|
29
50
|
|
|
30
|
-
export function loadInBatches(
|
|
51
|
+
export function loadInBatches(
|
|
52
|
+
files: FileType | FileType[] | FileList,
|
|
53
|
+
loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
|
|
54
|
+
options?: LoaderOptions,
|
|
55
|
+
context?: LoaderContext
|
|
56
|
+
) {
|
|
57
|
+
let loadersArray: LoaderWithParser | LoaderWithParser[] | undefined;
|
|
31
58
|
// Signature: load(url, options)
|
|
32
59
|
if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
33
60
|
context = undefined; // context not supported in short signature
|
|
34
|
-
options = loaders;
|
|
35
|
-
|
|
61
|
+
options = loaders as LoaderOptions;
|
|
62
|
+
loadersArray = undefined;
|
|
63
|
+
} else {
|
|
64
|
+
loadersArray = loaders as LoaderWithParser | LoaderWithParser[] | undefined;
|
|
36
65
|
}
|
|
37
66
|
|
|
38
67
|
// Select fetch function
|
|
@@ -40,21 +69,30 @@ export function loadInBatches(files, loaders, options, context) {
|
|
|
40
69
|
|
|
41
70
|
// Single url/file
|
|
42
71
|
if (!Array.isArray(files)) {
|
|
43
|
-
return loadOneFileInBatches(files,
|
|
72
|
+
return loadOneFileInBatches(files, loadersArray!, options || {}, fetch);
|
|
44
73
|
}
|
|
45
74
|
|
|
46
75
|
// Multiple URLs / files
|
|
47
|
-
const promises = files.map((file) =>
|
|
76
|
+
const promises = files.map((file) =>
|
|
77
|
+
loadOneFileInBatches(file, loadersArray!, options || {}, fetch)
|
|
78
|
+
);
|
|
48
79
|
|
|
49
80
|
// No point in waiting here for all responses before starting to stream individual streams?
|
|
50
81
|
return promises;
|
|
51
82
|
}
|
|
52
83
|
|
|
53
|
-
async function loadOneFileInBatches(
|
|
84
|
+
async function loadOneFileInBatches(
|
|
85
|
+
file: FileType,
|
|
86
|
+
loaders: LoaderWithParser | LoaderWithParser[],
|
|
87
|
+
options: LoaderOptions,
|
|
88
|
+
fetch: FetchLike
|
|
89
|
+
) {
|
|
54
90
|
if (typeof file === 'string') {
|
|
55
91
|
const url = file;
|
|
56
92
|
const response = await fetch(url);
|
|
93
|
+
// @ts-expect-error
|
|
57
94
|
return await parseInBatches(response, loaders, options);
|
|
58
95
|
}
|
|
96
|
+
// @ts-expect-error TODO
|
|
59
97
|
return await parseInBatches(file, loaders, options);
|
|
60
98
|
}
|
package/src/lib/api/load.ts
CHANGED
|
@@ -25,10 +25,7 @@ export async function load<LoaderT extends Loader>(
|
|
|
25
25
|
context?: LoaderContext
|
|
26
26
|
): Promise<LoaderReturnType<LoaderT>>;
|
|
27
27
|
|
|
28
|
-
export async function load<
|
|
29
|
-
LoaderT extends Loader, // eslint-disable-line @typescript-eslint/no-unused-vars
|
|
30
|
-
LoaderOptionsT extends LoaderOptions = LoaderOptions
|
|
31
|
-
>(
|
|
28
|
+
export async function load<LoaderOptionsT extends LoaderOptions = LoaderOptions>(
|
|
32
29
|
url: string | DataType,
|
|
33
30
|
loaders: Loader[] | LoaderOptions,
|
|
34
31
|
options?: LoaderOptionsT,
|
|
@@ -36,7 +33,7 @@ export async function load<
|
|
|
36
33
|
): Promise<any>;
|
|
37
34
|
|
|
38
35
|
// implementation signature
|
|
39
|
-
export async function load<LoaderOptionsT extends LoaderOptions>(
|
|
36
|
+
export async function load<LoaderOptionsT extends LoaderOptions = LoaderOptions>(
|
|
40
37
|
url: string | DataType,
|
|
41
38
|
loaders?: Loader[] | LoaderOptions,
|
|
42
39
|
options?: LoaderOptionsT,
|
|
@@ -1,11 +1,9 @@
|
|
|
1
|
+
// loaders.gl, MIT license
|
|
2
|
+
|
|
1
3
|
import type {Batch} from '@loaders.gl/schema';
|
|
2
|
-
import type {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
LoaderWithParser,
|
|
6
|
-
LoaderContext,
|
|
7
|
-
LoaderOptions
|
|
8
|
-
} from '@loaders.gl/loader-utils';
|
|
4
|
+
import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
|
|
5
|
+
import type {LoaderContext, BatchableDataType} from '@loaders.gl/loader-utils';
|
|
6
|
+
import type {LoaderBatchType, LoaderOptionsType} from '@loaders.gl/loader-utils';
|
|
9
7
|
import {assert, concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
|
|
10
8
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
11
9
|
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
@@ -17,6 +15,37 @@ import {selectLoader} from './select-loader';
|
|
|
17
15
|
// Ensure `parse` is available in context if loader falls back to `parse`
|
|
18
16
|
import {parse} from './parse';
|
|
19
17
|
|
|
18
|
+
/**
|
|
19
|
+
* Parses `data` synchronously using a specified loader
|
|
20
|
+
*/
|
|
21
|
+
export async function parseInBatches<
|
|
22
|
+
LoaderT extends Loader,
|
|
23
|
+
OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
|
|
24
|
+
>(
|
|
25
|
+
data: BatchableDataType,
|
|
26
|
+
loader: LoaderT,
|
|
27
|
+
options?: OptionsT,
|
|
28
|
+
context?: LoaderContext
|
|
29
|
+
): Promise<AsyncIterable<LoaderBatchType<LoaderT>>>;
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Parses `data` using one of the supplied loaders
|
|
33
|
+
*/
|
|
34
|
+
export async function parseInBatches(
|
|
35
|
+
data: BatchableDataType,
|
|
36
|
+
loaders: Loader[],
|
|
37
|
+
options?: LoaderOptions,
|
|
38
|
+
context?: LoaderContext
|
|
39
|
+
): Promise<AsyncIterable<any>>;
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Parses `data` in batches by selecting a pre-registered loader
|
|
43
|
+
*/
|
|
44
|
+
export async function parseInBatches(
|
|
45
|
+
data: BatchableDataType,
|
|
46
|
+
options?: LoaderOptions
|
|
47
|
+
): Promise<AsyncIterable<any>>;
|
|
48
|
+
|
|
20
49
|
/**
|
|
21
50
|
* Parses `data` using a specified loader
|
|
22
51
|
* @param data
|
|
@@ -29,7 +58,7 @@ export async function parseInBatches(
|
|
|
29
58
|
loaders?: Loader | Loader[] | LoaderOptions,
|
|
30
59
|
options?: LoaderOptions,
|
|
31
60
|
context?: LoaderContext
|
|
32
|
-
): Promise<AsyncIterable<
|
|
61
|
+
): Promise<AsyncIterable<unknown>> {
|
|
33
62
|
assert(!context || typeof context === 'object'); // parseInBatches no longer accepts final url
|
|
34
63
|
|
|
35
64
|
const loaderArray = Array.isArray(loaders) ? loaders : undefined;
|
|
@@ -75,7 +104,7 @@ async function parseWithLoaderInBatches(
|
|
|
75
104
|
data: BatchableDataType,
|
|
76
105
|
options: LoaderOptions,
|
|
77
106
|
context: LoaderContext
|
|
78
|
-
): Promise<AsyncIterable<
|
|
107
|
+
): Promise<AsyncIterable<unknown>> {
|
|
79
108
|
const outputIterator = await parseToOutputIterator(loader, data, options, context);
|
|
80
109
|
|
|
81
110
|
// Generate metadata batch if requested
|
|
@@ -112,7 +141,7 @@ async function parseToOutputIterator(
|
|
|
112
141
|
data: BatchableDataType,
|
|
113
142
|
options: LoaderOptions,
|
|
114
143
|
context: LoaderContext
|
|
115
|
-
): Promise<AsyncIterable<
|
|
144
|
+
): Promise<AsyncIterable<unknown>> {
|
|
116
145
|
// Get an iterator from the input
|
|
117
146
|
const inputIterator = await getAsyncIterableFromData(data, options);
|
|
118
147
|
|
|
@@ -1,10 +1,6 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
LoaderWithParser,
|
|
5
|
-
LoaderContext,
|
|
6
|
-
LoaderOptions
|
|
7
|
-
} from '@loaders.gl/loader-utils';
|
|
1
|
+
import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
|
|
2
|
+
import type {LoaderContext, SyncDataType} from '@loaders.gl/loader-utils';
|
|
3
|
+
import type {LoaderOptionsType, LoaderReturnType} from '@loaders.gl/loader-utils';
|
|
8
4
|
import {assert} from '@loaders.gl/loader-utils';
|
|
9
5
|
import {selectLoaderSync} from './select-loader';
|
|
10
6
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
@@ -13,19 +9,45 @@ import {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';
|
|
|
13
9
|
import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
|
|
14
10
|
import {getResourceUrl} from '../utils/resource-utils';
|
|
15
11
|
|
|
12
|
+
// OVERLOADS
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Parses `data` synchronously using the specified loader
|
|
16
|
+
*/
|
|
17
|
+
export function parseSync<
|
|
18
|
+
LoaderT extends Loader,
|
|
19
|
+
OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
|
|
20
|
+
>(
|
|
21
|
+
data: SyncDataType,
|
|
22
|
+
loader: LoaderT,
|
|
23
|
+
options?: OptionsT,
|
|
24
|
+
context?: LoaderContext
|
|
25
|
+
): LoaderReturnType<LoaderT>;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Parses `data` synchronously by matching one of the supplied loaders
|
|
29
|
+
*/
|
|
30
|
+
export function parseSync(
|
|
31
|
+
data: SyncDataType,
|
|
32
|
+
loaders: Loader[],
|
|
33
|
+
options?: LoaderOptions,
|
|
34
|
+
context?: LoaderContext
|
|
35
|
+
): any;
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Parses `data` synchronously by matching a pre=registered loader
|
|
39
|
+
*/
|
|
40
|
+
export function parseSync(data: SyncDataType, options?: LoaderOptions): any;
|
|
41
|
+
|
|
16
42
|
/**
|
|
17
43
|
* Parses `data` synchronously using a specified loader
|
|
18
|
-
* @param data
|
|
19
|
-
* @param loaders
|
|
20
|
-
* @param options
|
|
21
|
-
* @param context
|
|
22
44
|
*/
|
|
23
45
|
export function parseSync(
|
|
24
46
|
data: SyncDataType,
|
|
25
47
|
loaders?: Loader | Loader[] | LoaderOptions,
|
|
26
48
|
options?: LoaderOptions,
|
|
27
49
|
context?: LoaderContext
|
|
28
|
-
):
|
|
50
|
+
): unknown {
|
|
29
51
|
assert(!context || typeof context === 'object'); // parseSync no longer accepts final url
|
|
30
52
|
|
|
31
53
|
// Signature: parseSync(data, options)
|