@loaders.gl/core 4.2.0-alpha.4 → 4.2.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core-addons/write-file-browser.js +62 -1
- package/dist/dist.dev.js +366 -301
- package/dist/dist.min.js +15 -0
- package/dist/index.cjs +60 -71
- package/dist/index.cjs.map +7 -0
- package/dist/index.d.ts +20 -20
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +18 -1
- package/dist/iterators/batch-iterators/timed-batch-iterator.js +18 -13
- package/dist/iterators/make-iterator/make-array-buffer-iterator.d.ts +1 -1
- package/dist/iterators/make-iterator/make-array-buffer-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-array-buffer-iterator.js +21 -15
- package/dist/iterators/make-iterator/make-blob-iterator.d.ts +1 -1
- package/dist/iterators/make-iterator/make-blob-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-blob-iterator.js +18 -10
- package/dist/iterators/make-iterator/make-iterator.d.ts +1 -1
- package/dist/iterators/make-iterator/make-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-iterator.js +29 -18
- package/dist/iterators/make-iterator/make-stream-iterator.js +86 -23
- package/dist/iterators/make-iterator/make-string-iterator.d.ts +1 -1
- package/dist/iterators/make-iterator/make-string-iterator.d.ts.map +1 -1
- package/dist/iterators/make-iterator/make-string-iterator.js +20 -10
- package/dist/iterators/make-stream/make-stream.js +47 -29
- package/dist/javascript-utils/is-type.js +25 -19
- package/dist/lib/api/encode-table.js +40 -35
- package/dist/lib/api/encode.js +112 -73
- package/dist/lib/api/load-in-batches.js +35 -21
- package/dist/lib/api/load.js +35 -20
- package/dist/lib/api/loader-options.d.ts +2 -2
- package/dist/lib/api/loader-options.d.ts.map +1 -1
- package/dist/lib/api/loader-options.js +3 -1
- package/dist/lib/api/parse-in-batches.js +106 -69
- package/dist/lib/api/parse-sync.js +42 -33
- package/dist/lib/api/parse.js +73 -61
- package/dist/lib/api/register-loaders.js +23 -14
- package/dist/lib/api/select-loader.js +216 -163
- package/dist/lib/common.js +3 -1
- package/dist/lib/fetch/fetch-error-message.js +17 -12
- package/dist/lib/fetch/fetch-file.js +26 -15
- package/dist/lib/fetch/read-array-buffer.js +30 -15
- package/dist/lib/filesystems/browser-filesystem.js +127 -69
- package/dist/lib/filesystems/read-array-buffer.js +14 -6
- package/dist/lib/init.js +13 -7
- package/dist/lib/loader-utils/check-errors.js +37 -16
- package/dist/lib/loader-utils/get-data.js +110 -88
- package/dist/lib/loader-utils/get-fetch-function.js +24 -13
- package/dist/lib/loader-utils/loader-context.js +50 -31
- package/dist/lib/loader-utils/loggers.js +35 -44
- package/dist/lib/loader-utils/normalize-loader.js +45 -32
- package/dist/lib/loader-utils/option-defaults.js +38 -34
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +133 -80
- package/dist/lib/progress/fetch-progress.js +54 -47
- package/dist/lib/utils/log.js +4 -4
- package/dist/lib/utils/mime-type-utils.js +34 -11
- package/dist/lib/utils/resource-utils.js +77 -45
- package/dist/lib/utils/response-utils.js +97 -74
- package/dist/lib/utils/url-utils.js +6 -4
- package/dist/null-loader.js +44 -30
- package/dist/null-worker-node.js +4 -2
- package/dist/null-worker.js +4 -2
- package/dist/workers/null-worker-node.js +3 -1
- package/dist/workers/null-worker.js +3 -1
- package/package.json +12 -11
- package/src/lib/api/parse-in-batches.ts +6 -5
- package/src/lib/loader-utils/option-utils.ts +3 -1
- package/src/lib/utils/resource-utils.ts +1 -1
- package/dist/core-addons/README.md +0 -1
- package/dist/core-addons/write-file-browser.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/iterators/batch-iterators/timed-batch-iterator.js.map +0 -1
- package/dist/iterators/make-iterator/make-array-buffer-iterator.js.map +0 -1
- package/dist/iterators/make-iterator/make-blob-iterator.js.map +0 -1
- package/dist/iterators/make-iterator/make-iterator.js.map +0 -1
- package/dist/iterators/make-iterator/make-stream-iterator.js.map +0 -1
- package/dist/iterators/make-iterator/make-string-iterator.js.map +0 -1
- package/dist/iterators/make-stream/make-stream.js.map +0 -1
- package/dist/javascript-utils/is-type.js.map +0 -1
- package/dist/lib/api/encode-table.js.map +0 -1
- package/dist/lib/api/encode.js.map +0 -1
- package/dist/lib/api/load-in-batches.js.map +0 -1
- package/dist/lib/api/load.js.map +0 -1
- package/dist/lib/api/loader-options.js.map +0 -1
- package/dist/lib/api/parse-in-batches.js.map +0 -1
- package/dist/lib/api/parse-sync.js.map +0 -1
- package/dist/lib/api/parse.js.map +0 -1
- package/dist/lib/api/register-loaders.js.map +0 -1
- package/dist/lib/api/select-loader.js.map +0 -1
- package/dist/lib/common.js.map +0 -1
- package/dist/lib/fetch/fetch-error-message.js.map +0 -1
- package/dist/lib/fetch/fetch-file.js.map +0 -1
- package/dist/lib/fetch/read-array-buffer.js.map +0 -1
- package/dist/lib/filesystems/browser-filesystem.js.map +0 -1
- package/dist/lib/filesystems/read-array-buffer.js.map +0 -1
- package/dist/lib/init.js.map +0 -1
- package/dist/lib/loader-utils/check-errors.js.map +0 -1
- package/dist/lib/loader-utils/get-data.js.map +0 -1
- package/dist/lib/loader-utils/get-fetch-function.js.map +0 -1
- package/dist/lib/loader-utils/loader-context.js.map +0 -1
- package/dist/lib/loader-utils/loggers.js.map +0 -1
- package/dist/lib/loader-utils/normalize-loader.js.map +0 -1
- package/dist/lib/loader-utils/option-defaults.js.map +0 -1
- package/dist/lib/loader-utils/option-utils.js.map +0 -1
- package/dist/lib/progress/fetch-progress.js.map +0 -1
- package/dist/lib/utils/log.js.map +0 -1
- package/dist/lib/utils/mime-type-utils.js.map +0 -1
- package/dist/lib/utils/resource-utils.js.map +0 -1
- package/dist/lib/utils/response-utils.js.map +0 -1
- package/dist/lib/utils/url-utils.js.map +0 -1
- package/dist/null-loader.js.map +0 -1
- package/dist/workers/null-worker-node.js.map +0 -1
- package/dist/workers/null-worker.js.map +0 -1
|
@@ -1,25 +1,36 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { makeStringIterator } from "./make-string-iterator.js";
|
|
2
5
|
import { makeArrayBufferIterator } from "./make-array-buffer-iterator.js";
|
|
3
6
|
import { makeBlobIterator } from "./make-blob-iterator.js";
|
|
4
7
|
import { makeStreamIterator } from "./make-stream-iterator.js";
|
|
5
8
|
import { isBlob, isReadableStream, isResponse } from "../../javascript-utils/is-type.js";
|
|
9
|
+
/**
|
|
10
|
+
* Returns an iterator that breaks its input into chunks and yields them one-by-one.
|
|
11
|
+
* @param data
|
|
12
|
+
* @param options
|
|
13
|
+
* @returns
|
|
14
|
+
* This function can e.g. be used to enable data sources that can only be read atomically
|
|
15
|
+
* (such as `Blob` and `File` via `FileReader`) to still be parsed in batches.
|
|
16
|
+
*/
|
|
6
17
|
export function makeIterator(data, options) {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
18
|
+
if (typeof data === 'string') {
|
|
19
|
+
// Note: Converts string chunks to binary
|
|
20
|
+
return makeStringIterator(data, options);
|
|
21
|
+
}
|
|
22
|
+
if (data instanceof ArrayBuffer) {
|
|
23
|
+
return makeArrayBufferIterator(data, options);
|
|
24
|
+
}
|
|
25
|
+
if (isBlob(data)) {
|
|
26
|
+
return makeBlobIterator(data, options);
|
|
27
|
+
}
|
|
28
|
+
if (isReadableStream(data)) {
|
|
29
|
+
return makeStreamIterator(data, options);
|
|
30
|
+
}
|
|
31
|
+
if (isResponse(data)) {
|
|
32
|
+
const response = data;
|
|
33
|
+
return makeStreamIterator(response.body, options);
|
|
34
|
+
}
|
|
35
|
+
throw new Error('makeIterator');
|
|
24
36
|
}
|
|
25
|
-
//# sourceMappingURL=make-iterator.js.map
|
|
@@ -1,32 +1,95 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { isBrowser, toArrayBuffer } from '@loaders.gl/loader-utils';
|
|
5
|
+
/**
|
|
6
|
+
* Returns an async iterable that reads from a stream (works in both Node.js and browsers)
|
|
7
|
+
* @param stream stream to iterator over
|
|
8
|
+
*/
|
|
2
9
|
export function makeStreamIterator(stream, options) {
|
|
3
|
-
|
|
10
|
+
return isBrowser
|
|
11
|
+
? makeBrowserStreamIterator(stream, options)
|
|
12
|
+
: makeNodeStreamIterator(stream, options);
|
|
4
13
|
}
|
|
14
|
+
/**
|
|
15
|
+
* Returns an async iterable that reads from a DOM (browser) stream
|
|
16
|
+
* @param stream stream to iterate from
|
|
17
|
+
* @see https://jakearchibald.com/2017/async-iterators-and-generators/#making-streams-iterate
|
|
18
|
+
*/
|
|
5
19
|
async function* makeBrowserStreamIterator(stream, options) {
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
// WhatWG: stream is supposed to have a `getIterator` method
|
|
21
|
+
// if (typeof stream.getIterator === 'function') {
|
|
22
|
+
// return stream.getIterator();
|
|
23
|
+
// }
|
|
24
|
+
// if (typeof stream[Symbol.asyncIterator] === 'function') {
|
|
25
|
+
// return makeToArrayBufferIterator(stream);
|
|
26
|
+
// }
|
|
27
|
+
// In the browser, we first need to get a lock on the stream
|
|
28
|
+
const reader = stream.getReader();
|
|
29
|
+
let nextBatchPromise;
|
|
30
|
+
try {
|
|
31
|
+
// eslint-disable-next-line no-constant-condition
|
|
32
|
+
while (true) {
|
|
33
|
+
const currentBatchPromise = nextBatchPromise || reader.read();
|
|
34
|
+
// Issue a read for an additional batch, while we await the next batch
|
|
35
|
+
// Idea is to make fetching happen in parallel with processing / parsing
|
|
36
|
+
if (options?._streamReadAhead) {
|
|
37
|
+
nextBatchPromise = reader.read();
|
|
38
|
+
}
|
|
39
|
+
// Read from the stream
|
|
40
|
+
// value is a Uint8Array
|
|
41
|
+
const { done, value } = await currentBatchPromise;
|
|
42
|
+
// Exit if we're done
|
|
43
|
+
if (done) {
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
// Else yield the chunk
|
|
47
|
+
yield toArrayBuffer(value);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
// TODO - examples makes it look like this should always be called,
|
|
52
|
+
// but that generates exceptions so only call it if we do not reach the end
|
|
53
|
+
reader.releaseLock();
|
|
22
54
|
}
|
|
23
|
-
} catch (error) {
|
|
24
|
-
reader.releaseLock();
|
|
25
|
-
}
|
|
26
55
|
}
|
|
56
|
+
/**
|
|
57
|
+
* Returns an async iterable that reads from a DOM (browser) stream
|
|
58
|
+
* @param stream stream to iterate from
|
|
59
|
+
* @note Requires Node.js >= 10
|
|
60
|
+
*/
|
|
27
61
|
async function* makeNodeStreamIterator(stream, options) {
|
|
28
|
-
|
|
29
|
-
|
|
62
|
+
// Hacky test for node version to ensure we don't call bad polyfills
|
|
63
|
+
// NODE 10+: stream is an asyncIterator
|
|
64
|
+
for await (const chunk of stream) {
|
|
65
|
+
yield toArrayBuffer(chunk); // Coerce each chunk to ArrayBuffer
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/* TODO - remove NODE < 10
|
|
69
|
+
* @see https://github.com/bustle/streaming-iterables, MIT license
|
|
70
|
+
*
|
|
71
|
+
if (typeof stream[Symbol.asyncIterator] === 'function') {
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// TODO - check if is this ever used in Node 10+?
|
|
76
|
+
// eslint-disable-next-line no-constant-condition
|
|
77
|
+
while (true) {
|
|
78
|
+
const data = stream.read();
|
|
79
|
+
if (data !== null) {
|
|
80
|
+
yield toArrayBuffer(data);
|
|
81
|
+
// eslint-disable-next-line no-continue
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
if (stream._readableState?.ended) {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
await onceReadable(stream);
|
|
30
88
|
}
|
|
89
|
+
|
|
90
|
+
async function onceReadable(stream: Readable): Promise<any> {
|
|
91
|
+
return new Promise((resolve) => {
|
|
92
|
+
stream.once('readable', resolve);
|
|
93
|
+
});
|
|
31
94
|
}
|
|
32
|
-
|
|
95
|
+
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"make-string-iterator.d.ts","sourceRoot":"","sources":["../../../src/iterators/make-iterator/make-string-iterator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,
|
|
1
|
+
{"version":3,"file":"make-string-iterator.d.ts","sourceRoot":"","sources":["../../../src/iterators/make-iterator/make-string-iterator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,2BAAwB;AAIrD;;;;;GAKG;AACH,wBAAiB,kBAAkB,CACjC,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,eAAe,GACxB,QAAQ,CAAC,WAAW,CAAC,CAcvB"}
|
|
@@ -1,13 +1,23 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
const DEFAULT_CHUNK_SIZE = 256 * 1024;
|
|
5
|
+
/**
|
|
6
|
+
* Returns an iterator that breaks a big string into chunks and yields them one-by-one as ArrayBuffers
|
|
7
|
+
* @param blob string to iterate over
|
|
8
|
+
* @param options
|
|
9
|
+
* @param options.chunkSize
|
|
10
|
+
*/
|
|
2
11
|
export function* makeStringIterator(string, options) {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
+
const chunkSize = options?.chunkSize || DEFAULT_CHUNK_SIZE;
|
|
13
|
+
let offset = 0;
|
|
14
|
+
const textEncoder = new TextEncoder();
|
|
15
|
+
while (offset < string.length) {
|
|
16
|
+
// Create a chunk of the right size
|
|
17
|
+
const chunkLength = Math.min(string.length - offset, chunkSize);
|
|
18
|
+
const chunk = string.slice(offset, offset + chunkLength);
|
|
19
|
+
offset += chunkLength;
|
|
20
|
+
// yield an ArrayBuffer chunk
|
|
21
|
+
yield textEncoder.encode(chunk);
|
|
22
|
+
}
|
|
12
23
|
}
|
|
13
|
-
//# sourceMappingURL=make-string-iterator.js.map
|
|
@@ -1,32 +1,50 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
/**
|
|
5
|
+
* Builds a DOM stream from an iterator
|
|
6
|
+
* This stream is currently used in browsers only,
|
|
7
|
+
* but note that Web stream support is present in Node from Node 16
|
|
8
|
+
* https://nodejs.org/api/webstreams.html#webstreams_web_streams_api
|
|
9
|
+
*/
|
|
1
10
|
export function makeStream(source, options) {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
}
|
|
5
|
-
const iterator = source[Symbol.asyncIterator] ? source[Symbol.asyncIterator]() : source[Symbol.iterator]();
|
|
6
|
-
return new ReadableStream({
|
|
7
|
-
type: 'bytes',
|
|
8
|
-
async pull(controller) {
|
|
9
|
-
try {
|
|
10
|
-
const {
|
|
11
|
-
done,
|
|
12
|
-
value
|
|
13
|
-
} = await iterator.next();
|
|
14
|
-
if (done) {
|
|
15
|
-
controller.close();
|
|
16
|
-
} else {
|
|
17
|
-
controller.enqueue(new Uint8Array(value));
|
|
18
|
-
}
|
|
19
|
-
} catch (error) {
|
|
20
|
-
controller.error(error);
|
|
21
|
-
}
|
|
22
|
-
},
|
|
23
|
-
async cancel() {
|
|
24
|
-
var _iterator$return;
|
|
25
|
-
await (iterator === null || iterator === void 0 ? void 0 : (_iterator$return = iterator.return) === null || _iterator$return === void 0 ? void 0 : _iterator$return.call(iterator));
|
|
11
|
+
if (globalThis.loaders.makeNodeStream) {
|
|
12
|
+
return globalThis.loaders.makeNodeStream(source, options);
|
|
26
13
|
}
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
14
|
+
// TODO - add AsyncGenerator to parameter types?
|
|
15
|
+
const iterator = source[Symbol.asyncIterator]
|
|
16
|
+
? source[Symbol.asyncIterator]()
|
|
17
|
+
: source[Symbol.iterator]();
|
|
18
|
+
return new ReadableStream({
|
|
19
|
+
// Create a byte stream (enables `Response(stream).arrayBuffer()`)
|
|
20
|
+
// Only supported on Chrome
|
|
21
|
+
// See: https://developer.mozilla.org/en-US/docs/Web/API/ReadableByteStreamController
|
|
22
|
+
// @ts-ignore
|
|
23
|
+
type: 'bytes',
|
|
24
|
+
async pull(controller) {
|
|
25
|
+
try {
|
|
26
|
+
const { done, value } = await iterator.next();
|
|
27
|
+
if (done) {
|
|
28
|
+
controller.close();
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
// TODO - ignores controller.desiredSize
|
|
32
|
+
// @ts-expect-error Unclear why value is not correctly typed
|
|
33
|
+
controller.enqueue(new Uint8Array(value));
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
controller.error(error);
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
async cancel() {
|
|
41
|
+
await iterator?.return?.();
|
|
42
|
+
}
|
|
43
|
+
},
|
|
44
|
+
// options: QueingStrategy<Uint8Array>
|
|
45
|
+
{
|
|
46
|
+
// This is bytes, not chunks
|
|
47
|
+
highWaterMark: 2 ** 24,
|
|
48
|
+
...options
|
|
49
|
+
});
|
|
31
50
|
}
|
|
32
|
-
//# sourceMappingURL=make-stream.js.map
|
|
@@ -1,19 +1,25 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
export const
|
|
7
|
-
export const
|
|
8
|
-
export const
|
|
9
|
-
export const
|
|
10
|
-
export const
|
|
11
|
-
export const
|
|
12
|
-
export const
|
|
13
|
-
|
|
14
|
-
export const
|
|
15
|
-
export const
|
|
16
|
-
|
|
17
|
-
export const
|
|
18
|
-
export const
|
|
19
|
-
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
const isBoolean = (x) => typeof x === 'boolean';
|
|
5
|
+
const isFunction = (x) => typeof x === 'function';
|
|
6
|
+
export const isObject = (x) => x !== null && typeof x === 'object';
|
|
7
|
+
export const isPureObject = (x) => isObject(x) && x.constructor === {}.constructor;
|
|
8
|
+
export const isPromise = (x) => isObject(x) && isFunction(x.then);
|
|
9
|
+
export const isIterable = (x) => Boolean(x) && typeof x[Symbol.iterator] === 'function';
|
|
10
|
+
export const isAsyncIterable = (x) => x && typeof x[Symbol.asyncIterator] === 'function';
|
|
11
|
+
export const isIterator = (x) => x && isFunction(x.next);
|
|
12
|
+
export const isResponse = (x) => (typeof Response !== 'undefined' && x instanceof Response) ||
|
|
13
|
+
(x && x.arrayBuffer && x.text && x.json);
|
|
14
|
+
export const isFile = (x) => typeof File !== 'undefined' && x instanceof File;
|
|
15
|
+
export const isBlob = (x) => typeof Blob !== 'undefined' && x instanceof Blob;
|
|
16
|
+
/** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */
|
|
17
|
+
export const isBuffer = (x) => x && typeof x === 'object' && x.isBuffer;
|
|
18
|
+
export const isWritableDOMStream = (x) => isObject(x) && isFunction(x.abort) && isFunction(x.getWriter);
|
|
19
|
+
export const isReadableDOMStream = (x) => (typeof ReadableStream !== 'undefined' && x instanceof ReadableStream) ||
|
|
20
|
+
(isObject(x) && isFunction(x.tee) && isFunction(x.cancel) && isFunction(x.getReader));
|
|
21
|
+
// Not implemented in Firefox: && isFunction(x.pipeTo)
|
|
22
|
+
export const isWritableNodeStream = (x) => isObject(x) && isFunction(x.end) && isFunction(x.write) && isBoolean(x.writable);
|
|
23
|
+
export const isReadableNodeStream = (x) => isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
|
|
24
|
+
export const isReadableStream = (x) => isReadableDOMStream(x) || isReadableNodeStream(x);
|
|
25
|
+
export const isWritableStream = (x) => isWritableDOMStream(x) || isWritableNodeStream(x);
|
|
@@ -1,45 +1,50 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
// Copyright 2022 Foursquare Labs, Inc
|
|
5
|
+
/* global TextEncoder, TextDecoder */
|
|
1
6
|
import { concatenateArrayBuffers } from '@loaders.gl/loader-utils';
|
|
2
7
|
export async function encodeTable(data, writer, options) {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
}
|
|
6
|
-
if (writer.encodeText) {
|
|
7
|
-
const text = await writer.encodeText(data, options);
|
|
8
|
-
return new TextEncoder().encode(text);
|
|
9
|
-
}
|
|
10
|
-
if (writer.encodeInBatches) {
|
|
11
|
-
const batches = encodeTableInBatches(data, writer, options);
|
|
12
|
-
const chunks = [];
|
|
13
|
-
for await (const batch of batches) {
|
|
14
|
-
chunks.push(batch);
|
|
8
|
+
if (writer.encode) {
|
|
9
|
+
return await writer.encode(data, options);
|
|
15
10
|
}
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
11
|
+
if (writer.encodeText) {
|
|
12
|
+
const text = await writer.encodeText(data, options);
|
|
13
|
+
return new TextEncoder().encode(text);
|
|
14
|
+
}
|
|
15
|
+
if (writer.encodeInBatches) {
|
|
16
|
+
// Create an iterator representing the data
|
|
17
|
+
// TODO - Assumes this is a table
|
|
18
|
+
const batches = encodeTableInBatches(data, writer, options);
|
|
19
|
+
// Concatenate the output
|
|
20
|
+
const chunks = [];
|
|
21
|
+
for await (const batch of batches) {
|
|
22
|
+
chunks.push(batch);
|
|
23
|
+
}
|
|
24
|
+
return concatenateArrayBuffers(...chunks);
|
|
25
|
+
}
|
|
26
|
+
throw new Error('Writer could not encode data');
|
|
19
27
|
}
|
|
20
28
|
export async function encodeTableAsText(data, writer, options) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
+
if (writer.text && writer.encodeText) {
|
|
30
|
+
return await writer.encodeText(data, options);
|
|
31
|
+
}
|
|
32
|
+
if (writer.text) {
|
|
33
|
+
const arrayBuffer = await encodeTable(data, writer, options);
|
|
34
|
+
return new TextDecoder().decode(arrayBuffer);
|
|
35
|
+
}
|
|
36
|
+
throw new Error(`Writer ${writer.name} could not encode data as text`);
|
|
29
37
|
}
|
|
30
38
|
export function encodeTableInBatches(data, writer, options) {
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
39
|
+
if (writer.encodeInBatches) {
|
|
40
|
+
const dataIterator = getIterator(data);
|
|
41
|
+
// @ts-expect-error
|
|
42
|
+
return writer.encodeInBatches(dataIterator, options);
|
|
43
|
+
}
|
|
44
|
+
// TODO -fall back to atomic encode?
|
|
45
|
+
throw new Error('Writer could not encode data in batches');
|
|
36
46
|
}
|
|
37
47
|
function getIterator(data) {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
start: 0,
|
|
41
|
-
end: data.length
|
|
42
|
-
}];
|
|
43
|
-
return dataIterator;
|
|
48
|
+
const dataIterator = [{ ...data, start: 0, end: data.length }];
|
|
49
|
+
return dataIterator;
|
|
44
50
|
}
|
|
45
|
-
//# sourceMappingURL=encode-table.js.map
|
package/dist/lib/api/encode.js
CHANGED
|
@@ -1,98 +1,137 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
1
4
|
import { canEncodeWithWorker, NodeFile, resolvePath } from '@loaders.gl/loader-utils';
|
|
2
5
|
import { processOnWorker } from '@loaders.gl/worker-utils';
|
|
3
6
|
import { isBrowser } from '@loaders.gl/loader-utils';
|
|
4
7
|
import { fetchFile } from "../fetch/fetch-file.js";
|
|
5
8
|
import { getLoaderOptions } from "./loader-options.js";
|
|
9
|
+
/**
|
|
10
|
+
* Encode loaded data into a binary ArrayBuffer using the specified Writer.
|
|
11
|
+
*/
|
|
6
12
|
export async function encode(data, writer, options) {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
...globalOptions,
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
13
|
+
const globalOptions = getLoaderOptions();
|
|
14
|
+
// const globalOptions: WriterOptions = {}; // getWriterOptions();
|
|
15
|
+
options = { ...globalOptions, ...options };
|
|
16
|
+
// Handle the special case where we are invoking external command-line tools
|
|
17
|
+
if (writer.encodeURLtoURL) {
|
|
18
|
+
return encodeWithCommandLineTool(writer, data, options);
|
|
19
|
+
}
|
|
20
|
+
// Worker support
|
|
21
|
+
if (canEncodeWithWorker(writer, options)) {
|
|
22
|
+
return await processOnWorker(writer, data, options);
|
|
23
|
+
}
|
|
24
|
+
// TODO Merge default writer options with options argument like it is done in load module.
|
|
25
|
+
return await writer.encode(data, options);
|
|
19
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* Encode loaded data into a binary ArrayBuffer using the specified Writer.
|
|
29
|
+
*/
|
|
20
30
|
export function encodeSync(data, writer, options) {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
31
|
+
if (writer.encodeSync) {
|
|
32
|
+
return writer.encodeSync(data, options);
|
|
33
|
+
}
|
|
34
|
+
if (writer.encodeTextSync) {
|
|
35
|
+
return new TextEncoder().encode(writer.encodeTextSync(data, options));
|
|
36
|
+
}
|
|
37
|
+
throw new Error(`Writer ${writer.name} could not synchronously encode data`);
|
|
28
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* Encode loaded data to text using the specified Writer
|
|
41
|
+
* @note This is a convenience function not intended for production use on large input data.
|
|
42
|
+
* It is not optimized for performance. Data maybe converted from text to binary and back.
|
|
43
|
+
* @throws if the writer does not generate text output
|
|
44
|
+
*/
|
|
29
45
|
export async function encodeText(data, writer, options) {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
46
|
+
if (writer.encodeText) {
|
|
47
|
+
return await writer.encodeText(data, options);
|
|
48
|
+
}
|
|
49
|
+
if (writer.encodeTextSync) {
|
|
50
|
+
return writer.encodeTextSync(data, options);
|
|
51
|
+
}
|
|
52
|
+
if (writer.text) {
|
|
53
|
+
const arrayBuffer = await writer.encode(data, options);
|
|
54
|
+
return new TextDecoder().decode(arrayBuffer);
|
|
55
|
+
}
|
|
56
|
+
throw new Error(`Writer ${writer.name} could not encode data as text`);
|
|
41
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* Encode loaded data to text using the specified Writer
|
|
60
|
+
* @note This is a convenience function not intended for production use on large input data.
|
|
61
|
+
* It is not optimized for performance. Data maybe converted from text to binary and back.
|
|
62
|
+
* @throws if the writer does not generate text output
|
|
63
|
+
*/
|
|
42
64
|
export function encodeTextSync(data, writer, options) {
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
65
|
+
if (writer.encodeTextSync) {
|
|
66
|
+
return writer.encodeTextSync(data, options);
|
|
67
|
+
}
|
|
68
|
+
if (writer.text && writer.encodeSync) {
|
|
69
|
+
const arrayBuffer = encodeSync(data, writer, options);
|
|
70
|
+
return new TextDecoder().decode(arrayBuffer);
|
|
71
|
+
}
|
|
72
|
+
throw new Error(`Writer ${writer.name} could not encode data as text`);
|
|
51
73
|
}
|
|
74
|
+
/**
|
|
75
|
+
* Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
|
|
76
|
+
*/
|
|
52
77
|
export function encodeInBatches(data, writer, options) {
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
78
|
+
if (writer.encodeInBatches) {
|
|
79
|
+
const dataIterator = getIterator(data);
|
|
80
|
+
// @ts-expect-error
|
|
81
|
+
return writer.encodeInBatches(dataIterator, options);
|
|
82
|
+
}
|
|
83
|
+
// TODO -fall back to atomic encode?
|
|
84
|
+
throw new Error(`Writer ${writer.name} could not encode in batches`);
|
|
58
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
|
|
88
|
+
*/
|
|
59
89
|
export function encodeTextInBatches(data, writer, options) {
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
90
|
+
if (writer.encodeTextInBatches) {
|
|
91
|
+
const dataIterator = getIterator(data);
|
|
92
|
+
// @ts-expect-error
|
|
93
|
+
return writer.encodeTextInBatches(dataIterator, options);
|
|
94
|
+
}
|
|
95
|
+
// TODO -fall back to atomic encode?
|
|
96
|
+
throw new Error(`Writer ${writer.name} could not encode text in batches`);
|
|
65
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* Encode data stored in a file (on disk) to another file.
|
|
100
|
+
* @note Node.js only. This function enables using command-line converters as "writers".
|
|
101
|
+
*/
|
|
66
102
|
export async function encodeURLtoURL(inputUrl, outputUrl, writer, options) {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
103
|
+
inputUrl = resolvePath(inputUrl);
|
|
104
|
+
outputUrl = resolvePath(outputUrl);
|
|
105
|
+
if (isBrowser || !writer.encodeURLtoURL) {
|
|
106
|
+
throw new Error();
|
|
107
|
+
}
|
|
108
|
+
const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
|
|
109
|
+
return outputFilename;
|
|
74
110
|
}
|
|
111
|
+
/** Helper function to encode via external tool (typically command line execution in Node.js) */
|
|
75
112
|
async function encodeWithCommandLineTool(writer, data, options) {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
113
|
+
if (isBrowser) {
|
|
114
|
+
throw new Error(`Writer ${writer.name} not supported in browser`);
|
|
115
|
+
}
|
|
116
|
+
// TODO - how to generate filenames with correct extensions?
|
|
117
|
+
const tmpInputFilename = getTemporaryFilename('input');
|
|
118
|
+
const file = new NodeFile(tmpInputFilename, 'w');
|
|
119
|
+
await file.write(data);
|
|
120
|
+
const tmpOutputFilename = getTemporaryFilename('output');
|
|
121
|
+
const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);
|
|
122
|
+
const response = await fetchFile(outputFilename);
|
|
123
|
+
return response.arrayBuffer();
|
|
86
124
|
}
|
|
125
|
+
/**
|
|
126
|
+
* @todo TODO - this is an unacceptable hack!!!
|
|
127
|
+
*/
|
|
87
128
|
function getIterator(data) {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
start: 0,
|
|
91
|
-
end: data.length
|
|
92
|
-
}];
|
|
93
|
-
return dataIterator;
|
|
129
|
+
const dataIterator = [{ ...data, start: 0, end: data.length }];
|
|
130
|
+
return dataIterator;
|
|
94
131
|
}
|
|
132
|
+
/**
|
|
133
|
+
* @todo Move to utils
|
|
134
|
+
*/
|
|
95
135
|
function getTemporaryFilename(filename) {
|
|
96
|
-
|
|
136
|
+
return `/tmp/${filename}`;
|
|
97
137
|
}
|
|
98
|
-
//# sourceMappingURL=encode.js.map
|