@loaders.gl/core 4.2.0-alpha.3 → 4.2.0-alpha.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/core-addons/write-file-browser.js +62 -1
  2. package/dist/dist.dev.js +366 -332
  3. package/dist/dist.min.js +15 -0
  4. package/dist/index.cjs +55 -68
  5. package/dist/index.cjs.map +7 -0
  6. package/dist/index.d.ts +20 -20
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +18 -1
  9. package/dist/iterators/batch-iterators/timed-batch-iterator.js +18 -13
  10. package/dist/iterators/make-iterator/make-array-buffer-iterator.d.ts +1 -1
  11. package/dist/iterators/make-iterator/make-array-buffer-iterator.d.ts.map +1 -1
  12. package/dist/iterators/make-iterator/make-array-buffer-iterator.js +21 -15
  13. package/dist/iterators/make-iterator/make-blob-iterator.d.ts +1 -1
  14. package/dist/iterators/make-iterator/make-blob-iterator.d.ts.map +1 -1
  15. package/dist/iterators/make-iterator/make-blob-iterator.js +18 -10
  16. package/dist/iterators/make-iterator/make-iterator.d.ts +1 -1
  17. package/dist/iterators/make-iterator/make-iterator.d.ts.map +1 -1
  18. package/dist/iterators/make-iterator/make-iterator.js +29 -18
  19. package/dist/iterators/make-iterator/make-stream-iterator.js +86 -23
  20. package/dist/iterators/make-iterator/make-string-iterator.d.ts +1 -1
  21. package/dist/iterators/make-iterator/make-string-iterator.d.ts.map +1 -1
  22. package/dist/iterators/make-iterator/make-string-iterator.js +20 -10
  23. package/dist/iterators/make-stream/make-stream.js +47 -29
  24. package/dist/javascript-utils/is-type.js +25 -19
  25. package/dist/lib/api/encode-table.js +40 -35
  26. package/dist/lib/api/encode.js +112 -73
  27. package/dist/lib/api/load-in-batches.js +35 -21
  28. package/dist/lib/api/load.js +35 -20
  29. package/dist/lib/api/loader-options.d.ts +2 -2
  30. package/dist/lib/api/loader-options.d.ts.map +1 -1
  31. package/dist/lib/api/loader-options.js +3 -1
  32. package/dist/lib/api/parse-in-batches.js +105 -69
  33. package/dist/lib/api/parse-sync.js +42 -33
  34. package/dist/lib/api/parse.js +73 -61
  35. package/dist/lib/api/register-loaders.js +23 -14
  36. package/dist/lib/api/select-loader.js +216 -163
  37. package/dist/lib/common.js +3 -1
  38. package/dist/lib/fetch/fetch-error-message.js +17 -12
  39. package/dist/lib/fetch/fetch-file.js +26 -15
  40. package/dist/lib/fetch/read-array-buffer.js +30 -15
  41. package/dist/lib/filesystems/browser-filesystem.js +126 -69
  42. package/dist/lib/filesystems/read-array-buffer.js +14 -6
  43. package/dist/lib/init.js +13 -7
  44. package/dist/lib/loader-utils/check-errors.js +37 -16
  45. package/dist/lib/loader-utils/get-data.js +110 -88
  46. package/dist/lib/loader-utils/get-fetch-function.js +24 -13
  47. package/dist/lib/loader-utils/loader-context.js +50 -31
  48. package/dist/lib/loader-utils/loggers.js +34 -44
  49. package/dist/lib/loader-utils/normalize-loader.js +45 -32
  50. package/dist/lib/loader-utils/option-defaults.js +38 -34
  51. package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
  52. package/dist/lib/loader-utils/option-utils.js +133 -80
  53. package/dist/lib/progress/fetch-progress.js +54 -47
  54. package/dist/lib/utils/log.js +4 -4
  55. package/dist/lib/utils/mime-type-utils.js +34 -11
  56. package/dist/lib/utils/resource-utils.js +77 -45
  57. package/dist/lib/utils/response-utils.js +97 -74
  58. package/dist/lib/utils/url-utils.js +6 -4
  59. package/dist/null-loader.js +44 -30
  60. package/dist/null-worker-node.js +1 -1
  61. package/dist/null-worker.js +1 -1
  62. package/dist/workers/null-worker-node.js +3 -1
  63. package/dist/workers/null-worker.js +3 -1
  64. package/package.json +7 -7
  65. package/src/lib/loader-utils/option-utils.ts +3 -1
  66. package/dist/core-addons/README.md +0 -1
  67. package/dist/core-addons/write-file-browser.js.map +0 -1
  68. package/dist/index.js.map +0 -1
  69. package/dist/iterators/batch-iterators/timed-batch-iterator.js.map +0 -1
  70. package/dist/iterators/make-iterator/make-array-buffer-iterator.js.map +0 -1
  71. package/dist/iterators/make-iterator/make-blob-iterator.js.map +0 -1
  72. package/dist/iterators/make-iterator/make-iterator.js.map +0 -1
  73. package/dist/iterators/make-iterator/make-stream-iterator.js.map +0 -1
  74. package/dist/iterators/make-iterator/make-string-iterator.js.map +0 -1
  75. package/dist/iterators/make-stream/make-stream.js.map +0 -1
  76. package/dist/javascript-utils/is-type.js.map +0 -1
  77. package/dist/lib/api/encode-table.js.map +0 -1
  78. package/dist/lib/api/encode.js.map +0 -1
  79. package/dist/lib/api/load-in-batches.js.map +0 -1
  80. package/dist/lib/api/load.js.map +0 -1
  81. package/dist/lib/api/loader-options.js.map +0 -1
  82. package/dist/lib/api/parse-in-batches.js.map +0 -1
  83. package/dist/lib/api/parse-sync.js.map +0 -1
  84. package/dist/lib/api/parse.js.map +0 -1
  85. package/dist/lib/api/register-loaders.js.map +0 -1
  86. package/dist/lib/api/select-loader.js.map +0 -1
  87. package/dist/lib/common.js.map +0 -1
  88. package/dist/lib/fetch/fetch-error-message.js.map +0 -1
  89. package/dist/lib/fetch/fetch-file.js.map +0 -1
  90. package/dist/lib/fetch/read-array-buffer.js.map +0 -1
  91. package/dist/lib/filesystems/browser-filesystem.js.map +0 -1
  92. package/dist/lib/filesystems/read-array-buffer.js.map +0 -1
  93. package/dist/lib/init.js.map +0 -1
  94. package/dist/lib/loader-utils/check-errors.js.map +0 -1
  95. package/dist/lib/loader-utils/get-data.js.map +0 -1
  96. package/dist/lib/loader-utils/get-fetch-function.js.map +0 -1
  97. package/dist/lib/loader-utils/loader-context.js.map +0 -1
  98. package/dist/lib/loader-utils/loggers.js.map +0 -1
  99. package/dist/lib/loader-utils/normalize-loader.js.map +0 -1
  100. package/dist/lib/loader-utils/option-defaults.js.map +0 -1
  101. package/dist/lib/loader-utils/option-utils.js.map +0 -1
  102. package/dist/lib/progress/fetch-progress.js.map +0 -1
  103. package/dist/lib/utils/log.js.map +0 -1
  104. package/dist/lib/utils/mime-type-utils.js.map +0 -1
  105. package/dist/lib/utils/resource-utils.js.map +0 -1
  106. package/dist/lib/utils/response-utils.js.map +0 -1
  107. package/dist/lib/utils/url-utils.js.map +0 -1
  108. package/dist/null-loader.js.map +0 -1
  109. package/dist/workers/null-worker-node.js.map +0 -1
  110. package/dist/workers/null-worker.js.map +0 -1
@@ -1,25 +1,36 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { makeStringIterator } from "./make-string-iterator.js";
2
5
  import { makeArrayBufferIterator } from "./make-array-buffer-iterator.js";
3
6
  import { makeBlobIterator } from "./make-blob-iterator.js";
4
7
  import { makeStreamIterator } from "./make-stream-iterator.js";
5
8
  import { isBlob, isReadableStream, isResponse } from "../../javascript-utils/is-type.js";
9
+ /**
10
+ * Returns an iterator that breaks its input into chunks and yields them one-by-one.
11
+ * @param data
12
+ * @param options
13
+ * @returns
14
+ * This function can e.g. be used to enable data sources that can only be read atomically
15
+ * (such as `Blob` and `File` via `FileReader`) to still be parsed in batches.
16
+ */
6
17
  export function makeIterator(data, options) {
7
- if (typeof data === 'string') {
8
- return makeStringIterator(data, options);
9
- }
10
- if (data instanceof ArrayBuffer) {
11
- return makeArrayBufferIterator(data, options);
12
- }
13
- if (isBlob(data)) {
14
- return makeBlobIterator(data, options);
15
- }
16
- if (isReadableStream(data)) {
17
- return makeStreamIterator(data, options);
18
- }
19
- if (isResponse(data)) {
20
- const response = data;
21
- return makeStreamIterator(response.body, options);
22
- }
23
- throw new Error('makeIterator');
18
+ if (typeof data === 'string') {
19
+ // Note: Converts string chunks to binary
20
+ return makeStringIterator(data, options);
21
+ }
22
+ if (data instanceof ArrayBuffer) {
23
+ return makeArrayBufferIterator(data, options);
24
+ }
25
+ if (isBlob(data)) {
26
+ return makeBlobIterator(data, options);
27
+ }
28
+ if (isReadableStream(data)) {
29
+ return makeStreamIterator(data, options);
30
+ }
31
+ if (isResponse(data)) {
32
+ const response = data;
33
+ return makeStreamIterator(response.body, options);
34
+ }
35
+ throw new Error('makeIterator');
24
36
  }
25
- //# sourceMappingURL=make-iterator.js.map
@@ -1,32 +1,95 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { isBrowser, toArrayBuffer } from '@loaders.gl/loader-utils';
5
+ /**
6
+ * Returns an async iterable that reads from a stream (works in both Node.js and browsers)
7
+ * @param stream stream to iterator over
8
+ */
2
9
  export function makeStreamIterator(stream, options) {
3
- return isBrowser ? makeBrowserStreamIterator(stream, options) : makeNodeStreamIterator(stream, options);
10
+ return isBrowser
11
+ ? makeBrowserStreamIterator(stream, options)
12
+ : makeNodeStreamIterator(stream, options);
4
13
  }
14
+ /**
15
+ * Returns an async iterable that reads from a DOM (browser) stream
16
+ * @param stream stream to iterate from
17
+ * @see https://jakearchibald.com/2017/async-iterators-and-generators/#making-streams-iterate
18
+ */
5
19
  async function* makeBrowserStreamIterator(stream, options) {
6
- const reader = stream.getReader();
7
- let nextBatchPromise;
8
- try {
9
- while (true) {
10
- const currentBatchPromise = nextBatchPromise || reader.read();
11
- if (options !== null && options !== void 0 && options._streamReadAhead) {
12
- nextBatchPromise = reader.read();
13
- }
14
- const {
15
- done,
16
- value
17
- } = await currentBatchPromise;
18
- if (done) {
19
- return;
20
- }
21
- yield toArrayBuffer(value);
20
+ // WhatWG: stream is supposed to have a `getIterator` method
21
+ // if (typeof stream.getIterator === 'function') {
22
+ // return stream.getIterator();
23
+ // }
24
+ // if (typeof stream[Symbol.asyncIterator] === 'function') {
25
+ // return makeToArrayBufferIterator(stream);
26
+ // }
27
+ // In the browser, we first need to get a lock on the stream
28
+ const reader = stream.getReader();
29
+ let nextBatchPromise;
30
+ try {
31
+ // eslint-disable-next-line no-constant-condition
32
+ while (true) {
33
+ const currentBatchPromise = nextBatchPromise || reader.read();
34
+ // Issue a read for an additional batch, while we await the next batch
35
+ // Idea is to make fetching happen in parallel with processing / parsing
36
+ if (options?._streamReadAhead) {
37
+ nextBatchPromise = reader.read();
38
+ }
39
+ // Read from the stream
40
+ // value is a Uint8Array
41
+ const { done, value } = await currentBatchPromise;
42
+ // Exit if we're done
43
+ if (done) {
44
+ return;
45
+ }
46
+ // Else yield the chunk
47
+ yield toArrayBuffer(value);
48
+ }
49
+ }
50
+ catch (error) {
51
+ // TODO - examples makes it look like this should always be called,
52
+ // but that generates exceptions so only call it if we do not reach the end
53
+ reader.releaseLock();
22
54
  }
23
- } catch (error) {
24
- reader.releaseLock();
25
- }
26
55
  }
56
+ /**
57
+ * Returns an async iterable that reads from a DOM (browser) stream
58
+ * @param stream stream to iterate from
59
+ * @note Requires Node.js >= 10
60
+ */
27
61
  async function* makeNodeStreamIterator(stream, options) {
28
- for await (const chunk of stream) {
29
- yield toArrayBuffer(chunk);
62
+ // Hacky test for node version to ensure we don't call bad polyfills
63
+ // NODE 10+: stream is an asyncIterator
64
+ for await (const chunk of stream) {
65
+ yield toArrayBuffer(chunk); // Coerce each chunk to ArrayBuffer
66
+ }
67
+ }
68
+ /* TODO - remove NODE < 10
69
+ * @see https://github.com/bustle/streaming-iterables, MIT license
70
+ *
71
+ if (typeof stream[Symbol.asyncIterator] === 'function') {
72
+ return;
73
+ }
74
+
75
+ // TODO - check if is this ever used in Node 10+?
76
+ // eslint-disable-next-line no-constant-condition
77
+ while (true) {
78
+ const data = stream.read();
79
+ if (data !== null) {
80
+ yield toArrayBuffer(data);
81
+ // eslint-disable-next-line no-continue
82
+ continue;
83
+ }
84
+ if (stream._readableState?.ended) {
85
+ return;
86
+ }
87
+ await onceReadable(stream);
30
88
  }
89
+
90
+ async function onceReadable(stream: Readable): Promise<any> {
91
+ return new Promise((resolve) => {
92
+ stream.once('readable', resolve);
93
+ });
31
94
  }
32
- //# sourceMappingURL=make-stream-iterator.js.map
95
+ */
@@ -1,4 +1,4 @@
1
- import type { IteratorOptions } from './make-iterator';
1
+ import type { IteratorOptions } from "./make-iterator.js";
2
2
  /**
3
3
  * Returns an iterator that breaks a big string into chunks and yields them one-by-one as ArrayBuffers
4
4
  * @param blob string to iterate over
@@ -1 +1 @@
1
- {"version":3,"file":"make-string-iterator.d.ts","sourceRoot":"","sources":["../../../src/iterators/make-iterator/make-string-iterator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,iBAAiB,CAAC;AAIrD;;;;;GAKG;AACH,wBAAiB,kBAAkB,CACjC,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,eAAe,GACxB,QAAQ,CAAC,WAAW,CAAC,CAcvB"}
1
+ {"version":3,"file":"make-string-iterator.d.ts","sourceRoot":"","sources":["../../../src/iterators/make-iterator/make-string-iterator.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,eAAe,EAAC,2BAAwB;AAIrD;;;;;GAKG;AACH,wBAAiB,kBAAkB,CACjC,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,eAAe,GACxB,QAAQ,CAAC,WAAW,CAAC,CAcvB"}
@@ -1,13 +1,23 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  const DEFAULT_CHUNK_SIZE = 256 * 1024;
5
+ /**
6
+ * Returns an iterator that breaks a big string into chunks and yields them one-by-one as ArrayBuffers
7
+ * @param blob string to iterate over
8
+ * @param options
9
+ * @param options.chunkSize
10
+ */
2
11
  export function* makeStringIterator(string, options) {
3
- const chunkSize = (options === null || options === void 0 ? void 0 : options.chunkSize) || DEFAULT_CHUNK_SIZE;
4
- let offset = 0;
5
- const textEncoder = new TextEncoder();
6
- while (offset < string.length) {
7
- const chunkLength = Math.min(string.length - offset, chunkSize);
8
- const chunk = string.slice(offset, offset + chunkLength);
9
- offset += chunkLength;
10
- yield textEncoder.encode(chunk);
11
- }
12
+ const chunkSize = options?.chunkSize || DEFAULT_CHUNK_SIZE;
13
+ let offset = 0;
14
+ const textEncoder = new TextEncoder();
15
+ while (offset < string.length) {
16
+ // Create a chunk of the right size
17
+ const chunkLength = Math.min(string.length - offset, chunkSize);
18
+ const chunk = string.slice(offset, offset + chunkLength);
19
+ offset += chunkLength;
20
+ // yield an ArrayBuffer chunk
21
+ yield textEncoder.encode(chunk);
22
+ }
12
23
  }
13
- //# sourceMappingURL=make-string-iterator.js.map
@@ -1,32 +1,50 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ /**
5
+ * Builds a DOM stream from an iterator
6
+ * This stream is currently used in browsers only,
7
+ * but note that Web stream support is present in Node from Node 16
8
+ * https://nodejs.org/api/webstreams.html#webstreams_web_streams_api
9
+ */
1
10
  export function makeStream(source, options) {
2
- if (globalThis.loaders.makeNodeStream) {
3
- return globalThis.loaders.makeNodeStream(source, options);
4
- }
5
- const iterator = source[Symbol.asyncIterator] ? source[Symbol.asyncIterator]() : source[Symbol.iterator]();
6
- return new ReadableStream({
7
- type: 'bytes',
8
- async pull(controller) {
9
- try {
10
- const {
11
- done,
12
- value
13
- } = await iterator.next();
14
- if (done) {
15
- controller.close();
16
- } else {
17
- controller.enqueue(new Uint8Array(value));
18
- }
19
- } catch (error) {
20
- controller.error(error);
21
- }
22
- },
23
- async cancel() {
24
- var _iterator$return;
25
- await (iterator === null || iterator === void 0 ? void 0 : (_iterator$return = iterator.return) === null || _iterator$return === void 0 ? void 0 : _iterator$return.call(iterator));
11
+ if (globalThis.loaders.makeNodeStream) {
12
+ return globalThis.loaders.makeNodeStream(source, options);
26
13
  }
27
- }, {
28
- highWaterMark: 2 ** 24,
29
- ...options
30
- });
14
+ // TODO - add AsyncGenerator to parameter types?
15
+ const iterator = source[Symbol.asyncIterator]
16
+ ? source[Symbol.asyncIterator]()
17
+ : source[Symbol.iterator]();
18
+ return new ReadableStream({
19
+ // Create a byte stream (enables `Response(stream).arrayBuffer()`)
20
+ // Only supported on Chrome
21
+ // See: https://developer.mozilla.org/en-US/docs/Web/API/ReadableByteStreamController
22
+ // @ts-ignore
23
+ type: 'bytes',
24
+ async pull(controller) {
25
+ try {
26
+ const { done, value } = await iterator.next();
27
+ if (done) {
28
+ controller.close();
29
+ }
30
+ else {
31
+ // TODO - ignores controller.desiredSize
32
+ // @ts-expect-error Unclear why value is not correctly typed
33
+ controller.enqueue(new Uint8Array(value));
34
+ }
35
+ }
36
+ catch (error) {
37
+ controller.error(error);
38
+ }
39
+ },
40
+ async cancel() {
41
+ await iterator?.return?.();
42
+ }
43
+ },
44
+ // options: QueingStrategy<Uint8Array>
45
+ {
46
+ // This is bytes, not chunks
47
+ highWaterMark: 2 ** 24,
48
+ ...options
49
+ });
31
50
  }
32
- //# sourceMappingURL=make-stream.js.map
@@ -1,19 +1,25 @@
1
- const isBoolean = x => typeof x === 'boolean';
2
- const isFunction = x => typeof x === 'function';
3
- export const isObject = x => x !== null && typeof x === 'object';
4
- export const isPureObject = x => isObject(x) && x.constructor === {}.constructor;
5
- export const isPromise = x => isObject(x) && isFunction(x.then);
6
- export const isIterable = x => Boolean(x) && typeof x[Symbol.iterator] === 'function';
7
- export const isAsyncIterable = x => x && typeof x[Symbol.asyncIterator] === 'function';
8
- export const isIterator = x => x && isFunction(x.next);
9
- export const isResponse = x => typeof Response !== 'undefined' && x instanceof Response || x && x.arrayBuffer && x.text && x.json;
10
- export const isFile = x => typeof File !== 'undefined' && x instanceof File;
11
- export const isBlob = x => typeof Blob !== 'undefined' && x instanceof Blob;
12
- export const isBuffer = x => x && typeof x === 'object' && x.isBuffer;
13
- export const isWritableDOMStream = x => isObject(x) && isFunction(x.abort) && isFunction(x.getWriter);
14
- export const isReadableDOMStream = x => typeof ReadableStream !== 'undefined' && x instanceof ReadableStream || isObject(x) && isFunction(x.tee) && isFunction(x.cancel) && isFunction(x.getReader);
15
- export const isWritableNodeStream = x => isObject(x) && isFunction(x.end) && isFunction(x.write) && isBoolean(x.writable);
16
- export const isReadableNodeStream = x => isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
17
- export const isReadableStream = x => isReadableDOMStream(x) || isReadableNodeStream(x);
18
- export const isWritableStream = x => isWritableDOMStream(x) || isWritableNodeStream(x);
19
- //# sourceMappingURL=is-type.js.map
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ const isBoolean = (x) => typeof x === 'boolean';
5
+ const isFunction = (x) => typeof x === 'function';
6
+ export const isObject = (x) => x !== null && typeof x === 'object';
7
+ export const isPureObject = (x) => isObject(x) && x.constructor === {}.constructor;
8
+ export const isPromise = (x) => isObject(x) && isFunction(x.then);
9
+ export const isIterable = (x) => Boolean(x) && typeof x[Symbol.iterator] === 'function';
10
+ export const isAsyncIterable = (x) => x && typeof x[Symbol.asyncIterator] === 'function';
11
+ export const isIterator = (x) => x && isFunction(x.next);
12
+ export const isResponse = (x) => (typeof Response !== 'undefined' && x instanceof Response) ||
13
+ (x && x.arrayBuffer && x.text && x.json);
14
+ export const isFile = (x) => typeof File !== 'undefined' && x instanceof File;
15
+ export const isBlob = (x) => typeof Blob !== 'undefined' && x instanceof Blob;
16
+ /** Check for Node.js `Buffer` without triggering bundler to include buffer polyfill */
17
+ export const isBuffer = (x) => x && typeof x === 'object' && x.isBuffer;
18
+ export const isWritableDOMStream = (x) => isObject(x) && isFunction(x.abort) && isFunction(x.getWriter);
19
+ export const isReadableDOMStream = (x) => (typeof ReadableStream !== 'undefined' && x instanceof ReadableStream) ||
20
+ (isObject(x) && isFunction(x.tee) && isFunction(x.cancel) && isFunction(x.getReader));
21
+ // Not implemented in Firefox: && isFunction(x.pipeTo)
22
+ export const isWritableNodeStream = (x) => isObject(x) && isFunction(x.end) && isFunction(x.write) && isBoolean(x.writable);
23
+ export const isReadableNodeStream = (x) => isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
24
+ export const isReadableStream = (x) => isReadableDOMStream(x) || isReadableNodeStream(x);
25
+ export const isWritableStream = (x) => isWritableDOMStream(x) || isWritableNodeStream(x);
@@ -1,45 +1,50 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+ // Copyright 2022 Foursquare Labs, Inc
5
+ /* global TextEncoder, TextDecoder */
1
6
  import { concatenateArrayBuffers } from '@loaders.gl/loader-utils';
2
7
  export async function encodeTable(data, writer, options) {
3
- if (writer.encode) {
4
- return await writer.encode(data, options);
5
- }
6
- if (writer.encodeText) {
7
- const text = await writer.encodeText(data, options);
8
- return new TextEncoder().encode(text);
9
- }
10
- if (writer.encodeInBatches) {
11
- const batches = encodeTableInBatches(data, writer, options);
12
- const chunks = [];
13
- for await (const batch of batches) {
14
- chunks.push(batch);
8
+ if (writer.encode) {
9
+ return await writer.encode(data, options);
15
10
  }
16
- return concatenateArrayBuffers(...chunks);
17
- }
18
- throw new Error('Writer could not encode data');
11
+ if (writer.encodeText) {
12
+ const text = await writer.encodeText(data, options);
13
+ return new TextEncoder().encode(text);
14
+ }
15
+ if (writer.encodeInBatches) {
16
+ // Create an iterator representing the data
17
+ // TODO - Assumes this is a table
18
+ const batches = encodeTableInBatches(data, writer, options);
19
+ // Concatenate the output
20
+ const chunks = [];
21
+ for await (const batch of batches) {
22
+ chunks.push(batch);
23
+ }
24
+ return concatenateArrayBuffers(...chunks);
25
+ }
26
+ throw new Error('Writer could not encode data');
19
27
  }
20
28
  export async function encodeTableAsText(data, writer, options) {
21
- if (writer.text && writer.encodeText) {
22
- return await writer.encodeText(data, options);
23
- }
24
- if (writer.text) {
25
- const arrayBuffer = await encodeTable(data, writer, options);
26
- return new TextDecoder().decode(arrayBuffer);
27
- }
28
- throw new Error(`Writer ${writer.name} could not encode data as text`);
29
+ if (writer.text && writer.encodeText) {
30
+ return await writer.encodeText(data, options);
31
+ }
32
+ if (writer.text) {
33
+ const arrayBuffer = await encodeTable(data, writer, options);
34
+ return new TextDecoder().decode(arrayBuffer);
35
+ }
36
+ throw new Error(`Writer ${writer.name} could not encode data as text`);
29
37
  }
30
38
  export function encodeTableInBatches(data, writer, options) {
31
- if (writer.encodeInBatches) {
32
- const dataIterator = getIterator(data);
33
- return writer.encodeInBatches(dataIterator, options);
34
- }
35
- throw new Error('Writer could not encode data in batches');
39
+ if (writer.encodeInBatches) {
40
+ const dataIterator = getIterator(data);
41
+ // @ts-expect-error
42
+ return writer.encodeInBatches(dataIterator, options);
43
+ }
44
+ // TODO -fall back to atomic encode?
45
+ throw new Error('Writer could not encode data in batches');
36
46
  }
37
47
  function getIterator(data) {
38
- const dataIterator = [{
39
- ...data,
40
- start: 0,
41
- end: data.length
42
- }];
43
- return dataIterator;
48
+ const dataIterator = [{ ...data, start: 0, end: data.length }];
49
+ return dataIterator;
44
50
  }
45
- //# sourceMappingURL=encode-table.js.map
@@ -1,98 +1,137 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { canEncodeWithWorker, NodeFile, resolvePath } from '@loaders.gl/loader-utils';
2
5
  import { processOnWorker } from '@loaders.gl/worker-utils';
3
6
  import { isBrowser } from '@loaders.gl/loader-utils';
4
7
  import { fetchFile } from "../fetch/fetch-file.js";
5
8
  import { getLoaderOptions } from "./loader-options.js";
9
+ /**
10
+ * Encode loaded data into a binary ArrayBuffer using the specified Writer.
11
+ */
6
12
  export async function encode(data, writer, options) {
7
- const globalOptions = getLoaderOptions();
8
- options = {
9
- ...globalOptions,
10
- ...options
11
- };
12
- if (writer.encodeURLtoURL) {
13
- return encodeWithCommandLineTool(writer, data, options);
14
- }
15
- if (canEncodeWithWorker(writer, options)) {
16
- return await processOnWorker(writer, data, options);
17
- }
18
- return await writer.encode(data, options);
13
+ const globalOptions = getLoaderOptions();
14
+ // const globalOptions: WriterOptions = {}; // getWriterOptions();
15
+ options = { ...globalOptions, ...options };
16
+ // Handle the special case where we are invoking external command-line tools
17
+ if (writer.encodeURLtoURL) {
18
+ return encodeWithCommandLineTool(writer, data, options);
19
+ }
20
+ // Worker support
21
+ if (canEncodeWithWorker(writer, options)) {
22
+ return await processOnWorker(writer, data, options);
23
+ }
24
+ // TODO Merge default writer options with options argument like it is done in load module.
25
+ return await writer.encode(data, options);
19
26
  }
27
+ /**
28
+ * Encode loaded data into a binary ArrayBuffer using the specified Writer.
29
+ */
20
30
  export function encodeSync(data, writer, options) {
21
- if (writer.encodeSync) {
22
- return writer.encodeSync(data, options);
23
- }
24
- if (writer.encodeTextSync) {
25
- return new TextEncoder().encode(writer.encodeTextSync(data, options));
26
- }
27
- throw new Error(`Writer ${writer.name} could not synchronously encode data`);
31
+ if (writer.encodeSync) {
32
+ return writer.encodeSync(data, options);
33
+ }
34
+ if (writer.encodeTextSync) {
35
+ return new TextEncoder().encode(writer.encodeTextSync(data, options));
36
+ }
37
+ throw new Error(`Writer ${writer.name} could not synchronously encode data`);
28
38
  }
39
+ /**
40
+ * Encode loaded data to text using the specified Writer
41
+ * @note This is a convenience function not intended for production use on large input data.
42
+ * It is not optimized for performance. Data maybe converted from text to binary and back.
43
+ * @throws if the writer does not generate text output
44
+ */
29
45
  export async function encodeText(data, writer, options) {
30
- if (writer.encodeText) {
31
- return await writer.encodeText(data, options);
32
- }
33
- if (writer.encodeTextSync) {
34
- return writer.encodeTextSync(data, options);
35
- }
36
- if (writer.text) {
37
- const arrayBuffer = await writer.encode(data, options);
38
- return new TextDecoder().decode(arrayBuffer);
39
- }
40
- throw new Error(`Writer ${writer.name} could not encode data as text`);
46
+ if (writer.encodeText) {
47
+ return await writer.encodeText(data, options);
48
+ }
49
+ if (writer.encodeTextSync) {
50
+ return writer.encodeTextSync(data, options);
51
+ }
52
+ if (writer.text) {
53
+ const arrayBuffer = await writer.encode(data, options);
54
+ return new TextDecoder().decode(arrayBuffer);
55
+ }
56
+ throw new Error(`Writer ${writer.name} could not encode data as text`);
41
57
  }
58
+ /**
59
+ * Encode loaded data to text using the specified Writer
60
+ * @note This is a convenience function not intended for production use on large input data.
61
+ * It is not optimized for performance. Data maybe converted from text to binary and back.
62
+ * @throws if the writer does not generate text output
63
+ */
42
64
  export function encodeTextSync(data, writer, options) {
43
- if (writer.encodeTextSync) {
44
- return writer.encodeTextSync(data, options);
45
- }
46
- if (writer.text && writer.encodeSync) {
47
- const arrayBuffer = encodeSync(data, writer, options);
48
- return new TextDecoder().decode(arrayBuffer);
49
- }
50
- throw new Error(`Writer ${writer.name} could not encode data as text`);
65
+ if (writer.encodeTextSync) {
66
+ return writer.encodeTextSync(data, options);
67
+ }
68
+ if (writer.text && writer.encodeSync) {
69
+ const arrayBuffer = encodeSync(data, writer, options);
70
+ return new TextDecoder().decode(arrayBuffer);
71
+ }
72
+ throw new Error(`Writer ${writer.name} could not encode data as text`);
51
73
  }
74
+ /**
75
+ * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
76
+ */
52
77
  export function encodeInBatches(data, writer, options) {
53
- if (writer.encodeInBatches) {
54
- const dataIterator = getIterator(data);
55
- return writer.encodeInBatches(dataIterator, options);
56
- }
57
- throw new Error(`Writer ${writer.name} could not encode in batches`);
78
+ if (writer.encodeInBatches) {
79
+ const dataIterator = getIterator(data);
80
+ // @ts-expect-error
81
+ return writer.encodeInBatches(dataIterator, options);
82
+ }
83
+ // TODO -fall back to atomic encode?
84
+ throw new Error(`Writer ${writer.name} could not encode in batches`);
58
85
  }
86
+ /**
87
+ * Encode loaded data into a sequence (iterator) of binary ArrayBuffers using the specified Writer.
88
+ */
59
89
  export function encodeTextInBatches(data, writer, options) {
60
- if (writer.encodeTextInBatches) {
61
- const dataIterator = getIterator(data);
62
- return writer.encodeTextInBatches(dataIterator, options);
63
- }
64
- throw new Error(`Writer ${writer.name} could not encode text in batches`);
90
+ if (writer.encodeTextInBatches) {
91
+ const dataIterator = getIterator(data);
92
+ // @ts-expect-error
93
+ return writer.encodeTextInBatches(dataIterator, options);
94
+ }
95
+ // TODO -fall back to atomic encode?
96
+ throw new Error(`Writer ${writer.name} could not encode text in batches`);
65
97
  }
98
+ /**
99
+ * Encode data stored in a file (on disk) to another file.
100
+ * @note Node.js only. This function enables using command-line converters as "writers".
101
+ */
66
102
  export async function encodeURLtoURL(inputUrl, outputUrl, writer, options) {
67
- inputUrl = resolvePath(inputUrl);
68
- outputUrl = resolvePath(outputUrl);
69
- if (isBrowser || !writer.encodeURLtoURL) {
70
- throw new Error();
71
- }
72
- const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
73
- return outputFilename;
103
+ inputUrl = resolvePath(inputUrl);
104
+ outputUrl = resolvePath(outputUrl);
105
+ if (isBrowser || !writer.encodeURLtoURL) {
106
+ throw new Error();
107
+ }
108
+ const outputFilename = await writer.encodeURLtoURL(inputUrl, outputUrl, options);
109
+ return outputFilename;
74
110
  }
111
+ /** Helper function to encode via external tool (typically command line execution in Node.js) */
75
112
  async function encodeWithCommandLineTool(writer, data, options) {
76
- if (isBrowser) {
77
- throw new Error(`Writer ${writer.name} not supported in browser`);
78
- }
79
- const tmpInputFilename = getTemporaryFilename('input');
80
- const file = new NodeFile(tmpInputFilename, 'w');
81
- await file.write(data);
82
- const tmpOutputFilename = getTemporaryFilename('output');
83
- const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);
84
- const response = await fetchFile(outputFilename);
85
- return response.arrayBuffer();
113
+ if (isBrowser) {
114
+ throw new Error(`Writer ${writer.name} not supported in browser`);
115
+ }
116
+ // TODO - how to generate filenames with correct extensions?
117
+ const tmpInputFilename = getTemporaryFilename('input');
118
+ const file = new NodeFile(tmpInputFilename, 'w');
119
+ await file.write(data);
120
+ const tmpOutputFilename = getTemporaryFilename('output');
121
+ const outputFilename = await encodeURLtoURL(tmpInputFilename, tmpOutputFilename, writer, options);
122
+ const response = await fetchFile(outputFilename);
123
+ return response.arrayBuffer();
86
124
  }
125
+ /**
126
+ * @todo TODO - this is an unacceptable hack!!!
127
+ */
87
128
  function getIterator(data) {
88
- const dataIterator = [{
89
- ...data,
90
- start: 0,
91
- end: data.length
92
- }];
93
- return dataIterator;
129
+ const dataIterator = [{ ...data, start: 0, end: data.length }];
130
+ return dataIterator;
94
131
  }
132
+ /**
133
+ * @todo Move to utils
134
+ */
95
135
  function getTemporaryFilename(filename) {
96
- return `/tmp/${filename}`;
136
+ return `/tmp/${filename}`;
97
137
  }
98
- //# sourceMappingURL=encode.js.map