@loaders.gl/core 4.2.0-alpha.3 → 4.2.0-alpha.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/core-addons/write-file-browser.js +62 -1
  2. package/dist/dist.dev.js +366 -332
  3. package/dist/dist.min.js +15 -0
  4. package/dist/index.cjs +55 -68
  5. package/dist/index.cjs.map +7 -0
  6. package/dist/index.d.ts +20 -20
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +18 -1
  9. package/dist/iterators/batch-iterators/timed-batch-iterator.js +18 -13
  10. package/dist/iterators/make-iterator/make-array-buffer-iterator.d.ts +1 -1
  11. package/dist/iterators/make-iterator/make-array-buffer-iterator.d.ts.map +1 -1
  12. package/dist/iterators/make-iterator/make-array-buffer-iterator.js +21 -15
  13. package/dist/iterators/make-iterator/make-blob-iterator.d.ts +1 -1
  14. package/dist/iterators/make-iterator/make-blob-iterator.d.ts.map +1 -1
  15. package/dist/iterators/make-iterator/make-blob-iterator.js +18 -10
  16. package/dist/iterators/make-iterator/make-iterator.d.ts +1 -1
  17. package/dist/iterators/make-iterator/make-iterator.d.ts.map +1 -1
  18. package/dist/iterators/make-iterator/make-iterator.js +29 -18
  19. package/dist/iterators/make-iterator/make-stream-iterator.js +86 -23
  20. package/dist/iterators/make-iterator/make-string-iterator.d.ts +1 -1
  21. package/dist/iterators/make-iterator/make-string-iterator.d.ts.map +1 -1
  22. package/dist/iterators/make-iterator/make-string-iterator.js +20 -10
  23. package/dist/iterators/make-stream/make-stream.js +47 -29
  24. package/dist/javascript-utils/is-type.js +25 -19
  25. package/dist/lib/api/encode-table.js +40 -35
  26. package/dist/lib/api/encode.js +112 -73
  27. package/dist/lib/api/load-in-batches.js +35 -21
  28. package/dist/lib/api/load.js +35 -20
  29. package/dist/lib/api/loader-options.d.ts +2 -2
  30. package/dist/lib/api/loader-options.d.ts.map +1 -1
  31. package/dist/lib/api/loader-options.js +3 -1
  32. package/dist/lib/api/parse-in-batches.js +105 -69
  33. package/dist/lib/api/parse-sync.js +42 -33
  34. package/dist/lib/api/parse.js +73 -61
  35. package/dist/lib/api/register-loaders.js +23 -14
  36. package/dist/lib/api/select-loader.js +216 -163
  37. package/dist/lib/common.js +3 -1
  38. package/dist/lib/fetch/fetch-error-message.js +17 -12
  39. package/dist/lib/fetch/fetch-file.js +26 -15
  40. package/dist/lib/fetch/read-array-buffer.js +30 -15
  41. package/dist/lib/filesystems/browser-filesystem.js +126 -69
  42. package/dist/lib/filesystems/read-array-buffer.js +14 -6
  43. package/dist/lib/init.js +13 -7
  44. package/dist/lib/loader-utils/check-errors.js +37 -16
  45. package/dist/lib/loader-utils/get-data.js +110 -88
  46. package/dist/lib/loader-utils/get-fetch-function.js +24 -13
  47. package/dist/lib/loader-utils/loader-context.js +50 -31
  48. package/dist/lib/loader-utils/loggers.js +34 -44
  49. package/dist/lib/loader-utils/normalize-loader.js +45 -32
  50. package/dist/lib/loader-utils/option-defaults.js +38 -34
  51. package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
  52. package/dist/lib/loader-utils/option-utils.js +133 -80
  53. package/dist/lib/progress/fetch-progress.js +54 -47
  54. package/dist/lib/utils/log.js +4 -4
  55. package/dist/lib/utils/mime-type-utils.js +34 -11
  56. package/dist/lib/utils/resource-utils.js +77 -45
  57. package/dist/lib/utils/response-utils.js +97 -74
  58. package/dist/lib/utils/url-utils.js +6 -4
  59. package/dist/null-loader.js +44 -30
  60. package/dist/null-worker-node.js +1 -1
  61. package/dist/null-worker.js +1 -1
  62. package/dist/workers/null-worker-node.js +3 -1
  63. package/dist/workers/null-worker.js +3 -1
  64. package/package.json +7 -7
  65. package/src/lib/loader-utils/option-utils.ts +3 -1
  66. package/dist/core-addons/README.md +0 -1
  67. package/dist/core-addons/write-file-browser.js.map +0 -1
  68. package/dist/index.js.map +0 -1
  69. package/dist/iterators/batch-iterators/timed-batch-iterator.js.map +0 -1
  70. package/dist/iterators/make-iterator/make-array-buffer-iterator.js.map +0 -1
  71. package/dist/iterators/make-iterator/make-blob-iterator.js.map +0 -1
  72. package/dist/iterators/make-iterator/make-iterator.js.map +0 -1
  73. package/dist/iterators/make-iterator/make-stream-iterator.js.map +0 -1
  74. package/dist/iterators/make-iterator/make-string-iterator.js.map +0 -1
  75. package/dist/iterators/make-stream/make-stream.js.map +0 -1
  76. package/dist/javascript-utils/is-type.js.map +0 -1
  77. package/dist/lib/api/encode-table.js.map +0 -1
  78. package/dist/lib/api/encode.js.map +0 -1
  79. package/dist/lib/api/load-in-batches.js.map +0 -1
  80. package/dist/lib/api/load.js.map +0 -1
  81. package/dist/lib/api/loader-options.js.map +0 -1
  82. package/dist/lib/api/parse-in-batches.js.map +0 -1
  83. package/dist/lib/api/parse-sync.js.map +0 -1
  84. package/dist/lib/api/parse.js.map +0 -1
  85. package/dist/lib/api/register-loaders.js.map +0 -1
  86. package/dist/lib/api/select-loader.js.map +0 -1
  87. package/dist/lib/common.js.map +0 -1
  88. package/dist/lib/fetch/fetch-error-message.js.map +0 -1
  89. package/dist/lib/fetch/fetch-file.js.map +0 -1
  90. package/dist/lib/fetch/read-array-buffer.js.map +0 -1
  91. package/dist/lib/filesystems/browser-filesystem.js.map +0 -1
  92. package/dist/lib/filesystems/read-array-buffer.js.map +0 -1
  93. package/dist/lib/init.js.map +0 -1
  94. package/dist/lib/loader-utils/check-errors.js.map +0 -1
  95. package/dist/lib/loader-utils/get-data.js.map +0 -1
  96. package/dist/lib/loader-utils/get-fetch-function.js.map +0 -1
  97. package/dist/lib/loader-utils/loader-context.js.map +0 -1
  98. package/dist/lib/loader-utils/loggers.js.map +0 -1
  99. package/dist/lib/loader-utils/normalize-loader.js.map +0 -1
  100. package/dist/lib/loader-utils/option-defaults.js.map +0 -1
  101. package/dist/lib/loader-utils/option-utils.js.map +0 -1
  102. package/dist/lib/progress/fetch-progress.js.map +0 -1
  103. package/dist/lib/utils/log.js.map +0 -1
  104. package/dist/lib/utils/mime-type-utils.js.map +0 -1
  105. package/dist/lib/utils/resource-utils.js.map +0 -1
  106. package/dist/lib/utils/response-utils.js.map +0 -1
  107. package/dist/lib/utils/url-utils.js.map +0 -1
  108. package/dist/null-loader.js.map +0 -1
  109. package/dist/workers/null-worker-node.js.map +0 -1
  110. package/dist/workers/null-worker.js.map +0 -1
@@ -1,28 +1,42 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { isLoaderObject } from "../loader-utils/normalize-loader.js";
2
5
  import { getFetchFunction } from "../loader-utils/get-fetch-function.js";
3
6
  import { parseInBatches } from "./parse-in-batches.js";
4
7
  export function loadInBatches(files, loaders, options, context) {
5
- let loadersArray;
6
- if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
7
- context = undefined;
8
- options = loaders;
9
- loadersArray = undefined;
10
- } else {
11
- loadersArray = loaders;
12
- }
13
- const fetch = getFetchFunction(options || {});
14
- if (!Array.isArray(files)) {
15
- return loadOneFileInBatches(files, loadersArray, options || {}, fetch);
16
- }
17
- const promises = files.map(file => loadOneFileInBatches(file, loadersArray, options || {}, fetch));
18
- return promises;
8
+ let loadersArray;
9
+ // Signature: load(url, options)
10
+ if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
11
+ context = undefined; // context not supported in short signature
12
+ options = loaders;
13
+ loadersArray = undefined;
14
+ }
15
+ else {
16
+ loadersArray = loaders;
17
+ }
18
+ // Select fetch function
19
+ const fetch = getFetchFunction(options || {});
20
+ // Single url/file
21
+ if (!Array.isArray(files)) {
22
+ return loadOneFileInBatches(files, loadersArray, options || {}, fetch);
23
+ }
24
+ // Multiple URLs / files
25
+ const promises = files.map((file) => loadOneFileInBatches(file, loadersArray, options || {}, fetch));
26
+ // No point in waiting here for all responses before starting to stream individual streams?
27
+ return promises;
19
28
  }
20
29
  async function loadOneFileInBatches(file, loaders, options, fetch) {
21
- if (typeof file === 'string') {
22
- const url = file;
23
- const response = await fetch(url);
24
- return Array.isArray(loaders) ? await parseInBatches(response, loaders, options) : await parseInBatches(response, loaders, options);
25
- }
26
- return Array.isArray(loaders) ? await parseInBatches(file, loaders, options) : await parseInBatches(file, loaders, options);
30
+ if (typeof file === 'string') {
31
+ const url = file;
32
+ const response = await fetch(url);
33
+ // pick right overload
34
+ return Array.isArray(loaders)
35
+ ? await parseInBatches(response, loaders, options)
36
+ : await parseInBatches(response, loaders, options);
37
+ }
38
+ // pick right overload
39
+ return Array.isArray(loaders)
40
+ ? await parseInBatches(file, loaders, options)
41
+ : await parseInBatches(file, loaders, options);
27
42
  }
28
- //# sourceMappingURL=load-in-batches.js.map
@@ -1,26 +1,41 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { isBlob } from "../../javascript-utils/is-type.js";
2
5
  import { isLoaderObject } from "../loader-utils/normalize-loader.js";
3
6
  import { getFetchFunction } from "../loader-utils/get-fetch-function.js";
4
7
  import { parse } from "./parse.js";
8
+ // implementation signature
5
9
  export async function load(url, loaders, options, context) {
6
- let resolvedLoaders;
7
- let resolvedOptions;
8
- if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
9
- resolvedLoaders = [];
10
- resolvedOptions = loaders;
11
- context = undefined;
12
- } else {
13
- resolvedLoaders = loaders;
14
- resolvedOptions = options;
15
- }
16
- const fetch = getFetchFunction(resolvedOptions);
17
- let data = url;
18
- if (typeof url === 'string') {
19
- data = await fetch(url);
20
- }
21
- if (isBlob(url)) {
22
- data = await fetch(url);
23
- }
24
- return Array.isArray(resolvedLoaders) ? await parse(data, resolvedLoaders, resolvedOptions) : await parse(data, resolvedLoaders, resolvedOptions);
10
+ let resolvedLoaders;
11
+ let resolvedOptions;
12
+ // Signature: load(url, options)
13
+ if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
14
+ resolvedLoaders = [];
15
+ resolvedOptions = loaders;
16
+ context = undefined; // context not supported in short signature
17
+ }
18
+ else {
19
+ resolvedLoaders = loaders;
20
+ resolvedOptions = options;
21
+ }
22
+ // Select fetch function
23
+ const fetch = getFetchFunction(resolvedOptions);
24
+ // at this point, `url` could be already loaded binary data
25
+ let data = url;
26
+ // url is a string, fetch the url
27
+ if (typeof url === 'string') {
28
+ data = await fetch(url);
29
+ // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)
30
+ }
31
+ if (isBlob(url)) {
32
+ // The fetch response object will contain blob.name
33
+ // @ts-expect-error TODO - This may not work for overridden fetch functions
34
+ data = await fetch(url);
35
+ }
36
+ // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
37
+ // return await parse(data, loaders as Loader[], options);
38
+ return Array.isArray(resolvedLoaders)
39
+ ? await parse(data, resolvedLoaders, resolvedOptions) // loader array overload
40
+ : await parse(data, resolvedLoaders, resolvedOptions); // single loader overload
25
41
  }
26
- //# sourceMappingURL=load.js.map
@@ -1,3 +1,3 @@
1
- export { setGlobalOptions as setLoaderOptions } from '../loader-utils/option-utils';
2
- export { getGlobalLoaderOptions as getLoaderOptions } from '../loader-utils/option-utils';
1
+ export { setGlobalOptions as setLoaderOptions } from "../loader-utils/option-utils.js";
2
+ export { getGlobalLoaderOptions as getLoaderOptions } from "../loader-utils/option-utils.js";
3
3
  //# sourceMappingURL=loader-options.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"loader-options.d.ts","sourceRoot":"","sources":["../../../src/lib/api/loader-options.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,gBAAgB,IAAI,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAClF,OAAO,EAAC,sBAAsB,IAAI,gBAAgB,EAAC,MAAM,8BAA8B,CAAC"}
1
+ {"version":3,"file":"loader-options.d.ts","sourceRoot":"","sources":["../../../src/lib/api/loader-options.ts"],"names":[],"mappings":"AAIA,OAAO,EAAC,gBAAgB,IAAI,gBAAgB,EAAC,wCAAqC;AAClF,OAAO,EAAC,sBAAsB,IAAI,gBAAgB,EAAC,wCAAqC"}
@@ -1,3 +1,5 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  export { setGlobalOptions as setLoaderOptions } from "../loader-utils/option-utils.js";
2
5
  export { getGlobalLoaderOptions as getLoaderOptions } from "../loader-utils/option-utils.js";
3
- //# sourceMappingURL=loader-options.js.map
@@ -1,3 +1,6 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { isTable, makeBatchFromTable } from '@loaders.gl/schema';
2
5
  import { concatenateArrayBuffersAsync } from '@loaders.gl/loader-utils';
3
6
  import { isLoaderObject } from "../loader-utils/normalize-loader.js";
@@ -6,84 +9,117 @@ import { getLoaderContext } from "../loader-utils/loader-context.js";
6
9
  import { getAsyncIterableFromData } from "../loader-utils/get-data.js";
7
10
  import { getResourceUrl } from "../utils/resource-utils.js";
8
11
  import { selectLoader } from "./select-loader.js";
12
+ // Ensure `parse` is available in context if loader falls back to `parse`
9
13
  import { parse } from "./parse.js";
14
+ /**
15
+ * Parses `data` using a specified loader
16
+ * @param data
17
+ * @param loaders
18
+ * @param options
19
+ * @param context
20
+ */
10
21
  export async function parseInBatches(data, loaders, options, context) {
11
- const loaderArray = Array.isArray(loaders) ? loaders : undefined;
12
- if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
13
- context = undefined;
14
- options = loaders;
15
- loaders = undefined;
16
- }
17
- data = await data;
18
- options = options || {};
19
- const url = getResourceUrl(data);
20
- const loader = await selectLoader(data, loaders, options);
21
- if (!loader) {
22
- return [];
23
- }
24
- options = normalizeOptions(options, loader, loaderArray, url);
25
- context = getLoaderContext({
26
- url,
27
- _parseInBatches: parseInBatches,
28
- _parse: parse,
29
- loaders: loaderArray
30
- }, options, context || null);
31
- return await parseWithLoaderInBatches(loader, data, options, context);
22
+ const loaderArray = Array.isArray(loaders) ? loaders : undefined;
23
+ // Signature: parseInBatches(data, options, url) - Uses registered loaders
24
+ if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
25
+ context = undefined; // context not supported in short signature
26
+ options = loaders;
27
+ loaders = undefined;
28
+ }
29
+ data = await data; // Resolve any promise
30
+ options = options || {};
31
+ // Extract a url for auto detection
32
+ const url = getResourceUrl(data);
33
+ // Chooses a loader and normalizes it
34
+ // Note - only uses URL and contentType for streams and iterator inputs
35
+ const loader = await selectLoader(data, loaders, options);
36
+ // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
37
+ if (!loader) {
38
+ return [];
39
+ }
40
+ // Normalize options
41
+ options = normalizeOptions(options, loader, loaderArray, url);
42
+ context = getLoaderContext({ url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray }, options, context || null);
43
+ return await parseWithLoaderInBatches(loader, data, options, context);
32
44
  }
45
+ /**
46
+ * Loader has been selected and context has been prepared, see if we need to emit a metadata batch
47
+ */
33
48
  async function parseWithLoaderInBatches(loader, data, options, context) {
34
- const outputIterator = await parseToOutputIterator(loader, data, options, context);
35
- if (!options.metadata) {
36
- return outputIterator;
37
- }
38
- const metadataBatch = {
39
- shape: 'metadata',
40
- batchType: 'metadata',
41
- metadata: {
42
- _loader: loader,
43
- _context: context
44
- },
45
- data: [],
46
- bytesUsed: 0
47
- };
48
- async function* makeMetadataBatchIterator(iterator) {
49
- yield metadataBatch;
50
- yield* iterator;
51
- }
52
- return makeMetadataBatchIterator(outputIterator);
49
+ const outputIterator = await parseToOutputIterator(loader, data, options, context);
50
+ // Generate metadata batch if requested
51
+ if (!options.metadata) {
52
+ return outputIterator;
53
+ }
54
+ const metadataBatch = {
55
+ shape: 'metadata',
56
+ batchType: 'metadata',
57
+ metadata: {
58
+ _loader: loader,
59
+ _context: context
60
+ },
61
+ // Populate with some default fields to avoid crashing
62
+ data: [],
63
+ bytesUsed: 0
64
+ };
65
+ async function* makeMetadataBatchIterator(iterator) {
66
+ yield metadataBatch;
67
+ yield* iterator;
68
+ }
69
+ return makeMetadataBatchIterator(outputIterator);
53
70
  }
71
+ /**
72
+ * Prep work is done, now it is time to start parsing into an output operator
73
+ * The approach depends on which parse function the loader exposes
74
+ * `parseInBatches` (preferred), `parse` (fallback)
75
+ */
54
76
  async function parseToOutputIterator(loader, data, options, context) {
55
- const inputIterator = await getAsyncIterableFromData(data, options);
56
- const transformedIterator = await applyInputTransforms(inputIterator, (options === null || options === void 0 ? void 0 : options.transforms) || []);
57
- if (loader.parseInBatches) {
58
- return loader.parseInBatches(transformedIterator, options, context);
59
- }
60
- return parseChunkInBatches(transformedIterator, loader, options, context);
77
+ // Get an iterator from the input
78
+ const inputIterator = await getAsyncIterableFromData(data, options);
79
+ // Apply any iterator transforms (options.transforms)
80
+ const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);
81
+ // If loader supports parseInBatches, we are done
82
+ if (loader.parseInBatches) {
83
+ return loader.parseInBatches(transformedIterator, options, context);
84
+ }
85
+ return parseChunkInBatches(transformedIterator, loader, options, context);
61
86
  }
87
+ // Fallback: load atomically using `parse` concatenating input iterator into single chunk
62
88
  async function* parseChunkInBatches(transformedIterator, loader, options, context) {
63
- const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
64
- const parsedData = await parse(arrayBuffer, loader, {
65
- ...options,
66
- mimeType: loader.mimeTypes[0]
67
- }, context);
68
- const batch = convertDataToBatch(parsedData, loader);
69
- yield batch;
89
+ const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
90
+ // Call `parse` instead of `loader.parse` to ensure we can call workers etc.
91
+ const parsedData = await parse(arrayBuffer, loader,
92
+ // TODO - Hack: supply loaders MIME type to ensure we match it
93
+ { ...options, mimeType: loader.mimeTypes[0] }, context);
94
+ // yield a single batch, the output from loader.parse() repackaged as a batch
95
+ const batch = convertDataToBatch(parsedData, loader);
96
+ yield batch;
70
97
  }
98
+ /**
99
+ * Convert parsed data into a single batch
100
+ * @todo run through batch builder to apply options etc...
101
+ */
71
102
  function convertDataToBatch(parsedData, loader) {
72
- const batch = isTable(parsedData) ? makeBatchFromTable(parsedData) : {
73
- shape: 'unknown',
74
- batchType: 'data',
75
- data: parsedData,
76
- length: Array.isArray(parsedData) ? parsedData.length : 1
77
- };
78
- batch.mimeType = loader.mimeTypes[0];
79
- return batch;
103
+ const batch = isTable(parsedData)
104
+ ? makeBatchFromTable(parsedData)
105
+ : {
106
+ shape: 'unknown',
107
+ batchType: 'data',
108
+ data: parsedData,
109
+ length: Array.isArray(parsedData) ? parsedData.length : 1
110
+ };
111
+ batch.mimeType = loader.mimeTypes[0];
112
+ return batch;
80
113
  }
81
- async function applyInputTransforms(inputIterator) {
82
- let transforms = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : [];
83
- let iteratorChain = inputIterator;
84
- for await (const transformBatches of transforms) {
85
- iteratorChain = transformBatches(iteratorChain);
86
- }
87
- return iteratorChain;
114
+ /**
115
+ * Create an iterator chain with any transform iterators (crypto, decompression)
116
+ * @param inputIterator
117
+ * @param options
118
+ */
119
+ async function applyInputTransforms(inputIterator, transforms = []) {
120
+ let iteratorChain = inputIterator;
121
+ for await (const transformBatches of transforms) {
122
+ iteratorChain = transformBatches(iteratorChain);
123
+ }
124
+ return iteratorChain;
88
125
  }
89
- //# sourceMappingURL=parse-in-batches.js.map
@@ -1,43 +1,52 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { selectLoaderSync } from "./select-loader.js";
2
5
  import { isLoaderObject } from "../loader-utils/normalize-loader.js";
3
6
  import { normalizeOptions } from "../loader-utils/option-utils.js";
4
7
  import { getArrayBufferOrStringFromDataSync } from "../loader-utils/get-data.js";
5
8
  import { getLoaderContext, getLoadersFromContext } from "../loader-utils/loader-context.js";
6
9
  import { getResourceUrl } from "../utils/resource-utils.js";
10
+ /**
11
+ * Parses `data` synchronously using a specified loader
12
+ */
7
13
  export function parseSync(data, loaders, options, context) {
8
- if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
9
- context = undefined;
10
- options = loaders;
11
- loaders = undefined;
12
- }
13
- options = options || {};
14
- const typedLoaders = loaders;
15
- const candidateLoaders = getLoadersFromContext(typedLoaders, context);
16
- const loader = selectLoaderSync(data, candidateLoaders, options);
17
- if (!loader) {
18
- return null;
19
- }
20
- options = normalizeOptions(options, loader, candidateLoaders);
21
- const url = getResourceUrl(data);
22
- const parse = () => {
23
- throw new Error('parseSync called parse (which is async');
24
- };
25
- context = getLoaderContext({
26
- url,
27
- _parseSync: parse,
28
- _parse: parse,
29
- loaders: loaders
30
- }, options, context || null);
31
- return parseWithLoaderSync(loader, data, options, context);
14
+ // Signature: parseSync(data, options)
15
+ // Uses registered loaders
16
+ if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
17
+ context = undefined; // context not supported in short signature
18
+ options = loaders;
19
+ loaders = undefined;
20
+ }
21
+ options = options || {};
22
+ // Chooses a loader (and normalizes it)
23
+ // Also use any loaders in the context, new loaders take priority
24
+ const typedLoaders = loaders;
25
+ const candidateLoaders = getLoadersFromContext(typedLoaders, context);
26
+ const loader = selectLoaderSync(data, candidateLoaders, options);
27
+ // Note: if nothrow option was set, it is possible that no loader was found, if so just return null
28
+ if (!loader) {
29
+ return null;
30
+ }
31
+ // Normalize options
32
+ options = normalizeOptions(options, loader, candidateLoaders);
33
+ // Extract a url for auto detection
34
+ const url = getResourceUrl(data);
35
+ const parse = () => {
36
+ throw new Error('parseSync called parse (which is async');
37
+ };
38
+ context = getLoaderContext({ url, _parseSync: parse, _parse: parse, loaders: loaders }, options, context || null);
39
+ return parseWithLoaderSync(loader, data, options, context);
32
40
  }
41
+ // TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator
33
42
  function parseWithLoaderSync(loader, data, options, context) {
34
- data = getArrayBufferOrStringFromDataSync(data, loader, options);
35
- if (loader.parseTextSync && typeof data === 'string') {
36
- return loader.parseTextSync(data, options);
37
- }
38
- if (loader.parseSync && data instanceof ArrayBuffer) {
39
- return loader.parseSync(data, options, context);
40
- }
41
- throw new Error(`${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${context.url || ''}`);
43
+ data = getArrayBufferOrStringFromDataSync(data, loader, options);
44
+ if (loader.parseTextSync && typeof data === 'string') {
45
+ return loader.parseTextSync(data, options); // , context, loader);
46
+ }
47
+ if (loader.parseSync && data instanceof ArrayBuffer) {
48
+ return loader.parseSync(data, options, context); // , loader);
49
+ }
50
+ // TBD - If synchronous parser not available, return null
51
+ throw new Error(`${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${context.url || ''}`);
42
52
  }
43
- //# sourceMappingURL=parse-sync.js.map
@@ -1,3 +1,6 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { parseWithWorker, canParseWithWorker } from '@loaders.gl/loader-utils';
2
5
  import { assert, validateWorkerVersion } from '@loaders.gl/worker-utils';
3
6
  import { isLoaderObject } from "../loader-utils/normalize-loader.js";
@@ -8,68 +11,77 @@ import { getArrayBufferOrStringFromData } from "../loader-utils/get-data.js";
8
11
  import { getLoaderContext, getLoadersFromContext } from "../loader-utils/loader-context.js";
9
12
  import { getResourceUrl } from "../utils/resource-utils.js";
10
13
  import { selectLoader } from "./select-loader.js";
14
+ /**
15
+ * Parses `data` using a specified loader
16
+ * @param data
17
+ * @param loaders
18
+ * @param options
19
+ * @param context
20
+ */
21
+ // implementation signature
11
22
  export async function parse(data, loaders, options, context) {
12
- if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
13
- context = undefined;
14
- options = loaders;
15
- loaders = undefined;
16
- }
17
- data = await data;
18
- options = options || {};
19
- const url = getResourceUrl(data);
20
- const typedLoaders = loaders;
21
- const candidateLoaders = getLoadersFromContext(typedLoaders, context);
22
- const loader = await selectLoader(data, candidateLoaders, options);
23
- if (!loader) {
24
- return null;
25
- }
26
- options = normalizeOptions(options, loader, candidateLoaders, url);
27
- context = getLoaderContext({
28
- url,
29
- _parse: parse,
30
- loaders: candidateLoaders
31
- }, options, context || null);
32
- return await parseWithLoader(loader, data, options, context);
23
+ // Signature: parse(data, options, context | url)
24
+ // Uses registered loaders
25
+ if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
26
+ context = undefined; // context not supported in short signature
27
+ options = loaders;
28
+ loaders = undefined;
29
+ }
30
+ data = await data; // Resolve any promise
31
+ options = options || {}; // Could be invalid...
32
+ // Extract a url for auto detection
33
+ const url = getResourceUrl(data);
34
+ // Chooses a loader (and normalizes it)
35
+ // Also use any loaders in the context, new loaders take priority
36
+ const typedLoaders = loaders;
37
+ const candidateLoaders = getLoadersFromContext(typedLoaders, context);
38
+ // todo hacky type cast
39
+ const loader = await selectLoader(data, candidateLoaders, options);
40
+ // Note: if no loader was found, if so just return null
41
+ if (!loader) {
42
+ return null;
43
+ }
44
+ // Normalize options
45
+ // @ts-expect-error
46
+ options = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
47
+ // Get a context (if already present, will be unchanged)
48
+ context = getLoaderContext(
49
+ // @ts-expect-error
50
+ { url, _parse: parse, loaders: candidateLoaders }, options, context || null);
51
+ return await parseWithLoader(loader, data, options, context);
33
52
  }
53
+ // TODO: support progress and abort
54
+ // TODO - should accept loader.parseAsyncIterator and concatenate.
34
55
  async function parseWithLoader(loader, data, options, context) {
35
- validateWorkerVersion(loader);
36
- options = mergeLoaderOptions(loader.options, options);
37
- if (isResponse(data)) {
38
- const response = data;
39
- const {
40
- ok,
41
- redirected,
42
- status,
43
- statusText,
44
- type,
45
- url
46
- } = response;
47
- const headers = Object.fromEntries(response.headers.entries());
48
- context.response = {
49
- headers,
50
- ok,
51
- redirected,
52
- status,
53
- statusText,
54
- type,
55
- url
56
- };
57
- }
58
- data = await getArrayBufferOrStringFromData(data, loader, options);
59
- const loaderWithParser = loader;
60
- if (loaderWithParser.parseTextSync && typeof data === 'string') {
61
- return loaderWithParser.parseTextSync(data, options, context);
62
- }
63
- if (canParseWithWorker(loader, options)) {
64
- return await parseWithWorker(loader, data, options, context, parse);
65
- }
66
- if (loaderWithParser.parseText && typeof data === 'string') {
67
- return await loaderWithParser.parseText(data, options, context);
68
- }
69
- if (loaderWithParser.parse) {
70
- return await loaderWithParser.parse(data, options, context);
71
- }
72
- assert(!loaderWithParser.parseSync);
73
- throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
56
+ validateWorkerVersion(loader);
57
+ options = mergeLoaderOptions(loader.options, options);
58
+ if (isResponse(data)) {
59
+ // Serialize to support passing the response to web worker
60
+ const response = data;
61
+ const { ok, redirected, status, statusText, type, url } = response;
62
+ const headers = Object.fromEntries(response.headers.entries());
63
+ // @ts-expect-error TODO - fix this
64
+ context.response = { headers, ok, redirected, status, statusText, type, url };
65
+ }
66
+ data = await getArrayBufferOrStringFromData(data, loader, options);
67
+ const loaderWithParser = loader;
68
+ // First check for synchronous text parser, wrap results in promises
69
+ if (loaderWithParser.parseTextSync && typeof data === 'string') {
70
+ return loaderWithParser.parseTextSync(data, options, context);
71
+ }
72
+ // If we have a workerUrl and the loader can parse the given options efficiently in a worker
73
+ if (canParseWithWorker(loader, options)) {
74
+ return await parseWithWorker(loader, data, options, context, parse);
75
+ }
76
+ // Check for asynchronous parser
77
+ if (loaderWithParser.parseText && typeof data === 'string') {
78
+ return await loaderWithParser.parseText(data, options, context);
79
+ }
80
+ if (loaderWithParser.parse) {
81
+ return await loaderWithParser.parse(data, options, context);
82
+ }
83
+ // This should not happen, all sync loaders should also offer `parse` function
84
+ assert(!loaderWithParser.parseSync);
85
+ // TBD - If asynchronous parser not available, return null
86
+ throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
74
87
  }
75
- //# sourceMappingURL=parse.js.map
@@ -1,25 +1,34 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
1
4
  import { normalizeLoader } from "../loader-utils/normalize-loader.js";
2
5
  import { getGlobalLoaderState } from "../loader-utils/option-utils.js";
6
+ /**
7
+ * Store global registered loaders on the global object to increase chances of cross loaders-version interoperability
8
+ * This use case is not reliable but can help when testing new versions of loaders.gl with existing frameworks
9
+ */
3
10
  const getGlobalLoaderRegistry = () => {
4
- const state = getGlobalLoaderState();
5
- state.loaderRegistry = state.loaderRegistry || [];
6
- return state.loaderRegistry;
11
+ const state = getGlobalLoaderState();
12
+ state.loaderRegistry = state.loaderRegistry || [];
13
+ return state.loaderRegistry;
7
14
  };
15
+ /** Register a list of global loaders */
8
16
  export function registerLoaders(loaders) {
9
- const loaderRegistry = getGlobalLoaderRegistry();
10
- loaders = Array.isArray(loaders) ? loaders : [loaders];
11
- for (const loader of loaders) {
12
- const normalizedLoader = normalizeLoader(loader);
13
- if (!loaderRegistry.find(registeredLoader => normalizedLoader === registeredLoader)) {
14
- loaderRegistry.unshift(normalizedLoader);
17
+ const loaderRegistry = getGlobalLoaderRegistry();
18
+ loaders = Array.isArray(loaders) ? loaders : [loaders];
19
+ for (const loader of loaders) {
20
+ const normalizedLoader = normalizeLoader(loader);
21
+ if (!loaderRegistry.find((registeredLoader) => normalizedLoader === registeredLoader)) {
22
+ // add to the beginning of the loaderRegistry, so the last registeredLoader get picked
23
+ loaderRegistry.unshift(normalizedLoader);
24
+ }
15
25
  }
16
- }
17
26
  }
18
27
  export function getRegisteredLoaders() {
19
- return getGlobalLoaderRegistry();
28
+ return getGlobalLoaderRegistry();
20
29
  }
30
+ /** @deprecated For testing only */
21
31
  export function _unregisterLoaders() {
22
- const state = getGlobalLoaderState();
23
- state.loaderRegistry = [];
32
+ const state = getGlobalLoaderState();
33
+ state.loaderRegistry = [];
24
34
  }
25
- //# sourceMappingURL=register-loaders.js.map