@loaders.gl/core 4.4.0-alpha.2 → 4.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/README.md +4 -0
  2. package/dist/core-addons/write-file-browser.js +1 -0
  3. package/dist/core-addons/write-file-browser.js.map +1 -0
  4. package/dist/dist.dev.js +641 -286
  5. package/dist/dist.min.js +3 -3
  6. package/dist/index.cjs +411 -262
  7. package/dist/index.cjs.map +4 -4
  8. package/dist/index.d.ts +1 -1
  9. package/dist/index.d.ts.map +1 -1
  10. package/dist/index.js +2 -1
  11. package/dist/index.js.map +1 -0
  12. package/dist/iterators/batch-iterators/timed-batch-iterator.js +1 -0
  13. package/dist/iterators/batch-iterators/timed-batch-iterator.js.map +1 -0
  14. package/dist/iterators/make-iterator/make-array-buffer-iterator.js +1 -0
  15. package/dist/iterators/make-iterator/make-array-buffer-iterator.js.map +1 -0
  16. package/dist/iterators/make-iterator/make-blob-iterator.js +1 -0
  17. package/dist/iterators/make-iterator/make-blob-iterator.js.map +1 -0
  18. package/dist/iterators/make-iterator/make-iterator.d.ts.map +1 -1
  19. package/dist/iterators/make-iterator/make-iterator.js +7 -3
  20. package/dist/iterators/make-iterator/make-iterator.js.map +1 -0
  21. package/dist/iterators/make-iterator/make-stream-iterator.js +1 -0
  22. package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -0
  23. package/dist/iterators/make-iterator/make-string-iterator.d.ts.map +1 -1
  24. package/dist/iterators/make-iterator/make-string-iterator.js +3 -1
  25. package/dist/iterators/make-iterator/make-string-iterator.js.map +1 -0
  26. package/dist/iterators/make-stream/make-stream.js +1 -0
  27. package/dist/iterators/make-stream/make-stream.js.map +1 -0
  28. package/dist/lib/api/create-data-source.js +1 -0
  29. package/dist/lib/api/create-data-source.js.map +1 -0
  30. package/dist/lib/api/encode-table.d.ts.map +1 -1
  31. package/dist/lib/api/encode-table.js +3 -2
  32. package/dist/lib/api/encode-table.js.map +1 -0
  33. package/dist/lib/api/encode.d.ts.map +1 -1
  34. package/dist/lib/api/encode.js +3 -2
  35. package/dist/lib/api/encode.js.map +1 -0
  36. package/dist/lib/api/load-in-batches.js +1 -0
  37. package/dist/lib/api/load-in-batches.js.map +1 -0
  38. package/dist/lib/api/load.d.ts.map +1 -1
  39. package/dist/lib/api/load.js +15 -1
  40. package/dist/lib/api/load.js.map +1 -0
  41. package/dist/lib/api/loader-options.js +1 -0
  42. package/dist/lib/api/loader-options.js.map +1 -0
  43. package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
  44. package/dist/lib/api/parse-in-batches.js +7 -6
  45. package/dist/lib/api/parse-in-batches.js.map +1 -0
  46. package/dist/lib/api/parse-sync.d.ts.map +1 -1
  47. package/dist/lib/api/parse-sync.js +4 -3
  48. package/dist/lib/api/parse-sync.js.map +1 -0
  49. package/dist/lib/api/parse.d.ts.map +1 -1
  50. package/dist/lib/api/parse.js +8 -9
  51. package/dist/lib/api/parse.js.map +1 -0
  52. package/dist/lib/api/register-loaders.js +1 -0
  53. package/dist/lib/api/register-loaders.js.map +1 -0
  54. package/dist/lib/api/select-loader.d.ts +3 -3
  55. package/dist/lib/api/select-loader.d.ts.map +1 -1
  56. package/dist/lib/api/select-loader.js +36 -15
  57. package/dist/lib/api/select-loader.js.map +1 -0
  58. package/dist/lib/api/select-source.js +1 -0
  59. package/dist/lib/api/select-source.js.map +1 -0
  60. package/dist/lib/common.js +1 -0
  61. package/dist/lib/common.js.map +1 -0
  62. package/dist/lib/fetch/fetch-error-message.js +1 -0
  63. package/dist/lib/fetch/fetch-error-message.js.map +1 -0
  64. package/dist/lib/fetch/fetch-error.js +1 -0
  65. package/dist/lib/fetch/fetch-error.js.map +1 -0
  66. package/dist/lib/fetch/fetch-file.js +1 -0
  67. package/dist/lib/fetch/fetch-file.js.map +1 -0
  68. package/dist/lib/fetch/read-array-buffer.js +1 -0
  69. package/dist/lib/fetch/read-array-buffer.js.map +1 -0
  70. package/dist/lib/filesystems/browser-filesystem.js +1 -0
  71. package/dist/lib/filesystems/browser-filesystem.js.map +1 -0
  72. package/dist/lib/filesystems/read-array-buffer.js +1 -0
  73. package/dist/lib/filesystems/read-array-buffer.js.map +1 -0
  74. package/dist/lib/init.js +2 -1
  75. package/dist/lib/init.js.map +1 -0
  76. package/dist/lib/loader-utils/check-errors.js +1 -0
  77. package/dist/lib/loader-utils/check-errors.js.map +1 -0
  78. package/dist/lib/loader-utils/get-data.d.ts +18 -1
  79. package/dist/lib/loader-utils/get-data.d.ts.map +1 -1
  80. package/dist/lib/loader-utils/get-data.js +47 -38
  81. package/dist/lib/loader-utils/get-data.js.map +1 -0
  82. package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
  83. package/dist/lib/loader-utils/get-fetch-function.js +7 -5
  84. package/dist/lib/loader-utils/get-fetch-function.js.map +1 -0
  85. package/dist/lib/loader-utils/loader-context.d.ts +2 -2
  86. package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
  87. package/dist/lib/loader-utils/loader-context.js +1 -0
  88. package/dist/lib/loader-utils/loader-context.js.map +1 -0
  89. package/dist/lib/loader-utils/loggers.js +1 -0
  90. package/dist/lib/loader-utils/loggers.js.map +1 -0
  91. package/dist/lib/loader-utils/normalize-loader.js +1 -0
  92. package/dist/lib/loader-utils/normalize-loader.js.map +1 -0
  93. package/dist/lib/loader-utils/option-defaults.d.ts +22 -0
  94. package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
  95. package/dist/lib/loader-utils/option-defaults.js +63 -33
  96. package/dist/lib/loader-utils/option-defaults.js.map +1 -0
  97. package/dist/lib/loader-utils/option-utils.d.ts +10 -4
  98. package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
  99. package/dist/lib/loader-utils/option-utils.js +123 -19
  100. package/dist/lib/loader-utils/option-utils.js.map +1 -0
  101. package/dist/lib/progress/fetch-progress.js +1 -0
  102. package/dist/lib/progress/fetch-progress.js.map +1 -0
  103. package/dist/lib/utils/mime-type-utils.js +1 -0
  104. package/dist/lib/utils/mime-type-utils.js.map +1 -0
  105. package/dist/lib/utils/resource-utils.d.ts.map +1 -1
  106. package/dist/lib/utils/resource-utils.js +8 -10
  107. package/dist/lib/utils/resource-utils.js.map +1 -0
  108. package/dist/lib/utils/response-utils.js +2 -1
  109. package/dist/lib/utils/response-utils.js.map +1 -0
  110. package/dist/lib/utils/url-utils.js +1 -0
  111. package/dist/lib/utils/url-utils.js.map +1 -0
  112. package/dist/null-loader.d.ts +3 -3
  113. package/dist/null-loader.d.ts.map +1 -1
  114. package/dist/null-loader.js +2 -1
  115. package/dist/null-loader.js.map +1 -0
  116. package/dist/null-worker-node.js +5 -2
  117. package/dist/null-worker.js +5 -2
  118. package/dist/workers/null-worker-node.js +1 -0
  119. package/dist/workers/null-worker-node.js.map +1 -0
  120. package/dist/workers/null-worker.js +1 -0
  121. package/dist/workers/null-worker.js.map +1 -0
  122. package/package.json +13 -10
  123. package/src/index.ts +1 -1
  124. package/src/iterators/make-iterator/make-iterator.ts +8 -5
  125. package/src/iterators/make-iterator/make-string-iterator.ts +2 -1
  126. package/src/lib/api/encode-table.ts +2 -1
  127. package/src/lib/api/encode.ts +8 -2
  128. package/src/lib/api/load.ts +15 -1
  129. package/src/lib/api/parse-in-batches.ts +26 -19
  130. package/src/lib/api/parse-sync.ts +6 -5
  131. package/src/lib/api/parse.ts +15 -11
  132. package/src/lib/api/select-loader.ts +68 -24
  133. package/src/lib/loader-utils/get-data.ts +62 -48
  134. package/src/lib/loader-utils/get-fetch-function.ts +6 -5
  135. package/src/lib/loader-utils/loader-context.ts +2 -2
  136. package/src/lib/loader-utils/option-defaults.ts +68 -35
  137. package/src/lib/loader-utils/option-utils.ts +146 -29
  138. package/src/lib/utils/resource-utils.ts +9 -12
  139. package/src/lib/utils/response-utils.ts +2 -2
  140. package/src/null-loader.ts +6 -3
  141. package/dist/javascript-utils/is-type.d.ts +0 -21
  142. package/dist/javascript-utils/is-type.d.ts.map +0 -1
  143. package/dist/javascript-utils/is-type.js +0 -25
  144. package/src/javascript-utils/is-type.ts +0 -51
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@loaders.gl/core",
3
- "version": "4.4.0-alpha.2",
3
+ "version": "4.4.0",
4
4
  "description": "The core API for working with loaders.gl loaders and writers",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -32,13 +32,16 @@
32
32
  "types": "./dist/index.d.ts",
33
33
  "import": "./dist/index.js",
34
34
  "require": "./dist/index.cjs"
35
+ },
36
+ "./null-worker.js": {
37
+ "import": "./dist/null-worker.js"
38
+ },
39
+ "./null-worker-node.js": {
40
+ "import": "./dist/null-worker-node.js"
35
41
  }
36
42
  },
37
43
  "sideEffects": false,
38
44
  "browser": {
39
- "./dist/iterators/make-stream/make-node-stream.js": "./dist/iterators/make-stream/make-dom-stream.js",
40
- "./src/iterators/make-stream/make-node-stream.js": "./src/iterators/make-stream/make-dom-stream.js",
41
- "./src/iterators/make-stream/make-node-stream.ts": "./src/iterators/make-stream/make-dom-stream.ts",
42
45
  "fs": false,
43
46
  "stream": false
44
47
  },
@@ -55,11 +58,11 @@
55
58
  "build-worker-node": "esbuild src/workers/null-worker.ts --outfile=dist/null-worker-node.js --bundle --platform=node --target=node16 --define:__VERSION__=\\\"$npm_package_version\\\""
56
59
  },
57
60
  "dependencies": {
58
- "@loaders.gl/loader-utils": "4.4.0-alpha.2",
59
- "@loaders.gl/schema": "4.4.0-alpha.2",
60
- "@loaders.gl/schema-utils": "4.4.0-alpha.2",
61
- "@loaders.gl/worker-utils": "4.4.0-alpha.2",
62
- "@probe.gl/log": "^4.0.2"
61
+ "@loaders.gl/loader-utils": "4.4.0",
62
+ "@loaders.gl/schema": "4.4.0",
63
+ "@loaders.gl/schema-utils": "4.4.0",
64
+ "@loaders.gl/worker-utils": "4.4.0",
65
+ "@probe.gl/log": "^4.1.1"
63
66
  },
64
- "gitHead": "3d9fed050eabdc0812ddf2f4d5fb9914a34ee0c2"
67
+ "gitHead": "b8a23bc05946bde7089936ff3ea1651ee9a45536"
65
68
  }
package/src/index.ts CHANGED
@@ -94,4 +94,4 @@ export {
94
94
  isResponse,
95
95
  isReadableStream,
96
96
  isWritableStream
97
- } from './javascript-utils/is-type';
97
+ } from '@loaders.gl/loader-utils';
@@ -9,7 +9,7 @@ import {makeArrayBufferIterator} from './make-array-buffer-iterator';
9
9
  import {makeBlobIterator} from './make-blob-iterator';
10
10
  import type {StreamIteratorOptions} from './make-stream-iterator';
11
11
  import {makeStreamIterator} from './make-stream-iterator';
12
- import {isBlob, isReadableStream, isResponse} from '../../javascript-utils/is-type';
12
+ import {isBlob, isReadableStream, isResponse} from '@loaders.gl/loader-utils';
13
13
 
14
14
  /**
15
15
  * @param [options.chunkSize]
@@ -38,14 +38,17 @@ export function makeIterator(
38
38
  return makeArrayBufferIterator(data, options);
39
39
  }
40
40
  if (isBlob(data)) {
41
- return makeBlobIterator(data as Blob, options);
41
+ return makeBlobIterator(data, options);
42
42
  }
43
43
  if (isReadableStream(data)) {
44
- return makeStreamIterator(data as ReadableStream, options);
44
+ return makeStreamIterator(data, options);
45
45
  }
46
46
  if (isResponse(data)) {
47
- const response = data as Response;
48
- return makeStreamIterator(response.body as ReadableStream, options);
47
+ const responseBody = data.body;
48
+ if (!responseBody) {
49
+ throw new Error('Readable stream not available on Response');
50
+ }
51
+ return makeStreamIterator(responseBody as ReadableStream, options);
49
52
  }
50
53
  throw new Error('makeIterator');
51
54
  }
@@ -2,6 +2,7 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
+ import {ensureArrayBuffer} from '@loaders.gl/loader-utils';
5
6
  import type {IteratorOptions} from './make-iterator';
6
7
 
7
8
  const DEFAULT_CHUNK_SIZE = 256 * 1024;
@@ -27,6 +28,6 @@ export function* makeStringIterator(
27
28
  offset += chunkLength;
28
29
 
29
30
  // yield an ArrayBuffer chunk
30
- yield textEncoder.encode(chunk);
31
+ yield ensureArrayBuffer(textEncoder.encode(chunk));
31
32
  }
32
33
  }
@@ -6,6 +6,7 @@
6
6
  /* global TextEncoder, TextDecoder */
7
7
  import {
8
8
  concatenateArrayBuffers,
9
+ ensureArrayBuffer,
9
10
  WriterOptionsType,
10
11
  WriterWithEncoder
11
12
  } from '@loaders.gl/loader-utils';
@@ -22,7 +23,7 @@ export async function encodeTable<WriterT extends WriterWithEncoder = WriterWith
22
23
 
23
24
  if (writer.encodeText) {
24
25
  const text = await writer.encodeText(data, options);
25
- return new TextEncoder().encode(text);
26
+ return ensureArrayBuffer(new TextEncoder().encode(text));
26
27
  }
27
28
 
28
29
  if (writer.encodeInBatches) {
@@ -9,7 +9,13 @@ import type {
9
9
  WriterDataType,
10
10
  WriterBatchType
11
11
  } from '@loaders.gl/loader-utils';
12
- import {canEncodeWithWorker, NodeFile, resolvePath, isBrowser} from '@loaders.gl/loader-utils';
12
+ import {
13
+ canEncodeWithWorker,
14
+ NodeFile,
15
+ resolvePath,
16
+ isBrowser,
17
+ ensureArrayBuffer
18
+ } from '@loaders.gl/loader-utils';
13
19
  import {processOnWorker} from '@loaders.gl/worker-utils';
14
20
  import {fetchFile} from '../fetch/fetch-file';
15
21
  import {getLoaderOptions} from './loader-options';
@@ -52,7 +58,7 @@ export function encodeSync<WriterT extends WriterWithEncoder>(
52
58
  return writer.encodeSync(data, options);
53
59
  }
54
60
  if (writer.encodeTextSync) {
55
- return new TextEncoder().encode(writer.encodeTextSync(data, options));
61
+ return ensureArrayBuffer(new TextEncoder().encode(writer.encodeTextSync(data, options)));
56
62
  }
57
63
  throw new Error(`Writer ${writer.name} could not synchronously encode data`);
58
64
  }
@@ -12,9 +12,10 @@ import type {
12
12
  LoaderArrayOptionsType,
13
13
  LoaderArrayReturnType
14
14
  } from '@loaders.gl/loader-utils';
15
- import {isBlob} from '../../javascript-utils/is-type';
15
+ import {isBlob} from '@loaders.gl/loader-utils';
16
16
  import {isLoaderObject} from '../loader-utils/normalize-loader';
17
17
  import {getFetchFunction} from '../loader-utils/get-fetch-function';
18
+ import {normalizeLoaderOptions} from '../loader-utils/option-utils';
18
19
 
19
20
  import {parse} from './parse';
20
21
 
@@ -97,6 +98,19 @@ export async function load(
97
98
  data = await fetch(url);
98
99
  }
99
100
 
101
+ if (typeof url === 'string') {
102
+ const normalizedOptions = normalizeLoaderOptions(resolvedOptions || {});
103
+ if (!normalizedOptions.core?.baseUrl) {
104
+ resolvedOptions = {
105
+ ...resolvedOptions,
106
+ core: {
107
+ ...resolvedOptions?.core,
108
+ baseUrl: url
109
+ }
110
+ };
111
+ }
112
+ }
113
+
100
114
  // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
101
115
  // return await parse(data, loaders as Loader[], options);
102
116
  return Array.isArray(resolvedLoaders)
@@ -7,13 +7,15 @@ import {isTable, makeBatchFromTable} from '@loaders.gl/schema-utils';
7
7
  import type {
8
8
  Loader,
9
9
  LoaderWithParser,
10
+ StrictLoaderOptions,
10
11
  LoaderOptions,
11
12
  LoaderContext,
12
13
  BatchableDataType,
13
14
  LoaderOptionsType,
14
15
  LoaderBatchType,
15
16
  LoaderArrayOptionsType,
16
- LoaderArrayBatchType
17
+ LoaderArrayBatchType,
18
+ TransformBatches
17
19
  } from '@loaders.gl/loader-utils';
18
20
  import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
19
21
  import {isLoaderObject} from '../loader-utils/normalize-loader';
@@ -80,7 +82,7 @@ export async function parseInBatches(
80
82
  // Signature: parseInBatches(data, options, url) - Uses registered loaders
81
83
  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
82
84
  context = undefined; // context not supported in short signature
83
- options = loaders as LoaderOptions;
85
+ options = loaders as unknown as LoaderOptions;
84
86
  loaders = undefined;
85
87
  }
86
88
 
@@ -92,21 +94,21 @@ export async function parseInBatches(
92
94
 
93
95
  // Chooses a loader and normalizes it
94
96
  // Note - only uses URL and contentType for streams and iterator inputs
95
- const loader = await selectLoader(data as ArrayBuffer, loaders, options);
97
+ const loader = await selectLoader(data, loaders, options);
96
98
  // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
97
99
  if (!loader) {
98
100
  return [];
99
101
  }
100
102
 
101
103
  // Normalize options
102
- options = normalizeOptions(options, loader, loaderArray, url);
104
+ const strictOptions = normalizeOptions(options, loader, loaderArray, url);
103
105
  context = getLoaderContext(
104
106
  {url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},
105
- options,
107
+ strictOptions,
106
108
  context || null
107
109
  );
108
110
 
109
- return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);
111
+ return await parseWithLoaderInBatches(loader as LoaderWithParser, data, strictOptions, context);
110
112
  }
111
113
 
112
114
  /**
@@ -115,13 +117,13 @@ export async function parseInBatches(
115
117
  async function parseWithLoaderInBatches(
116
118
  loader: LoaderWithParser,
117
119
  data: BatchableDataType,
118
- options: LoaderOptions,
120
+ options: StrictLoaderOptions,
119
121
  context: LoaderContext
120
122
  ): Promise<AsyncIterable<unknown>> {
121
123
  const outputIterator = await parseToOutputIterator(loader, data, options, context);
122
124
 
123
125
  // Generate metadata batch if requested
124
- if (!options.metadata) {
126
+ if (!options?.core?.metadata) {
125
127
  return outputIterator;
126
128
  }
127
129
 
@@ -155,14 +157,17 @@ async function parseWithLoaderInBatches(
155
157
  async function parseToOutputIterator(
156
158
  loader: LoaderWithParser,
157
159
  data: BatchableDataType,
158
- options: LoaderOptions,
160
+ options: StrictLoaderOptions,
159
161
  context: LoaderContext
160
162
  ): Promise<AsyncIterable<unknown>> {
161
163
  // Get an iterator from the input
162
164
  const inputIterator = await getAsyncIterableFromData(data, options);
163
165
 
164
166
  // Apply any iterator transforms (options.transforms)
165
- const transformedIterator = await applyInputTransforms(inputIterator, options?.transforms || []);
167
+ const transformedIterator = await applyInputTransforms(
168
+ inputIterator,
169
+ options?.core?.transforms || []
170
+ );
166
171
 
167
172
  // If loader supports parseInBatches, we are done
168
173
  if (loader.parseInBatches) {
@@ -174,9 +179,11 @@ async function parseToOutputIterator(
174
179
 
175
180
  // Fallback: load atomically using `parse` concatenating input iterator into single chunk
176
181
  async function* parseChunkInBatches(
177
- transformedIterator: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,
182
+ transformedIterator:
183
+ | Iterable<ArrayBufferLike | ArrayBufferView>
184
+ | AsyncIterable<ArrayBufferLike | ArrayBufferView>,
178
185
  loader: Loader,
179
- options: LoaderOptions,
186
+ options: StrictLoaderOptions,
180
187
  context: LoaderContext
181
188
  ): AsyncIterable<Batch> {
182
189
  const arrayBuffer = await concatenateArrayBuffersAsync(transformedIterator);
@@ -185,7 +192,7 @@ async function* parseChunkInBatches(
185
192
  arrayBuffer,
186
193
  loader,
187
194
  // TODO - Hack: supply loaders MIME type to ensure we match it
188
- {...options, mimeType: loader.mimeTypes[0]},
195
+ {...options, core: {...options?.core, mimeType: loader.mimeTypes[0]}},
189
196
  context
190
197
  );
191
198
 
@@ -215,19 +222,19 @@ function convertDataToBatch(parsedData: unknown, loader: Loader): Batch {
215
222
  return batch;
216
223
  }
217
224
 
218
- type TransformBatches = (
219
- asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
220
- ) => AsyncIterable<ArrayBuffer>;
221
-
222
225
  /**
223
226
  * Create an iterator chain with any transform iterators (crypto, decompression)
224
227
  * @param inputIterator
225
228
  * @param options
226
229
  */
227
230
  async function applyInputTransforms(
228
- inputIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
231
+ inputIterator:
232
+ | AsyncIterable<ArrayBufferLike | ArrayBufferView>
233
+ | Iterable<ArrayBufferLike | ArrayBufferView>,
229
234
  transforms: TransformBatches[] = []
230
- ): Promise<AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>> {
235
+ ): Promise<
236
+ AsyncIterable<ArrayBufferLike | ArrayBufferView> | Iterable<ArrayBufferLike | ArrayBufferView>
237
+ > {
231
238
  let iteratorChain = inputIterator;
232
239
  for await (const transformBatches of transforms) {
233
240
  iteratorChain = transformBatches(iteratorChain);
@@ -11,7 +11,8 @@ import type {
11
11
  LoaderOptionsType,
12
12
  LoaderReturnType,
13
13
  LoaderArrayOptionsType,
14
- LoaderArrayReturnType
14
+ LoaderArrayReturnType,
15
+ StrictLoaderOptions
15
16
  } from '@loaders.gl/loader-utils';
16
17
  import {selectLoaderSync} from './select-loader';
17
18
  import {isLoaderObject} from '../loader-utils/normalize-loader';
@@ -84,7 +85,7 @@ export function parseSync(
84
85
  }
85
86
 
86
87
  // Normalize options
87
- options = normalizeOptions(options, loader, candidateLoaders as Loader[] | undefined);
88
+ const strictOptions = normalizeOptions(options, loader, candidateLoaders as Loader[] | undefined);
88
89
 
89
90
  // Extract a url for auto detection
90
91
  const url = getResourceUrl(data);
@@ -94,18 +95,18 @@ export function parseSync(
94
95
  };
95
96
  context = getLoaderContext(
96
97
  {url, _parseSync: parse, _parse: parse, loaders: loaders as Loader[]},
97
- options,
98
+ strictOptions,
98
99
  context || null
99
100
  );
100
101
 
101
- return parseWithLoaderSync(loader as LoaderWithParser, data, options, context);
102
+ return parseWithLoaderSync(loader as LoaderWithParser, data, strictOptions, context);
102
103
  }
103
104
 
104
105
  // TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator
105
106
  function parseWithLoaderSync(
106
107
  loader: LoaderWithParser,
107
108
  data: SyncDataType,
108
- options: LoaderOptions,
109
+ options: StrictLoaderOptions,
109
110
  context: LoaderContext
110
111
  ) {
111
112
  data = getArrayBufferOrStringFromDataSync(data, loader, options);
@@ -11,12 +11,17 @@ import type {
11
11
  LoaderOptionsType,
12
12
  LoaderReturnType,
13
13
  LoaderArrayOptionsType,
14
- LoaderArrayReturnType
14
+ LoaderArrayReturnType,
15
+ StrictLoaderOptions
16
+ } from '@loaders.gl/loader-utils';
17
+ import {
18
+ parseWithWorker,
19
+ canParseWithWorker,
20
+ mergeOptions,
21
+ isResponse
15
22
  } from '@loaders.gl/loader-utils';
16
- import {parseWithWorker, canParseWithWorker, mergeOptions} from '@loaders.gl/loader-utils';
17
23
  import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
18
24
  import {isLoaderObject} from '../loader-utils/normalize-loader';
19
- import {isResponse} from '../../javascript-utils/is-type';
20
25
  import {normalizeOptions} from '../loader-utils/option-utils';
21
26
  import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
22
27
  import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
@@ -100,18 +105,18 @@ export async function parse(
100
105
  }
101
106
 
102
107
  // Normalize options
103
- // @ts-expect-error
104
- options = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
108
+ // @ts-expect-error candidateLoaders
109
+ const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
105
110
 
106
111
  // Get a context (if already present, will be unchanged)
107
112
  context = getLoaderContext(
108
113
  // @ts-expect-error
109
114
  {url, _parse: parse, loaders: candidateLoaders},
110
- options,
115
+ strictOptions,
111
116
  context || null
112
117
  );
113
118
 
114
- return await parseWithLoader(loader, data, options, context);
119
+ return await parseWithLoader(loader, data, strictOptions, context);
115
120
  }
116
121
 
117
122
  // TODO: support progress and abort
@@ -119,7 +124,7 @@ export async function parse(
119
124
  async function parseWithLoader(
120
125
  loader: Loader,
121
126
  data,
122
- options: LoaderOptions,
127
+ options: StrictLoaderOptions,
123
128
  context: LoaderContext
124
129
  ): Promise<unknown> {
125
130
  validateWorkerVersion(loader);
@@ -128,9 +133,8 @@ async function parseWithLoader(
128
133
 
129
134
  if (isResponse(data)) {
130
135
  // Serialize to support passing the response to web worker
131
- const response = data as Response;
132
- const {ok, redirected, status, statusText, type, url} = response;
133
- const headers = Object.fromEntries(response.headers.entries());
136
+ const {ok, redirected, status, statusText, type, url} = data;
137
+ const headers = Object.fromEntries(data.headers.entries());
134
138
  // @ts-expect-error TODO - fix this
135
139
  context.response = {headers, ok, redirected, status, statusText, type, url};
136
140
  }
@@ -2,14 +2,21 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {LoaderContext, LoaderOptions, Loader} from '@loaders.gl/loader-utils';
6
- import {compareArrayBuffers, path, log} from '@loaders.gl/loader-utils';
5
+ import type {LoaderContext, LoaderOptions, Loader, DataType} from '@loaders.gl/loader-utils';
6
+ import {
7
+ compareArrayBuffers,
8
+ path,
9
+ log,
10
+ isBlob,
11
+ ensureArrayBuffer,
12
+ isArrayBufferLike
13
+ } from '@loaders.gl/loader-utils';
7
14
  import {TypedArray} from '@loaders.gl/schema';
8
15
  import {normalizeLoader} from '../loader-utils/normalize-loader';
16
+ import {normalizeLoaderOptions} from '../loader-utils/option-utils';
9
17
  import {getResourceUrl, getResourceMIMEType} from '../utils/resource-utils';
10
18
  import {compareMIMETypes} from '../utils/mime-type-utils';
11
19
  import {getRegisteredLoaders} from './register-loaders';
12
- import {isBlob} from '../../javascript-utils/is-type';
13
20
  import {stripQueryString} from '../utils/url-utils';
14
21
 
15
22
  const EXT_PATTERN = /\.([^.]+)$/;
@@ -28,7 +35,7 @@ const EXT_PATTERN = /\.([^.]+)$/;
28
35
  * @param context used internally, applications should not provide this parameter
29
36
  */
30
37
  export async function selectLoader(
31
- data: Response | Blob | ArrayBuffer | string,
38
+ data: DataType,
32
39
  loaders: Loader[] | Loader = [],
33
40
  options?: LoaderOptions,
34
41
  context?: LoaderContext
@@ -37,8 +44,29 @@ export async function selectLoader(
37
44
  return null;
38
45
  }
39
46
 
47
+ const normalizedOptions = normalizeLoaderOptions(options || {});
48
+ normalizedOptions.core ||= {};
49
+
50
+ if (data instanceof Response && mayContainText(data)) {
51
+ const text = await data.clone().text();
52
+ const textLoader = selectLoaderSync(
53
+ text,
54
+ loaders,
55
+ {...normalizedOptions, core: {...normalizedOptions.core, nothrow: true}},
56
+ context
57
+ );
58
+ if (textLoader) {
59
+ return textLoader;
60
+ }
61
+ }
62
+
40
63
  // First make a sync attempt, disabling exceptions
41
- let loader = selectLoaderSync(data, loaders, {...options, nothrow: true}, context);
64
+ let loader = selectLoaderSync(
65
+ data,
66
+ loaders,
67
+ {...normalizedOptions, core: {...normalizedOptions.core, nothrow: true}},
68
+ context
69
+ );
42
70
  if (loader) {
43
71
  return loader;
44
72
  }
@@ -46,18 +74,31 @@ export async function selectLoader(
46
74
  // For Blobs and Files, try to asynchronously read a small initial slice and test again with that
47
75
  // to see if we can detect by initial content
48
76
  if (isBlob(data)) {
49
- data = await (data as Blob).slice(0, 10).arrayBuffer();
50
- loader = selectLoaderSync(data, loaders, options, context);
77
+ data = await data.slice(0, 10).arrayBuffer();
78
+ loader = selectLoaderSync(data, loaders, normalizedOptions, context);
79
+ }
80
+
81
+ if (!loader && data instanceof Response && mayContainText(data)) {
82
+ const text = await data.clone().text();
83
+ loader = selectLoaderSync(text, loaders, normalizedOptions, context);
51
84
  }
52
85
 
53
86
  // no loader available
54
- if (!loader && !options?.nothrow) {
87
+ if (!loader && !normalizedOptions.core.nothrow) {
55
88
  throw new Error(getNoValidLoaderMessage(data));
56
89
  }
57
90
 
58
91
  return loader;
59
92
  }
60
93
 
94
+ function mayContainText(response: Response): boolean {
95
+ const mimeType = getResourceMIMEType(response);
96
+ return Boolean(
97
+ mimeType &&
98
+ (mimeType.startsWith('text/') || mimeType === 'application/json' || mimeType.endsWith('+json'))
99
+ );
100
+ }
101
+
61
102
  /**
62
103
  * Find a loader that matches file extension and/or initial file content
63
104
  * Search the loaders array argument for a loader that matches url extension or initial data
@@ -68,7 +109,7 @@ export async function selectLoader(
68
109
  * @param context used internally, applications should not provide this parameter
69
110
  */
70
111
  export function selectLoaderSync(
71
- data: Response | Blob | ArrayBuffer | string,
112
+ data: DataType,
72
113
  loaders: Loader[] | Loader = [],
73
114
  options?: LoaderOptions,
74
115
  context?: LoaderContext
@@ -77,6 +118,9 @@ export function selectLoaderSync(
77
118
  return null;
78
119
  }
79
120
 
121
+ const normalizedOptions = normalizeLoaderOptions(options || {});
122
+ normalizedOptions.core ||= {};
123
+
80
124
  // eslint-disable-next-line complexity
81
125
  // if only a single loader was provided (not as array), force its use
82
126
  // TODO - Should this behavior be kept and documented?
@@ -92,17 +136,17 @@ export function selectLoaderSync(
92
136
  candidateLoaders = candidateLoaders.concat(loaders);
93
137
  }
94
138
  // Then fall back to registered loaders
95
- if (!options?.ignoreRegisteredLoaders) {
139
+ if (!normalizedOptions.core.ignoreRegisteredLoaders) {
96
140
  candidateLoaders.push(...getRegisteredLoaders());
97
141
  }
98
142
 
99
143
  // TODO - remove support for legacy loaders
100
144
  normalizeLoaders(candidateLoaders);
101
145
 
102
- const loader = selectLoaderInternal(data, candidateLoaders, options, context);
146
+ const loader = selectLoaderInternal(data, candidateLoaders, normalizedOptions, context);
103
147
 
104
148
  // no loader available
105
- if (!loader && !options?.nothrow) {
149
+ if (!loader && !normalizedOptions.core.nothrow) {
106
150
  throw new Error(getNoValidLoaderMessage(data));
107
151
  }
108
152
 
@@ -112,7 +156,7 @@ export function selectLoaderSync(
112
156
  /** Implements loaders selection logic */
113
157
  // eslint-disable-next-line complexity
114
158
  function selectLoaderInternal(
115
- data: Response | Blob | ArrayBuffer | string,
159
+ data: DataType,
116
160
  loaders: Loader[],
117
161
  options?: LoaderOptions,
118
162
  context?: LoaderContext
@@ -126,9 +170,9 @@ function selectLoaderInternal(
126
170
  let reason: string = '';
127
171
 
128
172
  // if options.mimeType is supplied, it takes precedence
129
- if (options?.mimeType) {
130
- loader = findLoaderByMIMEType(loaders, options?.mimeType);
131
- reason = `match forced by supplied MIME type ${options?.mimeType}`;
173
+ if (options?.core?.mimeType) {
174
+ loader = findLoaderByMIMEType(loaders, options?.core?.mimeType);
175
+ reason = `match forced by supplied MIME type ${options?.core?.mimeType}`;
132
176
  }
133
177
 
134
178
  // Look up loader by url
@@ -146,8 +190,8 @@ function selectLoaderInternal(
146
190
  reason = reason || (loader ? `matched initial data ${getFirstCharacters(data)}` : '');
147
191
 
148
192
  // Look up loader by fallback mime type
149
- if (options?.fallbackMimeType) {
150
- loader = loader || findLoaderByMIMEType(loaders, options?.fallbackMimeType);
193
+ if (options?.core?.fallbackMimeType) {
194
+ loader = loader || findLoaderByMIMEType(loaders, options?.core?.fallbackMimeType);
151
195
  reason = reason || (loader ? `matched fallback MIME type ${type}` : '');
152
196
  }
153
197
 
@@ -171,7 +215,7 @@ function validHTTPResponse(data: unknown): boolean {
171
215
  }
172
216
 
173
217
  /** Generate a helpful message to help explain why loader selection failed. */
174
- function getNoValidLoaderMessage(data: string | ArrayBuffer | Response | Blob): string {
218
+ function getNoValidLoaderMessage(data: DataType): string {
175
219
  const url = getResourceUrl(data);
176
220
  const type = getResourceMIMEType(data);
177
221
 
@@ -264,23 +308,23 @@ function testDataAgainstText(data: string, loader: Loader): boolean {
264
308
  return tests.some((test) => data.startsWith(test as string));
265
309
  }
266
310
 
267
- function testDataAgainstBinary(data: ArrayBuffer, byteOffset: number, loader: Loader): boolean {
311
+ function testDataAgainstBinary(data: ArrayBufferLike, byteOffset: number, loader: Loader): boolean {
268
312
  const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
269
313
  return tests.some((test) => testBinary(data, byteOffset, loader, test));
270
314
  }
271
315
 
272
316
  function testBinary(
273
- data: ArrayBuffer,
317
+ data: ArrayBufferLike,
274
318
  byteOffset: number,
275
319
  loader: Loader,
276
320
  test?: ArrayBuffer | string | ((b: ArrayBuffer) => boolean)
277
321
  ): boolean {
278
- if (test instanceof ArrayBuffer) {
322
+ if (isArrayBufferLike(test)) {
279
323
  return compareArrayBuffers(test, data, test.byteLength);
280
324
  }
281
325
  switch (typeof test) {
282
326
  case 'function':
283
- return test(data);
327
+ return test(ensureArrayBuffer(data));
284
328
 
285
329
  case 'string':
286
330
  // Magic bytes check: If `test` is a string, check if binary data starts with that strings
@@ -305,7 +349,7 @@ function getFirstCharacters(data: string | ArrayBuffer | TypedArray, length: num
305
349
  return '';
306
350
  }
307
351
 
308
- function getMagicString(arrayBuffer: ArrayBuffer, byteOffset: number, length: number): string {
352
+ function getMagicString(arrayBuffer: ArrayBufferLike, byteOffset: number, length: number): string {
309
353
  if (arrayBuffer.byteLength < byteOffset + length) {
310
354
  return '';
311
355
  }