@loaders.gl/core 4.4.5 → 5.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/LICENSE +14 -3
  2. package/dist/dist.dev.js +800 -271
  3. package/dist/dist.min.js +4 -3
  4. package/dist/index.cjs +2191 -296
  5. package/dist/index.cjs.map +3 -4
  6. package/dist/index.d.ts +8 -2
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +7 -1
  9. package/dist/index.js.map +1 -1
  10. package/dist/iterators/make-iterator/make-stream-iterator.js +48 -9
  11. package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -1
  12. package/dist/lib/api/core-api.d.ts +6 -0
  13. package/dist/lib/api/core-api.d.ts.map +1 -0
  14. package/dist/lib/api/core-api.js +23 -0
  15. package/dist/lib/api/core-api.js.map +1 -0
  16. package/dist/lib/api/create-data-source.d.ts +2 -2
  17. package/dist/lib/api/create-data-source.d.ts.map +1 -1
  18. package/dist/lib/api/create-data-source.js +15 -14
  19. package/dist/lib/api/create-data-source.js.map +1 -1
  20. package/dist/lib/api/load-in-batches.d.ts +4 -4
  21. package/dist/lib/api/load-in-batches.d.ts.map +1 -1
  22. package/dist/lib/api/load-in-batches.js +1 -1
  23. package/dist/lib/api/load-in-batches.js.map +1 -1
  24. package/dist/lib/api/load-loader.d.ts +8 -0
  25. package/dist/lib/api/load-loader.d.ts.map +1 -0
  26. package/dist/lib/api/load-loader.js +110 -0
  27. package/dist/lib/api/load-loader.js.map +1 -0
  28. package/dist/lib/api/load.d.ts +2 -2
  29. package/dist/lib/api/load.d.ts.map +1 -1
  30. package/dist/lib/api/load.js +66 -3
  31. package/dist/lib/api/load.js.map +1 -1
  32. package/dist/lib/api/parse-file.d.ts +15 -0
  33. package/dist/lib/api/parse-file.d.ts.map +1 -0
  34. package/dist/lib/api/parse-file.js +70 -0
  35. package/dist/lib/api/parse-file.js.map +1 -0
  36. package/dist/lib/api/parse-in-batches.d.ts +2 -2
  37. package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
  38. package/dist/lib/api/parse-in-batches.js +6 -4
  39. package/dist/lib/api/parse-in-batches.js.map +1 -1
  40. package/dist/lib/api/parse-sync.d.ts +2 -2
  41. package/dist/lib/api/parse-sync.d.ts.map +1 -1
  42. package/dist/lib/api/parse-sync.js +10 -1
  43. package/dist/lib/api/parse-sync.js.map +1 -1
  44. package/dist/lib/api/parse.d.ts +2 -2
  45. package/dist/lib/api/parse.d.ts.map +1 -1
  46. package/dist/lib/api/parse.js +28 -16
  47. package/dist/lib/api/parse.js.map +1 -1
  48. package/dist/lib/api/preload.d.ts +11 -0
  49. package/dist/lib/api/preload.d.ts.map +1 -0
  50. package/dist/lib/api/preload.js +14 -0
  51. package/dist/lib/api/preload.js.map +1 -0
  52. package/dist/lib/api/register-loaders.js +1 -1
  53. package/dist/lib/api/register-loaders.js.map +1 -1
  54. package/dist/lib/api/select-loader.d.ts.map +1 -1
  55. package/dist/lib/api/select-loader.js +48 -5
  56. package/dist/lib/api/select-loader.js.map +1 -1
  57. package/dist/lib/api/select-source.d.ts +3 -3
  58. package/dist/lib/api/select-source.d.ts.map +1 -1
  59. package/dist/lib/api/select-source.js +2 -5
  60. package/dist/lib/api/select-source.js.map +1 -1
  61. package/dist/lib/fetch/fetch-error-message.js +1 -1
  62. package/dist/lib/fetch/fetch-error-message.js.map +1 -1
  63. package/dist/lib/init.js +1 -1
  64. package/dist/lib/init.js.map +1 -1
  65. package/dist/lib/loader-utils/check-errors.js +1 -1
  66. package/dist/lib/loader-utils/check-errors.js.map +1 -1
  67. package/dist/lib/loader-utils/get-data.d.ts +4 -0
  68. package/dist/lib/loader-utils/get-data.d.ts.map +1 -1
  69. package/dist/lib/loader-utils/get-data.js +26 -0
  70. package/dist/lib/loader-utils/get-data.js.map +1 -1
  71. package/dist/lib/loader-utils/get-fetch-function.d.ts +1 -1
  72. package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
  73. package/dist/lib/loader-utils/get-fetch-function.js +1 -1
  74. package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
  75. package/dist/lib/loader-utils/loader-context.d.ts +1 -1
  76. package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
  77. package/dist/lib/loader-utils/loader-context.js +3 -1
  78. package/dist/lib/loader-utils/loader-context.js.map +1 -1
  79. package/dist/lib/loader-utils/loggers.d.ts +19 -0
  80. package/dist/lib/loader-utils/loggers.d.ts.map +1 -1
  81. package/dist/lib/loader-utils/loggers.js +28 -2
  82. package/dist/lib/loader-utils/loggers.js.map +1 -1
  83. package/dist/lib/loader-utils/option-defaults.d.ts +1 -0
  84. package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
  85. package/dist/lib/loader-utils/option-defaults.js +2 -0
  86. package/dist/lib/loader-utils/option-defaults.js.map +1 -1
  87. package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
  88. package/dist/lib/loader-utils/option-utils.js +50 -2
  89. package/dist/lib/loader-utils/option-utils.js.map +1 -1
  90. package/dist/lib/utils/response-utils.js +3 -3
  91. package/dist/lib/utils/response-utils.js.map +1 -1
  92. package/dist/null-loader.d.ts.map +1 -1
  93. package/dist/null-loader.js +1 -1
  94. package/dist/null-loader.js.map +1 -1
  95. package/dist/null-worker-node.js +199 -36
  96. package/dist/null-worker.js +199 -36
  97. package/package.json +6 -6
  98. package/src/index.ts +10 -1
  99. package/src/iterators/make-iterator/make-stream-iterator.ts +47 -9
  100. package/src/lib/api/core-api.ts +25 -0
  101. package/src/lib/api/create-data-source.ts +24 -24
  102. package/src/lib/api/load-in-batches.ts +16 -11
  103. package/src/lib/api/load-loader.ts +152 -0
  104. package/src/lib/api/load.ts +98 -5
  105. package/src/lib/api/parse-file.ts +143 -0
  106. package/src/lib/api/parse-in-batches.ts +13 -5
  107. package/src/lib/api/parse-sync.ts +27 -2
  108. package/src/lib/api/parse.ts +49 -18
  109. package/src/lib/api/preload.ts +31 -0
  110. package/src/lib/api/register-loaders.ts +1 -1
  111. package/src/lib/api/select-loader.ts +57 -5
  112. package/src/lib/api/select-source.ts +7 -10
  113. package/src/lib/fetch/fetch-error-message.ts +1 -1
  114. package/src/lib/loader-utils/check-errors.ts +1 -1
  115. package/src/lib/loader-utils/get-data.ts +36 -0
  116. package/src/lib/loader-utils/get-fetch-function.ts +3 -2
  117. package/src/lib/loader-utils/loader-context.ts +5 -2
  118. package/src/lib/loader-utils/loggers.ts +28 -2
  119. package/src/lib/loader-utils/option-defaults.ts +2 -0
  120. package/src/lib/loader-utils/option-utils.ts +59 -2
  121. package/src/lib/utils/response-utils.ts +3 -3
  122. package/src/null-loader.ts +3 -3
@@ -10,8 +10,10 @@ import type {
10
10
  StrictLoaderOptions,
11
11
  LoaderOptions,
12
12
  LoaderContext,
13
+ LoaderOptionsWithShape,
13
14
  BatchableDataType,
14
15
  LoaderOptionsType,
16
+ LoaderShapeType,
15
17
  LoaderBatchType,
16
18
  LoaderArrayOptionsType,
17
19
  LoaderArrayBatchType,
@@ -23,6 +25,7 @@ import {normalizeOptions} from '../loader-utils/option-utils';
23
25
  import {getLoaderContext} from '../loader-utils/loader-context';
24
26
  import {getAsyncIterableFromData} from '../loader-utils/get-data';
25
27
  import {getResourceUrl} from '../utils/resource-utils';
28
+ import {getLoaderImplementation} from './load-loader';
26
29
  import {selectLoader} from './select-loader';
27
30
 
28
31
  // Ensure `parse` is available in context if loader falls back to `parse`
@@ -33,7 +36,10 @@ import {parse} from './parse';
33
36
  */
34
37
  export async function parseInBatches<
35
38
  LoaderT extends Loader,
36
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
39
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
40
+ LoaderOptionsType<LoaderT>,
41
+ LoaderShapeType<LoaderT>
42
+ >
37
43
  >(
38
44
  data: BatchableDataType,
39
45
  loader: LoaderT,
@@ -108,13 +114,15 @@ export async function parseInBatches(
108
114
  context || null
109
115
  );
110
116
 
111
- return await parseWithLoaderInBatches(loader as LoaderWithParser, data, strictOptions, context);
117
+ const loaderWithParser = await getLoaderImplementation(loader, strictOptions, context.url);
118
+ return await parseWithLoaderInBatches(loader, loaderWithParser, data, strictOptions, context);
112
119
  }
113
120
 
114
121
  /**
115
122
  * Loader has been selected and context has been prepared, see if we need to emit a metadata batch
116
123
  */
117
124
  async function parseWithLoaderInBatches(
125
+ originalLoader: Loader,
118
126
  loader: LoaderWithParser,
119
127
  data: BatchableDataType,
120
128
  options: StrictLoaderOptions,
@@ -131,7 +139,7 @@ async function parseWithLoaderInBatches(
131
139
  shape: 'metadata',
132
140
  batchType: 'metadata',
133
141
  metadata: {
134
- _loader: loader,
142
+ _loader: originalLoader,
135
143
  _context: context
136
144
  },
137
145
  // Populate with some default fields to avoid crashing
@@ -182,7 +190,7 @@ async function* parseChunkInBatches(
182
190
  transformedIterator:
183
191
  | Iterable<ArrayBufferLike | ArrayBufferView>
184
192
  | AsyncIterable<ArrayBufferLike | ArrayBufferView>,
185
- loader: Loader,
193
+ loader: LoaderWithParser,
186
194
  options: StrictLoaderOptions,
187
195
  context: LoaderContext
188
196
  ): AsyncIterable<Batch> {
@@ -207,7 +215,7 @@ async function* parseChunkInBatches(
207
215
  * @todo run through batch builder to apply options etc...
208
216
  */
209
217
  function convertDataToBatch(parsedData: unknown, loader: Loader): Batch {
210
- // prettier-ignore
218
+ // biome-ignore format: preserve intentional fixture layout
211
219
  const batch: Batch = isTable(parsedData)
212
220
  ? makeBatchFromTable(parsedData)
213
221
  : {
@@ -7,19 +7,23 @@ import type {
7
7
  LoaderWithParser,
8
8
  LoaderOptions,
9
9
  LoaderContext,
10
+ LoaderOptionsWithShape,
10
11
  SyncDataType,
11
12
  LoaderOptionsType,
13
+ LoaderShapeType,
12
14
  LoaderReturnType,
13
15
  LoaderArrayOptionsType,
14
16
  LoaderArrayReturnType,
15
17
  StrictLoaderOptions
16
18
  } from '@loaders.gl/loader-utils';
19
+ import {isSourceLoader} from '@loaders.gl/loader-utils';
17
20
  import {selectLoaderSync} from './select-loader';
18
21
  import {isLoaderObject} from '../loader-utils/normalize-loader';
19
22
  import {normalizeOptions} from '../loader-utils/option-utils';
20
23
  import {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';
21
24
  import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
22
25
  import {getResourceUrl} from '../utils/resource-utils';
26
+ import {getLoaderImplementationSync} from './load-loader';
23
27
 
24
28
  // OVERLOADS
25
29
 
@@ -28,7 +32,10 @@ import {getResourceUrl} from '../utils/resource-utils';
28
32
  */
29
33
  export function parseSync<
30
34
  LoaderT extends Loader,
31
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
35
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
36
+ LoaderOptionsType<LoaderT>,
37
+ LoaderShapeType<LoaderT>
38
+ >
32
39
  >(
33
40
  data: SyncDataType,
34
41
  loader: LoaderT,
@@ -84,6 +91,12 @@ export function parseSync(
84
91
  return null;
85
92
  }
86
93
 
94
+ if (isSourceLoader(loader)) {
95
+ throw new Error(
96
+ `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parseSync().`
97
+ );
98
+ }
99
+
87
100
  // Normalize options
88
101
  const strictOptions = normalizeOptions(options, loader, candidateLoaders as Loader[] | undefined);
89
102
 
@@ -99,7 +112,13 @@ export function parseSync(
99
112
  context || null
100
113
  );
101
114
 
102
- return parseWithLoaderSync(loader as LoaderWithParser, data, strictOptions, context);
115
+ const loaderWithParser = getLoaderImplementationSync(loader);
116
+ return parseWithLoaderSync(
117
+ loaderWithParser || (loader as LoaderWithParser),
118
+ data,
119
+ strictOptions,
120
+ context
121
+ );
103
122
  }
104
123
 
105
124
  // TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator
@@ -119,6 +138,12 @@ function parseWithLoaderSync(
119
138
  return loader.parseSync(data, options, context); // , loader);
120
139
  }
121
140
 
141
+ if (loader.preload) {
142
+ throw new Error(
143
+ `${loader.name} loader: 'parseSync' requires a parser-bearing loader. Import the loader implementation directly, or call preload(loader) before parseSync(). ${context.url || ''}`
144
+ );
145
+ }
146
+
122
147
  // TBD - If synchronous parser not available, return null
123
148
  throw new Error(
124
149
  `${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${
@@ -6,9 +6,11 @@ import type {
6
6
  Loader,
7
7
  LoaderContext,
8
8
  LoaderOptions,
9
+ LoaderOptionsWithShape,
9
10
  DataType,
10
11
  LoaderWithParser,
11
12
  LoaderOptionsType,
13
+ LoaderShapeType,
12
14
  LoaderReturnType,
13
15
  LoaderArrayOptionsType,
14
16
  LoaderArrayReturnType,
@@ -18,14 +20,18 @@ import {
18
20
  parseWithWorker,
19
21
  canParseWithWorker,
20
22
  mergeOptions,
21
- isResponse
23
+ isResponse,
24
+ isBlob,
25
+ isSourceLoader
22
26
  } from '@loaders.gl/loader-utils';
23
- import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
27
+ import {validateWorkerVersion} from '@loaders.gl/worker-utils';
24
28
  import {isLoaderObject} from '../loader-utils/normalize-loader';
25
29
  import {normalizeOptions} from '../loader-utils/option-utils';
26
30
  import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
31
+ import {getArrayBufferFromData} from '../loader-utils/get-data';
27
32
  import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
28
33
  import {getResourceUrl} from '../utils/resource-utils';
34
+ import {getLoaderImplementation} from './load-loader';
29
35
  import {selectLoader} from './select-loader';
30
36
 
31
37
  // type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T
@@ -35,7 +41,10 @@ import {selectLoader} from './select-loader';
35
41
  */
36
42
  export async function parse<
37
43
  LoaderT extends Loader,
38
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
44
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
45
+ LoaderOptionsType<LoaderT>,
46
+ LoaderShapeType<LoaderT>
47
+ >
39
48
  >(
40
49
  data: DataType | Promise<DataType>,
41
50
  loader: LoaderT,
@@ -104,6 +113,12 @@ export async function parse(
104
113
  return null;
105
114
  }
106
115
 
116
+ if (isSourceLoader(loader)) {
117
+ throw new Error(
118
+ `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parse().`
119
+ );
120
+ }
121
+
107
122
  // Normalize options
108
123
  // @ts-expect-error candidateLoaders
109
124
  const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
@@ -139,31 +154,47 @@ async function parseWithLoader(
139
154
  context.response = {headers, ok, redirected, status, statusText, type, url};
140
155
  }
141
156
 
142
- data = await getArrayBufferOrStringFromData(data, loader, options);
157
+ const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
143
158
 
144
- const loaderWithParser = loader as LoaderWithParser;
159
+ if (canParseWithWorker(loaderWithParser, options)) {
160
+ data = await getArrayBufferFromData(data, options);
161
+ return await parseWithWorker(loaderWithParser, data, options, context, parse);
162
+ }
145
163
 
146
- // First check for synchronous text parser, wrap results in promises
147
- if (loaderWithParser.parseTextSync && typeof data === 'string') {
148
- return loaderWithParser.parseTextSync(data, options, context);
164
+ if (isBlob(data) && loaderWithParser.parseBlob) {
165
+ return await loaderWithParser.parseBlob(data, options, context);
149
166
  }
150
167
 
151
- // If we have a workerUrl and the loader can parse the given options efficiently in a worker
152
- if (canParseWithWorker(loader, options)) {
153
- return await parseWithWorker(loader, data, options, context, parse);
168
+ data = await getArrayBufferOrStringFromData(data, loader, options);
169
+
170
+ return await parseWithLoaderImplementation(loaderWithParser, data, options, context);
171
+ }
172
+
173
+ async function parseWithLoaderImplementation(
174
+ loader: LoaderWithParser,
175
+ data: string | ArrayBuffer,
176
+ options: StrictLoaderOptions,
177
+ context: LoaderContext
178
+ ): Promise<unknown> {
179
+ if (loader.parseText && typeof data === 'string') {
180
+ return await loader.parseText(data, options, context);
154
181
  }
155
182
 
156
- // Check for asynchronous parser
157
- if (loaderWithParser.parseText && typeof data === 'string') {
158
- return await loaderWithParser.parseText(data, options, context);
183
+ // Fall back to synchronous text parser, wrap results in promises
184
+ if (loader.parseTextSync && typeof data === 'string') {
185
+ return loader.parseTextSync(data, options, context);
159
186
  }
160
187
 
161
- if (loaderWithParser.parse) {
162
- return await loaderWithParser.parse(data, options, context);
188
+ if (loader.parse) {
189
+ return await loader.parse(data as ArrayBuffer, options, context);
163
190
  }
164
191
 
165
- // This should not happen, all sync loaders should also offer `parse` function
166
- assert(!loaderWithParser.parseSync);
192
+ // All parser-bearing sync loaders should also offer `parse` so async APIs can use them.
193
+ if (loader.parseSync) {
194
+ throw new Error(
195
+ `${loader.name} loader: 'parse' not available on parser-bearing sync loader. Add an async 'parse' function to this loader, or call preload(loader) before parseSync(). ${context.url || ''}`
196
+ );
197
+ }
167
198
 
168
199
  // TBD - If asynchronous parser not available, return null
169
200
  throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
@@ -0,0 +1,31 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {Loader, LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
6
+ import {getLoaderImplementation, getLoaderImplementationSync} from './load-loader';
7
+
8
+ /**
9
+ * Preloads a loader implementation so later parse/load calls do not pay the resolution cost on first use.
10
+ * May return a parser-bearing loader that also supports `parseSync`.
11
+ */
12
+ export async function preload<LoaderT extends Loader>(
13
+ loader: LoaderT,
14
+ options?: LoaderOptions,
15
+ url?: string
16
+ ): Promise<LoaderWithParser>;
17
+
18
+ export async function preload(
19
+ loader: Loader,
20
+ options?: LoaderOptions,
21
+ url?: string
22
+ ): Promise<LoaderWithParser> {
23
+ return await getLoaderImplementation(loader, options, url);
24
+ }
25
+
26
+ /**
27
+ * Returns a cached parser-bearing loader implementation if one is already available.
28
+ */
29
+ export function preloadSync(loader: Loader): LoaderWithParser | null {
30
+ return getLoaderImplementationSync(loader);
31
+ }
@@ -28,7 +28,7 @@ export function registerLoaders(loaders: Loader[] | Loader) {
28
28
 
29
29
  for (const loader of loaders) {
30
30
  const normalizedLoader = normalizeLoader(loader);
31
- if (!loaderRegistry.find((registeredLoader) => normalizedLoader === registeredLoader)) {
31
+ if (!loaderRegistry.find(registeredLoader => normalizedLoader === registeredLoader)) {
32
32
  // add to the beginning of the loaderRegistry, so the last registeredLoader get picked
33
33
  loaderRegistry.unshift(normalizedLoader);
34
34
  }
@@ -9,7 +9,8 @@ import {
9
9
  log,
10
10
  isBlob,
11
11
  ensureArrayBuffer,
12
- isArrayBufferLike
12
+ isArrayBufferLike,
13
+ isSourceLoader
13
14
  } from '@loaders.gl/loader-utils';
14
15
  import {TypedArray} from '@loaders.gl/schema';
15
16
  import {normalizeLoader} from '../loader-utils/normalize-loader';
@@ -95,7 +96,9 @@ function mayContainText(response: Response): boolean {
95
96
  const mimeType = getResourceMIMEType(response);
96
97
  return Boolean(
97
98
  mimeType &&
98
- (mimeType.startsWith('text/') || mimeType === 'application/json' || mimeType.endsWith('+json'))
99
+ (mimeType.startsWith('text/') ||
100
+ mimeType === 'application/json' ||
101
+ mimeType.endsWith('+json'))
99
102
  );
100
103
  }
101
104
 
@@ -169,12 +172,23 @@ function selectLoaderInternal(
169
172
  let loader: Loader | null = null;
170
173
  let reason: string = '';
171
174
 
175
+ // if options.mimeType is supplied, it takes precedence
176
+ const sourceType =
177
+ options?.core && 'type' in options.core ? (options.core.type as string | undefined) : undefined;
178
+ if (sourceType && sourceType !== 'auto') {
179
+ loader = findSourceLoaderByType(loaders, sourceType);
180
+ reason = loader ? `match forced by supplied source type ${sourceType}` : '';
181
+ }
182
+
172
183
  // if options.mimeType is supplied, it takes precedence
173
184
  if (options?.core?.mimeType) {
174
185
  loader = findLoaderByMIMEType(loaders, options?.core?.mimeType);
175
186
  reason = `match forced by supplied MIME type ${options?.core?.mimeType}`;
176
187
  }
177
188
 
189
+ loader = loader || findSourceLoaderByTestURL(loaders, testUrl);
190
+ reason = reason || (loader ? `matched source url ${testUrl}` : '');
191
+
178
192
  // Look up loader by url
179
193
  loader = loader || findLoaderByUrl(loaders, testUrl);
180
194
  reason = reason || (loader ? `matched url ${testUrl}` : '');
@@ -189,6 +203,11 @@ function selectLoaderInternal(
189
203
  // @ts-ignore Blob | Response
190
204
  reason = reason || (loader ? `matched initial data ${getFirstCharacters(data)}` : '');
191
205
 
206
+ if (!loader && isBlob(data)) {
207
+ loader = findSourceLoaderByTestData(loaders, data);
208
+ reason = reason || (loader ? 'matched source testData' : '');
209
+ }
210
+
192
211
  // Look up loader by fallback mime type
193
212
  if (options?.core?.fallbackMimeType) {
194
213
  loader = loader || findLoaderByMIMEType(loaders, options?.core?.fallbackMimeType);
@@ -258,9 +277,42 @@ function findLoaderByExtension(loaders: Loader[], extension: string): Loader | n
258
277
  return null;
259
278
  }
260
279
 
280
+ function findSourceLoaderByType(loaders: Loader[], type: string): Loader | null {
281
+ for (const loader of loaders) {
282
+ if (isSourceLoader(loader) && loader.type === type) {
283
+ return loader;
284
+ }
285
+ }
286
+ return null;
287
+ }
288
+
289
+ function findSourceLoaderByTestURL(loaders: Loader[], url?: string): Loader | null {
290
+ if (!url) {
291
+ return null;
292
+ }
293
+
294
+ for (const loader of loaders) {
295
+ if (isSourceLoader(loader) && loader.testURL(url)) {
296
+ return loader;
297
+ }
298
+ }
299
+ return null;
300
+ }
301
+
302
+ function findSourceLoaderByTestData(loaders: Loader[], data: Blob): Loader | null {
303
+ for (const loader of loaders) {
304
+ if (isSourceLoader(loader)) {
305
+ if (loader.testData?.(data)) {
306
+ return loader;
307
+ }
308
+ }
309
+ }
310
+ return null;
311
+ }
312
+
261
313
  function findLoaderByMIMEType(loaders: Loader[], mimeType: string): Loader | null {
262
314
  for (const loader of loaders) {
263
- if (loader.mimeTypes?.some((mimeType1) => compareMIMETypes(mimeType, mimeType1))) {
315
+ if (loader.mimeTypes?.some(mimeType1 => compareMIMETypes(mimeType, mimeType1))) {
264
316
  return loader;
265
317
  }
266
318
 
@@ -305,12 +357,12 @@ function testDataAgainstText(data: string, loader: Loader): boolean {
305
357
  }
306
358
 
307
359
  const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
308
- return tests.some((test) => data.startsWith(test as string));
360
+ return tests.some(test => data.startsWith(test as string));
309
361
  }
310
362
 
311
363
  function testDataAgainstBinary(data: ArrayBufferLike, byteOffset: number, loader: Loader): boolean {
312
364
  const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
313
- return tests.some((test) => testBinary(data, byteOffset, loader, test));
365
+ return tests.some(test => testBinary(data, byteOffset, loader, test));
314
366
  }
315
367
 
316
368
  function testBinary(
@@ -2,26 +2,23 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {Source} from '@loaders.gl/loader-utils';
5
+ import type {SourceLoader} from '@loaders.gl/loader-utils';
6
+ import {selectLoaderSync} from './select-loader';
6
7
 
7
8
  /** Guess service type from URL */
8
9
  export function selectSource(
9
10
  url: string | Blob,
10
- sources: Source[],
11
+ sources: SourceLoader[],
11
12
  options?: {
12
13
  /** Provide id of a source to select that source. Omit or provide 'auto' to test the source*/
13
14
  type?: string;
14
15
  nothrow?: boolean;
15
16
  }
16
- ): Source | null {
17
+ ): SourceLoader | null {
17
18
  const type = options?.type || 'auto';
18
- let selectedSource: Source | null = null;
19
+ let selectedSource: SourceLoader | null = null;
19
20
  if (type === 'auto') {
20
- for (const source of sources) {
21
- if (typeof url === 'string' && source.testURL && source.testURL(url)) {
22
- return source;
23
- }
24
- }
21
+ selectedSource = selectLoaderSync(url, sources, {core: {nothrow: true}}) as SourceLoader | null;
25
22
  } else {
26
23
  selectedSource = getSourceOfType(type, sources);
27
24
  }
@@ -34,7 +31,7 @@ export function selectSource(
34
31
  }
35
32
 
36
33
  /** Guess service type from URL */
37
- function getSourceOfType(type: string, sources: Source[]): Source | null {
34
+ function getSourceOfType(type: string, sources: SourceLoader[]): SourceLoader | null {
38
35
  for (const service of sources) {
39
36
  if (service.type === type) {
40
37
  return service;
@@ -17,7 +17,7 @@ export async function getErrorMessageFromResponse(response: Response): Promise<s
17
17
  } else {
18
18
  message += response.statusText;
19
19
  }
20
- } catch (error) {
20
+ } catch (_error) {
21
21
  // eslint forbids return in finally statement
22
22
  return message;
23
23
  }
@@ -14,7 +14,7 @@ export async function checkFetchResponseStatus(response: Response): Promise<void
14
14
  if (text) {
15
15
  errorMessage += `: ${getErrorText(text)}`;
16
16
  }
17
- } catch (error) {
17
+ } catch (_error) {
18
18
  // ignore error
19
19
  }
20
20
  throw new Error(errorMessage);
@@ -95,6 +95,42 @@ export async function getArrayBufferOrStringFromData(
95
95
  throw new Error(ERR_DATA);
96
96
  }
97
97
 
98
+ /**
99
+ * Resolves the provided data into an {@link ArrayBuffer}, preserving bytes for worker transfer.
100
+ */
101
+ export async function getArrayBufferFromData(
102
+ data: DataType,
103
+ options: LoaderOptions
104
+ ): Promise<ArrayBuffer> {
105
+ if (typeof data === 'string') {
106
+ return new TextEncoder().encode(data).buffer;
107
+ }
108
+
109
+ if (isArrayBufferLike(data)) {
110
+ return toArrayBuffer(toArrayBufferView(data));
111
+ }
112
+
113
+ if (isBlob(data)) {
114
+ data = await makeResponse(data);
115
+ }
116
+
117
+ if (isResponse(data)) {
118
+ await checkResponse(data);
119
+ return await data.arrayBuffer();
120
+ }
121
+
122
+ if (isReadableStream(data)) {
123
+ // @ts-expect-error TS2559 options type
124
+ data = makeIterator(data as ReadableStream, options);
125
+ }
126
+
127
+ if (isIterable(data) || isAsyncIterable(data)) {
128
+ return concatenateArrayBuffersAsync(data as AsyncIterable<ArrayBufferLike>);
129
+ }
130
+
131
+ throw new Error(ERR_DATA);
132
+ }
133
+
98
134
  /**
99
135
  * Normalizes batchable inputs into async iterables for batch parsing flows.
100
136
  * Supports synchronous iterables, async iterables, fetch responses, readable streams, and
@@ -14,7 +14,8 @@ import {getGlobalLoaderOptions} from './option-utils';
14
14
  */
15
15
  export function getFetchFunction(
16
16
  options?: LoaderOptions,
17
- context?: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>
17
+ context?: Omit<LoaderContext, 'fetch' | 'coreApi'> &
18
+ Partial<Pick<LoaderContext, 'fetch' | 'coreApi'>>
18
19
  ): FetchLike {
19
20
  const globalOptions = getGlobalLoaderOptions();
20
21
 
@@ -28,7 +29,7 @@ export function getFetchFunction(
28
29
 
29
30
  // options.fetch can be an options object
30
31
  if (isObject(fetchOption)) {
31
- return (url) => fetchFile(url, fetchOption as RequestInit);
32
+ return url => fetchFile(url, fetchOption as RequestInit);
32
33
  }
33
34
 
34
35
  // else refer to context (from parent loader) if available
@@ -6,9 +6,11 @@ import type {Loader, LoaderContext, StrictLoaderOptions} from '@loaders.gl/loade
6
6
  import {getFetchFunction} from './get-fetch-function';
7
7
  import {extractQueryString, stripQueryString} from '../utils/url-utils';
8
8
  import {path} from '@loaders.gl/loader-utils';
9
+ import {coreApi} from '../api/core-api';
9
10
 
10
11
  /** Properties for creating an updated context */
11
- type LoaderContextProps = Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>;
12
+ type LoaderContextProps = Omit<LoaderContext, 'fetch' | 'coreApi'> &
13
+ Partial<Pick<LoaderContext, 'fetch' | 'coreApi'>>;
12
14
 
13
15
  /**
14
16
  * "sub" loaders invoked by other loaders get a "context" injected on `this`
@@ -31,8 +33,9 @@ export function getLoaderContext(
31
33
  }
32
34
 
33
35
  const newContext: LoaderContext = {
36
+ ...context,
34
37
  fetch: getFetchFunction(options, context),
35
- ...context
38
+ coreApi: context.coreApi || coreApi
36
39
  };
37
40
 
38
41
  // Parse URLs so that subloaders can easily generate correct strings
@@ -6,42 +6,68 @@
6
6
 
7
7
  import {Log} from '@probe.gl/log';
8
8
 
9
+ /** Default probe.gl logger used by loaders.gl. */
9
10
  export const probeLog = new Log({id: 'loaders.gl'});
10
11
 
11
12
  type LogFunction = () => void;
12
13
 
13
- // Logs nothing
14
+ /** Logger implementation that intentionally drops all messages. */
14
15
  export class NullLog {
16
+ /** Drops a log message. */
15
17
  log(): LogFunction {
16
18
  return () => {};
17
19
  }
20
+ /** Drops an info message. */
18
21
  info(): LogFunction {
19
22
  return () => {};
20
23
  }
24
+ /** Logs one message once; no-op for the null logger. */
25
+ once(): LogFunction {
26
+ return () => {};
27
+ }
28
+ /** Drops a warning message. */
21
29
  warn(): LogFunction {
22
30
  return () => {};
23
31
  }
32
+ /** Drops an error message. */
24
33
  error(): LogFunction {
25
34
  return () => {};
26
35
  }
27
36
  }
28
37
 
29
- // Logs to console
38
+ /** Logger implementation that forwards messages to the global console. */
30
39
  export class ConsoleLog {
40
+ /** Console-like object used for message output. */
31
41
  console;
42
+ /** Messages already emitted through `once`. */
43
+ private onceMessages = new Set<string>();
32
44
 
45
+ /** Creates a console-backed logger. */
33
46
  constructor() {
34
47
  this.console = console; // eslint-disable-line
35
48
  }
49
+ /** Logs a message to the console. */
36
50
  log(...args: unknown[]): LogFunction {
37
51
  return this.console.log.bind(this.console, ...args);
38
52
  }
53
+ /** Logs an informational message to the console. */
39
54
  info(...args: unknown[]): LogFunction {
40
55
  return this.console.info.bind(this.console, ...args);
41
56
  }
57
+ /** Logs one message once to the console. */
58
+ once(...args: unknown[]): LogFunction {
59
+ const message = String(args[0]);
60
+ if (this.onceMessages.has(message)) {
61
+ return () => {};
62
+ }
63
+ this.onceMessages.add(message);
64
+ return this.console.info.bind(this.console, ...args);
65
+ }
66
+ /** Logs a warning message to the console. */
42
67
  warn(...args: unknown[]): LogFunction {
43
68
  return this.console.warn.bind(this.console, ...args);
44
69
  }
70
+ /** Logs an error message to the console. */
45
71
  error(...args: unknown[]): LogFunction {
46
72
  return this.console.error.bind(this.console, ...args);
47
73
  }
@@ -15,6 +15,7 @@ export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
15
15
  fallbackMimeType: undefined,
16
16
  ignoreRegisteredLoaders: undefined,
17
17
  nothrow: false,
18
+ shape: undefined,
18
19
  log: new ConsoleLog(), // A probe.gl compatible (`log.log()()` syntax) that just logs to console
19
20
  useLocalLibraries: false,
20
21
 
@@ -57,6 +58,7 @@ export const REMOVED_LOADER_OPTIONS = {
57
58
 
58
59
  limit: 'core.limit',
59
60
  _limitMB: 'core._limitMB',
61
+ shape: 'core.shape',
60
62
  batchSize: 'core.batchSize',
61
63
  batchDebounceMs: 'core.batchDebounceMs',
62
64
  metadata: 'core.metadata',