@loaders.gl/core 5.0.0-alpha.1 → 5.0.0-alpha.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. package/LICENSE +14 -3
  2. package/dist/dist.dev.js +1246 -227
  3. package/dist/dist.min.js +3 -3
  4. package/dist/index.cjs +2695 -270
  5. package/dist/index.cjs.map +3 -4
  6. package/dist/iterators/make-iterator/make-stream-iterator.js +49 -10
  7. package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -1
  8. package/dist/lib/api/create-data-source.d.ts +1 -1
  9. package/dist/lib/api/create-data-source.d.ts.map +1 -1
  10. package/dist/lib/api/create-data-source.js +23 -2
  11. package/dist/lib/api/create-data-source.js.map +1 -1
  12. package/dist/lib/api/load-in-batches.d.ts.map +1 -1
  13. package/dist/lib/api/load-in-batches.js +17 -6
  14. package/dist/lib/api/load-in-batches.js.map +1 -1
  15. package/dist/lib/api/load-loader.d.ts +2 -0
  16. package/dist/lib/api/load-loader.d.ts.map +1 -1
  17. package/dist/lib/api/load-loader.js +113 -7
  18. package/dist/lib/api/load-loader.js.map +1 -1
  19. package/dist/lib/api/load.d.ts +3 -3
  20. package/dist/lib/api/load.d.ts.map +1 -1
  21. package/dist/lib/api/load.js +36 -6
  22. package/dist/lib/api/load.js.map +1 -1
  23. package/dist/lib/api/parse-file.d.ts.map +1 -1
  24. package/dist/lib/api/parse-file.js +6 -2
  25. package/dist/lib/api/parse-file.js.map +1 -1
  26. package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
  27. package/dist/lib/api/parse-in-batches.js +12 -3
  28. package/dist/lib/api/parse-in-batches.js.map +1 -1
  29. package/dist/lib/api/parse.d.ts.map +1 -1
  30. package/dist/lib/api/parse.js +11 -3
  31. package/dist/lib/api/parse.js.map +1 -1
  32. package/dist/lib/api/preload.d.ts +6 -3
  33. package/dist/lib/api/preload.d.ts.map +1 -1
  34. package/dist/lib/api/preload.js +4 -5
  35. package/dist/lib/api/preload.js.map +1 -1
  36. package/dist/lib/filesystems/browser-filesystem.d.ts.map +1 -1
  37. package/dist/lib/filesystems/browser-filesystem.js +25 -11
  38. package/dist/lib/filesystems/browser-filesystem.js.map +1 -1
  39. package/dist/lib/init.js +1 -1
  40. package/dist/lib/init.js.map +1 -1
  41. package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
  42. package/dist/lib/loader-utils/get-fetch-function.js +16 -10
  43. package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
  44. package/dist/lib/loader-utils/loader-context.d.ts +1 -1
  45. package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
  46. package/dist/lib/loader-utils/loader-context.js +8 -3
  47. package/dist/lib/loader-utils/loader-context.js.map +1 -1
  48. package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
  49. package/dist/lib/loader-utils/option-defaults.js +3 -0
  50. package/dist/lib/loader-utils/option-defaults.js.map +1 -1
  51. package/dist/lib/loader-utils/option-utils.d.ts +7 -0
  52. package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
  53. package/dist/lib/loader-utils/option-utils.js +41 -1
  54. package/dist/lib/loader-utils/option-utils.js.map +1 -1
  55. package/dist/lib/utils/response-utils.js +2 -11
  56. package/dist/lib/utils/response-utils.js.map +1 -1
  57. package/dist/null-loader.js +1 -1
  58. package/dist/null-loader.js.map +1 -1
  59. package/dist/null-worker-node.js +130 -58
  60. package/dist/null-worker.js +130 -58
  61. package/package.json +6 -6
  62. package/src/iterators/make-iterator/make-stream-iterator.ts +48 -10
  63. package/src/lib/api/create-data-source.ts +31 -2
  64. package/src/lib/api/load-in-batches.ts +18 -11
  65. package/src/lib/api/load-loader.ts +157 -14
  66. package/src/lib/api/load.ts +87 -9
  67. package/src/lib/api/parse-file.ts +15 -2
  68. package/src/lib/api/parse-in-batches.ts +22 -3
  69. package/src/lib/api/parse.ts +18 -3
  70. package/src/lib/api/preload.ts +22 -7
  71. package/src/lib/filesystems/browser-filesystem.ts +25 -11
  72. package/src/lib/loader-utils/get-fetch-function.ts +15 -14
  73. package/src/lib/loader-utils/loader-context.ts +8 -3
  74. package/src/lib/loader-utils/option-defaults.ts +3 -0
  75. package/src/lib/loader-utils/option-utils.ts +47 -1
  76. package/src/lib/utils/response-utils.ts +2 -12
@@ -2,7 +2,13 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {Loader, LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
5
+ import {isSourceLoader, mergeOptions} from '@loaders.gl/loader-utils';
6
+ import type {
7
+ Loader,
8
+ LoaderOptions,
9
+ LoaderWithParser,
10
+ StrictLoaderOptions
11
+ } from '@loaders.gl/loader-utils';
6
12
 
7
13
  const loaderImplementationPromises = new Map<Loader, Map<string, Promise<LoaderWithParser>>>();
8
14
  const loaderImplementations = new Map<Loader, Map<string, LoaderWithParser>>();
@@ -13,6 +19,15 @@ export async function getLoaderImplementation(
13
19
  options?: LoaderOptions,
14
20
  url?: string
15
21
  ): Promise<LoaderWithParser> {
22
+ if (isSourceLoader(loader)) {
23
+ const source = loader.preload ? await loader.preload(url || '', options) : loader;
24
+ if (!isSourceLoader(source)) {
25
+ throw new Error(
26
+ `${loader.id} source loader preload() did not return a runtime source loader`
27
+ );
28
+ }
29
+ return source as unknown as LoaderWithParser;
30
+ }
16
31
  if (isLoaderWithParser(loader)) {
17
32
  return loader;
18
33
  }
@@ -57,6 +72,11 @@ export async function getLoaderImplementation(
57
72
 
58
73
  /** Gets a cached parser-bearing implementation for a loader without loading it. */
59
74
  export function getLoaderImplementationSync(loader: Loader): LoaderWithParser | null {
75
+ if (loader.subloaders && !preparedLoaders.has(loader)) {
76
+ const prepared = preparedDefaults.get(loader);
77
+ if (prepared) return prepared;
78
+ if (!isLoaderWithParser(loader)) return null;
79
+ }
60
80
  if (isLoaderWithParser(loader)) {
61
81
  return loader;
62
82
  }
@@ -121,20 +141,9 @@ function getLoaderImplementationSpecifier(loader: Loader, options?: LoaderOption
121
141
 
122
142
  /** Gets the cache key for option-selected parser implementations. */
123
143
  function getLoaderImplementationCacheKey(loader: Loader, options?: LoaderOptions): string {
124
- const loaderOptions = options?.[loader.id] as
125
- | {backend?: unknown; implementation?: unknown}
126
- | undefined;
144
+ const loaderOptions = options?.[loader.id] as {backend?: unknown} | undefined;
127
145
  const defaultLoaderOptions = loader.options?.[loader.id] as {backend?: unknown} | undefined;
128
- return String(
129
- loaderOptions?.backend ||
130
- getDeprecatedImplementationBackend(loaderOptions?.implementation) ||
131
- defaultLoaderOptions?.backend ||
132
- ''
133
- );
134
- }
135
-
136
- function getDeprecatedImplementationBackend(implementation: unknown): unknown {
137
- return implementation === 'js' ? 'typescript' : implementation;
146
+ return String(loaderOptions?.backend || defaultLoaderOptions?.backend || '');
138
147
  }
139
148
 
140
149
  /** Imports a module and finds the parser-bearing loader implementation with the requested id. */
@@ -161,3 +170,137 @@ async function importLoaderImplementation(
161
170
  `Could not find parser implementation for ${loaderId} in ${implementationSpecifier}`
162
171
  );
163
172
  }
173
+
174
+ /** Prepared loaders carry caller-local parser bindings rather than shared metadata. */
175
+ const preparedLoaders = new WeakSet<Loader>();
176
+ /** Successfully prepared default graphs used by synchronous parsing. */
177
+ const preparedDefaults = new WeakMap<Loader, LoaderWithParser>();
178
+ /** In-flight preparations for default dependency graphs. */
179
+ const subloaderPreparations = new WeakMap<Loader, Promise<LoaderWithParser>>();
180
+
181
+ /** Resolves declared dependencies and binds parser entry points to their implementations. */
182
+ export async function preloadLoaderDependencies(
183
+ loader: Loader,
184
+ options?: LoaderOptions,
185
+ url?: string
186
+ ): Promise<LoaderWithParser> {
187
+ if (preparedLoaders.has(loader)) return await getLoaderImplementation(loader, options, url);
188
+ validateSubloaderGraph(loader, options);
189
+ return await prepareLoaderDependencies(loader, options, url);
190
+ }
191
+
192
+ /** Prepares a node after the complete dependency graph has been validated. */
193
+ async function prepareLoaderDependencies(
194
+ loader: Loader,
195
+ options?: LoaderOptions,
196
+ url?: string
197
+ ): Promise<LoaderWithParser> {
198
+ // Options can select nested backends or overrides; do not share their bindings.
199
+ if (!options) {
200
+ const cached = subloaderPreparations.get(loader);
201
+ if (cached) return await cached;
202
+ }
203
+ const preparation = bindSubloaders(loader, options, url);
204
+ if (!options) {
205
+ subloaderPreparations.set(loader, preparation);
206
+ void preparation.catch(() => subloaderPreparations.delete(loader));
207
+ }
208
+ const prepared = await preparation;
209
+ if (!options) preparedDefaults.set(loader, prepared);
210
+ return prepared;
211
+ }
212
+
213
+ /** Returns declared dependencies with the shared flat override map applied. */
214
+ function getSubloaders(loader: Loader, options?: LoaderOptions): Record<string, Loader> {
215
+ const overrides = options?.core?.subloaders || {};
216
+ return Object.fromEntries(
217
+ Object.entries(loader.subloaders || {}).map(([name, dependency]) => [
218
+ name,
219
+ Object.hasOwn(overrides, name) ? overrides[name] : dependency
220
+ ])
221
+ );
222
+ }
223
+
224
+ /** Rejects cycles in the dependency graph after applying the shared overrides. */
225
+ function validateSubloaderGraph(loader: Loader, options?: LoaderOptions): void {
226
+ visitLoader(loader, []);
227
+
228
+ /** Detects cycles along each resolved dependency path. */
229
+ function visitLoader(currentLoader: Loader, ancestors: Loader[]): void {
230
+ if (ancestors.includes(currentLoader)) {
231
+ throw new Error(
232
+ `Subloader cycle: ${[...ancestors, currentLoader].map(value => value.id).join(' -> ')}`
233
+ );
234
+ }
235
+ for (const dependency of Object.values(getSubloaders(currentLoader, options))) {
236
+ visitLoader(dependency, [...ancestors, currentLoader]);
237
+ }
238
+ }
239
+ }
240
+
241
+ /** Prepares dependencies concurrently and injects them into parser calls' scoped options. */
242
+ async function bindSubloaders(
243
+ loader: Loader,
244
+ options?: LoaderOptions,
245
+ url?: string
246
+ ): Promise<LoaderWithParser> {
247
+ const [implementation, entries] = await Promise.all([
248
+ getLoaderImplementation(loader, options, url),
249
+ Promise.all(
250
+ Object.entries(getSubloaders(loader, options)).map(
251
+ async ([name, dependency]) =>
252
+ [name, await prepareLoaderDependencies(dependency, options, url)] as const
253
+ )
254
+ )
255
+ ]);
256
+ const subloaders = Object.fromEntries(entries);
257
+ // Expose preload settings as defaults before core normalizes a later parse call.
258
+ const defaultOptions = mergeOptions(
259
+ implementation.options,
260
+ (options || {}) as StrictLoaderOptions
261
+ );
262
+ defaultOptions.core = {...defaultOptions.core, worker: false};
263
+ const prepared = {...implementation, options: defaultOptions, subloaders};
264
+ delete prepared.preload;
265
+ // Function-valued dependency bindings cannot be transferred to a worker.
266
+ prepared.worker = false;
267
+ for (const method of [
268
+ 'parseBlob',
269
+ 'createDataSource',
270
+ 'parse',
271
+ 'parseSync',
272
+ 'parseText',
273
+ 'parseTextSync',
274
+ 'parseInBatches',
275
+ 'parseFile',
276
+ 'parseFileInBatches',
277
+ 'parseUrl'
278
+ ] as const) {
279
+ const parser = (implementation as Record<string, unknown>)[method];
280
+ if (typeof parser === 'function') {
281
+ (prepared as Record<string, unknown>)[method] = (
282
+ data: unknown,
283
+ parseOptions?: LoaderOptions,
284
+ context?: unknown
285
+ ) => {
286
+ const forwardedOptions = mergeOptions(options, parseOptions || {});
287
+ return parser.call(
288
+ implementation,
289
+ data,
290
+ {
291
+ ...forwardedOptions,
292
+ core: {
293
+ ...forwardedOptions.core,
294
+ subloaders: options?.core?.subloaders || parseOptions?.core?.subloaders,
295
+ worker: false
296
+ },
297
+ [loader.id]: {...(forwardedOptions[loader.id] as object), subloaders}
298
+ },
299
+ context
300
+ );
301
+ };
302
+ }
303
+ }
304
+ preparedLoaders.add(prepared);
305
+ return prepared;
306
+ }
@@ -13,12 +13,25 @@ import type {
13
13
  LoaderReturnType,
14
14
  LoaderArrayOptionsType,
15
15
  LoaderArrayReturnType,
16
- SourceLoader
16
+ SourceLoader,
17
+ SourcePropsType,
18
+ SourceArrayOptionsType,
19
+ DataSourceOptions,
20
+ LoaderWithParser
21
+ } from '@loaders.gl/loader-utils';
22
+ import {
23
+ isBlob,
24
+ isSourceLoader,
25
+ resolveAuthenticationOptions,
26
+ resolveLoaderAuthenticationOptions
17
27
  } from '@loaders.gl/loader-utils';
18
- import {isBlob, isSourceLoader} from '@loaders.gl/loader-utils';
19
28
  import {isLoaderObject} from '../loader-utils/normalize-loader';
20
29
  import {getFetchFunction} from '../loader-utils/get-fetch-function';
21
- import {normalizeLoaderOptions} from '../loader-utils/option-utils';
30
+ import {
31
+ applySearchParamsToUrl,
32
+ normalizeLoaderOptions,
33
+ getGlobalLoaderOptions
34
+ } from '../loader-utils/option-utils';
22
35
  import {fetchFile} from '../fetch/fetch-file';
23
36
 
24
37
  import {parse} from './parse';
@@ -27,6 +40,7 @@ import {parseSync} from './parse-sync';
27
40
  import {parseInBatches} from './parse-in-batches';
28
41
  import {loadInBatches} from './load-in-batches';
29
42
  import {selectLoader} from './select-loader';
43
+ import {getLoaderImplementation} from './load-loader';
30
44
 
31
45
  /**
32
46
  * Parses `data` using a specified loader
@@ -47,7 +61,7 @@ export async function load<
47
61
  >(
48
62
  url: string | DataType,
49
63
  loader: LoaderT,
50
- options?: OptionsT,
64
+ options?: LoaderT extends SourceLoader ? SourcePropsType<LoaderT> & DataSourceOptions : OptionsT,
51
65
  context?: LoaderContext
52
66
  ): Promise<LoaderReturnType<LoaderT>>;
53
67
 
@@ -57,7 +71,7 @@ export async function load<
57
71
  >(
58
72
  url: string | DataType,
59
73
  loaders: LoaderArrayT,
60
- options?: OptionsT,
74
+ options?: LoaderArrayT extends SourceLoader[] ? SourceArrayOptionsType<LoaderArrayT> : OptionsT,
61
75
  context?: LoaderContext
62
76
  ): Promise<LoaderArrayReturnType<LoaderArrayT>>;
63
77
 
@@ -93,7 +107,15 @@ export async function load(
93
107
  resolvedOptions = options as LoaderOptions;
94
108
  }
95
109
 
110
+ resolvedOptions ||= getGlobalLoaderOptions();
111
+
96
112
  if (!Array.isArray(resolvedLoaders) && isSourceLoader(resolvedLoaders)) {
113
+ const sourceLoader = await resolveSourceLoader(resolvedLoaders, url, resolvedOptions);
114
+ resolvedOptions = await resolveLoaderAuthenticationOptions(
115
+ resolvedLoaders.getAuthentications ? resolvedLoaders : sourceLoader,
116
+ typeof url === 'string' ? url : '',
117
+ resolvedOptions || {}
118
+ );
97
119
  const runtimeCoreApi = {
98
120
  fetchFile,
99
121
  parse,
@@ -103,7 +125,7 @@ export async function load(
103
125
  load,
104
126
  loadInBatches
105
127
  };
106
- return resolvedLoaders.createDataSource(
128
+ return sourceLoader.createDataSource(
107
129
  url as string | Blob,
108
130
  (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
109
131
  runtimeCoreApi
@@ -115,6 +137,12 @@ export async function load(
115
137
  resolvedLoaders.length === 1 &&
116
138
  isSourceLoader(resolvedLoaders[0])
117
139
  ) {
140
+ const sourceLoader = await resolveSourceLoader(resolvedLoaders[0], url, resolvedOptions);
141
+ resolvedOptions = await resolveLoaderAuthenticationOptions(
142
+ resolvedLoaders[0].getAuthentications ? resolvedLoaders[0] : sourceLoader,
143
+ typeof url === 'string' ? url : '',
144
+ resolvedOptions || {}
145
+ );
118
146
  const runtimeCoreApi = {
119
147
  fetchFile,
120
148
  parse,
@@ -124,7 +152,7 @@ export async function load(
124
152
  load,
125
153
  loadInBatches
126
154
  };
127
- return resolvedLoaders[0].createDataSource(
155
+ return sourceLoader.createDataSource(
128
156
  url as string | Blob,
129
157
  (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
130
158
  runtimeCoreApi
@@ -138,7 +166,13 @@ export async function load(
138
166
  });
139
167
 
140
168
  if (selectedLoader && isSourceLoader(selectedLoader)) {
141
- return selectedLoader.createDataSource(
169
+ const sourceLoader = await resolveSourceLoader(selectedLoader, url, resolvedOptions);
170
+ resolvedOptions = await resolveLoaderAuthenticationOptions(
171
+ selectedLoader.getAuthentications ? selectedLoader : sourceLoader,
172
+ typeof url === 'string' ? url : '',
173
+ resolvedOptions || {}
174
+ );
175
+ return sourceLoader.createDataSource(
142
176
  url,
143
177
  (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
144
178
  {
@@ -152,16 +186,43 @@ export async function load(
152
186
  }
153
187
  );
154
188
  }
189
+
190
+ if (selectedLoader && typeof url === 'string') {
191
+ const loaderImplementation = await getLoaderImplementation(
192
+ selectedLoader,
193
+ resolvedOptions,
194
+ url
195
+ );
196
+ resolvedOptions = await resolveLoaderAuthenticationOptions(
197
+ selectedLoader.getAuthentications ? selectedLoader : loaderImplementation,
198
+ url,
199
+ resolvedOptions || {}
200
+ );
201
+ const parseUrl = (
202
+ loaderImplementation as LoaderWithParser & {
203
+ parseUrl?: (
204
+ url: string,
205
+ options?: LoaderOptions,
206
+ context?: LoaderContext
207
+ ) => Promise<unknown>;
208
+ }
209
+ ).parseUrl;
210
+ if (parseUrl) {
211
+ const requestUrl = applySearchParamsToUrl(url, resolvedOptions);
212
+ return await parseUrl(requestUrl, resolvedOptions, context);
213
+ }
214
+ }
155
215
  }
156
216
 
157
217
  // Select fetch function
218
+ if (resolvedOptions) resolvedOptions = resolveAuthenticationOptions(resolvedOptions);
158
219
  const fetch = getFetchFunction(resolvedOptions);
159
220
 
160
221
  // at this point, `url` could be already loaded binary data
161
222
  let data = url;
162
223
  // url is a string, fetch the url
163
224
  if (typeof url === 'string') {
164
- data = await fetch(url);
225
+ data = await fetch(applySearchParamsToUrl(url, resolvedOptions));
165
226
  // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)
166
227
  }
167
228
 
@@ -190,3 +251,20 @@ export async function load(
190
251
  ? await parse(data, resolvedLoaders, resolvedOptions, context) // loader array overload
191
252
  : await parse(data, resolvedLoaders, resolvedOptions, context); // single loader overload
192
253
  }
254
+
255
+ /** Resolves a lightweight source loader into its runtime implementation when necessary. */
256
+ async function resolveSourceLoader(
257
+ loader: SourceLoader,
258
+ url: string | DataType,
259
+ options?: LoaderOptions
260
+ ): Promise<SourceLoader> {
261
+ if (!loader.preload) {
262
+ return loader;
263
+ }
264
+
265
+ const sourceLoader = await loader.preload(typeof url === 'string' ? url : '', options);
266
+ if (!isSourceLoader(sourceLoader)) {
267
+ throw new Error(`${loader.id} source loader preload() did not return a runtime source loader`);
268
+ }
269
+ return sourceLoader;
270
+ }
@@ -16,7 +16,11 @@ import type {
16
16
  ReadableFile,
17
17
  StrictLoaderOptions
18
18
  } from '@loaders.gl/loader-utils';
19
- import {isSourceLoader, mergeOptions} from '@loaders.gl/loader-utils';
19
+ import {
20
+ isSourceLoader,
21
+ mergeOptions,
22
+ resolveLoaderAuthenticationOptions
23
+ } from '@loaders.gl/loader-utils';
20
24
  import {validateWorkerVersion} from '@loaders.gl/worker-utils';
21
25
  import {isLoaderObject} from '../loader-utils/normalize-loader';
22
26
  import {normalizeOptions} from '../loader-utils/option-utils';
@@ -96,7 +100,16 @@ export async function parseFile(
96
100
  }
97
101
 
98
102
  // @ts-expect-error candidateLoaders may be a single forced loader
99
- const strictOptions = normalizeOptions(options, loader, candidateLoaders, url);
103
+ const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url);
104
+ const authenticationLoader =
105
+ !loader.getAuthentications && normalizedOptions.core?.credentials?.length
106
+ ? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
107
+ : loader;
108
+ const strictOptions = await resolveLoaderAuthenticationOptions(
109
+ authenticationLoader,
110
+ url,
111
+ normalizedOptions
112
+ );
100
113
  context = getLoaderContext(
101
114
  // @ts-expect-error candidateLoaders may be a single forced loader
102
115
  {url, _parse: parse, loaders: candidateLoaders},
@@ -4,6 +4,7 @@
4
4
 
5
5
  import type {Batch} from '@loaders.gl/schema';
6
6
  import {isTable, makeBatchFromTable} from '@loaders.gl/schema-utils';
7
+ import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
7
8
  import type {
8
9
  Loader,
9
10
  LoaderWithParser,
@@ -19,7 +20,12 @@ import type {
19
20
  LoaderArrayBatchType,
20
21
  TransformBatches
21
22
  } from '@loaders.gl/loader-utils';
22
- import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
23
+ import {
24
+ canParseWithWorker,
25
+ concatenateArrayBuffersAsync,
26
+ isBrowser,
27
+ parseWithWorkerInBatches
28
+ } from '@loaders.gl/loader-utils';
23
29
  import {isLoaderObject} from '../loader-utils/normalize-loader';
24
30
  import {normalizeOptions} from '../loader-utils/option-utils';
25
31
  import {getLoaderContext} from '../loader-utils/loader-context';
@@ -107,7 +113,16 @@ export async function parseInBatches(
107
113
  }
108
114
 
109
115
  // Normalize options
110
- const strictOptions = normalizeOptions(options, loader, loaderArray, url);
116
+ const normalizedOptions = normalizeOptions(options, loader, loaderArray, url);
117
+ const authenticationLoader =
118
+ !loader.getAuthentications && normalizedOptions.core?.credentials?.length
119
+ ? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
120
+ : loader;
121
+ const strictOptions = await resolveLoaderAuthenticationOptions(
122
+ authenticationLoader,
123
+ url || '',
124
+ normalizedOptions
125
+ );
111
126
  context = getLoaderContext(
112
127
  {url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},
113
128
  strictOptions,
@@ -177,8 +192,12 @@ async function parseToOutputIterator(
177
192
  options?.core?.transforms || []
178
193
  );
179
194
 
180
- // If loader supports parseInBatches, we are done
195
+ // Stream supported browser batch parsers through one stateful worker session.
181
196
  if (loader.parseInBatches) {
197
+ // The loader-level adapter is intentionally piloted with CSV Arrow batches first.
198
+ if (loader.id === 'csv' && isBrowser && canParseWithWorker(loader, options)) {
199
+ return parseWithWorkerInBatches(loader, transformedIterator, options, context, parse);
200
+ }
182
201
  return loader.parseInBatches(transformedIterator, options, context);
183
202
  }
184
203
 
@@ -2,6 +2,7 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
+ import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
5
6
  import type {
6
7
  Loader,
7
8
  LoaderContext,
@@ -18,9 +19,10 @@ import type {
18
19
  } from '@loaders.gl/loader-utils';
19
20
  import {
20
21
  parseWithWorker,
21
- canParseWithWorker,
22
+ shouldParseWithWorker,
22
23
  mergeOptions,
23
24
  isResponse,
25
+ isBlob,
24
26
  isSourceLoader
25
27
  } from '@loaders.gl/loader-utils';
26
28
  import {validateWorkerVersion} from '@loaders.gl/worker-utils';
@@ -120,7 +122,16 @@ export async function parse(
120
122
 
121
123
  // Normalize options
122
124
  // @ts-expect-error candidateLoaders
123
- const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
125
+ const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
126
+ const authenticationLoader =
127
+ !loader.getAuthentications && normalizedOptions.core?.credentials?.length
128
+ ? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
129
+ : loader;
130
+ const strictOptions = await resolveLoaderAuthenticationOptions(
131
+ authenticationLoader,
132
+ url || '',
133
+ normalizedOptions
134
+ );
124
135
 
125
136
  // Get a context (if already present, will be unchanged)
126
137
  context = getLoaderContext(
@@ -155,11 +166,15 @@ async function parseWithLoader(
155
166
 
156
167
  const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
157
168
 
158
- if (canParseWithWorker(loaderWithParser, options)) {
169
+ if (shouldParseWithWorker(loaderWithParser, data, options, context, loader)) {
159
170
  data = await getArrayBufferFromData(data, options);
160
171
  return await parseWithWorker(loaderWithParser, data, options, context, parse);
161
172
  }
162
173
 
174
+ if (isBlob(data) && loaderWithParser.parseBlob) {
175
+ return await loaderWithParser.parseBlob(data, options, context);
176
+ }
177
+
163
178
  data = await getArrayBufferOrStringFromData(data, loader, options);
164
179
 
165
180
  return await parseWithLoaderImplementation(loaderWithParser, data, options, context);
@@ -2,13 +2,23 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import type {Loader, LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
6
- import {getLoaderImplementation, getLoaderImplementationSync} from './load-loader';
5
+ import type {Loader, LoaderOptions, LoaderWithParser, SourceLoader} from '@loaders.gl/loader-utils';
6
+ import {
7
+ getLoaderImplementation,
8
+ getLoaderImplementationSync,
9
+ preloadLoaderDependencies
10
+ } from './load-loader';
7
11
 
8
12
  /**
9
- * Preloads a loader implementation so later parse/load calls do not pay the resolution cost on first use.
10
- * May return a parser-bearing loader that also supports `parseSync`.
13
+ * Prepares a source implementation and its declared dependencies before source creation.
11
14
  */
15
+ export async function preload<SourceT extends SourceLoader>(
16
+ loader: SourceT,
17
+ options?: LoaderOptions,
18
+ url?: string
19
+ ): Promise<SourceT>;
20
+
21
+ /** Prepares a parser and its recursively declared dependencies. */
12
22
  export async function preload<LoaderT extends Loader>(
13
23
  loader: LoaderT,
14
24
  options?: LoaderOptions,
@@ -19,13 +29,18 @@ export async function preload(
19
29
  loader: Loader,
20
30
  options?: LoaderOptions,
21
31
  url?: string
22
- ): Promise<LoaderWithParser> {
23
- return await getLoaderImplementation(loader, options, url);
32
+ ): Promise<LoaderWithParser | SourceLoader> {
33
+ return loader.subloaders
34
+ ? await preloadLoaderDependencies(loader, options, url)
35
+ : await getLoaderImplementation(loader, options, url);
24
36
  }
25
37
 
26
38
  /**
27
39
  * Returns a cached parser-bearing loader implementation if one is already available.
28
40
  */
29
- export function preloadSync(loader: Loader): LoaderWithParser | null {
41
+ export function preloadSync<SourceT extends SourceLoader>(loader: SourceT): SourceT | null;
42
+ /** Returns a cached parser, including prepared default dependency bindings. */
43
+ export function preloadSync(loader: Loader): LoaderWithParser | null;
44
+ export function preloadSync(loader: Loader): LoaderWithParser | SourceLoader | null {
30
45
  return getLoaderImplementationSync(loader);
31
46
  }
@@ -50,23 +50,34 @@ export class BrowserFileSystem implements FileSystem {
50
50
  }
51
51
 
52
52
  // Local fetches are served from the list of files
53
- const file = this.files[path];
53
+ const file = this._getFile(path, true);
54
54
  if (!file) {
55
55
  return new Response(path, {status: 400, statusText: 'NOT FOUND'});
56
56
  }
57
57
 
58
58
  const headers = new Headers(options?.headers);
59
59
  const range = headers.get('Range');
60
- const bytes = range && /bytes=($1)-($2)/.exec(range);
60
+ const bytes = range && /bytes=(\d+)-(\d+)/.exec(range);
61
61
 
62
62
  if (bytes) {
63
63
  const start = parseInt(bytes[1]);
64
- const end = parseInt(bytes[2]);
64
+ const end = Math.min(parseInt(bytes[2]), file.size - 1);
65
+ if (start > end) {
66
+ const response = new Response(null, {
67
+ status: 416,
68
+ headers: {'Content-Range': `bytes */${file.size}`}
69
+ });
70
+ Object.defineProperty(response, 'url', {value: path});
71
+ return response;
72
+ }
65
73
  // The trick when reading File objects is to read successive "slices" of the File
66
74
  // Per spec https://w3c.github.io/FileAPI/, slicing a File should only update the start and end fields
67
75
  // Actually reading from file should happen in `readAsArrayBuffer` (and as far we can tell it does)
68
- const data = await file.slice(start, end).arrayBuffer();
69
- const response = new Response(data);
76
+ const data = await file.slice(start, end + 1).arrayBuffer();
77
+ const response = new Response(data, {
78
+ status: 206,
79
+ headers: {'Content-Range': `bytes ${start}-${end}/${file.size}`}
80
+ });
70
81
  Object.defineProperty(response, 'url', {value: path});
71
82
  return response;
72
83
  }
@@ -95,7 +106,7 @@ export class BrowserFileSystem implements FileSystem {
95
106
  * Return information (size) about files in this file system
96
107
  */
97
108
  async stat(path: string, options?: object): Promise<{size: number}> {
98
- const file = this.files[path];
109
+ const file = this._getFile(path, false);
99
110
  if (!file) {
100
111
  throw new Error(path);
101
112
  }
@@ -106,16 +117,19 @@ export class BrowserFileSystem implements FileSystem {
106
117
  * Just removes the file from the list
107
118
  */
108
119
  async unlink(path: string): Promise<void> {
109
- delete this.files[path];
110
- delete this.lowerCaseFiles[path];
111
- this.usedFiles[path] = true;
120
+ const file = this._getFile(path, false);
121
+ if (file) {
122
+ delete this.files[file.name];
123
+ delete this.lowerCaseFiles[file.name.toLowerCase()];
124
+ this.usedFiles[file.name] = true;
125
+ }
112
126
  }
113
127
 
114
128
  // implements IRandomAccessFileSystem
115
129
 
116
130
  // RANDOM ACCESS
117
131
  async openReadableFile(pathname: string, flags: unknown): Promise<ReadableFile> {
118
- return new BlobFile(this.files[pathname]);
132
+ return new BlobFile(this._getFile(pathname, true));
119
133
  }
120
134
 
121
135
  // PRIVATE
@@ -123,7 +137,7 @@ export class BrowserFileSystem implements FileSystem {
123
137
  // Supports case independent paths, and file usage tracking
124
138
  _getFile(path: string, used: boolean): File {
125
139
  // Prefer case match, but fall back to case independent.
126
- const file = this.files[path] || this.lowerCaseFiles[path];
140
+ const file = this.files[path] || this.lowerCaseFiles[path.toLowerCase()];
127
141
  if (file && used) {
128
142
  this.usedFiles[path] = true;
129
143
  }