@loaders.gl/core 5.0.0-alpha.1 → 5.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -3
- package/dist/dist.dev.js +1246 -227
- package/dist/dist.min.js +3 -3
- package/dist/index.cjs +2695 -270
- package/dist/index.cjs.map +3 -4
- package/dist/iterators/make-iterator/make-stream-iterator.js +49 -10
- package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -1
- package/dist/lib/api/create-data-source.d.ts +1 -1
- package/dist/lib/api/create-data-source.d.ts.map +1 -1
- package/dist/lib/api/create-data-source.js +23 -2
- package/dist/lib/api/create-data-source.js.map +1 -1
- package/dist/lib/api/load-in-batches.d.ts.map +1 -1
- package/dist/lib/api/load-in-batches.js +17 -6
- package/dist/lib/api/load-in-batches.js.map +1 -1
- package/dist/lib/api/load-loader.d.ts +2 -0
- package/dist/lib/api/load-loader.d.ts.map +1 -1
- package/dist/lib/api/load-loader.js +113 -7
- package/dist/lib/api/load-loader.js.map +1 -1
- package/dist/lib/api/load.d.ts +3 -3
- package/dist/lib/api/load.d.ts.map +1 -1
- package/dist/lib/api/load.js +36 -6
- package/dist/lib/api/load.js.map +1 -1
- package/dist/lib/api/parse-file.d.ts.map +1 -1
- package/dist/lib/api/parse-file.js +6 -2
- package/dist/lib/api/parse-file.js.map +1 -1
- package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.js +12 -3
- package/dist/lib/api/parse-in-batches.js.map +1 -1
- package/dist/lib/api/parse.d.ts.map +1 -1
- package/dist/lib/api/parse.js +11 -3
- package/dist/lib/api/parse.js.map +1 -1
- package/dist/lib/api/preload.d.ts +6 -3
- package/dist/lib/api/preload.d.ts.map +1 -1
- package/dist/lib/api/preload.js +4 -5
- package/dist/lib/api/preload.js.map +1 -1
- package/dist/lib/filesystems/browser-filesystem.d.ts.map +1 -1
- package/dist/lib/filesystems/browser-filesystem.js +25 -11
- package/dist/lib/filesystems/browser-filesystem.js.map +1 -1
- package/dist/lib/init.js +1 -1
- package/dist/lib/init.js.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.js +16 -10
- package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
- package/dist/lib/loader-utils/loader-context.js +8 -3
- package/dist/lib/loader-utils/loader-context.js.map +1 -1
- package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-defaults.js +3 -0
- package/dist/lib/loader-utils/option-defaults.js.map +1 -1
- package/dist/lib/loader-utils/option-utils.d.ts +7 -0
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +41 -1
- package/dist/lib/loader-utils/option-utils.js.map +1 -1
- package/dist/lib/utils/response-utils.js +2 -11
- package/dist/lib/utils/response-utils.js.map +1 -1
- package/dist/null-loader.js +1 -1
- package/dist/null-loader.js.map +1 -1
- package/dist/null-worker-node.js +130 -58
- package/dist/null-worker.js +130 -58
- package/package.json +6 -6
- package/src/iterators/make-iterator/make-stream-iterator.ts +48 -10
- package/src/lib/api/create-data-source.ts +31 -2
- package/src/lib/api/load-in-batches.ts +18 -11
- package/src/lib/api/load-loader.ts +157 -14
- package/src/lib/api/load.ts +87 -9
- package/src/lib/api/parse-file.ts +15 -2
- package/src/lib/api/parse-in-batches.ts +22 -3
- package/src/lib/api/parse.ts +18 -3
- package/src/lib/api/preload.ts +22 -7
- package/src/lib/filesystems/browser-filesystem.ts +25 -11
- package/src/lib/loader-utils/get-fetch-function.ts +15 -14
- package/src/lib/loader-utils/loader-context.ts +8 -3
- package/src/lib/loader-utils/option-defaults.ts +3 -0
- package/src/lib/loader-utils/option-utils.ts +47 -1
- package/src/lib/utils/response-utils.ts +2 -12
|
@@ -2,7 +2,13 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import
|
|
5
|
+
import {isSourceLoader, mergeOptions} from '@loaders.gl/loader-utils';
|
|
6
|
+
import type {
|
|
7
|
+
Loader,
|
|
8
|
+
LoaderOptions,
|
|
9
|
+
LoaderWithParser,
|
|
10
|
+
StrictLoaderOptions
|
|
11
|
+
} from '@loaders.gl/loader-utils';
|
|
6
12
|
|
|
7
13
|
const loaderImplementationPromises = new Map<Loader, Map<string, Promise<LoaderWithParser>>>();
|
|
8
14
|
const loaderImplementations = new Map<Loader, Map<string, LoaderWithParser>>();
|
|
@@ -13,6 +19,15 @@ export async function getLoaderImplementation(
|
|
|
13
19
|
options?: LoaderOptions,
|
|
14
20
|
url?: string
|
|
15
21
|
): Promise<LoaderWithParser> {
|
|
22
|
+
if (isSourceLoader(loader)) {
|
|
23
|
+
const source = loader.preload ? await loader.preload(url || '', options) : loader;
|
|
24
|
+
if (!isSourceLoader(source)) {
|
|
25
|
+
throw new Error(
|
|
26
|
+
`${loader.id} source loader preload() did not return a runtime source loader`
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
return source as unknown as LoaderWithParser;
|
|
30
|
+
}
|
|
16
31
|
if (isLoaderWithParser(loader)) {
|
|
17
32
|
return loader;
|
|
18
33
|
}
|
|
@@ -57,6 +72,11 @@ export async function getLoaderImplementation(
|
|
|
57
72
|
|
|
58
73
|
/** Gets a cached parser-bearing implementation for a loader without loading it. */
|
|
59
74
|
export function getLoaderImplementationSync(loader: Loader): LoaderWithParser | null {
|
|
75
|
+
if (loader.subloaders && !preparedLoaders.has(loader)) {
|
|
76
|
+
const prepared = preparedDefaults.get(loader);
|
|
77
|
+
if (prepared) return prepared;
|
|
78
|
+
if (!isLoaderWithParser(loader)) return null;
|
|
79
|
+
}
|
|
60
80
|
if (isLoaderWithParser(loader)) {
|
|
61
81
|
return loader;
|
|
62
82
|
}
|
|
@@ -121,20 +141,9 @@ function getLoaderImplementationSpecifier(loader: Loader, options?: LoaderOption
|
|
|
121
141
|
|
|
122
142
|
/** Gets the cache key for option-selected parser implementations. */
|
|
123
143
|
function getLoaderImplementationCacheKey(loader: Loader, options?: LoaderOptions): string {
|
|
124
|
-
const loaderOptions = options?.[loader.id] as
|
|
125
|
-
| {backend?: unknown; implementation?: unknown}
|
|
126
|
-
| undefined;
|
|
144
|
+
const loaderOptions = options?.[loader.id] as {backend?: unknown} | undefined;
|
|
127
145
|
const defaultLoaderOptions = loader.options?.[loader.id] as {backend?: unknown} | undefined;
|
|
128
|
-
return String(
|
|
129
|
-
loaderOptions?.backend ||
|
|
130
|
-
getDeprecatedImplementationBackend(loaderOptions?.implementation) ||
|
|
131
|
-
defaultLoaderOptions?.backend ||
|
|
132
|
-
''
|
|
133
|
-
);
|
|
134
|
-
}
|
|
135
|
-
|
|
136
|
-
function getDeprecatedImplementationBackend(implementation: unknown): unknown {
|
|
137
|
-
return implementation === 'js' ? 'typescript' : implementation;
|
|
146
|
+
return String(loaderOptions?.backend || defaultLoaderOptions?.backend || '');
|
|
138
147
|
}
|
|
139
148
|
|
|
140
149
|
/** Imports a module and finds the parser-bearing loader implementation with the requested id. */
|
|
@@ -161,3 +170,137 @@ async function importLoaderImplementation(
|
|
|
161
170
|
`Could not find parser implementation for ${loaderId} in ${implementationSpecifier}`
|
|
162
171
|
);
|
|
163
172
|
}
|
|
173
|
+
|
|
174
|
+
/** Prepared loaders carry caller-local parser bindings rather than shared metadata. */
|
|
175
|
+
const preparedLoaders = new WeakSet<Loader>();
|
|
176
|
+
/** Successfully prepared default graphs used by synchronous parsing. */
|
|
177
|
+
const preparedDefaults = new WeakMap<Loader, LoaderWithParser>();
|
|
178
|
+
/** In-flight preparations for default dependency graphs. */
|
|
179
|
+
const subloaderPreparations = new WeakMap<Loader, Promise<LoaderWithParser>>();
|
|
180
|
+
|
|
181
|
+
/** Resolves declared dependencies and binds parser entry points to their implementations. */
|
|
182
|
+
export async function preloadLoaderDependencies(
|
|
183
|
+
loader: Loader,
|
|
184
|
+
options?: LoaderOptions,
|
|
185
|
+
url?: string
|
|
186
|
+
): Promise<LoaderWithParser> {
|
|
187
|
+
if (preparedLoaders.has(loader)) return await getLoaderImplementation(loader, options, url);
|
|
188
|
+
validateSubloaderGraph(loader, options);
|
|
189
|
+
return await prepareLoaderDependencies(loader, options, url);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/** Prepares a node after the complete dependency graph has been validated. */
|
|
193
|
+
async function prepareLoaderDependencies(
|
|
194
|
+
loader: Loader,
|
|
195
|
+
options?: LoaderOptions,
|
|
196
|
+
url?: string
|
|
197
|
+
): Promise<LoaderWithParser> {
|
|
198
|
+
// Options can select nested backends or overrides; do not share their bindings.
|
|
199
|
+
if (!options) {
|
|
200
|
+
const cached = subloaderPreparations.get(loader);
|
|
201
|
+
if (cached) return await cached;
|
|
202
|
+
}
|
|
203
|
+
const preparation = bindSubloaders(loader, options, url);
|
|
204
|
+
if (!options) {
|
|
205
|
+
subloaderPreparations.set(loader, preparation);
|
|
206
|
+
void preparation.catch(() => subloaderPreparations.delete(loader));
|
|
207
|
+
}
|
|
208
|
+
const prepared = await preparation;
|
|
209
|
+
if (!options) preparedDefaults.set(loader, prepared);
|
|
210
|
+
return prepared;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Returns declared dependencies with the shared flat override map applied. */
|
|
214
|
+
function getSubloaders(loader: Loader, options?: LoaderOptions): Record<string, Loader> {
|
|
215
|
+
const overrides = options?.core?.subloaders || {};
|
|
216
|
+
return Object.fromEntries(
|
|
217
|
+
Object.entries(loader.subloaders || {}).map(([name, dependency]) => [
|
|
218
|
+
name,
|
|
219
|
+
Object.hasOwn(overrides, name) ? overrides[name] : dependency
|
|
220
|
+
])
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/** Rejects cycles in the dependency graph after applying the shared overrides. */
|
|
225
|
+
function validateSubloaderGraph(loader: Loader, options?: LoaderOptions): void {
|
|
226
|
+
visitLoader(loader, []);
|
|
227
|
+
|
|
228
|
+
/** Detects cycles along each resolved dependency path. */
|
|
229
|
+
function visitLoader(currentLoader: Loader, ancestors: Loader[]): void {
|
|
230
|
+
if (ancestors.includes(currentLoader)) {
|
|
231
|
+
throw new Error(
|
|
232
|
+
`Subloader cycle: ${[...ancestors, currentLoader].map(value => value.id).join(' -> ')}`
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
for (const dependency of Object.values(getSubloaders(currentLoader, options))) {
|
|
236
|
+
visitLoader(dependency, [...ancestors, currentLoader]);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** Prepares dependencies concurrently and injects them into parser calls' scoped options. */
|
|
242
|
+
async function bindSubloaders(
|
|
243
|
+
loader: Loader,
|
|
244
|
+
options?: LoaderOptions,
|
|
245
|
+
url?: string
|
|
246
|
+
): Promise<LoaderWithParser> {
|
|
247
|
+
const [implementation, entries] = await Promise.all([
|
|
248
|
+
getLoaderImplementation(loader, options, url),
|
|
249
|
+
Promise.all(
|
|
250
|
+
Object.entries(getSubloaders(loader, options)).map(
|
|
251
|
+
async ([name, dependency]) =>
|
|
252
|
+
[name, await prepareLoaderDependencies(dependency, options, url)] as const
|
|
253
|
+
)
|
|
254
|
+
)
|
|
255
|
+
]);
|
|
256
|
+
const subloaders = Object.fromEntries(entries);
|
|
257
|
+
// Expose preload settings as defaults before core normalizes a later parse call.
|
|
258
|
+
const defaultOptions = mergeOptions(
|
|
259
|
+
implementation.options,
|
|
260
|
+
(options || {}) as StrictLoaderOptions
|
|
261
|
+
);
|
|
262
|
+
defaultOptions.core = {...defaultOptions.core, worker: false};
|
|
263
|
+
const prepared = {...implementation, options: defaultOptions, subloaders};
|
|
264
|
+
delete prepared.preload;
|
|
265
|
+
// Function-valued dependency bindings cannot be transferred to a worker.
|
|
266
|
+
prepared.worker = false;
|
|
267
|
+
for (const method of [
|
|
268
|
+
'parseBlob',
|
|
269
|
+
'createDataSource',
|
|
270
|
+
'parse',
|
|
271
|
+
'parseSync',
|
|
272
|
+
'parseText',
|
|
273
|
+
'parseTextSync',
|
|
274
|
+
'parseInBatches',
|
|
275
|
+
'parseFile',
|
|
276
|
+
'parseFileInBatches',
|
|
277
|
+
'parseUrl'
|
|
278
|
+
] as const) {
|
|
279
|
+
const parser = (implementation as Record<string, unknown>)[method];
|
|
280
|
+
if (typeof parser === 'function') {
|
|
281
|
+
(prepared as Record<string, unknown>)[method] = (
|
|
282
|
+
data: unknown,
|
|
283
|
+
parseOptions?: LoaderOptions,
|
|
284
|
+
context?: unknown
|
|
285
|
+
) => {
|
|
286
|
+
const forwardedOptions = mergeOptions(options, parseOptions || {});
|
|
287
|
+
return parser.call(
|
|
288
|
+
implementation,
|
|
289
|
+
data,
|
|
290
|
+
{
|
|
291
|
+
...forwardedOptions,
|
|
292
|
+
core: {
|
|
293
|
+
...forwardedOptions.core,
|
|
294
|
+
subloaders: options?.core?.subloaders || parseOptions?.core?.subloaders,
|
|
295
|
+
worker: false
|
|
296
|
+
},
|
|
297
|
+
[loader.id]: {...(forwardedOptions[loader.id] as object), subloaders}
|
|
298
|
+
},
|
|
299
|
+
context
|
|
300
|
+
);
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
preparedLoaders.add(prepared);
|
|
305
|
+
return prepared;
|
|
306
|
+
}
|
package/src/lib/api/load.ts
CHANGED
|
@@ -13,12 +13,25 @@ import type {
|
|
|
13
13
|
LoaderReturnType,
|
|
14
14
|
LoaderArrayOptionsType,
|
|
15
15
|
LoaderArrayReturnType,
|
|
16
|
-
SourceLoader
|
|
16
|
+
SourceLoader,
|
|
17
|
+
SourcePropsType,
|
|
18
|
+
SourceArrayOptionsType,
|
|
19
|
+
DataSourceOptions,
|
|
20
|
+
LoaderWithParser
|
|
21
|
+
} from '@loaders.gl/loader-utils';
|
|
22
|
+
import {
|
|
23
|
+
isBlob,
|
|
24
|
+
isSourceLoader,
|
|
25
|
+
resolveAuthenticationOptions,
|
|
26
|
+
resolveLoaderAuthenticationOptions
|
|
17
27
|
} from '@loaders.gl/loader-utils';
|
|
18
|
-
import {isBlob, isSourceLoader} from '@loaders.gl/loader-utils';
|
|
19
28
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
20
29
|
import {getFetchFunction} from '../loader-utils/get-fetch-function';
|
|
21
|
-
import {
|
|
30
|
+
import {
|
|
31
|
+
applySearchParamsToUrl,
|
|
32
|
+
normalizeLoaderOptions,
|
|
33
|
+
getGlobalLoaderOptions
|
|
34
|
+
} from '../loader-utils/option-utils';
|
|
22
35
|
import {fetchFile} from '../fetch/fetch-file';
|
|
23
36
|
|
|
24
37
|
import {parse} from './parse';
|
|
@@ -27,6 +40,7 @@ import {parseSync} from './parse-sync';
|
|
|
27
40
|
import {parseInBatches} from './parse-in-batches';
|
|
28
41
|
import {loadInBatches} from './load-in-batches';
|
|
29
42
|
import {selectLoader} from './select-loader';
|
|
43
|
+
import {getLoaderImplementation} from './load-loader';
|
|
30
44
|
|
|
31
45
|
/**
|
|
32
46
|
* Parses `data` using a specified loader
|
|
@@ -47,7 +61,7 @@ export async function load<
|
|
|
47
61
|
>(
|
|
48
62
|
url: string | DataType,
|
|
49
63
|
loader: LoaderT,
|
|
50
|
-
options?: OptionsT,
|
|
64
|
+
options?: LoaderT extends SourceLoader ? SourcePropsType<LoaderT> & DataSourceOptions : OptionsT,
|
|
51
65
|
context?: LoaderContext
|
|
52
66
|
): Promise<LoaderReturnType<LoaderT>>;
|
|
53
67
|
|
|
@@ -57,7 +71,7 @@ export async function load<
|
|
|
57
71
|
>(
|
|
58
72
|
url: string | DataType,
|
|
59
73
|
loaders: LoaderArrayT,
|
|
60
|
-
options?: OptionsT,
|
|
74
|
+
options?: LoaderArrayT extends SourceLoader[] ? SourceArrayOptionsType<LoaderArrayT> : OptionsT,
|
|
61
75
|
context?: LoaderContext
|
|
62
76
|
): Promise<LoaderArrayReturnType<LoaderArrayT>>;
|
|
63
77
|
|
|
@@ -93,7 +107,15 @@ export async function load(
|
|
|
93
107
|
resolvedOptions = options as LoaderOptions;
|
|
94
108
|
}
|
|
95
109
|
|
|
110
|
+
resolvedOptions ||= getGlobalLoaderOptions();
|
|
111
|
+
|
|
96
112
|
if (!Array.isArray(resolvedLoaders) && isSourceLoader(resolvedLoaders)) {
|
|
113
|
+
const sourceLoader = await resolveSourceLoader(resolvedLoaders, url, resolvedOptions);
|
|
114
|
+
resolvedOptions = await resolveLoaderAuthenticationOptions(
|
|
115
|
+
resolvedLoaders.getAuthentications ? resolvedLoaders : sourceLoader,
|
|
116
|
+
typeof url === 'string' ? url : '',
|
|
117
|
+
resolvedOptions || {}
|
|
118
|
+
);
|
|
97
119
|
const runtimeCoreApi = {
|
|
98
120
|
fetchFile,
|
|
99
121
|
parse,
|
|
@@ -103,7 +125,7 @@ export async function load(
|
|
|
103
125
|
load,
|
|
104
126
|
loadInBatches
|
|
105
127
|
};
|
|
106
|
-
return
|
|
128
|
+
return sourceLoader.createDataSource(
|
|
107
129
|
url as string | Blob,
|
|
108
130
|
(resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
|
|
109
131
|
runtimeCoreApi
|
|
@@ -115,6 +137,12 @@ export async function load(
|
|
|
115
137
|
resolvedLoaders.length === 1 &&
|
|
116
138
|
isSourceLoader(resolvedLoaders[0])
|
|
117
139
|
) {
|
|
140
|
+
const sourceLoader = await resolveSourceLoader(resolvedLoaders[0], url, resolvedOptions);
|
|
141
|
+
resolvedOptions = await resolveLoaderAuthenticationOptions(
|
|
142
|
+
resolvedLoaders[0].getAuthentications ? resolvedLoaders[0] : sourceLoader,
|
|
143
|
+
typeof url === 'string' ? url : '',
|
|
144
|
+
resolvedOptions || {}
|
|
145
|
+
);
|
|
118
146
|
const runtimeCoreApi = {
|
|
119
147
|
fetchFile,
|
|
120
148
|
parse,
|
|
@@ -124,7 +152,7 @@ export async function load(
|
|
|
124
152
|
load,
|
|
125
153
|
loadInBatches
|
|
126
154
|
};
|
|
127
|
-
return
|
|
155
|
+
return sourceLoader.createDataSource(
|
|
128
156
|
url as string | Blob,
|
|
129
157
|
(resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
|
|
130
158
|
runtimeCoreApi
|
|
@@ -138,7 +166,13 @@ export async function load(
|
|
|
138
166
|
});
|
|
139
167
|
|
|
140
168
|
if (selectedLoader && isSourceLoader(selectedLoader)) {
|
|
141
|
-
|
|
169
|
+
const sourceLoader = await resolveSourceLoader(selectedLoader, url, resolvedOptions);
|
|
170
|
+
resolvedOptions = await resolveLoaderAuthenticationOptions(
|
|
171
|
+
selectedLoader.getAuthentications ? selectedLoader : sourceLoader,
|
|
172
|
+
typeof url === 'string' ? url : '',
|
|
173
|
+
resolvedOptions || {}
|
|
174
|
+
);
|
|
175
|
+
return sourceLoader.createDataSource(
|
|
142
176
|
url,
|
|
143
177
|
(resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
|
|
144
178
|
{
|
|
@@ -152,16 +186,43 @@ export async function load(
|
|
|
152
186
|
}
|
|
153
187
|
);
|
|
154
188
|
}
|
|
189
|
+
|
|
190
|
+
if (selectedLoader && typeof url === 'string') {
|
|
191
|
+
const loaderImplementation = await getLoaderImplementation(
|
|
192
|
+
selectedLoader,
|
|
193
|
+
resolvedOptions,
|
|
194
|
+
url
|
|
195
|
+
);
|
|
196
|
+
resolvedOptions = await resolveLoaderAuthenticationOptions(
|
|
197
|
+
selectedLoader.getAuthentications ? selectedLoader : loaderImplementation,
|
|
198
|
+
url,
|
|
199
|
+
resolvedOptions || {}
|
|
200
|
+
);
|
|
201
|
+
const parseUrl = (
|
|
202
|
+
loaderImplementation as LoaderWithParser & {
|
|
203
|
+
parseUrl?: (
|
|
204
|
+
url: string,
|
|
205
|
+
options?: LoaderOptions,
|
|
206
|
+
context?: LoaderContext
|
|
207
|
+
) => Promise<unknown>;
|
|
208
|
+
}
|
|
209
|
+
).parseUrl;
|
|
210
|
+
if (parseUrl) {
|
|
211
|
+
const requestUrl = applySearchParamsToUrl(url, resolvedOptions);
|
|
212
|
+
return await parseUrl(requestUrl, resolvedOptions, context);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
155
215
|
}
|
|
156
216
|
|
|
157
217
|
// Select fetch function
|
|
218
|
+
if (resolvedOptions) resolvedOptions = resolveAuthenticationOptions(resolvedOptions);
|
|
158
219
|
const fetch = getFetchFunction(resolvedOptions);
|
|
159
220
|
|
|
160
221
|
// at this point, `url` could be already loaded binary data
|
|
161
222
|
let data = url;
|
|
162
223
|
// url is a string, fetch the url
|
|
163
224
|
if (typeof url === 'string') {
|
|
164
|
-
data = await fetch(url);
|
|
225
|
+
data = await fetch(applySearchParamsToUrl(url, resolvedOptions));
|
|
165
226
|
// URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)
|
|
166
227
|
}
|
|
167
228
|
|
|
@@ -190,3 +251,20 @@ export async function load(
|
|
|
190
251
|
? await parse(data, resolvedLoaders, resolvedOptions, context) // loader array overload
|
|
191
252
|
: await parse(data, resolvedLoaders, resolvedOptions, context); // single loader overload
|
|
192
253
|
}
|
|
254
|
+
|
|
255
|
+
/** Resolves a lightweight source loader into its runtime implementation when necessary. */
|
|
256
|
+
async function resolveSourceLoader(
|
|
257
|
+
loader: SourceLoader,
|
|
258
|
+
url: string | DataType,
|
|
259
|
+
options?: LoaderOptions
|
|
260
|
+
): Promise<SourceLoader> {
|
|
261
|
+
if (!loader.preload) {
|
|
262
|
+
return loader;
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
const sourceLoader = await loader.preload(typeof url === 'string' ? url : '', options);
|
|
266
|
+
if (!isSourceLoader(sourceLoader)) {
|
|
267
|
+
throw new Error(`${loader.id} source loader preload() did not return a runtime source loader`);
|
|
268
|
+
}
|
|
269
|
+
return sourceLoader;
|
|
270
|
+
}
|
|
@@ -16,7 +16,11 @@ import type {
|
|
|
16
16
|
ReadableFile,
|
|
17
17
|
StrictLoaderOptions
|
|
18
18
|
} from '@loaders.gl/loader-utils';
|
|
19
|
-
import {
|
|
19
|
+
import {
|
|
20
|
+
isSourceLoader,
|
|
21
|
+
mergeOptions,
|
|
22
|
+
resolveLoaderAuthenticationOptions
|
|
23
|
+
} from '@loaders.gl/loader-utils';
|
|
20
24
|
import {validateWorkerVersion} from '@loaders.gl/worker-utils';
|
|
21
25
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
22
26
|
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
@@ -96,7 +100,16 @@ export async function parseFile(
|
|
|
96
100
|
}
|
|
97
101
|
|
|
98
102
|
// @ts-expect-error candidateLoaders may be a single forced loader
|
|
99
|
-
const
|
|
103
|
+
const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url);
|
|
104
|
+
const authenticationLoader =
|
|
105
|
+
!loader.getAuthentications && normalizedOptions.core?.credentials?.length
|
|
106
|
+
? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
|
|
107
|
+
: loader;
|
|
108
|
+
const strictOptions = await resolveLoaderAuthenticationOptions(
|
|
109
|
+
authenticationLoader,
|
|
110
|
+
url,
|
|
111
|
+
normalizedOptions
|
|
112
|
+
);
|
|
100
113
|
context = getLoaderContext(
|
|
101
114
|
// @ts-expect-error candidateLoaders may be a single forced loader
|
|
102
115
|
{url, _parse: parse, loaders: candidateLoaders},
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
import type {Batch} from '@loaders.gl/schema';
|
|
6
6
|
import {isTable, makeBatchFromTable} from '@loaders.gl/schema-utils';
|
|
7
|
+
import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
|
|
7
8
|
import type {
|
|
8
9
|
Loader,
|
|
9
10
|
LoaderWithParser,
|
|
@@ -19,7 +20,12 @@ import type {
|
|
|
19
20
|
LoaderArrayBatchType,
|
|
20
21
|
TransformBatches
|
|
21
22
|
} from '@loaders.gl/loader-utils';
|
|
22
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
canParseWithWorker,
|
|
25
|
+
concatenateArrayBuffersAsync,
|
|
26
|
+
isBrowser,
|
|
27
|
+
parseWithWorkerInBatches
|
|
28
|
+
} from '@loaders.gl/loader-utils';
|
|
23
29
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
24
30
|
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
25
31
|
import {getLoaderContext} from '../loader-utils/loader-context';
|
|
@@ -107,7 +113,16 @@ export async function parseInBatches(
|
|
|
107
113
|
}
|
|
108
114
|
|
|
109
115
|
// Normalize options
|
|
110
|
-
const
|
|
116
|
+
const normalizedOptions = normalizeOptions(options, loader, loaderArray, url);
|
|
117
|
+
const authenticationLoader =
|
|
118
|
+
!loader.getAuthentications && normalizedOptions.core?.credentials?.length
|
|
119
|
+
? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
|
|
120
|
+
: loader;
|
|
121
|
+
const strictOptions = await resolveLoaderAuthenticationOptions(
|
|
122
|
+
authenticationLoader,
|
|
123
|
+
url || '',
|
|
124
|
+
normalizedOptions
|
|
125
|
+
);
|
|
111
126
|
context = getLoaderContext(
|
|
112
127
|
{url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},
|
|
113
128
|
strictOptions,
|
|
@@ -177,8 +192,12 @@ async function parseToOutputIterator(
|
|
|
177
192
|
options?.core?.transforms || []
|
|
178
193
|
);
|
|
179
194
|
|
|
180
|
-
//
|
|
195
|
+
// Stream supported browser batch parsers through one stateful worker session.
|
|
181
196
|
if (loader.parseInBatches) {
|
|
197
|
+
// The loader-level adapter is intentionally piloted with CSV Arrow batches first.
|
|
198
|
+
if (loader.id === 'csv' && isBrowser && canParseWithWorker(loader, options)) {
|
|
199
|
+
return parseWithWorkerInBatches(loader, transformedIterator, options, context, parse);
|
|
200
|
+
}
|
|
182
201
|
return loader.parseInBatches(transformedIterator, options, context);
|
|
183
202
|
}
|
|
184
203
|
|
package/src/lib/api/parse.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
+
import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
|
|
5
6
|
import type {
|
|
6
7
|
Loader,
|
|
7
8
|
LoaderContext,
|
|
@@ -18,9 +19,10 @@ import type {
|
|
|
18
19
|
} from '@loaders.gl/loader-utils';
|
|
19
20
|
import {
|
|
20
21
|
parseWithWorker,
|
|
21
|
-
|
|
22
|
+
shouldParseWithWorker,
|
|
22
23
|
mergeOptions,
|
|
23
24
|
isResponse,
|
|
25
|
+
isBlob,
|
|
24
26
|
isSourceLoader
|
|
25
27
|
} from '@loaders.gl/loader-utils';
|
|
26
28
|
import {validateWorkerVersion} from '@loaders.gl/worker-utils';
|
|
@@ -120,7 +122,16 @@ export async function parse(
|
|
|
120
122
|
|
|
121
123
|
// Normalize options
|
|
122
124
|
// @ts-expect-error candidateLoaders
|
|
123
|
-
const
|
|
125
|
+
const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
|
|
126
|
+
const authenticationLoader =
|
|
127
|
+
!loader.getAuthentications && normalizedOptions.core?.credentials?.length
|
|
128
|
+
? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
|
|
129
|
+
: loader;
|
|
130
|
+
const strictOptions = await resolveLoaderAuthenticationOptions(
|
|
131
|
+
authenticationLoader,
|
|
132
|
+
url || '',
|
|
133
|
+
normalizedOptions
|
|
134
|
+
);
|
|
124
135
|
|
|
125
136
|
// Get a context (if already present, will be unchanged)
|
|
126
137
|
context = getLoaderContext(
|
|
@@ -155,11 +166,15 @@ async function parseWithLoader(
|
|
|
155
166
|
|
|
156
167
|
const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
|
|
157
168
|
|
|
158
|
-
if (
|
|
169
|
+
if (shouldParseWithWorker(loaderWithParser, data, options, context, loader)) {
|
|
159
170
|
data = await getArrayBufferFromData(data, options);
|
|
160
171
|
return await parseWithWorker(loaderWithParser, data, options, context, parse);
|
|
161
172
|
}
|
|
162
173
|
|
|
174
|
+
if (isBlob(data) && loaderWithParser.parseBlob) {
|
|
175
|
+
return await loaderWithParser.parseBlob(data, options, context);
|
|
176
|
+
}
|
|
177
|
+
|
|
163
178
|
data = await getArrayBufferOrStringFromData(data, loader, options);
|
|
164
179
|
|
|
165
180
|
return await parseWithLoaderImplementation(loaderWithParser, data, options, context);
|
package/src/lib/api/preload.ts
CHANGED
|
@@ -2,13 +2,23 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import type {Loader, LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
|
|
6
|
-
import {
|
|
5
|
+
import type {Loader, LoaderOptions, LoaderWithParser, SourceLoader} from '@loaders.gl/loader-utils';
|
|
6
|
+
import {
|
|
7
|
+
getLoaderImplementation,
|
|
8
|
+
getLoaderImplementationSync,
|
|
9
|
+
preloadLoaderDependencies
|
|
10
|
+
} from './load-loader';
|
|
7
11
|
|
|
8
12
|
/**
|
|
9
|
-
*
|
|
10
|
-
* May return a parser-bearing loader that also supports `parseSync`.
|
|
13
|
+
* Prepares a source implementation and its declared dependencies before source creation.
|
|
11
14
|
*/
|
|
15
|
+
export async function preload<SourceT extends SourceLoader>(
|
|
16
|
+
loader: SourceT,
|
|
17
|
+
options?: LoaderOptions,
|
|
18
|
+
url?: string
|
|
19
|
+
): Promise<SourceT>;
|
|
20
|
+
|
|
21
|
+
/** Prepares a parser and its recursively declared dependencies. */
|
|
12
22
|
export async function preload<LoaderT extends Loader>(
|
|
13
23
|
loader: LoaderT,
|
|
14
24
|
options?: LoaderOptions,
|
|
@@ -19,13 +29,18 @@ export async function preload(
|
|
|
19
29
|
loader: Loader,
|
|
20
30
|
options?: LoaderOptions,
|
|
21
31
|
url?: string
|
|
22
|
-
): Promise<LoaderWithParser> {
|
|
23
|
-
return
|
|
32
|
+
): Promise<LoaderWithParser | SourceLoader> {
|
|
33
|
+
return loader.subloaders
|
|
34
|
+
? await preloadLoaderDependencies(loader, options, url)
|
|
35
|
+
: await getLoaderImplementation(loader, options, url);
|
|
24
36
|
}
|
|
25
37
|
|
|
26
38
|
/**
|
|
27
39
|
* Returns a cached parser-bearing loader implementation if one is already available.
|
|
28
40
|
*/
|
|
29
|
-
export function preloadSync(loader:
|
|
41
|
+
export function preloadSync<SourceT extends SourceLoader>(loader: SourceT): SourceT | null;
|
|
42
|
+
/** Returns a cached parser, including prepared default dependency bindings. */
|
|
43
|
+
export function preloadSync(loader: Loader): LoaderWithParser | null;
|
|
44
|
+
export function preloadSync(loader: Loader): LoaderWithParser | SourceLoader | null {
|
|
30
45
|
return getLoaderImplementationSync(loader);
|
|
31
46
|
}
|
|
@@ -50,23 +50,34 @@ export class BrowserFileSystem implements FileSystem {
|
|
|
50
50
|
}
|
|
51
51
|
|
|
52
52
|
// Local fetches are served from the list of files
|
|
53
|
-
const file = this.
|
|
53
|
+
const file = this._getFile(path, true);
|
|
54
54
|
if (!file) {
|
|
55
55
|
return new Response(path, {status: 400, statusText: 'NOT FOUND'});
|
|
56
56
|
}
|
|
57
57
|
|
|
58
58
|
const headers = new Headers(options?.headers);
|
|
59
59
|
const range = headers.get('Range');
|
|
60
|
-
const bytes = range && /bytes=(
|
|
60
|
+
const bytes = range && /bytes=(\d+)-(\d+)/.exec(range);
|
|
61
61
|
|
|
62
62
|
if (bytes) {
|
|
63
63
|
const start = parseInt(bytes[1]);
|
|
64
|
-
const end = parseInt(bytes[2]);
|
|
64
|
+
const end = Math.min(parseInt(bytes[2]), file.size - 1);
|
|
65
|
+
if (start > end) {
|
|
66
|
+
const response = new Response(null, {
|
|
67
|
+
status: 416,
|
|
68
|
+
headers: {'Content-Range': `bytes */${file.size}`}
|
|
69
|
+
});
|
|
70
|
+
Object.defineProperty(response, 'url', {value: path});
|
|
71
|
+
return response;
|
|
72
|
+
}
|
|
65
73
|
// The trick when reading File objects is to read successive "slices" of the File
|
|
66
74
|
// Per spec https://w3c.github.io/FileAPI/, slicing a File should only update the start and end fields
|
|
67
75
|
// Actually reading from file should happen in `readAsArrayBuffer` (and as far we can tell it does)
|
|
68
|
-
const data = await file.slice(start, end).arrayBuffer();
|
|
69
|
-
const response = new Response(data
|
|
76
|
+
const data = await file.slice(start, end + 1).arrayBuffer();
|
|
77
|
+
const response = new Response(data, {
|
|
78
|
+
status: 206,
|
|
79
|
+
headers: {'Content-Range': `bytes ${start}-${end}/${file.size}`}
|
|
80
|
+
});
|
|
70
81
|
Object.defineProperty(response, 'url', {value: path});
|
|
71
82
|
return response;
|
|
72
83
|
}
|
|
@@ -95,7 +106,7 @@ export class BrowserFileSystem implements FileSystem {
|
|
|
95
106
|
* Return information (size) about files in this file system
|
|
96
107
|
*/
|
|
97
108
|
async stat(path: string, options?: object): Promise<{size: number}> {
|
|
98
|
-
const file = this.
|
|
109
|
+
const file = this._getFile(path, false);
|
|
99
110
|
if (!file) {
|
|
100
111
|
throw new Error(path);
|
|
101
112
|
}
|
|
@@ -106,16 +117,19 @@ export class BrowserFileSystem implements FileSystem {
|
|
|
106
117
|
* Just removes the file from the list
|
|
107
118
|
*/
|
|
108
119
|
async unlink(path: string): Promise<void> {
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
120
|
+
const file = this._getFile(path, false);
|
|
121
|
+
if (file) {
|
|
122
|
+
delete this.files[file.name];
|
|
123
|
+
delete this.lowerCaseFiles[file.name.toLowerCase()];
|
|
124
|
+
this.usedFiles[file.name] = true;
|
|
125
|
+
}
|
|
112
126
|
}
|
|
113
127
|
|
|
114
128
|
// implements IRandomAccessFileSystem
|
|
115
129
|
|
|
116
130
|
// RANDOM ACCESS
|
|
117
131
|
async openReadableFile(pathname: string, flags: unknown): Promise<ReadableFile> {
|
|
118
|
-
return new BlobFile(this.
|
|
132
|
+
return new BlobFile(this._getFile(pathname, true));
|
|
119
133
|
}
|
|
120
134
|
|
|
121
135
|
// PRIVATE
|
|
@@ -123,7 +137,7 @@ export class BrowserFileSystem implements FileSystem {
|
|
|
123
137
|
// Supports case independent paths, and file usage tracking
|
|
124
138
|
_getFile(path: string, used: boolean): File {
|
|
125
139
|
// Prefer case match, but fall back to case independent.
|
|
126
|
-
const file = this.files[path] || this.lowerCaseFiles[path];
|
|
140
|
+
const file = this.files[path] || this.lowerCaseFiles[path.toLowerCase()];
|
|
127
141
|
if (file && used) {
|
|
128
142
|
this.usedFiles[path] = true;
|
|
129
143
|
}
|