@loaders.gl/core 4.5.3 → 5.0.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -3
- package/dist/dist.dev.js +1789 -352
- package/dist/dist.min.js +4 -3
- package/dist/index.cjs +3087 -311
- package/dist/index.cjs.map +3 -4
- package/dist/index.d.ts +8 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/iterators/make-iterator/make-stream-iterator.js +48 -9
- package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -1
- package/dist/lib/api/core-api.d.ts +6 -0
- package/dist/lib/api/core-api.d.ts.map +1 -0
- package/dist/lib/api/core-api.js +23 -0
- package/dist/lib/api/core-api.js.map +1 -0
- package/dist/lib/api/create-data-source.d.ts +2 -2
- package/dist/lib/api/create-data-source.d.ts.map +1 -1
- package/dist/lib/api/create-data-source.js +36 -14
- package/dist/lib/api/create-data-source.js.map +1 -1
- package/dist/lib/api/load-in-batches.d.ts +4 -4
- package/dist/lib/api/load-in-batches.d.ts.map +1 -1
- package/dist/lib/api/load-in-batches.js +17 -6
- package/dist/lib/api/load-in-batches.js.map +1 -1
- package/dist/lib/api/load-loader.d.ts +10 -0
- package/dist/lib/api/load-loader.d.ts.map +1 -0
- package/dist/lib/api/load-loader.js +222 -0
- package/dist/lib/api/load-loader.js.map +1 -0
- package/dist/lib/api/load.d.ts +3 -3
- package/dist/lib/api/load.d.ts.map +1 -1
- package/dist/lib/api/load.js +84 -5
- package/dist/lib/api/load.js.map +1 -1
- package/dist/lib/api/parse-file.d.ts +15 -0
- package/dist/lib/api/parse-file.d.ts.map +1 -0
- package/dist/lib/api/parse-file.js +74 -0
- package/dist/lib/api/parse-file.js.map +1 -0
- package/dist/lib/api/parse-in-batches.d.ts +2 -2
- package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.js +18 -7
- package/dist/lib/api/parse-in-batches.js.map +1 -1
- package/dist/lib/api/parse-sync.d.ts +2 -2
- package/dist/lib/api/parse-sync.d.ts.map +1 -1
- package/dist/lib/api/parse-sync.js +10 -1
- package/dist/lib/api/parse-sync.js.map +1 -1
- package/dist/lib/api/parse.d.ts +2 -2
- package/dist/lib/api/parse.d.ts.map +1 -1
- package/dist/lib/api/parse.js +34 -17
- package/dist/lib/api/parse.js.map +1 -1
- package/dist/lib/api/preload.d.ts +14 -0
- package/dist/lib/api/preload.d.ts.map +1 -0
- package/dist/lib/api/preload.js +13 -0
- package/dist/lib/api/preload.js.map +1 -0
- package/dist/lib/api/register-loaders.js +1 -1
- package/dist/lib/api/register-loaders.js.map +1 -1
- package/dist/lib/api/select-loader.d.ts.map +1 -1
- package/dist/lib/api/select-loader.js +48 -5
- package/dist/lib/api/select-loader.js.map +1 -1
- package/dist/lib/api/select-source.d.ts +3 -3
- package/dist/lib/api/select-source.d.ts.map +1 -1
- package/dist/lib/api/select-source.js +2 -5
- package/dist/lib/api/select-source.js.map +1 -1
- package/dist/lib/fetch/fetch-error-message.js +1 -1
- package/dist/lib/fetch/fetch-error-message.js.map +1 -1
- package/dist/lib/filesystems/browser-filesystem.d.ts.map +1 -1
- package/dist/lib/filesystems/browser-filesystem.js +25 -11
- package/dist/lib/filesystems/browser-filesystem.js.map +1 -1
- package/dist/lib/init.js +1 -1
- package/dist/lib/init.js.map +1 -1
- package/dist/lib/loader-utils/check-errors.js +1 -1
- package/dist/lib/loader-utils/check-errors.js.map +1 -1
- package/dist/lib/loader-utils/get-data.d.ts +4 -0
- package/dist/lib/loader-utils/get-data.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-data.js +26 -0
- package/dist/lib/loader-utils/get-data.js.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.d.ts +1 -1
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.js +16 -10
- package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts +2 -2
- package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
- package/dist/lib/loader-utils/loader-context.js +11 -4
- package/dist/lib/loader-utils/loader-context.js.map +1 -1
- package/dist/lib/loader-utils/loggers.d.ts +19 -0
- package/dist/lib/loader-utils/loggers.d.ts.map +1 -1
- package/dist/lib/loader-utils/loggers.js +28 -2
- package/dist/lib/loader-utils/loggers.js.map +1 -1
- package/dist/lib/loader-utils/option-defaults.d.ts +1 -0
- package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-defaults.js +5 -0
- package/dist/lib/loader-utils/option-defaults.js.map +1 -1
- package/dist/lib/loader-utils/option-utils.d.ts +7 -0
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +91 -3
- package/dist/lib/loader-utils/option-utils.js.map +1 -1
- package/dist/lib/utils/response-utils.js +5 -14
- package/dist/lib/utils/response-utils.js.map +1 -1
- package/dist/null-loader.d.ts.map +1 -1
- package/dist/null-loader.js +1 -1
- package/dist/null-loader.js.map +1 -1
- package/dist/null-worker-node.js +270 -42
- package/dist/null-worker.js +270 -42
- package/package.json +6 -6
- package/src/index.ts +10 -1
- package/src/iterators/make-iterator/make-stream-iterator.ts +47 -9
- package/src/lib/api/core-api.ts +25 -0
- package/src/lib/api/create-data-source.ts +53 -24
- package/src/lib/api/load-in-batches.ts +33 -21
- package/src/lib/api/load-loader.ts +306 -0
- package/src/lib/api/load.ts +160 -9
- package/src/lib/api/parse-file.ts +156 -0
- package/src/lib/api/parse-in-batches.ts +35 -8
- package/src/lib/api/parse-sync.ts +27 -2
- package/src/lib/api/parse.ts +61 -20
- package/src/lib/api/preload.ts +46 -0
- package/src/lib/api/register-loaders.ts +1 -1
- package/src/lib/api/select-loader.ts +57 -5
- package/src/lib/api/select-source.ts +7 -10
- package/src/lib/fetch/fetch-error-message.ts +1 -1
- package/src/lib/filesystems/browser-filesystem.ts +25 -11
- package/src/lib/loader-utils/check-errors.ts +1 -1
- package/src/lib/loader-utils/get-data.ts +36 -0
- package/src/lib/loader-utils/get-fetch-function.ts +17 -15
- package/src/lib/loader-utils/loader-context.ts +13 -5
- package/src/lib/loader-utils/loggers.ts +28 -2
- package/src/lib/loader-utils/option-defaults.ts +5 -0
- package/src/lib/loader-utils/option-utils.ts +106 -3
- package/src/lib/utils/response-utils.ts +5 -15
- package/src/null-loader.ts +3 -3
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {
|
|
6
|
+
Loader,
|
|
7
|
+
LoaderContext,
|
|
8
|
+
LoaderOptions,
|
|
9
|
+
LoaderOptionsWithShape,
|
|
10
|
+
LoaderOptionsType,
|
|
11
|
+
LoaderShapeType,
|
|
12
|
+
LoaderReturnType,
|
|
13
|
+
LoaderArrayOptionsType,
|
|
14
|
+
LoaderArrayReturnType,
|
|
15
|
+
LoaderWithParser,
|
|
16
|
+
ReadableFile,
|
|
17
|
+
StrictLoaderOptions
|
|
18
|
+
} from '@loaders.gl/loader-utils';
|
|
19
|
+
import {
|
|
20
|
+
isSourceLoader,
|
|
21
|
+
mergeOptions,
|
|
22
|
+
resolveLoaderAuthenticationOptions
|
|
23
|
+
} from '@loaders.gl/loader-utils';
|
|
24
|
+
import {validateWorkerVersion} from '@loaders.gl/worker-utils';
|
|
25
|
+
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
26
|
+
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
27
|
+
import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
|
|
28
|
+
import {getLoaderImplementation} from './load-loader';
|
|
29
|
+
import {parse} from './parse';
|
|
30
|
+
import {selectLoader} from './select-loader';
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Parses a random access readable file asynchronously using the supplied loader.
|
|
34
|
+
*/
|
|
35
|
+
export async function parseFile<
|
|
36
|
+
LoaderT extends Loader,
|
|
37
|
+
OptionsT extends LoaderOptions = LoaderOptionsWithShape<
|
|
38
|
+
LoaderOptionsType<LoaderT>,
|
|
39
|
+
LoaderShapeType<LoaderT>
|
|
40
|
+
>
|
|
41
|
+
>(
|
|
42
|
+
file: ReadableFile,
|
|
43
|
+
loader: LoaderT,
|
|
44
|
+
options?: OptionsT,
|
|
45
|
+
context?: LoaderContext
|
|
46
|
+
): Promise<LoaderReturnType<LoaderT>>;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Parses a random access readable file asynchronously by matching one of the supplied loaders.
|
|
50
|
+
*/
|
|
51
|
+
export async function parseFile<
|
|
52
|
+
LoaderArrayT extends Loader[],
|
|
53
|
+
OptionsT extends LoaderOptions = LoaderArrayOptionsType<LoaderArrayT>
|
|
54
|
+
>(
|
|
55
|
+
file: ReadableFile,
|
|
56
|
+
loaders: LoaderArrayT,
|
|
57
|
+
options?: OptionsT,
|
|
58
|
+
context?: LoaderContext
|
|
59
|
+
): Promise<LoaderArrayReturnType<LoaderArrayT>>;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Parses a random access readable file asynchronously by matching a pre-registered loader.
|
|
63
|
+
* @deprecated Loader registration is deprecated, use parseFile(file, loaders, options) instead.
|
|
64
|
+
*/
|
|
65
|
+
export async function parseFile(file: ReadableFile, options?: LoaderOptions): Promise<unknown>;
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Parses a random access readable file using a specified loader.
|
|
69
|
+
* @param file readable file
|
|
70
|
+
* @param loaders loader or loaders to use
|
|
71
|
+
* @param options loader options
|
|
72
|
+
* @param context loader context
|
|
73
|
+
* @returns parsed data
|
|
74
|
+
*/
|
|
75
|
+
export async function parseFile(
|
|
76
|
+
file: ReadableFile,
|
|
77
|
+
loaders?: Loader | Loader[] | LoaderOptions,
|
|
78
|
+
options?: LoaderOptions,
|
|
79
|
+
context?: LoaderContext
|
|
80
|
+
): Promise<unknown> {
|
|
81
|
+
if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
82
|
+
context = undefined;
|
|
83
|
+
options = loaders as LoaderOptions;
|
|
84
|
+
loaders = undefined;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
options = options || {};
|
|
88
|
+
const url = file.url || '';
|
|
89
|
+
const typedLoaders = loaders as Loader | Loader[] | undefined;
|
|
90
|
+
const candidateLoaders = getLoadersFromContext(typedLoaders, context);
|
|
91
|
+
const loader = await selectLoader(url, candidateLoaders, options);
|
|
92
|
+
if (!loader) {
|
|
93
|
+
return null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (isSourceLoader(loader)) {
|
|
97
|
+
throw new Error(
|
|
98
|
+
`${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parseFile().`
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// @ts-expect-error candidateLoaders may be a single forced loader
|
|
103
|
+
const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url);
|
|
104
|
+
const authenticationLoader =
|
|
105
|
+
!loader.getAuthentications && normalizedOptions.core?.credentials?.length
|
|
106
|
+
? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
|
|
107
|
+
: loader;
|
|
108
|
+
const strictOptions = await resolveLoaderAuthenticationOptions(
|
|
109
|
+
authenticationLoader,
|
|
110
|
+
url,
|
|
111
|
+
normalizedOptions
|
|
112
|
+
);
|
|
113
|
+
context = getLoaderContext(
|
|
114
|
+
// @ts-expect-error candidateLoaders may be a single forced loader
|
|
115
|
+
{url, _parse: parse, loaders: candidateLoaders},
|
|
116
|
+
strictOptions,
|
|
117
|
+
context || null
|
|
118
|
+
);
|
|
119
|
+
|
|
120
|
+
return await parseFileWithLoader(loader, file, strictOptions, context);
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Parses a readable file after a loader has been selected.
|
|
125
|
+
* @param loader selected loader
|
|
126
|
+
* @param file readable file
|
|
127
|
+
* @param options strict loader options
|
|
128
|
+
* @param context loader context
|
|
129
|
+
* @returns parsed data
|
|
130
|
+
*/
|
|
131
|
+
async function parseFileWithLoader(
|
|
132
|
+
loader: Loader,
|
|
133
|
+
file: ReadableFile,
|
|
134
|
+
options: StrictLoaderOptions,
|
|
135
|
+
context: LoaderContext
|
|
136
|
+
): Promise<unknown> {
|
|
137
|
+
validateWorkerVersion(loader);
|
|
138
|
+
options = mergeOptions(loader.options, options);
|
|
139
|
+
|
|
140
|
+
const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
|
|
141
|
+
if (loaderWithParser.parseFile) {
|
|
142
|
+
return await loaderWithParser.parseFile(file, options, context);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const parseFunction = (loaderWithParser as {parse?: LoaderWithParser['parse']}).parse;
|
|
146
|
+
if (parseFunction) {
|
|
147
|
+
const length = file.bigsize > 0n ? Number(file.bigsize) : file.size;
|
|
148
|
+
if (!length) {
|
|
149
|
+
throw new Error(`${loader.id} loader - cannot fall back to parse without a file size`);
|
|
150
|
+
}
|
|
151
|
+
const data = await file.read(0n, length);
|
|
152
|
+
return await parse(data, loaderWithParser, options, context);
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
throw new Error(`${loader.id} loader - no file parser found`);
|
|
156
|
+
}
|
|
@@ -4,25 +4,34 @@
|
|
|
4
4
|
|
|
5
5
|
import type {Batch} from '@loaders.gl/schema';
|
|
6
6
|
import {isTable, makeBatchFromTable} from '@loaders.gl/schema-utils';
|
|
7
|
+
import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
|
|
7
8
|
import type {
|
|
8
9
|
Loader,
|
|
9
10
|
LoaderWithParser,
|
|
10
11
|
StrictLoaderOptions,
|
|
11
12
|
LoaderOptions,
|
|
12
13
|
LoaderContext,
|
|
14
|
+
LoaderOptionsWithShape,
|
|
13
15
|
BatchableDataType,
|
|
14
16
|
LoaderOptionsType,
|
|
17
|
+
LoaderShapeType,
|
|
15
18
|
LoaderBatchType,
|
|
16
19
|
LoaderArrayOptionsType,
|
|
17
20
|
LoaderArrayBatchType,
|
|
18
21
|
TransformBatches
|
|
19
22
|
} from '@loaders.gl/loader-utils';
|
|
20
|
-
import {
|
|
23
|
+
import {
|
|
24
|
+
canParseWithWorker,
|
|
25
|
+
concatenateArrayBuffersAsync,
|
|
26
|
+
isBrowser,
|
|
27
|
+
parseWithWorkerInBatches
|
|
28
|
+
} from '@loaders.gl/loader-utils';
|
|
21
29
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
22
30
|
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
23
31
|
import {getLoaderContext} from '../loader-utils/loader-context';
|
|
24
32
|
import {getAsyncIterableFromData} from '../loader-utils/get-data';
|
|
25
33
|
import {getResourceUrl} from '../utils/resource-utils';
|
|
34
|
+
import {getLoaderImplementation} from './load-loader';
|
|
26
35
|
import {selectLoader} from './select-loader';
|
|
27
36
|
|
|
28
37
|
// Ensure `parse` is available in context if loader falls back to `parse`
|
|
@@ -33,7 +42,10 @@ import {parse} from './parse';
|
|
|
33
42
|
*/
|
|
34
43
|
export async function parseInBatches<
|
|
35
44
|
LoaderT extends Loader,
|
|
36
|
-
OptionsT extends LoaderOptions =
|
|
45
|
+
OptionsT extends LoaderOptions = LoaderOptionsWithShape<
|
|
46
|
+
LoaderOptionsType<LoaderT>,
|
|
47
|
+
LoaderShapeType<LoaderT>
|
|
48
|
+
>
|
|
37
49
|
>(
|
|
38
50
|
data: BatchableDataType,
|
|
39
51
|
loader: LoaderT,
|
|
@@ -101,20 +113,31 @@ export async function parseInBatches(
|
|
|
101
113
|
}
|
|
102
114
|
|
|
103
115
|
// Normalize options
|
|
104
|
-
const
|
|
116
|
+
const normalizedOptions = normalizeOptions(options, loader, loaderArray, url);
|
|
117
|
+
const authenticationLoader =
|
|
118
|
+
!loader.getAuthentications && normalizedOptions.core?.credentials?.length
|
|
119
|
+
? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
|
|
120
|
+
: loader;
|
|
121
|
+
const strictOptions = await resolveLoaderAuthenticationOptions(
|
|
122
|
+
authenticationLoader,
|
|
123
|
+
url || '',
|
|
124
|
+
normalizedOptions
|
|
125
|
+
);
|
|
105
126
|
context = getLoaderContext(
|
|
106
127
|
{url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},
|
|
107
128
|
strictOptions,
|
|
108
129
|
context || null
|
|
109
130
|
);
|
|
110
131
|
|
|
111
|
-
|
|
132
|
+
const loaderWithParser = await getLoaderImplementation(loader, strictOptions, context.url);
|
|
133
|
+
return await parseWithLoaderInBatches(loader, loaderWithParser, data, strictOptions, context);
|
|
112
134
|
}
|
|
113
135
|
|
|
114
136
|
/**
|
|
115
137
|
* Loader has been selected and context has been prepared, see if we need to emit a metadata batch
|
|
116
138
|
*/
|
|
117
139
|
async function parseWithLoaderInBatches(
|
|
140
|
+
originalLoader: Loader,
|
|
118
141
|
loader: LoaderWithParser,
|
|
119
142
|
data: BatchableDataType,
|
|
120
143
|
options: StrictLoaderOptions,
|
|
@@ -131,7 +154,7 @@ async function parseWithLoaderInBatches(
|
|
|
131
154
|
shape: 'metadata',
|
|
132
155
|
batchType: 'metadata',
|
|
133
156
|
metadata: {
|
|
134
|
-
_loader:
|
|
157
|
+
_loader: originalLoader,
|
|
135
158
|
_context: context
|
|
136
159
|
},
|
|
137
160
|
// Populate with some default fields to avoid crashing
|
|
@@ -169,8 +192,12 @@ async function parseToOutputIterator(
|
|
|
169
192
|
options?.core?.transforms || []
|
|
170
193
|
);
|
|
171
194
|
|
|
172
|
-
//
|
|
195
|
+
// Stream supported browser batch parsers through one stateful worker session.
|
|
173
196
|
if (loader.parseInBatches) {
|
|
197
|
+
// The loader-level adapter is intentionally piloted with CSV Arrow batches first.
|
|
198
|
+
if (loader.id === 'csv' && isBrowser && canParseWithWorker(loader, options)) {
|
|
199
|
+
return parseWithWorkerInBatches(loader, transformedIterator, options, context, parse);
|
|
200
|
+
}
|
|
174
201
|
return loader.parseInBatches(transformedIterator, options, context);
|
|
175
202
|
}
|
|
176
203
|
|
|
@@ -182,7 +209,7 @@ async function* parseChunkInBatches(
|
|
|
182
209
|
transformedIterator:
|
|
183
210
|
| Iterable<ArrayBufferLike | ArrayBufferView>
|
|
184
211
|
| AsyncIterable<ArrayBufferLike | ArrayBufferView>,
|
|
185
|
-
loader:
|
|
212
|
+
loader: LoaderWithParser,
|
|
186
213
|
options: StrictLoaderOptions,
|
|
187
214
|
context: LoaderContext
|
|
188
215
|
): AsyncIterable<Batch> {
|
|
@@ -207,7 +234,7 @@ async function* parseChunkInBatches(
|
|
|
207
234
|
* @todo run through batch builder to apply options etc...
|
|
208
235
|
*/
|
|
209
236
|
function convertDataToBatch(parsedData: unknown, loader: Loader): Batch {
|
|
210
|
-
//
|
|
237
|
+
// biome-ignore format: preserve intentional fixture layout
|
|
211
238
|
const batch: Batch = isTable(parsedData)
|
|
212
239
|
? makeBatchFromTable(parsedData)
|
|
213
240
|
: {
|
|
@@ -7,19 +7,23 @@ import type {
|
|
|
7
7
|
LoaderWithParser,
|
|
8
8
|
LoaderOptions,
|
|
9
9
|
LoaderContext,
|
|
10
|
+
LoaderOptionsWithShape,
|
|
10
11
|
SyncDataType,
|
|
11
12
|
LoaderOptionsType,
|
|
13
|
+
LoaderShapeType,
|
|
12
14
|
LoaderReturnType,
|
|
13
15
|
LoaderArrayOptionsType,
|
|
14
16
|
LoaderArrayReturnType,
|
|
15
17
|
StrictLoaderOptions
|
|
16
18
|
} from '@loaders.gl/loader-utils';
|
|
19
|
+
import {isSourceLoader} from '@loaders.gl/loader-utils';
|
|
17
20
|
import {selectLoaderSync} from './select-loader';
|
|
18
21
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
19
22
|
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
20
23
|
import {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';
|
|
21
24
|
import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
|
|
22
25
|
import {getResourceUrl} from '../utils/resource-utils';
|
|
26
|
+
import {getLoaderImplementationSync} from './load-loader';
|
|
23
27
|
|
|
24
28
|
// OVERLOADS
|
|
25
29
|
|
|
@@ -28,7 +32,10 @@ import {getResourceUrl} from '../utils/resource-utils';
|
|
|
28
32
|
*/
|
|
29
33
|
export function parseSync<
|
|
30
34
|
LoaderT extends Loader,
|
|
31
|
-
OptionsT extends LoaderOptions =
|
|
35
|
+
OptionsT extends LoaderOptions = LoaderOptionsWithShape<
|
|
36
|
+
LoaderOptionsType<LoaderT>,
|
|
37
|
+
LoaderShapeType<LoaderT>
|
|
38
|
+
>
|
|
32
39
|
>(
|
|
33
40
|
data: SyncDataType,
|
|
34
41
|
loader: LoaderT,
|
|
@@ -84,6 +91,12 @@ export function parseSync(
|
|
|
84
91
|
return null;
|
|
85
92
|
}
|
|
86
93
|
|
|
94
|
+
if (isSourceLoader(loader)) {
|
|
95
|
+
throw new Error(
|
|
96
|
+
`${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parseSync().`
|
|
97
|
+
);
|
|
98
|
+
}
|
|
99
|
+
|
|
87
100
|
// Normalize options
|
|
88
101
|
const strictOptions = normalizeOptions(options, loader, candidateLoaders as Loader[] | undefined);
|
|
89
102
|
|
|
@@ -99,7 +112,13 @@ export function parseSync(
|
|
|
99
112
|
context || null
|
|
100
113
|
);
|
|
101
114
|
|
|
102
|
-
|
|
115
|
+
const loaderWithParser = getLoaderImplementationSync(loader);
|
|
116
|
+
return parseWithLoaderSync(
|
|
117
|
+
loaderWithParser || (loader as LoaderWithParser),
|
|
118
|
+
data,
|
|
119
|
+
strictOptions,
|
|
120
|
+
context
|
|
121
|
+
);
|
|
103
122
|
}
|
|
104
123
|
|
|
105
124
|
// TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator
|
|
@@ -119,6 +138,12 @@ function parseWithLoaderSync(
|
|
|
119
138
|
return loader.parseSync(data, options, context); // , loader);
|
|
120
139
|
}
|
|
121
140
|
|
|
141
|
+
if (loader.preload) {
|
|
142
|
+
throw new Error(
|
|
143
|
+
`${loader.name} loader: 'parseSync' requires a parser-bearing loader. Import the loader implementation directly, or call preload(loader) before parseSync(). ${context.url || ''}`
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
|
|
122
147
|
// TBD - If synchronous parser not available, return null
|
|
123
148
|
throw new Error(
|
|
124
149
|
`${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${
|
package/src/lib/api/parse.ts
CHANGED
|
@@ -2,13 +2,16 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
+
import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
|
|
5
6
|
import type {
|
|
6
7
|
Loader,
|
|
7
8
|
LoaderContext,
|
|
8
9
|
LoaderOptions,
|
|
10
|
+
LoaderOptionsWithShape,
|
|
9
11
|
DataType,
|
|
10
12
|
LoaderWithParser,
|
|
11
13
|
LoaderOptionsType,
|
|
14
|
+
LoaderShapeType,
|
|
12
15
|
LoaderReturnType,
|
|
13
16
|
LoaderArrayOptionsType,
|
|
14
17
|
LoaderArrayReturnType,
|
|
@@ -16,16 +19,20 @@ import type {
|
|
|
16
19
|
} from '@loaders.gl/loader-utils';
|
|
17
20
|
import {
|
|
18
21
|
parseWithWorker,
|
|
19
|
-
|
|
22
|
+
shouldParseWithWorker,
|
|
20
23
|
mergeOptions,
|
|
21
|
-
isResponse
|
|
24
|
+
isResponse,
|
|
25
|
+
isBlob,
|
|
26
|
+
isSourceLoader
|
|
22
27
|
} from '@loaders.gl/loader-utils';
|
|
23
|
-
import {
|
|
28
|
+
import {validateWorkerVersion} from '@loaders.gl/worker-utils';
|
|
24
29
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
25
30
|
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
26
31
|
import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
|
|
32
|
+
import {getArrayBufferFromData} from '../loader-utils/get-data';
|
|
27
33
|
import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
|
|
28
34
|
import {getResourceUrl} from '../utils/resource-utils';
|
|
35
|
+
import {getLoaderImplementation} from './load-loader';
|
|
29
36
|
import {selectLoader} from './select-loader';
|
|
30
37
|
|
|
31
38
|
// type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T
|
|
@@ -35,7 +42,10 @@ import {selectLoader} from './select-loader';
|
|
|
35
42
|
*/
|
|
36
43
|
export async function parse<
|
|
37
44
|
LoaderT extends Loader,
|
|
38
|
-
OptionsT extends LoaderOptions =
|
|
45
|
+
OptionsT extends LoaderOptions = LoaderOptionsWithShape<
|
|
46
|
+
LoaderOptionsType<LoaderT>,
|
|
47
|
+
LoaderShapeType<LoaderT>
|
|
48
|
+
>
|
|
39
49
|
>(
|
|
40
50
|
data: DataType | Promise<DataType>,
|
|
41
51
|
loader: LoaderT,
|
|
@@ -104,9 +114,24 @@ export async function parse(
|
|
|
104
114
|
return null;
|
|
105
115
|
}
|
|
106
116
|
|
|
117
|
+
if (isSourceLoader(loader)) {
|
|
118
|
+
throw new Error(
|
|
119
|
+
`${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parse().`
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
|
|
107
123
|
// Normalize options
|
|
108
124
|
// @ts-expect-error candidateLoaders
|
|
109
|
-
const
|
|
125
|
+
const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
|
|
126
|
+
const authenticationLoader =
|
|
127
|
+
!loader.getAuthentications && normalizedOptions.core?.credentials?.length
|
|
128
|
+
? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
|
|
129
|
+
: loader;
|
|
130
|
+
const strictOptions = await resolveLoaderAuthenticationOptions(
|
|
131
|
+
authenticationLoader,
|
|
132
|
+
url || '',
|
|
133
|
+
normalizedOptions
|
|
134
|
+
);
|
|
110
135
|
|
|
111
136
|
// Get a context (if already present, will be unchanged)
|
|
112
137
|
context = getLoaderContext(
|
|
@@ -139,31 +164,47 @@ async function parseWithLoader(
|
|
|
139
164
|
context.response = {headers, ok, redirected, status, statusText, type, url};
|
|
140
165
|
}
|
|
141
166
|
|
|
142
|
-
|
|
167
|
+
const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
|
|
143
168
|
|
|
144
|
-
|
|
169
|
+
if (shouldParseWithWorker(loaderWithParser, data, options, context, loader)) {
|
|
170
|
+
data = await getArrayBufferFromData(data, options);
|
|
171
|
+
return await parseWithWorker(loaderWithParser, data, options, context, parse);
|
|
172
|
+
}
|
|
145
173
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
return loaderWithParser.parseTextSync(data, options, context);
|
|
174
|
+
if (isBlob(data) && loaderWithParser.parseBlob) {
|
|
175
|
+
return await loaderWithParser.parseBlob(data, options, context);
|
|
149
176
|
}
|
|
150
177
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
178
|
+
data = await getArrayBufferOrStringFromData(data, loader, options);
|
|
179
|
+
|
|
180
|
+
return await parseWithLoaderImplementation(loaderWithParser, data, options, context);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async function parseWithLoaderImplementation(
|
|
184
|
+
loader: LoaderWithParser,
|
|
185
|
+
data: string | ArrayBuffer,
|
|
186
|
+
options: StrictLoaderOptions,
|
|
187
|
+
context: LoaderContext
|
|
188
|
+
): Promise<unknown> {
|
|
189
|
+
if (loader.parseText && typeof data === 'string') {
|
|
190
|
+
return await loader.parseText(data, options, context);
|
|
154
191
|
}
|
|
155
192
|
|
|
156
|
-
//
|
|
157
|
-
if (
|
|
158
|
-
return
|
|
193
|
+
// Fall back to synchronous text parser, wrap results in promises
|
|
194
|
+
if (loader.parseTextSync && typeof data === 'string') {
|
|
195
|
+
return loader.parseTextSync(data, options, context);
|
|
159
196
|
}
|
|
160
197
|
|
|
161
|
-
if (
|
|
162
|
-
return await
|
|
198
|
+
if (loader.parse) {
|
|
199
|
+
return await loader.parse(data as ArrayBuffer, options, context);
|
|
163
200
|
}
|
|
164
201
|
|
|
165
|
-
//
|
|
166
|
-
|
|
202
|
+
// All parser-bearing sync loaders should also offer `parse` so async APIs can use them.
|
|
203
|
+
if (loader.parseSync) {
|
|
204
|
+
throw new Error(
|
|
205
|
+
`${loader.name} loader: 'parse' not available on parser-bearing sync loader. Add an async 'parse' function to this loader, or call preload(loader) before parseSync(). ${context.url || ''}`
|
|
206
|
+
);
|
|
207
|
+
}
|
|
167
208
|
|
|
168
209
|
// TBD - If asynchronous parser not available, return null
|
|
169
210
|
throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {Loader, LoaderOptions, LoaderWithParser, SourceLoader} from '@loaders.gl/loader-utils';
|
|
6
|
+
import {
|
|
7
|
+
getLoaderImplementation,
|
|
8
|
+
getLoaderImplementationSync,
|
|
9
|
+
preloadLoaderDependencies
|
|
10
|
+
} from './load-loader';
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Prepares a source implementation and its declared dependencies before source creation.
|
|
14
|
+
*/
|
|
15
|
+
export async function preload<SourceT extends SourceLoader>(
|
|
16
|
+
loader: SourceT,
|
|
17
|
+
options?: LoaderOptions,
|
|
18
|
+
url?: string
|
|
19
|
+
): Promise<SourceT>;
|
|
20
|
+
|
|
21
|
+
/** Prepares a parser and its recursively declared dependencies. */
|
|
22
|
+
export async function preload<LoaderT extends Loader>(
|
|
23
|
+
loader: LoaderT,
|
|
24
|
+
options?: LoaderOptions,
|
|
25
|
+
url?: string
|
|
26
|
+
): Promise<LoaderWithParser>;
|
|
27
|
+
|
|
28
|
+
export async function preload(
|
|
29
|
+
loader: Loader,
|
|
30
|
+
options?: LoaderOptions,
|
|
31
|
+
url?: string
|
|
32
|
+
): Promise<LoaderWithParser | SourceLoader> {
|
|
33
|
+
return loader.subloaders
|
|
34
|
+
? await preloadLoaderDependencies(loader, options, url)
|
|
35
|
+
: await getLoaderImplementation(loader, options, url);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Returns a cached parser-bearing loader implementation if one is already available.
|
|
40
|
+
*/
|
|
41
|
+
export function preloadSync<SourceT extends SourceLoader>(loader: SourceT): SourceT | null;
|
|
42
|
+
/** Returns a cached parser, including prepared default dependency bindings. */
|
|
43
|
+
export function preloadSync(loader: Loader): LoaderWithParser | null;
|
|
44
|
+
export function preloadSync(loader: Loader): LoaderWithParser | SourceLoader | null {
|
|
45
|
+
return getLoaderImplementationSync(loader);
|
|
46
|
+
}
|
|
@@ -28,7 +28,7 @@ export function registerLoaders(loaders: Loader[] | Loader) {
|
|
|
28
28
|
|
|
29
29
|
for (const loader of loaders) {
|
|
30
30
|
const normalizedLoader = normalizeLoader(loader);
|
|
31
|
-
if (!loaderRegistry.find(
|
|
31
|
+
if (!loaderRegistry.find(registeredLoader => normalizedLoader === registeredLoader)) {
|
|
32
32
|
// add to the beginning of the loaderRegistry, so the last registeredLoader get picked
|
|
33
33
|
loaderRegistry.unshift(normalizedLoader);
|
|
34
34
|
}
|
|
@@ -9,7 +9,8 @@ import {
|
|
|
9
9
|
log,
|
|
10
10
|
isBlob,
|
|
11
11
|
ensureArrayBuffer,
|
|
12
|
-
isArrayBufferLike
|
|
12
|
+
isArrayBufferLike,
|
|
13
|
+
isSourceLoader
|
|
13
14
|
} from '@loaders.gl/loader-utils';
|
|
14
15
|
import {TypedArray} from '@loaders.gl/schema';
|
|
15
16
|
import {normalizeLoader} from '../loader-utils/normalize-loader';
|
|
@@ -95,7 +96,9 @@ function mayContainText(response: Response): boolean {
|
|
|
95
96
|
const mimeType = getResourceMIMEType(response);
|
|
96
97
|
return Boolean(
|
|
97
98
|
mimeType &&
|
|
98
|
-
|
|
99
|
+
(mimeType.startsWith('text/') ||
|
|
100
|
+
mimeType === 'application/json' ||
|
|
101
|
+
mimeType.endsWith('+json'))
|
|
99
102
|
);
|
|
100
103
|
}
|
|
101
104
|
|
|
@@ -169,12 +172,23 @@ function selectLoaderInternal(
|
|
|
169
172
|
let loader: Loader | null = null;
|
|
170
173
|
let reason: string = '';
|
|
171
174
|
|
|
175
|
+
// if options.mimeType is supplied, it takes precedence
|
|
176
|
+
const sourceType =
|
|
177
|
+
options?.core && 'type' in options.core ? (options.core.type as string | undefined) : undefined;
|
|
178
|
+
if (sourceType && sourceType !== 'auto') {
|
|
179
|
+
loader = findSourceLoaderByType(loaders, sourceType);
|
|
180
|
+
reason = loader ? `match forced by supplied source type ${sourceType}` : '';
|
|
181
|
+
}
|
|
182
|
+
|
|
172
183
|
// if options.mimeType is supplied, it takes precedence
|
|
173
184
|
if (options?.core?.mimeType) {
|
|
174
185
|
loader = findLoaderByMIMEType(loaders, options?.core?.mimeType);
|
|
175
186
|
reason = `match forced by supplied MIME type ${options?.core?.mimeType}`;
|
|
176
187
|
}
|
|
177
188
|
|
|
189
|
+
loader = loader || findSourceLoaderByTestURL(loaders, testUrl);
|
|
190
|
+
reason = reason || (loader ? `matched source url ${testUrl}` : '');
|
|
191
|
+
|
|
178
192
|
// Look up loader by url
|
|
179
193
|
loader = loader || findLoaderByUrl(loaders, testUrl);
|
|
180
194
|
reason = reason || (loader ? `matched url ${testUrl}` : '');
|
|
@@ -189,6 +203,11 @@ function selectLoaderInternal(
|
|
|
189
203
|
// @ts-ignore Blob | Response
|
|
190
204
|
reason = reason || (loader ? `matched initial data ${getFirstCharacters(data)}` : '');
|
|
191
205
|
|
|
206
|
+
if (!loader && isBlob(data)) {
|
|
207
|
+
loader = findSourceLoaderByTestData(loaders, data);
|
|
208
|
+
reason = reason || (loader ? 'matched source testData' : '');
|
|
209
|
+
}
|
|
210
|
+
|
|
192
211
|
// Look up loader by fallback mime type
|
|
193
212
|
if (options?.core?.fallbackMimeType) {
|
|
194
213
|
loader = loader || findLoaderByMIMEType(loaders, options?.core?.fallbackMimeType);
|
|
@@ -258,9 +277,42 @@ function findLoaderByExtension(loaders: Loader[], extension: string): Loader | n
|
|
|
258
277
|
return null;
|
|
259
278
|
}
|
|
260
279
|
|
|
280
|
+
function findSourceLoaderByType(loaders: Loader[], type: string): Loader | null {
|
|
281
|
+
for (const loader of loaders) {
|
|
282
|
+
if (isSourceLoader(loader) && loader.type === type) {
|
|
283
|
+
return loader;
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
function findSourceLoaderByTestURL(loaders: Loader[], url?: string): Loader | null {
|
|
290
|
+
if (!url) {
|
|
291
|
+
return null;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
for (const loader of loaders) {
|
|
295
|
+
if (isSourceLoader(loader) && loader.testURL(url)) {
|
|
296
|
+
return loader;
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
return null;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
function findSourceLoaderByTestData(loaders: Loader[], data: Blob): Loader | null {
|
|
303
|
+
for (const loader of loaders) {
|
|
304
|
+
if (isSourceLoader(loader)) {
|
|
305
|
+
if (loader.testData?.(data)) {
|
|
306
|
+
return loader;
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
return null;
|
|
311
|
+
}
|
|
312
|
+
|
|
261
313
|
function findLoaderByMIMEType(loaders: Loader[], mimeType: string): Loader | null {
|
|
262
314
|
for (const loader of loaders) {
|
|
263
|
-
if (loader.mimeTypes?.some(
|
|
315
|
+
if (loader.mimeTypes?.some(mimeType1 => compareMIMETypes(mimeType, mimeType1))) {
|
|
264
316
|
return loader;
|
|
265
317
|
}
|
|
266
318
|
|
|
@@ -305,12 +357,12 @@ function testDataAgainstText(data: string, loader: Loader): boolean {
|
|
|
305
357
|
}
|
|
306
358
|
|
|
307
359
|
const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
|
|
308
|
-
return tests.some(
|
|
360
|
+
return tests.some(test => data.startsWith(test as string));
|
|
309
361
|
}
|
|
310
362
|
|
|
311
363
|
function testDataAgainstBinary(data: ArrayBufferLike, byteOffset: number, loader: Loader): boolean {
|
|
312
364
|
const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
|
|
313
|
-
return tests.some(
|
|
365
|
+
return tests.some(test => testBinary(data, byteOffset, loader, test));
|
|
314
366
|
}
|
|
315
367
|
|
|
316
368
|
function testBinary(
|