@loaders.gl/core 4.4.5 → 5.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +14 -3
- package/dist/dist.dev.js +800 -271
- package/dist/dist.min.js +4 -3
- package/dist/index.cjs +2191 -296
- package/dist/index.cjs.map +3 -4
- package/dist/index.d.ts +8 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/iterators/make-iterator/make-stream-iterator.js +48 -9
- package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -1
- package/dist/lib/api/core-api.d.ts +6 -0
- package/dist/lib/api/core-api.d.ts.map +1 -0
- package/dist/lib/api/core-api.js +23 -0
- package/dist/lib/api/core-api.js.map +1 -0
- package/dist/lib/api/create-data-source.d.ts +2 -2
- package/dist/lib/api/create-data-source.d.ts.map +1 -1
- package/dist/lib/api/create-data-source.js +15 -14
- package/dist/lib/api/create-data-source.js.map +1 -1
- package/dist/lib/api/load-in-batches.d.ts +4 -4
- package/dist/lib/api/load-in-batches.d.ts.map +1 -1
- package/dist/lib/api/load-in-batches.js +1 -1
- package/dist/lib/api/load-in-batches.js.map +1 -1
- package/dist/lib/api/load-loader.d.ts +8 -0
- package/dist/lib/api/load-loader.d.ts.map +1 -0
- package/dist/lib/api/load-loader.js +110 -0
- package/dist/lib/api/load-loader.js.map +1 -0
- package/dist/lib/api/load.d.ts +2 -2
- package/dist/lib/api/load.d.ts.map +1 -1
- package/dist/lib/api/load.js +66 -3
- package/dist/lib/api/load.js.map +1 -1
- package/dist/lib/api/parse-file.d.ts +15 -0
- package/dist/lib/api/parse-file.d.ts.map +1 -0
- package/dist/lib/api/parse-file.js +70 -0
- package/dist/lib/api/parse-file.js.map +1 -0
- package/dist/lib/api/parse-in-batches.d.ts +2 -2
- package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
- package/dist/lib/api/parse-in-batches.js +6 -4
- package/dist/lib/api/parse-in-batches.js.map +1 -1
- package/dist/lib/api/parse-sync.d.ts +2 -2
- package/dist/lib/api/parse-sync.d.ts.map +1 -1
- package/dist/lib/api/parse-sync.js +10 -1
- package/dist/lib/api/parse-sync.js.map +1 -1
- package/dist/lib/api/parse.d.ts +2 -2
- package/dist/lib/api/parse.d.ts.map +1 -1
- package/dist/lib/api/parse.js +28 -16
- package/dist/lib/api/parse.js.map +1 -1
- package/dist/lib/api/preload.d.ts +11 -0
- package/dist/lib/api/preload.d.ts.map +1 -0
- package/dist/lib/api/preload.js +14 -0
- package/dist/lib/api/preload.js.map +1 -0
- package/dist/lib/api/register-loaders.js +1 -1
- package/dist/lib/api/register-loaders.js.map +1 -1
- package/dist/lib/api/select-loader.d.ts.map +1 -1
- package/dist/lib/api/select-loader.js +48 -5
- package/dist/lib/api/select-loader.js.map +1 -1
- package/dist/lib/api/select-source.d.ts +3 -3
- package/dist/lib/api/select-source.d.ts.map +1 -1
- package/dist/lib/api/select-source.js +2 -5
- package/dist/lib/api/select-source.js.map +1 -1
- package/dist/lib/fetch/fetch-error-message.js +1 -1
- package/dist/lib/fetch/fetch-error-message.js.map +1 -1
- package/dist/lib/init.js +1 -1
- package/dist/lib/init.js.map +1 -1
- package/dist/lib/loader-utils/check-errors.js +1 -1
- package/dist/lib/loader-utils/check-errors.js.map +1 -1
- package/dist/lib/loader-utils/get-data.d.ts +4 -0
- package/dist/lib/loader-utils/get-data.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-data.js +26 -0
- package/dist/lib/loader-utils/get-data.js.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.d.ts +1 -1
- package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
- package/dist/lib/loader-utils/get-fetch-function.js +1 -1
- package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts +1 -1
- package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
- package/dist/lib/loader-utils/loader-context.js +3 -1
- package/dist/lib/loader-utils/loader-context.js.map +1 -1
- package/dist/lib/loader-utils/loggers.d.ts +19 -0
- package/dist/lib/loader-utils/loggers.d.ts.map +1 -1
- package/dist/lib/loader-utils/loggers.js +28 -2
- package/dist/lib/loader-utils/loggers.js.map +1 -1
- package/dist/lib/loader-utils/option-defaults.d.ts +1 -0
- package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-defaults.js +2 -0
- package/dist/lib/loader-utils/option-defaults.js.map +1 -1
- package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
- package/dist/lib/loader-utils/option-utils.js +50 -2
- package/dist/lib/loader-utils/option-utils.js.map +1 -1
- package/dist/lib/utils/response-utils.js +3 -3
- package/dist/lib/utils/response-utils.js.map +1 -1
- package/dist/null-loader.d.ts.map +1 -1
- package/dist/null-loader.js +1 -1
- package/dist/null-loader.js.map +1 -1
- package/dist/null-worker-node.js +199 -36
- package/dist/null-worker.js +199 -36
- package/package.json +6 -6
- package/src/index.ts +10 -1
- package/src/iterators/make-iterator/make-stream-iterator.ts +47 -9
- package/src/lib/api/core-api.ts +25 -0
- package/src/lib/api/create-data-source.ts +24 -24
- package/src/lib/api/load-in-batches.ts +16 -11
- package/src/lib/api/load-loader.ts +152 -0
- package/src/lib/api/load.ts +98 -5
- package/src/lib/api/parse-file.ts +143 -0
- package/src/lib/api/parse-in-batches.ts +13 -5
- package/src/lib/api/parse-sync.ts +27 -2
- package/src/lib/api/parse.ts +49 -18
- package/src/lib/api/preload.ts +31 -0
- package/src/lib/api/register-loaders.ts +1 -1
- package/src/lib/api/select-loader.ts +57 -5
- package/src/lib/api/select-source.ts +7 -10
- package/src/lib/fetch/fetch-error-message.ts +1 -1
- package/src/lib/loader-utils/check-errors.ts +1 -1
- package/src/lib/loader-utils/get-data.ts +36 -0
- package/src/lib/loader-utils/get-fetch-function.ts +3 -2
- package/src/lib/loader-utils/loader-context.ts +5 -2
- package/src/lib/loader-utils/loggers.ts +28 -2
- package/src/lib/loader-utils/option-defaults.ts +2 -0
- package/src/lib/loader-utils/option-utils.ts +59 -2
- package/src/lib/utils/response-utils.ts +3 -3
- package/src/null-loader.ts +3 -3
|
@@ -43,30 +43,68 @@ async function* makeBrowserStreamIterator(
|
|
|
43
43
|
const reader = stream.getReader();
|
|
44
44
|
|
|
45
45
|
let nextBatchPromise: Promise<{done?: boolean; value?: Uint8Array}> | undefined;
|
|
46
|
+
let streamFinished = false;
|
|
47
|
+
let streamReadFailed = false;
|
|
48
|
+
let iteratorFailed = false;
|
|
46
49
|
|
|
47
50
|
try {
|
|
48
51
|
// eslint-disable-next-line no-constant-condition
|
|
49
52
|
while (true) {
|
|
50
53
|
const currentBatchPromise = nextBatchPromise || reader.read();
|
|
51
|
-
|
|
52
|
-
// Idea is to make fetching happen in parallel with processing / parsing
|
|
53
|
-
if (options?._streamReadAhead) {
|
|
54
|
-
nextBatchPromise = reader.read();
|
|
55
|
-
}
|
|
54
|
+
nextBatchPromise = undefined;
|
|
56
55
|
// Read from the stream
|
|
57
56
|
// value is a Uint8Array
|
|
58
|
-
|
|
57
|
+
let batch;
|
|
58
|
+
try {
|
|
59
|
+
batch = await currentBatchPromise;
|
|
60
|
+
} catch (error) {
|
|
61
|
+
streamReadFailed = true;
|
|
62
|
+
throw error;
|
|
63
|
+
}
|
|
64
|
+
const {done, value} = batch;
|
|
59
65
|
// Exit if we're done
|
|
60
66
|
if (done) {
|
|
67
|
+
streamFinished = true;
|
|
61
68
|
return;
|
|
62
69
|
}
|
|
70
|
+
// Issue a read for an additional batch while the current batch is processed
|
|
71
|
+
if (options?._streamReadAhead) {
|
|
72
|
+
nextBatchPromise = reader.read();
|
|
73
|
+
// Observe a prefetched rejection even if the consumer stops before requesting it
|
|
74
|
+
void nextBatchPromise.catch(() => {});
|
|
75
|
+
}
|
|
63
76
|
// Else yield the chunk
|
|
64
77
|
yield toArrayBuffer(value);
|
|
65
78
|
}
|
|
66
79
|
} catch (error) {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
80
|
+
iteratorFailed = true;
|
|
81
|
+
throw error;
|
|
82
|
+
} finally {
|
|
83
|
+
let cleanupFailed = false;
|
|
84
|
+
let cleanupError: unknown;
|
|
85
|
+
|
|
86
|
+
if (!streamFinished && !streamReadFailed) {
|
|
87
|
+
try {
|
|
88
|
+
await reader.cancel();
|
|
89
|
+
} catch (error) {
|
|
90
|
+
cleanupFailed = true;
|
|
91
|
+
cleanupError = error;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
try {
|
|
96
|
+
reader.releaseLock();
|
|
97
|
+
} catch (error) {
|
|
98
|
+
if (!cleanupFailed) {
|
|
99
|
+
cleanupFailed = true;
|
|
100
|
+
cleanupError = error;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Preserve errors from iteration instead of replacing them with cleanup errors
|
|
105
|
+
if (!iteratorFailed && cleanupFailed) {
|
|
106
|
+
throw cleanupError;
|
|
107
|
+
}
|
|
70
108
|
}
|
|
71
109
|
}
|
|
72
110
|
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {CoreAPI} from '@loaders.gl/loader-utils';
|
|
6
|
+
import {fetchFile} from '../fetch/fetch-file';
|
|
7
|
+
import {parse} from './parse';
|
|
8
|
+
import {parseFile} from './parse-file';
|
|
9
|
+
import {parseSync} from './parse-sync';
|
|
10
|
+
import {parseInBatches} from './parse-in-batches';
|
|
11
|
+
import {load} from './load';
|
|
12
|
+
import {loadInBatches} from './load-in-batches';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Shared Core API surface that can be injected into contexts and sources.
|
|
16
|
+
*/
|
|
17
|
+
export const coreApi: CoreAPI = Object.freeze({
|
|
18
|
+
fetchFile,
|
|
19
|
+
parse,
|
|
20
|
+
parseFile,
|
|
21
|
+
parseSync,
|
|
22
|
+
parseInBatches,
|
|
23
|
+
load,
|
|
24
|
+
loadInBatches
|
|
25
|
+
});
|
|
@@ -2,7 +2,13 @@
|
|
|
2
2
|
// SPDX-License-Identifier: MIT
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
SourceLoader,
|
|
7
|
+
SourceArrayOptionsType,
|
|
8
|
+
SourceArrayDataSourceType
|
|
9
|
+
} from '@loaders.gl/loader-utils';
|
|
10
|
+
import {coreApi} from './core-api';
|
|
11
|
+
import {selectLoaderSync} from './select-loader';
|
|
6
12
|
|
|
7
13
|
/**
|
|
8
14
|
* Creates a source from a service
|
|
@@ -13,39 +19,33 @@ import {Source, SourceArrayOptionsType, SourceArrayDataSourceType} from '@loader
|
|
|
13
19
|
*/
|
|
14
20
|
// DataSourceOptionsT extends DataSourceOptions = DataSourceOptions,
|
|
15
21
|
// DataSourceT extends DataSource = DataSource
|
|
16
|
-
export function createDataSource<SourceArrayT extends
|
|
17
|
-
data:
|
|
22
|
+
export function createDataSource<SourceArrayT extends SourceLoader[]>(
|
|
23
|
+
data: unknown,
|
|
18
24
|
sources: Readonly<SourceArrayT>,
|
|
19
25
|
options: Readonly<SourceArrayOptionsType<SourceArrayT>>
|
|
20
26
|
): SourceArrayDataSourceType<SourceArrayT> {
|
|
21
|
-
const
|
|
22
|
-
const
|
|
27
|
+
const resolvedOptions = (options || {}) as SourceArrayOptionsType<SourceArrayT>;
|
|
28
|
+
const type = resolvedOptions?.core?.type || (resolvedOptions.type as unknown as string) || 'auto';
|
|
29
|
+
const source =
|
|
30
|
+
sources.length === 1
|
|
31
|
+
? sources[0]
|
|
32
|
+
: type === 'auto'
|
|
33
|
+
? (selectLoaderSync(data as never, sources as unknown as SourceLoader[], {
|
|
34
|
+
// `createDataSource` supports non-fetch inputs such as in-memory tables.
|
|
35
|
+
// Selection remains best-effort for those cases.
|
|
36
|
+
...resolvedOptions,
|
|
37
|
+
core: {...resolvedOptions?.core, type: 'auto'}
|
|
38
|
+
}) as SourceArrayT[number] | null)
|
|
39
|
+
: getSourceOfType(type, sources);
|
|
23
40
|
|
|
24
41
|
if (!source) {
|
|
25
42
|
throw new Error('Not a valid source type');
|
|
26
43
|
}
|
|
27
|
-
return source.createDataSource(data,
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
// TODO - use selectSource...
|
|
31
|
-
|
|
32
|
-
/** Guess service type from URL */
|
|
33
|
-
function selectSource<SourceArrayT extends Source[]>(
|
|
34
|
-
url: string | Blob,
|
|
35
|
-
sources: Readonly<SourceArrayT>
|
|
36
|
-
): SourceArrayT[number] | null {
|
|
37
|
-
for (const service of sources) {
|
|
38
|
-
// @ts-expect-error
|
|
39
|
-
if (service.testURL && service.testURL(url)) {
|
|
40
|
-
return service;
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
return null;
|
|
44
|
+
return source.createDataSource(data as string | Blob, resolvedOptions, coreApi);
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
/** Guess service type from URL */
|
|
48
|
-
function getSourceOfType(type: string, sources: Readonly<
|
|
48
|
+
function getSourceOfType(type: string, sources: Readonly<SourceLoader[]>): SourceLoader | null {
|
|
49
49
|
for (const service of sources) {
|
|
50
50
|
if (service.type === type) {
|
|
51
51
|
return service;
|
|
@@ -3,13 +3,15 @@
|
|
|
3
3
|
// Copyright (c) vis.gl contributors
|
|
4
4
|
|
|
5
5
|
import type {
|
|
6
|
-
|
|
6
|
+
Loader,
|
|
7
7
|
LoaderOptions,
|
|
8
8
|
LoaderContext,
|
|
9
9
|
FetchLike,
|
|
10
10
|
BatchableDataType,
|
|
11
11
|
LoaderBatchType,
|
|
12
|
-
LoaderOptionsType
|
|
12
|
+
LoaderOptionsType,
|
|
13
|
+
LoaderOptionsWithShape,
|
|
14
|
+
LoaderShapeType
|
|
13
15
|
} from '@loaders.gl/loader-utils';
|
|
14
16
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
15
17
|
import {getFetchFunction} from '../loader-utils/get-fetch-function';
|
|
@@ -22,8 +24,11 @@ type FileType = string | File | Blob | Response | (string | File | Blob | Respon
|
|
|
22
24
|
* Parses `data` synchronously using a specified loader
|
|
23
25
|
*/
|
|
24
26
|
export async function loadInBatches<
|
|
25
|
-
LoaderT extends
|
|
26
|
-
OptionsT extends LoaderOptions =
|
|
27
|
+
LoaderT extends Loader,
|
|
28
|
+
OptionsT extends LoaderOptions = LoaderOptionsWithShape<
|
|
29
|
+
LoaderOptionsType<LoaderT>,
|
|
30
|
+
LoaderShapeType<LoaderT>
|
|
31
|
+
>
|
|
27
32
|
>(
|
|
28
33
|
files: FileType,
|
|
29
34
|
loader: LoaderT,
|
|
@@ -40,32 +45,32 @@ export async function loadInBatches<
|
|
|
40
45
|
*/
|
|
41
46
|
export function loadInBatches(
|
|
42
47
|
files: FileType,
|
|
43
|
-
loaders?:
|
|
48
|
+
loaders?: Loader | Loader[] | LoaderOptions,
|
|
44
49
|
options?: LoaderOptions,
|
|
45
50
|
context?: LoaderContext
|
|
46
51
|
): Promise<AsyncIterable<unknown>>;
|
|
47
52
|
|
|
48
53
|
export function loadInBatches(
|
|
49
54
|
files: FileType[] | FileList,
|
|
50
|
-
loaders?:
|
|
55
|
+
loaders?: Loader | Loader[] | LoaderOptions,
|
|
51
56
|
options?: LoaderOptions,
|
|
52
57
|
context?: LoaderContext
|
|
53
58
|
): Promise<AsyncIterable<unknown>>[];
|
|
54
59
|
|
|
55
60
|
export function loadInBatches(
|
|
56
61
|
files: FileType | FileType[] | FileList,
|
|
57
|
-
loaders?:
|
|
62
|
+
loaders?: Loader | Loader[] | LoaderOptions,
|
|
58
63
|
options?: LoaderOptions,
|
|
59
64
|
context?: LoaderContext
|
|
60
65
|
): Promise<AsyncIterable<unknown>> | Promise<AsyncIterable<unknown>>[] {
|
|
61
|
-
let loadersArray:
|
|
66
|
+
let loadersArray: Loader | Loader[] | undefined;
|
|
62
67
|
// Signature: load(url, options)
|
|
63
68
|
if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
64
69
|
context = undefined; // context not supported in short signature
|
|
65
70
|
options = loaders as LoaderOptions;
|
|
66
71
|
loadersArray = undefined;
|
|
67
72
|
} else {
|
|
68
|
-
loadersArray = loaders as
|
|
73
|
+
loadersArray = loaders as Loader | Loader[] | undefined;
|
|
69
74
|
}
|
|
70
75
|
|
|
71
76
|
// Select fetch function
|
|
@@ -77,7 +82,7 @@ export function loadInBatches(
|
|
|
77
82
|
}
|
|
78
83
|
|
|
79
84
|
// Multiple URLs / files
|
|
80
|
-
const promises = files.map(
|
|
85
|
+
const promises = files.map(file =>
|
|
81
86
|
loadOneFileInBatches(file, loadersArray!, options || {}, fetch)
|
|
82
87
|
);
|
|
83
88
|
|
|
@@ -87,7 +92,7 @@ export function loadInBatches(
|
|
|
87
92
|
|
|
88
93
|
async function loadOneFileInBatches(
|
|
89
94
|
file: FileType,
|
|
90
|
-
loaders:
|
|
95
|
+
loaders: Loader | Loader[],
|
|
91
96
|
options: LoaderOptions,
|
|
92
97
|
fetch: FetchLike
|
|
93
98
|
): Promise<AsyncIterable<unknown>> {
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {Loader, LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
|
|
6
|
+
|
|
7
|
+
const loaderImplementationPromises = new Map<Loader, Map<string, Promise<LoaderWithParser>>>();
|
|
8
|
+
const loaderImplementations = new Map<Loader, Map<string, LoaderWithParser>>();
|
|
9
|
+
|
|
10
|
+
/** Gets a parser-bearing implementation for a loader, loading it if needed. */
|
|
11
|
+
export async function getLoaderImplementation(
|
|
12
|
+
loader: Loader,
|
|
13
|
+
options?: LoaderOptions,
|
|
14
|
+
url?: string
|
|
15
|
+
): Promise<LoaderWithParser> {
|
|
16
|
+
if (isLoaderWithParser(loader)) {
|
|
17
|
+
return loader;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const implementationCacheKey = getLoaderImplementationCacheKey(loader, options);
|
|
21
|
+
const loaderImplementation = loaderImplementations.get(loader)?.get(implementationCacheKey);
|
|
22
|
+
if (loaderImplementation) {
|
|
23
|
+
return loaderImplementation;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
let loaderImplementationPromisesForLoader = loaderImplementationPromises.get(loader);
|
|
27
|
+
if (!loaderImplementationPromisesForLoader) {
|
|
28
|
+
loaderImplementationPromisesForLoader = new Map();
|
|
29
|
+
loaderImplementationPromises.set(loader, loaderImplementationPromisesForLoader);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
let loaderImplementationPromise =
|
|
33
|
+
loaderImplementationPromisesForLoader.get(implementationCacheKey);
|
|
34
|
+
if (!loaderImplementationPromise) {
|
|
35
|
+
loaderImplementationPromise = resolveLoaderImplementation(loader, options, url)
|
|
36
|
+
.then(implementation => {
|
|
37
|
+
let loaderImplementationsForLoader = loaderImplementations.get(loader);
|
|
38
|
+
if (!loaderImplementationsForLoader) {
|
|
39
|
+
loaderImplementationsForLoader = new Map();
|
|
40
|
+
loaderImplementations.set(loader, loaderImplementationsForLoader);
|
|
41
|
+
}
|
|
42
|
+
loaderImplementationsForLoader.set(implementationCacheKey, implementation);
|
|
43
|
+
return implementation;
|
|
44
|
+
})
|
|
45
|
+
.catch(error => {
|
|
46
|
+
loaderImplementationPromisesForLoader.delete(implementationCacheKey);
|
|
47
|
+
if (loaderImplementationPromisesForLoader.size === 0) {
|
|
48
|
+
loaderImplementationPromises.delete(loader);
|
|
49
|
+
}
|
|
50
|
+
throw error;
|
|
51
|
+
});
|
|
52
|
+
loaderImplementationPromisesForLoader.set(implementationCacheKey, loaderImplementationPromise);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return await loaderImplementationPromise;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Gets a cached parser-bearing implementation for a loader without loading it. */
|
|
59
|
+
export function getLoaderImplementationSync(loader: Loader): LoaderWithParser | null {
|
|
60
|
+
if (isLoaderWithParser(loader)) {
|
|
61
|
+
return loader;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const implementationCacheKey = getLoaderImplementationCacheKey(loader);
|
|
65
|
+
return loaderImplementations.get(loader)?.get(implementationCacheKey) || null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Returns true when a loader object already includes parser methods. */
|
|
69
|
+
export function isLoaderWithParser(loader: Loader): loader is LoaderWithParser {
|
|
70
|
+
const candidate = loader as LoaderWithParser;
|
|
71
|
+
return Boolean(
|
|
72
|
+
candidate.parse ||
|
|
73
|
+
candidate.parseSync ||
|
|
74
|
+
candidate.parseInBatches ||
|
|
75
|
+
candidate.parseText ||
|
|
76
|
+
candidate.parseTextSync ||
|
|
77
|
+
candidate.parseFile ||
|
|
78
|
+
candidate.parseFileInBatches
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Resolves a parser-bearing implementation through loader preload or the test import fallback. */
|
|
83
|
+
async function resolveLoaderImplementation(
|
|
84
|
+
loader: Loader,
|
|
85
|
+
options?: LoaderOptions,
|
|
86
|
+
url?: string
|
|
87
|
+
): Promise<LoaderWithParser> {
|
|
88
|
+
if (loader.preload) {
|
|
89
|
+
const preloadedLoader = await loader.preload(url || '', options);
|
|
90
|
+
if (isLoaderWithParser(preloadedLoader as Loader)) {
|
|
91
|
+
return preloadedLoader as LoaderWithParser;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
throw new Error(`${loader.id} loader preload() did not return a parser-bearing loader`);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const implementationSpecifier = getLoaderImplementationSpecifier(loader, options);
|
|
98
|
+
return await importLoaderImplementation(implementationSpecifier, loader.id);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Gets the dynamic implementation specifier for the test worker fallback path. */
|
|
102
|
+
function getLoaderImplementationSpecifier(loader: Loader, options?: LoaderOptions): string {
|
|
103
|
+
const workerType = options?._workerType || options?.core?._workerType;
|
|
104
|
+
if (workerType === 'test') {
|
|
105
|
+
const sourcePath = `modules/${loader.module}/src/${loader.id}-loader.ts`;
|
|
106
|
+
if (typeof window !== 'undefined') {
|
|
107
|
+
return `/${sourcePath}`;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
if (typeof process !== 'undefined' && process.cwd) {
|
|
111
|
+
return new URL(sourcePath, `file://${process.cwd()}/`).toString();
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return sourcePath;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
throw new Error(
|
|
118
|
+
`${loader.id} loader does not provide a parser implementation. Import a parser-bearing loader directly, or use preload() before parse/load.`
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Gets the cache key for option-selected parser implementations. */
|
|
123
|
+
function getLoaderImplementationCacheKey(loader: Loader, options?: LoaderOptions): string {
|
|
124
|
+
const loaderOptions = options?.[loader.id] as {backend?: unknown} | undefined;
|
|
125
|
+
const defaultLoaderOptions = loader.options?.[loader.id] as {backend?: unknown} | undefined;
|
|
126
|
+
return String(loaderOptions?.backend || defaultLoaderOptions?.backend || '');
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Imports a module and finds the parser-bearing loader implementation with the requested id. */
|
|
130
|
+
async function importLoaderImplementation(
|
|
131
|
+
implementationSpecifier: string,
|
|
132
|
+
loaderId: string
|
|
133
|
+
): Promise<LoaderWithParser> {
|
|
134
|
+
const moduleExports = await import(
|
|
135
|
+
/* webpackIgnore: true */ /* @vite-ignore */ implementationSpecifier
|
|
136
|
+
);
|
|
137
|
+
|
|
138
|
+
for (const exportValue of Object.values(moduleExports)) {
|
|
139
|
+
if (
|
|
140
|
+
exportValue &&
|
|
141
|
+
typeof exportValue === 'object' &&
|
|
142
|
+
(exportValue as Loader).id === loaderId &&
|
|
143
|
+
isLoaderWithParser(exportValue as Loader)
|
|
144
|
+
) {
|
|
145
|
+
return exportValue as LoaderWithParser;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
throw new Error(
|
|
150
|
+
`Could not find parser implementation for ${loaderId} in ${implementationSpecifier}`
|
|
151
|
+
);
|
|
152
|
+
}
|
package/src/lib/api/load.ts
CHANGED
|
@@ -7,17 +7,26 @@ import type {
|
|
|
7
7
|
Loader,
|
|
8
8
|
LoaderContext,
|
|
9
9
|
LoaderOptions,
|
|
10
|
+
LoaderOptionsWithShape,
|
|
10
11
|
LoaderOptionsType,
|
|
12
|
+
LoaderShapeType,
|
|
11
13
|
LoaderReturnType,
|
|
12
14
|
LoaderArrayOptionsType,
|
|
13
|
-
LoaderArrayReturnType
|
|
15
|
+
LoaderArrayReturnType,
|
|
16
|
+
SourceLoader
|
|
14
17
|
} from '@loaders.gl/loader-utils';
|
|
15
|
-
import {isBlob} from '@loaders.gl/loader-utils';
|
|
18
|
+
import {isBlob, isSourceLoader} from '@loaders.gl/loader-utils';
|
|
16
19
|
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
17
20
|
import {getFetchFunction} from '../loader-utils/get-fetch-function';
|
|
18
21
|
import {normalizeLoaderOptions} from '../loader-utils/option-utils';
|
|
22
|
+
import {fetchFile} from '../fetch/fetch-file';
|
|
19
23
|
|
|
20
24
|
import {parse} from './parse';
|
|
25
|
+
import {parseFile} from './parse-file';
|
|
26
|
+
import {parseSync} from './parse-sync';
|
|
27
|
+
import {parseInBatches} from './parse-in-batches';
|
|
28
|
+
import {loadInBatches} from './load-in-batches';
|
|
29
|
+
import {selectLoader} from './select-loader';
|
|
21
30
|
|
|
22
31
|
/**
|
|
23
32
|
* Parses `data` using a specified loader
|
|
@@ -31,7 +40,10 @@ import {parse} from './parse';
|
|
|
31
40
|
|
|
32
41
|
export async function load<
|
|
33
42
|
LoaderT extends Loader,
|
|
34
|
-
OptionsT extends LoaderOptions =
|
|
43
|
+
OptionsT extends LoaderOptions = LoaderOptionsWithShape<
|
|
44
|
+
LoaderOptionsType<LoaderT>,
|
|
45
|
+
LoaderShapeType<LoaderT>
|
|
46
|
+
>
|
|
35
47
|
>(
|
|
36
48
|
url: string | DataType,
|
|
37
49
|
loader: LoaderT,
|
|
@@ -81,6 +93,70 @@ export async function load(
|
|
|
81
93
|
resolvedOptions = options as LoaderOptions;
|
|
82
94
|
}
|
|
83
95
|
|
|
96
|
+
if (!Array.isArray(resolvedLoaders) && isSourceLoader(resolvedLoaders)) {
|
|
97
|
+
const sourceLoader = await resolveSourceLoader(resolvedLoaders, url, resolvedOptions);
|
|
98
|
+
const runtimeCoreApi = {
|
|
99
|
+
fetchFile,
|
|
100
|
+
parse,
|
|
101
|
+
parseFile,
|
|
102
|
+
parseSync,
|
|
103
|
+
parseInBatches,
|
|
104
|
+
load,
|
|
105
|
+
loadInBatches
|
|
106
|
+
};
|
|
107
|
+
return sourceLoader.createDataSource(
|
|
108
|
+
url as string | Blob,
|
|
109
|
+
(resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
|
|
110
|
+
runtimeCoreApi
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (
|
|
115
|
+
Array.isArray(resolvedLoaders) &&
|
|
116
|
+
resolvedLoaders.length === 1 &&
|
|
117
|
+
isSourceLoader(resolvedLoaders[0])
|
|
118
|
+
) {
|
|
119
|
+
const sourceLoader = await resolveSourceLoader(resolvedLoaders[0], url, resolvedOptions);
|
|
120
|
+
const runtimeCoreApi = {
|
|
121
|
+
fetchFile,
|
|
122
|
+
parse,
|
|
123
|
+
parseFile,
|
|
124
|
+
parseSync,
|
|
125
|
+
parseInBatches,
|
|
126
|
+
load,
|
|
127
|
+
loadInBatches
|
|
128
|
+
};
|
|
129
|
+
return sourceLoader.createDataSource(
|
|
130
|
+
url as string | Blob,
|
|
131
|
+
(resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
|
|
132
|
+
runtimeCoreApi
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (typeof url === 'string' || isBlob(url)) {
|
|
137
|
+
const selectedLoader = await selectLoader(url, resolvedLoaders as Loader | Loader[], {
|
|
138
|
+
...resolvedOptions,
|
|
139
|
+
core: {...resolvedOptions?.core, nothrow: true}
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
if (selectedLoader && isSourceLoader(selectedLoader)) {
|
|
143
|
+
const sourceLoader = await resolveSourceLoader(selectedLoader, url, resolvedOptions);
|
|
144
|
+
return sourceLoader.createDataSource(
|
|
145
|
+
url,
|
|
146
|
+
(resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
|
|
147
|
+
{
|
|
148
|
+
fetchFile,
|
|
149
|
+
parse,
|
|
150
|
+
parseFile,
|
|
151
|
+
parseSync,
|
|
152
|
+
parseInBatches,
|
|
153
|
+
load,
|
|
154
|
+
loadInBatches
|
|
155
|
+
}
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
84
160
|
// Select fetch function
|
|
85
161
|
const fetch = getFetchFunction(resolvedOptions);
|
|
86
162
|
|
|
@@ -114,6 +190,23 @@ export async function load(
|
|
|
114
190
|
// Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
|
|
115
191
|
// return await parse(data, loaders as Loader[], options);
|
|
116
192
|
return Array.isArray(resolvedLoaders)
|
|
117
|
-
? await parse(data, resolvedLoaders, resolvedOptions) // loader array overload
|
|
118
|
-
: await parse(data, resolvedLoaders, resolvedOptions); // single loader overload
|
|
193
|
+
? await parse(data, resolvedLoaders, resolvedOptions, context) // loader array overload
|
|
194
|
+
: await parse(data, resolvedLoaders, resolvedOptions, context); // single loader overload
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
/** Resolves a lightweight source loader into its runtime implementation when necessary. */
|
|
198
|
+
async function resolveSourceLoader(
|
|
199
|
+
loader: SourceLoader,
|
|
200
|
+
url: string | DataType,
|
|
201
|
+
options?: LoaderOptions
|
|
202
|
+
): Promise<SourceLoader> {
|
|
203
|
+
if (!loader.preload) {
|
|
204
|
+
return loader;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
const sourceLoader = await loader.preload(typeof url === 'string' ? url : '', options);
|
|
208
|
+
if (!isSourceLoader(sourceLoader)) {
|
|
209
|
+
throw new Error(`${loader.id} source loader preload() did not return a runtime source loader`);
|
|
210
|
+
}
|
|
211
|
+
return sourceLoader;
|
|
119
212
|
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
// loaders.gl
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
// Copyright (c) vis.gl contributors
|
|
4
|
+
|
|
5
|
+
import type {
|
|
6
|
+
Loader,
|
|
7
|
+
LoaderContext,
|
|
8
|
+
LoaderOptions,
|
|
9
|
+
LoaderOptionsWithShape,
|
|
10
|
+
LoaderOptionsType,
|
|
11
|
+
LoaderShapeType,
|
|
12
|
+
LoaderReturnType,
|
|
13
|
+
LoaderArrayOptionsType,
|
|
14
|
+
LoaderArrayReturnType,
|
|
15
|
+
LoaderWithParser,
|
|
16
|
+
ReadableFile,
|
|
17
|
+
StrictLoaderOptions
|
|
18
|
+
} from '@loaders.gl/loader-utils';
|
|
19
|
+
import {isSourceLoader, mergeOptions} from '@loaders.gl/loader-utils';
|
|
20
|
+
import {validateWorkerVersion} from '@loaders.gl/worker-utils';
|
|
21
|
+
import {isLoaderObject} from '../loader-utils/normalize-loader';
|
|
22
|
+
import {normalizeOptions} from '../loader-utils/option-utils';
|
|
23
|
+
import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
|
|
24
|
+
import {getLoaderImplementation} from './load-loader';
|
|
25
|
+
import {parse} from './parse';
|
|
26
|
+
import {selectLoader} from './select-loader';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Parses a random access readable file asynchronously using the supplied loader.
|
|
30
|
+
*/
|
|
31
|
+
export async function parseFile<
|
|
32
|
+
LoaderT extends Loader,
|
|
33
|
+
OptionsT extends LoaderOptions = LoaderOptionsWithShape<
|
|
34
|
+
LoaderOptionsType<LoaderT>,
|
|
35
|
+
LoaderShapeType<LoaderT>
|
|
36
|
+
>
|
|
37
|
+
>(
|
|
38
|
+
file: ReadableFile,
|
|
39
|
+
loader: LoaderT,
|
|
40
|
+
options?: OptionsT,
|
|
41
|
+
context?: LoaderContext
|
|
42
|
+
): Promise<LoaderReturnType<LoaderT>>;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Parses a random access readable file asynchronously by matching one of the supplied loaders.
|
|
46
|
+
*/
|
|
47
|
+
export async function parseFile<
|
|
48
|
+
LoaderArrayT extends Loader[],
|
|
49
|
+
OptionsT extends LoaderOptions = LoaderArrayOptionsType<LoaderArrayT>
|
|
50
|
+
>(
|
|
51
|
+
file: ReadableFile,
|
|
52
|
+
loaders: LoaderArrayT,
|
|
53
|
+
options?: OptionsT,
|
|
54
|
+
context?: LoaderContext
|
|
55
|
+
): Promise<LoaderArrayReturnType<LoaderArrayT>>;
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Parses a random access readable file asynchronously by matching a pre-registered loader.
|
|
59
|
+
* @deprecated Loader registration is deprecated, use parseFile(file, loaders, options) instead.
|
|
60
|
+
*/
|
|
61
|
+
export async function parseFile(file: ReadableFile, options?: LoaderOptions): Promise<unknown>;
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Parses a random access readable file using a specified loader.
|
|
65
|
+
* @param file readable file
|
|
66
|
+
* @param loaders loader or loaders to use
|
|
67
|
+
* @param options loader options
|
|
68
|
+
* @param context loader context
|
|
69
|
+
* @returns parsed data
|
|
70
|
+
*/
|
|
71
|
+
export async function parseFile(
|
|
72
|
+
file: ReadableFile,
|
|
73
|
+
loaders?: Loader | Loader[] | LoaderOptions,
|
|
74
|
+
options?: LoaderOptions,
|
|
75
|
+
context?: LoaderContext
|
|
76
|
+
): Promise<unknown> {
|
|
77
|
+
if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
|
|
78
|
+
context = undefined;
|
|
79
|
+
options = loaders as LoaderOptions;
|
|
80
|
+
loaders = undefined;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
options = options || {};
|
|
84
|
+
const url = file.url || '';
|
|
85
|
+
const typedLoaders = loaders as Loader | Loader[] | undefined;
|
|
86
|
+
const candidateLoaders = getLoadersFromContext(typedLoaders, context);
|
|
87
|
+
const loader = await selectLoader(url, candidateLoaders, options);
|
|
88
|
+
if (!loader) {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (isSourceLoader(loader)) {
|
|
93
|
+
throw new Error(
|
|
94
|
+
`${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parseFile().`
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// @ts-expect-error candidateLoaders may be a single forced loader
|
|
99
|
+
const strictOptions = normalizeOptions(options, loader, candidateLoaders, url);
|
|
100
|
+
context = getLoaderContext(
|
|
101
|
+
// @ts-expect-error candidateLoaders may be a single forced loader
|
|
102
|
+
{url, _parse: parse, loaders: candidateLoaders},
|
|
103
|
+
strictOptions,
|
|
104
|
+
context || null
|
|
105
|
+
);
|
|
106
|
+
|
|
107
|
+
return await parseFileWithLoader(loader, file, strictOptions, context);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Parses a readable file after a loader has been selected.
|
|
112
|
+
* @param loader selected loader
|
|
113
|
+
* @param file readable file
|
|
114
|
+
* @param options strict loader options
|
|
115
|
+
* @param context loader context
|
|
116
|
+
* @returns parsed data
|
|
117
|
+
*/
|
|
118
|
+
async function parseFileWithLoader(
|
|
119
|
+
loader: Loader,
|
|
120
|
+
file: ReadableFile,
|
|
121
|
+
options: StrictLoaderOptions,
|
|
122
|
+
context: LoaderContext
|
|
123
|
+
): Promise<unknown> {
|
|
124
|
+
validateWorkerVersion(loader);
|
|
125
|
+
options = mergeOptions(loader.options, options);
|
|
126
|
+
|
|
127
|
+
const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
|
|
128
|
+
if (loaderWithParser.parseFile) {
|
|
129
|
+
return await loaderWithParser.parseFile(file, options, context);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const parseFunction = (loaderWithParser as {parse?: LoaderWithParser['parse']}).parse;
|
|
133
|
+
if (parseFunction) {
|
|
134
|
+
const length = file.bigsize > 0n ? Number(file.bigsize) : file.size;
|
|
135
|
+
if (!length) {
|
|
136
|
+
throw new Error(`${loader.id} loader - cannot fall back to parse without a file size`);
|
|
137
|
+
}
|
|
138
|
+
const data = await file.read(0n, length);
|
|
139
|
+
return await parse(data, loaderWithParser, options, context);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
throw new Error(`${loader.id} loader - no file parser found`);
|
|
143
|
+
}
|