@loaders.gl/core 4.4.5 → 5.0.0-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/LICENSE +14 -3
  2. package/dist/dist.dev.js +800 -271
  3. package/dist/dist.min.js +4 -3
  4. package/dist/index.cjs +2191 -296
  5. package/dist/index.cjs.map +3 -4
  6. package/dist/index.d.ts +8 -2
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +7 -1
  9. package/dist/index.js.map +1 -1
  10. package/dist/iterators/make-iterator/make-stream-iterator.js +48 -9
  11. package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -1
  12. package/dist/lib/api/core-api.d.ts +6 -0
  13. package/dist/lib/api/core-api.d.ts.map +1 -0
  14. package/dist/lib/api/core-api.js +23 -0
  15. package/dist/lib/api/core-api.js.map +1 -0
  16. package/dist/lib/api/create-data-source.d.ts +2 -2
  17. package/dist/lib/api/create-data-source.d.ts.map +1 -1
  18. package/dist/lib/api/create-data-source.js +15 -14
  19. package/dist/lib/api/create-data-source.js.map +1 -1
  20. package/dist/lib/api/load-in-batches.d.ts +4 -4
  21. package/dist/lib/api/load-in-batches.d.ts.map +1 -1
  22. package/dist/lib/api/load-in-batches.js +1 -1
  23. package/dist/lib/api/load-in-batches.js.map +1 -1
  24. package/dist/lib/api/load-loader.d.ts +8 -0
  25. package/dist/lib/api/load-loader.d.ts.map +1 -0
  26. package/dist/lib/api/load-loader.js +110 -0
  27. package/dist/lib/api/load-loader.js.map +1 -0
  28. package/dist/lib/api/load.d.ts +2 -2
  29. package/dist/lib/api/load.d.ts.map +1 -1
  30. package/dist/lib/api/load.js +66 -3
  31. package/dist/lib/api/load.js.map +1 -1
  32. package/dist/lib/api/parse-file.d.ts +15 -0
  33. package/dist/lib/api/parse-file.d.ts.map +1 -0
  34. package/dist/lib/api/parse-file.js +70 -0
  35. package/dist/lib/api/parse-file.js.map +1 -0
  36. package/dist/lib/api/parse-in-batches.d.ts +2 -2
  37. package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
  38. package/dist/lib/api/parse-in-batches.js +6 -4
  39. package/dist/lib/api/parse-in-batches.js.map +1 -1
  40. package/dist/lib/api/parse-sync.d.ts +2 -2
  41. package/dist/lib/api/parse-sync.d.ts.map +1 -1
  42. package/dist/lib/api/parse-sync.js +10 -1
  43. package/dist/lib/api/parse-sync.js.map +1 -1
  44. package/dist/lib/api/parse.d.ts +2 -2
  45. package/dist/lib/api/parse.d.ts.map +1 -1
  46. package/dist/lib/api/parse.js +28 -16
  47. package/dist/lib/api/parse.js.map +1 -1
  48. package/dist/lib/api/preload.d.ts +11 -0
  49. package/dist/lib/api/preload.d.ts.map +1 -0
  50. package/dist/lib/api/preload.js +14 -0
  51. package/dist/lib/api/preload.js.map +1 -0
  52. package/dist/lib/api/register-loaders.js +1 -1
  53. package/dist/lib/api/register-loaders.js.map +1 -1
  54. package/dist/lib/api/select-loader.d.ts.map +1 -1
  55. package/dist/lib/api/select-loader.js +48 -5
  56. package/dist/lib/api/select-loader.js.map +1 -1
  57. package/dist/lib/api/select-source.d.ts +3 -3
  58. package/dist/lib/api/select-source.d.ts.map +1 -1
  59. package/dist/lib/api/select-source.js +2 -5
  60. package/dist/lib/api/select-source.js.map +1 -1
  61. package/dist/lib/fetch/fetch-error-message.js +1 -1
  62. package/dist/lib/fetch/fetch-error-message.js.map +1 -1
  63. package/dist/lib/init.js +1 -1
  64. package/dist/lib/init.js.map +1 -1
  65. package/dist/lib/loader-utils/check-errors.js +1 -1
  66. package/dist/lib/loader-utils/check-errors.js.map +1 -1
  67. package/dist/lib/loader-utils/get-data.d.ts +4 -0
  68. package/dist/lib/loader-utils/get-data.d.ts.map +1 -1
  69. package/dist/lib/loader-utils/get-data.js +26 -0
  70. package/dist/lib/loader-utils/get-data.js.map +1 -1
  71. package/dist/lib/loader-utils/get-fetch-function.d.ts +1 -1
  72. package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
  73. package/dist/lib/loader-utils/get-fetch-function.js +1 -1
  74. package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
  75. package/dist/lib/loader-utils/loader-context.d.ts +1 -1
  76. package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
  77. package/dist/lib/loader-utils/loader-context.js +3 -1
  78. package/dist/lib/loader-utils/loader-context.js.map +1 -1
  79. package/dist/lib/loader-utils/loggers.d.ts +19 -0
  80. package/dist/lib/loader-utils/loggers.d.ts.map +1 -1
  81. package/dist/lib/loader-utils/loggers.js +28 -2
  82. package/dist/lib/loader-utils/loggers.js.map +1 -1
  83. package/dist/lib/loader-utils/option-defaults.d.ts +1 -0
  84. package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
  85. package/dist/lib/loader-utils/option-defaults.js +2 -0
  86. package/dist/lib/loader-utils/option-defaults.js.map +1 -1
  87. package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
  88. package/dist/lib/loader-utils/option-utils.js +50 -2
  89. package/dist/lib/loader-utils/option-utils.js.map +1 -1
  90. package/dist/lib/utils/response-utils.js +3 -3
  91. package/dist/lib/utils/response-utils.js.map +1 -1
  92. package/dist/null-loader.d.ts.map +1 -1
  93. package/dist/null-loader.js +1 -1
  94. package/dist/null-loader.js.map +1 -1
  95. package/dist/null-worker-node.js +199 -36
  96. package/dist/null-worker.js +199 -36
  97. package/package.json +6 -6
  98. package/src/index.ts +10 -1
  99. package/src/iterators/make-iterator/make-stream-iterator.ts +47 -9
  100. package/src/lib/api/core-api.ts +25 -0
  101. package/src/lib/api/create-data-source.ts +24 -24
  102. package/src/lib/api/load-in-batches.ts +16 -11
  103. package/src/lib/api/load-loader.ts +152 -0
  104. package/src/lib/api/load.ts +98 -5
  105. package/src/lib/api/parse-file.ts +143 -0
  106. package/src/lib/api/parse-in-batches.ts +13 -5
  107. package/src/lib/api/parse-sync.ts +27 -2
  108. package/src/lib/api/parse.ts +49 -18
  109. package/src/lib/api/preload.ts +31 -0
  110. package/src/lib/api/register-loaders.ts +1 -1
  111. package/src/lib/api/select-loader.ts +57 -5
  112. package/src/lib/api/select-source.ts +7 -10
  113. package/src/lib/fetch/fetch-error-message.ts +1 -1
  114. package/src/lib/loader-utils/check-errors.ts +1 -1
  115. package/src/lib/loader-utils/get-data.ts +36 -0
  116. package/src/lib/loader-utils/get-fetch-function.ts +3 -2
  117. package/src/lib/loader-utils/loader-context.ts +5 -2
  118. package/src/lib/loader-utils/loggers.ts +28 -2
  119. package/src/lib/loader-utils/option-defaults.ts +2 -0
  120. package/src/lib/loader-utils/option-utils.ts +59 -2
  121. package/src/lib/utils/response-utils.ts +3 -3
  122. package/src/null-loader.ts +3 -3
@@ -43,30 +43,68 @@ async function* makeBrowserStreamIterator(
43
43
  const reader = stream.getReader();
44
44
 
45
45
  let nextBatchPromise: Promise<{done?: boolean; value?: Uint8Array}> | undefined;
46
+ let streamFinished = false;
47
+ let streamReadFailed = false;
48
+ let iteratorFailed = false;
46
49
 
47
50
  try {
48
51
  // eslint-disable-next-line no-constant-condition
49
52
  while (true) {
50
53
  const currentBatchPromise = nextBatchPromise || reader.read();
51
- // Issue a read for an additional batch, while we await the next batch
52
- // Idea is to make fetching happen in parallel with processing / parsing
53
- if (options?._streamReadAhead) {
54
- nextBatchPromise = reader.read();
55
- }
54
+ nextBatchPromise = undefined;
56
55
  // Read from the stream
57
56
  // value is a Uint8Array
58
- const {done, value} = await currentBatchPromise;
57
+ let batch;
58
+ try {
59
+ batch = await currentBatchPromise;
60
+ } catch (error) {
61
+ streamReadFailed = true;
62
+ throw error;
63
+ }
64
+ const {done, value} = batch;
59
65
  // Exit if we're done
60
66
  if (done) {
67
+ streamFinished = true;
61
68
  return;
62
69
  }
70
+ // Issue a read for an additional batch while the current batch is processed
71
+ if (options?._streamReadAhead) {
72
+ nextBatchPromise = reader.read();
73
+ // Observe a prefetched rejection even if the consumer stops before requesting it
74
+ void nextBatchPromise.catch(() => {});
75
+ }
63
76
  // Else yield the chunk
64
77
  yield toArrayBuffer(value);
65
78
  }
66
79
  } catch (error) {
67
- // TODO - examples makes it look like this should always be called,
68
- // but that generates exceptions so only call it if we do not reach the end
69
- reader.releaseLock();
80
+ iteratorFailed = true;
81
+ throw error;
82
+ } finally {
83
+ let cleanupFailed = false;
84
+ let cleanupError: unknown;
85
+
86
+ if (!streamFinished && !streamReadFailed) {
87
+ try {
88
+ await reader.cancel();
89
+ } catch (error) {
90
+ cleanupFailed = true;
91
+ cleanupError = error;
92
+ }
93
+ }
94
+
95
+ try {
96
+ reader.releaseLock();
97
+ } catch (error) {
98
+ if (!cleanupFailed) {
99
+ cleanupFailed = true;
100
+ cleanupError = error;
101
+ }
102
+ }
103
+
104
+ // Preserve errors from iteration instead of replacing them with cleanup errors
105
+ if (!iteratorFailed && cleanupFailed) {
106
+ throw cleanupError;
107
+ }
70
108
  }
71
109
  }
72
110
 
@@ -0,0 +1,25 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {CoreAPI} from '@loaders.gl/loader-utils';
6
+ import {fetchFile} from '../fetch/fetch-file';
7
+ import {parse} from './parse';
8
+ import {parseFile} from './parse-file';
9
+ import {parseSync} from './parse-sync';
10
+ import {parseInBatches} from './parse-in-batches';
11
+ import {load} from './load';
12
+ import {loadInBatches} from './load-in-batches';
13
+
14
+ /**
15
+ * Shared Core API surface that can be injected into contexts and sources.
16
+ */
17
+ export const coreApi: CoreAPI = Object.freeze({
18
+ fetchFile,
19
+ parse,
20
+ parseFile,
21
+ parseSync,
22
+ parseInBatches,
23
+ load,
24
+ loadInBatches
25
+ });
@@ -2,7 +2,13 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import {Source, SourceArrayOptionsType, SourceArrayDataSourceType} from '@loaders.gl/loader-utils';
5
+ import {
6
+ SourceLoader,
7
+ SourceArrayOptionsType,
8
+ SourceArrayDataSourceType
9
+ } from '@loaders.gl/loader-utils';
10
+ import {coreApi} from './core-api';
11
+ import {selectLoaderSync} from './select-loader';
6
12
 
7
13
  /**
8
14
  * Creates a source from a service
@@ -13,39 +19,33 @@ import {Source, SourceArrayOptionsType, SourceArrayDataSourceType} from '@loader
13
19
  */
14
20
  // DataSourceOptionsT extends DataSourceOptions = DataSourceOptions,
15
21
  // DataSourceT extends DataSource = DataSource
16
- export function createDataSource<SourceArrayT extends Source[]>(
17
- data: string | Blob,
22
+ export function createDataSource<SourceArrayT extends SourceLoader[]>(
23
+ data: unknown,
18
24
  sources: Readonly<SourceArrayT>,
19
25
  options: Readonly<SourceArrayOptionsType<SourceArrayT>>
20
26
  ): SourceArrayDataSourceType<SourceArrayT> {
21
- const type = options?.core?.type || (options.type as unknown as string) || 'auto';
22
- const source = type === 'auto' ? selectSource(data, sources) : getSourceOfType(type, sources);
27
+ const resolvedOptions = (options || {}) as SourceArrayOptionsType<SourceArrayT>;
28
+ const type = resolvedOptions?.core?.type || (resolvedOptions.type as unknown as string) || 'auto';
29
+ const source =
30
+ sources.length === 1
31
+ ? sources[0]
32
+ : type === 'auto'
33
+ ? (selectLoaderSync(data as never, sources as unknown as SourceLoader[], {
34
+ // `createDataSource` supports non-fetch inputs such as in-memory tables.
35
+ // Selection remains best-effort for those cases.
36
+ ...resolvedOptions,
37
+ core: {...resolvedOptions?.core, type: 'auto'}
38
+ }) as SourceArrayT[number] | null)
39
+ : getSourceOfType(type, sources);
23
40
 
24
41
  if (!source) {
25
42
  throw new Error('Not a valid source type');
26
43
  }
27
- return source.createDataSource(data, options);
28
- }
29
-
30
- // TODO - use selectSource...
31
-
32
- /** Guess service type from URL */
33
- function selectSource<SourceArrayT extends Source[]>(
34
- url: string | Blob,
35
- sources: Readonly<SourceArrayT>
36
- ): SourceArrayT[number] | null {
37
- for (const service of sources) {
38
- // @ts-expect-error
39
- if (service.testURL && service.testURL(url)) {
40
- return service;
41
- }
42
- }
43
-
44
- return null;
44
+ return source.createDataSource(data as string | Blob, resolvedOptions, coreApi);
45
45
  }
46
46
 
47
47
  /** Guess service type from URL */
48
- function getSourceOfType(type: string, sources: Readonly<Source[]>): Source | null {
48
+ function getSourceOfType(type: string, sources: Readonly<SourceLoader[]>): SourceLoader | null {
49
49
  for (const service of sources) {
50
50
  if (service.type === type) {
51
51
  return service;
@@ -3,13 +3,15 @@
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
5
  import type {
6
- LoaderWithParser,
6
+ Loader,
7
7
  LoaderOptions,
8
8
  LoaderContext,
9
9
  FetchLike,
10
10
  BatchableDataType,
11
11
  LoaderBatchType,
12
- LoaderOptionsType
12
+ LoaderOptionsType,
13
+ LoaderOptionsWithShape,
14
+ LoaderShapeType
13
15
  } from '@loaders.gl/loader-utils';
14
16
  import {isLoaderObject} from '../loader-utils/normalize-loader';
15
17
  import {getFetchFunction} from '../loader-utils/get-fetch-function';
@@ -22,8 +24,11 @@ type FileType = string | File | Blob | Response | (string | File | Blob | Respon
22
24
  * Parses `data` synchronously using a specified loader
23
25
  */
24
26
  export async function loadInBatches<
25
- LoaderT extends LoaderWithParser,
26
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
27
+ LoaderT extends Loader,
28
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
29
+ LoaderOptionsType<LoaderT>,
30
+ LoaderShapeType<LoaderT>
31
+ >
27
32
  >(
28
33
  files: FileType,
29
34
  loader: LoaderT,
@@ -40,32 +45,32 @@ export async function loadInBatches<
40
45
  */
41
46
  export function loadInBatches(
42
47
  files: FileType,
43
- loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
48
+ loaders?: Loader | Loader[] | LoaderOptions,
44
49
  options?: LoaderOptions,
45
50
  context?: LoaderContext
46
51
  ): Promise<AsyncIterable<unknown>>;
47
52
 
48
53
  export function loadInBatches(
49
54
  files: FileType[] | FileList,
50
- loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
55
+ loaders?: Loader | Loader[] | LoaderOptions,
51
56
  options?: LoaderOptions,
52
57
  context?: LoaderContext
53
58
  ): Promise<AsyncIterable<unknown>>[];
54
59
 
55
60
  export function loadInBatches(
56
61
  files: FileType | FileType[] | FileList,
57
- loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
62
+ loaders?: Loader | Loader[] | LoaderOptions,
58
63
  options?: LoaderOptions,
59
64
  context?: LoaderContext
60
65
  ): Promise<AsyncIterable<unknown>> | Promise<AsyncIterable<unknown>>[] {
61
- let loadersArray: LoaderWithParser | LoaderWithParser[] | undefined;
66
+ let loadersArray: Loader | Loader[] | undefined;
62
67
  // Signature: load(url, options)
63
68
  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
64
69
  context = undefined; // context not supported in short signature
65
70
  options = loaders as LoaderOptions;
66
71
  loadersArray = undefined;
67
72
  } else {
68
- loadersArray = loaders as LoaderWithParser | LoaderWithParser[] | undefined;
73
+ loadersArray = loaders as Loader | Loader[] | undefined;
69
74
  }
70
75
 
71
76
  // Select fetch function
@@ -77,7 +82,7 @@ export function loadInBatches(
77
82
  }
78
83
 
79
84
  // Multiple URLs / files
80
- const promises = files.map((file) =>
85
+ const promises = files.map(file =>
81
86
  loadOneFileInBatches(file, loadersArray!, options || {}, fetch)
82
87
  );
83
88
 
@@ -87,7 +92,7 @@ export function loadInBatches(
87
92
 
88
93
  async function loadOneFileInBatches(
89
94
  file: FileType,
90
- loaders: LoaderWithParser | LoaderWithParser[],
95
+ loaders: Loader | Loader[],
91
96
  options: LoaderOptions,
92
97
  fetch: FetchLike
93
98
  ): Promise<AsyncIterable<unknown>> {
@@ -0,0 +1,152 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {Loader, LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
6
+
7
+ const loaderImplementationPromises = new Map<Loader, Map<string, Promise<LoaderWithParser>>>();
8
+ const loaderImplementations = new Map<Loader, Map<string, LoaderWithParser>>();
9
+
10
+ /** Gets a parser-bearing implementation for a loader, loading it if needed. */
11
+ export async function getLoaderImplementation(
12
+ loader: Loader,
13
+ options?: LoaderOptions,
14
+ url?: string
15
+ ): Promise<LoaderWithParser> {
16
+ if (isLoaderWithParser(loader)) {
17
+ return loader;
18
+ }
19
+
20
+ const implementationCacheKey = getLoaderImplementationCacheKey(loader, options);
21
+ const loaderImplementation = loaderImplementations.get(loader)?.get(implementationCacheKey);
22
+ if (loaderImplementation) {
23
+ return loaderImplementation;
24
+ }
25
+
26
+ let loaderImplementationPromisesForLoader = loaderImplementationPromises.get(loader);
27
+ if (!loaderImplementationPromisesForLoader) {
28
+ loaderImplementationPromisesForLoader = new Map();
29
+ loaderImplementationPromises.set(loader, loaderImplementationPromisesForLoader);
30
+ }
31
+
32
+ let loaderImplementationPromise =
33
+ loaderImplementationPromisesForLoader.get(implementationCacheKey);
34
+ if (!loaderImplementationPromise) {
35
+ loaderImplementationPromise = resolveLoaderImplementation(loader, options, url)
36
+ .then(implementation => {
37
+ let loaderImplementationsForLoader = loaderImplementations.get(loader);
38
+ if (!loaderImplementationsForLoader) {
39
+ loaderImplementationsForLoader = new Map();
40
+ loaderImplementations.set(loader, loaderImplementationsForLoader);
41
+ }
42
+ loaderImplementationsForLoader.set(implementationCacheKey, implementation);
43
+ return implementation;
44
+ })
45
+ .catch(error => {
46
+ loaderImplementationPromisesForLoader.delete(implementationCacheKey);
47
+ if (loaderImplementationPromisesForLoader.size === 0) {
48
+ loaderImplementationPromises.delete(loader);
49
+ }
50
+ throw error;
51
+ });
52
+ loaderImplementationPromisesForLoader.set(implementationCacheKey, loaderImplementationPromise);
53
+ }
54
+
55
+ return await loaderImplementationPromise;
56
+ }
57
+
58
+ /** Gets a cached parser-bearing implementation for a loader without loading it. */
59
+ export function getLoaderImplementationSync(loader: Loader): LoaderWithParser | null {
60
+ if (isLoaderWithParser(loader)) {
61
+ return loader;
62
+ }
63
+
64
+ const implementationCacheKey = getLoaderImplementationCacheKey(loader);
65
+ return loaderImplementations.get(loader)?.get(implementationCacheKey) || null;
66
+ }
67
+
68
+ /** Returns true when a loader object already includes parser methods. */
69
+ export function isLoaderWithParser(loader: Loader): loader is LoaderWithParser {
70
+ const candidate = loader as LoaderWithParser;
71
+ return Boolean(
72
+ candidate.parse ||
73
+ candidate.parseSync ||
74
+ candidate.parseInBatches ||
75
+ candidate.parseText ||
76
+ candidate.parseTextSync ||
77
+ candidate.parseFile ||
78
+ candidate.parseFileInBatches
79
+ );
80
+ }
81
+
82
+ /** Resolves a parser-bearing implementation through loader preload or the test import fallback. */
83
+ async function resolveLoaderImplementation(
84
+ loader: Loader,
85
+ options?: LoaderOptions,
86
+ url?: string
87
+ ): Promise<LoaderWithParser> {
88
+ if (loader.preload) {
89
+ const preloadedLoader = await loader.preload(url || '', options);
90
+ if (isLoaderWithParser(preloadedLoader as Loader)) {
91
+ return preloadedLoader as LoaderWithParser;
92
+ }
93
+
94
+ throw new Error(`${loader.id} loader preload() did not return a parser-bearing loader`);
95
+ }
96
+
97
+ const implementationSpecifier = getLoaderImplementationSpecifier(loader, options);
98
+ return await importLoaderImplementation(implementationSpecifier, loader.id);
99
+ }
100
+
101
+ /** Gets the dynamic implementation specifier for the test worker fallback path. */
102
+ function getLoaderImplementationSpecifier(loader: Loader, options?: LoaderOptions): string {
103
+ const workerType = options?._workerType || options?.core?._workerType;
104
+ if (workerType === 'test') {
105
+ const sourcePath = `modules/${loader.module}/src/${loader.id}-loader.ts`;
106
+ if (typeof window !== 'undefined') {
107
+ return `/${sourcePath}`;
108
+ }
109
+
110
+ if (typeof process !== 'undefined' && process.cwd) {
111
+ return new URL(sourcePath, `file://${process.cwd()}/`).toString();
112
+ }
113
+
114
+ return sourcePath;
115
+ }
116
+
117
+ throw new Error(
118
+ `${loader.id} loader does not provide a parser implementation. Import a parser-bearing loader directly, or use preload() before parse/load.`
119
+ );
120
+ }
121
+
122
+ /** Gets the cache key for option-selected parser implementations. */
123
+ function getLoaderImplementationCacheKey(loader: Loader, options?: LoaderOptions): string {
124
+ const loaderOptions = options?.[loader.id] as {backend?: unknown} | undefined;
125
+ const defaultLoaderOptions = loader.options?.[loader.id] as {backend?: unknown} | undefined;
126
+ return String(loaderOptions?.backend || defaultLoaderOptions?.backend || '');
127
+ }
128
+
129
+ /** Imports a module and finds the parser-bearing loader implementation with the requested id. */
130
+ async function importLoaderImplementation(
131
+ implementationSpecifier: string,
132
+ loaderId: string
133
+ ): Promise<LoaderWithParser> {
134
+ const moduleExports = await import(
135
+ /* webpackIgnore: true */ /* @vite-ignore */ implementationSpecifier
136
+ );
137
+
138
+ for (const exportValue of Object.values(moduleExports)) {
139
+ if (
140
+ exportValue &&
141
+ typeof exportValue === 'object' &&
142
+ (exportValue as Loader).id === loaderId &&
143
+ isLoaderWithParser(exportValue as Loader)
144
+ ) {
145
+ return exportValue as LoaderWithParser;
146
+ }
147
+ }
148
+
149
+ throw new Error(
150
+ `Could not find parser implementation for ${loaderId} in ${implementationSpecifier}`
151
+ );
152
+ }
@@ -7,17 +7,26 @@ import type {
7
7
  Loader,
8
8
  LoaderContext,
9
9
  LoaderOptions,
10
+ LoaderOptionsWithShape,
10
11
  LoaderOptionsType,
12
+ LoaderShapeType,
11
13
  LoaderReturnType,
12
14
  LoaderArrayOptionsType,
13
- LoaderArrayReturnType
15
+ LoaderArrayReturnType,
16
+ SourceLoader
14
17
  } from '@loaders.gl/loader-utils';
15
- import {isBlob} from '@loaders.gl/loader-utils';
18
+ import {isBlob, isSourceLoader} from '@loaders.gl/loader-utils';
16
19
  import {isLoaderObject} from '../loader-utils/normalize-loader';
17
20
  import {getFetchFunction} from '../loader-utils/get-fetch-function';
18
21
  import {normalizeLoaderOptions} from '../loader-utils/option-utils';
22
+ import {fetchFile} from '../fetch/fetch-file';
19
23
 
20
24
  import {parse} from './parse';
25
+ import {parseFile} from './parse-file';
26
+ import {parseSync} from './parse-sync';
27
+ import {parseInBatches} from './parse-in-batches';
28
+ import {loadInBatches} from './load-in-batches';
29
+ import {selectLoader} from './select-loader';
21
30
 
22
31
  /**
23
32
  * Parses `data` using a specified loader
@@ -31,7 +40,10 @@ import {parse} from './parse';
31
40
 
32
41
  export async function load<
33
42
  LoaderT extends Loader,
34
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
43
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
44
+ LoaderOptionsType<LoaderT>,
45
+ LoaderShapeType<LoaderT>
46
+ >
35
47
  >(
36
48
  url: string | DataType,
37
49
  loader: LoaderT,
@@ -81,6 +93,70 @@ export async function load(
81
93
  resolvedOptions = options as LoaderOptions;
82
94
  }
83
95
 
96
+ if (!Array.isArray(resolvedLoaders) && isSourceLoader(resolvedLoaders)) {
97
+ const sourceLoader = await resolveSourceLoader(resolvedLoaders, url, resolvedOptions);
98
+ const runtimeCoreApi = {
99
+ fetchFile,
100
+ parse,
101
+ parseFile,
102
+ parseSync,
103
+ parseInBatches,
104
+ load,
105
+ loadInBatches
106
+ };
107
+ return sourceLoader.createDataSource(
108
+ url as string | Blob,
109
+ (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
110
+ runtimeCoreApi
111
+ );
112
+ }
113
+
114
+ if (
115
+ Array.isArray(resolvedLoaders) &&
116
+ resolvedLoaders.length === 1 &&
117
+ isSourceLoader(resolvedLoaders[0])
118
+ ) {
119
+ const sourceLoader = await resolveSourceLoader(resolvedLoaders[0], url, resolvedOptions);
120
+ const runtimeCoreApi = {
121
+ fetchFile,
122
+ parse,
123
+ parseFile,
124
+ parseSync,
125
+ parseInBatches,
126
+ load,
127
+ loadInBatches
128
+ };
129
+ return sourceLoader.createDataSource(
130
+ url as string | Blob,
131
+ (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
132
+ runtimeCoreApi
133
+ );
134
+ }
135
+
136
+ if (typeof url === 'string' || isBlob(url)) {
137
+ const selectedLoader = await selectLoader(url, resolvedLoaders as Loader | Loader[], {
138
+ ...resolvedOptions,
139
+ core: {...resolvedOptions?.core, nothrow: true}
140
+ });
141
+
142
+ if (selectedLoader && isSourceLoader(selectedLoader)) {
143
+ const sourceLoader = await resolveSourceLoader(selectedLoader, url, resolvedOptions);
144
+ return sourceLoader.createDataSource(
145
+ url,
146
+ (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
147
+ {
148
+ fetchFile,
149
+ parse,
150
+ parseFile,
151
+ parseSync,
152
+ parseInBatches,
153
+ load,
154
+ loadInBatches
155
+ }
156
+ );
157
+ }
158
+ }
159
+
84
160
  // Select fetch function
85
161
  const fetch = getFetchFunction(resolvedOptions);
86
162
 
@@ -114,6 +190,23 @@ export async function load(
114
190
  // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
115
191
  // return await parse(data, loaders as Loader[], options);
116
192
  return Array.isArray(resolvedLoaders)
117
- ? await parse(data, resolvedLoaders, resolvedOptions) // loader array overload
118
- : await parse(data, resolvedLoaders, resolvedOptions); // single loader overload
193
+ ? await parse(data, resolvedLoaders, resolvedOptions, context) // loader array overload
194
+ : await parse(data, resolvedLoaders, resolvedOptions, context); // single loader overload
195
+ }
196
+
197
+ /** Resolves a lightweight source loader into its runtime implementation when necessary. */
198
+ async function resolveSourceLoader(
199
+ loader: SourceLoader,
200
+ url: string | DataType,
201
+ options?: LoaderOptions
202
+ ): Promise<SourceLoader> {
203
+ if (!loader.preload) {
204
+ return loader;
205
+ }
206
+
207
+ const sourceLoader = await loader.preload(typeof url === 'string' ? url : '', options);
208
+ if (!isSourceLoader(sourceLoader)) {
209
+ throw new Error(`${loader.id} source loader preload() did not return a runtime source loader`);
210
+ }
211
+ return sourceLoader;
119
212
  }
@@ -0,0 +1,143 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {
6
+ Loader,
7
+ LoaderContext,
8
+ LoaderOptions,
9
+ LoaderOptionsWithShape,
10
+ LoaderOptionsType,
11
+ LoaderShapeType,
12
+ LoaderReturnType,
13
+ LoaderArrayOptionsType,
14
+ LoaderArrayReturnType,
15
+ LoaderWithParser,
16
+ ReadableFile,
17
+ StrictLoaderOptions
18
+ } from '@loaders.gl/loader-utils';
19
+ import {isSourceLoader, mergeOptions} from '@loaders.gl/loader-utils';
20
+ import {validateWorkerVersion} from '@loaders.gl/worker-utils';
21
+ import {isLoaderObject} from '../loader-utils/normalize-loader';
22
+ import {normalizeOptions} from '../loader-utils/option-utils';
23
+ import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
24
+ import {getLoaderImplementation} from './load-loader';
25
+ import {parse} from './parse';
26
+ import {selectLoader} from './select-loader';
27
+
28
+ /**
29
+ * Parses a random access readable file asynchronously using the supplied loader.
30
+ */
31
+ export async function parseFile<
32
+ LoaderT extends Loader,
33
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
34
+ LoaderOptionsType<LoaderT>,
35
+ LoaderShapeType<LoaderT>
36
+ >
37
+ >(
38
+ file: ReadableFile,
39
+ loader: LoaderT,
40
+ options?: OptionsT,
41
+ context?: LoaderContext
42
+ ): Promise<LoaderReturnType<LoaderT>>;
43
+
44
+ /**
45
+ * Parses a random access readable file asynchronously by matching one of the supplied loaders.
46
+ */
47
+ export async function parseFile<
48
+ LoaderArrayT extends Loader[],
49
+ OptionsT extends LoaderOptions = LoaderArrayOptionsType<LoaderArrayT>
50
+ >(
51
+ file: ReadableFile,
52
+ loaders: LoaderArrayT,
53
+ options?: OptionsT,
54
+ context?: LoaderContext
55
+ ): Promise<LoaderArrayReturnType<LoaderArrayT>>;
56
+
57
+ /**
58
+ * Parses a random access readable file asynchronously by matching a pre-registered loader.
59
+ * @deprecated Loader registration is deprecated, use parseFile(file, loaders, options) instead.
60
+ */
61
+ export async function parseFile(file: ReadableFile, options?: LoaderOptions): Promise<unknown>;
62
+
63
+ /**
64
+ * Parses a random access readable file using a specified loader.
65
+ * @param file readable file
66
+ * @param loaders loader or loaders to use
67
+ * @param options loader options
68
+ * @param context loader context
69
+ * @returns parsed data
70
+ */
71
+ export async function parseFile(
72
+ file: ReadableFile,
73
+ loaders?: Loader | Loader[] | LoaderOptions,
74
+ options?: LoaderOptions,
75
+ context?: LoaderContext
76
+ ): Promise<unknown> {
77
+ if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
78
+ context = undefined;
79
+ options = loaders as LoaderOptions;
80
+ loaders = undefined;
81
+ }
82
+
83
+ options = options || {};
84
+ const url = file.url || '';
85
+ const typedLoaders = loaders as Loader | Loader[] | undefined;
86
+ const candidateLoaders = getLoadersFromContext(typedLoaders, context);
87
+ const loader = await selectLoader(url, candidateLoaders, options);
88
+ if (!loader) {
89
+ return null;
90
+ }
91
+
92
+ if (isSourceLoader(loader)) {
93
+ throw new Error(
94
+ `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parseFile().`
95
+ );
96
+ }
97
+
98
+ // @ts-expect-error candidateLoaders may be a single forced loader
99
+ const strictOptions = normalizeOptions(options, loader, candidateLoaders, url);
100
+ context = getLoaderContext(
101
+ // @ts-expect-error candidateLoaders may be a single forced loader
102
+ {url, _parse: parse, loaders: candidateLoaders},
103
+ strictOptions,
104
+ context || null
105
+ );
106
+
107
+ return await parseFileWithLoader(loader, file, strictOptions, context);
108
+ }
109
+
110
+ /**
111
+ * Parses a readable file after a loader has been selected.
112
+ * @param loader selected loader
113
+ * @param file readable file
114
+ * @param options strict loader options
115
+ * @param context loader context
116
+ * @returns parsed data
117
+ */
118
+ async function parseFileWithLoader(
119
+ loader: Loader,
120
+ file: ReadableFile,
121
+ options: StrictLoaderOptions,
122
+ context: LoaderContext
123
+ ): Promise<unknown> {
124
+ validateWorkerVersion(loader);
125
+ options = mergeOptions(loader.options, options);
126
+
127
+ const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
128
+ if (loaderWithParser.parseFile) {
129
+ return await loaderWithParser.parseFile(file, options, context);
130
+ }
131
+
132
+ const parseFunction = (loaderWithParser as {parse?: LoaderWithParser['parse']}).parse;
133
+ if (parseFunction) {
134
+ const length = file.bigsize > 0n ? Number(file.bigsize) : file.size;
135
+ if (!length) {
136
+ throw new Error(`${loader.id} loader - cannot fall back to parse without a file size`);
137
+ }
138
+ const data = await file.read(0n, length);
139
+ return await parse(data, loaderWithParser, options, context);
140
+ }
141
+
142
+ throw new Error(`${loader.id} loader - no file parser found`);
143
+ }