@loaders.gl/core 4.5.3 → 5.0.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/LICENSE +14 -3
  2. package/dist/dist.dev.js +1789 -352
  3. package/dist/dist.min.js +4 -3
  4. package/dist/index.cjs +3087 -311
  5. package/dist/index.cjs.map +3 -4
  6. package/dist/index.d.ts +8 -2
  7. package/dist/index.d.ts.map +1 -1
  8. package/dist/index.js +7 -1
  9. package/dist/index.js.map +1 -1
  10. package/dist/iterators/make-iterator/make-stream-iterator.js +48 -9
  11. package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -1
  12. package/dist/lib/api/core-api.d.ts +6 -0
  13. package/dist/lib/api/core-api.d.ts.map +1 -0
  14. package/dist/lib/api/core-api.js +23 -0
  15. package/dist/lib/api/core-api.js.map +1 -0
  16. package/dist/lib/api/create-data-source.d.ts +2 -2
  17. package/dist/lib/api/create-data-source.d.ts.map +1 -1
  18. package/dist/lib/api/create-data-source.js +36 -14
  19. package/dist/lib/api/create-data-source.js.map +1 -1
  20. package/dist/lib/api/load-in-batches.d.ts +4 -4
  21. package/dist/lib/api/load-in-batches.d.ts.map +1 -1
  22. package/dist/lib/api/load-in-batches.js +17 -6
  23. package/dist/lib/api/load-in-batches.js.map +1 -1
  24. package/dist/lib/api/load-loader.d.ts +10 -0
  25. package/dist/lib/api/load-loader.d.ts.map +1 -0
  26. package/dist/lib/api/load-loader.js +222 -0
  27. package/dist/lib/api/load-loader.js.map +1 -0
  28. package/dist/lib/api/load.d.ts +3 -3
  29. package/dist/lib/api/load.d.ts.map +1 -1
  30. package/dist/lib/api/load.js +84 -5
  31. package/dist/lib/api/load.js.map +1 -1
  32. package/dist/lib/api/parse-file.d.ts +15 -0
  33. package/dist/lib/api/parse-file.d.ts.map +1 -0
  34. package/dist/lib/api/parse-file.js +74 -0
  35. package/dist/lib/api/parse-file.js.map +1 -0
  36. package/dist/lib/api/parse-in-batches.d.ts +2 -2
  37. package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
  38. package/dist/lib/api/parse-in-batches.js +18 -7
  39. package/dist/lib/api/parse-in-batches.js.map +1 -1
  40. package/dist/lib/api/parse-sync.d.ts +2 -2
  41. package/dist/lib/api/parse-sync.d.ts.map +1 -1
  42. package/dist/lib/api/parse-sync.js +10 -1
  43. package/dist/lib/api/parse-sync.js.map +1 -1
  44. package/dist/lib/api/parse.d.ts +2 -2
  45. package/dist/lib/api/parse.d.ts.map +1 -1
  46. package/dist/lib/api/parse.js +34 -17
  47. package/dist/lib/api/parse.js.map +1 -1
  48. package/dist/lib/api/preload.d.ts +14 -0
  49. package/dist/lib/api/preload.d.ts.map +1 -0
  50. package/dist/lib/api/preload.js +13 -0
  51. package/dist/lib/api/preload.js.map +1 -0
  52. package/dist/lib/api/register-loaders.js +1 -1
  53. package/dist/lib/api/register-loaders.js.map +1 -1
  54. package/dist/lib/api/select-loader.d.ts.map +1 -1
  55. package/dist/lib/api/select-loader.js +48 -5
  56. package/dist/lib/api/select-loader.js.map +1 -1
  57. package/dist/lib/api/select-source.d.ts +3 -3
  58. package/dist/lib/api/select-source.d.ts.map +1 -1
  59. package/dist/lib/api/select-source.js +2 -5
  60. package/dist/lib/api/select-source.js.map +1 -1
  61. package/dist/lib/fetch/fetch-error-message.js +1 -1
  62. package/dist/lib/fetch/fetch-error-message.js.map +1 -1
  63. package/dist/lib/filesystems/browser-filesystem.d.ts.map +1 -1
  64. package/dist/lib/filesystems/browser-filesystem.js +25 -11
  65. package/dist/lib/filesystems/browser-filesystem.js.map +1 -1
  66. package/dist/lib/init.js +1 -1
  67. package/dist/lib/init.js.map +1 -1
  68. package/dist/lib/loader-utils/check-errors.js +1 -1
  69. package/dist/lib/loader-utils/check-errors.js.map +1 -1
  70. package/dist/lib/loader-utils/get-data.d.ts +4 -0
  71. package/dist/lib/loader-utils/get-data.d.ts.map +1 -1
  72. package/dist/lib/loader-utils/get-data.js +26 -0
  73. package/dist/lib/loader-utils/get-data.js.map +1 -1
  74. package/dist/lib/loader-utils/get-fetch-function.d.ts +1 -1
  75. package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
  76. package/dist/lib/loader-utils/get-fetch-function.js +16 -10
  77. package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
  78. package/dist/lib/loader-utils/loader-context.d.ts +2 -2
  79. package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
  80. package/dist/lib/loader-utils/loader-context.js +11 -4
  81. package/dist/lib/loader-utils/loader-context.js.map +1 -1
  82. package/dist/lib/loader-utils/loggers.d.ts +19 -0
  83. package/dist/lib/loader-utils/loggers.d.ts.map +1 -1
  84. package/dist/lib/loader-utils/loggers.js +28 -2
  85. package/dist/lib/loader-utils/loggers.js.map +1 -1
  86. package/dist/lib/loader-utils/option-defaults.d.ts +1 -0
  87. package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
  88. package/dist/lib/loader-utils/option-defaults.js +5 -0
  89. package/dist/lib/loader-utils/option-defaults.js.map +1 -1
  90. package/dist/lib/loader-utils/option-utils.d.ts +7 -0
  91. package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
  92. package/dist/lib/loader-utils/option-utils.js +91 -3
  93. package/dist/lib/loader-utils/option-utils.js.map +1 -1
  94. package/dist/lib/utils/response-utils.js +5 -14
  95. package/dist/lib/utils/response-utils.js.map +1 -1
  96. package/dist/null-loader.d.ts.map +1 -1
  97. package/dist/null-loader.js +1 -1
  98. package/dist/null-loader.js.map +1 -1
  99. package/dist/null-worker-node.js +270 -42
  100. package/dist/null-worker.js +270 -42
  101. package/package.json +6 -6
  102. package/src/index.ts +10 -1
  103. package/src/iterators/make-iterator/make-stream-iterator.ts +47 -9
  104. package/src/lib/api/core-api.ts +25 -0
  105. package/src/lib/api/create-data-source.ts +53 -24
  106. package/src/lib/api/load-in-batches.ts +33 -21
  107. package/src/lib/api/load-loader.ts +306 -0
  108. package/src/lib/api/load.ts +160 -9
  109. package/src/lib/api/parse-file.ts +156 -0
  110. package/src/lib/api/parse-in-batches.ts +35 -8
  111. package/src/lib/api/parse-sync.ts +27 -2
  112. package/src/lib/api/parse.ts +61 -20
  113. package/src/lib/api/preload.ts +46 -0
  114. package/src/lib/api/register-loaders.ts +1 -1
  115. package/src/lib/api/select-loader.ts +57 -5
  116. package/src/lib/api/select-source.ts +7 -10
  117. package/src/lib/fetch/fetch-error-message.ts +1 -1
  118. package/src/lib/filesystems/browser-filesystem.ts +25 -11
  119. package/src/lib/loader-utils/check-errors.ts +1 -1
  120. package/src/lib/loader-utils/get-data.ts +36 -0
  121. package/src/lib/loader-utils/get-fetch-function.ts +17 -15
  122. package/src/lib/loader-utils/loader-context.ts +13 -5
  123. package/src/lib/loader-utils/loggers.ts +28 -2
  124. package/src/lib/loader-utils/option-defaults.ts +5 -0
  125. package/src/lib/loader-utils/option-utils.ts +106 -3
  126. package/src/lib/utils/response-utils.ts +5 -15
  127. package/src/null-loader.ts +3 -3
@@ -0,0 +1,156 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {
6
+ Loader,
7
+ LoaderContext,
8
+ LoaderOptions,
9
+ LoaderOptionsWithShape,
10
+ LoaderOptionsType,
11
+ LoaderShapeType,
12
+ LoaderReturnType,
13
+ LoaderArrayOptionsType,
14
+ LoaderArrayReturnType,
15
+ LoaderWithParser,
16
+ ReadableFile,
17
+ StrictLoaderOptions
18
+ } from '@loaders.gl/loader-utils';
19
+ import {
20
+ isSourceLoader,
21
+ mergeOptions,
22
+ resolveLoaderAuthenticationOptions
23
+ } from '@loaders.gl/loader-utils';
24
+ import {validateWorkerVersion} from '@loaders.gl/worker-utils';
25
+ import {isLoaderObject} from '../loader-utils/normalize-loader';
26
+ import {normalizeOptions} from '../loader-utils/option-utils';
27
+ import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
28
+ import {getLoaderImplementation} from './load-loader';
29
+ import {parse} from './parse';
30
+ import {selectLoader} from './select-loader';
31
+
32
+ /**
33
+ * Parses a random access readable file asynchronously using the supplied loader.
34
+ */
35
+ export async function parseFile<
36
+ LoaderT extends Loader,
37
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
38
+ LoaderOptionsType<LoaderT>,
39
+ LoaderShapeType<LoaderT>
40
+ >
41
+ >(
42
+ file: ReadableFile,
43
+ loader: LoaderT,
44
+ options?: OptionsT,
45
+ context?: LoaderContext
46
+ ): Promise<LoaderReturnType<LoaderT>>;
47
+
48
+ /**
49
+ * Parses a random access readable file asynchronously by matching one of the supplied loaders.
50
+ */
51
+ export async function parseFile<
52
+ LoaderArrayT extends Loader[],
53
+ OptionsT extends LoaderOptions = LoaderArrayOptionsType<LoaderArrayT>
54
+ >(
55
+ file: ReadableFile,
56
+ loaders: LoaderArrayT,
57
+ options?: OptionsT,
58
+ context?: LoaderContext
59
+ ): Promise<LoaderArrayReturnType<LoaderArrayT>>;
60
+
61
+ /**
62
+ * Parses a random access readable file asynchronously by matching a pre-registered loader.
63
+ * @deprecated Loader registration is deprecated, use parseFile(file, loaders, options) instead.
64
+ */
65
+ export async function parseFile(file: ReadableFile, options?: LoaderOptions): Promise<unknown>;
66
+
67
+ /**
68
+ * Parses a random access readable file using a specified loader.
69
+ * @param file readable file
70
+ * @param loaders loader or loaders to use
71
+ * @param options loader options
72
+ * @param context loader context
73
+ * @returns parsed data
74
+ */
75
+ export async function parseFile(
76
+ file: ReadableFile,
77
+ loaders?: Loader | Loader[] | LoaderOptions,
78
+ options?: LoaderOptions,
79
+ context?: LoaderContext
80
+ ): Promise<unknown> {
81
+ if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
82
+ context = undefined;
83
+ options = loaders as LoaderOptions;
84
+ loaders = undefined;
85
+ }
86
+
87
+ options = options || {};
88
+ const url = file.url || '';
89
+ const typedLoaders = loaders as Loader | Loader[] | undefined;
90
+ const candidateLoaders = getLoadersFromContext(typedLoaders, context);
91
+ const loader = await selectLoader(url, candidateLoaders, options);
92
+ if (!loader) {
93
+ return null;
94
+ }
95
+
96
+ if (isSourceLoader(loader)) {
97
+ throw new Error(
98
+ `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parseFile().`
99
+ );
100
+ }
101
+
102
+ // @ts-expect-error candidateLoaders may be a single forced loader
103
+ const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url);
104
+ const authenticationLoader =
105
+ !loader.getAuthentications && normalizedOptions.core?.credentials?.length
106
+ ? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
107
+ : loader;
108
+ const strictOptions = await resolveLoaderAuthenticationOptions(
109
+ authenticationLoader,
110
+ url,
111
+ normalizedOptions
112
+ );
113
+ context = getLoaderContext(
114
+ // @ts-expect-error candidateLoaders may be a single forced loader
115
+ {url, _parse: parse, loaders: candidateLoaders},
116
+ strictOptions,
117
+ context || null
118
+ );
119
+
120
+ return await parseFileWithLoader(loader, file, strictOptions, context);
121
+ }
122
+
123
+ /**
124
+ * Parses a readable file after a loader has been selected.
125
+ * @param loader selected loader
126
+ * @param file readable file
127
+ * @param options strict loader options
128
+ * @param context loader context
129
+ * @returns parsed data
130
+ */
131
+ async function parseFileWithLoader(
132
+ loader: Loader,
133
+ file: ReadableFile,
134
+ options: StrictLoaderOptions,
135
+ context: LoaderContext
136
+ ): Promise<unknown> {
137
+ validateWorkerVersion(loader);
138
+ options = mergeOptions(loader.options, options);
139
+
140
+ const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
141
+ if (loaderWithParser.parseFile) {
142
+ return await loaderWithParser.parseFile(file, options, context);
143
+ }
144
+
145
+ const parseFunction = (loaderWithParser as {parse?: LoaderWithParser['parse']}).parse;
146
+ if (parseFunction) {
147
+ const length = file.bigsize > 0n ? Number(file.bigsize) : file.size;
148
+ if (!length) {
149
+ throw new Error(`${loader.id} loader - cannot fall back to parse without a file size`);
150
+ }
151
+ const data = await file.read(0n, length);
152
+ return await parse(data, loaderWithParser, options, context);
153
+ }
154
+
155
+ throw new Error(`${loader.id} loader - no file parser found`);
156
+ }
@@ -4,25 +4,34 @@
4
4
 
5
5
  import type {Batch} from '@loaders.gl/schema';
6
6
  import {isTable, makeBatchFromTable} from '@loaders.gl/schema-utils';
7
+ import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
7
8
  import type {
8
9
  Loader,
9
10
  LoaderWithParser,
10
11
  StrictLoaderOptions,
11
12
  LoaderOptions,
12
13
  LoaderContext,
14
+ LoaderOptionsWithShape,
13
15
  BatchableDataType,
14
16
  LoaderOptionsType,
17
+ LoaderShapeType,
15
18
  LoaderBatchType,
16
19
  LoaderArrayOptionsType,
17
20
  LoaderArrayBatchType,
18
21
  TransformBatches
19
22
  } from '@loaders.gl/loader-utils';
20
- import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
23
+ import {
24
+ canParseWithWorker,
25
+ concatenateArrayBuffersAsync,
26
+ isBrowser,
27
+ parseWithWorkerInBatches
28
+ } from '@loaders.gl/loader-utils';
21
29
  import {isLoaderObject} from '../loader-utils/normalize-loader';
22
30
  import {normalizeOptions} from '../loader-utils/option-utils';
23
31
  import {getLoaderContext} from '../loader-utils/loader-context';
24
32
  import {getAsyncIterableFromData} from '../loader-utils/get-data';
25
33
  import {getResourceUrl} from '../utils/resource-utils';
34
+ import {getLoaderImplementation} from './load-loader';
26
35
  import {selectLoader} from './select-loader';
27
36
 
28
37
  // Ensure `parse` is available in context if loader falls back to `parse`
@@ -33,7 +42,10 @@ import {parse} from './parse';
33
42
  */
34
43
  export async function parseInBatches<
35
44
  LoaderT extends Loader,
36
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
45
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
46
+ LoaderOptionsType<LoaderT>,
47
+ LoaderShapeType<LoaderT>
48
+ >
37
49
  >(
38
50
  data: BatchableDataType,
39
51
  loader: LoaderT,
@@ -101,20 +113,31 @@ export async function parseInBatches(
101
113
  }
102
114
 
103
115
  // Normalize options
104
- const strictOptions = normalizeOptions(options, loader, loaderArray, url);
116
+ const normalizedOptions = normalizeOptions(options, loader, loaderArray, url);
117
+ const authenticationLoader =
118
+ !loader.getAuthentications && normalizedOptions.core?.credentials?.length
119
+ ? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
120
+ : loader;
121
+ const strictOptions = await resolveLoaderAuthenticationOptions(
122
+ authenticationLoader,
123
+ url || '',
124
+ normalizedOptions
125
+ );
105
126
  context = getLoaderContext(
106
127
  {url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},
107
128
  strictOptions,
108
129
  context || null
109
130
  );
110
131
 
111
- return await parseWithLoaderInBatches(loader as LoaderWithParser, data, strictOptions, context);
132
+ const loaderWithParser = await getLoaderImplementation(loader, strictOptions, context.url);
133
+ return await parseWithLoaderInBatches(loader, loaderWithParser, data, strictOptions, context);
112
134
  }
113
135
 
114
136
  /**
115
137
  * Loader has been selected and context has been prepared, see if we need to emit a metadata batch
116
138
  */
117
139
  async function parseWithLoaderInBatches(
140
+ originalLoader: Loader,
118
141
  loader: LoaderWithParser,
119
142
  data: BatchableDataType,
120
143
  options: StrictLoaderOptions,
@@ -131,7 +154,7 @@ async function parseWithLoaderInBatches(
131
154
  shape: 'metadata',
132
155
  batchType: 'metadata',
133
156
  metadata: {
134
- _loader: loader,
157
+ _loader: originalLoader,
135
158
  _context: context
136
159
  },
137
160
  // Populate with some default fields to avoid crashing
@@ -169,8 +192,12 @@ async function parseToOutputIterator(
169
192
  options?.core?.transforms || []
170
193
  );
171
194
 
172
- // If loader supports parseInBatches, we are done
195
+ // Stream supported browser batch parsers through one stateful worker session.
173
196
  if (loader.parseInBatches) {
197
+ // The loader-level adapter is intentionally piloted with CSV Arrow batches first.
198
+ if (loader.id === 'csv' && isBrowser && canParseWithWorker(loader, options)) {
199
+ return parseWithWorkerInBatches(loader, transformedIterator, options, context, parse);
200
+ }
174
201
  return loader.parseInBatches(transformedIterator, options, context);
175
202
  }
176
203
 
@@ -182,7 +209,7 @@ async function* parseChunkInBatches(
182
209
  transformedIterator:
183
210
  | Iterable<ArrayBufferLike | ArrayBufferView>
184
211
  | AsyncIterable<ArrayBufferLike | ArrayBufferView>,
185
- loader: Loader,
212
+ loader: LoaderWithParser,
186
213
  options: StrictLoaderOptions,
187
214
  context: LoaderContext
188
215
  ): AsyncIterable<Batch> {
@@ -207,7 +234,7 @@ async function* parseChunkInBatches(
207
234
  * @todo run through batch builder to apply options etc...
208
235
  */
209
236
  function convertDataToBatch(parsedData: unknown, loader: Loader): Batch {
210
- // prettier-ignore
237
+ // biome-ignore format: preserve intentional fixture layout
211
238
  const batch: Batch = isTable(parsedData)
212
239
  ? makeBatchFromTable(parsedData)
213
240
  : {
@@ -7,19 +7,23 @@ import type {
7
7
  LoaderWithParser,
8
8
  LoaderOptions,
9
9
  LoaderContext,
10
+ LoaderOptionsWithShape,
10
11
  SyncDataType,
11
12
  LoaderOptionsType,
13
+ LoaderShapeType,
12
14
  LoaderReturnType,
13
15
  LoaderArrayOptionsType,
14
16
  LoaderArrayReturnType,
15
17
  StrictLoaderOptions
16
18
  } from '@loaders.gl/loader-utils';
19
+ import {isSourceLoader} from '@loaders.gl/loader-utils';
17
20
  import {selectLoaderSync} from './select-loader';
18
21
  import {isLoaderObject} from '../loader-utils/normalize-loader';
19
22
  import {normalizeOptions} from '../loader-utils/option-utils';
20
23
  import {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';
21
24
  import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
22
25
  import {getResourceUrl} from '../utils/resource-utils';
26
+ import {getLoaderImplementationSync} from './load-loader';
23
27
 
24
28
  // OVERLOADS
25
29
 
@@ -28,7 +32,10 @@ import {getResourceUrl} from '../utils/resource-utils';
28
32
  */
29
33
  export function parseSync<
30
34
  LoaderT extends Loader,
31
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
35
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
36
+ LoaderOptionsType<LoaderT>,
37
+ LoaderShapeType<LoaderT>
38
+ >
32
39
  >(
33
40
  data: SyncDataType,
34
41
  loader: LoaderT,
@@ -84,6 +91,12 @@ export function parseSync(
84
91
  return null;
85
92
  }
86
93
 
94
+ if (isSourceLoader(loader)) {
95
+ throw new Error(
96
+ `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parseSync().`
97
+ );
98
+ }
99
+
87
100
  // Normalize options
88
101
  const strictOptions = normalizeOptions(options, loader, candidateLoaders as Loader[] | undefined);
89
102
 
@@ -99,7 +112,13 @@ export function parseSync(
99
112
  context || null
100
113
  );
101
114
 
102
- return parseWithLoaderSync(loader as LoaderWithParser, data, strictOptions, context);
115
+ const loaderWithParser = getLoaderImplementationSync(loader);
116
+ return parseWithLoaderSync(
117
+ loaderWithParser || (loader as LoaderWithParser),
118
+ data,
119
+ strictOptions,
120
+ context
121
+ );
103
122
  }
104
123
 
105
124
  // TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator
@@ -119,6 +138,12 @@ function parseWithLoaderSync(
119
138
  return loader.parseSync(data, options, context); // , loader);
120
139
  }
121
140
 
141
+ if (loader.preload) {
142
+ throw new Error(
143
+ `${loader.name} loader: 'parseSync' requires a parser-bearing loader. Import the loader implementation directly, or call preload(loader) before parseSync(). ${context.url || ''}`
144
+ );
145
+ }
146
+
122
147
  // TBD - If synchronous parser not available, return null
123
148
  throw new Error(
124
149
  `${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${
@@ -2,13 +2,16 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
+ import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
5
6
  import type {
6
7
  Loader,
7
8
  LoaderContext,
8
9
  LoaderOptions,
10
+ LoaderOptionsWithShape,
9
11
  DataType,
10
12
  LoaderWithParser,
11
13
  LoaderOptionsType,
14
+ LoaderShapeType,
12
15
  LoaderReturnType,
13
16
  LoaderArrayOptionsType,
14
17
  LoaderArrayReturnType,
@@ -16,16 +19,20 @@ import type {
16
19
  } from '@loaders.gl/loader-utils';
17
20
  import {
18
21
  parseWithWorker,
19
- canParseWithWorker,
22
+ shouldParseWithWorker,
20
23
  mergeOptions,
21
- isResponse
24
+ isResponse,
25
+ isBlob,
26
+ isSourceLoader
22
27
  } from '@loaders.gl/loader-utils';
23
- import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
28
+ import {validateWorkerVersion} from '@loaders.gl/worker-utils';
24
29
  import {isLoaderObject} from '../loader-utils/normalize-loader';
25
30
  import {normalizeOptions} from '../loader-utils/option-utils';
26
31
  import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
32
+ import {getArrayBufferFromData} from '../loader-utils/get-data';
27
33
  import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
28
34
  import {getResourceUrl} from '../utils/resource-utils';
35
+ import {getLoaderImplementation} from './load-loader';
29
36
  import {selectLoader} from './select-loader';
30
37
 
31
38
  // type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T
@@ -35,7 +42,10 @@ import {selectLoader} from './select-loader';
35
42
  */
36
43
  export async function parse<
37
44
  LoaderT extends Loader,
38
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
45
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
46
+ LoaderOptionsType<LoaderT>,
47
+ LoaderShapeType<LoaderT>
48
+ >
39
49
  >(
40
50
  data: DataType | Promise<DataType>,
41
51
  loader: LoaderT,
@@ -104,9 +114,24 @@ export async function parse(
104
114
  return null;
105
115
  }
106
116
 
117
+ if (isSourceLoader(loader)) {
118
+ throw new Error(
119
+ `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parse().`
120
+ );
121
+ }
122
+
107
123
  // Normalize options
108
124
  // @ts-expect-error candidateLoaders
109
- const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
125
+ const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
126
+ const authenticationLoader =
127
+ !loader.getAuthentications && normalizedOptions.core?.credentials?.length
128
+ ? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
129
+ : loader;
130
+ const strictOptions = await resolveLoaderAuthenticationOptions(
131
+ authenticationLoader,
132
+ url || '',
133
+ normalizedOptions
134
+ );
110
135
 
111
136
  // Get a context (if already present, will be unchanged)
112
137
  context = getLoaderContext(
@@ -139,31 +164,47 @@ async function parseWithLoader(
139
164
  context.response = {headers, ok, redirected, status, statusText, type, url};
140
165
  }
141
166
 
142
- data = await getArrayBufferOrStringFromData(data, loader, options);
167
+ const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
143
168
 
144
- const loaderWithParser = loader as LoaderWithParser;
169
+ if (shouldParseWithWorker(loaderWithParser, data, options, context, loader)) {
170
+ data = await getArrayBufferFromData(data, options);
171
+ return await parseWithWorker(loaderWithParser, data, options, context, parse);
172
+ }
145
173
 
146
- // First check for synchronous text parser, wrap results in promises
147
- if (loaderWithParser.parseTextSync && typeof data === 'string') {
148
- return loaderWithParser.parseTextSync(data, options, context);
174
+ if (isBlob(data) && loaderWithParser.parseBlob) {
175
+ return await loaderWithParser.parseBlob(data, options, context);
149
176
  }
150
177
 
151
- // If we have a workerUrl and the loader can parse the given options efficiently in a worker
152
- if (canParseWithWorker(loader, options)) {
153
- return await parseWithWorker(loader, data, options, context, parse);
178
+ data = await getArrayBufferOrStringFromData(data, loader, options);
179
+
180
+ return await parseWithLoaderImplementation(loaderWithParser, data, options, context);
181
+ }
182
+
183
+ async function parseWithLoaderImplementation(
184
+ loader: LoaderWithParser,
185
+ data: string | ArrayBuffer,
186
+ options: StrictLoaderOptions,
187
+ context: LoaderContext
188
+ ): Promise<unknown> {
189
+ if (loader.parseText && typeof data === 'string') {
190
+ return await loader.parseText(data, options, context);
154
191
  }
155
192
 
156
- // Check for asynchronous parser
157
- if (loaderWithParser.parseText && typeof data === 'string') {
158
- return await loaderWithParser.parseText(data, options, context);
193
+ // Fall back to synchronous text parser, wrap results in promises
194
+ if (loader.parseTextSync && typeof data === 'string') {
195
+ return loader.parseTextSync(data, options, context);
159
196
  }
160
197
 
161
- if (loaderWithParser.parse) {
162
- return await loaderWithParser.parse(data, options, context);
198
+ if (loader.parse) {
199
+ return await loader.parse(data as ArrayBuffer, options, context);
163
200
  }
164
201
 
165
- // This should not happen, all sync loaders should also offer `parse` function
166
- assert(!loaderWithParser.parseSync);
202
+ // All parser-bearing sync loaders should also offer `parse` so async APIs can use them.
203
+ if (loader.parseSync) {
204
+ throw new Error(
205
+ `${loader.name} loader: 'parse' not available on parser-bearing sync loader. Add an async 'parse' function to this loader, or call preload(loader) before parseSync(). ${context.url || ''}`
206
+ );
207
+ }
167
208
 
168
209
  // TBD - If asynchronous parser not available, return null
169
210
  throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
@@ -0,0 +1,46 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {Loader, LoaderOptions, LoaderWithParser, SourceLoader} from '@loaders.gl/loader-utils';
6
+ import {
7
+ getLoaderImplementation,
8
+ getLoaderImplementationSync,
9
+ preloadLoaderDependencies
10
+ } from './load-loader';
11
+
12
+ /**
13
+ * Prepares a source implementation and its declared dependencies before source creation.
14
+ */
15
+ export async function preload<SourceT extends SourceLoader>(
16
+ loader: SourceT,
17
+ options?: LoaderOptions,
18
+ url?: string
19
+ ): Promise<SourceT>;
20
+
21
+ /** Prepares a parser and its recursively declared dependencies. */
22
+ export async function preload<LoaderT extends Loader>(
23
+ loader: LoaderT,
24
+ options?: LoaderOptions,
25
+ url?: string
26
+ ): Promise<LoaderWithParser>;
27
+
28
+ export async function preload(
29
+ loader: Loader,
30
+ options?: LoaderOptions,
31
+ url?: string
32
+ ): Promise<LoaderWithParser | SourceLoader> {
33
+ return loader.subloaders
34
+ ? await preloadLoaderDependencies(loader, options, url)
35
+ : await getLoaderImplementation(loader, options, url);
36
+ }
37
+
38
+ /**
39
+ * Returns a cached parser-bearing loader implementation if one is already available.
40
+ */
41
+ export function preloadSync<SourceT extends SourceLoader>(loader: SourceT): SourceT | null;
42
+ /** Returns a cached parser, including prepared default dependency bindings. */
43
+ export function preloadSync(loader: Loader): LoaderWithParser | null;
44
+ export function preloadSync(loader: Loader): LoaderWithParser | SourceLoader | null {
45
+ return getLoaderImplementationSync(loader);
46
+ }
@@ -28,7 +28,7 @@ export function registerLoaders(loaders: Loader[] | Loader) {
28
28
 
29
29
  for (const loader of loaders) {
30
30
  const normalizedLoader = normalizeLoader(loader);
31
- if (!loaderRegistry.find((registeredLoader) => normalizedLoader === registeredLoader)) {
31
+ if (!loaderRegistry.find(registeredLoader => normalizedLoader === registeredLoader)) {
32
32
  // add to the beginning of the loaderRegistry, so the last registeredLoader get picked
33
33
  loaderRegistry.unshift(normalizedLoader);
34
34
  }
@@ -9,7 +9,8 @@ import {
9
9
  log,
10
10
  isBlob,
11
11
  ensureArrayBuffer,
12
- isArrayBufferLike
12
+ isArrayBufferLike,
13
+ isSourceLoader
13
14
  } from '@loaders.gl/loader-utils';
14
15
  import {TypedArray} from '@loaders.gl/schema';
15
16
  import {normalizeLoader} from '../loader-utils/normalize-loader';
@@ -95,7 +96,9 @@ function mayContainText(response: Response): boolean {
95
96
  const mimeType = getResourceMIMEType(response);
96
97
  return Boolean(
97
98
  mimeType &&
98
- (mimeType.startsWith('text/') || mimeType === 'application/json' || mimeType.endsWith('+json'))
99
+ (mimeType.startsWith('text/') ||
100
+ mimeType === 'application/json' ||
101
+ mimeType.endsWith('+json'))
99
102
  );
100
103
  }
101
104
 
@@ -169,12 +172,23 @@ function selectLoaderInternal(
169
172
  let loader: Loader | null = null;
170
173
  let reason: string = '';
171
174
 
175
+ // if options.mimeType is supplied, it takes precedence
176
+ const sourceType =
177
+ options?.core && 'type' in options.core ? (options.core.type as string | undefined) : undefined;
178
+ if (sourceType && sourceType !== 'auto') {
179
+ loader = findSourceLoaderByType(loaders, sourceType);
180
+ reason = loader ? `match forced by supplied source type ${sourceType}` : '';
181
+ }
182
+
172
183
  // if options.mimeType is supplied, it takes precedence
173
184
  if (options?.core?.mimeType) {
174
185
  loader = findLoaderByMIMEType(loaders, options?.core?.mimeType);
175
186
  reason = `match forced by supplied MIME type ${options?.core?.mimeType}`;
176
187
  }
177
188
 
189
+ loader = loader || findSourceLoaderByTestURL(loaders, testUrl);
190
+ reason = reason || (loader ? `matched source url ${testUrl}` : '');
191
+
178
192
  // Look up loader by url
179
193
  loader = loader || findLoaderByUrl(loaders, testUrl);
180
194
  reason = reason || (loader ? `matched url ${testUrl}` : '');
@@ -189,6 +203,11 @@ function selectLoaderInternal(
189
203
  // @ts-ignore Blob | Response
190
204
  reason = reason || (loader ? `matched initial data ${getFirstCharacters(data)}` : '');
191
205
 
206
+ if (!loader && isBlob(data)) {
207
+ loader = findSourceLoaderByTestData(loaders, data);
208
+ reason = reason || (loader ? 'matched source testData' : '');
209
+ }
210
+
192
211
  // Look up loader by fallback mime type
193
212
  if (options?.core?.fallbackMimeType) {
194
213
  loader = loader || findLoaderByMIMEType(loaders, options?.core?.fallbackMimeType);
@@ -258,9 +277,42 @@ function findLoaderByExtension(loaders: Loader[], extension: string): Loader | n
258
277
  return null;
259
278
  }
260
279
 
280
+ function findSourceLoaderByType(loaders: Loader[], type: string): Loader | null {
281
+ for (const loader of loaders) {
282
+ if (isSourceLoader(loader) && loader.type === type) {
283
+ return loader;
284
+ }
285
+ }
286
+ return null;
287
+ }
288
+
289
+ function findSourceLoaderByTestURL(loaders: Loader[], url?: string): Loader | null {
290
+ if (!url) {
291
+ return null;
292
+ }
293
+
294
+ for (const loader of loaders) {
295
+ if (isSourceLoader(loader) && loader.testURL(url)) {
296
+ return loader;
297
+ }
298
+ }
299
+ return null;
300
+ }
301
+
302
+ function findSourceLoaderByTestData(loaders: Loader[], data: Blob): Loader | null {
303
+ for (const loader of loaders) {
304
+ if (isSourceLoader(loader)) {
305
+ if (loader.testData?.(data)) {
306
+ return loader;
307
+ }
308
+ }
309
+ }
310
+ return null;
311
+ }
312
+
261
313
  function findLoaderByMIMEType(loaders: Loader[], mimeType: string): Loader | null {
262
314
  for (const loader of loaders) {
263
- if (loader.mimeTypes?.some((mimeType1) => compareMIMETypes(mimeType, mimeType1))) {
315
+ if (loader.mimeTypes?.some(mimeType1 => compareMIMETypes(mimeType, mimeType1))) {
264
316
  return loader;
265
317
  }
266
318
 
@@ -305,12 +357,12 @@ function testDataAgainstText(data: string, loader: Loader): boolean {
305
357
  }
306
358
 
307
359
  const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
308
- return tests.some((test) => data.startsWith(test as string));
360
+ return tests.some(test => data.startsWith(test as string));
309
361
  }
310
362
 
311
363
  function testDataAgainstBinary(data: ArrayBufferLike, byteOffset: number, loader: Loader): boolean {
312
364
  const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
313
- return tests.some((test) => testBinary(data, byteOffset, loader, test));
365
+ return tests.some(test => testBinary(data, byteOffset, loader, test));
314
366
  }
315
367
 
316
368
  function testBinary(