@loaders.gl/core 4.4.2 → 5.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/dist.dev.js +583 -162
  2. package/dist/dist.min.js +4 -3
  3. package/dist/index.cjs +499 -148
  4. package/dist/index.cjs.map +4 -4
  5. package/dist/index.d.ts +8 -2
  6. package/dist/index.d.ts.map +1 -1
  7. package/dist/index.js +7 -1
  8. package/dist/index.js.map +1 -1
  9. package/dist/iterators/make-iterator/make-stream-iterator.js +1 -1
  10. package/dist/iterators/make-iterator/make-stream-iterator.js.map +1 -1
  11. package/dist/lib/api/core-api.d.ts +6 -0
  12. package/dist/lib/api/core-api.d.ts.map +1 -0
  13. package/dist/lib/api/core-api.js +23 -0
  14. package/dist/lib/api/core-api.js.map +1 -0
  15. package/dist/lib/api/create-data-source.d.ts +2 -2
  16. package/dist/lib/api/create-data-source.d.ts.map +1 -1
  17. package/dist/lib/api/create-data-source.js +15 -14
  18. package/dist/lib/api/create-data-source.js.map +1 -1
  19. package/dist/lib/api/load-in-batches.d.ts +4 -4
  20. package/dist/lib/api/load-in-batches.d.ts.map +1 -1
  21. package/dist/lib/api/load-in-batches.js +1 -1
  22. package/dist/lib/api/load-in-batches.js.map +1 -1
  23. package/dist/lib/api/load-loader.d.ts +8 -0
  24. package/dist/lib/api/load-loader.d.ts.map +1 -0
  25. package/dist/lib/api/load-loader.js +116 -0
  26. package/dist/lib/api/load-loader.js.map +1 -0
  27. package/dist/lib/api/load.d.ts +2 -2
  28. package/dist/lib/api/load.d.ts.map +1 -1
  29. package/dist/lib/api/load.js +52 -3
  30. package/dist/lib/api/load.js.map +1 -1
  31. package/dist/lib/api/parse-file.d.ts +15 -0
  32. package/dist/lib/api/parse-file.d.ts.map +1 -0
  33. package/dist/lib/api/parse-file.js +70 -0
  34. package/dist/lib/api/parse-file.js.map +1 -0
  35. package/dist/lib/api/parse-in-batches.d.ts +2 -2
  36. package/dist/lib/api/parse-in-batches.d.ts.map +1 -1
  37. package/dist/lib/api/parse-in-batches.js +6 -4
  38. package/dist/lib/api/parse-in-batches.js.map +1 -1
  39. package/dist/lib/api/parse-sync.d.ts +2 -2
  40. package/dist/lib/api/parse-sync.d.ts.map +1 -1
  41. package/dist/lib/api/parse-sync.js +10 -1
  42. package/dist/lib/api/parse-sync.js.map +1 -1
  43. package/dist/lib/api/parse.d.ts +2 -2
  44. package/dist/lib/api/parse.d.ts.map +1 -1
  45. package/dist/lib/api/parse.js +25 -16
  46. package/dist/lib/api/parse.js.map +1 -1
  47. package/dist/lib/api/preload.d.ts +11 -0
  48. package/dist/lib/api/preload.d.ts.map +1 -0
  49. package/dist/lib/api/preload.js +14 -0
  50. package/dist/lib/api/preload.js.map +1 -0
  51. package/dist/lib/api/register-loaders.js +1 -1
  52. package/dist/lib/api/register-loaders.js.map +1 -1
  53. package/dist/lib/api/select-loader.d.ts.map +1 -1
  54. package/dist/lib/api/select-loader.js +48 -5
  55. package/dist/lib/api/select-loader.js.map +1 -1
  56. package/dist/lib/api/select-source.d.ts +3 -3
  57. package/dist/lib/api/select-source.d.ts.map +1 -1
  58. package/dist/lib/api/select-source.js +2 -5
  59. package/dist/lib/api/select-source.js.map +1 -1
  60. package/dist/lib/fetch/fetch-error-message.js +1 -1
  61. package/dist/lib/fetch/fetch-error-message.js.map +1 -1
  62. package/dist/lib/init.js +1 -1
  63. package/dist/lib/init.js.map +1 -1
  64. package/dist/lib/loader-utils/check-errors.js +1 -1
  65. package/dist/lib/loader-utils/check-errors.js.map +1 -1
  66. package/dist/lib/loader-utils/get-data.d.ts +4 -0
  67. package/dist/lib/loader-utils/get-data.d.ts.map +1 -1
  68. package/dist/lib/loader-utils/get-data.js +26 -0
  69. package/dist/lib/loader-utils/get-data.js.map +1 -1
  70. package/dist/lib/loader-utils/get-fetch-function.d.ts +1 -1
  71. package/dist/lib/loader-utils/get-fetch-function.d.ts.map +1 -1
  72. package/dist/lib/loader-utils/get-fetch-function.js +1 -1
  73. package/dist/lib/loader-utils/get-fetch-function.js.map +1 -1
  74. package/dist/lib/loader-utils/loader-context.d.ts +1 -1
  75. package/dist/lib/loader-utils/loader-context.d.ts.map +1 -1
  76. package/dist/lib/loader-utils/loader-context.js +3 -1
  77. package/dist/lib/loader-utils/loader-context.js.map +1 -1
  78. package/dist/lib/loader-utils/loggers.d.ts +19 -0
  79. package/dist/lib/loader-utils/loggers.d.ts.map +1 -1
  80. package/dist/lib/loader-utils/loggers.js +28 -2
  81. package/dist/lib/loader-utils/loggers.js.map +1 -1
  82. package/dist/lib/loader-utils/option-defaults.d.ts +1 -0
  83. package/dist/lib/loader-utils/option-defaults.d.ts.map +1 -1
  84. package/dist/lib/loader-utils/option-defaults.js +2 -0
  85. package/dist/lib/loader-utils/option-defaults.js.map +1 -1
  86. package/dist/lib/loader-utils/option-utils.d.ts.map +1 -1
  87. package/dist/lib/loader-utils/option-utils.js +50 -2
  88. package/dist/lib/loader-utils/option-utils.js.map +1 -1
  89. package/dist/lib/utils/response-utils.js +3 -3
  90. package/dist/lib/utils/response-utils.js.map +1 -1
  91. package/dist/null-loader.d.ts.map +1 -1
  92. package/dist/null-loader.js +1 -1
  93. package/dist/null-loader.js.map +1 -1
  94. package/dist/null-worker-node.js +199 -36
  95. package/dist/null-worker.js +199 -36
  96. package/package.json +6 -6
  97. package/src/index.ts +10 -1
  98. package/src/iterators/make-iterator/make-stream-iterator.ts +1 -1
  99. package/src/lib/api/core-api.ts +25 -0
  100. package/src/lib/api/create-data-source.ts +24 -24
  101. package/src/lib/api/load-in-batches.ts +16 -11
  102. package/src/lib/api/load-loader.ts +163 -0
  103. package/src/lib/api/load.ts +78 -5
  104. package/src/lib/api/parse-file.ts +143 -0
  105. package/src/lib/api/parse-in-batches.ts +13 -5
  106. package/src/lib/api/parse-sync.ts +27 -2
  107. package/src/lib/api/parse.ts +44 -18
  108. package/src/lib/api/preload.ts +31 -0
  109. package/src/lib/api/register-loaders.ts +1 -1
  110. package/src/lib/api/select-loader.ts +57 -5
  111. package/src/lib/api/select-source.ts +7 -10
  112. package/src/lib/fetch/fetch-error-message.ts +1 -1
  113. package/src/lib/loader-utils/check-errors.ts +1 -1
  114. package/src/lib/loader-utils/get-data.ts +36 -0
  115. package/src/lib/loader-utils/get-fetch-function.ts +3 -2
  116. package/src/lib/loader-utils/loader-context.ts +5 -2
  117. package/src/lib/loader-utils/loggers.ts +28 -2
  118. package/src/lib/loader-utils/option-defaults.ts +2 -0
  119. package/src/lib/loader-utils/option-utils.ts +59 -2
  120. package/src/lib/utils/response-utils.ts +3 -3
  121. package/src/null-loader.ts +3 -3
@@ -0,0 +1,25 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {CoreAPI} from '@loaders.gl/loader-utils';
6
+ import {fetchFile} from '../fetch/fetch-file';
7
+ import {parse} from './parse';
8
+ import {parseFile} from './parse-file';
9
+ import {parseSync} from './parse-sync';
10
+ import {parseInBatches} from './parse-in-batches';
11
+ import {load} from './load';
12
+ import {loadInBatches} from './load-in-batches';
13
+
14
+ /**
15
+ * Shared Core API surface that can be injected into contexts and sources.
16
+ */
17
+ export const coreApi: CoreAPI = Object.freeze({
18
+ fetchFile,
19
+ parse,
20
+ parseFile,
21
+ parseSync,
22
+ parseInBatches,
23
+ load,
24
+ loadInBatches
25
+ });
@@ -2,7 +2,13 @@
2
2
  // SPDX-License-Identifier: MIT
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
- import {Source, SourceArrayOptionsType, SourceArrayDataSourceType} from '@loaders.gl/loader-utils';
5
+ import {
6
+ SourceLoader,
7
+ SourceArrayOptionsType,
8
+ SourceArrayDataSourceType
9
+ } from '@loaders.gl/loader-utils';
10
+ import {coreApi} from './core-api';
11
+ import {selectLoaderSync} from './select-loader';
6
12
 
7
13
  /**
8
14
  * Creates a source from a service
@@ -13,39 +19,33 @@ import {Source, SourceArrayOptionsType, SourceArrayDataSourceType} from '@loader
13
19
  */
14
20
  // DataSourceOptionsT extends DataSourceOptions = DataSourceOptions,
15
21
  // DataSourceT extends DataSource = DataSource
16
- export function createDataSource<SourceArrayT extends Source[]>(
17
- data: string | Blob,
22
+ export function createDataSource<SourceArrayT extends SourceLoader[]>(
23
+ data: unknown,
18
24
  sources: Readonly<SourceArrayT>,
19
25
  options: Readonly<SourceArrayOptionsType<SourceArrayT>>
20
26
  ): SourceArrayDataSourceType<SourceArrayT> {
21
- const type = options?.core?.type || (options.type as unknown as string) || 'auto';
22
- const source = type === 'auto' ? selectSource(data, sources) : getSourceOfType(type, sources);
27
+ const resolvedOptions = (options || {}) as SourceArrayOptionsType<SourceArrayT>;
28
+ const type = resolvedOptions?.core?.type || (resolvedOptions.type as unknown as string) || 'auto';
29
+ const source =
30
+ sources.length === 1
31
+ ? sources[0]
32
+ : type === 'auto'
33
+ ? (selectLoaderSync(data as never, sources as unknown as SourceLoader[], {
34
+ // `createDataSource` supports non-fetch inputs such as in-memory tables.
35
+ // Selection remains best-effort for those cases.
36
+ ...resolvedOptions,
37
+ core: {...resolvedOptions?.core, type: 'auto'}
38
+ }) as SourceArrayT[number] | null)
39
+ : getSourceOfType(type, sources);
23
40
 
24
41
  if (!source) {
25
42
  throw new Error('Not a valid source type');
26
43
  }
27
- return source.createDataSource(data, options);
28
- }
29
-
30
- // TODO - use selectSource...
31
-
32
- /** Guess service type from URL */
33
- function selectSource<SourceArrayT extends Source[]>(
34
- url: string | Blob,
35
- sources: Readonly<SourceArrayT>
36
- ): SourceArrayT[number] | null {
37
- for (const service of sources) {
38
- // @ts-expect-error
39
- if (service.testURL && service.testURL(url)) {
40
- return service;
41
- }
42
- }
43
-
44
- return null;
44
+ return source.createDataSource(data as string | Blob, resolvedOptions, coreApi);
45
45
  }
46
46
 
47
47
  /** Guess service type from URL */
48
- function getSourceOfType(type: string, sources: Readonly<Source[]>): Source | null {
48
+ function getSourceOfType(type: string, sources: Readonly<SourceLoader[]>): SourceLoader | null {
49
49
  for (const service of sources) {
50
50
  if (service.type === type) {
51
51
  return service;
@@ -3,13 +3,15 @@
3
3
  // Copyright (c) vis.gl contributors
4
4
 
5
5
  import type {
6
- LoaderWithParser,
6
+ Loader,
7
7
  LoaderOptions,
8
8
  LoaderContext,
9
9
  FetchLike,
10
10
  BatchableDataType,
11
11
  LoaderBatchType,
12
- LoaderOptionsType
12
+ LoaderOptionsType,
13
+ LoaderOptionsWithShape,
14
+ LoaderShapeType
13
15
  } from '@loaders.gl/loader-utils';
14
16
  import {isLoaderObject} from '../loader-utils/normalize-loader';
15
17
  import {getFetchFunction} from '../loader-utils/get-fetch-function';
@@ -22,8 +24,11 @@ type FileType = string | File | Blob | Response | (string | File | Blob | Respon
22
24
  * Parses `data` synchronously using a specified loader
23
25
  */
24
26
  export async function loadInBatches<
25
- LoaderT extends LoaderWithParser,
26
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
27
+ LoaderT extends Loader,
28
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
29
+ LoaderOptionsType<LoaderT>,
30
+ LoaderShapeType<LoaderT>
31
+ >
27
32
  >(
28
33
  files: FileType,
29
34
  loader: LoaderT,
@@ -40,32 +45,32 @@ export async function loadInBatches<
40
45
  */
41
46
  export function loadInBatches(
42
47
  files: FileType,
43
- loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
48
+ loaders?: Loader | Loader[] | LoaderOptions,
44
49
  options?: LoaderOptions,
45
50
  context?: LoaderContext
46
51
  ): Promise<AsyncIterable<unknown>>;
47
52
 
48
53
  export function loadInBatches(
49
54
  files: FileType[] | FileList,
50
- loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
55
+ loaders?: Loader | Loader[] | LoaderOptions,
51
56
  options?: LoaderOptions,
52
57
  context?: LoaderContext
53
58
  ): Promise<AsyncIterable<unknown>>[];
54
59
 
55
60
  export function loadInBatches(
56
61
  files: FileType | FileType[] | FileList,
57
- loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
62
+ loaders?: Loader | Loader[] | LoaderOptions,
58
63
  options?: LoaderOptions,
59
64
  context?: LoaderContext
60
65
  ): Promise<AsyncIterable<unknown>> | Promise<AsyncIterable<unknown>>[] {
61
- let loadersArray: LoaderWithParser | LoaderWithParser[] | undefined;
66
+ let loadersArray: Loader | Loader[] | undefined;
62
67
  // Signature: load(url, options)
63
68
  if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
64
69
  context = undefined; // context not supported in short signature
65
70
  options = loaders as LoaderOptions;
66
71
  loadersArray = undefined;
67
72
  } else {
68
- loadersArray = loaders as LoaderWithParser | LoaderWithParser[] | undefined;
73
+ loadersArray = loaders as Loader | Loader[] | undefined;
69
74
  }
70
75
 
71
76
  // Select fetch function
@@ -77,7 +82,7 @@ export function loadInBatches(
77
82
  }
78
83
 
79
84
  // Multiple URLs / files
80
- const promises = files.map((file) =>
85
+ const promises = files.map(file =>
81
86
  loadOneFileInBatches(file, loadersArray!, options || {}, fetch)
82
87
  );
83
88
 
@@ -87,7 +92,7 @@ export function loadInBatches(
87
92
 
88
93
  async function loadOneFileInBatches(
89
94
  file: FileType,
90
- loaders: LoaderWithParser | LoaderWithParser[],
95
+ loaders: Loader | Loader[],
91
96
  options: LoaderOptions,
92
97
  fetch: FetchLike
93
98
  ): Promise<AsyncIterable<unknown>> {
@@ -0,0 +1,163 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {Loader, LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
6
+
7
+ const loaderImplementationPromises = new Map<Loader, Map<string, Promise<LoaderWithParser>>>();
8
+ const loaderImplementations = new Map<Loader, Map<string, LoaderWithParser>>();
9
+
10
+ /** Gets a parser-bearing implementation for a loader, loading it if needed. */
11
+ export async function getLoaderImplementation(
12
+ loader: Loader,
13
+ options?: LoaderOptions,
14
+ url?: string
15
+ ): Promise<LoaderWithParser> {
16
+ if (isLoaderWithParser(loader)) {
17
+ return loader;
18
+ }
19
+
20
+ const implementationCacheKey = getLoaderImplementationCacheKey(loader, options);
21
+ const loaderImplementation = loaderImplementations.get(loader)?.get(implementationCacheKey);
22
+ if (loaderImplementation) {
23
+ return loaderImplementation;
24
+ }
25
+
26
+ let loaderImplementationPromisesForLoader = loaderImplementationPromises.get(loader);
27
+ if (!loaderImplementationPromisesForLoader) {
28
+ loaderImplementationPromisesForLoader = new Map();
29
+ loaderImplementationPromises.set(loader, loaderImplementationPromisesForLoader);
30
+ }
31
+
32
+ let loaderImplementationPromise =
33
+ loaderImplementationPromisesForLoader.get(implementationCacheKey);
34
+ if (!loaderImplementationPromise) {
35
+ loaderImplementationPromise = resolveLoaderImplementation(loader, options, url)
36
+ .then(implementation => {
37
+ let loaderImplementationsForLoader = loaderImplementations.get(loader);
38
+ if (!loaderImplementationsForLoader) {
39
+ loaderImplementationsForLoader = new Map();
40
+ loaderImplementations.set(loader, loaderImplementationsForLoader);
41
+ }
42
+ loaderImplementationsForLoader.set(implementationCacheKey, implementation);
43
+ return implementation;
44
+ })
45
+ .catch(error => {
46
+ loaderImplementationPromisesForLoader.delete(implementationCacheKey);
47
+ if (loaderImplementationPromisesForLoader.size === 0) {
48
+ loaderImplementationPromises.delete(loader);
49
+ }
50
+ throw error;
51
+ });
52
+ loaderImplementationPromisesForLoader.set(implementationCacheKey, loaderImplementationPromise);
53
+ }
54
+
55
+ return await loaderImplementationPromise;
56
+ }
57
+
58
+ /** Gets a cached parser-bearing implementation for a loader without loading it. */
59
+ export function getLoaderImplementationSync(loader: Loader): LoaderWithParser | null {
60
+ if (isLoaderWithParser(loader)) {
61
+ return loader;
62
+ }
63
+
64
+ const implementationCacheKey = getLoaderImplementationCacheKey(loader);
65
+ return loaderImplementations.get(loader)?.get(implementationCacheKey) || null;
66
+ }
67
+
68
+ /** Returns true when a loader object already includes parser methods. */
69
+ export function isLoaderWithParser(loader: Loader): loader is LoaderWithParser {
70
+ const candidate = loader as LoaderWithParser;
71
+ return Boolean(
72
+ candidate.parse ||
73
+ candidate.parseSync ||
74
+ candidate.parseInBatches ||
75
+ candidate.parseText ||
76
+ candidate.parseTextSync ||
77
+ candidate.parseFile ||
78
+ candidate.parseFileInBatches
79
+ );
80
+ }
81
+
82
+ /** Resolves a parser-bearing implementation through loader preload or the test import fallback. */
83
+ async function resolveLoaderImplementation(
84
+ loader: Loader,
85
+ options?: LoaderOptions,
86
+ url?: string
87
+ ): Promise<LoaderWithParser> {
88
+ if (loader.preload) {
89
+ const preloadedLoader = await loader.preload(url || '', options);
90
+ if (isLoaderWithParser(preloadedLoader as Loader)) {
91
+ return preloadedLoader as LoaderWithParser;
92
+ }
93
+
94
+ throw new Error(`${loader.id} loader preload() did not return a parser-bearing loader`);
95
+ }
96
+
97
+ const implementationSpecifier = getLoaderImplementationSpecifier(loader, options);
98
+ return await importLoaderImplementation(implementationSpecifier, loader.id);
99
+ }
100
+
101
+ /** Gets the dynamic implementation specifier for the test worker fallback path. */
102
+ function getLoaderImplementationSpecifier(loader: Loader, options?: LoaderOptions): string {
103
+ const workerType = options?._workerType || options?.core?._workerType;
104
+ if (workerType === 'test') {
105
+ const sourcePath = `modules/${loader.module}/src/${loader.id}-loader.ts`;
106
+ if (typeof window !== 'undefined') {
107
+ return `/${sourcePath}`;
108
+ }
109
+
110
+ if (typeof process !== 'undefined' && process.cwd) {
111
+ return new URL(sourcePath, `file://${process.cwd()}/`).toString();
112
+ }
113
+
114
+ return sourcePath;
115
+ }
116
+
117
+ throw new Error(
118
+ `${loader.id} loader does not provide a parser implementation. Import a parser-bearing loader directly, or use preload() before parse/load.`
119
+ );
120
+ }
121
+
122
+ /** Gets the cache key for option-selected parser implementations. */
123
+ function getLoaderImplementationCacheKey(loader: Loader, options?: LoaderOptions): string {
124
+ const loaderOptions = options?.[loader.id] as
125
+ | {backend?: unknown; implementation?: unknown}
126
+ | undefined;
127
+ const defaultLoaderOptions = loader.options?.[loader.id] as {backend?: unknown} | undefined;
128
+ return String(
129
+ loaderOptions?.backend ||
130
+ getDeprecatedImplementationBackend(loaderOptions?.implementation) ||
131
+ defaultLoaderOptions?.backend ||
132
+ ''
133
+ );
134
+ }
135
+
136
+ function getDeprecatedImplementationBackend(implementation: unknown): unknown {
137
+ return implementation === 'js' ? 'typescript' : implementation;
138
+ }
139
+
140
+ /** Imports a module and finds the parser-bearing loader implementation with the requested id. */
141
+ async function importLoaderImplementation(
142
+ implementationSpecifier: string,
143
+ loaderId: string
144
+ ): Promise<LoaderWithParser> {
145
+ const moduleExports = await import(
146
+ /* webpackIgnore: true */ /* @vite-ignore */ implementationSpecifier
147
+ );
148
+
149
+ for (const exportValue of Object.values(moduleExports)) {
150
+ if (
151
+ exportValue &&
152
+ typeof exportValue === 'object' &&
153
+ (exportValue as Loader).id === loaderId &&
154
+ isLoaderWithParser(exportValue as Loader)
155
+ ) {
156
+ return exportValue as LoaderWithParser;
157
+ }
158
+ }
159
+
160
+ throw new Error(
161
+ `Could not find parser implementation for ${loaderId} in ${implementationSpecifier}`
162
+ );
163
+ }
@@ -7,17 +7,26 @@ import type {
7
7
  Loader,
8
8
  LoaderContext,
9
9
  LoaderOptions,
10
+ LoaderOptionsWithShape,
10
11
  LoaderOptionsType,
12
+ LoaderShapeType,
11
13
  LoaderReturnType,
12
14
  LoaderArrayOptionsType,
13
- LoaderArrayReturnType
15
+ LoaderArrayReturnType,
16
+ SourceLoader
14
17
  } from '@loaders.gl/loader-utils';
15
- import {isBlob} from '@loaders.gl/loader-utils';
18
+ import {isBlob, isSourceLoader} from '@loaders.gl/loader-utils';
16
19
  import {isLoaderObject} from '../loader-utils/normalize-loader';
17
20
  import {getFetchFunction} from '../loader-utils/get-fetch-function';
18
21
  import {normalizeLoaderOptions} from '../loader-utils/option-utils';
22
+ import {fetchFile} from '../fetch/fetch-file';
19
23
 
20
24
  import {parse} from './parse';
25
+ import {parseFile} from './parse-file';
26
+ import {parseSync} from './parse-sync';
27
+ import {parseInBatches} from './parse-in-batches';
28
+ import {loadInBatches} from './load-in-batches';
29
+ import {selectLoader} from './select-loader';
21
30
 
22
31
  /**
23
32
  * Parses `data` using a specified loader
@@ -31,7 +40,10 @@ import {parse} from './parse';
31
40
 
32
41
  export async function load<
33
42
  LoaderT extends Loader,
34
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
43
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
44
+ LoaderOptionsType<LoaderT>,
45
+ LoaderShapeType<LoaderT>
46
+ >
35
47
  >(
36
48
  url: string | DataType,
37
49
  loader: LoaderT,
@@ -81,6 +93,67 @@ export async function load(
81
93
  resolvedOptions = options as LoaderOptions;
82
94
  }
83
95
 
96
+ if (!Array.isArray(resolvedLoaders) && isSourceLoader(resolvedLoaders)) {
97
+ const runtimeCoreApi = {
98
+ fetchFile,
99
+ parse,
100
+ parseFile,
101
+ parseSync,
102
+ parseInBatches,
103
+ load,
104
+ loadInBatches
105
+ };
106
+ return resolvedLoaders.createDataSource(
107
+ url as string | Blob,
108
+ (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
109
+ runtimeCoreApi
110
+ );
111
+ }
112
+
113
+ if (
114
+ Array.isArray(resolvedLoaders) &&
115
+ resolvedLoaders.length === 1 &&
116
+ isSourceLoader(resolvedLoaders[0])
117
+ ) {
118
+ const runtimeCoreApi = {
119
+ fetchFile,
120
+ parse,
121
+ parseFile,
122
+ parseSync,
123
+ parseInBatches,
124
+ load,
125
+ loadInBatches
126
+ };
127
+ return resolvedLoaders[0].createDataSource(
128
+ url as string | Blob,
129
+ (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
130
+ runtimeCoreApi
131
+ );
132
+ }
133
+
134
+ if (typeof url === 'string' || isBlob(url)) {
135
+ const selectedLoader = await selectLoader(url, resolvedLoaders as Loader | Loader[], {
136
+ ...resolvedOptions,
137
+ core: {...resolvedOptions?.core, nothrow: true}
138
+ });
139
+
140
+ if (selectedLoader && isSourceLoader(selectedLoader)) {
141
+ return selectedLoader.createDataSource(
142
+ url,
143
+ (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
144
+ {
145
+ fetchFile,
146
+ parse,
147
+ parseFile,
148
+ parseSync,
149
+ parseInBatches,
150
+ load,
151
+ loadInBatches
152
+ }
153
+ );
154
+ }
155
+ }
156
+
84
157
  // Select fetch function
85
158
  const fetch = getFetchFunction(resolvedOptions);
86
159
 
@@ -114,6 +187,6 @@ export async function load(
114
187
  // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
115
188
  // return await parse(data, loaders as Loader[], options);
116
189
  return Array.isArray(resolvedLoaders)
117
- ? await parse(data, resolvedLoaders, resolvedOptions) // loader array overload
118
- : await parse(data, resolvedLoaders, resolvedOptions); // single loader overload
190
+ ? await parse(data, resolvedLoaders, resolvedOptions, context) // loader array overload
191
+ : await parse(data, resolvedLoaders, resolvedOptions, context); // single loader overload
119
192
  }
@@ -0,0 +1,143 @@
1
+ // loaders.gl
2
+ // SPDX-License-Identifier: MIT
3
+ // Copyright (c) vis.gl contributors
4
+
5
+ import type {
6
+ Loader,
7
+ LoaderContext,
8
+ LoaderOptions,
9
+ LoaderOptionsWithShape,
10
+ LoaderOptionsType,
11
+ LoaderShapeType,
12
+ LoaderReturnType,
13
+ LoaderArrayOptionsType,
14
+ LoaderArrayReturnType,
15
+ LoaderWithParser,
16
+ ReadableFile,
17
+ StrictLoaderOptions
18
+ } from '@loaders.gl/loader-utils';
19
+ import {isSourceLoader, mergeOptions} from '@loaders.gl/loader-utils';
20
+ import {validateWorkerVersion} from '@loaders.gl/worker-utils';
21
+ import {isLoaderObject} from '../loader-utils/normalize-loader';
22
+ import {normalizeOptions} from '../loader-utils/option-utils';
23
+ import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
24
+ import {getLoaderImplementation} from './load-loader';
25
+ import {parse} from './parse';
26
+ import {selectLoader} from './select-loader';
27
+
28
+ /**
29
+ * Parses a random access readable file asynchronously using the supplied loader.
30
+ */
31
+ export async function parseFile<
32
+ LoaderT extends Loader,
33
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
34
+ LoaderOptionsType<LoaderT>,
35
+ LoaderShapeType<LoaderT>
36
+ >
37
+ >(
38
+ file: ReadableFile,
39
+ loader: LoaderT,
40
+ options?: OptionsT,
41
+ context?: LoaderContext
42
+ ): Promise<LoaderReturnType<LoaderT>>;
43
+
44
+ /**
45
+ * Parses a random access readable file asynchronously by matching one of the supplied loaders.
46
+ */
47
+ export async function parseFile<
48
+ LoaderArrayT extends Loader[],
49
+ OptionsT extends LoaderOptions = LoaderArrayOptionsType<LoaderArrayT>
50
+ >(
51
+ file: ReadableFile,
52
+ loaders: LoaderArrayT,
53
+ options?: OptionsT,
54
+ context?: LoaderContext
55
+ ): Promise<LoaderArrayReturnType<LoaderArrayT>>;
56
+
57
+ /**
58
+ * Parses a random access readable file asynchronously by matching a pre-registered loader.
59
+ * @deprecated Loader registration is deprecated, use parseFile(file, loaders, options) instead.
60
+ */
61
+ export async function parseFile(file: ReadableFile, options?: LoaderOptions): Promise<unknown>;
62
+
63
+ /**
64
+ * Parses a random access readable file using a specified loader.
65
+ * @param file readable file
66
+ * @param loaders loader or loaders to use
67
+ * @param options loader options
68
+ * @param context loader context
69
+ * @returns parsed data
70
+ */
71
+ export async function parseFile(
72
+ file: ReadableFile,
73
+ loaders?: Loader | Loader[] | LoaderOptions,
74
+ options?: LoaderOptions,
75
+ context?: LoaderContext
76
+ ): Promise<unknown> {
77
+ if (loaders && !Array.isArray(loaders) && !isLoaderObject(loaders)) {
78
+ context = undefined;
79
+ options = loaders as LoaderOptions;
80
+ loaders = undefined;
81
+ }
82
+
83
+ options = options || {};
84
+ const url = file.url || '';
85
+ const typedLoaders = loaders as Loader | Loader[] | undefined;
86
+ const candidateLoaders = getLoadersFromContext(typedLoaders, context);
87
+ const loader = await selectLoader(url, candidateLoaders, options);
88
+ if (!loader) {
89
+ return null;
90
+ }
91
+
92
+ if (isSourceLoader(loader)) {
93
+ throw new Error(
94
+ `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parseFile().`
95
+ );
96
+ }
97
+
98
+ // @ts-expect-error candidateLoaders may be a single forced loader
99
+ const strictOptions = normalizeOptions(options, loader, candidateLoaders, url);
100
+ context = getLoaderContext(
101
+ // @ts-expect-error candidateLoaders may be a single forced loader
102
+ {url, _parse: parse, loaders: candidateLoaders},
103
+ strictOptions,
104
+ context || null
105
+ );
106
+
107
+ return await parseFileWithLoader(loader, file, strictOptions, context);
108
+ }
109
+
110
+ /**
111
+ * Parses a readable file after a loader has been selected.
112
+ * @param loader selected loader
113
+ * @param file readable file
114
+ * @param options strict loader options
115
+ * @param context loader context
116
+ * @returns parsed data
117
+ */
118
+ async function parseFileWithLoader(
119
+ loader: Loader,
120
+ file: ReadableFile,
121
+ options: StrictLoaderOptions,
122
+ context: LoaderContext
123
+ ): Promise<unknown> {
124
+ validateWorkerVersion(loader);
125
+ options = mergeOptions(loader.options, options);
126
+
127
+ const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
128
+ if (loaderWithParser.parseFile) {
129
+ return await loaderWithParser.parseFile(file, options, context);
130
+ }
131
+
132
+ const parseFunction = (loaderWithParser as {parse?: LoaderWithParser['parse']}).parse;
133
+ if (parseFunction) {
134
+ const length = file.bigsize > 0n ? Number(file.bigsize) : file.size;
135
+ if (!length) {
136
+ throw new Error(`${loader.id} loader - cannot fall back to parse without a file size`);
137
+ }
138
+ const data = await file.read(0n, length);
139
+ return await parse(data, loaderWithParser, options, context);
140
+ }
141
+
142
+ throw new Error(`${loader.id} loader - no file parser found`);
143
+ }
@@ -10,8 +10,10 @@ import type {
10
10
  StrictLoaderOptions,
11
11
  LoaderOptions,
12
12
  LoaderContext,
13
+ LoaderOptionsWithShape,
13
14
  BatchableDataType,
14
15
  LoaderOptionsType,
16
+ LoaderShapeType,
15
17
  LoaderBatchType,
16
18
  LoaderArrayOptionsType,
17
19
  LoaderArrayBatchType,
@@ -23,6 +25,7 @@ import {normalizeOptions} from '../loader-utils/option-utils';
23
25
  import {getLoaderContext} from '../loader-utils/loader-context';
24
26
  import {getAsyncIterableFromData} from '../loader-utils/get-data';
25
27
  import {getResourceUrl} from '../utils/resource-utils';
28
+ import {getLoaderImplementation} from './load-loader';
26
29
  import {selectLoader} from './select-loader';
27
30
 
28
31
  // Ensure `parse` is available in context if loader falls back to `parse`
@@ -33,7 +36,10 @@ import {parse} from './parse';
33
36
  */
34
37
  export async function parseInBatches<
35
38
  LoaderT extends Loader,
36
- OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
39
+ OptionsT extends LoaderOptions = LoaderOptionsWithShape<
40
+ LoaderOptionsType<LoaderT>,
41
+ LoaderShapeType<LoaderT>
42
+ >
37
43
  >(
38
44
  data: BatchableDataType,
39
45
  loader: LoaderT,
@@ -108,13 +114,15 @@ export async function parseInBatches(
108
114
  context || null
109
115
  );
110
116
 
111
- return await parseWithLoaderInBatches(loader as LoaderWithParser, data, strictOptions, context);
117
+ const loaderWithParser = await getLoaderImplementation(loader, strictOptions, context.url);
118
+ return await parseWithLoaderInBatches(loader, loaderWithParser, data, strictOptions, context);
112
119
  }
113
120
 
114
121
  /**
115
122
  * Loader has been selected and context has been prepared, see if we need to emit a metadata batch
116
123
  */
117
124
  async function parseWithLoaderInBatches(
125
+ originalLoader: Loader,
118
126
  loader: LoaderWithParser,
119
127
  data: BatchableDataType,
120
128
  options: StrictLoaderOptions,
@@ -131,7 +139,7 @@ async function parseWithLoaderInBatches(
131
139
  shape: 'metadata',
132
140
  batchType: 'metadata',
133
141
  metadata: {
134
- _loader: loader,
142
+ _loader: originalLoader,
135
143
  _context: context
136
144
  },
137
145
  // Populate with some default fields to avoid crashing
@@ -182,7 +190,7 @@ async function* parseChunkInBatches(
182
190
  transformedIterator:
183
191
  | Iterable<ArrayBufferLike | ArrayBufferView>
184
192
  | AsyncIterable<ArrayBufferLike | ArrayBufferView>,
185
- loader: Loader,
193
+ loader: LoaderWithParser,
186
194
  options: StrictLoaderOptions,
187
195
  context: LoaderContext
188
196
  ): AsyncIterable<Batch> {
@@ -207,7 +215,7 @@ async function* parseChunkInBatches(
207
215
  * @todo run through batch builder to apply options etc...
208
216
  */
209
217
  function convertDataToBatch(parsedData: unknown, loader: Loader): Batch {
210
- // prettier-ignore
218
+ // biome-ignore format: preserve intentional fixture layout
211
219
  const batch: Batch = isTable(parsedData)
212
220
  ? makeBatchFromTable(parsedData)
213
221
  : {