@loaders.gl/core 4.5.3 → 5.0.0-alpha.10

package/src/lib/api/load-in-batches.ts

@@ -3,18 +3,23 @@
 // Copyright (c) vis.gl contributors
 
 import type {
-  LoaderWithParser,
+  Loader,
   LoaderOptions,
   LoaderContext,
-  FetchLike,
   BatchableDataType,
   LoaderBatchType,
-  LoaderOptionsType
+  LoaderOptionsType,
+  LoaderOptionsWithShape,
+  LoaderShapeType
 } from '@loaders.gl/loader-utils';
+import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {getFetchFunction} from '../loader-utils/get-fetch-function';
+import {applySearchParamsToUrl} from '../loader-utils/option-utils';
 
 import {parseInBatches} from './parse-in-batches';
+import {selectLoader} from './select-loader';
+import {getLoaderImplementation} from './load-loader';
 
 type FileType = string | File | Blob | Response | (string | File | Blob | Response)[] | FileList;
 
@@ -22,8 +27,11 @@ type FileType = string | File | Blob | Response | (string | File | Blob | Respon
  * Parses `data` synchronously using a specified loader
  */
 export async function loadInBatches<
-  LoaderT extends LoaderWithParser,
-  OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
+  LoaderT extends Loader,
+  OptionsT extends LoaderOptions = LoaderOptionsWithShape<
+    LoaderOptionsType<LoaderT>,
+    LoaderShapeType<LoaderT>
+  >
 >(
   files: FileType,
   loader: LoaderT,
@@ -40,46 +48,41 @@ export async function loadInBatches<
  */
 export function loadInBatches(
   files: FileType,
-  loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
+  loaders?: Loader | Loader[] | LoaderOptions,
   options?: LoaderOptions,
   context?: LoaderContext
 ): Promise<AsyncIterable<unknown>>;
 
 export function loadInBatches(
   files: FileType[] | FileList,
-  loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
+  loaders?: Loader | Loader[] | LoaderOptions,
   options?: LoaderOptions,
   context?: LoaderContext
 ): Promise<AsyncIterable<unknown>>[];
 
 export function loadInBatches(
   files: FileType | FileType[] | FileList,
-  loaders?: LoaderWithParser | LoaderWithParser[] | LoaderOptions,
+  loaders?: Loader | Loader[] | LoaderOptions,
   options?: LoaderOptions,
   context?: LoaderContext
 ): Promise<AsyncIterable<unknown>> | Promise<AsyncIterable<unknown>>[] {
-  let loadersArray: LoaderWithParser | LoaderWithParser[] | undefined;
+  let loadersArray: Loader | Loader[] | undefined;
   // Signature: load(url, options)
   if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
     context = undefined; // context not supported in short signature
     options = loaders as LoaderOptions;
     loadersArray = undefined;
   } else {
-    loadersArray = loaders as LoaderWithParser | LoaderWithParser[] | undefined;
+    loadersArray = loaders as Loader | Loader[] | undefined;
   }
 
-  // Select fetch function
-  const fetch = getFetchFunction(options || {});
-
   // Single url/file
   if (!Array.isArray(files)) {
-    return loadOneFileInBatches(files, loadersArray!, options || {}, fetch);
+    return loadOneFileInBatches(files, loadersArray!, options || {});
   }
 
   // Multiple URLs / files
-  const promises = files.map((file) =>
-    loadOneFileInBatches(file, loadersArray!, options || {}, fetch)
-  );
+  const promises = files.map(file => loadOneFileInBatches(file, loadersArray!, options || {}));
 
   // No point in waiting here for all responses before starting to stream individual streams?
   return promises;
@@ -87,13 +90,22 @@ export function loadInBatches(
 
 async function loadOneFileInBatches(
   file: FileType,
-  loaders: LoaderWithParser | LoaderWithParser[],
-  options: LoaderOptions,
-  fetch: FetchLike
+  loaders: Loader | Loader[],
+  options: LoaderOptions
 ): Promise<AsyncIterable<unknown>> {
   if (typeof file === 'string') {
     const url = file;
-    const response = await fetch(url);
+    const loader = await selectLoader(url, loaders, {
+      ...options,
+      core: {...options.core, nothrow: true}
+    });
+    const authenticationLoader =
+      loader && !loader.getAuthentications && options.core?.credentials?.length
+        ? await getLoaderImplementation(loader, options, url)
+        : loader;
+    options = await resolveLoaderAuthenticationOptions(authenticationLoader, url, options);
+    const fetch = getFetchFunction(options);
+    const response = await fetch(applySearchParamsToUrl(url, options));
     // pick right overload
     return Array.isArray(loaders)
       ? await parseInBatches(response, loaders, options)

package/src/lib/api/load-loader.ts

@@ -0,0 +1,306 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {isSourceLoader, mergeOptions} from '@loaders.gl/loader-utils';
+import type {
+  Loader,
+  LoaderOptions,
+  LoaderWithParser,
+  StrictLoaderOptions
+} from '@loaders.gl/loader-utils';
+
+const loaderImplementationPromises = new Map<Loader, Map<string, Promise<LoaderWithParser>>>();
+const loaderImplementations = new Map<Loader, Map<string, LoaderWithParser>>();
+
+/** Gets a parser-bearing implementation for a loader, loading it if needed. */
+export async function getLoaderImplementation(
+  loader: Loader,
+  options?: LoaderOptions,
+  url?: string
+): Promise<LoaderWithParser> {
+  if (isSourceLoader(loader)) {
+    const source = loader.preload ? await loader.preload(url || '', options) : loader;
+    if (!isSourceLoader(source)) {
+      throw new Error(
+        `${loader.id} source loader preload() did not return a runtime source loader`
+      );
+    }
+    return source as unknown as LoaderWithParser;
+  }
+  if (isLoaderWithParser(loader)) {
+    return loader;
+  }
+
+  const implementationCacheKey = getLoaderImplementationCacheKey(loader, options);
+  const loaderImplementation = loaderImplementations.get(loader)?.get(implementationCacheKey);
+  if (loaderImplementation) {
+    return loaderImplementation;
+  }
+
+  let loaderImplementationPromisesForLoader = loaderImplementationPromises.get(loader);
+  if (!loaderImplementationPromisesForLoader) {
+    loaderImplementationPromisesForLoader = new Map();
+    loaderImplementationPromises.set(loader, loaderImplementationPromisesForLoader);
+  }
+
+  let loaderImplementationPromise =
+    loaderImplementationPromisesForLoader.get(implementationCacheKey);
+  if (!loaderImplementationPromise) {
+    loaderImplementationPromise = resolveLoaderImplementation(loader, options, url)
+      .then(implementation => {
+        let loaderImplementationsForLoader = loaderImplementations.get(loader);
+        if (!loaderImplementationsForLoader) {
+          loaderImplementationsForLoader = new Map();
+          loaderImplementations.set(loader, loaderImplementationsForLoader);
+        }
+        loaderImplementationsForLoader.set(implementationCacheKey, implementation);
+        return implementation;
+      })
+      .catch(error => {
+        loaderImplementationPromisesForLoader.delete(implementationCacheKey);
+        if (loaderImplementationPromisesForLoader.size === 0) {
+          loaderImplementationPromises.delete(loader);
+        }
+        throw error;
+      });
+    loaderImplementationPromisesForLoader.set(implementationCacheKey, loaderImplementationPromise);
+  }
+
+  return await loaderImplementationPromise;
+}
+
+/** Gets a cached parser-bearing implementation for a loader without loading it. */
+export function getLoaderImplementationSync(loader: Loader): LoaderWithParser | null {
+  if (loader.subloaders && !preparedLoaders.has(loader)) {
+    const prepared = preparedDefaults.get(loader);
+    if (prepared) return prepared;
+    if (!isLoaderWithParser(loader)) return null;
+  }
+  if (isLoaderWithParser(loader)) {
+    return loader;
+  }
+
+  const implementationCacheKey = getLoaderImplementationCacheKey(loader);
+  return loaderImplementations.get(loader)?.get(implementationCacheKey) || null;
+}
+
+/** Returns true when a loader object already includes parser methods. */
+export function isLoaderWithParser(loader: Loader): loader is LoaderWithParser {
+  const candidate = loader as LoaderWithParser;
+  return Boolean(
+    candidate.parse ||
+      candidate.parseSync ||
+      candidate.parseInBatches ||
+      candidate.parseText ||
+      candidate.parseTextSync ||
+      candidate.parseFile ||
+      candidate.parseFileInBatches
+  );
+}
+
+/** Resolves a parser-bearing implementation through loader preload or the test import fallback. */
+async function resolveLoaderImplementation(
+  loader: Loader,
+  options?: LoaderOptions,
+  url?: string
+): Promise<LoaderWithParser> {
+  if (loader.preload) {
+    const preloadedLoader = await loader.preload(url || '', options);
+    if (isLoaderWithParser(preloadedLoader as Loader)) {
+      return preloadedLoader as LoaderWithParser;
+    }
+
+    throw new Error(`${loader.id} loader preload() did not return a parser-bearing loader`);
+  }
+
+  const implementationSpecifier = getLoaderImplementationSpecifier(loader, options);
+  return await importLoaderImplementation(implementationSpecifier, loader.id);
+}
+
+/** Gets the dynamic implementation specifier for the test worker fallback path. */
+function getLoaderImplementationSpecifier(loader: Loader, options?: LoaderOptions): string {
+  const workerType = options?._workerType || options?.core?._workerType;
+  if (workerType === 'test') {
+    const sourcePath = `modules/${loader.module}/src/${loader.id}-loader.ts`;
+    if (typeof window !== 'undefined') {
+      return `/${sourcePath}`;
+    }
+
+    if (typeof process !== 'undefined' && process.cwd) {
+      return new URL(sourcePath, `file://${process.cwd()}/`).toString();
+    }
+
+    return sourcePath;
+  }
+
+  throw new Error(
+    `${loader.id} loader does not provide a parser implementation. Import a parser-bearing loader directly, or use preload() before parse/load.`
+  );
+}
+
+/** Gets the cache key for option-selected parser implementations. */
+function getLoaderImplementationCacheKey(loader: Loader, options?: LoaderOptions): string {
+  const loaderOptions = options?.[loader.id] as {backend?: unknown} | undefined;
+  const defaultLoaderOptions = loader.options?.[loader.id] as {backend?: unknown} | undefined;
+  return String(loaderOptions?.backend || defaultLoaderOptions?.backend || '');
+}
+
+/** Imports a module and finds the parser-bearing loader implementation with the requested id. */
+async function importLoaderImplementation(
+  implementationSpecifier: string,
+  loaderId: string
+): Promise<LoaderWithParser> {
+  const moduleExports = await import(
+    /* webpackIgnore: true */ /* @vite-ignore */ implementationSpecifier
+  );
+
+  for (const exportValue of Object.values(moduleExports)) {
+    if (
+      exportValue &&
+      typeof exportValue === 'object' &&
+      (exportValue as Loader).id === loaderId &&
+      isLoaderWithParser(exportValue as Loader)
+    ) {
+      return exportValue as LoaderWithParser;
+    }
+  }
+
+  throw new Error(
+    `Could not find parser implementation for ${loaderId} in ${implementationSpecifier}`
+  );
+}
+
+/** Prepared loaders carry caller-local parser bindings rather than shared metadata. */
+const preparedLoaders = new WeakSet<Loader>();
+/** Successfully prepared default graphs used by synchronous parsing. */
+const preparedDefaults = new WeakMap<Loader, LoaderWithParser>();
+/** In-flight preparations for default dependency graphs. */
+const subloaderPreparations = new WeakMap<Loader, Promise<LoaderWithParser>>();
+
+/** Resolves declared dependencies and binds parser entry points to their implementations. */
+export async function preloadLoaderDependencies(
+  loader: Loader,
+  options?: LoaderOptions,
+  url?: string
+): Promise<LoaderWithParser> {
+  if (preparedLoaders.has(loader)) return await getLoaderImplementation(loader, options, url);
+  validateSubloaderGraph(loader, options);
+  return await prepareLoaderDependencies(loader, options, url);
+}
+
+/** Prepares a node after the complete dependency graph has been validated. */
+async function prepareLoaderDependencies(
+  loader: Loader,
+  options?: LoaderOptions,
+  url?: string
+): Promise<LoaderWithParser> {
+  // Options can select nested backends or overrides; do not share their bindings.
+  if (!options) {
+    const cached = subloaderPreparations.get(loader);
+    if (cached) return await cached;
+  }
+  const preparation = bindSubloaders(loader, options, url);
+  if (!options) {
+    subloaderPreparations.set(loader, preparation);
+    void preparation.catch(() => subloaderPreparations.delete(loader));
+  }
+  const prepared = await preparation;
+  if (!options) preparedDefaults.set(loader, prepared);
+  return prepared;
+}
+
+/** Returns declared dependencies with the shared flat override map applied. */
+function getSubloaders(loader: Loader, options?: LoaderOptions): Record<string, Loader> {
+  const overrides = options?.core?.subloaders || {};
+  return Object.fromEntries(
+    Object.entries(loader.subloaders || {}).map(([name, dependency]) => [
+      name,
+      Object.hasOwn(overrides, name) ? overrides[name] : dependency
+    ])
+  );
+}
+
+/** Rejects cycles in the dependency graph after applying the shared overrides. */
+function validateSubloaderGraph(loader: Loader, options?: LoaderOptions): void {
+  visitLoader(loader, []);
+
+  /** Detects cycles along each resolved dependency path. */
+  function visitLoader(currentLoader: Loader, ancestors: Loader[]): void {
+    if (ancestors.includes(currentLoader)) {
+      throw new Error(
+        `Subloader cycle: ${[...ancestors, currentLoader].map(value => value.id).join(' -> ')}`
+      );
+    }
+    for (const dependency of Object.values(getSubloaders(currentLoader, options))) {
+      visitLoader(dependency, [...ancestors, currentLoader]);
+    }
+  }
+}
+
+/** Prepares dependencies concurrently and injects them into parser calls' scoped options. */
+async function bindSubloaders(
+  loader: Loader,
+  options?: LoaderOptions,
+  url?: string
+): Promise<LoaderWithParser> {
+  const [implementation, entries] = await Promise.all([
+    getLoaderImplementation(loader, options, url),
+    Promise.all(
+      Object.entries(getSubloaders(loader, options)).map(
+        async ([name, dependency]) =>
+          [name, await prepareLoaderDependencies(dependency, options, url)] as const
+      )
+    )
+  ]);
+  const subloaders = Object.fromEntries(entries);
+  // Expose preload settings as defaults before core normalizes a later parse call.
+  const defaultOptions = mergeOptions(
+    implementation.options,
+    (options || {}) as StrictLoaderOptions
+  );
+  defaultOptions.core = {...defaultOptions.core, worker: false};
+  const prepared = {...implementation, options: defaultOptions, subloaders};
+  delete prepared.preload;
+  // Function-valued dependency bindings cannot be transferred to a worker.
+  prepared.worker = false;
+  for (const method of [
+    'parseBlob',
+    'createDataSource',
+    'parse',
+    'parseSync',
+    'parseText',
+    'parseTextSync',
+    'parseInBatches',
+    'parseFile',
+    'parseFileInBatches',
+    'parseUrl'
+  ] as const) {
+    const parser = (implementation as Record<string, unknown>)[method];
+    if (typeof parser === 'function') {
+      (prepared as Record<string, unknown>)[method] = (
+        data: unknown,
+        parseOptions?: LoaderOptions,
+        context?: unknown
+      ) => {
+        const forwardedOptions = mergeOptions(options, parseOptions || {});
+        return parser.call(
+          implementation,
+          data,
+          {
+            ...forwardedOptions,
+            core: {
+              ...forwardedOptions.core,
+              subloaders: options?.core?.subloaders || parseOptions?.core?.subloaders,
+              worker: false
+            },
+            [loader.id]: {...(forwardedOptions[loader.id] as object), subloaders}
+          },
+          context
+        );
+      };
+    }
+  }
+  preparedLoaders.add(prepared);
+  return prepared;
+}

package/src/lib/api/load.ts

@@ -7,17 +7,40 @@ import type {
   Loader,
   LoaderContext,
   LoaderOptions,
+  LoaderOptionsWithShape,
   LoaderOptionsType,
+  LoaderShapeType,
   LoaderReturnType,
   LoaderArrayOptionsType,
-  LoaderArrayReturnType
+  LoaderArrayReturnType,
+  SourceLoader,
+  SourcePropsType,
+  SourceArrayOptionsType,
+  DataSourceOptions,
+  LoaderWithParser
+} from '@loaders.gl/loader-utils';
+import {
+  isBlob,
+  isSourceLoader,
+  resolveAuthenticationOptions,
+  resolveLoaderAuthenticationOptions
 } from '@loaders.gl/loader-utils';
-import {isBlob} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {getFetchFunction} from '../loader-utils/get-fetch-function';
-import {normalizeLoaderOptions} from '../loader-utils/option-utils';
+import {
+  applySearchParamsToUrl,
+  normalizeLoaderOptions,
+  getGlobalLoaderOptions
+} from '../loader-utils/option-utils';
+import {fetchFile} from '../fetch/fetch-file';
 
 import {parse} from './parse';
+import {parseFile} from './parse-file';
+import {parseSync} from './parse-sync';
+import {parseInBatches} from './parse-in-batches';
+import {loadInBatches} from './load-in-batches';
+import {selectLoader} from './select-loader';
+import {getLoaderImplementation} from './load-loader';
 
 /**
  * Parses `data` using a specified loader
@@ -31,11 +54,14 @@ import {parse} from './parse';
 
 export async function load<
   LoaderT extends Loader,
-  OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
+  OptionsT extends LoaderOptions = LoaderOptionsWithShape<
+    LoaderOptionsType<LoaderT>,
+    LoaderShapeType<LoaderT>
+  >
 >(
   url: string | DataType,
   loader: LoaderT,
-  options?: OptionsT,
+  options?: LoaderT extends SourceLoader ? SourcePropsType<LoaderT> & DataSourceOptions : OptionsT,
   context?: LoaderContext
 ): Promise<LoaderReturnType<LoaderT>>;
 
@@ -45,7 +71,7 @@ export async function load<
 >(
   url: string | DataType,
   loaders: LoaderArrayT,
-  options?: OptionsT,
+  options?: LoaderArrayT extends SourceLoader[] ? SourceArrayOptionsType<LoaderArrayT> : OptionsT,
   context?: LoaderContext
 ): Promise<LoaderArrayReturnType<LoaderArrayT>>;
 
@@ -81,14 +107,122 @@ export async function load(
     resolvedOptions = options as LoaderOptions;
   }
 
+  resolvedOptions ||= getGlobalLoaderOptions();
+
+  if (!Array.isArray(resolvedLoaders) && isSourceLoader(resolvedLoaders)) {
+    const sourceLoader = await resolveSourceLoader(resolvedLoaders, url, resolvedOptions);
+    resolvedOptions = await resolveLoaderAuthenticationOptions(
+      resolvedLoaders.getAuthentications ? resolvedLoaders : sourceLoader,
+      typeof url === 'string' ? url : '',
+      resolvedOptions || {}
+    );
+    const runtimeCoreApi = {
+      fetchFile,
+      parse,
+      parseFile,
+      parseSync,
+      parseInBatches,
+      load,
+      loadInBatches
+    };
+    return sourceLoader.createDataSource(
+      url as string | Blob,
+      (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
+      runtimeCoreApi
+    );
+  }
+
+  if (
+    Array.isArray(resolvedLoaders) &&
+    resolvedLoaders.length === 1 &&
+    isSourceLoader(resolvedLoaders[0])
+  ) {
+    const sourceLoader = await resolveSourceLoader(resolvedLoaders[0], url, resolvedOptions);
+    resolvedOptions = await resolveLoaderAuthenticationOptions(
+      resolvedLoaders[0].getAuthentications ? resolvedLoaders[0] : sourceLoader,
+      typeof url === 'string' ? url : '',
+      resolvedOptions || {}
+    );
+    const runtimeCoreApi = {
+      fetchFile,
+      parse,
+      parseFile,
+      parseSync,
+      parseInBatches,
+      load,
+      loadInBatches
+    };
+    return sourceLoader.createDataSource(
+      url as string | Blob,
+      (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
+      runtimeCoreApi
+    );
+  }
+
+  if (typeof url === 'string' || isBlob(url)) {
+    const selectedLoader = await selectLoader(url, resolvedLoaders as Loader | Loader[], {
+      ...resolvedOptions,
+      core: {...resolvedOptions?.core, nothrow: true}
+    });
+
+    if (selectedLoader && isSourceLoader(selectedLoader)) {
+      const sourceLoader = await resolveSourceLoader(selectedLoader, url, resolvedOptions);
+      resolvedOptions = await resolveLoaderAuthenticationOptions(
+        selectedLoader.getAuthentications ? selectedLoader : sourceLoader,
+        typeof url === 'string' ? url : '',
+        resolvedOptions || {}
+      );
+      return sourceLoader.createDataSource(
+        url,
+        (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
+        {
+          fetchFile,
+          parse,
+          parseFile,
+          parseSync,
+          parseInBatches,
+          load,
+          loadInBatches
+        }
+      );
+    }
+
+    if (selectedLoader && typeof url === 'string') {
+      const loaderImplementation = await getLoaderImplementation(
+        selectedLoader,
+        resolvedOptions,
+        url
+      );
+      resolvedOptions = await resolveLoaderAuthenticationOptions(
+        selectedLoader.getAuthentications ? selectedLoader : loaderImplementation,
+        url,
+        resolvedOptions || {}
+      );
+      const parseUrl = (
+        loaderImplementation as LoaderWithParser & {
+          parseUrl?: (
+            url: string,
+            options?: LoaderOptions,
+            context?: LoaderContext
+          ) => Promise<unknown>;
+        }
+      ).parseUrl;
+      if (parseUrl) {
+        const requestUrl = applySearchParamsToUrl(url, resolvedOptions);
+        return await parseUrl(requestUrl, resolvedOptions, context);
+      }
+    }
+  }
+
   // Select fetch function
+  if (resolvedOptions) resolvedOptions = resolveAuthenticationOptions(resolvedOptions);
   const fetch = getFetchFunction(resolvedOptions);
 
   // at this point, `url` could be already loaded binary data
   let data = url;
   // url is a string, fetch the url
   if (typeof url === 'string') {
-    data = await fetch(url);
+    data = await fetch(applySearchParamsToUrl(url, resolvedOptions));
     // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)
   }
 
@@ -114,6 +248,23 @@ export async function load(
   // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
   // return await parse(data, loaders as Loader[], options);
   return Array.isArray(resolvedLoaders)
-    ? await parse(data, resolvedLoaders, resolvedOptions) // loader array overload
-    : await parse(data, resolvedLoaders, resolvedOptions); // single loader overload
+    ? await parse(data, resolvedLoaders, resolvedOptions, context) // loader array overload
+    : await parse(data, resolvedLoaders, resolvedOptions, context); // single loader overload
+}
+
+/** Resolves a lightweight source loader into its runtime implementation when necessary. */
+async function resolveSourceLoader(
+  loader: SourceLoader,
+  url: string | DataType,
+  options?: LoaderOptions
+): Promise<SourceLoader> {
+  if (!loader.preload) {
+    return loader;
+  }
+
+  const sourceLoader = await loader.preload(typeof url === 'string' ? url : '', options);
+  if (!isSourceLoader(sourceLoader)) {
+    throw new Error(`${loader.id} source loader preload() did not return a runtime source loader`);
+  }
+  return sourceLoader;
 }