@loaders.gl/core 5.0.0-alpha.7 → 5.0.0-alpha.9

package/src/lib/api/load-in-batches.ts

@@ -6,17 +6,20 @@ import type {
   Loader,
   LoaderOptions,
   LoaderContext,
-  FetchLike,
   BatchableDataType,
   LoaderBatchType,
   LoaderOptionsType,
   LoaderOptionsWithShape,
   LoaderShapeType
 } from '@loaders.gl/loader-utils';
+import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {getFetchFunction} from '../loader-utils/get-fetch-function';
+import {applySearchParamsToUrl} from '../loader-utils/option-utils';
 
 import {parseInBatches} from './parse-in-batches';
+import {selectLoader} from './select-loader';
+import {getLoaderImplementation} from './load-loader';
 
 type FileType = string | File | Blob | Response | (string | File | Blob | Response)[] | FileList;
 
@@ -73,18 +76,13 @@ export function loadInBatches(
     loadersArray = loaders as Loader | Loader[] | undefined;
   }
 
-  // Select fetch function
-  const fetch = getFetchFunction(options || {});
-
   // Single url/file
   if (!Array.isArray(files)) {
-    return loadOneFileInBatches(files, loadersArray!, options || {}, fetch);
+    return loadOneFileInBatches(files, loadersArray!, options || {});
   }
 
   // Multiple URLs / files
-  const promises = files.map(file =>
-    loadOneFileInBatches(file, loadersArray!, options || {}, fetch)
-  );
+  const promises = files.map(file => loadOneFileInBatches(file, loadersArray!, options || {}));
 
   // No point in waiting here for all responses before starting to stream individual streams?
   return promises;
@@ -93,12 +91,21 @@ export function loadInBatches(
 async function loadOneFileInBatches(
   file: FileType,
   loaders: Loader | Loader[],
-  options: LoaderOptions,
-  fetch: FetchLike
+  options: LoaderOptions
 ): Promise<AsyncIterable<unknown>> {
   if (typeof file === 'string') {
     const url = file;
-    const response = await fetch(url);
+    const loader = await selectLoader(url, loaders, {
+      ...options,
+      core: {...options.core, nothrow: true}
+    });
+    const authenticationLoader =
+      loader && !loader.getAuthentications && options.core?.credentials?.length
+        ? await getLoaderImplementation(loader, options, url)
+        : loader;
+    options = await resolveLoaderAuthenticationOptions(authenticationLoader, url, options);
+    const fetch = getFetchFunction(options);
+    const response = await fetch(applySearchParamsToUrl(url, options));
     // pick right overload
     return Array.isArray(loaders)
       ? await parseInBatches(response, loaders, options)

package/src/lib/api/load.ts

@@ -14,12 +14,24 @@ import type {
   LoaderArrayOptionsType,
   LoaderArrayReturnType,
   SourceLoader,
+  SourcePropsType,
+  SourceArrayOptionsType,
+  DataSourceOptions,
   LoaderWithParser
 } from '@loaders.gl/loader-utils';
-import {isBlob, isSourceLoader} from '@loaders.gl/loader-utils';
+import {
+  isBlob,
+  isSourceLoader,
+  resolveAuthenticationOptions,
+  resolveLoaderAuthenticationOptions
+} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {getFetchFunction} from '../loader-utils/get-fetch-function';
-import {normalizeLoaderOptions} from '../loader-utils/option-utils';
+import {
+  applySearchParamsToUrl,
+  normalizeLoaderOptions,
+  getGlobalLoaderOptions
+} from '../loader-utils/option-utils';
 import {fetchFile} from '../fetch/fetch-file';
 
 import {parse} from './parse';
@@ -49,7 +61,7 @@ export async function load<
 >(
   url: string | DataType,
   loader: LoaderT,
-  options?: OptionsT,
+  options?: LoaderT extends SourceLoader ? SourcePropsType<LoaderT> & DataSourceOptions : OptionsT,
   context?: LoaderContext
 ): Promise<LoaderReturnType<LoaderT>>;
 
@@ -59,7 +71,7 @@ export async function load<
 >(
   url: string | DataType,
   loaders: LoaderArrayT,
-  options?: OptionsT,
+  options?: LoaderArrayT extends SourceLoader[] ? SourceArrayOptionsType<LoaderArrayT> : OptionsT,
   context?: LoaderContext
 ): Promise<LoaderArrayReturnType<LoaderArrayT>>;
 
@@ -95,8 +107,15 @@ export async function load(
     resolvedOptions = options as LoaderOptions;
   }
 
+  resolvedOptions ||= getGlobalLoaderOptions();
+
   if (!Array.isArray(resolvedLoaders) && isSourceLoader(resolvedLoaders)) {
     const sourceLoader = await resolveSourceLoader(resolvedLoaders, url, resolvedOptions);
+    resolvedOptions = await resolveLoaderAuthenticationOptions(
+      resolvedLoaders.getAuthentications ? resolvedLoaders : sourceLoader,
+      typeof url === 'string' ? url : '',
+      resolvedOptions || {}
+    );
     const runtimeCoreApi = {
       fetchFile,
       parse,
@@ -119,6 +138,11 @@ export async function load(
     isSourceLoader(resolvedLoaders[0])
   ) {
     const sourceLoader = await resolveSourceLoader(resolvedLoaders[0], url, resolvedOptions);
+    resolvedOptions = await resolveLoaderAuthenticationOptions(
+      resolvedLoaders[0].getAuthentications ? resolvedLoaders[0] : sourceLoader,
+      typeof url === 'string' ? url : '',
+      resolvedOptions || {}
+    );
     const runtimeCoreApi = {
       fetchFile,
       parse,
@@ -143,6 +167,11 @@ export async function load(
 
     if (selectedLoader && isSourceLoader(selectedLoader)) {
       const sourceLoader = await resolveSourceLoader(selectedLoader, url, resolvedOptions);
+      resolvedOptions = await resolveLoaderAuthenticationOptions(
+        selectedLoader.getAuthentications ? selectedLoader : sourceLoader,
+        typeof url === 'string' ? url : '',
+        resolvedOptions || {}
+      );
       return sourceLoader.createDataSource(
         url,
         (resolvedOptions || {}) as LoaderOptionsType<SourceLoader>,
@@ -164,6 +193,11 @@ export async function load(
         resolvedOptions,
         url
       );
+      resolvedOptions = await resolveLoaderAuthenticationOptions(
+        selectedLoader.getAuthentications ? selectedLoader : loaderImplementation,
+        url,
+        resolvedOptions || {}
+      );
       const parseUrl = (
         loaderImplementation as LoaderWithParser & {
           parseUrl?: (
@@ -173,18 +207,22 @@ export async function load(
           ) => Promise<unknown>;
         }
       ).parseUrl;
-      if (parseUrl) return await parseUrl(url, resolvedOptions, context);
+      if (parseUrl) {
+        const requestUrl = applySearchParamsToUrl(url, resolvedOptions);
+        return await parseUrl(requestUrl, resolvedOptions, context);
+      }
     }
   }
 
   // Select fetch function
+  if (resolvedOptions) resolvedOptions = resolveAuthenticationOptions(resolvedOptions);
   const fetch = getFetchFunction(resolvedOptions);
 
   // at this point, `url` could be already loaded binary data
   let data = url;
   // url is a string, fetch the url
   if (typeof url === 'string') {
-    data = await fetch(url);
+    data = await fetch(applySearchParamsToUrl(url, resolvedOptions));
     // URL is Blob or File, fetchFile handles it (alt: we could generate ObjectURL here)
   }
 

package/src/lib/api/parse-file.ts

@@ -16,7 +16,11 @@ import type {
   ReadableFile,
   StrictLoaderOptions
 } from '@loaders.gl/loader-utils';
-import {isSourceLoader, mergeOptions} from '@loaders.gl/loader-utils';
+import {
+  isSourceLoader,
+  mergeOptions,
+  resolveLoaderAuthenticationOptions
+} from '@loaders.gl/loader-utils';
 import {validateWorkerVersion} from '@loaders.gl/worker-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
@@ -96,7 +100,16 @@ export async function parseFile(
   }
 
   // @ts-expect-error candidateLoaders may be a single forced loader
-  const strictOptions = normalizeOptions(options, loader, candidateLoaders, url);
+  const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url);
+  const authenticationLoader =
+    !loader.getAuthentications && normalizedOptions.core?.credentials?.length
+      ? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
+      : loader;
+  const strictOptions = await resolveLoaderAuthenticationOptions(
+    authenticationLoader,
+    url,
+    normalizedOptions
+  );
   context = getLoaderContext(
     // @ts-expect-error candidateLoaders may be a single forced loader
     {url, _parse: parse, loaders: candidateLoaders},

package/src/lib/api/parse-in-batches.ts

@@ -4,6 +4,7 @@
 
 import type {Batch} from '@loaders.gl/schema';
 import {isTable, makeBatchFromTable} from '@loaders.gl/schema-utils';
+import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
 import type {
   Loader,
   LoaderWithParser,
@@ -112,7 +113,16 @@ export async function parseInBatches(
   }
 
   // Normalize options
-  const strictOptions = normalizeOptions(options, loader, loaderArray, url);
+  const normalizedOptions = normalizeOptions(options, loader, loaderArray, url);
+  const authenticationLoader =
+    !loader.getAuthentications && normalizedOptions.core?.credentials?.length
+      ? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
+      : loader;
+  const strictOptions = await resolveLoaderAuthenticationOptions(
+    authenticationLoader,
+    url || '',
+    normalizedOptions
+  );
   context = getLoaderContext(
     {url, _parseInBatches: parseInBatches, _parse: parse, loaders: loaderArray},
     strictOptions,

package/src/lib/api/parse.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import {resolveLoaderAuthenticationOptions} from '@loaders.gl/loader-utils';
 import type {
   Loader,
   LoaderContext,
@@ -121,7 +122,16 @@ export async function parse(
 
   // Normalize options
   // @ts-expect-error candidateLoaders
-  const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
+  const normalizedOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
+  const authenticationLoader =
+    !loader.getAuthentications && normalizedOptions.core?.credentials?.length
+      ? await getLoaderImplementation(loader, normalizedOptions, url || context?.url)
+      : loader;
+  const strictOptions = await resolveLoaderAuthenticationOptions(
+    authenticationLoader,
+    url || '',
+    normalizedOptions
+  );
 
   // Get a context (if already present, will be unchanged)
   context = getLoaderContext(

package/src/lib/loader-utils/get-fetch-function.ts

@@ -27,9 +27,7 @@ export function getFetchFunction(
   if (typeof fetchOption === 'function') {
     fetchFunction = fetchOption;
   } else if (isObject(fetchOption)) {
-    const baseFetch = context?.fetch || fetchFile;
-    fetchFunction = (url, requestOptions) =>
-      baseFetch(url, mergeFetchOptions(fetchOption as RequestInit, requestOptions));
+    fetchFunction = context?.fetch || fetchFile;
   } else if (context?.fetch) {
     fetchFunction = context.fetch;
   } else {
@@ -38,17 +36,8 @@ export function getFetchFunction(
 
   return createAuthenticatedFetch({
     fetch: fetchFunction,
-    credentials: loaderOptions.core?.credentials || []
+    credentials: loaderOptions.core?.credentials || [],
+    authentications: loaderOptions.core?.authentications,
+    fetchOptions: isObject(fetchOption) ? (fetchOption as RequestInit) : undefined
   });
 }
-
-/** Combines static and per-request fetch options without replacing either header collection. */
-function mergeFetchOptions(defaultOptions: RequestInit, requestOptions?: RequestInit): RequestInit {
-  const options = {...defaultOptions, ...requestOptions};
-  if (defaultOptions.headers || requestOptions?.headers) {
-    const headers = new Headers(defaultOptions.headers);
-    new Headers(requestOptions?.headers).forEach((value, key) => headers.set(key, value));
-    options.headers = headers;
-  }
-  return options;
-}

package/src/lib/loader-utils/option-defaults.ts

@@ -7,6 +7,7 @@ import {isBrowser} from '@loaders.gl/loader-utils';
 import {ConsoleLog} from './loggers';
 
 export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
+  searchParams: undefined,
   core: {
     baseUrl: undefined,
     // baseUrl

package/src/lib/loader-utils/option-utils.ts

@@ -42,6 +42,40 @@ const CORE_LOADER_OPTION_KEYS = [
   'transforms'
 ] as const;
 
+/**
+ * Appends loader search parameters to a URL while preserving existing parameters and fragments.
+ * @param url URL to update
+ * @param options Loader options containing optional search parameters
+ * @returns URL with the configured parameters appended
+ */
+export function applySearchParamsToUrl(url: string, options?: LoaderOptions): string {
+  const searchParams = options?.searchParams;
+  if (!searchParams || Object.keys(searchParams).length === 0 || /^(?:data|blob):/i.test(url)) {
+    return url;
+  }
+
+  const hashIndex = url.indexOf('#');
+  const hash = hashIndex >= 0 ? url.slice(hashIndex) : '';
+  const urlWithoutHash = hashIndex >= 0 ? url.slice(0, hashIndex) : url;
+  const queryIndex = urlWithoutHash.indexOf('?');
+  const path = queryIndex >= 0 ? urlWithoutHash.slice(0, queryIndex) : urlWithoutHash;
+  const query = queryIndex >= 0 ? urlWithoutHash.slice(queryIndex + 1) : '';
+  const existingSearchParams = new URLSearchParams(query);
+  const appendedSearchParams = new URLSearchParams();
+  for (const [key, value] of Object.entries(searchParams)) {
+    if (!existingSearchParams.has(key)) {
+      appendedSearchParams.set(key, String(value));
+    }
+  }
+
+  const serializedAppendedSearchParams = appendedSearchParams.toString();
+  return `${path}${queryIndex >= 0 ? `?${query}` : ''}${
+    serializedAppendedSearchParams
+      ? `${queryIndex >= 0 ? '&' : '?'}${serializedAppendedSearchParams}`
+      : ''
+  }${hash}`;
+}
+
 /**
  * Global state for loaders.gl. Stored on `globalThis.loaders._state`
  */

package/src/lib/utils/response-utils.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {isResponse} from '@loaders.gl/loader-utils';
+import {encodeArrayBufferToBase64, isResponse} from '@loaders.gl/loader-utils';
 import {FetchError} from '../fetch/fetch-error';
 import {getResourceContentLength, getResourceUrl, getResourceMIMEType} from './resource-utils';
 import {shortenUrlForDisplay} from './url-utils';
@@ -120,18 +120,8 @@ async function getInitialDataUrl(
   }
   if (resource instanceof ArrayBuffer) {
     const slice = resource.slice(0, INITIAL_DATA_LENGTH);
-    const base64 = arrayBufferToBase64(slice);
+    const base64 = encodeArrayBufferToBase64(slice);
     return `data:base64,${base64}`;
   }
   return null;
 }
-
-// https://stackoverflow.com/questions/9267899/arraybuffer-to-base64-encoded-string
-function arrayBufferToBase64(buffer: ArrayBuffer): string {
-  let binary = '';
-  const bytes = new Uint8Array(buffer);
-  for (let i = 0; i < bytes.byteLength; i++) {
-    binary += String.fromCharCode(bytes[i]);
-  }
-  return btoa(binary);
-}