@loaders.gl/core 4.0.0-alpha.5 → 4.0.0-alpha.7

package/src/lib/api/load.ts

@@ -1,7 +1,10 @@
+// loaders.gl, MIT license
+
 import type {DataType, Loader, LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {LoaderOptionsType, LoaderReturnType} from '@loaders.gl/loader-utils';
 import {isBlob} from '../../javascript-utils/is-type';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
-import {getFetchFunction} from '../loader-utils/option-utils';
+import {getFetchFunction} from '../loader-utils/get-fetch-function';
 
 import {parse} from './parse';
 
@@ -14,22 +17,46 @@ import {parse} from './parse';
  * @param options
  * @param context
  */
+
+export async function load<LoaderT extends Loader>(
+  url: string | DataType,
+  loader: LoaderT,
+  options?: LoaderOptionsType<LoaderT>,
+  context?: LoaderContext
+): Promise<LoaderReturnType<LoaderT>>;
+
+export async function load<
+  LoaderT extends Loader, // eslint-disable-line  @typescript-eslint/no-unused-vars
+  LoaderOptionsT extends LoaderOptions = LoaderOptions
+>(
+  url: string | DataType,
+  loaders: Loader[] | LoaderOptions,
+  options?: LoaderOptionsT,
+  context?: LoaderContext
+): Promise<any>;
+
 // implementation signature
-export async function load(
+export async function load<LoaderOptionsT extends LoaderOptions>(
   url: string | DataType,
-  loaders?: Loader | Loader[] | LoaderOptions,
-  options?: LoaderOptions,
+  loaders?: Loader[] | LoaderOptions,
+  options?: LoaderOptionsT,
   context?: LoaderContext
 ): Promise<any> {
+  let resolvedLoaders: Loader | Loader[];
+  let resolvedOptions: LoaderOptionsT | undefined;
+
   // Signature: load(url, options)
   if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
+    resolvedLoaders = [];
+    resolvedOptions = loaders as LoaderOptionsT;
     context = undefined; // context not supported in short signature
-    options = loaders as LoaderOptions;
-    loaders = undefined;
+  } else {
+    resolvedLoaders = loaders as Loader | Loader[];
+    resolvedOptions = options;
   }
 
   // Select fetch function
-  const fetch = getFetchFunction(options);
+  const fetch = getFetchFunction(resolvedOptions);
 
   // at this point, `url` could be already loaded binary data
   let data = url;
@@ -41,9 +68,13 @@ export async function load(
 
   if (isBlob(url)) {
     // The fetch response object will contain blob.name
+    // @ts-expect-error TODO - This may not work for overridden fetch functions
     data = await fetch(url);
   }
 
   // Data is loaded (at least we have a `Response` object) so time to hand over to `parse`
-  return await parse(data, loaders, options);
+  // return await parse(data, loaders as Loader[], options);
+  return Array.isArray(resolvedLoaders)
+    ? await parse(data, resolvedLoaders, resolvedOptions) // loader array overload
+    : await parse(data, resolvedLoaders, resolvedOptions); // single loader overload
 }

package/src/lib/api/loader-options.ts

@@ -0,0 +1,2 @@
+export {setGlobalOptions as setLoaderOptions} from '../loader-utils/option-utils';
+export {getGlobalLoaderOptions as getLoaderOptions} from '../loader-utils/option-utils';

package/src/lib/api/parse-in-batches.ts

@@ -11,7 +11,7 @@ import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getLoaderContext} from '../loader-utils/loader-context';
 import {getAsyncIterableFromData} from '../loader-utils/get-data';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 import {selectLoader} from './select-loader';
 
 // Ensure `parse` is available in context if loader falls back to `parse`
@@ -32,6 +32,8 @@ export async function parseInBatches(
 ): Promise<AsyncIterable<any>> {
   assert(!context || typeof context === 'object'); // parseInBatches no longer accepts final url
 
+  const loaderArray = Array.isArray(loaders) ? loaders : undefined;
+
   // Signature: parseInBatches(data, options, url) - Uses registered loaders
   if (!Array.isArray(loaders) && !isLoaderObject(loaders)) {
     context = undefined; // context not supported in short signature
@@ -43,11 +45,11 @@ export async function parseInBatches(
   options = options || {};
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   // Chooses a loader and normalizes it
   // Note - only uses URL and contentType for streams and iterator inputs
-  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader[], options);
+  const loader = await selectLoader(data as ArrayBuffer, loaders as Loader | Loader[], options);
   // Note: if options.nothrow was set, it is possible that no loader was found, if so just return null
   if (!loader) {
     // @ts-ignore
@@ -55,14 +57,11 @@ export async function parseInBatches(
   }
 
   // Normalize options
-  // @ts-ignore
-  options = normalizeOptions(options, loader, loaders, url);
-  // @ts-ignore
+  options = normalizeOptions(options, loader, loaderArray, url);
   context = getLoaderContext(
-    // @ts-ignore
-    {url, parseInBatches, parse, loaders: loaders as Loader[]},
+    {url, parseInBatches, parse, loaders: loaderArray},
     options,
-    context
+    context || null
   );
 
   return await parseWithLoaderInBatches(loader as LoaderWithParser, data, options, context);

package/src/lib/api/parse-sync.ts

@@ -11,7 +11,7 @@ import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 
 /**
  * Parses `data` synchronously using a specified loader
@@ -52,12 +52,16 @@ export function parseSync(
   options = normalizeOptions(options, loader, candidateLoaders);
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   const parse = () => {
-    throw new Error('parseSync called parse');
+    throw new Error('parseSync called parse (which is async');
   };
-  context = getLoaderContext({url, parseSync, parse, loaders: loaders as Loader[]}, options);
+  context = getLoaderContext(
+    {url, parseSync, parse, loaders: loaders as Loader[]},
+    options,
+    context || null
+  );
 
   return parseWithLoaderSync(loader as LoaderWithParser, data, options, context);
 }

package/src/lib/api/parse.ts

@@ -1,13 +1,41 @@
+// loaders.gl, MIT license
+
 import type {DataType, Loader, LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {LoaderOptionsType, LoaderReturnType} from '@loaders.gl/loader-utils';
 import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
 import {parseWithWorker, canParseWithWorker} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
+import {isResponse} from '../../javascript-utils/is-type';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {getResourceUrl} from '../utils/resource-utils';
 import {selectLoader} from './select-loader';
 
+// type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T
+
+export async function parse<
+  LoaderT extends Loader,
+  OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
+>(
+  data: DataType | Promise<DataType>,
+  loader: LoaderT,
+  options?: OptionsT,
+  context?: LoaderContext
+): Promise<LoaderReturnType<LoaderT>>;
+
+export async function parse(
+  data: DataType | Promise<DataType>,
+  loaders: Loader[],
+  options?: LoaderOptions,
+  context?: LoaderContext
+): Promise<any>;
+
+export async function parse(
+  data: DataType | Promise<DataType>,
+  options?: LoaderOptions
+): Promise<any>;
+
 /**
  * Parses `data` using a specified loader
  * @param data
@@ -15,6 +43,7 @@ import {selectLoader} from './select-loader';
  * @param options
  * @param context
  */
+// implementation signature
 export async function parse(
   data: DataType | Promise<DataType>,
   loaders?: Loader | Loader[] | LoaderOptions,
@@ -32,10 +61,10 @@ export async function parse(
   }
 
   data = await data; // Resolve any promise
-  options = options || {};
+  options = options || ({} as LoaderOptions); // Could be invalid...
 
   // Extract a url for auto detection
-  const {url} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
 
   // Chooses a loader (and normalizes it)
   // Also use any loaders in the context, new loaders take priority
@@ -49,10 +78,10 @@ export async function parse(
   }
 
   // Normalize options
-  options = normalizeOptions(options, loader, candidateLoaders, url);
+  options = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
 
   // Get a context (if already present, will be unchanged)
-  context = getLoaderContext({url, parse, loaders: candidateLoaders}, options, context);
+  context = getLoaderContext({url, parse, loaders: candidateLoaders}, options, context || null);
 
   return await parseWithLoader(loader, data, options, context);
 }
@@ -62,6 +91,14 @@ export async function parse(
 async function parseWithLoader(loader, data, options, context) {
   validateWorkerVersion(loader);
 
+  if (isResponse(data)) {
+    // Serialize to support passing the response to web worker
+    const response = data as Response;
+    const {ok, redirected, status, statusText, type, url} = response;
+    const headers = Object.fromEntries(response.headers.entries());
+    context.response = {headers, ok, redirected, status, statusText, type, url};
+  }
+
   data = await getArrayBufferOrStringFromData(data, loader, options);
 
   // First check for synchronous text parser, wrap results in promises

package/src/lib/api/register-loaders.ts

@@ -11,7 +11,7 @@ const getGlobalLoaderRegistry = () => {
 };
 
 /** Register a list of global loaders */
-export function registerLoaders(loaders: Loader[]) {
+export function registerLoaders(loaders: Loader[] | Loader) {
   const loaderRegistry = getGlobalLoaderRegistry();
 
   loaders = Array.isArray(loaders) ? loaders : [loaders];

package/src/lib/api/select-loader.ts

@@ -1,9 +1,11 @@
 import type {LoaderContext, LoaderOptions, Loader} from '@loaders.gl/loader-utils';
 import {compareArrayBuffers, path} from '@loaders.gl/loader-utils';
 import {normalizeLoader} from '../loader-utils/normalize-loader';
-import {getResourceUrlAndType} from '../utils/resource-utils';
+import {log} from '../utils/log';
+import {getResourceUrl, getResourceMIMEType} from '../utils/resource-utils';
 import {getRegisteredLoaders} from './register-loaders';
 import {isBlob} from '../../javascript-utils/is-type';
+import {stripQueryString} from '../utils/url-utils';
 
 const EXT_PATTERN = /\.([^.]+)$/;
 
@@ -103,30 +105,46 @@ export function selectLoaderSync(
 }
 
 /** Implements loaders selection logic */
+// eslint-disable-next-line complexity
 function selectLoaderInternal(
   data: Response | Blob | ArrayBuffer | string,
   loaders: Loader[],
   options?: LoaderOptions,
   context?: LoaderContext
 ) {
-  const {url, type} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
+  const type = getResourceMIMEType(data);
 
-  const testUrl = url || context?.url;
+  const testUrl = stripQueryString(url) || context?.url;
 
   let loader: Loader | null = null;
+  let reason: string = '';
 
   // if options.mimeType is supplied, it takes precedence
   if (options?.mimeType) {
     loader = findLoaderByMIMEType(loaders, options?.mimeType);
+    reason = `match forced by supplied MIME type ${options?.mimeType}`;
   }
+
   // Look up loader by url
   loader = loader || findLoaderByUrl(loaders, testUrl);
+  reason = reason || (loader ? `matched url ${testUrl}` : '');
+
   // Look up loader by mime type
   loader = loader || findLoaderByMIMEType(loaders, type);
+  reason = reason || (loader ? `matched MIME type ${type}` : '');
+
   // Look for loader via initial bytes (Note: not always accessible (e.g. Response, stream, async iterator)
   loader = loader || findLoaderByInitialBytes(loaders, data);
+  reason = reason || (loader ? `matched initial data ${getFirstCharacters(data)}` : '');
+
   // Look up loader by fallback mime type
   loader = loader || findLoaderByMIMEType(loaders, options?.fallbackMimeType);
+  reason = reason || (loader ? `matched fallback MIME type ${type}` : '');
+
+  if (reason) {
+    log.log(1, `selectLoader selected ${loader?.name}: ${reason}.`);
+  }
 
   return loader;
 }
@@ -145,7 +163,8 @@ function validHTTPResponse(data: any): boolean {
 
 /** Generate a helpful message to help explain why loader selection failed. */
 function getNoValidLoaderMessage(data): string {
-  const {url, type} = getResourceUrlAndType(data);
+  const url = getResourceUrl(data);
+  const type = getResourceMIMEType(data);
 
   let message = 'No valid loader found (';
   message += url ? `${path.filename(url)}, ` : 'no url provided, ';

package/src/lib/init.ts

@@ -1,10 +1,17 @@
+// loaders.gl, MIT license
+import {log} from './utils/log';
+
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const version = typeof __VERSION__ !== 'undefined' ? __VERSION__ : '';
 
 // @ts-ignore
-globalThis.loaders = Object.assign(globalThis.loaders || {}, {
-  VERSION: version
-});
+if (!globalThis.loaders) {
+  log.log(1, `loaders.gl ${version}`)();
 
+  globalThis.loaders = Object.assign(globalThis.loaders || {}, {
+    VERSION: version,
+    log
+  });
+}
 // @ts-ignore
 export default globalThis.loaders;

package/src/lib/loader-utils/get-fetch-function.ts

@@ -0,0 +1,38 @@
+// loaders.gl, MIT license
+
+import type {LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
+import {isObject} from '../../javascript-utils/is-type';
+import {fetchFile} from '../fetch/fetch-file';
+import {getGlobalLoaderOptions} from './option-utils';
+
+/**
+ * Gets the current fetch function from options and context
+ * @param options
+ * @param context
+ */
+export function getFetchFunction(
+  options?: LoaderOptions,
+  context?: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>
+) {
+  const globalOptions = getGlobalLoaderOptions();
+
+  const fetchOptions = options || globalOptions;
+
+  // options.fetch can be a function
+  if (typeof fetchOptions.fetch === 'function') {
+    return fetchOptions.fetch;
+  }
+
+  // options.fetch can be an options object
+  if (isObject(fetchOptions.fetch)) {
+    return (url) => fetchFile(url, fetchOptions as RequestInit);
+  }
+
+  // else refer to context (from parent loader) if available
+  if (context?.fetch) {
+    return context?.fetch;
+  }
+
+  // else return the default fetch function
+  return fetchFile;
+}

package/src/lib/loader-utils/loader-context.ts

@@ -1,5 +1,7 @@
 import type {Loader, LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
-import {getFetchFunction} from './option-utils';
+import {getFetchFunction} from './get-fetch-function';
+import {extractQueryString, stripQueryString} from '../utils/url-utils';
+import {path} from '@loaders.gl/loader-utils';
 
 /**
  * "sub" loaders invoked by other loaders get a "context" injected on `this`
@@ -12,26 +14,35 @@ import {getFetchFunction} from './option-utils';
  */
 export function getLoaderContext(
   context: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>,
-  options?: LoaderOptions,
-  previousContext: LoaderContext | null = null
+  options: LoaderOptions,
+  parentContext: LoaderContext | null
 ): LoaderContext {
   // For recursive calls, we already have a context
   // TODO - add any additional loaders to context?
-  if (previousContext) {
-    return previousContext;
+  if (parentContext) {
+    return parentContext;
   }
 
-  const resolvedContext: LoaderContext = {
+  const newContext: LoaderContext = {
     fetch: getFetchFunction(options, context),
     ...context
   };
 
+  // Parse URLs so that subloaders can easily generate correct strings
+  if (newContext.url) {
+    const baseUrl = stripQueryString(newContext.url);
+    newContext.baseUrl = baseUrl;
+    newContext.queryString = extractQueryString(newContext.url);
+    newContext.filename = path.filename(baseUrl);
+    newContext.baseUrl = path.dirname(baseUrl);
+  }
+
   // Recursive loading does not use single loader
-  if (!Array.isArray(resolvedContext.loaders)) {
-    resolvedContext.loaders = null;
+  if (!Array.isArray(newContext.loaders)) {
+    newContext.loaders = null;
   }
 
-  return resolvedContext;
+  return newContext;
 }
 
 // eslint-disable-next-line complexity

package/src/lib/loader-utils/loggers.ts

@@ -1,5 +1,5 @@
 // probe.gl Log compatible loggers
-import {Log} from 'probe.gl';
+import {Log} from '@probe.gl/log';
 
 export const probeLog = new Log({id: 'loaders.gl'});
 

package/src/lib/loader-utils/option-defaults.ts

@@ -1,4 +1,5 @@
 import type {LoaderOptions} from '@loaders.gl/loader-utils';
+import {isBrowser} from '@loaders.gl/loader-utils';
 import {ConsoleLog} from './loggers';
 
 export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
@@ -12,7 +13,8 @@ export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
   worker: true, // By default, use worker if provided by loader.
   maxConcurrency: 3, // How many worker instances should be created for each loader.
   maxMobileConcurrency: 1, // How many worker instances should be created for each loader on mobile devices.
-  reuseWorkers: true, // By default reuse workers,
+  reuseWorkers: isBrowser, // By default reuse workers in browser (Node.js refuses to terminate if browsers are running)
+  _nodeWorkers: false, // By default do not support node workers
   _workerType: '', // 'test' to use locally generated workers
 
   limit: 0,

package/src/lib/loader-utils/option-utils.ts

@@ -1,14 +1,16 @@
-import type {Loader, LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
+// loaders.gl, MIT license
+
+import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
 import {isPureObject, isObject} from '../../javascript-utils/is-type';
-import {fetchFile} from '../fetch/fetch-file';
 import {probeLog, NullLog} from './loggers';
 import {DEFAULT_LOADER_OPTIONS, REMOVED_LOADER_OPTIONS} from './option-defaults';
+
 /**
  * Global state for loaders.gl. Stored on `global.loaders._state`
  */
 type GlobalLoaderState = {
   loaderRegistry: Loader[];
-  globalOptions: {[key: string]: any};
+  globalOptions: LoaderOptions;
 };
 
 /**
@@ -26,9 +28,12 @@ export function getGlobalLoaderState(): GlobalLoaderState {
   return loaders._state;
 }
 
-// Store global loader options on the global object to increase chances of cross loaders-version interoperability
-// NOTE: This use case is not reliable but can help when testing new versions of loaders.gl with existing frameworks
-const getGlobalLoaderOptions = () => {
+/**
+ * Store global loader options on the global object to increase chances of cross loaders-version interoperability
+ * NOTE: This use case is not reliable but can help when testing new versions of loaders.gl with existing frameworks
+ * @returns global loader options merged with default loader options
+ */
+export const getGlobalLoaderOptions = (): LoaderOptions => {
   const state = getGlobalLoaderState();
   // Ensure all default loader options from this library are mentioned
   state.globalOptions = state.globalOptions || {...DEFAULT_LOADER_OPTIONS};
@@ -39,7 +44,7 @@ const getGlobalLoaderOptions = () => {
  * Set global loader options
  * @param options
  */
-export function setGlobalOptions(options: object): void {
+export function setGlobalOptions(options: LoaderOptions): void {
   const state = getGlobalLoaderState();
   const globalOptions = getGlobalLoaderOptions();
   state.globalOptions = normalizeOptionsInternal(globalOptions, options);
@@ -53,11 +58,11 @@ export function setGlobalOptions(options: object): void {
  * @param url
  */
 export function normalizeOptions(
-  options: object,
+  options: LoaderOptions,
   loader: Loader,
   loaders?: Loader[],
   url?: string
-): object {
+): LoaderOptions {
   loaders = loaders || [];
   loaders = Array.isArray(loaders) ? loaders : [loaders];
 
@@ -65,38 +70,6 @@ export function normalizeOptions(
   return normalizeOptionsInternal(loader, options, url);
 }
 
-/**
- * Gets the current fetch function from options and context
- * @param options
- * @param context
- */
-export function getFetchFunction(
-  options?: LoaderOptions,
-  context?: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>
-) {
-  const globalOptions = getGlobalLoaderOptions();
-
-  const fetchOptions = options || globalOptions;
-
-  // options.fetch can be a function
-  if (typeof fetchOptions.fetch === 'function') {
-    return fetchOptions.fetch;
-  }
-
-  // options.fetch can be an options object
-  if (isObject(fetchOptions.fetch)) {
-    return (url) => fetchFile(url, fetchOptions);
-  }
-
-  // else refer to context (from parent loader) if available
-  if (context?.fetch) {
-    return context?.fetch;
-  }
-
-  // else return the default fetch function
-  return fetchFile;
-}
-
 // VALIDATE OPTIONS
 
 /**
@@ -211,11 +184,14 @@ function mergeNestedFields(mergedOptions, options) {
   }
 }
 
-// Harvest information from the url
-// TODO - baseUri should be a directory, i.e. remove file component from baseUri
-// TODO - extract extension?
-// TODO - extract query parameters?
-// TODO - should these be injected on context instead of options?
+/**
+ * Harvest information from the url
+ * @deprecated This is mainly there to support a hack in the GLTFLoader
+ * TODO - baseUri should be a directory, i.e. remove file component from baseUri
+ * TODO - extract extension?
+ * TODO - extract query parameters?
+ * TODO - should these be injected on context instead of options?
+ */
 function addUrlOptions(options, url?: string) {
   if (url && !('baseUri' in options)) {
     options.baseUri = url;

package/src/lib/progress/fetch-progress.ts

@@ -20,8 +20,8 @@ export default async function fetchProgress(
     return response;
   }
   const contentLength = response.headers.get('content-length') || 0;
-  const totalBytes = contentLength && parseInt(contentLength);
-  if (!(contentLength > 0)) {
+  const totalBytes = contentLength ? parseInt(contentLength) : 0;
+  if (!(totalBytes > 0)) {
     return response;
   }
   // Currently override only implemented in browser

package/src/lib/utils/log.ts

@@ -0,0 +1,4 @@
+// loaders.gl, MIT license
+import {Log} from '@probe.gl/log';
+
+export const log = new Log({id: 'loaders.gl'});