@loaders.gl/core 4.4.3 → 5.0.0-alpha.1

package/src/lib/api/parse-sync.ts

@@ -7,19 +7,23 @@ import type {
   LoaderWithParser,
   LoaderOptions,
   LoaderContext,
+  LoaderOptionsWithShape,
   SyncDataType,
   LoaderOptionsType,
+  LoaderShapeType,
   LoaderReturnType,
   LoaderArrayOptionsType,
   LoaderArrayReturnType,
   StrictLoaderOptions
 } from '@loaders.gl/loader-utils';
+import {isSourceLoader} from '@loaders.gl/loader-utils';
 import {selectLoaderSync} from './select-loader';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
 import {getResourceUrl} from '../utils/resource-utils';
+import {getLoaderImplementationSync} from './load-loader';
 
 // OVERLOADS
 
@@ -28,7 +32,10 @@ import {getResourceUrl} from '../utils/resource-utils';
  */
 export function parseSync<
   LoaderT extends Loader,
-  OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
+  OptionsT extends LoaderOptions = LoaderOptionsWithShape<
+    LoaderOptionsType<LoaderT>,
+    LoaderShapeType<LoaderT>
+  >
 >(
   data: SyncDataType,
   loader: LoaderT,
@@ -84,6 +91,12 @@ export function parseSync(
     return null;
   }
 
+  if (isSourceLoader(loader)) {
+    throw new Error(
+      `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parseSync().`
+    );
+  }
+
   // Normalize options
   const strictOptions = normalizeOptions(options, loader, candidateLoaders as Loader[] | undefined);
 
@@ -99,7 +112,13 @@ export function parseSync(
     context || null
   );
 
-  return parseWithLoaderSync(loader as LoaderWithParser, data, strictOptions, context);
+  const loaderWithParser = getLoaderImplementationSync(loader);
+  return parseWithLoaderSync(
+    loaderWithParser || (loader as LoaderWithParser),
+    data,
+    strictOptions,
+    context
+  );
 }
 
 // TODO - should accept loader.parseSync/parse and generate 1 chunk asyncIterator
@@ -119,6 +138,12 @@ function parseWithLoaderSync(
     return loader.parseSync(data, options, context); // , loader);
   }
 
+  if (loader.preload) {
+    throw new Error(
+      `${loader.name} loader: 'parseSync' requires a parser-bearing loader. Import the loader implementation directly, or call preload(loader) before parseSync(). ${context.url || ''}`
+    );
+  }
+
   // TBD - If synchronous parser not available, return null
   throw new Error(
     `${loader.name} loader: 'parseSync' not supported by this loader, use 'parse' instead. ${

package/src/lib/api/parse.ts

@@ -6,9 +6,11 @@ import type {
   Loader,
   LoaderContext,
   LoaderOptions,
+  LoaderOptionsWithShape,
   DataType,
   LoaderWithParser,
   LoaderOptionsType,
+  LoaderShapeType,
   LoaderReturnType,
   LoaderArrayOptionsType,
   LoaderArrayReturnType,
@@ -18,14 +20,17 @@ import {
   parseWithWorker,
   canParseWithWorker,
   mergeOptions,
-  isResponse
+  isResponse,
+  isSourceLoader
 } from '@loaders.gl/loader-utils';
-import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
+import {validateWorkerVersion} from '@loaders.gl/worker-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
 import {getArrayBufferOrStringFromData} from '../loader-utils/get-data';
+import {getArrayBufferFromData} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
 import {getResourceUrl} from '../utils/resource-utils';
+import {getLoaderImplementation} from './load-loader';
 import {selectLoader} from './select-loader';
 
 // type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T
@@ -35,7 +40,10 @@ import {selectLoader} from './select-loader';
  */
 export async function parse<
   LoaderT extends Loader,
-  OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
+  OptionsT extends LoaderOptions = LoaderOptionsWithShape<
+    LoaderOptionsType<LoaderT>,
+    LoaderShapeType<LoaderT>
+  >
 >(
   data: DataType | Promise<DataType>,
   loader: LoaderT,
@@ -104,6 +112,12 @@ export async function parse(
     return null;
   }
 
+  if (isSourceLoader(loader)) {
+    throw new Error(
+      `${loader.id} is a SourceLoader. Use load() to create a runtime source object instead of parse().`
+    );
+  }
+
   // Normalize options
   // @ts-expect-error candidateLoaders
   const strictOptions = normalizeOptions(options, loader, candidateLoaders, url); // Could be invalid...
@@ -139,32 +153,44 @@ async function parseWithLoader(
     context.response = {headers, ok, redirected, status, statusText, type, url};
   }
 
+  const loaderWithParser = await getLoaderImplementation(loader, options, context.url);
+
+  if (canParseWithWorker(loaderWithParser, options)) {
+    data = await getArrayBufferFromData(data, options);
+    return await parseWithWorker(loaderWithParser, data, options, context, parse);
+  }
+
   data = await getArrayBufferOrStringFromData(data, loader, options);
 
-  const loaderWithParser = loader as LoaderWithParser;
+  return await parseWithLoaderImplementation(loaderWithParser, data, options, context);
+}
 
-  // First check for synchronous text parser, wrap results in promises
-  if (loaderWithParser.parseTextSync && typeof data === 'string') {
-    return loaderWithParser.parseTextSync(data, options, context);
+async function parseWithLoaderImplementation(
+  loader: LoaderWithParser,
+  data: string | ArrayBuffer,
+  options: StrictLoaderOptions,
+  context: LoaderContext
+): Promise<unknown> {
+  if (loader.parseText && typeof data === 'string') {
+    return await loader.parseText(data, options, context);
   }
 
-  // If we have a workerUrl and the loader can parse the given options efficiently in a worker
-  if (canParseWithWorker(loader, options)) {
-    return await parseWithWorker(loader, data, options, context, parse);
+  // Fall back to synchronous text parser, wrap results in promises
+  if (loader.parseTextSync && typeof data === 'string') {
+    return loader.parseTextSync(data, options, context);
   }
 
-  // Check for asynchronous parser
-  if (loaderWithParser.parseText && typeof data === 'string') {
-    return await loaderWithParser.parseText(data, options, context);
+  if (loader.parse) {
+    return await loader.parse(data as ArrayBuffer, options, context);
   }
 
-  if (loaderWithParser.parse) {
-    return await loaderWithParser.parse(data, options, context);
+  // All parser-bearing sync loaders should also offer `parse` so async APIs can use them.
+  if (loader.parseSync) {
+    throw new Error(
+      `${loader.name} loader: 'parse' not available on parser-bearing sync loader. Add an async 'parse' function to this loader, or call preload(loader) before parseSync(). ${context.url || ''}`
+    );
   }
 
-  // This should not happen, all sync loaders should also offer `parse` function
-  assert(!loaderWithParser.parseSync);
-
   // TBD - If asynchronous parser not available, return null
   throw new Error(`${loader.id} loader - no parser found and worker is disabled`);
 }

package/src/lib/api/preload.ts

@@ -0,0 +1,31 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Loader, LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
+import {getLoaderImplementation, getLoaderImplementationSync} from './load-loader';
+
+/**
+ * Preloads a loader implementation so later parse/load calls do not pay the resolution cost on first use.
+ * May return a parser-bearing loader that also supports `parseSync`.
+ */
+export async function preload<LoaderT extends Loader>(
+  loader: LoaderT,
+  options?: LoaderOptions,
+  url?: string
+): Promise<LoaderWithParser>;
+
+export async function preload(
+  loader: Loader,
+  options?: LoaderOptions,
+  url?: string
+): Promise<LoaderWithParser> {
+  return await getLoaderImplementation(loader, options, url);
+}
+
+/**
+ * Returns a cached parser-bearing loader implementation if one is already available.
+ */
+export function preloadSync(loader: Loader): LoaderWithParser | null {
+  return getLoaderImplementationSync(loader);
+}

package/src/lib/api/register-loaders.ts

@@ -28,7 +28,7 @@ export function registerLoaders(loaders: Loader[] | Loader) {
 
   for (const loader of loaders) {
     const normalizedLoader = normalizeLoader(loader);
-    if (!loaderRegistry.find((registeredLoader) => normalizedLoader === registeredLoader)) {
+    if (!loaderRegistry.find(registeredLoader => normalizedLoader === registeredLoader)) {
       // add to the beginning of the loaderRegistry, so the last registeredLoader get picked
       loaderRegistry.unshift(normalizedLoader);
     }

package/src/lib/api/select-loader.ts

@@ -9,7 +9,8 @@ import {
   log,
   isBlob,
   ensureArrayBuffer,
-  isArrayBufferLike
+  isArrayBufferLike,
+  isSourceLoader
 } from '@loaders.gl/loader-utils';
 import {TypedArray} from '@loaders.gl/schema';
 import {normalizeLoader} from '../loader-utils/normalize-loader';
@@ -95,7 +96,9 @@ function mayContainText(response: Response): boolean {
   const mimeType = getResourceMIMEType(response);
   return Boolean(
     mimeType &&
-    (mimeType.startsWith('text/') || mimeType === 'application/json' || mimeType.endsWith('+json'))
+      (mimeType.startsWith('text/') ||
+        mimeType === 'application/json' ||
+        mimeType.endsWith('+json'))
   );
 }
 
@@ -169,12 +172,23 @@ function selectLoaderInternal(
   let loader: Loader | null = null;
   let reason: string = '';
 
+  // if options.mimeType is supplied, it takes precedence
+  const sourceType =
+    options?.core && 'type' in options.core ? (options.core.type as string | undefined) : undefined;
+  if (sourceType && sourceType !== 'auto') {
+    loader = findSourceLoaderByType(loaders, sourceType);
+    reason = loader ? `match forced by supplied source type ${sourceType}` : '';
+  }
+
   // if options.mimeType is supplied, it takes precedence
   if (options?.core?.mimeType) {
     loader = findLoaderByMIMEType(loaders, options?.core?.mimeType);
     reason = `match forced by supplied MIME type ${options?.core?.mimeType}`;
   }
 
+  loader = loader || findSourceLoaderByTestURL(loaders, testUrl);
+  reason = reason || (loader ? `matched source url ${testUrl}` : '');
+
   // Look up loader by url
   loader = loader || findLoaderByUrl(loaders, testUrl);
   reason = reason || (loader ? `matched url ${testUrl}` : '');
@@ -189,6 +203,11 @@ function selectLoaderInternal(
   // @ts-ignore Blob | Response
   reason = reason || (loader ? `matched initial data ${getFirstCharacters(data)}` : '');
 
+  if (!loader && isBlob(data)) {
+    loader = findSourceLoaderByTestData(loaders, data);
+    reason = reason || (loader ? 'matched source testData' : '');
+  }
+
   // Look up loader by fallback mime type
   if (options?.core?.fallbackMimeType) {
     loader = loader || findLoaderByMIMEType(loaders, options?.core?.fallbackMimeType);
@@ -258,9 +277,42 @@ function findLoaderByExtension(loaders: Loader[], extension: string): Loader | n
   return null;
 }
 
+function findSourceLoaderByType(loaders: Loader[], type: string): Loader | null {
+  for (const loader of loaders) {
+    if (isSourceLoader(loader) && loader.type === type) {
+      return loader;
+    }
+  }
+  return null;
+}
+
+function findSourceLoaderByTestURL(loaders: Loader[], url?: string): Loader | null {
+  if (!url) {
+    return null;
+  }
+
+  for (const loader of loaders) {
+    if (isSourceLoader(loader) && loader.testURL(url)) {
+      return loader;
+    }
+  }
+  return null;
+}
+
+function findSourceLoaderByTestData(loaders: Loader[], data: Blob): Loader | null {
+  for (const loader of loaders) {
+    if (isSourceLoader(loader)) {
+      if (loader.testData?.(data)) {
+        return loader;
+      }
+    }
+  }
+  return null;
+}
+
 function findLoaderByMIMEType(loaders: Loader[], mimeType: string): Loader | null {
   for (const loader of loaders) {
-    if (loader.mimeTypes?.some((mimeType1) => compareMIMETypes(mimeType, mimeType1))) {
+    if (loader.mimeTypes?.some(mimeType1 => compareMIMETypes(mimeType, mimeType1))) {
       return loader;
     }
 
@@ -305,12 +357,12 @@ function testDataAgainstText(data: string, loader: Loader): boolean {
   }
 
   const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
-  return tests.some((test) => data.startsWith(test as string));
+  return tests.some(test => data.startsWith(test as string));
 }
 
 function testDataAgainstBinary(data: ArrayBufferLike, byteOffset: number, loader: Loader): boolean {
   const tests = Array.isArray(loader.tests) ? loader.tests : [loader.tests];
-  return tests.some((test) => testBinary(data, byteOffset, loader, test));
+  return tests.some(test => testBinary(data, byteOffset, loader, test));
 }
 
 function testBinary(

package/src/lib/api/select-source.ts

@@ -2,26 +2,23 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {Source} from '@loaders.gl/loader-utils';
+import type {SourceLoader} from '@loaders.gl/loader-utils';
+import {selectLoaderSync} from './select-loader';
 
 /** Guess service type from URL */
 export function selectSource(
   url: string | Blob,
-  sources: Source[],
+  sources: SourceLoader[],
   options?: {
     /** Provide id of a source to select that source. Omit or provide 'auto' to test the source*/
     type?: string;
     nothrow?: boolean;
   }
-): Source | null {
+): SourceLoader | null {
   const type = options?.type || 'auto';
-  let selectedSource: Source | null = null;
+  let selectedSource: SourceLoader | null = null;
   if (type === 'auto') {
-    for (const source of sources) {
-      if (typeof url === 'string' && source.testURL && source.testURL(url)) {
-        return source;
-      }
-    }
+    selectedSource = selectLoaderSync(url, sources, {core: {nothrow: true}}) as SourceLoader | null;
   } else {
     selectedSource = getSourceOfType(type, sources);
   }
@@ -34,7 +31,7 @@ export function selectSource(
 }
 
 /** Guess service type from URL */
-function getSourceOfType(type: string, sources: Source[]): Source | null {
+function getSourceOfType(type: string, sources: SourceLoader[]): SourceLoader | null {
   for (const service of sources) {
     if (service.type === type) {
       return service;

package/src/lib/fetch/fetch-error-message.ts

@@ -17,7 +17,7 @@ export async function getErrorMessageFromResponse(response: Response): Promise<s
     } else {
       message += response.statusText;
     }
-  } catch (error) {
+  } catch (_error) {
     // eslint forbids return in finally statement
     return message;
   }

package/src/lib/loader-utils/check-errors.ts

@@ -14,7 +14,7 @@ export async function checkFetchResponseStatus(response: Response): Promise<void
       if (text) {
         errorMessage += `: ${getErrorText(text)}`;
       }
-    } catch (error) {
+    } catch (_error) {
       // ignore error
     }
     throw new Error(errorMessage);

package/src/lib/loader-utils/get-data.ts

@@ -95,6 +95,42 @@ export async function getArrayBufferOrStringFromData(
   throw new Error(ERR_DATA);
 }
 
+/**
+ * Resolves the provided data into an {@link ArrayBuffer}, preserving bytes for worker transfer.
+ */
+export async function getArrayBufferFromData(
+  data: DataType,
+  options: LoaderOptions
+): Promise<ArrayBuffer> {
+  if (typeof data === 'string') {
+    return new TextEncoder().encode(data).buffer;
+  }
+
+  if (isArrayBufferLike(data)) {
+    return toArrayBuffer(toArrayBufferView(data));
+  }
+
+  if (isBlob(data)) {
+    data = await makeResponse(data);
+  }
+
+  if (isResponse(data)) {
+    await checkResponse(data);
+    return await data.arrayBuffer();
+  }
+
+  if (isReadableStream(data)) {
+    // @ts-expect-error TS2559 options type
+    data = makeIterator(data as ReadableStream, options);
+  }
+
+  if (isIterable(data) || isAsyncIterable(data)) {
+    return concatenateArrayBuffersAsync(data as AsyncIterable<ArrayBufferLike>);
+  }
+
+  throw new Error(ERR_DATA);
+}
+
 /**
  * Normalizes batchable inputs into async iterables for batch parsing flows.
  * Supports synchronous iterables, async iterables, fetch responses, readable streams, and

package/src/lib/loader-utils/get-fetch-function.ts

@@ -14,7 +14,8 @@ import {getGlobalLoaderOptions} from './option-utils';
  */
 export function getFetchFunction(
   options?: LoaderOptions,
-  context?: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>
+  context?: Omit<LoaderContext, 'fetch' | 'coreApi'> &
+    Partial<Pick<LoaderContext, 'fetch' | 'coreApi'>>
 ): FetchLike {
   const globalOptions = getGlobalLoaderOptions();
 
@@ -28,7 +29,7 @@ export function getFetchFunction(
 
   // options.fetch can be an options object
   if (isObject(fetchOption)) {
-    return (url) => fetchFile(url, fetchOption as RequestInit);
+    return url => fetchFile(url, fetchOption as RequestInit);
   }
 
   // else refer to context (from parent loader) if available

package/src/lib/loader-utils/loader-context.ts

@@ -6,9 +6,11 @@ import type {Loader, LoaderContext, StrictLoaderOptions} from '@loaders.gl/loade
 import {getFetchFunction} from './get-fetch-function';
 import {extractQueryString, stripQueryString} from '../utils/url-utils';
 import {path} from '@loaders.gl/loader-utils';
+import {coreApi} from '../api/core-api';
 
 /** Properties for creating an updated context */
-type LoaderContextProps = Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>;
+type LoaderContextProps = Omit<LoaderContext, 'fetch' | 'coreApi'> &
+  Partial<Pick<LoaderContext, 'fetch' | 'coreApi'>>;
 
 /**
  * "sub" loaders invoked by other loaders get a "context" injected on `this`
@@ -31,8 +33,9 @@ export function getLoaderContext(
   }
 
   const newContext: LoaderContext = {
+    ...context,
     fetch: getFetchFunction(options, context),
-    ...context
+    coreApi: context.coreApi || coreApi
   };
 
   // Parse URLs so that subloaders can easily generate correct strings

package/src/lib/loader-utils/loggers.ts

@@ -6,42 +6,68 @@
 
 import {Log} from '@probe.gl/log';
 
+/** Default probe.gl logger used by loaders.gl. */
 export const probeLog = new Log({id: 'loaders.gl'});
 
 type LogFunction = () => void;
 
-// Logs nothing
+/** Logger implementation that intentionally drops all messages. */
 export class NullLog {
+  /** Drops a log message. */
   log(): LogFunction {
     return () => {};
   }
+  /** Drops an info message. */
   info(): LogFunction {
     return () => {};
   }
+  /** Logs one message once; no-op for the null logger. */
+  once(): LogFunction {
+    return () => {};
+  }
+  /** Drops a warning message. */
   warn(): LogFunction {
     return () => {};
   }
+  /** Drops an error message. */
   error(): LogFunction {
     return () => {};
   }
 }
 
-// Logs to console
+/** Logger implementation that forwards messages to the global console. */
 export class ConsoleLog {
+  /** Console-like object used for message output. */
   console;
+  /** Messages already emitted through `once`. */
+  private onceMessages = new Set<string>();
 
+  /** Creates a console-backed logger. */
   constructor() {
     this.console = console; // eslint-disable-line
   }
+  /** Logs a message to the console. */
   log(...args: unknown[]): LogFunction {
     return this.console.log.bind(this.console, ...args);
   }
+  /** Logs an informational message to the console. */
   info(...args: unknown[]): LogFunction {
     return this.console.info.bind(this.console, ...args);
   }
+  /** Logs one message once to the console. */
+  once(...args: unknown[]): LogFunction {
+    const message = String(args[0]);
+    if (this.onceMessages.has(message)) {
+      return () => {};
+    }
+    this.onceMessages.add(message);
+    return this.console.info.bind(this.console, ...args);
+  }
+  /** Logs a warning message to the console. */
   warn(...args: unknown[]): LogFunction {
     return this.console.warn.bind(this.console, ...args);
   }
+  /** Logs an error message to the console. */
   error(...args: unknown[]): LogFunction {
     return this.console.error.bind(this.console, ...args);
   }

package/src/lib/loader-utils/option-defaults.ts

@@ -15,6 +15,7 @@ export const DEFAULT_LOADER_OPTIONS: LoaderOptions = {
     fallbackMimeType: undefined,
     ignoreRegisteredLoaders: undefined,
     nothrow: false,
+    shape: undefined,
     log: new ConsoleLog(), // A probe.gl compatible (`log.log()()` syntax) that just logs to console
     useLocalLibraries: false,
 
@@ -57,6 +58,7 @@ export const REMOVED_LOADER_OPTIONS = {
 
   limit: 'core.limit',
   _limitMB: 'core._limitMB',
+  shape: 'core.shape',
   batchSize: 'core.batchSize',
   batchDebounceMs: 'core.batchDebounceMs',
   metadata: 'core.metadata',