@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.14

package/src/loader-utils/loader-types.ts

@@ -0,0 +1,543 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Format} from './format-types';
+import type {LoadWorker, WorkerOptions, WorkerProgress} from '@loaders.gl/core/worker-utils';
+import {DataType, FetchLike, TransformBatches} from './types';
+import {ReadableFile} from './lib/files/file';
+import type {CoreAPI} from './lib/sources/data-source';
+import type {Credential, AuthenticationConstructor} from './lib/request-utils/authentication';
+import type {ExperimentalScanOptions} from './lib/scan-utils/experimental-scan-options';
+
+/** Query parameters appended to URL-backed loader requests. */
+export type SearchParams = Record<string, string | number | boolean>;
+
+// LOADERS
+
+/**
+ * Core Loader Options
+ */
+export type StrictLoaderOptions = Pick<WorkerOptions, 'signal' | 'onProgress'> & {
+  /** Experimental common scan request. Format loaders implement the fields they support. */
+  _scan?: ExperimentalScanOptions;
+  /** Query parameters appended to the initial URL request. */
+  searchParams?: SearchParams;
+  core?: {
+    /** Base URL for resolving relative paths */
+    baseUrl?: string;
+    /** fetch options or a custom fetch function */
+    fetch?: typeof fetch | FetchLike | RequestInit | null;
+    /** Exact-origin credentials applied to top-level and nested requests. */
+    credentials?: readonly Credential[];
+    /** Authentication classes used to instantiate typed entries in `credentials`. */
+    authentications?: readonly AuthenticationConstructor[];
+    /** Do not throw on errors */
+    nothrow?: boolean;
+    /** Shared default shape for loaders that support shape selection. Loader-scoped `shape` options override this default. */
+    shape?: string;
+
+    /** loader selection, search first for supplied mimeType */
+    mimeType?: string;
+    /** loader selection, provide fallback mimeType is server does not provide */
+    fallbackMimeType?: string;
+    /** loader selection, avoid searching registered loaders */
+    ignoreRegisteredLoaders?: boolean;
+
+    // general
+    /** Experimental: Supply a logger to the parser */
+    log?: any;
+
+    // batched parsing
+
+    /** Size of each batch. `auto` matches batches to size of incoming chunks */
+    batchSize?: number | 'auto';
+    /** Minimal amount of time between batches */
+    batchDebounceMs?: number;
+    /** Stop loading after a given number of rows (compare SQL limit clause) */
+    limit?: 0;
+    /** Experimental: Stop loading after reaching */
+    _limitMB?: 0;
+    /** Generate metadata batches */
+    metadata?: boolean;
+    /** Transforms to run on incoming batches */
+    transforms?: TransformBatches[];
+
+    // module loading
+
+    /** Shared named loader replacements forwarded through recursive dependency preparation and parsing. */
+    subloaders?: Readonly<Record<string, Loader>>;
+
+    /** Force to load WASM libraries from local file system in NodeJS or from loaders.gl CDN in a web browser */
+    useLocalLibraries?: boolean;
+
+    // workers
+
+    /** CDN load workers from */
+    CDN?: string | null;
+    /** Set to `false` to disable workers, or `'auto'` to use loader work estimates. */
+    worker?: boolean | 'auto';
+    /** Declares batch support for a custom worker URL, or disables batch offloading. The loader must also advertise workerInBatches. */
+    workerInBatches?: boolean;
+    /** Work-score threshold used by `worker: 'auto'`. */
+    workerThreshold?: number;
+    /** Number of concurrent workers (per loader) on desktop browser */
+    maxConcurrency?: number;
+    /** Number of concurrent workers (per loader) on mobile browsers */
+    maxMobileConcurrency?: number;
+    /** Set to `false` to prevent reuse workers */
+    reuseWorkers?: boolean;
+    /** Controls whether worker result transfer prepares standalone ArrayBuffers. */
+    workerTransferBufferCopy?: 'none' | 'sliced' | 'all';
+    /** Whether to use workers under Node.js (experimental) */
+    _nodeWorkers?: boolean;
+    /** set to 'test' to run local worker */
+    _workerType?: string;
+  };
+
+  /** Any additional JS libraries */
+  modules?: Record<string, unknown>;
+
+  // Accept other keys (loader options objects, e.g. `options.csv`, `options.json` ...)
+  [loaderId: string]:
+    | Record<string, unknown>
+    | AbortSignal
+    | ((progress: WorkerProgress) => void)
+    | undefined;
+};
+
+/**
+ * Core Loader Options
+ */
+export type LoaderOptions = Pick<WorkerOptions, 'signal' | 'onProgress'> & {
+  /** Experimental common scan request. Format loaders implement the fields they support. */
+  _scan?: ExperimentalScanOptions;
+  /** Query parameters appended to the initial URL request. */
+  searchParams?: SearchParams;
+  core?: StrictLoaderOptions['core'];
+  modules?: StrictLoaderOptions['modules'];
+
+  // Deprecated top-level aliases for core options
+  /** @deprecated Use options.core.baseUrl */
+  baseUri?: string;
+  /** @deprecated Use options.core.fetch */
+  fetch?: typeof fetch | FetchLike | RequestInit | null;
+  /** @deprecated Use options.core.mimeType */
+  mimeType?: string;
+  /** @deprecated Use options.core.fallbackMimeType */
+  fallbackMimeType?: string;
+  /** @deprecated Use options.core.ignoreRegisteredLoaders */
+  ignoreRegisteredLoaders?: boolean;
+  /** @deprecated Use options.core.nothrow */
+  nothrow?: boolean;
+  /** @deprecated Use options.core.log */
+  log?: any;
+  /** @deprecated Use options.core.useLocalLibraries */
+  useLocalLibraries?: boolean;
+  /** @deprecated Use options.core.CDN */
+  CDN?: string | null;
+  /** @deprecated Use options.core.worker */
+  worker?: boolean | 'auto';
+  /** @deprecated Use options.core.maxConcurrency */
+  maxConcurrency?: number;
+  /** @deprecated Use options.core.maxMobileConcurrency */
+  maxMobileConcurrency?: number;
+  /** @deprecated Use options.core.reuseWorkers */
+  reuseWorkers?: boolean;
+  /** @deprecated Use options.core.workerTransferBufferCopy */
+  workerTransferBufferCopy?: 'none' | 'sliced' | 'all';
+  /** @deprecated Use options.core._nodeWorkers */
+  _nodeWorkers?: boolean;
+  /** @deprecated Use options.core._workerType */
+  _workerType?: string;
+  /** @deprecated Use options.core._workerType */
+  _worker?: string;
+  /** @deprecated Use options.core.limit */
+  limit?: 0;
+  /** @deprecated Use options.core._limitMB */
+  _limitMB?: 0;
+  /** @deprecated Use options.core.shape */
+  shape?: string;
+  /** @deprecated Use options.core.batchSize */
+  batchSize?: number | 'auto';
+  /** @deprecated Use options.core.batchDebounceMs */
+  batchDebounceMs?: number;
+  /** @deprecated Use options.core.metadata */
+  metadata?: boolean;
+  /** @deprecated Use options.core.transforms */
+  transforms?: TransformBatches[];
+
+  // Accept other keys (loader options objects, e.g. `options.csv`, `options.json` ...)
+  [loaderId: string]: unknown;
+};
+
+type PreloadOptions = {
+  [key: string]: unknown;
+};
+
+/**
+ * A worker loader definition that can be used with `@loaders.gl/core` functions
+ */
+export type Loader<DataT = any, BatchT = any, LoaderOptionsT = StrictLoaderOptions> = Format & {
+  /** The result type of this loader  */
+  dataType?: DataT;
+  /** The batched result type of this loader  */
+  batchType?: BatchT;
+
+  /**
+   * Named metadata loaders used by this loader or source.
+   * Worker loaders declare nested decoders here so the caller can resolve their identifiers
+   * without transferring parser or preload functions between threads.
+   */
+  subloaders?: Readonly<Record<string, Loader>>;
+
+  /** Default Options */
+  options: LoaderOptionsT;
+  /** Deprecated Options */
+  deprecatedOptions?: Record<string, string | Record<string, string>>;
+  /** Version should be injected by build tools */
+  version: string;
+  /** A boolean, or a URL */
+  worker?: string | boolean;
+  /** Whether the worker and its batch transport support this loader's batch parser. */
+  workerInBatches?: boolean;
+  /** Browser worker filename when it differs from the loader id. */
+  workerFile?: string;
+  /** Optional Node.js worker filename, such as a CommonJS `.cjs` bundle. */
+  workerNode?: string;
+  /** Repository-relative source path for a Node test worker after package grouping. */
+  workerSourceNode?: string;
+  /** Creates a built-in browser worker, typically using `type: 'module'`. */
+  loadWorker?: LoadWorker;
+  /**
+   * Optionally warm the loader before parse/load is invoked.
+   * Can be used to avoid a later delay and may return a parser-bearing loader that also supports `parseSync`.
+   */
+  preload?: Preload;
+  /** Contributes authentication constructors before requests; application constructors take precedence. */
+  getAuthentications?: (
+    url: string,
+    options: LoaderOptions
+  ) => readonly AuthenticationConstructor[] | Promise<readonly AuthenticationConstructor[]>;
+  /** Serializes parser output before returning it from a worker. */
+  serializeWorkerResult?: (
+    result: DataT,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => unknown;
+  /** Deserializes parser output returned from a worker. */
+  deserializeWorkerResult?: (
+    result: unknown,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => DataT;
+  /** Estimates parser work without consuming the input. Scores range from 0 (trivial) to 1 (expensive). */
+  getWorkerEstimate?: (
+    data: DataType,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => number | undefined;
+  // end Worker
+
+  /** Human readable name */
+  name: string;
+  /** id should be the same as the field used in LoaderOptions */
+  id: string;
+  /** module is used to generate worker threads, need to be the module directory name */
+  module: string;
+  /** Which category does this loader belong to */
+  category?: string;
+  /** File extensions that are potential matches with this loader. */
+  extensions: string[];
+  /** MIMETypes that indicate a match with this loader. @note Some MIMETypes are generic and supported by many loaders */
+  mimeTypes: string[];
+  /** Is the input of this loader binary */
+  binary?: boolean;
+  /** Is the input of this loader text */
+  text?: boolean;
+
+  /** Test some initial bytes of content to see if this loader might be a match */
+  tests?: (((ArrayBuffer: ArrayBuffer) => boolean) | ArrayBuffer | string)[];
+
+  /** @deprecated */
+  supported?: boolean;
+  /** @deprecated */
+  testText?: (string: string) => boolean;
+};
+
+/**
+ * A "bundled" loader definition that can be used with `@loaders.gl/core` functions
+ * If a worker loader is supported it will also be supported.
+ */
+export type LoaderWithParser<
+  DataT = any,
+  BatchT = any,
+  LoaderOptionsT = StrictLoaderOptions
+> = Loader<DataT, BatchT, LoaderOptionsT> & {
+  /** Parse asynchronously and atomically from an arraybuffer */
+  parse: (
+    arrayBuffer: ArrayBuffer,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => Promise<DataT>;
+  /** Parse asynchronously and atomically from a browser Blob without first copying to an ArrayBuffer. */
+  parseBlob?: (blob: Blob, options?: LoaderOptionsT, context?: LoaderContext) => Promise<DataT>;
+  /** Parse asynchronously and atomically from a random access "file like" input */
+  parseFile?: (
+    file: ReadableFile,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => Promise<DataT>;
+  /** Parse synchronously and atomically from an arraybuffer */
+  parseSync?: (
+    arrayBuffer: ArrayBuffer,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => DataT;
+  /** Parse batches of data from an iterator (e.g. fetch stream), return an iterator that yield parsed batches. */
+  parseInBatches?: (
+    iterator:
+      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+      | Iterable<ArrayBufferLike | ArrayBufferView>,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => AsyncIterable<BatchT>;
+  /** Serializes one parser batch before returning it from a worker. */
+  serializeWorkerBatch?: (
+    batch: BatchT,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => unknown;
+  /** Deserializes one parser batch returned from a worker. */
+  deserializeWorkerBatch?: (
+    batch: unknown,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => BatchT;
+  /** For random access, file like sources, source that don't integrate with fetch. */
+  parseFileInBatches?: (
+    file: ReadableFile,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => AsyncIterable<BatchT>;
+
+  /** Parse atomically from a string asynchronously */
+  parseText?: (text: string, options?: LoaderOptionsT, context?: LoaderContext) => Promise<DataT>;
+  /** Parse atomically from a string synchronously */
+  parseTextSync?: (text: string, options?: LoaderOptionsT, context?: LoaderContext) => DataT;
+};
+
+/**
+ * A Loader context is provided as a third parameters to a loader object's
+ * parse functions when that loader is called by other loaders rather then
+ * directly by the application.
+ *
+ * - The context object allows the subloaders to be aware of the parameters and
+ *   options that the application provided in the top level call.
+ * - The context also providedsaccess to parse functions so that the subloader
+ *   does not need to include the core module.
+ * - In addition, the context's parse functions may also redirect loads from worker
+ *   threads back to main thread.
+ */
+export type LoaderContext = {
+  /** Cancels nested asynchronous operations where supported. */
+  signal?: AbortSignal;
+  /** Reports progress within a loader-defined phase. */
+  reportProgress?: (progress: WorkerProgress) => void;
+  /** Loader list provided to top-level loader call plus any sub loaders */
+  loaders?: Loader[] | null;
+  /** If URL is available.  */
+  url?: string;
+  /** the file name component of the URL (leading path and query string removed) */
+  filename?: string;
+  /** the directory name component of the URL (leading path excluding file name and query string) */
+  baseUrl?: string;
+  /** Query string (characters after `?`) */
+  queryString?: string;
+
+  /** Provides access to any application overrides of fetch() */
+  fetch: typeof fetch | FetchLike;
+  /** Provides access to the core parsing and loading API without importing `@loaders.gl/core`. */
+  coreApi: CoreAPI;
+
+  /** TBD */
+  response?: Response;
+
+  /**
+   * Parse function for subloaders. Avoids importing `core`. In workers, may redirect to main thread
+   */
+  _parse: (
+    arrayBuffer: ArrayBuffer,
+    loaders?: Loader | Loader[] | LoaderOptions,
+    options?: LoaderOptions,
+    context?: LoaderContext
+  ) => Promise<any>;
+
+  /**
+   * ParseSync function. Avoids importing `core`. In workers, may redirect to main thread
+   * @deprecated Do not call directly, use `parseSyncFromContext` instead
+   */
+  _parseSync?: (
+    arrayBuffer: ArrayBuffer,
+    loaders?: Loader | Loader[] | LoaderOptions,
+    options?: LoaderOptions,
+    context?: LoaderContext
+  ) => any;
+
+  /**
+   * ParseInBatches function. Avoids importing `core`.
+   * @deprecated Do not call directly, use `parseInBatchesFromContext` instead
+   */
+  _parseInBatches?: (
+    iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer> | Response,
+    loaders?: Loader | Loader[] | LoaderOptions,
+    options?: LoaderOptions,
+    context?: LoaderContext
+  ) => AsyncIterable<any> | Promise<AsyncIterable<any>>;
+};
+
+// type Parse = (
+//   arrayBuffer: ArrayBuffer,
+//   options?: LoaderOptions,
+//   context?: LoaderContext
+// ) => Promise<any>;
+// type ParseSync = (
+//   arrayBuffer: ArrayBuffer,
+//   options?: LoaderOptions,
+//   context?: LoaderContext
+// ) => any;
+// type ParseText = (text: string, options?: LoaderOptions) => Promise<any>;
+// type ParseTextSync = (text: string, options?: LoaderOptions) => any;
+// type ParseInBatches = (
+//   iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+//   options?: LoaderOptions,
+//   context?: LoaderContext
+// ) => AsyncIterable<any>;
+// type ParseFileInBatches = (
+//   file: Blob,
+//   options?: LoaderOptions,
+//   context?: LoaderContext
+// ) => AsyncIterable<any>;
+
+/**
+ * Optionally resolves a loader to a parser-bearing implementation before parsing begins.
+ * Can be used to avoid a later delay and may return a loader that supports `parseSync`.
+ */
+type Preload = (url: string, options?: PreloadOptions) => any;
+
+/** Typescript helper to extract options type from a loader type */
+export type LoaderOptionsType<T = Loader> =
+  T extends Loader<unknown, unknown, infer Options> ? Options : never;
+export type LoaderShapeType<T extends Loader = Loader> = LoaderOptionsType<T>[T['id']] extends {
+  shape?: infer Shape;
+}
+  ? Extract<Shape, string>
+  : LoaderOptionsType<T>[T['id']] extends {shape: infer Shape}
+    ? Extract<Shape, string>
+    : never;
+type LoaderShapeOrString<Shape extends string> = [Shape] extends [never] ? string : Shape;
+export type LoaderOptionsWithShape<
+  Options extends LoaderOptions,
+  Shape extends string
+> = Options extends LoaderOptions
+  ? Omit<Options, 'core'> & {
+      core?: Omit<NonNullable<Options['core']>, 'shape'> & {shape?: LoaderShapeOrString<Shape>};
+    }
+  : never;
+/** Typescript helper to extract data type from a loader type */
+export type LoaderReturnType<T = Loader> =
+  T extends Loader<infer Return, unknown, unknown> ? Return : never;
+/** Typescript helper to extract batch type from a loader type */
+export type LoaderBatchType<T = Loader> =
+  T extends Loader<unknown, infer Batch, unknown> ? Batch : never;
+
+/** Typescript helper to extract options type from an array of loader types */
+export type LoaderArrayOptionsType<LoadersT extends Loader[] = Loader[]> = LoaderOptionsWithShape<
+  LoadersT[number]['options'],
+  LoaderShapeType<LoadersT[number]>
+>;
+/** Typescript helper to extract data type from a loader type */
+export type LoaderArrayReturnType<LoadersT extends Loader[] = Loader[]> =
+  LoadersT[number]['dataType'];
+/** Typescript helper to extract batch type from a loader type */
+export type LoaderArrayBatchType<LoadersT extends Loader[] = Loader[]> =
+  LoadersT[number]['batchType'];
+
+/**
+ * Parses `data` asynchronously using the supplied loader, parse function provided via the loader context
+ */
+export async function parseFromContext<
+  LoaderT extends Loader,
+  OptionsT extends StrictLoaderOptions = LoaderOptionsWithShape<
+    LoaderOptionsType<LoaderT>,
+    LoaderShapeType<LoaderT>
+  >
+>(
+  data: ArrayBuffer,
+  loader: LoaderT,
+  options: OptionsT | undefined,
+  context: LoaderContext
+): Promise<LoaderReturnType<LoaderT>>;
+
+/**
+ * Parses `data` asynchronously by matching one of the supplied loader
+ */
+export async function parseFromContext(
+  data: ArrayBuffer,
+  loaders: Loader[],
+  options: LoaderOptions | undefined,
+  context: LoaderContext
+): Promise<unknown>;
+
+/**
+ * Parses `data` using a specified loader
+ * @param data
+ * @param loaders
+ * @param options
+ * @param context
+ */
+// implementation signature
+export async function parseFromContext(
+  data: ArrayBuffer,
+  loaders: Loader | Loader[],
+  options: LoaderOptions | undefined,
+  context: LoaderContext
+): Promise<unknown> {
+  return context._parse(data, loaders, options, context);
+}
+
+/**
+ * Parses `data` synchronously using the specified loader, parse function provided via the loader context
+ */
+export function parseSyncFromContext<
+  LoaderT extends Loader,
+  OptionsT extends StrictLoaderOptions = LoaderOptionsType<LoaderT>
+>(
+  data: ArrayBuffer,
+  loader: LoaderT,
+  options: OptionsT | undefined,
+  context: LoaderContext
+): LoaderReturnType<LoaderT> {
+  if (!context._parseSync) {
+    throw new Error('parseSync');
+  }
+  return context._parseSync(data, loader, options, context);
+}
+
+/**
+ * Parses `data` synchronously using a specified loader, parse function provided via the loader context
+ */
+export async function parseInBatchesFromContext<
+  LoaderT extends Loader,
+  OptionsT extends StrictLoaderOptions = LoaderOptionsType<LoaderT>
+>(
+  data: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer> | Response,
+  loader: LoaderT,
+  options: OptionsT | undefined,
+  context: LoaderContext
+): Promise<AsyncIterable<LoaderBatchType<LoaderT>>> {
+  if (!context._parseInBatches) {
+    throw new Error('parseInBatches');
+  }
+  return context._parseInBatches(data, loader, options, context);
+}

package/src/loader-utils/source-types.ts

@@ -0,0 +1,76 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Loader, LoaderOptions} from './loader-types';
+import type {CoreAPI, DataSource, DataSourceOptions} from './lib/sources/data-source';
+
+/**
+ * A `SourceLoader` is a top-level loader that constructs a runtime `DataSource`.
+ * It allows source-style loaders to participate in core loader selection and `load()`.
+ * @example
+ *  `createDataSource(... , [MVTSourceLoader, PMTilesSourceLoader, ...])
+ */
+export type SourceLoader<
+  DataSourceT extends DataSource<unknown, DataSourceOptions> = DataSource<
+    unknown,
+    DataSourceOptions
+  >,
+  LoaderOptionsT extends LoaderOptions & DataSourceOptions = LoaderOptions & DataSourceOptions
+> = Loader<DataSourceT, never, LoaderOptionsT> & {
+  /** Type of source created by this source loader */
+  dataSource?: DataSourceT;
+  /** Type of options used when creating sources */
+  options?: DataSourceT['optionsType'];
+
+  /** Type string identifying this source loader, e.g. 'wms' */
+  type: string;
+  /** Can source be created from a URL */
+  fromUrl: boolean;
+  /** Can source be created from a Blob or File */
+  fromBlob: boolean;
+
+  /** Default options for creating the runtime data source */
+  defaultOptions: DataSourceT['optionsType'];
+
+  /** Check if a URL can support this source loader */
+  testURL: (url: string) => boolean;
+  /** Test data */
+  testData?: (data: Blob) => boolean;
+  /** Create a runtime data source  */
+  createDataSource(
+    data: string | Blob,
+    options: Readonly<DataSourceT['optionsType']>,
+    coreApi?: CoreAPI
+  ): DataSourceT;
+};
+
+export function isSourceLoader(loader?: unknown): loader is SourceLoader {
+  return Boolean(loader && typeof loader === 'object' && 'createDataSource' in loader);
+}
+
+/** Extracts the full public options contract, including fields absent from runtime defaults. */
+export type SourcePropsType<SourceT extends SourceLoader> = NonNullable<
+  Parameters<SourceT['createDataSource']>[1]
+>;
+
+/** Typescript helper to extract the source options type from a source type */
+export type SourceDataSourceType<SourceT extends SourceLoader> = ReturnType<
+  SourceT['createDataSource']
+>;
+
+/** Combines options for all candidate sources instead of accepting one candidate's untyped keys. */
+type UnionToIntersection<Value> = (Value extends unknown ? (value: Value) => void : never) extends (
+  value: infer Intersection
+) => void
+  ? Intersection
+  : never;
+
+/** Typescript helper to extract options type from an array of source types */
+export type SourceArrayOptionsType<SourcesT extends SourceLoader[] = SourceLoader[]> =
+  UnionToIntersection<SourcePropsType<SourcesT[number]>> & DataSourceOptions;
+
+/** Typescript helper to extract data type from a source type */
+export type SourceArrayDataSourceType<SourcesT extends SourceLoader[] = SourceLoader[]> =
+  SourceDataSourceType<SourcesT[number]>;
+/** Typescript helper to extract batch type from a source type */

package/src/loader-utils/types.ts

@@ -0,0 +1,83 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// Typed arrays
+
+// eslint-disable-next-line spaced-comment
+/// <reference lib="es2025.float16" preserve="true" />
+
+export type TypedIntArray =
+  | Int8Array
+  | Uint8Array
+  | Uint8ClampedArray
+  | Int16Array
+  | Uint16Array
+  | Int32Array
+  | Uint32Array;
+
+export type TypedFloatArray = Uint16Array | Float16Array | Float32Array | Float64Array;
+
+export type TypedArray = TypedIntArray | TypedFloatArray;
+
+export type BigTypedArray = TypedArray | BigInt64Array | BigUint64Array;
+
+export type TypedArrayConstructor =
+  | Int8ArrayConstructor
+  | Uint8ArrayConstructor
+  | Int16ArrayConstructor
+  | Uint16ArrayConstructor
+  | Int32ArrayConstructor
+  | Uint32ArrayConstructor
+  | Float16ArrayConstructor
+  | Float32ArrayConstructor
+  | Float64ArrayConstructor;
+
+export type BigTypedArrayConstructor =
+  | TypedArrayConstructor
+  | BigInt64ArrayConstructor
+  | BigUint64ArrayConstructor;
+
+/** Any numeric array: typed array or `number[]` */
+export type NumberArray = number[] | TypedArray;
+
+export type NumericArray = number[] | TypedArray;
+
+// FETCH
+
+export type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
+
+// MISC TYPES
+
+/**
+ * Transformation callback used by batch parsers to reshape streamed binary chunks.
+ * Accepts iterables of `ArrayBuffer`, `ArrayBufferView`, or `ArrayBufferLike` values and
+ * returns an async iterable of the same surface.
+ */
+export type TransformBatches = (
+  asyncIterator:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>
+) => AsyncIterable<ArrayBufferLike | ArrayBufferView>;
+
+/** Types that can be synchronously parsed */
+export type SyncDataType = string | ArrayBufferLike | ArrayBufferView; // TODO File | Blob can be read synchronously...
+
+/** Types that can be parsed async */
+export type DataType =
+  | string
+  | ArrayBufferLike
+  | ArrayBufferView
+  | File
+  | Blob
+  | Response
+  | ReadableStream
+  | Iterable<ArrayBufferLike | ArrayBufferView>
+  | AsyncIterable<ArrayBufferLike | ArrayBufferView>;
+
+/** Types that can be parsed in batches */
+export type BatchableDataType =
+  | DataType
+  | Iterable<ArrayBufferLike | ArrayBufferView>
+  | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+  | Promise<AsyncIterable<ArrayBufferLike | ArrayBufferView>>;