@loaders.gl/core 4.0.0-alpha.19 → 4.0.0-alpha.21

package/src/lib/api/parse-in-batches.ts

@@ -1,11 +1,9 @@
+// loaders.gl, MIT license
+
 import type {Batch} from '@loaders.gl/schema';
-import type {
-  BatchableDataType,
-  Loader,
-  LoaderWithParser,
-  LoaderContext,
-  LoaderOptions
-} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {LoaderContext, BatchableDataType} from '@loaders.gl/loader-utils';
+import type {LoaderBatchType, LoaderOptionsType} from '@loaders.gl/loader-utils';
 import {assert, concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {normalizeOptions} from '../loader-utils/option-utils';
@@ -17,6 +15,37 @@ import {selectLoader} from './select-loader';
 // Ensure `parse` is available in context if loader falls back to `parse`
 import {parse} from './parse';
 
+/**
+ * Parses `data` synchronously using a specified loader
+ */
+export async function parseInBatches<
+  LoaderT extends Loader,
+  OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
+>(
+  data: BatchableDataType,
+  loader: LoaderT,
+  options?: OptionsT,
+  context?: LoaderContext
+): Promise<AsyncIterable<LoaderBatchType<LoaderT>>>;
+
+/**
+ * Parses `data` using one of the supplied loaders
+ */
+export async function parseInBatches(
+  data: BatchableDataType,
+  loaders: Loader[],
+  options?: LoaderOptions,
+  context?: LoaderContext
+): Promise<AsyncIterable<any>>;
+
+/**
+ * Parses `data` in batches by selecting a pre-registered loader
+ */
+export async function parseInBatches(
+  data: BatchableDataType,
+  options?: LoaderOptions
+): Promise<AsyncIterable<any>>;
+
 /**
  * Parses `data` using a specified loader
  * @param data
@@ -29,7 +58,7 @@ export async function parseInBatches(
   loaders?: Loader | Loader[] | LoaderOptions,
   options?: LoaderOptions,
   context?: LoaderContext
-): Promise<AsyncIterable<any>> {
+): Promise<AsyncIterable<unknown>> {
   assert(!context || typeof context === 'object'); // parseInBatches no longer accepts final url
 
   const loaderArray = Array.isArray(loaders) ? loaders : undefined;
@@ -75,7 +104,7 @@ async function parseWithLoaderInBatches(
   data: BatchableDataType,
   options: LoaderOptions,
   context: LoaderContext
-): Promise<AsyncIterable<any>> {
+): Promise<AsyncIterable<unknown>> {
   const outputIterator = await parseToOutputIterator(loader, data, options, context);
 
   // Generate metadata batch if requested
@@ -112,7 +141,7 @@ async function parseToOutputIterator(
   data: BatchableDataType,
   options: LoaderOptions,
   context: LoaderContext
-): Promise<AsyncIterable<any>> {
+): Promise<AsyncIterable<unknown>> {
   // Get an iterator from the input
   const inputIterator = await getAsyncIterableFromData(data, options);
 

package/src/lib/api/parse-sync.ts

@@ -1,10 +1,6 @@
-import type {
-  SyncDataType,
-  Loader,
-  LoaderWithParser,
-  LoaderContext,
-  LoaderOptions
-} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {LoaderContext, SyncDataType} from '@loaders.gl/loader-utils';
+import type {LoaderOptionsType, LoaderReturnType} from '@loaders.gl/loader-utils';
 import {assert} from '@loaders.gl/loader-utils';
 import {selectLoaderSync} from './select-loader';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
@@ -13,19 +9,45 @@ import {getArrayBufferOrStringFromDataSync} from '../loader-utils/get-data';
 import {getLoaderContext, getLoadersFromContext} from '../loader-utils/loader-context';
 import {getResourceUrl} from '../utils/resource-utils';
 
+// OVERLOADS
+
+/**
+ * Parses `data` synchronously using the specified loader
+ */
+export function parseSync<
+  LoaderT extends Loader,
+  OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
+>(
+  data: SyncDataType,
+  loader: LoaderT,
+  options?: OptionsT,
+  context?: LoaderContext
+): LoaderReturnType<LoaderT>;
+
+/**
+ * Parses `data` synchronously by matching one of the supplied loaders
+ */
+export function parseSync(
+  data: SyncDataType,
+  loaders: Loader[],
+  options?: LoaderOptions,
+  context?: LoaderContext
+): any;
+
+/**
+ * Parses `data` synchronously by matching a pre=registered loader
+ */
+export function parseSync(data: SyncDataType, options?: LoaderOptions): any;
+
 /**
  * Parses `data` synchronously using a specified loader
- * @param data
- * @param loaders
- * @param options
- * @param context
  */
 export function parseSync(
   data: SyncDataType,
   loaders?: Loader | Loader[] | LoaderOptions,
   options?: LoaderOptions,
   context?: LoaderContext
-): any {
+): unknown {
   assert(!context || typeof context === 'object'); // parseSync no longer accepts final url
 
   // Signature: parseSync(data, options)

package/src/lib/api/parse.ts

@@ -1,9 +1,10 @@
 // loaders.gl, MIT license
 
-import type {DataType, Loader, LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {DataType, LoaderWithParser} from '@loaders.gl/loader-utils';
 import type {LoaderOptionsType, LoaderReturnType} from '@loaders.gl/loader-utils';
-import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
 import {parseWithWorker, canParseWithWorker} from '@loaders.gl/loader-utils';
+import {assert, validateWorkerVersion} from '@loaders.gl/worker-utils';
 import {isLoaderObject} from '../loader-utils/normalize-loader';
 import {isResponse} from '../../javascript-utils/is-type';
 import {normalizeOptions} from '../loader-utils/option-utils';
@@ -14,6 +15,9 @@ import {selectLoader} from './select-loader';
 
 // type LoaderArrayType<T> = T extends (infer Loader)[] ? LoaderOptionsType<Loader> : T
 
+/**
+ * Parses `data` asynchronously using the supplied loaders
+ */
 export async function parse<
   LoaderT extends Loader,
   OptionsT extends LoaderOptions = LoaderOptionsType<LoaderT>
@@ -24,6 +28,9 @@ export async function parse<
   context?: LoaderContext
 ): Promise<LoaderReturnType<LoaderT>>;
 
+/**
+ * Parses `data` asynchronously by matching one of the supplied loaders
+ */
 export async function parse(
   data: DataType | Promise<DataType>,
   loaders: Loader[],
@@ -31,6 +38,9 @@ export async function parse(
   context?: LoaderContext
 ): Promise<any>;
 
+/**
+ * Parses data asynchronously by matching a pre-registered loader
+ */
 export async function parse(
   data: DataType | Promise<DataType>,
   options?: LoaderOptions
@@ -49,7 +59,7 @@ export async function parse(
   loaders?: Loader | Loader[] | LoaderOptions,
   options?: LoaderOptions,
   context?: LoaderContext
-): Promise<any> {
+): Promise<unknown> {
   assert(!context || typeof context === 'object'); // parse no longer accepts final url
 
   // Signature: parse(data, options, context | url)
@@ -88,7 +98,12 @@ export async function parse(
 
 // TODO: support progress and abort
 // TODO - should accept loader.parseAsyncIterator and concatenate.
-async function parseWithLoader(loader, data, options, context) {
+async function parseWithLoader(
+  loader: Loader,
+  data,
+  options: LoaderOptions,
+  context: LoaderContext
+): Promise<unknown> {
   validateWorkerVersion(loader);
 
   if (isResponse(data)) {
@@ -96,15 +111,17 @@ async function parseWithLoader(loader, data, options, context) {
     const response = data as Response;
     const {ok, redirected, status, statusText, type, url} = response;
     const headers = Object.fromEntries(response.headers.entries());
+    // @ts-expect-error TODO - fix this
     context.response = {headers, ok, redirected, status, statusText, type, url};
   }
 
   data = await getArrayBufferOrStringFromData(data, loader, options);
 
+  const loaderWithParser = loader as LoaderWithParser;
+
   // First check for synchronous text parser, wrap results in promises
-  if (loader.parseTextSync && typeof data === 'string') {
-    options.dataType = 'text';
-    return loader.parseTextSync(data, options, context, loader);
+  if (loaderWithParser.parseTextSync && typeof data === 'string') {
+    return loaderWithParser.parseTextSync(data, options, context);
   }
 
   // If we have a workerUrl and the loader can parse the given options efficiently in a worker
@@ -113,16 +130,16 @@ async function parseWithLoader(loader, data, options, context) {
   }
 
   // Check for asynchronous parser
-  if (loader.parseText && typeof data === 'string') {
-    return await loader.parseText(data, options, context, loader);
+  if (loaderWithParser.parseText && typeof data === 'string') {
+    return await loaderWithParser.parseText(data, options, context);
   }
 
-  if (loader.parse) {
-    return await loader.parse(data, options, context, loader);
+  if (loaderWithParser.parse) {
+    return await loaderWithParser.parse(data, options, context);
   }
 
   // This should not happen, all sync loaders should also offer `parse` function
-  assert(!loader.parseSync);
+  assert(!loaderWithParser.parseSync);
 
   // TBD - If asynchronous parser not available, return null
   throw new Error(`${loader.id} loader - no parser found and worker is disabled`);

package/src/lib/filesystems/filesystem.ts

@@ -36,9 +36,9 @@ export interface FileSystem {
  * A random access file system
  */
 export interface RandomAccessReadFileSystem extends FileSystem {
-  open(path: string, flags, mode?): Promise<any>;
-  close(fd: any): Promise<void>;
-  fstat(fd: any): Promise<Stat>;
+  open(path: string, flags: unknown, mode?: unknown): Promise<any>;
+  close(fd: unknown): Promise<void>;
+  fstat(fd: unknown): Promise<Stat>;
   read(
     fd: any,
     buffer: ArrayBuffer | ArrayBufferView,

package/src/lib/loader-utils/get-fetch-function.ts

@@ -1,6 +1,6 @@
 // loaders.gl, MIT license
 
-import type {LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {LoaderContext, LoaderOptions, FetchLike} from '@loaders.gl/loader-utils';
 import {isObject} from '../../javascript-utils/is-type';
 import {fetchFile} from '../fetch/fetch-file';
 import {getGlobalLoaderOptions} from './option-utils';
@@ -13,19 +13,19 @@ import {getGlobalLoaderOptions} from './option-utils';
 export function getFetchFunction(
   options?: LoaderOptions,
   context?: Omit<LoaderContext, 'fetch'> & Partial<Pick<LoaderContext, 'fetch'>>
-) {
+): FetchLike {
   const globalOptions = getGlobalLoaderOptions();
 
-  const fetchOptions = options || globalOptions;
+  const loaderOptions = options || globalOptions;
 
   // options.fetch can be a function
-  if (typeof fetchOptions.fetch === 'function') {
-    return fetchOptions.fetch;
+  if (typeof loaderOptions.fetch === 'function') {
+    return loaderOptions.fetch;
   }
 
   // options.fetch can be an options object
-  if (isObject(fetchOptions.fetch)) {
-    return (url) => fetchFile(url, fetchOptions as RequestInit);
+  if (isObject(loaderOptions.fetch)) {
+    return (url) => fetchFile(url, loaderOptions.fetch as RequestInit);
   }
 
   // else refer to context (from parent loader) if available

package/src/null-loader.ts

@@ -2,12 +2,17 @@
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-import {Loader, LoaderWithParser} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import {LoaderContext} from 'modules/loader-utils/dist';
+
+export type NullLoaderOptions = LoaderOptions & {
+  null?: {};
+};
 
 /**
  * Loads any data and returns null (or optionally passes through data unparsed)
  */
-export const NullWorkerLoader: Loader = {
+export const NullWorkerLoader: Loader<null, never, NullLoaderOptions> = {
   name: 'Null loader',
   id: 'null',
   module: 'core',
@@ -21,27 +26,18 @@ export const NullWorkerLoader: Loader = {
   }
 };
 
-/**
- * Returns arguments passed to the parse API in a format that can be transfered to a
- * web worker. The `context` parameter is stripped using JSON.stringify & parse.
- */
-function parseSync(arrayBuffer, options, context) {
-  if (!options.null.echoParameters) return null;
-  context = context && JSON.parse(JSON.stringify(context));
-  return {arrayBuffer, options, context};
-}
-
 /**
  * Loads any data and returns null (or optionally passes through data unparsed)
  */
-export const NullLoader: LoaderWithParser = {
+export const NullLoader: LoaderWithParser<null, null, NullLoaderOptions> = {
   name: 'Null loader',
   id: 'null',
   module: 'core',
   version: VERSION,
   mimeTypes: ['application/x.empty'],
   extensions: ['null'],
-  parse: async (arrayBuffer, options, context) => parseSync(arrayBuffer, options, context),
+  parse: async (arrayBuffer: ArrayBuffer, options?: NullLoaderOptions, context?: LoaderContext) =>
+    parseSync(arrayBuffer, options || {}, context),
   parseSync,
   parseInBatches: async function* generator(asyncIterator, options, context) {
     for await (const batch of asyncIterator) {
@@ -50,8 +46,18 @@ export const NullLoader: LoaderWithParser = {
   },
   tests: [() => false],
   options: {
-    null: {
-      echoParameters: false
-    }
+    null: {}
   }
 };
+
+/**
+ * Returns arguments passed to the parse API in a format that can be transferred to a
+ * web worker. The `context` parameter is stripped using JSON.stringify & parse.
+ */
+function parseSync(
+  arrayBuffer: ArrayBuffer,
+  options?: NullLoaderOptions,
+  context?: LoaderContext
+): null {
+  return null;
+}