@loaders.gl/loader-utils 4.0.0-alpha.1 → 4.0.0-alpha.11

package/src/lib/path-utils/path.ts

@@ -1,12 +1,14 @@
 // Beginning of a minimal implementation of the Node.js path API, that doesn't pull in big polyfills.
 
+import {getCWD} from './get-cwd';
+
 /**
  * Replacement for Node.js path.filename
  * @param url
  */
 export function filename(url: string): string {
-  const slashIndex = url && url.lastIndexOf('/');
-  return slashIndex >= 0 ? url.substr((slashIndex as number) + 1) : '';
+  const slashIndex = url ? url.lastIndexOf('/') : -1;
+  return slashIndex >= 0 ? url.substr(slashIndex + 1) : '';
 }
 
 /**
@@ -14,8 +16,8 @@ export function filename(url: string): string {
  * @param url
  */
 export function dirname(url: string): string {
-  const slashIndex = url && url.lastIndexOf('/');
-  return slashIndex >= 0 ? url.substr(0, slashIndex as number) : '';
+  const slashIndex = url ? url.lastIndexOf('/') : -1;
+  return slashIndex >= 0 ? url.substr(0, slashIndex) : '';
 }
 
 /**
@@ -35,3 +37,135 @@ export function join(...parts: string[]): string {
   });
   return parts.join(separator);
 }
+
+/* eslint-disable no-continue */
+
+/**
+ * https://nodejs.org/api/path.html#path_path_resolve_paths
+ * @param paths A sequence of paths or path segments.
+ * @return resolved path
+ * Forked from BTOdell/path-resolve under MIT license
+ * @see https://github.com/BTOdell/path-resolve/blob/master/LICENSE
+ */
+export function resolve(...components: string[]): string {
+  const paths: string[] = [];
+  for (let _i = 0; _i < components.length; _i++) {
+    paths[_i] = components[_i];
+  }
+  let resolvedPath = '';
+  let resolvedAbsolute = false;
+  let cwd: string | undefined;
+  for (let i = paths.length - 1; i >= -1 && !resolvedAbsolute; i--) {
+    let path: string | undefined;
+    if (i >= 0) {
+      path = paths[i];
+    } else {
+      if (cwd === undefined) {
+        cwd = getCWD();
+      }
+      path = cwd;
+    }
+    // Skip empty entries
+    if (path.length === 0) {
+      continue;
+    }
+    resolvedPath = `${path}/${resolvedPath}`;
+    resolvedAbsolute = path.charCodeAt(0) === SLASH;
+  }
+  // At this point the path should be resolved to a full absolute path, but
+  // handle relative paths to be safe (might happen when process.cwd() fails)
+  // Normalize the path (removes leading slash)
+  resolvedPath = normalizeStringPosix(resolvedPath, !resolvedAbsolute);
+  if (resolvedAbsolute) {
+    return `/${resolvedPath}`;
+  } else if (resolvedPath.length > 0) {
+    return resolvedPath;
+  }
+  return '.';
+}
+
+const SLASH = 47;
+const DOT = 46;
+
+/**
+ * Resolves . and .. elements in a path with directory names
+ * Forked from BTOdell/path-resolve under MIT license
+ * @see https://github.com/BTOdell/path-resolve/blob/master/LICENSE
+ */
+/* eslint-disable max-depth */
+// eslint-disable-next-line complexity, max-statements
+function normalizeStringPosix(path: string, allowAboveRoot: boolean): string {
+  let res = '';
+  let lastSlash = -1;
+  let dots = 0;
+  let code: number | undefined;
+  let isAboveRoot = false;
+
+  for (let i = 0; i <= path.length; ++i) {
+    if (i < path.length) {
+      code = path.charCodeAt(i);
+    } else if (code === SLASH) {
+      break;
+    } else {
+      code = SLASH;
+    }
+    if (code === SLASH) {
+      if (lastSlash === i - 1 || dots === 1) {
+        // NOOP
+      } else if (lastSlash !== i - 1 && dots === 2) {
+        if (
+          res.length < 2 ||
+          !isAboveRoot ||
+          res.charCodeAt(res.length - 1) !== DOT ||
+          res.charCodeAt(res.length - 2) !== DOT
+        ) {
+          if (res.length > 2) {
+            const start = res.length - 1;
+            let j = start;
+            for (; j >= 0; --j) {
+              if (res.charCodeAt(j) === SLASH) {
+                break;
+              }
+            }
+            if (j !== start) {
+              res = j === -1 ? '' : res.slice(0, j);
+              lastSlash = i;
+              dots = 0;
+              isAboveRoot = false;
+              continue;
+            }
+          } else if (res.length === 2 || res.length === 1) {
+            res = '';
+            lastSlash = i;
+            dots = 0;
+            isAboveRoot = false;
+            continue;
+          }
+        }
+        if (allowAboveRoot) {
+          if (res.length > 0) {
+            res += '/..';
+          } else {
+            res = '..';
+          }
+          isAboveRoot = true;
+        }
+      } else {
+        const slice = path.slice(lastSlash + 1, i);
+        if (res.length > 0) {
+          res += `/${slice}`;
+        } else {
+          res = slice;
+        }
+        isAboveRoot = false;
+      }
+      lastSlash = i;
+      dots = 0;
+    } else if (code === DOT && dots !== -1) {
+      ++dots;
+    } else {
+      dots = -1;
+    }
+  }
+  return res;
+}

package/src/lib/worker-loader-utils/create-loader-worker.ts

@@ -11,7 +11,7 @@ let requestId = 0;
  */
 export function createLoaderWorker(loader: LoaderWithParser) {
   // Check that we are actually in a worker thread
-  if (typeof self === 'undefined') {
+  if (!WorkerBody.inWorkerThread()) {
     return;
   }
 
@@ -21,13 +21,14 @@ export function createLoaderWorker(loader: LoaderWithParser) {
         try {
           // validateLoaderVersion(loader, data.source.split('@')[1]);
 
-          const {input, options = {}} = payload;
+          const {input, options = {}, context = {}} = payload;
 
           const result = await parseData({
             loader,
             arrayBuffer: input,
             options,
             context: {
+              ...context,
               parse: parseOnMainThread
             }
           });

package/src/lib/worker-loader-utils/encode-with-worker.ts

@@ -0,0 +1,21 @@
+import {WorkerFarm} from '@loaders.gl/worker-utils';
+import {Writer, WriterOptions} from '../../types';
+import {isBrowser} from '../env-utils/globals';
+
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+export function canEncodeWithWorker(writer: Writer, options?: WriterOptions) {
+  if (!WorkerFarm.isSupported()) {
+    return false;
+  }
+
+  // Node workers are still experimental
+  if (!isBrowser && !options?._nodeWorkers) {
+    return false;
+  }
+
+  return writer.worker && options?.worker;
+}

package/src/lib/worker-loader-utils/parse-with-worker.ts

@@ -1,4 +1,9 @@
-import type {WorkerJob, WorkerMessageType, WorkerMessagePayload} from '@loaders.gl/worker-utils';
+import {
+  WorkerJob,
+  WorkerMessageType,
+  WorkerMessagePayload,
+  isBrowser
+} from '@loaders.gl/worker-utils';
 import type {Loader, LoaderOptions, LoaderContext} from '../../types';
 import {WorkerFarm, getWorkerURL} from '@loaders.gl/worker-utils';
 
@@ -12,6 +17,11 @@ export function canParseWithWorker(loader: Loader, options?: LoaderOptions) {
     return false;
   }
 
+  // Node workers are still experimental
+  if (!isBrowser && !options?._nodeWorkers) {
+    return false;
+  }
+
   return loader.worker && options?.worker;
 }
 
@@ -33,19 +43,22 @@ export async function parseWithWorker(
   const workerPool = workerFarm.getWorkerPool({name, url});
 
   // options.log object contains functions which cannot be transferred
+  // context.fetch & context.parse functions cannot be transferred
   // TODO - decide how to handle logging on workers
   options = JSON.parse(JSON.stringify(options));
+  context = JSON.parse(JSON.stringify(context || {}));
 
   const job = await workerPool.startJob(
     'process-on-worker',
     // @ts-expect-error
-    onMessage.bind(null, parseOnMainThread) // eslint-disable-this-line
+    onMessage.bind(null, parseOnMainThread) // eslint-disable-line @typescript-eslint/no-misused-promises
   );
 
   job.postMessage('process', {
     // @ts-ignore
     input: data,
-    options
+    options,
+    context
   });
 
   const result = await job.result;

package/src/types.ts

@@ -15,11 +15,37 @@ export type TypedFloatArray = Uint16Array | Float32Array | Float64Array;
 
 export type TypedArray = TypedIntArray | TypedFloatArray;
 
-export type NumericArray = Array<number> | TypedIntArray | TypedFloatArray;
+export type BigTypedArray = TypedArray | BigInt64Array | BigUint64Array;
+
+export type TypedArrayConstructor =
+  | Int8ArrayConstructor
+  | Uint8ArrayConstructor
+  | Int16ArrayConstructor
+  | Uint16ArrayConstructor
+  | Int32ArrayConstructor
+  | Uint32ArrayConstructor
+  | Int32ArrayConstructor
+  | Uint32ArrayConstructor
+  | Float32ArrayConstructor
+  | Float64ArrayConstructor;
+
+export type BigTypedArrayConstructor =
+  | TypedArrayConstructor
+  | BigInt64ArrayConstructor
+  | BigUint64ArrayConstructor;
+
+/** Any numeric array: typed array or `number[]` */
+export type NumberArray = number[] | TypedArray;
+
+export type NumericArray = number[] | TypedArray;
 
 type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
 
-// Core Loader Options
+// LOADERS
+
+/**
+ * Core Loader Options
+ */
 export type LoaderOptions = {
   /** fetch options or a custom fetch function */
   fetch?: typeof fetch | FetchLike | RequestInit | null;
@@ -55,7 +81,7 @@ export type LoaderOptions = {
   // workers
 
   /** CDN load workers from */
-  CDN?: string;
+  CDN?: string | null;
   /** Set to `false` to disable workers */
   worker?: boolean;
   /** Number of concurrent workers (per loader) on desktop browser */
@@ -64,7 +90,9 @@ export type LoaderOptions = {
   maxMobileConcurrency?: number;
   /** Set to `false` to prevent reuse workers */
   reuseWorkers?: boolean;
-  /** set to 'test' to run worker type */
+  /** Whether to use workers under Node.js (experimental) */
+  _nodeWorkers?: boolean;
+  /** set to 'test' to run local worker */
   _workerType?: string;
 
   /** @deprecated `options.batchType` removed, Use `options.<loader>.type` instead */
@@ -72,175 +100,265 @@ export type LoaderOptions = {
   /** @deprecated `options.throw removed`, Use `options.nothrow` instead */
   throws?: boolean;
   /** @deprecated `options.dataType` no longer used */
-  dataType?: any;
+  dataType?: never;
   /** @deprecated `options.uri` no longer used */
-  uri?: any;
+  uri?: never;
   /** @deprecated `options.method` removed. Use `options.fetch.method` */
-  method?: any;
+  method?: never;
   /** @deprecated `options.headers` removed. Use `options.fetch.headers` */
-  headers?: any;
+  headers?: never;
   /** @deprecated `options.body` removed. Use `options.fetch.body` */
-  body?: any;
+  body?: never;
   /** @deprecated `options.mode` removed. Use `options.fetch.mode` */
-  mode?: any;
+  mode?: never;
   /** @deprecated `options.credentials` removed. Use `options.fetch.credentials` */
-  credentials?: any;
+  credentials?: never;
   /** @deprecated `options.cache` removed. Use `options.fetch.cache` */
-  cache?: any;
+  cache?: never;
   /** @deprecated `options.redirect` removed. Use `options.fetch.redirect` */
-  redirect?: any;
+  redirect?: never;
   /** @deprecated `options.referrer` removed. Use `options.fetch.referrer` */
-  referrer?: any;
+  referrer?: never;
   /** @deprecated `options.referrerPolicy` removed. Use `options.fetch.referrerPolicy` */
-  referrerPolicy?: any;
+  referrerPolicy?: never;
   /** @deprecated `options.integrity` removed. Use `options.fetch.integrity` */
-  integrity?: any;
+  integrity?: never;
   /** @deprecated `options.keepalive` removed. Use `options.fetch.keepalive` */
-  keepalive?: any;
+  keepalive?: never;
   /** @deprecated `options.signal` removed. Use `options.fetch.signal` */
-  signal?: any;
+  signal?: never;
 
   // Accept other keys (loader options objects, e.g. `options.csv`, `options.json` ...)
-  [loaderId: string]: any;
+  [loaderId: string]: unknown;
 };
 
 type PreloadOptions = {
-  [key: string]: any;
+  [key: string]: unknown;
 };
 
 /**
  * A worker loader definition that can be used with `@loaders.gl/core` functions
  */
-export type Loader = {
+export type Loader<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> = {
+  // Types
+  dataType?: DataT;
+  batchType?: BatchT;
+
+  /** Default Options */
+  options: LoaderOptionsT;
+  /** Deprecated Options */
+  deprecatedOptions?: Record<string, string | Record<string, string>>;
+
   // Worker
   name: string;
+  /** id should be the same as the field used in LoaderOptions */
   id: string;
+  /** module is used to generate worker threads, need to be the module directory name */
   module: string;
+  /** Version should be injected by build tools */
   version: string;
+  /** A boolean, or a URL */
   worker?: string | boolean;
-  options: object;
-  deprecatedOptions?: object;
   // end Worker
 
+  /** Which category does this loader belong to */
   category?: string;
+  /** What extensions does this loader generate */
   extensions: string[];
   mimeTypes: string[];
 
   binary?: boolean;
   text?: boolean;
 
-  tests?: (((ArrayBuffer) => boolean) | ArrayBuffer | string)[];
+  tests?: (((ArrayBuffer: ArrayBuffer) => boolean) | ArrayBuffer | string)[];
 
   // TODO - deprecated
   supported?: boolean;
-  testText?: (string) => boolean;
+  testText?: (string: string) => boolean;
 };
 
 /**
  * A "bundled" loader definition that can be used with `@loaders.gl/core` functions
  * If a worker loader is supported it will also be supported.
  */
-export type LoaderWithParser = Loader & {
+export type LoaderWithParser<DataT = any, BatchT = any, LoaderOptionsT = LoaderOptions> = Loader<
+  DataT,
+  BatchT,
+  LoaderOptionsT
+> & {
   // TODO - deprecated
-  testText?: (string) => boolean;
+  testText?: (string: string) => boolean;
 
-  parse: Parse;
+  parse: (
+    arrayBuffer: ArrayBuffer,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => Promise<DataT>;
   preload?: Preload;
-  parseSync?: ParseSync;
-  parseText?: ParseText;
-  parseTextSync?: ParseTextSync;
-  parseInBatches?: ParseInBatches;
-  parseFileInBatches?: ParseFileInBatches;
+  parseSync?: (
+    arrayBuffer: ArrayBuffer,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => DataT;
+  parseText?: (text: string, options?: LoaderOptionsT) => Promise<DataT>;
+  parseTextSync?: (text: string, options?: LoaderOptionsT) => DataT;
+  parseInBatches?: (
+    iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => AsyncIterable<BatchT>;
+  parseFileInBatches?: (
+    file: Blob,
+    options?: LoaderOptionsT,
+    context?: LoaderContext
+  ) => AsyncIterable<BatchT>;
 };
 
-/** Options for writers */
-export type WriterOptions = {};
-
 /**
- * A writer definition that can be used with `@loaders.gl/core` functions
+ * A Loader context is provided as a third parameters to a loader object's
+ * parse functions when that loader is called by other loaders rather then
+ * directly by the application.
+ *
+ * - The context object allows the subloaders to be aware of the parameters and
+ *   options that the application provided in the top level call.
+ * - The context also providedsaccess to parse functions so that the subloader
+ *   does not need to include the core module.
+ * - In addition, the context's parse functions may also redirect loads from worker
+ *   threads back to main thread.
  */
-export type Writer = {
-  name: string;
-
-  id: string;
-  module: string;
-  version: string;
-
-  options: object;
-  deprecatedOptions?: object;
-
-  // TODO - are these are needed?
-  binary?: boolean;
-  extensions?: string[];
-  mimeTypes?: string[];
-  text?: boolean;
-
-  encode?: Encode;
-  encodeSync?: EncodeSync;
-  encodeInBatches?: EncodeInBatches;
-  encodeURLtoURL?: EncodeURLtoURL;
-  encodeText?: EncodeText;
-};
-
 export type LoaderContext = {
   loaders?: Loader[] | null;
+  /** If URL is available.  */
   url?: string;
-
-  fetch: typeof fetch;
+  /** the file name component of the URL (leading path and query string removed) */
+  filename?: string;
+  /** the directory name component of the URL (leading path excluding file name and query string) */
+  baseUrl?: string;
+  /** Query string (characters after `?`) */
+  queryString?: string;
+
+  /** Provides access to any application overrides of fetch() */
+  fetch: typeof fetch | FetchLike;
+  /** TBD */
+  response?: Response;
+  /** Parse function. Use instead of importing `core`. In workers, may redirect to main thread */
   parse: (
     arrayBuffer: ArrayBuffer,
-    loaders?,
+    loaders?: Loader | Loader[] | LoaderOptions,
     options?: LoaderOptions,
     context?: LoaderContext
   ) => Promise<any>;
+  /** ParseSync function. Use instead of importing `core`. In workers, may redirect to main thread */
   parseSync?: (
     arrayBuffer: ArrayBuffer,
-    loaders?,
+    loaders?: Loader | Loader[] | LoaderOptions,
     options?: LoaderOptions,
     context?: LoaderContext
   ) => any;
+  /** ParseInBatches function. Use instead of importing `core`.  */
   parseInBatches?: (
     iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
-    loaders?,
+    loaders?: Loader | Loader[] | LoaderOptions,
     options?: LoaderOptions,
     context?: LoaderContext
   ) => AsyncIterable<any> | Promise<AsyncIterable<any>>;
 };
 
-type Parse = (
-  arrayBuffer: ArrayBuffer,
-  options?: LoaderOptions,
-  context?: LoaderContext
-) => Promise<any>;
-type ParseSync = (
-  arrayBuffer: ArrayBuffer,
-  options?: LoaderOptions,
-  context?: LoaderContext
-) => any;
-type ParseText = (text: string, options?: LoaderOptions) => Promise<any>;
-type ParseTextSync = (text: string, options?: LoaderOptions) => any;
-type ParseInBatches = (
-  iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
-  options?: LoaderOptions,
-  context?: LoaderContext
-) => AsyncIterable<any>;
-type ParseFileInBatches = (
-  file: Blob,
-  options?: LoaderOptions,
-  context?: LoaderContext
-) => AsyncIterable<any>;
-
-type Encode = (data: any, options?: WriterOptions) => Promise<ArrayBuffer>;
+// type Parse = (
+//   arrayBuffer: ArrayBuffer,
+//   options?: LoaderOptions,
+//   context?: LoaderContext
+// ) => Promise<any>;
+// type ParseSync = (
+//   arrayBuffer: ArrayBuffer,
+//   options?: LoaderOptions,
+//   context?: LoaderContext
+// ) => any;
+// type ParseText = (text: string, options?: LoaderOptions) => Promise<any>;
+// type ParseTextSync = (text: string, options?: LoaderOptions) => any;
+// type ParseInBatches = (
+//   iterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+//   options?: LoaderOptions,
+//   context?: LoaderContext
+// ) => AsyncIterable<any>;
+// type ParseFileInBatches = (
+//   file: Blob,
+//   options?: LoaderOptions,
+//   context?: LoaderContext
+// ) => AsyncIterable<any>;
+
+type Preload = (url: string, options?: PreloadOptions) => any;
+
+/** Typescript helper to extract options type from a generic loader type */
+export type LoaderOptionsType<T = Loader> = T extends Loader<any, any, infer Options>
+  ? Options
+  : never;
+/** Typescript helper to extract data type from a generic loader type */
+export type LoaderReturnType<T = Loader> = T extends Loader<infer Return, any, any>
+  ? Return
+  : never;
+/** Typescript helper to extract batch type from a generic loader type */
+export type LoaderBatchType<T = Loader> = T extends Loader<any, infer Batch, any> ? Batch : never;
+
+// WRITERS
+
+/** Options for writers */
+export type WriterOptions = {
+  /** worker source. If is set will be used instead of loading worker from the Internet */
+  souce?: string | null;
+  /** writer-specific options */
+  [writerId: string]: any;
+};
+
+/**
+ * A writer definition that can be used with `@loaders.gl/core` functions
+ */
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export type Writer<DataT = unknown, BatchT = unknown, WriterOptionsT = WriterOptions> = {
+  name: string;
+
+  id: string;
+  module: string;
+  version: string;
+  worker?: string | boolean;
+
+  // TODO - are these are needed?
+  extensions?: string[];
+  mimeTypes?: string[];
+  binary?: boolean;
+  text?: boolean;
+
+  options: WriterOptionsT;
+  deprecatedOptions?: Record<string, string>;
+
+  // encodeText?: EncodeText;
+  // encode?: Encode;
+  encodeSync?: EncodeSync;
+  // encodeInBatches?: EncodeInBatches;
+  encodeURLtoURL?: EncodeURLtoURL;
+
+  encode?(data: DataT, options?: WriterOptionsT): Promise<ArrayBuffer>;
+  encodeText?(table: DataT, options?: WriterOptionsT): Promise<string> | string;
+  encodeInBatches?(data: AsyncIterable<any>, options?: WriterOptionsT): AsyncIterable<ArrayBuffer>;
+};
+
+//  type Encode = (data: any, options?: WriterOptions) => Promise<ArrayBuffer>;
 type EncodeSync = (data: any, options?: WriterOptions) => ArrayBuffer;
 // TODO
-type EncodeText = Function;
-type EncodeInBatches = Function;
+// type EncodeText = Function;
+// type EncodeInBatches = Function;
 type EncodeURLtoURL = (
   inputUrl: string,
   outputUrl: string,
   options?: WriterOptions
 ) => Promise<string>;
-type Preload = (url: string, options?: PreloadOptions) => any;
+
+/** Typescript helper to extract the writer options type from a generic writer type */
+export type WriterOptionsType<T = Writer> = T extends Writer<unknown, unknown, infer Options>
+  ? Options
+  : never;
+
+// MISC TYPES
 
 export type TransformBatches = (
   asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
@@ -295,7 +413,7 @@ export interface IFileSystem {
 
 type ReadOptions = {buffer?: ArrayBuffer; offset?: number; length?: number; position?: number};
 export interface IRandomAccessReadFileSystem extends IFileSystem {
-  open(path: string, flags, mode?): Promise<any>;
+  open(path: string, flags: string | number, mode?: any): Promise<any>;
   close(fd: any): Promise<void>;
   fstat(fd: any): Promise<object>;
   read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Buffer}>;