@loaders.gl/loader-utils 4.0.0-alpha.5 → 4.0.0-alpha.6

package/src/lib/node/fs.ts

@@ -1,9 +1,9 @@
 // fs wrapper (promisified fs + avoids bundling fs in browsers)
 import fs from 'fs';
 import {toArrayBuffer} from './buffer';
-import {promisify} from './util';
+import {promisify2, promisify3} from './promisify';
 
-export type {Stats} from 'fs';
+export type {Stats, WriteStream} from 'fs';
 
 export let readdir;
 /** Wrapper for Node.js fs method */
@@ -23,41 +23,46 @@ export let writeFileSync;
 /** Wrapper for Node.js fs method */
 export let open;
 /** Wrapper for Node.js fs method */
-export let close;
+export let close: (fd: number) => Promise<void>;
 /** Wrapper for Node.js fs method */
 export let read;
 /** Wrapper for Node.js fs method */
 export let fstat;
 
+export let createWriteStream: typeof fs.createWriteStream;
+
 export let isSupported = Boolean(fs);
 
 // paths
 
 try {
   /** Wrapper for Node.js fs method */
-  readdir = promisify(fs.readdir);
+  readdir = promisify2(fs.readdir);
   /** Wrapper for Node.js fs method */
-  stat = promisify(fs.stat);
+  stat = promisify2(fs.stat);
 
   /** Wrapper for Node.js fs method */
-  readFile = promisify(fs.readFile);
+  readFile = fs.readFile;
   /** Wrapper for Node.js fs method */
   readFileSync = fs.readFileSync;
   /** Wrapper for Node.js fs method */
-  writeFile = promisify(fs.writeFile);
+  writeFile = promisify3(fs.writeFile);
   /** Wrapper for Node.js fs method */
   writeFileSync = fs.writeFileSync;
 
   // file descriptors
 
   /** Wrapper for Node.js fs method */
-  open = promisify(fs.open);
+  open = fs.open;
   /** Wrapper for Node.js fs method */
-  close = promisify(fs.close);
+  close = (fd: number) =>
+    new Promise((resolve, reject) => fs.close(fd, (err) => (err ? reject(err) : resolve())));
   /** Wrapper for Node.js fs method */
-  read = promisify(fs.read);
+  read = fs.read;
   /** Wrapper for Node.js fs method */
-  fstat = promisify(fs.fstat);
+  fstat = fs.fstat;
+
+  createWriteStream = fs.createWriteStream;
 
   isSupported = Boolean(fs);
 } catch {

package/src/lib/node/promisify.ts

@@ -0,0 +1,70 @@
+// @loaders.gl, MIT license
+
+// type Parameter1<T extends (arg1: any, ...args: unknown[]) => unknown> = T extends (
+//   arg1: infer P,
+//   ...args: unknown[]
+// ) => unknown
+//   ? P
+//   : never;
+
+//   type Parameter2<T extends (arg1: unknown, arg2: any, ...args: unknown[]) => unknown> = T extends (
+//   arg1: unknown,
+//   arg2: infer P,
+//   ...args: unknown[]
+// ) => unknown
+//   ? P
+//   : never;
+
+// type CallbackParameter2<
+//   T extends (arg: unknown, cb: (error: any, value: any) => void) => unknown
+// > = T extends (arg: unknown, cb: (error: any, value: infer P) => void) => unknown ? P : never;
+
+// type CallbackParameter3<
+//   T extends (arg: unknown, arg2: unknown, cb: (error: any, value: any) => void) => unknown
+// > = T extends (arg: unknown, arg2: unknown, cb: (error: any, value: infer P) => void) => unknown
+//   ? P
+//   : never;
+
+// /** Extract the parameters of a function type in a tuple */
+// export type Promisified<F extends (arg1, cb: (error: unknown, value: any) => void) => any> = (
+//   arg1: Parameter1<F>
+// ) => Promise<CallbackParameter2<F>>;
+// /** Extract the parameters of a function type in a tuple */
+// export type Promisified2<F extends (arg1, arg2, cb: (error: unknown, value: any) => void) => any> = (
+//   arg1: Parameter1<F>,
+//   arg2: Parameter2<F>
+// ) => Promise<CallbackParameter3<F>>;
+
+/** Wrapper for Node.js promisify */
+type Callback<A> = (error: unknown, args: A) => void;
+
+/**
+ * Typesafe promisify implementation
+ * @link https://dev.to/_gdelgado/implement-a-type-safe-version-of-node-s-promisify-in-7-lines-of-code-in-typescript-2j34
+ * @param fn
+ * @returns
+ */
+export function promisify1<T, A>(fn: (args: T, cb: Callback<A>) => void): (args: T) => Promise<A> {
+  return (args: T) =>
+    new Promise((resolve, reject) =>
+      fn(args, (error, callbackArgs) => (error ? reject(error) : resolve(callbackArgs)))
+    );
+}
+
+export function promisify2<T1, T2, A>(
+  fn: (arg1: T1, arg2: T2, cb: Callback<A>) => void
+): (arg1: T1, arg2: T2) => Promise<A> {
+  return (arg1: T1, arg2: T2) =>
+    new Promise((resolve, reject) =>
+      fn(arg1, arg2, (error, callbackArgs) => (error ? reject(error) : resolve(callbackArgs)))
+    );
+}
+
+export function promisify3<T1, T2, T3, A>(
+  fn: (arg1: T1, arg2: T2, arg3: T3, cb: Callback<A>) => void
+): (arg1: T1, arg2: T2, arg3: T3) => Promise<A> {
+  return (arg1: T1, arg2: T2, arg3: T3) =>
+    new Promise((resolve, reject) =>
+      fn(arg1, arg2, arg3, (error, callbackArgs) => (error ? reject(error) : resolve(callbackArgs)))
+    );
+}

package/src/lib/node/stream.ts

@@ -0,0 +1,18 @@
+// loaders.gl, MIT license
+
+import stream from 'stream';
+
+export type {Writable} from 'stream';
+
+export let Transform;
+
+export const isSupported = Boolean(stream);
+
+// paths
+
+try {
+  /** Wrapper for Node.js fs method */
+  Transform = stream.Transform;
+} catch {
+  // ignore
+}

package/src/lib/option-utils/merge-loader-options.ts

@@ -0,0 +1,35 @@
+// loaders.gl, MIT license
+
+import {LoaderOptions} from '../../types';
+
+/**
+ *
+ * @param baseOptions Can be undefined, in which case a fresh options object will be minted
+ * @param newOptions
+ * @returns
+ */
+export function mergeLoaderOptions<Options extends LoaderOptions>(
+  baseOptions: Options | undefined,
+  newOptions: Options
+): Options {
+  return mergeOptionsRecursively(baseOptions || {}, newOptions) as Options;
+}
+
+function mergeOptionsRecursively(
+  baseOptions: Record<string, unknown>,
+  newOptions: Record<string, unknown>
+): Record<string, unknown> {
+  const options = {...baseOptions};
+  for (const [key, newValue] of Object.entries(newOptions)) {
+    if (newValue && typeof newValue === 'object') {
+      options[key] = mergeOptionsRecursively(
+        (options[key] as Record<string, unknown>) || {},
+        newOptions[key] as Record<string, unknown>
+      );
+      // Object.assign(options[key] as object, newOptions[key]);
+    } else {
+      options[key] = newOptions[key];
+    }
+  }
+  return options as Record<string, unknown>;
+}

package/src/lib/path-utils/get-cwd.ts

@@ -0,0 +1,9 @@
+// loaders.gl MIT license
+
+export function getCWD() {
+  if (typeof process !== 'undefined' && typeof process.cwd !== 'undefined') {
+    return process.cwd();
+  }
+  const pathname = window.location?.pathname;
+  return pathname?.slice(0, pathname.lastIndexOf('/') + 1) || '';
+}

package/src/lib/path-utils/path.ts

@@ -1,12 +1,14 @@
 // Beginning of a minimal implementation of the Node.js path API, that doesn't pull in big polyfills.
 
+import {getCWD} from './get-cwd';
+
 /**
  * Replacement for Node.js path.filename
  * @param url
  */
 export function filename(url: string): string {
-  const slashIndex = url && url.lastIndexOf('/');
-  return slashIndex >= 0 ? url.substr((slashIndex as number) + 1) : '';
+  const slashIndex = url ? url.lastIndexOf('/') : -1;
+  return slashIndex >= 0 ? url.substr(slashIndex + 1) : '';
 }
 
 /**
@@ -14,8 +16,8 @@ export function filename(url: string): string {
  * @param url
  */
 export function dirname(url: string): string {
-  const slashIndex = url && url.lastIndexOf('/');
-  return slashIndex >= 0 ? url.substr(0, slashIndex as number) : '';
+  const slashIndex = url ? url.lastIndexOf('/') : -1;
+  return slashIndex >= 0 ? url.substr(0, slashIndex) : '';
 }
 
 /**
@@ -35,3 +37,135 @@ export function join(...parts: string[]): string {
   });
   return parts.join(separator);
 }
+
+/* eslint-disable no-continue */
+
+/**
+ * https://nodejs.org/api/path.html#path_path_resolve_paths
+ * @param paths A sequence of paths or path segments.
+ * @return resolved path
+ * Forked from BTOdell/path-resolve under MIT license
+ * @see https://github.com/BTOdell/path-resolve/blob/master/LICENSE
+ */
+export function resolve(...components: string[]): string {
+  const paths: string[] = [];
+  for (let _i = 0; _i < components.length; _i++) {
+    paths[_i] = components[_i];
+  }
+  let resolvedPath = '';
+  let resolvedAbsolute = false;
+  let cwd: string | undefined;
+  for (let i = paths.length - 1; i >= -1 && !resolvedAbsolute; i--) {
+    let path: string | undefined;
+    if (i >= 0) {
+      path = paths[i];
+    } else {
+      if (cwd === undefined) {
+        cwd = getCWD();
+      }
+      path = cwd;
+    }
+    // Skip empty entries
+    if (path.length === 0) {
+      continue;
+    }
+    resolvedPath = `${path}/${resolvedPath}`;
+    resolvedAbsolute = path.charCodeAt(0) === SLASH;
+  }
+  // At this point the path should be resolved to a full absolute path, but
+  // handle relative paths to be safe (might happen when process.cwd() fails)
+  // Normalize the path (removes leading slash)
+  resolvedPath = normalizeStringPosix(resolvedPath, !resolvedAbsolute);
+  if (resolvedAbsolute) {
+    return `/${resolvedPath}`;
+  } else if (resolvedPath.length > 0) {
+    return resolvedPath;
+  }
+  return '.';
+}
+
+const SLASH = 47;
+const DOT = 46;
+
+/**
+ * Resolves . and .. elements in a path with directory names
+ * Forked from BTOdell/path-resolve under MIT license
+ * @see https://github.com/BTOdell/path-resolve/blob/master/LICENSE
+ */
+/* eslint-disable max-depth */
+// eslint-disable-next-line complexity, max-statements
+function normalizeStringPosix(path: string, allowAboveRoot: boolean): string {
+  let res = '';
+  let lastSlash = -1;
+  let dots = 0;
+  let code: number | undefined;
+  let isAboveRoot = false;
+
+  for (let i = 0; i <= path.length; ++i) {
+    if (i < path.length) {
+      code = path.charCodeAt(i);
+    } else if (code === SLASH) {
+      break;
+    } else {
+      code = SLASH;
+    }
+    if (code === SLASH) {
+      if (lastSlash === i - 1 || dots === 1) {
+        // NOOP
+      } else if (lastSlash !== i - 1 && dots === 2) {
+        if (
+          res.length < 2 ||
+          !isAboveRoot ||
+          res.charCodeAt(res.length - 1) !== DOT ||
+          res.charCodeAt(res.length - 2) !== DOT
+        ) {
+          if (res.length > 2) {
+            const start = res.length - 1;
+            let j = start;
+            for (; j >= 0; --j) {
+              if (res.charCodeAt(j) === SLASH) {
+                break;
+              }
+            }
+            if (j !== start) {
+              res = j === -1 ? '' : res.slice(0, j);
+              lastSlash = i;
+              dots = 0;
+              isAboveRoot = false;
+              continue;
+            }
+          } else if (res.length === 2 || res.length === 1) {
+            res = '';
+            lastSlash = i;
+            dots = 0;
+            isAboveRoot = false;
+            continue;
+          }
+        }
+        if (allowAboveRoot) {
+          if (res.length > 0) {
+            res += '/..';
+          } else {
+            res = '..';
+          }
+          isAboveRoot = true;
+        }
+      } else {
+        const slice = path.slice(lastSlash + 1, i);
+        if (res.length > 0) {
+          res += `/${slice}`;
+        } else {
+          res = slice;
+        }
+        isAboveRoot = false;
+      }
+      lastSlash = i;
+      dots = 0;
+    } else if (code === DOT && dots !== -1) {
+      ++dots;
+    } else {
+      dots = -1;
+    }
+  }
+  return res;
+}

package/src/lib/worker-loader-utils/create-loader-worker.ts

@@ -11,7 +11,7 @@ let requestId = 0;
  */
 export function createLoaderWorker(loader: LoaderWithParser) {
   // Check that we are actually in a worker thread
-  if (typeof self === 'undefined') {
+  if (!WorkerBody.inWorkerThread()) {
     return;
   }
 
@@ -21,13 +21,14 @@ export function createLoaderWorker(loader: LoaderWithParser) {
         try {
           // validateLoaderVersion(loader, data.source.split('@')[1]);
 
-          const {input, options = {}} = payload;
+          const {input, options = {}, context = {}} = payload;
 
           const result = await parseData({
             loader,
             arrayBuffer: input,
             options,
             context: {
+              ...context,
               parse: parseOnMainThread
             }
           });

package/src/lib/worker-loader-utils/encode-with-worker.ts

@@ -0,0 +1,21 @@
+import {WorkerFarm} from '@loaders.gl/worker-utils';
+import {Writer, WriterOptions} from '../../types';
+import {isBrowser} from '../env-utils/globals';
+
+/**
+ * Determines if a loader can parse with worker
+ * @param loader
+ * @param options
+ */
+export function canEncodeWithWorker(writer: Writer, options?: WriterOptions) {
+  if (!WorkerFarm.isSupported()) {
+    return false;
+  }
+
+  // Node workers are still experimental
+  if (!isBrowser && !options?._nodeWorkers) {
+    return false;
+  }
+
+  return writer.worker && options?.worker;
+}

package/src/lib/worker-loader-utils/parse-with-worker.ts

@@ -1,4 +1,9 @@
-import type {WorkerJob, WorkerMessageType, WorkerMessagePayload} from '@loaders.gl/worker-utils';
+import {
+  WorkerJob,
+  WorkerMessageType,
+  WorkerMessagePayload,
+  isBrowser
+} from '@loaders.gl/worker-utils';
 import type {Loader, LoaderOptions, LoaderContext} from '../../types';
 import {WorkerFarm, getWorkerURL} from '@loaders.gl/worker-utils';
 
@@ -12,6 +17,11 @@ export function canParseWithWorker(loader: Loader, options?: LoaderOptions) {
     return false;
   }
 
+  // Node workers are still experimental
+  if (!isBrowser && !options?._nodeWorkers) {
+    return false;
+  }
+
   return loader.worker && options?.worker;
 }
 
@@ -33,8 +43,10 @@ export async function parseWithWorker(
   const workerPool = workerFarm.getWorkerPool({name, url});
 
   // options.log object contains functions which cannot be transferred
+  // context.fetch & context.parse functions cannot be transferred
   // TODO - decide how to handle logging on workers
   options = JSON.parse(JSON.stringify(options));
+  context = JSON.parse(JSON.stringify(context || {}));
 
   const job = await workerPool.startJob(
     'process-on-worker',
@@ -45,7 +57,8 @@ export async function parseWithWorker(
   job.postMessage('process', {
     // @ts-ignore
     input: data,
-    options
+    options,
+    context
   });
 
   const result = await job.result;