@loaders.gl/loader-utils 4.0.0-alpha.5 → 4.0.0-alpha.7

package/src/lib/binary-utils/{binary-copy-utils.ts → dataview-copy-utils.ts}

@@ -1,13 +1,65 @@
+// loaders./gl, MIT license
+
 import {TypedArray} from '../../types';
 import {padToNBytes} from './memory-copy-utils';
 
+/**
+ * Helper function that pads a string with spaces to fit a certain byte alignment
+ * @param string
+ * @param byteAlignment
+ * @returns
+ *
+ * @todo PERFORMANCE IDEA: No need to copy string twice...
+ */
+export function padStringToByteAlignment(string: string, byteAlignment: number): string {
+  const length = string.length;
+  const paddedLength = Math.ceil(length / byteAlignment) * byteAlignment; // Round up to the required alignment
+  const padding = paddedLength - length;
+  let whitespace = '';
+  for (let i = 0; i < padding; ++i) {
+    whitespace += ' ';
+  }
+  return string + whitespace;
+}
+
+/**
+ *
+ * @param dataView
+ * @param byteOffset
+ * @param string
+ * @param byteLength
+ * @returns
+ */
+export function copyStringToDataView(
+  dataView: DataView,
+  byteOffset: number,
+  string: string,
+  byteLength: number
+): number {
+  if (dataView) {
+    for (let i = 0; i < byteLength; i++) {
+      dataView.setUint8(byteOffset + i, string.charCodeAt(i));
+    }
+  }
+  return byteOffset + byteLength;
+}
+
+export function copyBinaryToDataView(dataView, byteOffset, binary, byteLength) {
+  if (dataView) {
+    for (let i = 0; i < byteLength; i++) {
+      dataView.setUint8(byteOffset + i, binary[i]);
+    }
+  }
+  return byteOffset + byteLength;
+}
+
 /**
  * Copy sourceBuffer to dataView with some padding
  *
- * @param {DataView | null} dataView - destination data container. If null - only new offset is calculated
- * @param {number} byteOffset - destination byte offset to copy to
- * @param {Array | TypedArray} sourceBuffer - source data buffer
- * @param {number} padding - pad the resulting array to multiple of "padding" bytes. Additional bytes are filled with 0x20 (ASCII space)
+ * @param dataView - destination data container. If null - only new offset is calculated
+ * @param byteOffset - destination byte offset to copy to
+ * @param sourceBuffer - source data buffer
+ * @param padding - pad the resulting array to multiple of "padding" bytes. Additional bytes are filled with 0x20 (ASCII space)
  *
  * @return new byteOffset of resulting dataView
  */
@@ -16,7 +68,7 @@ export function copyPaddedArrayBufferToDataView(
   byteOffset: number,
   sourceBuffer: TypedArray,
   padding: number
-) {
+): number {
   const paddedLength = padToNBytes(sourceBuffer.byteLength, padding);
   const padLength = paddedLength - sourceBuffer.byteLength;
 

package/src/lib/binary-utils/get-first-characters.ts

@@ -1,4 +1,12 @@
-export function getFirstCharacters(data, length = 5): string {
+// loaders.gl, MIT license
+
+/**
+ * Get the first characters from a binary file (interpret the first bytes as an ASCII string)
+ * @param data
+ * @param length
+ * @returns
+ */
+export function getFirstCharacters(data: string | ArrayBuffer, length: number = 5): string {
   if (typeof data === 'string') {
     return data.slice(0, length);
   } else if (ArrayBuffer.isView(data)) {
@@ -11,6 +19,14 @@ export function getFirstCharacters(data, length = 5): string {
   return '';
 }
 
+/**
+ * Gets a magic string from a "file"
+ * Typically used to check or detect file format
+ * @param arrayBuffer
+ * @param byteOffset
+ * @param length
+ * @returns
+ */
 export function getMagicString(
   arrayBuffer: ArrayBuffer,
   byteOffset: number,

package/src/lib/binary-utils/memory-conversion-utils.ts

@@ -0,0 +1,53 @@
+// loaders.gl, MIT license
+
+import * as node from '../node/buffer';
+
+/**
+ * Check for Node.js `Buffer` (without triggering bundler to include Buffer polyfill on browser)
+ */
+export function isBuffer(value: any): boolean {
+  return value && typeof value === 'object' && value.isBuffer;
+}
+
+/**
+ * Converts to Node.js `Buffer` (without triggering bundler to include Buffer polyfill on browser)
+ * @todo better data type
+ */
+export function toBuffer(data: any): Buffer {
+  return node.toBuffer ? node.toBuffer(data) : data;
+}
+
+/**
+ * Convert an object to an array buffer
+ */
+export function toArrayBuffer(data: unknown): ArrayBuffer {
+  // Note: Should be called first, Buffers can trigger other detections below
+  if (isBuffer(data)) {
+    return node.toArrayBuffer(data);
+  }
+
+  if (data instanceof ArrayBuffer) {
+    return data;
+  }
+
+  // Careful - Node Buffers look like Uint8Arrays (keep after isBuffer)
+  if (ArrayBuffer.isView(data)) {
+    if (data.byteOffset === 0 && data.byteLength === data.buffer.byteLength) {
+      return data.buffer;
+    }
+    return data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
+  }
+
+  if (typeof data === 'string') {
+    const text = data;
+    const uint8Array = new TextEncoder().encode(text);
+    return uint8Array.buffer;
+  }
+
+  // HACK to support Blob polyfill
+  if (data && typeof data === 'object' && (data as any)._toArrayBuffer) {
+    return (data as any)._toArrayBuffer();
+  }
+
+  throw new Error('toArrayBuffer');
+}

package/src/lib/filesystems/readable-file.ts

@@ -0,0 +1,30 @@
+// loaders.gl, MIT license
+
+export type ReadableFile = {
+  read: (position: number, length: number) => Promise<Buffer>;
+  close: () => Promise<void>;
+  /** Length of file in bytes */
+  size: number;
+};
+
+/** Helper function to create an envelope reader for a binary memory input */
+export function makeReadableFile(data: Blob | ArrayBuffer): ReadableFile {
+  if (data instanceof ArrayBuffer) {
+    const arrayBuffer: ArrayBuffer = data;
+    return {
+      read: async (start: number, length: number) => Buffer.from(data, start, length),
+      close: async () => {},
+      size: arrayBuffer.byteLength
+    };
+  }
+
+  const blob: Blob = data;
+  return {
+    read: async (start: number, length: number) => {
+      const arrayBuffer = await blob.slice(start, start + length).arrayBuffer();
+      return Buffer.from(arrayBuffer);
+    },
+    close: async () => {},
+    size: blob.size
+  };
+}

package/src/lib/filesystems/writable-file.ts

@@ -0,0 +1,44 @@
+// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
+import {isBrowser} from '../env-utils/globals';
+import * as fs from '../node/fs';
+import type {Writable} from 'stream';
+
+export type WritableFile = {
+  write: (buf: Buffer) => Promise<void>;
+  close: () => Promise<void>;
+};
+
+export interface WriteStreamOptions {
+  flags?: string;
+  encoding?: 'utf8';
+  fd?: number;
+  mode?: number;
+  autoClose?: boolean;
+  start?: number;
+}
+
+/** Helper function to create an envelope reader for a binary memory input */
+export function makeWritableFile(
+  pathOrStream: string | Writable,
+  options?: WriteStreamOptions
+): WritableFile {
+  if (isBrowser) {
+    return {
+      write: async () => {},
+      close: async () => {}
+    };
+  }
+
+  const outputStream: Writable =
+    typeof pathOrStream === 'string' ? fs.createWriteStream(pathOrStream, options) : pathOrStream;
+  return {
+    write: async (buffer: Buffer) =>
+      new Promise((resolve, reject) => {
+        outputStream.write(buffer, (err) => (err ? reject(err) : resolve()));
+      }),
+    close: () =>
+      new Promise((resolve, reject) => {
+        (outputStream as any).close((err) => (err ? reject(err) : resolve()));
+      })
+  };
+}

package/src/lib/node/buffer.browser.ts

@@ -0,0 +1,20 @@
+// loaders.gl, MIT license
+
+// Isolates Buffer references to ensure they are only bundled under Node.js (avoids big webpack polyfill)
+// this file is selected by the package.json "browser" field).
+
+/**
+ * Convert Buffer to ArrayBuffer
+ * Converts Node.js `Buffer` to `ArrayBuffer` (without triggering bundler to include Buffer polyfill on browser)
+ * @todo better data type
+ */
+export function toArrayBuffer(buffer) {
+  return buffer;
+}
+
+/**
+ * Convert (copy) ArrayBuffer to Buffer
+ */
+export function toBuffer(binaryData: ArrayBuffer | ArrayBuffer | Buffer): Buffer {
+  throw new Error('Buffer not supported in browser');
+}

package/src/lib/node/buffer.ts

@@ -1,10 +1,12 @@
-// Isolates Buffer references to ensure they are only bundled under Node.js (avoids big webpack polyfill)
-// this file is not visible to webpack (it is excluded in the package.json "browser" field).
+// loaders.gl, MIT license
 
-import {assert} from '../env-utils/assert';
+// Isolates Buffer references to ensure they are only bundled under Node.js (avoids big webpack polyfill)
+// this file is selected by the package.json "browser" field).
 
 /**
  * Convert Buffer to ArrayBuffer
+ * Converts Node.js `Buffer` to `ArrayBuffer` (without triggering bundler to include Buffer polyfill on browser)
+ * @todo better data type
  */
 export function toArrayBuffer(buffer) {
   // TODO - per docs we should just be able to call buffer.buffer, but there are issues
@@ -18,19 +20,18 @@ export function toArrayBuffer(buffer) {
 /**
  * Convert (copy) ArrayBuffer to Buffer
  */
-export function toBuffer(binaryData) {
+export function toBuffer(binaryData: ArrayBuffer | ArrayBuffer | Buffer): Buffer {
+  if (Buffer.isBuffer(binaryData)) {
+    return binaryData;
+  }
+
   if (ArrayBuffer.isView(binaryData)) {
     binaryData = binaryData.buffer;
   }
 
   if (typeof Buffer !== 'undefined' && binaryData instanceof ArrayBuffer) {
-    const buffer = new Buffer(binaryData.byteLength);
-    const view = new Uint8Array(binaryData);
-    for (let i = 0; i < buffer.length; ++i) {
-      buffer[i] = view[i];
-    }
-    return buffer;
+    return Buffer.from(binaryData);
   }
 
-  return assert(false);
+  throw new Error('toBuffer');
 }

package/src/lib/node/fs.ts

@@ -1,9 +1,9 @@
 // fs wrapper (promisified fs + avoids bundling fs in browsers)
 import fs from 'fs';
 import {toArrayBuffer} from './buffer';
-import {promisify} from './util';
+import {promisify2, promisify3} from './promisify';
 
-export type {Stats} from 'fs';
+export type {Stats, WriteStream} from 'fs';
 
 export let readdir;
 /** Wrapper for Node.js fs method */
@@ -23,41 +23,46 @@ export let writeFileSync;
 /** Wrapper for Node.js fs method */
 export let open;
 /** Wrapper for Node.js fs method */
-export let close;
+export let close: (fd: number) => Promise<void>;
 /** Wrapper for Node.js fs method */
 export let read;
 /** Wrapper for Node.js fs method */
 export let fstat;
 
+export let createWriteStream: typeof fs.createWriteStream;
+
 export let isSupported = Boolean(fs);
 
 // paths
 
 try {
   /** Wrapper for Node.js fs method */
-  readdir = promisify(fs.readdir);
+  readdir = promisify2(fs.readdir);
   /** Wrapper for Node.js fs method */
-  stat = promisify(fs.stat);
+  stat = promisify2(fs.stat);
 
   /** Wrapper for Node.js fs method */
-  readFile = promisify(fs.readFile);
+  readFile = fs.readFile;
   /** Wrapper for Node.js fs method */
   readFileSync = fs.readFileSync;
   /** Wrapper for Node.js fs method */
-  writeFile = promisify(fs.writeFile);
+  writeFile = promisify3(fs.writeFile);
   /** Wrapper for Node.js fs method */
   writeFileSync = fs.writeFileSync;
 
   // file descriptors
 
   /** Wrapper for Node.js fs method */
-  open = promisify(fs.open);
+  open = fs.open;
   /** Wrapper for Node.js fs method */
-  close = promisify(fs.close);
+  close = (fd: number) =>
+    new Promise((resolve, reject) => fs.close(fd, (err) => (err ? reject(err) : resolve())));
   /** Wrapper for Node.js fs method */
-  read = promisify(fs.read);
+  read = fs.read;
   /** Wrapper for Node.js fs method */
-  fstat = promisify(fs.fstat);
+  fstat = fs.fstat;
+
+  createWriteStream = fs.createWriteStream;
 
   isSupported = Boolean(fs);
 } catch {

package/src/lib/node/promisify.ts

@@ -0,0 +1,70 @@
+// @loaders.gl, MIT license
+
+// type Parameter1<T extends (arg1: any, ...args: unknown[]) => unknown> = T extends (
+//   arg1: infer P,
+//   ...args: unknown[]
+// ) => unknown
+//   ? P
+//   : never;
+
+//   type Parameter2<T extends (arg1: unknown, arg2: any, ...args: unknown[]) => unknown> = T extends (
+//   arg1: unknown,
+//   arg2: infer P,
+//   ...args: unknown[]
+// ) => unknown
+//   ? P
+//   : never;
+
+// type CallbackParameter2<
+//   T extends (arg: unknown, cb: (error: any, value: any) => void) => unknown
+// > = T extends (arg: unknown, cb: (error: any, value: infer P) => void) => unknown ? P : never;
+
+// type CallbackParameter3<
+//   T extends (arg: unknown, arg2: unknown, cb: (error: any, value: any) => void) => unknown
+// > = T extends (arg: unknown, arg2: unknown, cb: (error: any, value: infer P) => void) => unknown
+//   ? P
+//   : never;
+
+// /** Extract the parameters of a function type in a tuple */
+// export type Promisified<F extends (arg1, cb: (error: unknown, value: any) => void) => any> = (
+//   arg1: Parameter1<F>
+// ) => Promise<CallbackParameter2<F>>;
+// /** Extract the parameters of a function type in a tuple */
+// export type Promisified2<F extends (arg1, arg2, cb: (error: unknown, value: any) => void) => any> = (
+//   arg1: Parameter1<F>,
+//   arg2: Parameter2<F>
+// ) => Promise<CallbackParameter3<F>>;
+
+/** Wrapper for Node.js promisify */
+type Callback<A> = (error: unknown, args: A) => void;
+
+/**
+ * Typesafe promisify implementation
+ * @link https://dev.to/_gdelgado/implement-a-type-safe-version-of-node-s-promisify-in-7-lines-of-code-in-typescript-2j34
+ * @param fn
+ * @returns
+ */
+export function promisify1<T, A>(fn: (args: T, cb: Callback<A>) => void): (args: T) => Promise<A> {
+  return (args: T) =>
+    new Promise((resolve, reject) =>
+      fn(args, (error, callbackArgs) => (error ? reject(error) : resolve(callbackArgs)))
+    );
+}
+
+export function promisify2<T1, T2, A>(
+  fn: (arg1: T1, arg2: T2, cb: Callback<A>) => void
+): (arg1: T1, arg2: T2) => Promise<A> {
+  return (arg1: T1, arg2: T2) =>
+    new Promise((resolve, reject) =>
+      fn(arg1, arg2, (error, callbackArgs) => (error ? reject(error) : resolve(callbackArgs)))
+    );
+}
+
+export function promisify3<T1, T2, T3, A>(
+  fn: (arg1: T1, arg2: T2, arg3: T3, cb: Callback<A>) => void
+): (arg1: T1, arg2: T2, arg3: T3) => Promise<A> {
+  return (arg1: T1, arg2: T2, arg3: T3) =>
+    new Promise((resolve, reject) =>
+      fn(arg1, arg2, arg3, (error, callbackArgs) => (error ? reject(error) : resolve(callbackArgs)))
+    );
+}

package/src/lib/node/stream.ts

@@ -0,0 +1,18 @@
+// loaders.gl, MIT license
+
+import stream from 'stream';
+
+export type {Writable} from 'stream';
+
+export let Transform;
+
+export const isSupported = Boolean(stream);
+
+// paths
+
+try {
+  /** Wrapper for Node.js fs method */
+  Transform = stream.Transform;
+} catch {
+  // ignore
+}

package/src/lib/option-utils/merge-loader-options.ts

@@ -0,0 +1,35 @@
+// loaders.gl, MIT license
+
+import {LoaderOptions} from '../../types';
+
+/**
+ *
+ * @param baseOptions Can be undefined, in which case a fresh options object will be minted
+ * @param newOptions
+ * @returns
+ */
+export function mergeLoaderOptions<Options extends LoaderOptions>(
+  baseOptions: Options | undefined,
+  newOptions: Options
+): Options {
+  return mergeOptionsRecursively(baseOptions || {}, newOptions) as Options;
+}
+
+function mergeOptionsRecursively(
+  baseOptions: Record<string, unknown>,
+  newOptions: Record<string, unknown>
+): Record<string, unknown> {
+  const options = {...baseOptions};
+  for (const [key, newValue] of Object.entries(newOptions)) {
+    if (newValue && typeof newValue === 'object') {
+      options[key] = mergeOptionsRecursively(
+        (options[key] as Record<string, unknown>) || {},
+        newOptions[key] as Record<string, unknown>
+      );
+      // Object.assign(options[key] as object, newOptions[key]);
+    } else {
+      options[key] = newOptions[key];
+    }
+  }
+  return options as Record<string, unknown>;
+}

package/src/lib/path-utils/get-cwd.ts

@@ -0,0 +1,9 @@
+// loaders.gl MIT license
+
+export function getCWD() {
+  if (typeof process !== 'undefined' && typeof process.cwd !== 'undefined') {
+    return process.cwd();
+  }
+  const pathname = window.location?.pathname;
+  return pathname?.slice(0, pathname.lastIndexOf('/') + 1) || '';
+}

package/src/lib/path-utils/path.ts

@@ -1,12 +1,14 @@
 // Beginning of a minimal implementation of the Node.js path API, that doesn't pull in big polyfills.
 
+import {getCWD} from './get-cwd';
+
 /**
  * Replacement for Node.js path.filename
  * @param url
  */
 export function filename(url: string): string {
-  const slashIndex = url && url.lastIndexOf('/');
-  return slashIndex >= 0 ? url.substr((slashIndex as number) + 1) : '';
+  const slashIndex = url ? url.lastIndexOf('/') : -1;
+  return slashIndex >= 0 ? url.substr(slashIndex + 1) : '';
 }
 
 /**
@@ -14,8 +16,8 @@ export function filename(url: string): string {
  * @param url
  */
 export function dirname(url: string): string {
-  const slashIndex = url && url.lastIndexOf('/');
-  return slashIndex >= 0 ? url.substr(0, slashIndex as number) : '';
+  const slashIndex = url ? url.lastIndexOf('/') : -1;
+  return slashIndex >= 0 ? url.substr(0, slashIndex) : '';
 }
 
 /**
@@ -35,3 +37,135 @@ export function join(...parts: string[]): string {
   });
   return parts.join(separator);
 }
+
+/* eslint-disable no-continue */
+
+/**
+ * https://nodejs.org/api/path.html#path_path_resolve_paths
+ * @param paths A sequence of paths or path segments.
+ * @return resolved path
+ * Forked from BTOdell/path-resolve under MIT license
+ * @see https://github.com/BTOdell/path-resolve/blob/master/LICENSE
+ */
+export function resolve(...components: string[]): string {
+  const paths: string[] = [];
+  for (let _i = 0; _i < components.length; _i++) {
+    paths[_i] = components[_i];
+  }
+  let resolvedPath = '';
+  let resolvedAbsolute = false;
+  let cwd: string | undefined;
+  for (let i = paths.length - 1; i >= -1 && !resolvedAbsolute; i--) {
+    let path: string | undefined;
+    if (i >= 0) {
+      path = paths[i];
+    } else {
+      if (cwd === undefined) {
+        cwd = getCWD();
+      }
+      path = cwd;
+    }
+    // Skip empty entries
+    if (path.length === 0) {
+      continue;
+    }
+    resolvedPath = `${path}/${resolvedPath}`;
+    resolvedAbsolute = path.charCodeAt(0) === SLASH;
+  }
+  // At this point the path should be resolved to a full absolute path, but
+  // handle relative paths to be safe (might happen when process.cwd() fails)
+  // Normalize the path (removes leading slash)
+  resolvedPath = normalizeStringPosix(resolvedPath, !resolvedAbsolute);
+  if (resolvedAbsolute) {
+    return `/${resolvedPath}`;
+  } else if (resolvedPath.length > 0) {
+    return resolvedPath;
+  }
+  return '.';
+}
+
+const SLASH = 47;
+const DOT = 46;
+
+/**
+ * Resolves . and .. elements in a path with directory names
+ * Forked from BTOdell/path-resolve under MIT license
+ * @see https://github.com/BTOdell/path-resolve/blob/master/LICENSE
+ */
+/* eslint-disable max-depth */
+// eslint-disable-next-line complexity, max-statements
+function normalizeStringPosix(path: string, allowAboveRoot: boolean): string {
+  let res = '';
+  let lastSlash = -1;
+  let dots = 0;
+  let code: number | undefined;
+  let isAboveRoot = false;
+
+  for (let i = 0; i <= path.length; ++i) {
+    if (i < path.length) {
+      code = path.charCodeAt(i);
+    } else if (code === SLASH) {
+      break;
+    } else {
+      code = SLASH;
+    }
+    if (code === SLASH) {
+      if (lastSlash === i - 1 || dots === 1) {
+        // NOOP
+      } else if (lastSlash !== i - 1 && dots === 2) {
+        if (
+          res.length < 2 ||
+          !isAboveRoot ||
+          res.charCodeAt(res.length - 1) !== DOT ||
+          res.charCodeAt(res.length - 2) !== DOT
+        ) {
+          if (res.length > 2) {
+            const start = res.length - 1;
+            let j = start;
+            for (; j >= 0; --j) {
+              if (res.charCodeAt(j) === SLASH) {
+                break;
+              }
+            }
+            if (j !== start) {
+              res = j === -1 ? '' : res.slice(0, j);
+              lastSlash = i;
+              dots = 0;
+              isAboveRoot = false;
+              continue;
+            }
+          } else if (res.length === 2 || res.length === 1) {
+            res = '';
+            lastSlash = i;
+            dots = 0;
+            isAboveRoot = false;
+            continue;
+          }
+        }
+        if (allowAboveRoot) {
+          if (res.length > 0) {
+            res += '/..';
+          } else {
+            res = '..';
+          }
+          isAboveRoot = true;
+        }
+      } else {
+        const slice = path.slice(lastSlash + 1, i);
+        if (res.length > 0) {
+          res += `/${slice}`;
+        } else {
+          res = slice;
+        }
+        isAboveRoot = false;
+      }
+      lastSlash = i;
+      dots = 0;
+    } else if (code === DOT && dots !== -1) {
+      ++dots;
+    } else {
+      dots = -1;
+    }
+  }
+  return res;
+}