@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/loader-utils/writer-types.ts

@@ -0,0 +1,107 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Format} from './format-types';
+
+// WRITERS
+
+/** Options for writers */
+export type WriterOptions = {
+  core?: {
+    /** worker source. If is set will be used instead of loading worker from the Internet */
+    source?: string | null;
+
+    // module loading
+
+    /** Force to load WASM libraries from local file system in NodeJS or from loaders.gl CDN in a web browser */
+    useLocalLibraries?: boolean;
+    /** Whether to use workers under Node.js (experimental) */
+    _nodeWorkers?: boolean;
+    /** Set to `false` to disable workers */
+    worker?: boolean;
+    log?: any;
+  };
+
+  /** Any additional JS libraries */
+  modules?: Record<string, any>;
+
+  /** writer-specific options */
+  [writerId: string]: Record<string, unknown> | undefined;
+};
+
+/**
+ * A writer definition that can be used with `@loaders.gl/core` functions
+ */
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+export type Writer<DataT = unknown, BatchT = unknown, WriterOptionsT = WriterOptions> = Format & {
+  /** The result type of this loader  */
+  dataType?: DataT;
+  /** The batched result type of this loader  */
+  batchType?: BatchT;
+  /** Version should be injected by build tools */
+  version: string;
+  /** A boolean, or a URL */
+  worker?: string | boolean;
+  // end Worker
+  options: WriterOptionsT;
+  deprecatedOptions?: Record<string, string>;
+
+  /** Human readable name */
+  name: string;
+  /** id should be the same as the field used in LoaderOptions */
+  id: string;
+  /** module is used to generate worker threads, need to be the module directory name */
+  module: string;
+  /** Which category does this loader belong to */
+  category?: string;
+  /** File extensions that are potential matches with this loader. */
+  extensions: string[];
+  /** MIMETypes that indicate a match with this loader. @note Some MIMETypes are generic and supported by many loaders */
+  mimeTypes?: string[];
+  /** Is the input of this loader binary */
+  binary?: boolean;
+  /** Is the input of this loader text */
+  text?: boolean;
+};
+
+/**
+ * A writer definition that can be used with `@loaders.gl/core` functions
+ */
+export type WriterWithEncoder<
+  DataT = unknown,
+  BatchT = unknown,
+  WriterOptionsT = WriterOptions
+> = Writer<DataT, BatchT, WriterOptionsT> & {
+  /** Encode to binary,  asynchronously */
+  encode(data: DataT, options?: WriterOptionsT): Promise<ArrayBuffer>;
+  /** Encode to binary, synchronously */
+  encodeSync?(data: DataT, options?: WriterOptionsT): ArrayBuffer;
+  /** Encode to binary in batches */
+  encodeInBatches?(data: AsyncIterable<any>, options?: WriterOptionsT): AsyncIterable<ArrayBuffer>;
+
+  /** Encode to text, asynchronously. For text formats. */
+  encodeText?(table: DataT, options?: WriterOptionsT): Promise<string>;
+  /** Encode to text, synchronously. For text formats. */
+  encodeTextSync?(table: DataT, options?: WriterOptionsT): string;
+  /** Encode to text in batched. For text formats. */
+  encodeTextInBatches?(
+    data: AsyncIterable<any>,
+    options?: WriterOptionsT
+  ): AsyncIterable<ArrayBuffer>;
+
+  encodeURLtoURL?: (
+    inputUrl: string,
+    outputUrl: string,
+    options?: WriterOptionsT
+  ) => Promise<string>;
+};
+
+/** Typescript helper to extract the writer options type from a writer type */
+export type WriterOptionsType<T = Writer> =
+  T extends Writer<unknown, unknown, infer OptionsType> ? OptionsType : never;
+/** Typescript helper to extract input data type from a writer type */
+export type WriterDataType<T = Writer> =
+  T extends Writer<infer DataType, any, any> ? DataType : never;
+export type WriterBatchType<T = Writer> =
+  T extends Writer<any, infer BatchType, any> ? BatchType : never;

package/src/null-loader.ts

@@ -2,8 +2,8 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
-import {LoaderContext} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/core/loader-utils';
+import {LoaderContext} from '@loaders.gl/core/loader-utils';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.

package/src/polyfills/README.md

@@ -0,0 +1,16 @@
+# @loaders.gl/core/polyfills
+
+[loaders.gl](https://loaders.gl/docs) is a collection of framework-independent 3D and geospatial parsers and encoders.
+
+The `@loaders.gl/core/polyfills` subpath installs Node.js adapters without importing the core root entry point or another loaders.gl module. Install `@loaders.gl/core` and import this subpath for its side effects.
+
+Under Node.js, importing `@loaders.gl/core/polyfills` also installs the minimal image support used by `@loaders.gl/images`:
+
+- a global `ImageBitmap` polyfill backed by decoded pixel data
+- a global `getImageBitmapData(image)` helper that unwraps that bitmap back to `{data, width, height}`
+
+This image support is intentionally minimal. It is designed for `ImageBitmapLoader` and parsed-image helpers, not as a full `createImageBitmap()` implementation.
+
+Image decoding and encoding also require the optional `get-pixels`, `ndarray`, and `save-pixels` packages. Install them only when those image operations are needed.
+
+For documentation please visit the [website](https://loaders.gl).

package/src/polyfills/buffer/btoa.node.ts

@@ -0,0 +1,15 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// btoa, atob polyfills for Node.js
+// Note: The atob and btoa functions (not just the polyfills!) are not unicode safe
+// But still useful for unit testing
+
+export function atob(string) {
+  return Buffer.from(string).toString('base64');
+}
+
+export function btoa(base64) {
+  return Buffer.from(base64, 'base64').toString('ascii');
+}

package/src/polyfills/buffer/to-array-buffer.node.ts

@@ -0,0 +1,12 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export function bufferToArrayBuffer(buffer) {
+  // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+  if (Buffer.isBuffer(buffer)) {
+    const typedArray = new Uint8Array(buffer);
+    return typedArray.buffer;
+  }
+  return buffer;
+}

package/src/polyfills/crypto/node-hash.ts

@@ -0,0 +1,78 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import * as crypto from 'crypto'; // Node.js builtin
+
+type CryptoHashOptions = {
+  crypto: {
+    algorithm: string;
+    onEnd?: (result: {hash: string}) => any;
+  };
+};
+
+/**
+ * Calculates Cryptographic Hash using Node.js crypto library
+ * @deprecated Warning, experimental class
+ */
+export class NodeHash {
+  readonly name = 'crypto-node';
+
+  options: CryptoHashOptions;
+  // @ts-ignore
+  private _algorithm;
+  // @ts-ignore
+  private _hash;
+
+  constructor(options: CryptoHashOptions) {
+    this.options = options;
+    this.hashBatches = this.hashBatches.bind(this);
+    if (!this.options?.crypto?.algorithm) {
+      throw new Error(this.name);
+    }
+  }
+
+  /** Complete the optional hash initialization step. */
+  async preload(): Promise<void> {
+    return;
+  }
+
+  /**
+   * Atomic hash calculation
+   * @returns base64 encoded hash
+   */
+  async hash(input: ArrayBuffer, encoding: 'hex' | 'base64'): Promise<string> {
+    // await this.preload();
+    const algorithm = this.options?.crypto?.algorithm?.toLowerCase();
+    try {
+      if (!crypto.createHash) {
+        throw new Error('crypto.createHash not available');
+      }
+      const hash = crypto.createHash?.(algorithm);
+      const inputArray = new Uint8Array(input);
+      return hash.update(inputArray).digest('base64');
+    } catch (error) {
+      throw Error(`${algorithm} hash not available. ${error}`);
+    }
+  }
+
+  async *hashBatches(
+    asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+    encoding: 'hex' | 'base64' = 'base64'
+  ): AsyncIterable<ArrayBuffer> {
+    // await this.preload();
+    if (!crypto.createHash) {
+      throw new Error('crypto.createHash not available');
+    }
+    const hash = crypto.createHash?.(this.options?.crypto?.algorithm?.toLowerCase());
+    for await (const chunk of asyncIterator) {
+      // https://stackoverflow.com/questions/25567468/how-to-decrypt-an-arraybuffer
+      const inputArray = new Uint8Array(chunk);
+      hash.update(inputArray);
+      yield chunk;
+    }
+    // We can pass our encoding constant directly to Node.js digest as it already supports `hex` and `base64`
+    const digest = hash.digest(encoding);
+    this.options?.crypto?.onEnd?.({hash: digest});
+  }
+}

package/src/polyfills/fetch/decode-data-uri.ts

@@ -0,0 +1,71 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// Based on binary-gltf-utils under MIT license: Copyright (c) 2016-17 Karl Cheng
+import {toArrayBuffer} from '../utils/buffer-utils';
+
+/**
+ * Parses a data URI into a buffer, as well as retrieving its declared MIME type.
+ *
+ * @param {string} uri - a data URI (assumed to be valid)
+ * @returns {Object} { buffer, mimeType }
+ */
+export function decodeDataUri(uri: string): {arrayBuffer: ArrayBuffer; mimeType: string} {
+  const dataIndex = uri.indexOf(',');
+
+  let buffer;
+  let mimeType;
+  if (uri.slice(dataIndex - 7, dataIndex) === ';base64') {
+    buffer = Buffer.from(uri.slice(dataIndex + 1), 'base64');
+    mimeType = uri.slice(5, dataIndex - 7).trim();
+  } else {
+    buffer = Buffer.from(decodeURIComponent(uri.slice(dataIndex + 1)));
+    mimeType = uri.slice(5, dataIndex).trim();
+  }
+
+  if (!mimeType) {
+    mimeType = 'text/plain;charset=US-ASCII';
+  } else if (mimeType.startsWith(';')) {
+    mimeType = `text/plain${mimeType}`;
+  }
+
+  return {arrayBuffer: toArrayBuffer(buffer), mimeType};
+}
+
+// /**
+//  * @param data
+//  * @todo Duplicate of core
+//  */
+// export function toArrayBuffer(data: unknown): ArrayBuffer {
+//   if (isArrayBuffer(data)) {
+//     return data as ArrayBuffer;
+//   }
+
+//   // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+//   if (isBuffer(data)) {
+//     // @ts-expect-error
+//     const typedArray = new Uint8Array(data);
+//     return typedArray.buffer;
+//   }
+
+//   // Careful - Node Buffers will look like ArrayBuffers (keep after isBuffer)
+//   if (ArrayBuffer.isView(data)) {
+//     return data.buffer;
+//   }
+
+//   if (typeof data === 'string') {
+//     const text = data;
+//     const uint8Array = new TextEncoder().encode(text);
+//     return uint8Array.buffer;
+//   }
+
+//   // HACK to support Blob polyfill
+//   // @ts-expect-error
+//   if (data && typeof data === 'object' && data._toArrayBuffer) {
+//     // @ts-expect-error
+//     return data._toArrayBuffer();
+//   }
+
+//   throw new Error(`toArrayBuffer(${JSON.stringify(data, null, 2).slice(10)})`);
+// }

package/src/polyfills/fetch/fetch-polyfill.ts

@@ -0,0 +1,165 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import http from 'http';
+import https from 'https';
+import {Response} from './response-polyfill';
+import {Headers} from './headers-polyfill';
+import {decodeDataUri} from './decode-data-uri';
+
+const isDataURL = (url: string): boolean => url.startsWith('data:');
+const isRequestURL = (url: string): boolean => url.startsWith('http:') || url.startsWith('https:');
+
+/**
+ * Emulation of Browser fetch for Node.js
+ * @param url
+ * @param options
+ */
+// eslint-disable-next-line complexity
+export async function fetchNode(url: string, options: RequestInit): Promise<Response> {
+  try {
+    // Handle file streams in node
+    // @ts-expect-error
+    if (globalThis.fetch !== fetchNode && (isRequestURL(url) || isDataURL(url))) {
+      // @ts-expect-error
+      return await fetch(url, options);
+    }
+
+    // Handle data urls in node, to match `fetch``
+    // Note - this loses the MIME type, data URIs are handled directly in fetch
+    if (isDataURL(url)) {
+      const {arrayBuffer, mimeType} = decodeDataUri(url);
+      const response = new Response(arrayBuffer, {
+        headers: {'content-type': mimeType},
+        url
+      });
+      return response;
+    }
+
+    // Automatically decompress gzipped files with .gz extension
+    const syntheticResponseHeaders = {};
+    const originalUrl = url;
+    if (url.endsWith('.gz')) {
+      url = url.slice(0, -3);
+      syntheticResponseHeaders['content-encoding'] = 'gzip';
+    }
+
+    // Need to create the stream in advance since Response constructor needs to be sync
+    const body = await createHTTPRequestReadStream(originalUrl, options);
+    const headers = getHeaders(url, body, syntheticResponseHeaders);
+    const {status, statusText} = getStatus(body);
+
+    const followRedirect =
+      // @ts-expect-error
+      !options || options.followRedirect || options.followRedirect === undefined;
+
+    if (status >= 300 && status < 400 && headers.has('location') && followRedirect) {
+      const redirectUrl = generateRedirectUrl(url, headers.get('location'));
+
+      // Redirect
+      return await fetchNode(redirectUrl, options);
+    }
+    return new Response(body, {headers, status, statusText, url});
+  } catch (error) {
+    // TODO - what error code to use here?
+    return new Response(null, {status: 400, statusText: String(error), url});
+  }
+}
+
+/** Returns a promise that resolves to a readable stream */
+export async function createHTTPRequestReadStream(
+  url: string,
+  options
+): Promise<http.IncomingMessage> {
+  // HANDLE HTTP/HTTPS REQUESTS IN NODE
+  // TODO: THIS IS BAD SINCE WE RETURN A PROMISE INSTEAD OF A STREAM
+  return await new Promise((resolve, reject) => {
+    const requestOptions = getRequestOptions(url, options);
+    const req = url.startsWith('https:')
+      ? https.request(requestOptions, res => resolve(res))
+      : http.request(requestOptions, res => resolve(res));
+    req.on('error', error => reject(error));
+    req.end();
+  });
+}
+
+/**
+ * Generate redirect url from location without origin and protocol.
+ * @param originalUrl
+ * @param redirectUrl
+ */
+function generateRedirectUrl(originalUrl: string, location: string): string {
+  if (location.startsWith('http')) {
+    return location;
+  }
+  // If url doesn't have origin and protocol just extend current url origin with location.
+  const url = new URL(originalUrl);
+  url.pathname = location;
+
+  return url.href;
+}
+
+// HELPER FUNCTIONS
+
+function getRequestOptions(url: string, options?: {fetch?: typeof fetch; headers?}) {
+  // Ensure header keys are lower case so that we can merge without duplicates
+  const originalHeaders = options?.headers || {};
+  const headers = {};
+  for (const key of Object.keys(originalHeaders)) {
+    headers[key.toLowerCase()] = originalHeaders[key];
+  }
+
+  // Add default accept-encoding to headers
+  headers['accept-encoding'] = headers['accept-encoding'] || 'gzip,br,deflate';
+
+  const urlObject = new URL(url);
+  return {
+    hostname: urlObject.hostname,
+    path: urlObject.pathname,
+    method: 'GET',
+    // Add options and user provided 'options.fetch' overrides if available
+    ...options,
+    ...options?.fetch,
+    // Override with updated headers with accepted encodings:
+    headers,
+    port: urlObject.port
+  };
+}
+
+function getStatus(httpResponse: http.IncomingMessage): {status: number; statusText: string} {
+  if (httpResponse.statusCode) {
+    return {status: httpResponse.statusCode, statusText: httpResponse.statusMessage || 'NA'};
+  }
+  return {status: 200, statusText: 'OK'};
+}
+
+function getHeaders(url, httpResponse, additionalHeaders = {}) {
+  const headers = {};
+
+  if (httpResponse && httpResponse.headers) {
+    const httpHeaders = httpResponse.headers;
+    for (const key in httpHeaders) {
+      const header = httpHeaders[key];
+      headers[key.toLowerCase()] = String(header);
+    }
+  }
+
+  // Fix up content length if we can for best progress experience
+  if (!headers['content-length']) {
+    const contentLength = getContentLength(url);
+    if (Number.isFinite(contentLength)) {
+      headers['content-length'] = contentLength;
+    }
+  }
+
+  Object.assign(headers, additionalHeaders);
+
+  return new Headers(headers);
+}
+
+/** Needs to be read from actual headers */
+function getContentLength(url: string): number | null {
+  // TODO - remove media type etc
+  return isDataURL(url) ? url.length - 'data:'.length : null;
+}

package/src/polyfills/fetch/headers-polyfill.ts

@@ -0,0 +1,119 @@
+// SPDX-License-Identifier: MIT
+
+/**
+ * Polyfill for Browser Headers
+ * Based on https://github.com/github/fetch under MIT license
+ */
+export class Headers {
+  map: {};
+
+  constructor(headers) {
+    this.map = {};
+
+    if (headers instanceof Headers) {
+      headers.forEach((value, name) => this.append(name, value));
+    } else if (Array.isArray(headers)) {
+      headers.forEach(header => this.append(header[0], header[1]));
+    } else if (headers) {
+      Object.getOwnPropertyNames(headers).forEach(name => this.append(name, headers[name]));
+    }
+  }
+
+  append(name, value) {
+    name = normalizeName(name);
+    value = normalizeValue(value);
+    const oldValue = this.map[name];
+    this.map[name] = oldValue ? `${oldValue}, ${value}` : value;
+  }
+
+  delete(name) {
+    delete this.map[normalizeName(name)];
+  }
+
+  get(name) {
+    name = normalizeName(name);
+    return this.has(name) ? this.map[name] : null;
+  }
+
+  has(name) {
+    return this.map.hasOwnProperty(normalizeName(name));
+  }
+
+  set(name, value) {
+    this.map[normalizeName(name)] = normalizeValue(value);
+  }
+
+  forEach(visitor, thisArg = null) {
+    for (const name in this.map) {
+      if (this.map.hasOwnProperty(name)) {
+        if (thisArg) {
+          visitor.call(thisArg, this.map[name], name, this);
+        } else {
+          visitor(this.map[name], name, this);
+        }
+      }
+    }
+  }
+
+  keys() {
+    const items: any[] = [];
+    this.forEach(function (value, name) {
+      items.push(name);
+    });
+    return iteratorFor(items);
+  }
+
+  values() {
+    const items: any[] = [];
+    this.forEach(function (value) {
+      items.push(value);
+    });
+    return iteratorFor(items);
+  }
+
+  entries() {
+    const items: any[] = [];
+    this.forEach(function (value, name) {
+      items.push([name, value]);
+    });
+    return iteratorFor(items);
+  }
+
+  *[Symbol.iterator]() {
+    // @ts-ignore must have a '[Symbol.iterator]()' method that returns an iterator.
+    yield* this.entries();
+  }
+}
+
+function normalizeName(name) {
+  if (typeof name !== 'string') {
+    name = String(name);
+  }
+  if (/[^a-z0-9\-#$%&'*+.^_`|~]/i.test(name) || name === '') {
+    throw new TypeError('Invalid character in header field name');
+  }
+  return name.toLowerCase();
+}
+
+function normalizeValue(value) {
+  if (typeof value !== 'string') {
+    value = String(value);
+  }
+  return value;
+}
+
+// Build a destructive iterator for the value list
+function iteratorFor(items) {
+  const iterator = {
+    next() {
+      const value = items.shift();
+      return {done: value === undefined, value};
+    }
+  };
+
+  iterator[Symbol.iterator] = function () {
+    return iterator;
+  };
+
+  return iterator;
+}