@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/polyfills/fetch/response-polyfill.ts

@@ -0,0 +1,101 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {assert} from '../utils/assert';
+import {decompressReadStream, concatenateReadStream} from '../filesystems/stream-utils.node.js';
+import {Headers} from './headers-polyfill';
+
+const isBoolean = x => typeof x === 'boolean';
+const isFunction = x => typeof x === 'function';
+const isObject = x => x !== null && typeof x === 'object';
+const isReadableNodeStream = x =>
+  isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
+
+/**
+ * Polyfill for Browser Response
+ *
+ * Under Node.js we return a mock "fetch response object"
+ * so that apps can use the same API as in the browser.
+ *
+ * Note: This is intended to be a "lightweight" implementation and will have limitations.
+ *
+ * See https://developer.mozilla.org/en-US/docs/Web/API/Response
+ */
+import * as stream from 'stream';
+
+export class Response {
+  readonly ok: boolean;
+  readonly status: number;
+  readonly statusText: string;
+  readonly headers: Headers;
+  readonly url: string;
+  bodyUsed: boolean = false;
+  private readonly _body;
+
+  // TODO - handle ArrayBuffer, ArrayBufferView, Buffer
+  constructor(
+    body,
+    options: {
+      headers?;
+      status?: number;
+      statusText?: string;
+      url: string;
+    }
+  ) {
+    const {headers, status = 200, statusText = 'OK', url} = options || {};
+
+    this.url = url;
+    this.ok = status === 200;
+    this.status = status; // TODO - handle errors and set status
+    this.statusText = statusText;
+    this.headers = new Headers(options?.headers || {});
+
+    // Check for content-encoding and create a decompression stream
+    if (isReadableNodeStream(body)) {
+      this._body = decompressReadStream(body, headers);
+    } else if (typeof body === 'string') {
+      this._body = stream.Readable.from([new TextEncoder().encode(body)]);
+    } else {
+      this._body = stream.Readable.from([body || new ArrayBuffer(0)]);
+    }
+  }
+
+  // Subset of Properties
+
+  // Returns a readable stream to the "body" of the response (or file)
+  get body() {
+    assert(!this.bodyUsed);
+    assert(isReadableNodeStream(this._body)); // Not implemented: conversion of ArrayBuffer etc to stream
+    this.bodyUsed = true;
+    return this._body;
+  }
+
+  // Subset of Methods
+
+  async arrayBuffer() {
+    if (!isReadableNodeStream(this._body)) {
+      return this._body || new ArrayBuffer(0);
+    }
+    const data = await concatenateReadStream(this._body);
+    return data;
+  }
+
+  async text() {
+    const arrayBuffer = await this.arrayBuffer();
+    const textDecoder = new TextDecoder();
+    return textDecoder.decode(arrayBuffer);
+  }
+
+  async json() {
+    const text = await this.text();
+    return JSON.parse(text);
+  }
+
+  async blob() {
+    if (typeof Blob === 'undefined') {
+      throw new Error('Blob polyfill not installed');
+    }
+    return new Blob([await this.arrayBuffer()]);
+  }
+}

package/src/polyfills/fetch/utils/decode-data-uri.node.ts

@@ -0,0 +1,34 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// Based on binary-gltf-utils under MIT license: Copyright (c) 2016-17 Karl Cheng
+import {toArrayBuffer} from '../../utils/buffer-utils';
+
+/**
+ * Parses a data URI into a buffer, as well as retrieving its declared MIME type.
+ *
+ * @param {string} uri - a data URI (assumed to be valid)
+ * @returns {Object} { buffer, mimeType }
+ */
+export function decodeDataUri(uri: string): {arrayBuffer: ArrayBuffer; mimeType: string} {
+  const dataIndex = uri.indexOf(',');
+
+  let buffer;
+  let mimeType;
+  if (uri.slice(dataIndex - 7, dataIndex) === ';base64') {
+    buffer = Buffer.from(uri.slice(dataIndex + 1), 'base64');
+    mimeType = uri.slice(5, dataIndex - 7).trim();
+  } else {
+    buffer = Buffer.from(decodeURIComponent(uri.slice(dataIndex + 1)));
+    mimeType = uri.slice(5, dataIndex).trim();
+  }
+
+  if (!mimeType) {
+    mimeType = 'text/plain;charset=US-ASCII';
+  } else if (mimeType.startsWith(';')) {
+    mimeType = `text/plain${mimeType}`;
+  }
+
+  return {arrayBuffer: toArrayBuffer(buffer), mimeType};
+}

package/src/polyfills/file/blob-stream-controller.ts

@@ -0,0 +1,69 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+/**
+ * Forked from @gozala's web-blob under MIT license
+ * @see https://github.com/Gozala/web-blob
+ */
+export class BlobStreamController {
+  private chunks: Iterator<Uint8Array>;
+  private isWorking: boolean = false;
+  private isCancelled: boolean = false;
+
+  /**
+   * @param chunks
+   */
+  constructor(chunks: Iterator<Uint8Array>) {
+    this.chunks = chunks;
+  }
+
+  /**
+   * @param controller
+   */
+  start(controller: ReadableStreamDefaultController) {
+    this.work(controller); // eslint-disable-line @typescript-eslint/no-floating-promises
+  }
+
+  /**
+   *
+   * @param controller
+   */
+  async work(controller: ReadableStreamDefaultController) {
+    const {chunks} = this;
+
+    this.isWorking = true;
+    while (!this.isCancelled && (controller.desiredSize || 0) > 0) {
+      let next: {done?: boolean; value?: Uint8Array} | undefined;
+      try {
+        next = chunks.next();
+      } catch (error) {
+        controller.error(error);
+        break;
+      }
+
+      if (next) {
+        if (!next.done && !this.isCancelled) {
+          controller.enqueue(next.value);
+        } else {
+          controller.close();
+        }
+      }
+    }
+
+    this.isWorking = false;
+  }
+
+  /**
+   *
+   * @param {ReadableStreamDefaultController} controller
+   */
+  pull(controller) {
+    if (!this.isWorking) {
+      this.work(controller); // eslint-disable-line @typescript-eslint/no-floating-promises
+    }
+  }
+  cancel() {
+    this.isCancelled = true;
+  }
+}

package/src/polyfills/file/blob-stream.ts

@@ -0,0 +1,40 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+/**
+ * Forked from @gozala's web-blob under MIT license
+ * @see https://github.com/Gozala/web-blob
+ */
+import {ReadableStreamPolyfill} from './readable-stream';
+import {BlobStreamController} from './blob-stream-controller';
+
+/**
+ * Blob stream is a `ReadableStream` extension optimized to have minimal
+ * overhead when consumed as `AsyncIterable<Uint8Array>`.
+ * extends {ReadableStream<Uint8Array>}
+ * implements {AsyncIterable<Uint8Array>}
+ */
+// @ts-ignore
+export class BlobStream<T> extends ReadableStreamPolyfill<T> {
+  private readonly _chunks: Uint8Array[];
+  /**
+   * @param chunks
+   */
+  constructor(chunks) {
+    // @ts-ignore
+    super(new BlobStreamController(chunks.values()), {type: 'bytes'});
+    /** @private */
+    this._chunks = chunks;
+  }
+
+  /**
+   * @property [_options.preventCancel]
+   */
+  // @ts-ignore
+  async *[Symbol.asyncIterator](_options?: {preventCancel?: boolean}): AsyncIterable<Uint8Array> {
+    const reader = this.getReader();
+    yield* this._chunks;
+    reader.releaseLock();
+  }
+}

package/src/polyfills/file/blob.ts

@@ -0,0 +1,177 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// Forked from @gozala's web-blob under MIT license https://github.com/Gozala/web-blob
+import {BlobStream} from './blob-stream';
+
+/**
+ * Forked from @gozala's web-blob under MIT license
+ * @see https://github.com/Gozala/web-blob
+ */
+export class BlobPolyfill {
+  // implements Blob {
+  /** The MIME type of the data contained in the Blob. If type is unknown, string is empty. */
+  readonly type: string;
+  /** The size, in bytes, of the data contained in the Blob object. */
+  size: number;
+  private parts: Uint8Array[];
+  /**
+   * @param [init]
+   * @param [options]
+   */
+  constructor(init: BlobPart[] = [], options: BlobPropertyBag = {}) {
+    this.parts = [];
+
+    this.size = 0;
+    for (const part of init) {
+      if (typeof part === 'string') {
+        const bytes = new TextEncoder().encode(part);
+        this.parts.push(bytes);
+        this.size += bytes.byteLength;
+      } else if (part instanceof BlobPolyfill) {
+        this.size += part.size;
+        // @ts-ignore - `parts` is marked private so TS will complain about
+        // accessing it.
+        this.parts.push(...part.parts);
+      } else if (part instanceof ArrayBuffer) {
+        this.parts.push(new Uint8Array(part));
+        this.size += part.byteLength;
+      } else if (part instanceof Uint8Array) {
+        this.parts.push(part);
+        this.size += part.byteLength;
+      } else if (ArrayBuffer.isView(part)) {
+        const {buffer, byteOffset, byteLength} = part;
+        this.parts.push(new Uint8Array(buffer, byteOffset, byteLength));
+        this.size += byteLength;
+      } else {
+        const bytes = new TextEncoder().encode(String(part));
+        this.parts.push(bytes);
+        this.size += bytes.byteLength;
+      }
+    }
+
+    /** @private */
+    this.type = readType(options.type);
+  }
+
+  /**
+   * Returns a new Blob object containing the data in the specified range of
+   * bytes of the blob on which it's called.
+   * @param start=0 - An index into the Blob indicating the first
+   * byte to include in the new Blob. If you specify a negative value, it's
+   * treated as an offset from the end of the Blob toward the beginning. For
+   * example, `-10` would be the 10th from last byte in the Blob. The default
+   * value is `0`. If you specify a value for start that is larger than the
+   * size of the source Blob, the returned Blob has size 0 and contains no
+   * data.
+   * @param end - An index into the `Blob` indicating the first byte
+   *  that will *not* be included in the new `Blob` (i.e. the byte exactly at
+   * this index is not included). If you specify a negative value, it's treated
+   * as an offset from the end of the Blob toward the beginning. For example,
+   * `-10` would be the 10th from last byte in the `Blob`. The default value is
+   * size.
+   * @param type - The content type to assign to the new Blob;
+   * this will be the value of its type property. The default value is an empty
+   * string.
+   */
+  slice(start: number = 0, end: number = this.size, type: string = ''): Blob {
+    const {size, parts: parts} = this;
+    let offset = start < 0 ? Math.max(size + start, 0) : Math.min(start, size);
+
+    let limit = end < 0 ? Math.max(size + end, 0) : Math.min(end, size);
+    const span = Math.max(limit - offset, 0);
+    const blob = new BlobPolyfill([], {type});
+
+    if (span === 0) {
+      // @ts-ignore
+      return blob;
+    }
+
+    let blobSize = 0;
+    const blobParts: Uint8Array[] = [];
+    for (const part of parts) {
+      const {byteLength} = part;
+      if (offset > 0 && byteLength <= offset) {
+        offset -= byteLength;
+        limit -= byteLength;
+      } else {
+        const chunk = part.subarray(offset, Math.min(byteLength, limit));
+        blobParts.push(chunk);
+        blobSize += chunk.byteLength;
+        // no longer need to take that into account
+        offset = 0;
+
+        // don't add the overflow to new blobParts
+        if (blobSize >= span) {
+          break;
+        }
+      }
+    }
+
+    blob.parts = blobParts;
+    blob.size = blobSize;
+
+    // @ts-ignore
+    return blob;
+  }
+
+  /**
+   * Returns a promise that resolves with an ArrayBuffer containing the entire
+   * contents of the Blob as binary data.
+   */
+  // eslint-disable-next-line require-await
+  async arrayBuffer(): Promise<ArrayBuffer> {
+    return this._toArrayBuffer();
+  }
+
+  /**
+   * Returns a promise that resolves with a USVString containing the entire
+   * contents of the Blob interpreted as UTF-8 text.
+   */
+  // eslint-disable-next-line require-await
+  async text(): Promise<string> {
+    const decoder = new TextDecoder();
+    let text = '';
+    for (const part of this.parts) {
+      text += decoder.decode(part);
+    }
+    return text;
+  }
+
+  /**
+   */
+  // @ts-ignore
+  stream(): BlobStream<any> {
+    return new BlobStream<any>(this.parts);
+  }
+
+  /**
+   * @returns {string}
+   */
+  toString() {
+    return '[object Blob]';
+  }
+
+  get [Symbol.toStringTag]() {
+    return 'Blob';
+  }
+
+  _toArrayBuffer(): ArrayBuffer {
+    const buffer = new ArrayBuffer(this.size);
+    const bytes = new Uint8Array(buffer);
+    let offset = 0;
+    for (const part of this.parts) {
+      bytes.set(part, offset);
+      offset += part.byteLength;
+    }
+    return buffer;
+  }
+}
+
+/**
+ */
+function readType(input: string = ''): string {
+  const type = String(input).toLowerCase();
+  return /[^\u0020-\u007E]/.test(type) ? '' : type;
+}

package/src/polyfills/file/file-reader.ts

@@ -0,0 +1,59 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import {atob} from '../buffer/btoa.node.js';
+
+export class FileReaderPolyfill implements FileReader {
+  // onload: ({result: any}) => void;
+  onload;
+  onabort;
+  onerror;
+  error;
+  onloadstart;
+  onloadend;
+  onprogress;
+
+  readyState;
+  result;
+  DONE;
+  EMPTY;
+  LOADING;
+  addEventListener;
+  removeEventListener;
+  dispatchEvent;
+
+  constructor() {
+    this.onload = null;
+  }
+
+  abort(): void {
+    return;
+  }
+
+  async readAsArrayBuffer(blob: Blob): Promise<void> {
+    const arrayBuffer = await blob.arrayBuffer();
+    if (this.onload) {
+      this.onload({target: {result: arrayBuffer}});
+    }
+  }
+
+  async readAsBinaryString(blob) {
+    throw Error('Not implemented');
+  }
+
+  async readAsDataURL(blob) {
+    const text = await blob.text();
+    const dataUrl = `data://;base64,${atob(text)}`;
+    if (this.onload) {
+      this.onload({target: {result: dataUrl}});
+    }
+  }
+
+  async readAsText(blob) {
+    const text = await blob.text();
+    if (this.onload) {
+      this.onload({target: {result: text}});
+    }
+  }
+}

package/src/polyfills/file/file.ts

@@ -0,0 +1,45 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// Forked from @gozala's web-file under MIT license https://github.com/Gozala/web-file
+
+/**
+ * Forked from @gozala's web-file under MIT license
+ * @see https://github.com/Gozala/web-file
+ */
+// @ts-ignore
+export class FilePolyfill extends globalThis.Blob {
+  // implements File {
+  // public API
+  /** The name of the file referenced by the File object. */
+  name: string = '';
+  /** The path the URL of the File is relative to. */
+  webkitRelativePath: string = '';
+
+  /**
+   * Returns the last modified time of the file, in millisecond since the UNIX
+   * epoch (January 1st, 1970 at Midnight).
+   */
+  lastModified: number;
+
+  /**
+   * @param init
+   * @param name - A USVString representing the file name or the path
+   * to the file.
+   * @param [options]
+   */
+  constructor(init: BlobPart[], name: string, options: FilePropertyBag = {}) {
+    super(init, options);
+    // Per File API spec https://w3c.github.io/FileAPI/#file-constructor
+    // Every "/" character of file name must be replaced with a ":".
+    /** @private */
+    this.name = String(name).replace(/\//g, ':');
+    /** @private */
+    this.lastModified = options?.lastModified || Date.now();
+  }
+
+  get [Symbol.toStringTag]() {
+    return 'File';
+  }
+}

package/src/polyfills/file/install-blob-polyfills.ts

@@ -0,0 +1,17 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// import {ReadableStreamPolyfill} from './readable-stream';
+import {BlobPolyfill} from './blob';
+
+export function instalBlobPolyfills() {
+  if (typeof Blob === 'undefined' && !globalThis.Blob) {
+    // @ts-ignore;
+    globalThis.Blob = BlobPolyfill;
+  }
+
+  return globalThis.Blob;
+}
+
+export const Blob_ = instalBlobPolyfills();

package/src/polyfills/file/install-file-polyfills.ts

@@ -0,0 +1,24 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+// import {ReadableStreamPolyfill} from './readable-stream';
+import {FileReaderPolyfill} from './file-reader';
+import {FilePolyfill} from './file';
+
+export function installFilePolyfills() {
+  if (typeof FileReader === 'undefined' && !globalThis.FileReader) {
+    // @ts-ignore;
+    globalThis.FileReader = FileReaderPolyfill;
+  }
+
+  // Install minimal Node.js File polyfill
+  if (typeof File === 'undefined' && !globalThis.File) {
+    // @ts-ignore;
+    globalThis.File = FilePolyfill;
+  }
+
+  return global;
+}
+
+export const File_ = installFilePolyfills();

package/src/polyfills/file/readable-stream.ts

@@ -0,0 +1,8 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+import {ReadableStream as NodeReadableStream} from 'node:stream/web';
+
+/** Compatibility stream class backed by Node's native Web Streams implementation. */
+export class ReadableStreamPolyfill<T> extends NodeReadableStream<T> {}

package/src/polyfills/filesystems/fetch-node.ts

@@ -0,0 +1,98 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import fs from 'fs';
+import {Readable} from 'stream';
+import {resolvePath} from '../utils/path-utils';
+import {decompressReadStream} from './stream-utils.node.js';
+
+const isBoolean = x => typeof x === 'boolean';
+const isFunction = x => typeof x === 'function';
+const isObject = x => x !== null && typeof x === 'object';
+const isReadableNodeStream = x =>
+  isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
+
+/**
+ * Enables
+ * @param url
+ * @param options
+ * @returns
+ */
+// eslint-disable-next-line max-statements
+export async function fetchNode(url: string, options?: RequestInit): Promise<Response> {
+  // Support `file://` protocol
+  const FILE_PROTOCOL_REGEX = /^file:\/\//;
+  url.replace(FILE_PROTOCOL_REGEX, '/');
+
+  // Remove any query parameters, as they have no meaning
+  let noqueryUrl = url.split('?')[0];
+  noqueryUrl = resolvePath(noqueryUrl);
+
+  const responseHeaders = new Headers();
+  // Automatically decompress gzipped files with .gz extension
+  if (url.endsWith('.gz')) {
+    // url = url.slice(0, -3);
+    responseHeaders.set('content-encoding', 'gzip');
+  }
+  if (url.endsWith('.br')) {
+    // url = url.slice(0, -3);
+    responseHeaders.set('content-encoding', 'br');
+  }
+
+  try {
+    // Now open the stream
+    const body = await new Promise<fs.ReadStream>((resolve, reject) => {
+      // @ts-ignore
+      const stream = fs.createReadStream(noqueryUrl, {encoding: null});
+      stream.once('readable', () => resolve(stream));
+      stream.on('error', error => reject(error));
+    });
+
+    let bodyStream: Readable = body;
+
+    // Check for content-encoding and create a decompression stream
+    if (isReadableNodeStream(body)) {
+      bodyStream = decompressReadStream(body, responseHeaders);
+    } else if (typeof body === 'string') {
+      bodyStream = Readable.from([new TextEncoder().encode(body)]);
+    } else {
+      bodyStream = Readable.from([body || new ArrayBuffer(0)]);
+    }
+
+    const status = 200;
+    const statusText = 'OK';
+    const headers = getHeadersForFile(noqueryUrl);
+    // @ts-expect-error
+    const response = new Response(bodyStream, {headers, status, statusText});
+    Object.defineProperty(response, 'url', {value: url});
+    return response;
+  } catch (error) {
+    // console.error(error);
+    const errorMessage = (error as Error).message;
+    const status = 400;
+    const statusText = errorMessage;
+    const headers = {};
+    const response = new Response(errorMessage, {headers, status, statusText});
+    Object.defineProperty(response, 'url', {value: url});
+    return response;
+  }
+}
+
+function getHeadersForFile(noqueryUrl: string): Headers {
+  const headers = {};
+
+  // Fix up content length if we can for best progress experience
+  if (!headers['content-length']) {
+    const stats = fs.statSync(noqueryUrl);
+    headers['content-length'] = stats.size;
+  }
+
+  // Automatically decompress gzipped files with .gz extension
+  if (noqueryUrl.endsWith('.gz')) {
+    noqueryUrl = noqueryUrl.slice(0, -3);
+    headers['content-encoding'] = 'gzip';
+  }
+
+  return new Headers(headers);
+}