@loaders.gl/polyfills 4.0.0-beta.2 → 4.0.0-beta.4

package/src/fetch/utils/decode-data-uri.node.ts

@@ -0,0 +1,69 @@
+// Based on binary-gltf-utils under MIT license: Copyright (c) 2016-17 Karl Cheng
+
+const isArrayBuffer = (x) => x && x instanceof ArrayBuffer;
+const isBuffer = (x) => x && x instanceof Buffer;
+
+/**
+ * Parses a data URI into a buffer, as well as retrieving its declared MIME type.
+ *
+ * @param {string} uri - a data URI (assumed to be valid)
+ * @returns {Object} { buffer, mimeType }
+ */
+export function decodeDataUri(uri: string): {arrayBuffer: ArrayBuffer; mimeType: string} {
+  const dataIndex = uri.indexOf(',');
+
+  let buffer;
+  let mimeType;
+  if (uri.slice(dataIndex - 7, dataIndex) === ';base64') {
+    buffer = Buffer.from(uri.slice(dataIndex + 1), 'base64');
+    mimeType = uri.slice(5, dataIndex - 7).trim();
+  } else {
+    buffer = Buffer.from(decodeURIComponent(uri.slice(dataIndex + 1)));
+    mimeType = uri.slice(5, dataIndex).trim();
+  }
+
+  if (!mimeType) {
+    mimeType = 'text/plain;charset=US-ASCII';
+  } else if (mimeType.startsWith(';')) {
+    mimeType = `text/plain${mimeType}`;
+  }
+
+  return {arrayBuffer: toArrayBuffer(buffer), mimeType};
+}
+
+/**
+ * @param data
+ * @todo Duplicate of core
+ */
+export function toArrayBuffer(data: unknown): ArrayBuffer {
+  if (isArrayBuffer(data)) {
+    return data as ArrayBuffer;
+  }
+
+  // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+  if (isBuffer(data)) {
+    // @ts-expect-error
+    const typedArray = new Uint8Array(data);
+    return typedArray.buffer;
+  }
+
+  // Careful - Node Buffers will look like ArrayBuffers (keep after isBuffer)
+  if (ArrayBuffer.isView(data)) {
+    return data.buffer;
+  }
+
+  if (typeof data === 'string') {
+    const text = data;
+    const uint8Array = new TextEncoder().encode(text);
+    return uint8Array.buffer;
+  }
+
+  // HACK to support Blob polyfill
+  // @ts-expect-error
+  if (data && typeof data === 'object' && data._toArrayBuffer) {
+    // @ts-expect-error
+    return data._toArrayBuffer();
+  }
+
+  throw new Error(`toArrayBuffer(${JSON.stringify(data, null, 2).slice(10)})`);
+}

package/src/{node/file → file}/file.ts

@@ -1,12 +1,11 @@
 // Forked from @gozala's web-file under MIT license https://github.com/Gozala/web-file
-import {BlobPolyfill} from './blob';
 
 /**
  * Forked from @gozala's web-file under MIT license
  * @see https://github.com/Gozala/web-file
  */
 // @ts-ignore
-export class FilePolyfill extends BlobPolyfill {
+export class FilePolyfill extends globalThis.Blob {
   // implements File {
   // public API
   /** The name of the file referenced by the File object. */

package/src/file/install-blob-polyfills.ts

@@ -0,0 +1,13 @@
+// import {ReadableStreamPolyfill} from './readable-stream';
+import {BlobPolyfill} from './blob';
+
+export function instalBlobPolyfills() {
+  if (typeof Blob === 'undefined' && !globalThis.Blob) {
+    // @ts-ignore;
+    globalThis.Blob = BlobPolyfill;
+  }
+
+  return globalThis.Blob;
+}
+
+export const Blob_ = instalBlobPolyfills();

package/src/file/install-file-polyfills.ts

@@ -0,0 +1,20 @@
+// import {ReadableStreamPolyfill} from './readable-stream';
+import {FileReaderPolyfill} from './file-reader';
+import {FilePolyfill} from './file';
+
+export function installFilePolyfills() {
+  if (typeof FileReader === 'undefined' && !globalThis.FileReader) {
+    // @ts-ignore;
+    globalThis.FileReader = FileReaderPolyfill;
+  }
+
+  // Install minimal Node.js File polyfill
+  if (typeof File === 'undefined' && !globalThis.File) {
+    // @ts-ignore;
+    globalThis.File = FilePolyfill;
+  }
+
+  return global;
+}
+
+export const File_ = installFilePolyfills();

package/src/{node/file → file}/readable-stream.ts

@@ -2,7 +2,7 @@ import {ReadableStream as WSPReadableStream} from 'web-streams-polyfill';
 
 // Want a polyfill, but please don't install it
 // @ts-ignore
-delete global.ReadableStream;
+delete globalThis.ReadableStream;
 
 // @ts-ignore
 export class ReadableStreamPolyfill<T> extends WSPReadableStream<T> implements ReadableStream {}

package/src/filesystems/fetch-node.ts

@@ -0,0 +1,96 @@
+// loaders.gl, MIT license
+
+import fs from 'fs';
+import {Readable} from 'stream';
+import {resolvePath} from '@loaders.gl/loader-utils';
+import {decompressReadStream} from './stream-utils.node';
+
+const isBoolean = (x) => typeof x === 'boolean';
+const isFunction = (x) => typeof x === 'function';
+const isObject = (x) => x !== null && typeof x === 'object';
+const isReadableNodeStream = (x) =>
+  isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
+
+/**
+ * Enables
+ * @param url
+ * @param options
+ * @returns
+ */
+// eslint-disable-next-line max-statements
+export async function fetchNode(url: string, options?: RequestInit): Promise<Response> {
+  // Support `file://` protocol
+  const FILE_PROTOCOL_REGEX = /^file:\/\//;
+  url.replace(FILE_PROTOCOL_REGEX, '/');
+
+  // Remove any query parameters, as they have no meaning
+  let noqueryUrl = url.split('?')[0];
+  noqueryUrl = resolvePath(noqueryUrl);
+
+  const responseHeaders = new Headers();
+  // Automatically decompress gzipped files with .gz extension
+  if (url.endsWith('.gz')) {
+    // url = url.slice(0, -3);
+    responseHeaders['content-encoding'] = 'gzip';
+  }
+  if (url.endsWith('.br')) {
+    // url = url.slice(0, -3);
+    responseHeaders['content-encoding'] = 'br';
+  }
+
+  try {
+    // Now open the stream
+    const body = await new Promise<fs.ReadStream>((resolve, reject) => {
+      // @ts-ignore
+      const stream = fs.createReadStream(noqueryUrl, {encoding: null});
+      stream.once('readable', () => resolve(stream));
+      stream.on('error', (error) => reject(error));
+    });
+
+    let bodyStream: Readable = body;
+
+    // Check for content-encoding and create a decompression stream
+    if (isReadableNodeStream(body)) {
+      bodyStream = decompressReadStream(body, responseHeaders);
+    } else if (typeof body === 'string') {
+      bodyStream = Readable.from([new TextEncoder().encode(body)]);
+    } else {
+      bodyStream = Readable.from([body || new ArrayBuffer(0)]);
+    }
+
+    const status = 200;
+    const statusText = 'OK';
+    const headers = getHeadersForFile(noqueryUrl);
+    // @ts-expect-error
+    const response = new Response(bodyStream, {headers, status, statusText});
+    Object.defineProperty(response, 'url', {value: url});
+    return response;
+  } catch (error) {
+    // console.error(error);
+    const errorMessage = (error as Error).message;
+    const status = 400;
+    const statusText = errorMessage;
+    const headers = {};
+    const response = new Response(errorMessage, {headers, status, statusText});
+    Object.defineProperty(response, 'url', {value: url});
+    return response;
+  }
+}
+
+function getHeadersForFile(noqueryUrl: string): Headers {
+  const headers = {};
+
+  // Fix up content length if we can for best progress experience
+  if (!headers['content-length']) {
+    const stats = fs.statSync(noqueryUrl);
+    headers['content-length'] = stats.size;
+  }
+
+  // Automatically decompress gzipped files with .gz extension
+  if (noqueryUrl.endsWith('.gz')) {
+    noqueryUrl = noqueryUrl.slice(0, -3);
+    headers['content-encoding'] = 'gzip';
+  }
+
+  return new Headers(headers);
+}

package/src/filesystems/node-file.ts

@@ -0,0 +1,137 @@
+import type {ReadableFile, WritableFile, Stat} from '@loaders.gl/loader-utils';
+import {resolvePath} from '@loaders.gl/loader-utils';
+import fs from 'fs';
+
+export class NodeFile implements ReadableFile, WritableFile {
+  handle: number;
+  size: number;
+  bigsize: bigint;
+  url: string;
+
+  constructor(path: string, flags: 'r' | 'w' | 'wx', mode?: number) {
+    path = resolvePath(path);
+    this.handle = fs.openSync(path, flags, mode);
+    const stats = fs.fstatSync(this.handle, {bigint: true});
+    this.size = Number(stats.size);
+    this.bigsize = stats.size;
+    this.url = path;
+  }
+
+  async close(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      fs.close(this.handle, (err) => (err ? reject(err) : resolve()));
+    });
+  }
+
+  async stat(): Promise<Stat> {
+    return await new Promise((resolve, reject) =>
+      fs.fstat(this.handle, {bigint: true}, (err, info) => {
+        const stats: Stat = {
+          size: Number(info.size),
+          bigsize: info.size,
+          isDirectory: info.isDirectory()
+        };
+        if (err) {
+          reject(err);
+        } else {
+          resolve(stats);
+        }
+      })
+    );
+  }
+
+  async read(offset: number | bigint, length: number): Promise<ArrayBuffer> {
+    const arrayBuffer = new ArrayBuffer(length);
+    let bigOffset = BigInt(offset);
+
+    let totalBytesRead = 0;
+    const uint8Array = new Uint8Array(arrayBuffer);
+
+    let position;
+    // Read in loop until we get required number of bytes
+    while (length > 0) {
+      const bytesRead = await readBytes(this.handle, uint8Array, 0, length, bigOffset);
+
+      // Check if end of file reached
+      if (bytesRead === 0) {
+        break;
+      }
+
+      totalBytesRead += bytesRead;
+      bigOffset += BigInt(bytesRead);
+      length -= bytesRead;
+
+      // Advance position unless we are using built-in position advancement
+      if (position !== undefined) {
+        position += bytesRead;
+      }
+    }
+    return totalBytesRead < length ? arrayBuffer.slice(0, totalBytesRead) : arrayBuffer;
+  }
+
+  async write(
+    arrayBuffer: ArrayBuffer,
+    offset: number | bigint = 0,
+    length: number = arrayBuffer.byteLength
+  ): Promise<number> {
+    return new Promise((resolve, reject) => {
+      // TODO - Node.js doesn't offer write with bigint offsets???
+      const nOffset = Number(offset);
+      const uint8Array = new Uint8Array(arrayBuffer, Number(offset), length);
+      fs.write(this.handle, uint8Array, 0, length, nOffset, (err, bytesWritten) =>
+        err ? reject(err) : resolve(bytesWritten)
+      );
+    });
+  }
+}
+
+async function readBytes(
+  fd: number,
+  uint8Array: Uint8Array,
+  offset: number,
+  length: number,
+  position: number | bigint | null
+): Promise<number> {
+  return await new Promise<number>((resolve, reject) =>
+    fs.read(fd, uint8Array, offset, length, position, (err, bytesRead) =>
+      err ? reject(err) : resolve(bytesRead)
+    )
+  );
+}
+
+// TODO - implement streaming write
+/*
+export interface WriteStreamOptions {
+  flags?: string;
+  encoding?: 'utf8';
+  fd?: number;
+  mode?: number;
+  autoClose?: boolean;
+  start?: number;
+}
+
+export class NodeStreamWritableFile implements WritableFile {
+  outputStream: fs.WriteStream | Writable;
+
+  constructor(pathOrStream: string | Writable, options?: WriteStreamOptions) {
+    this.outputStream =
+      typeof pathOrStream === 'string' ? fs.createWriteStream(pathOrStream, options) : pathOrStream;
+  }
+
+  async write(buffer: ArrayBuffer): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const uint8Array = new Uint8Array(buffer);
+      this.outputStream.write(uint8Array, (err) => (err ? reject(err) : resolve()));
+    });
+  }
+
+  async close(): Promise<void> {
+    if (this.outputStream instanceof fs.WriteStream) {
+      return new Promise((resolve, reject) => {
+        const stream = this.outputStream as fs.WriteStream;
+        stream.close((err) => (err ? reject(err) : resolve()));
+      });
+    }
+  }
+}
+*/

package/src/filesystems/node-filesystem.ts

@@ -1,87 +1,52 @@
-import {FileSystem, RandomAccessReadFileSystem} from '@loaders.gl/loader-utils';
-import fs from 'fs';
+// loaders.gl, MIT license
+
+import {Stat, RandomAccessFileSystem} from '@loaders.gl/loader-utils';
 import fsPromise from 'fs/promises';
+import {NodeFile} from './node-file';
+import {fetchNode} from './fetch-node';
 
 // import {fetchFile} from "../fetch/fetch-file"
 // import {selectLoader} from "../api/select-loader";
 
-type Stat = {
-  size: number;
-  isDirectory: () => boolean;
-  info?: fs.Stats;
-};
-
-type ReadOptions = {
-  buffer?: Buffer;
-  offset?: number;
-  length?: number;
-  position?: number;
-};
-
 /**
  * FileSystem pass-through for Node.js
  * Compatible with BrowserFileSystem.
  * @param options
  */
-export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem {
+export class NodeFileSystem implements RandomAccessFileSystem {
+  readable: boolean = true;
+  writable: boolean = true;
+
   // implements FileSystem
-  constructor(options: {[key: string]: any}) {
-    this.fetch = options._fetch;
-  }
+  constructor() {}
 
   async readdir(dirname = '.', options?: {}): Promise<any[]> {
     return await fsPromise.readdir(dirname, options);
   }
 
-  async stat(path: string, options?: {}): Promise<Stat> {
-    const info = await fsPromise.stat(path, options);
-    return {size: Number(info.size), isDirectory: () => false, info};
+  async stat(path: string): Promise<Stat> {
+    const info = await fsPromise.stat(path, {bigint: true});
+    return {
+      size: Number(info.size),
+      bigsize: info.size,
+      isDirectory: info.isDirectory()
+    };
   }
 
-  async fetch(path: string, options: {[key: string]: any}) {
-    // Falls back to handle https:/http:/data: etc fetches
-    // eslint-disable-next-line
-    const fallbackFetch = options.fetch || this.fetch;
-    return fallbackFetch(path, options);
+  async unlink(path: string): Promise<void> {
+    return await fsPromise.unlink(path);
   }
 
-  // implements IRandomAccessFileSystem
-  async open(path: string, flags: string | number, mode?: any): Promise<number> {
-    return (await fsPromise.open(path, flags)) as unknown as number;
-  }
-
-  async close(fd: number): Promise<void> {
-    fs.close(fd);
+  async fetch(path: string, options: RequestInit): Promise<Response> {
+    return await fetchNode(path, options);
   }
 
-  async fstat(fd: number): Promise<Stat> {
-    return await new Promise((resolve, reject) =>
-      fs.fstat(fd, (err, info) => (err ? reject(err) : resolve(info)))
-    );
+  // implements IRandomAccessFileSystem
+  async openReadableFile(path: string, flags: 'r' = 'r'): Promise<NodeFile> {
+    return new NodeFile(path, flags);
   }
 
-  async read(
-    fd: number,
-    // @ts-ignore Possibly null
-    {buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions
-  ): Promise<{bytesRead: number; buffer: Uint8Array}> {
-    let totalBytesRead = 0;
-    // Read in loop until we get required number of bytes
-    while (totalBytesRead < length) {
-      const {bytesRead} = await new Promise<{bytesRead: number; buffer: Buffer}>(
-        // eslint-disable-next-line no-loop-func
-        (resolve, reject) =>
-          fs.read(
-            fd,
-            buffer,
-            offset + totalBytesRead,
-            length - totalBytesRead,
-            position + totalBytesRead,
-            (err, bytesRead, buffer) => (err ? reject(err) : resolve({bytesRead, buffer}))
-          )
-      );
-      totalBytesRead += bytesRead;
-    }
-    return {bytesRead: totalBytesRead, buffer};
+  async openWritableFile(path: string, flags: 'w' | 'wx' = 'w', mode?: any): Promise<NodeFile> {
+    return new NodeFile(path, flags, mode);
   }
 }

package/src/{node/fetch/utils → filesystems}/stream-utils.node.ts

@@ -1,14 +1,16 @@
 // loaders.gl, MIT license
 
 import zlib from 'zlib';
+import {Readable} from 'stream';
 
-import {toArrayBuffer} from './decode-data-uri.node';
+const isArrayBuffer = (x) => x && x instanceof ArrayBuffer;
+const isBuffer = (x) => x && x instanceof Buffer;
 
 /**
  *
  */
-export function decompressReadStream(readStream, headers) {
-  switch (headers.get('content-encoding')) {
+export function decompressReadStream(readStream: Readable, headers?: Headers) {
+  switch (headers?.get('content-encoding')) {
     case 'br':
       return readStream.pipe(zlib.createBrotliDecompress());
     case 'gzip':
@@ -77,3 +79,40 @@ export function concatenateArrayBuffers(sources: (ArrayBuffer | Uint8Array)[]):
   // We work with ArrayBuffers, discard the typed array wrapper
   return result.buffer;
 }
+
+/**
+ * @param data
+ * @todo Duplicate of core
+ */
+export function toArrayBuffer(data: unknown): ArrayBuffer {
+  if (isArrayBuffer(data)) {
+    return data as ArrayBuffer;
+  }
+
+  // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+  if (isBuffer(data)) {
+    // @ts-expect-error
+    const typedArray = new Uint8Array(data);
+    return typedArray.buffer;
+  }
+
+  // Careful - Node Buffers will look like ArrayBuffers (keep after isBuffer)
+  if (ArrayBuffer.isView(data)) {
+    return data.buffer;
+  }
+
+  if (typeof data === 'string') {
+    const text = data;
+    const uint8Array = new TextEncoder().encode(text);
+    return uint8Array.buffer;
+  }
+
+  // HACK to support Blob polyfill
+  // @ts-expect-error
+  if (data && typeof data === 'object' && data._toArrayBuffer) {
+    // @ts-expect-error
+    return data._toArrayBuffer();
+  }
+
+  throw new Error(`toArrayBuffer(${JSON.stringify(data, null, 2).slice(10)})`);
+}

package/src/images/encode-image.node.ts

@@ -0,0 +1,41 @@
+// Use stackgl modules for DOM-less reading and writing of images
+
+import savePixels from 'save-pixels';
+import ndarray from 'ndarray';
+import {bufferToArrayBuffer} from '../buffer/to-array-buffer.node';
+
+/**
+ * Returns data bytes representing a compressed image in PNG or JPG format,
+ * This data can be saved using file system (f) methods or
+ * used in a request.
+ * @param image to save
+ * @param options
+ * @param options.type='png' - png, jpg or image/png, image/jpg are valid
+ * @param options.dataURI - Whether to include a data URI header
+ * @return {*} bytes
+ */
+export function encodeImageToStreamNode(
+  image: {data: any; width: number; height: number},
+  options: {type?: string; dataURI?: string}
+) {
+  // Support MIME type strings
+  const type = options.type ? options.type.replace('image/', '') : 'jpeg';
+  const pixels = ndarray(image.data, [image.width, image.height, 4], [4, image.width * 4, 1], 0);
+
+  // Note: savePixels returns a stream
+  return savePixels(pixels, type, options);
+}
+
+export function encodeImageNode(image, options) {
+  const imageStream = encodeImageToStreamNode(image, options);
+
+  return new Promise((resolve) => {
+    const buffers: any[] = [];
+    imageStream.on('data', (buffer) => buffers.push(buffer));
+    // TODO - convert to arraybuffer?
+    imageStream.on('end', () => {
+      const buffer = Buffer.concat(buffers);
+      resolve(bufferToArrayBuffer(buffer));
+    });
+  });
+}

package/src/images/parse-image.node.ts

@@ -0,0 +1,53 @@
+// loaders.gl, MIT license
+
+import getPixels from 'get-pixels';
+
+/** Declares which image format mime types this loader polyfill supports */
+export const NODE_FORMAT_SUPPORT = ['image/png', 'image/jpeg', 'image/gif'];
+
+// Note: These types are also defined in @loaders.gl/images and need to be kept in sync
+type NDArray = {
+  shape: number[];
+  data: Uint8Array;
+  width: number;
+  height: number;
+  components: number;
+  layers: number[];
+};
+
+export async function parseImageNode(arrayBuffer: ArrayBuffer, mimeType: string): Promise<NDArray> {
+  if (!mimeType) {
+    throw new Error('MIMEType is required to parse image under Node.js');
+  }
+
+  const buffer = arrayBuffer instanceof Buffer ? arrayBuffer : Buffer.from(arrayBuffer);
+  const ndarray = await getPixelsAsync(buffer, mimeType);
+  return ndarray;
+}
+
+// TODO - check if getPixels callback is asynchronous if provided with buffer input
+// if not, parseImage can be a sync function
+function getPixelsAsync(buffer: Buffer, mimeType: string): Promise<NDArray> {
+  return new Promise<NDArray>((resolve) =>
+    getPixels(buffer, mimeType, (err, ndarray) => {
+      if (err) {
+        throw err;
+      }
+
+      const shape = [...ndarray.shape];
+      const layers = ndarray.shape.length === 4 ? ndarray.shape.shift() : 1;
+      const data = ndarray.data instanceof Buffer ? new Uint8Array(ndarray.data) : ndarray.data;
+
+      // extract width/height etc
+      resolve({
+        shape,
+        data,
+        width: ndarray.shape[0],
+        height: ndarray.shape[1],
+        components: ndarray.shape[2],
+        // TODO - error
+        layers: layers ? [layers] : []
+      });
+    })
+  );
+}

package/src/index.browser.ts

@@ -1,14 +1,10 @@
 // loaders.gl, MIT License
 
-import {allSettled} from './promise/all-settled';
-
-if (!('allSettled' in Promise)) {
-  // @ts-ignore
-  Promise.allSettled = allSettled;
-}
-
-// eslint-disable-next-line @typescript-eslint/no-empty-function
 export function installFilePolyfills() {}
 
 // Dummy export to avoid import errors in browser tests
 export const NodeFileSystem = null;
+
+export function fetchNode(path: string, options: RequestInit): Promise<Response> {
+  throw new Error('fetchNode not available in browser');
+}