@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/polyfills/filesystems/node-file.ts

@@ -0,0 +1,169 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {resolvePath} from '../utils/path-utils';
+import type {FileStat} from '../utils/filesystem-types';
+import fs from 'fs';
+
+export class NodeFile {
+  handle: number;
+  size: number;
+  bigsize: bigint;
+  url: string;
+
+  constructor(path: string, flags: 'r' | 'w' | 'wx' | 'a+', mode?: number) {
+    path = resolvePath(path);
+    this.handle = fs.openSync(path, flags, mode);
+    const stats = fs.fstatSync(this.handle, {bigint: true});
+    this.size = Number(stats.size);
+    this.bigsize = stats.size;
+    this.url = path;
+  }
+
+  async close(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      fs.close(this.handle, err => (err ? reject(err) : resolve()));
+    });
+  }
+
+  async truncate(length: number): Promise<void> {
+    return new Promise((resolve, reject) => {
+      fs.ftruncate(this.handle, length, err => {
+        if (err) {
+          reject(err);
+        } else {
+          this.bigsize = BigInt(length);
+          this.size = Number(this.bigsize);
+          resolve();
+        }
+      });
+    });
+  }
+
+  async append(data: Uint8Array): Promise<void> {
+    return new Promise((resolve, reject) => {
+      fs.appendFile(this.handle, data, err => {
+        if (err) {
+          reject(err);
+        } else {
+          this.bigsize = this.bigsize + BigInt(data.length);
+          this.size = Number(this.bigsize);
+          resolve();
+        }
+      });
+    });
+  }
+
+  async stat(): Promise<FileStat> {
+    return await new Promise((resolve, reject) =>
+      fs.fstat(this.handle, {bigint: true}, (err, info) => {
+        const stats: FileStat = {
+          size: Number(info.size),
+          bigsize: info.size,
+          isDirectory: info.isDirectory()
+        };
+        if (err) {
+          reject(err);
+        } else {
+          resolve(stats);
+        }
+      })
+    );
+  }
+
+  async read(offset: number | bigint, length: number): Promise<ArrayBuffer> {
+    const arrayBuffer = new ArrayBuffer(length);
+    let bigOffset = BigInt(offset);
+
+    let totalBytesRead = 0;
+    const uint8Array = new Uint8Array(arrayBuffer);
+
+    let position;
+    // Read in loop until we get required number of bytes
+    while (length > 0) {
+      const bytesRead = await readBytes(this.handle, uint8Array, 0, length, bigOffset);
+
+      // Check if end of file reached
+      if (bytesRead === 0) {
+        break;
+      }
+
+      totalBytesRead += bytesRead;
+      bigOffset += BigInt(bytesRead);
+      length -= bytesRead;
+
+      // Advance position unless we are using built-in position advancement
+      if (position !== undefined) {
+        position += bytesRead;
+      }
+    }
+    return totalBytesRead < length ? arrayBuffer.slice(0, totalBytesRead) : arrayBuffer;
+  }
+
+  async write(
+    arrayBuffer: ArrayBuffer,
+    offset: number | bigint = 0,
+    length: number = arrayBuffer.byteLength
+  ): Promise<number> {
+    return new Promise((resolve, reject) => {
+      // TODO - Node.js doesn't offer write with bigint offsets???
+      const nOffset = Number(offset);
+      const uint8Array = new Uint8Array(arrayBuffer, Number(offset), length);
+      fs.write(this.handle, uint8Array, 0, length, nOffset, (err, bytesWritten) =>
+        err ? reject(err) : resolve(bytesWritten)
+      );
+    });
+  }
+}
+
+async function readBytes(
+  fd: number,
+  uint8Array: Uint8Array,
+  offset: number,
+  length: number,
+  position: number | bigint | null
+): Promise<number> {
+  return await new Promise<number>((resolve, reject) =>
+    fs.read(fd, uint8Array, offset, length, position, (err, bytesRead) =>
+      err ? reject(err) : resolve(bytesRead)
+    )
+  );
+}
+
+// TODO - implement streaming write
+/*
+export interface WriteStreamOptions {
+  flags?: string;
+  encoding?: 'utf8';
+  fd?: number;
+  mode?: number;
+  autoClose?: boolean;
+  start?: number;
+}
+
+export class NodeStreamWritableFile implements WritableFile {
+  outputStream: fs.WriteStream | Writable;
+
+  constructor(pathOrStream: string | Writable, options?: WriteStreamOptions) {
+    this.outputStream =
+      typeof pathOrStream === 'string' ? fs.createWriteStream(pathOrStream, options) : pathOrStream;
+  }
+
+  async write(buffer: ArrayBuffer): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const uint8Array = new Uint8Array(buffer);
+      this.outputStream.write(uint8Array, (err) => (err ? reject(err) : resolve()));
+    });
+  }
+
+  async close(): Promise<void> {
+    if (this.outputStream instanceof fs.WriteStream) {
+      return new Promise((resolve, reject) => {
+        const stream = this.outputStream as fs.WriteStream;
+        stream.close((err) => (err ? reject(err) : resolve()));
+      });
+    }
+  }
+}
+*/

package/src/polyfills/filesystems/node-filesystem.ts

@@ -0,0 +1,54 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import fsPromise from 'fs/promises';
+import type {FileStat} from '../utils/filesystem-types';
+import {NodeFile} from './node-file';
+import {fetchNode} from './fetch-node';
+
+// import {fetchFile} from "../fetch/fetch-file"
+// import {selectLoader} from "../api/select-loader";
+
+/**
+ * FileSystem pass-through for Node.js
+ * Compatible with BrowserFileSystem.
+ * @param options
+ */
+export class NodeFileSystem {
+  readable: boolean = true;
+  writable: boolean = true;
+
+  // implements FileSystem
+  constructor() {}
+
+  async readdir(dirname = '.', options?: {}): Promise<any[]> {
+    return await fsPromise.readdir(dirname, options);
+  }
+
+  async stat(path: string): Promise<FileStat> {
+    const info = await fsPromise.stat(path, {bigint: true});
+    return {
+      size: Number(info.size),
+      bigsize: info.size,
+      isDirectory: info.isDirectory()
+    };
+  }
+
+  async unlink(path: string): Promise<void> {
+    return await fsPromise.unlink(path);
+  }
+
+  async fetch(path: string, options: RequestInit): Promise<Response> {
+    return await fetchNode(path, options);
+  }
+
+  // implements IRandomAccessFileSystem
+  async openReadableFile(path: string, flags: 'r' = 'r'): Promise<NodeFile> {
+    return new NodeFile(path, flags);
+  }
+
+  async openWritableFile(path: string, flags: 'w' | 'wx' = 'w', mode?: any): Promise<NodeFile> {
+    return new NodeFile(path, flags, mode);
+  }
+}

package/src/polyfills/filesystems/stream-utils.node.ts

@@ -0,0 +1,81 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import zlib from 'zlib';
+import {Readable} from 'stream';
+import {toArrayBuffer} from '../utils/buffer-utils';
+
+/**
+ *
+ */
+export function decompressReadStream(readStream: Readable, headers?: Headers) {
+  switch (headers?.get('content-encoding')) {
+    case 'br':
+      return readStream.pipe(zlib.createBrotliDecompress());
+    case 'gzip':
+      return readStream.pipe(zlib.createGunzip());
+    case 'deflate':
+      return readStream.pipe(zlib.createDeflate());
+    default:
+      // No compression or an unknown one, just return it as is
+      return readStream;
+  }
+}
+
+/**
+ *
+ * @param readStream
+ * @returns
+ */
+export async function concatenateReadStream(readStream): Promise<ArrayBuffer> {
+  const arrayBufferChunks: ArrayBuffer[] = [];
+
+  return await new Promise((resolve, reject) => {
+    readStream.on('error', error => reject(error));
+
+    // Once the readable callback has been added, stream switches to "flowing mode"
+    // In Node 10 (but not 12 and 14) this causes `data` and `end` to never be called unless we read data here
+    readStream.on('readable', () => readStream.read());
+
+    readStream.on('data', chunk => {
+      if (typeof chunk === 'string') {
+        reject(new Error('Read stream not binary'));
+      }
+      arrayBufferChunks.push(toArrayBuffer(chunk));
+    });
+
+    readStream.on('end', () => {
+      const arrayBuffer = concatenateArrayBuffers(arrayBufferChunks);
+      resolve(arrayBuffer);
+    });
+  });
+}
+
+/**
+ * Concatenate a sequence of ArrayBuffers
+ * @return A concatenated ArrayBuffer
+ * @note duplicates loader-utils since polyfills should be independent
+ */
+export function concatenateArrayBuffers(sources: (ArrayBuffer | Uint8Array)[]): ArrayBuffer {
+  // Make sure all inputs are wrapped in typed arrays
+  const sourceArrays = sources.map(source2 =>
+    source2 instanceof ArrayBuffer ? new Uint8Array(source2) : source2
+  );
+
+  // Get length of all inputs
+  const byteLength = sourceArrays.reduce((length, typedArray) => length + typedArray.byteLength, 0);
+
+  // Allocate array with space for all inputs
+  const result = new Uint8Array(byteLength);
+
+  // Copy the subarrays
+  let offset = 0;
+  for (const sourceArray of sourceArrays) {
+    result.set(sourceArray, offset);
+    offset += sourceArray.byteLength;
+  }
+
+  // We work with ArrayBuffers, discard the typed array wrapper
+  return result.buffer;
+}

package/src/polyfills/images/encode-image-node.ts

@@ -0,0 +1,45 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// Use stackgl modules for DOM-less reading and writing of images
+
+import savePixels from 'save-pixels';
+import ndarray from 'ndarray';
+import {bufferToArrayBuffer} from '../buffer/to-array-buffer.node.js';
+
+/**
+ * Returns data bytes representing a compressed image in PNG or JPG format,
+ * This data can be saved using file system (f) methods or
+ * used in a request.
+ * @param image to save
+ * @param options
+ * @param options.type='png' - png, jpg or image/png, image/jpg are valid
+ * @param options.dataURI - Whether to include a data URI header
+ * @return {*} bytes
+ */
+export function encodeImageToStreamNode(
+  image: {data: any; width: number; height: number},
+  options: {type?: string; dataURI?: string}
+) {
+  // Support MIME type strings
+  const type = options.type ? options.type.replace('image/', '') : 'jpeg';
+  const pixels = ndarray(image.data, [image.width, image.height, 4], [4, image.width * 4, 1], 0);
+
+  // Note: savePixels returns a stream
+  return savePixels(pixels, type, options);
+}
+
+export function encodeImageNode(image, options) {
+  const imageStream = encodeImageToStreamNode(image, options);
+
+  return new Promise(resolve => {
+    const buffers: any[] = [];
+    imageStream.on('data', buffer => buffers.push(buffer));
+    // TODO - convert to arraybuffer?
+    imageStream.on('end', () => {
+      const buffer = Buffer.concat(buffers);
+      resolve(bufferToArrayBuffer(buffer));
+    });
+  });
+}

package/src/polyfills/images/encode-image.node.ts.disabled

@@ -0,0 +1,41 @@
+// Use stackgl modules for DOM-less reading and writing of images
+
+import savePixels from 'save-pixels';
+import ndarray from 'ndarray';
+import {bufferToArrayBuffer} from '../buffer/to-array-buffer.node';
+
+/**
+ * Returns data bytes representing a compressed image in PNG or JPG format,
+ * This data can be saved using file system (f) methods or
+ * used in a request.
+ * @param image to save
+ * @param options
+ * @param options.type='png' - png, jpg or image/png, image/jpg are valid
+ * @param options.dataURI - Whether to include a data URI header
+ * @return {*} bytes
+ */
+export function encodeImageToStreamNode(
+  image: {data: any; width: number; height: number},
+  options: {type?: string; dataURI?: string}
+) {
+  // Support MIME type strings
+  const type = options.type ? options.type.replace('image/', '') : 'jpeg';
+  const pixels = ndarray(image.data, [image.width, image.height, 4], [4, image.width * 4, 1], 0);
+
+  // Note: savePixels returns a stream
+  return savePixels(pixels, type, options);
+}
+
+export function encodeImageNode(image, options) {
+  const imageStream = encodeImageToStreamNode(image, options);
+
+  return new Promise((resolve) => {
+    const buffers: any[] = [];
+    imageStream.on('data', (buffer) => buffers.push(buffer));
+    // TODO - convert to arraybuffer?
+    imageStream.on('end', () => {
+      const buffer = Buffer.concat(buffers);
+      resolve(bufferToArrayBuffer(buffer));
+    });
+  });
+}

package/src/polyfills/images/image-codecs-node.ts

@@ -0,0 +1,40 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {parseImageNode as parseImageImplementation} from './parse-image-node';
+import type {encodeImageNode as encodeImageImplementation} from './encode-image-node';
+
+export {NODE_FORMAT_SUPPORT} from './node-image-formats';
+
+/** Loads the optional image decoder only when pixel decoding is requested. */
+export async function parseImageNode(
+  ...parameters: Parameters<typeof parseImageImplementation>
+): Promise<Awaited<ReturnType<typeof parseImageImplementation>>> {
+  let decoder: typeof import('./parse-image-node');
+  try {
+    decoder = await import('./parse-image-node');
+  } catch (error) {
+    throw new Error(
+      'Node image decoding requires get-pixels. Install get-pixels to decode images.',
+      {cause: error}
+    );
+  }
+  return decoder.parseImageNode(...parameters);
+}
+
+/** Loads the optional image encoder only when image encoding is requested. */
+export async function encodeImageNode(
+  ...parameters: Parameters<typeof encodeImageImplementation>
+): Promise<Awaited<ReturnType<typeof encodeImageImplementation>>> {
+  let encoder: typeof import('./encode-image-node');
+  try {
+    encoder = await import('./encode-image-node');
+  } catch (error) {
+    throw new Error(
+      'Node image encoding requires save-pixels and ndarray. Install save-pixels and ndarray to encode images.',
+      {cause: error}
+    );
+  }
+  return encoder.encodeImageNode(...parameters);
+}

package/src/polyfills/images/node-image-bitmap.ts

@@ -0,0 +1,81 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+type NodeImageData = {
+  data: Uint8Array;
+  width: number;
+  height: number;
+};
+
+/**
+ * Minimal Node.js `ImageBitmap` polyfill backed by decoded pixel data.
+ * This is intentionally limited to the functionality needed by `ImageBitmapLoader`
+ * and `getImageBitmapData()`.
+ */
+export class NodeImageBitmap {
+  /** Width of the decoded bitmap in pixels. */
+  width: number;
+
+  /** Height of the decoded bitmap in pixels. */
+  height: number;
+
+  /** Backing pixel data retained until `close()` is called. */
+  private data: Uint8Array | null;
+
+  /** Creates a bitmap wrapper around decoded Node.js image data. */
+  constructor(imageData: NodeImageData) {
+    if (!imageData || !imageData.data || !imageData.width || !imageData.height) {
+      throw new Error('NodeImageBitmap requires decoded image data');
+    }
+
+    this.width = imageData.width;
+    this.height = imageData.height;
+    this.data = imageData.data;
+  }
+
+  /** Releases the backing pixel data and invalidates further reads. */
+  close(): void {
+    this.width = 0;
+    this.height = 0;
+    this.data = null;
+  }
+
+  /** Returns the decoded image data backing this bitmap. */
+  _getImageData(): NodeImageData {
+    if (!this.data) {
+      throw new Error('ImageBitmap is closed');
+    }
+
+    return {
+      data: this.data,
+      width: this.width,
+      height: this.height
+    };
+  }
+}
+
+/**
+ * Detects whether a value is a Node.js `ImageBitmap` installed by `@loaders.gl/core/polyfills`.
+ */
+export function isNodeImageBitmap(image: unknown): image is NodeImageBitmap {
+  return image instanceof NodeImageBitmap;
+}
+
+/**
+ * Wraps decoded Node.js image data in the loaders.gl `NodeImageBitmap` polyfill.
+ */
+export function createImageBitmapNode(imageData: NodeImageData): NodeImageBitmap {
+  return new NodeImageBitmap(imageData);
+}
+
+/**
+ * Returns decoded image data from a Node.js `ImageBitmap`.
+ */
+export function getImageBitmapDataNode(image: unknown): NodeImageData {
+  if (!isNodeImageBitmap(image)) {
+    throw new Error('getImageBitmapData');
+  }
+
+  return image._getImageData();
+}

package/src/polyfills/images/node-image-formats.ts

@@ -0,0 +1,6 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Image MIME types supported when the optional Node decoder is installed. */
+export const NODE_FORMAT_SUPPORT = ['image/png', 'image/jpeg', 'image/gif'];

package/src/polyfills/images/parse-image-node.ts

@@ -0,0 +1,54 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import getPixels from 'get-pixels';
+
+export {NODE_FORMAT_SUPPORT} from './node-image-formats';
+
+// Note: These types are also defined in @loaders.gl/images and need to be kept in sync
+type NDArray = {
+  shape: number[];
+  data: Uint8Array;
+  width: number;
+  height: number;
+  components: number;
+  layers: number[];
+};
+
+export async function parseImageNode(arrayBuffer: ArrayBuffer, mimeType: string): Promise<NDArray> {
+  if (!mimeType) {
+    throw new Error('MIMEType is required to parse image under Node.js');
+  }
+
+  const buffer = arrayBuffer instanceof Buffer ? arrayBuffer : Buffer.from(arrayBuffer);
+  const ndarray = await getPixelsAsync(buffer, mimeType);
+  return ndarray;
+}
+
+// TODO - check if getPixels callback is asynchronous if provided with buffer input
+// if not, parseImage can be a sync function
+function getPixelsAsync(buffer: Buffer, mimeType: string): Promise<NDArray> {
+  return new Promise<NDArray>(resolve =>
+    getPixels(buffer as any, mimeType, (err, ndarray) => {
+      if (err) {
+        throw err;
+      }
+
+      const shape = [...ndarray.shape];
+      const layers = ndarray.shape.length === 4 ? ndarray.shape.shift() : 1;
+      const data = ndarray.data instanceof Buffer ? new Uint8Array(ndarray.data) : ndarray.data;
+
+      // extract width/height etc
+      resolve({
+        shape,
+        data,
+        width: ndarray.shape[0],
+        height: ndarray.shape[1],
+        components: ndarray.shape[2],
+        // TODO - error
+        layers: layers ? [layers] : []
+      });
+    })
+  );
+}

package/src/polyfills/images/parse-image.node.ts.disabled

@@ -0,0 +1,55 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import getPixels from 'get-pixels';
+
+/** Declares which image format mime types this loader polyfill supports */
+export const NODE_FORMAT_SUPPORT = ['image/png', 'image/jpeg', 'image/gif'];
+
+// Note: These types are also defined in @loaders.gl/images and need to be kept in sync
+type NDArray = {
+  shape: number[];
+  data: Uint8Array;
+  width: number;
+  height: number;
+  components: number;
+  layers: number[];
+};
+
+export async function parseImageNode(arrayBuffer: ArrayBuffer, mimeType: string): Promise<NDArray> {
+  if (!mimeType) {
+    throw new Error('MIMEType is required to parse image under Node.js');
+  }
+
+  const buffer = arrayBuffer instanceof Buffer ? arrayBuffer : Buffer.from(arrayBuffer);
+  const ndarray = await getPixelsAsync(buffer, mimeType);
+  return ndarray;
+}
+
+// TODO - check if getPixels callback is asynchronous if provided with buffer input
+// if not, parseImage can be a sync function
+function getPixelsAsync(buffer: Buffer, mimeType: string): Promise<NDArray> {
+  return new Promise<NDArray>((resolve) =>
+    getPixels(buffer as any, mimeType, (err, ndarray) => {
+      if (err) {
+        throw err;
+      }
+
+      const shape = [...ndarray.shape];
+      const layers = ndarray.shape.length === 4 ? ndarray.shape.shift() : 1;
+      const data = ndarray.data instanceof Buffer ? new Uint8Array(ndarray.data) : ndarray.data;
+
+      // extract width/height etc
+      resolve({
+        shape,
+        data,
+        width: ndarray.shape[0],
+        height: ndarray.shape[1],
+        components: ndarray.shape[2],
+        // TODO - error
+        layers: layers ? [layers] : []
+      });
+    })
+  );
+}

package/src/polyfills/index.browser.ts

@@ -0,0 +1,23 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+
+export function installFilePolyfills() {}
+
+/** Browser placeholder for the Node path-prefix configuration. */
+export function setPathPrefix(_prefix: string): void {}
+
+/** Browser placeholder for the Node path-alias configuration. */
+export function addAliases(_aliases: Record<string, string>): void {}
+
+// Dummy export to avoid import errors in browser tests
+export const NodeFileSystem = null;
+
+export function fetchNode(path: string, options: RequestInit): Promise<Response> {
+  throw new Error('fetchNode not available in browser');
+}
+
+// Ensure process and process.env are defined, as Node code tends to crash on these
+// @ts-ignore
+globalThis.process = globalThis.process || {};
+// @ts-ignore
+globalThis.process.env = globalThis.process.env || {};