@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.14

package/src/loader-utils/lib/binary-utils/memory-copy-utils.ts

@@ -0,0 +1,69 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {assert} from '../env-utils/assert';
+
+/**
+ * Calculate new size of an arrayBuffer to be aligned to an n-byte boundary
+ * This function increases `byteLength` by the minimum delta,
+ * allowing the total length to be divided by `padding`
+ * @param byteLength
+ * @param padding
+ */
+export function padToNBytes(byteLength: number, padding: number): number {
+  assert(byteLength >= 0); // `Incorrect 'byteLength' value: ${byteLength}`
+  assert(padding > 0); // `Incorrect 'padding' value: ${padding}`
+  return (byteLength + (padding - 1)) & ~(padding - 1);
+}
+
+/**
+ * Creates a new Uint8Array based on two different ArrayBuffers
+ * @param targetBuffer The first buffer.
+ * @param sourceBuffer The second buffer.
+ * @return The new ArrayBuffer created out of the two.
+ */
+export function copyArrayBuffer(
+  targetBuffer: ArrayBuffer,
+  sourceBuffer: ArrayBuffer,
+  byteOffset: number,
+  byteLength: number = sourceBuffer.byteLength
+): ArrayBuffer {
+  const targetArray = new Uint8Array(targetBuffer, byteOffset, byteLength);
+  const sourceArray = new Uint8Array(sourceBuffer);
+  targetArray.set(sourceArray);
+  return targetBuffer;
+}
+
+/**
+ * Copy from source to target at the targetOffset
+ *
+ * @param source - The data to copy
+ * @param target - The destination to copy data into
+ * @param targetOffset - The start offset into target to place the copied data
+ * @returns the new offset taking into account proper padding
+ */
+export function copyToArray(source: ArrayBuffer | any, target: any, targetOffset: number): number {
+  let sourceArray;
+
+  if (source instanceof ArrayBuffer) {
+    sourceArray = new Uint8Array(source);
+  } else {
+    // Pack buffer onto the big target array
+    //
+    // 'source.data.buffer' could be a view onto a larger buffer.
+    // We MUST use this constructor to ensure the byteOffset and byteLength is
+    // set to correct values from 'source.data' and not the underlying
+    // buffer for target.set() to work properly.
+    const srcByteOffset = source.byteOffset;
+    const srcByteLength = source.byteLength;
+    // In gltf parser it is set as "arrayBuffer" instead of "buffer"
+    // https://github.com/visgl/loaders.gl/blob/1e3a82a0a65d7b6a67b1e60633453e5edda2960a/modules/gltf/src/lib/parse-gltf.js#L85
+    sourceArray = new Uint8Array(source.buffer || source.arrayBuffer, srcByteOffset, srcByteLength);
+  }
+
+  // Pack buffer onto the big target array
+  target.set(sourceArray, targetOffset);
+
+  return targetOffset + padToNBytes(sourceArray.byteLength, 4);
+}

package/src/loader-utils/lib/crs/crs-reprojection.ts

@@ -0,0 +1,52 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ReadonlyCRSDefinition} from '@math.gl/geospatial/crs';
+
+/**
+ * Common opt-in coordinate transformation options for vector loaders.
+ *
+ * Loader geometry and bounds use `xy` order (x/easting or longitude first), regardless of the
+ * authoritative axis order advertised by a CRS. The CRS definition itself is preserved separately
+ * in metadata. When `reproject` is true, `targetCrs` defaults to WGS84 for compatibility.
+ */
+export type CRSReprojectionOptions = {
+  /** Transform decoded coordinates into the requested output CRS. Defaults to `false`. */
+  reproject?: boolean;
+  /** CRS of returned coordinates. Defaults to `WGS84` when reprojection is enabled. */
+  targetCrs?: ReadonlyCRSDefinition;
+};
+
+/** Stable categories for failures while an opt-in CRS transformation is being prepared. */
+export type CRSReprojectionErrorCode =
+  | 'missing-source-crs'
+  | 'unsupported-source-crs'
+  | 'unsupported-target-crs'
+  | 'transformation-failed';
+
+/** Error thrown when a requested CRS transformation cannot be performed. */
+export class CRSReprojectionError extends Error {
+  /** Machine-readable reason for the failed transformation request. */
+  readonly code: CRSReprojectionErrorCode;
+  /** Source CRS when it was discovered or supplied. */
+  readonly sourceCrs?: ReadonlyCRSDefinition;
+  /** Requested output CRS when it was supplied. */
+  readonly targetCrs?: ReadonlyCRSDefinition;
+
+  constructor(
+    code: CRSReprojectionErrorCode,
+    message: string,
+    options: {
+      sourceCrs?: ReadonlyCRSDefinition;
+      targetCrs?: ReadonlyCRSDefinition;
+      cause?: unknown;
+    } = {}
+  ) {
+    super(message, {cause: options.cause});
+    this.name = 'CRSReprojectionError';
+    this.code = code;
+    this.sourceCrs = options.sourceCrs;
+    this.targetCrs = options.targetCrs;
+  }
+}

package/src/loader-utils/lib/env-utils/assert.ts

@@ -0,0 +1,13 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * Throws an `Error` with the optional `message` if `condition` is falsy
+ * @note Replacement for the external assert method to reduce bundle size
+ */
+export function assert(condition: any, message?: string): void {
+  if (!condition) {
+    throw new Error(message || 'loader assertion failed.');
+  }
+}

package/src/loader-utils/lib/env-utils/globals.ts

@@ -0,0 +1,36 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// Purpose: include this in your module to avoid
+// dependencies on micro modules like 'global' and 'is-browser';
+
+/* eslint-disable no-restricted-globals */
+const globals = {
+  self: typeof self !== 'undefined' && self,
+  window: typeof window !== 'undefined' && window,
+  global: typeof global !== 'undefined' && global,
+  document: typeof document !== 'undefined' && document
+};
+
+type obj = {[key: string]: any};
+const self_: obj = globals.self || globals.window || globals.global || {};
+const window_: obj = globals.window || globals.self || globals.global || {};
+const global_: obj = globals.global || globals.self || globals.window || {};
+const document_: obj = globals.document || {};
+
+export {self_ as self, window_ as window, global_ as global, document_ as document};
+
+/** true if running in a browser */
+export const isBrowser: boolean =
+  // @ts-ignore process does not exist on browser
+  Boolean(typeof process !== 'object' || String(process) !== '[object process]' || process.browser);
+
+/** true if running in a worker thread */
+export const isWorker: boolean = typeof importScripts === 'function';
+
+// Extract node major version
+const matches =
+  typeof process !== 'undefined' && process.version && /v([0-9]*)/.exec(process.version);
+/** Major Node version (as a number) */
+export const nodeVersion: number = (matches && parseFloat(matches[1])) || 0;

package/src/loader-utils/lib/file-provider/data-view-file.ts

@@ -0,0 +1,83 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {FileProviderInterface} from './file-provider-interface';
+import {copyToArrayBuffer} from '../binary-utils/memory-conversion-utils';
+
+/**
+ * Checks if bigint can be converted to number and convert it if possible
+ * @param bigint bigint to be converted
+ * @returns number
+ */
+const toNumber = (bigint: bigint) => {
+  if (bigint > Number.MAX_SAFE_INTEGER) {
+    throw new Error('Offset is out of bounds');
+  }
+  return Number(bigint);
+};
+
+/**
+ * Provides file data using DataView
+ * @deprecated - will be replaced with ReadableFile
+ */
+export class DataViewFile implements FileProviderInterface {
+  /** The DataView from which data is provided */
+  private file: DataView;
+
+  constructor(file: DataView) {
+    this.file = file;
+  }
+
+  async destroy(): Promise<void> {}
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: bigint): Promise<number> {
+    return this.file.getUint8(toNumber(offset));
+  }
+
+  /**
+   * Gets an unsigned 16-bit intege at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: bigint): Promise<number> {
+    return this.file.getUint16(toNumber(offset), true);
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: bigint): Promise<number> {
+    return this.file.getUint32(toNumber(offset), true);
+  }
+
+  /**
+   * Gets an unsigned 64-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: bigint): Promise<bigint> {
+    return this.file.getBigUint64(toNumber(offset), true);
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    return copyToArrayBuffer(
+      this.file.buffer,
+      toNumber(startOffset + BigInt(this.file.byteOffset)),
+      toNumber(endOffset - startOffset)
+    );
+  }
+
+  /** the length (in bytes) of the data. */
+  get length() {
+    return BigInt(this.file.byteLength);
+  }
+}

package/src/loader-utils/lib/file-provider/file-handle-file.ts

@@ -0,0 +1,114 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {FileProviderInterface} from './file-provider-interface';
+import {NodeFileFacade as NodeFile} from '../files/node-file-facade';
+
+/**
+ * Provides file data using node fs library
+ * @deprecated - will be replaced with ReadableFile
+ */
+export class FileHandleFile implements FileProviderInterface {
+  /** The FileHandle from which data is provided */
+  private file: NodeFile;
+
+  /** Create a new FileHandleFile */
+  constructor(path: string, append: boolean = false) {
+    this.file = new NodeFile(path, append ? 'a+' : 'r');
+  }
+
+  /**
+   * Truncates the file descriptor.
+   * @param length desired file lenght
+   */
+  async truncate(length: number): Promise<void> {
+    await this.file.truncate(length);
+  }
+
+  /**
+   * Append data to a file.
+   * @param buffer data to append
+   */
+  async append(buffer: Uint8Array): Promise<void> {
+    await this.file.append(buffer);
+  }
+
+  /** Close file */
+  async destroy(): Promise<void> {
+    await this.file.close();
+  }
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 1);
+    const val = new Uint8Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 2);
+    const val = new Uint16Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 4);
+    const val = new Uint32Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: number | bigint): Promise<bigint> {
+    const arrayBuffer = await this.file.read(offset, 8);
+    const val = new BigInt64Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in byte, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    const bigLength = endOffset - startOffset;
+    if (bigLength > Number.MAX_SAFE_INTEGER) {
+      throw new Error('too big slice');
+    }
+    const length = Number(bigLength);
+    return await this.file.read(startOffset, length);
+  }
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  get length(): bigint {
+    return this.file.bigsize;
+  }
+}

package/src/loader-utils/lib/file-provider/file-provider-interface.ts

@@ -0,0 +1,60 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * Interface for providing file data
+ */
+export interface FileProviderInterface {
+  /**
+   * Cleanup class data
+   */
+  destroy(): Promise<void>;
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint8(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint16(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the file of the view where to read the data.
+   */
+  getUint32(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in byte, from the file of the view where to read the data.
+   */
+  getBigUint64(offset: bigint): Promise<bigint>;
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer>;
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  length: bigint;
+}
+
+/**
+ * Check is the object has FileProvider members
+ * @param fileProvider - tested object
+ */
+export const isFileProvider = (fileProvider: unknown) => {
+  return (
+    (fileProvider as FileProviderInterface)?.getUint8 &&
+    (fileProvider as FileProviderInterface)?.slice &&
+    (fileProvider as FileProviderInterface)?.length
+  );
+};

package/src/loader-utils/lib/file-provider/file-provider.ts

@@ -0,0 +1,130 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import {ReadableFile} from '../files/file';
+import {FileProviderInterface} from './file-provider-interface';
+
+/**
+ * Provides file data using range requests to the server
+ * @deprecated - will be replaced with ReadableFile
+ */
+export class FileProvider implements FileProviderInterface {
+  /** The File object from which data is provided */
+  private file: ReadableFile;
+  private size: bigint;
+
+  /** Create a new BrowserFile */
+  private constructor(file: ReadableFile, size: bigint | number) {
+    this.file = file;
+    this.size = BigInt(size);
+  }
+
+  static async create(file: ReadableFile): Promise<FileProvider> {
+    let size: bigint | number = 0n;
+    if (file.bigsize > 0n) {
+      size = file.bigsize;
+    } else if (file.size > 0) {
+      size = file.size;
+    } else {
+      const stats = await file.stat?.();
+      size = stats?.bigsize ?? 0n;
+    }
+
+    return new FileProvider(file, size);
+  }
+
+  /**
+   * Truncates the file descriptor.
+   * @param length desired file lenght
+   */
+  async truncate(length: number): Promise<void> {
+    throw new Error('file loaded via range requests cannot be changed');
+  }
+
+  /**
+   * Append data to a file.
+   * @param buffer data to append
+   */
+  async append(buffer: Uint8Array): Promise<void> {
+    throw new Error('file loaded via range requests cannot be changed');
+  }
+
+  /** Close file */
+  async destroy(): Promise<void> {
+    throw new Error('file loaded via range requests cannot be changed');
+  }
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 1);
+    const val = new Uint8Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 2);
+    const val = new Uint16Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 4);
+    const val = new Uint32Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: number | bigint): Promise<bigint> {
+    const arrayBuffer = await this.file.read(offset, 8);
+    const val = new BigInt64Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in byte, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffset: bigint | number, endOffset: bigint | number): Promise<ArrayBuffer> {
+    const bigLength = BigInt(endOffset) - BigInt(startOffset);
+    if (bigLength > Number.MAX_SAFE_INTEGER) {
+      throw new Error('too big slice');
+    }
+    const length = Number(bigLength);
+
+    return await this.file.read(startOffset, length);
+  }
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  get length(): bigint {
+    return this.size;
+  }
+}

package/src/loader-utils/lib/files/array-buffer-file.ts

@@ -0,0 +1,46 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ReadableFile, Stat} from './file';
+
+/** Random-access file over an in-memory ArrayBuffer without routing reads through Blob. */
+export class ArrayBufferFile implements ReadableFile {
+  /** Complete in-memory file. */
+  readonly handle: ArrayBuffer;
+  /** File length in bytes. */
+  readonly size: number;
+  /** File length in bytes as a bigint. */
+  readonly bigsize: bigint;
+  /** In-memory files do not have a URL. */
+  readonly url = '';
+
+  /** Creates a random-access view of an in-memory file. */
+  constructor(arrayBuffer: ArrayBuffer) {
+    this.handle = arrayBuffer;
+    this.size = arrayBuffer.byteLength;
+    this.bigsize = BigInt(this.size);
+  }
+
+  /** Releases the file; in-memory buffers do not require cleanup. */
+  async close(): Promise<void> {}
+
+  /** Returns the in-memory file size. */
+  async stat(): Promise<Stat> {
+    return {size: this.size, bigsize: this.bigsize, isDirectory: false};
+  }
+
+  /** Copies one byte range from the in-memory file. */
+  async read(
+    start: number | bigint = 0,
+    length?: number,
+    signal?: AbortSignal
+  ): Promise<ArrayBuffer> {
+    if (signal?.aborted) {
+      throw new Error('Request aborted');
+    }
+    const startOffset = Number(start);
+    const endOffset = length === undefined ? this.size : startOffset + length;
+    return this.handle.slice(startOffset, endOffset);
+  }
+}

package/src/loader-utils/lib/files/blob-file.ts

@@ -0,0 +1,45 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {ReadableFile} from './file';
+
+/**
+ * BlobFile provides a "file like interface" to the data in a Blob or File object
+ */
+export class BlobFile implements ReadableFile {
+  readonly handle: Blob;
+  readonly size: number;
+  readonly bigsize: bigint;
+  readonly url: string;
+
+  constructor(blob: Blob | File | ArrayBuffer) {
+    this.handle = blob instanceof ArrayBuffer ? new Blob([blob]) : blob;
+    this.size = blob instanceof ArrayBuffer ? blob.byteLength : blob.size;
+    this.bigsize = BigInt(this.size);
+    this.url = blob instanceof File ? blob.name : '';
+  }
+
+  async close() {}
+
+  async stat() {
+    return {
+      size: this.handle.size,
+      bigsize: BigInt(this.handle.size),
+      isDirectory: false
+    };
+  }
+
+  async read(start?: number | bigint, length?: number, signal?: AbortSignal): Promise<ArrayBuffer> {
+    if (signal?.aborted) {
+      throw new Error('Request aborted');
+    }
+    const arrayBuffer = await this.handle
+      .slice(Number(start), Number(start) + Number(length))
+      .arrayBuffer();
+    if (signal?.aborted) {
+      throw new Error('Request aborted');
+    }
+    return arrayBuffer;
+  }
+}