@loaders.gl/core 5.0.0-alpha.12 → 5.0.0-alpha.13

package/src/loader-utils/lib/binary-utils/memory-conversion-utils.ts

@@ -0,0 +1,105 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {isSharedArrayBuffer} from '../javascript-utils/is-type';
+import * as node from '../node/buffer';
+
+/**
+ * Check for Node.js `Buffer` (without triggering bundler to include Buffer polyfill on browser)
+ */
+export function isBuffer(value: any): value is Buffer {
+  return value && typeof value === 'object' && value.isBuffer;
+}
+
+/**
+ * Converts to Node.js `Buffer` (without triggering bundler to include Buffer polyfill on browser)
+ * @todo better data type
+ */
+export function toBuffer(data: unknown): Buffer {
+  return node.toBuffer ? node.toBuffer(data as any) : (data as Buffer);
+}
+
+/**
+ * Convert an object to an array buffer. Handles SharedArrayBuffers.
+ */
+export function toArrayBuffer(
+  data: Buffer | ArrayBufferLike | ArrayBufferView | string | Blob
+): ArrayBuffer {
+  // Note: Should be called first, Buffers can trigger other detections below
+  if (isBuffer(data)) {
+    return node.toArrayBuffer(data);
+  }
+
+  if (data instanceof ArrayBuffer) {
+    return data;
+  }
+
+  if (isSharedArrayBuffer(data)) {
+    return copyToArrayBuffer(data);
+  }
+
+  // Node Buffers look like Uint8Arrays (Check isBuffer first)
+  if (ArrayBuffer.isView(data)) {
+    // TODO - ArrayBufferLike type mess
+    const buffer = data.buffer as ArrayBuffer;
+    if (data.byteOffset === 0 && data.byteLength === data.buffer.byteLength) {
+      return buffer;
+    }
+    return buffer.slice(data.byteOffset, data.byteOffset + data.byteLength);
+  }
+
+  if (typeof data === 'string') {
+    const text = data;
+    const uint8Array = new TextEncoder().encode(text);
+    return uint8Array.buffer;
+  }
+
+  // HACK to support Blob polyfill
+  if (data && typeof data === 'object' && (data as any)._toArrayBuffer) {
+    return (data as any)._toArrayBuffer();
+  }
+
+  throw new Error('toArrayBuffer');
+}
+
+/** Ensure that SharedArrayBuffers are copied into ArrayBuffers */
+export function ensureArrayBuffer(bufferSource: ArrayBufferLike | ArrayBufferView): ArrayBuffer {
+  if (bufferSource instanceof ArrayBuffer) {
+    return bufferSource;
+  }
+
+  if (isSharedArrayBuffer(bufferSource)) {
+    return copyToArrayBuffer(bufferSource);
+  }
+
+  const {buffer, byteOffset, byteLength} = bufferSource;
+  if (buffer instanceof ArrayBuffer && byteOffset === 0 && byteLength === buffer.byteLength) {
+    return buffer;
+  }
+  return copyToArrayBuffer(buffer, byteOffset, byteLength);
+}
+
+/** Copies an ArrayBuffer or a section of an ArrayBuffer to a new ArrayBuffer, handles SharedArrayBuffers */
+export function copyToArrayBuffer(
+  buffer: ArrayBufferLike,
+  byteOffset = 0,
+  byteLength = buffer.byteLength - byteOffset
+): ArrayBuffer {
+  const view = new Uint8Array(buffer, byteOffset, byteLength);
+  const copy = new Uint8Array(view.length);
+  copy.set(view);
+  return copy.buffer;
+}
+
+/** Convert an object to an ArrayBufferView, handles SharedArrayBuffers */
+export function toArrayBufferView(
+  data: ArrayBufferLike | ArrayBufferView
+): ArrayBuffer | ArrayBufferView {
+  if (ArrayBuffer.isView(data)) {
+    return data;
+  }
+
+  // Create a view to support ArrayBufferLike sources such as SharedArrayBuffer
+  return new Uint8Array(data);
+}

package/src/loader-utils/lib/binary-utils/memory-copy-utils.ts

@@ -0,0 +1,69 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {assert} from '../env-utils/assert';
+
+/**
+ * Calculate new size of an arrayBuffer to be aligned to an n-byte boundary
+ * This function increases `byteLength` by the minimum delta,
+ * allowing the total length to be divided by `padding`
+ * @param byteLength
+ * @param padding
+ */
+export function padToNBytes(byteLength: number, padding: number): number {
+  assert(byteLength >= 0); // `Incorrect 'byteLength' value: ${byteLength}`
+  assert(padding > 0); // `Incorrect 'padding' value: ${padding}`
+  return (byteLength + (padding - 1)) & ~(padding - 1);
+}
+
+/**
+ * Creates a new Uint8Array based on two different ArrayBuffers
+ * @param targetBuffer The first buffer.
+ * @param sourceBuffer The second buffer.
+ * @return The new ArrayBuffer created out of the two.
+ */
+export function copyArrayBuffer(
+  targetBuffer: ArrayBuffer,
+  sourceBuffer: ArrayBuffer,
+  byteOffset: number,
+  byteLength: number = sourceBuffer.byteLength
+): ArrayBuffer {
+  const targetArray = new Uint8Array(targetBuffer, byteOffset, byteLength);
+  const sourceArray = new Uint8Array(sourceBuffer);
+  targetArray.set(sourceArray);
+  return targetBuffer;
+}
+
+/**
+ * Copy from source to target at the targetOffset
+ *
+ * @param source - The data to copy
+ * @param target - The destination to copy data into
+ * @param targetOffset - The start offset into target to place the copied data
+ * @returns the new offset taking into account proper padding
+ */
+export function copyToArray(source: ArrayBuffer | any, target: any, targetOffset: number): number {
+  let sourceArray;
+
+  if (source instanceof ArrayBuffer) {
+    sourceArray = new Uint8Array(source);
+  } else {
+    // Pack buffer onto the big target array
+    //
+    // 'source.data.buffer' could be a view onto a larger buffer.
+    // We MUST use this constructor to ensure the byteOffset and byteLength is
+    // set to correct values from 'source.data' and not the underlying
+    // buffer for target.set() to work properly.
+    const srcByteOffset = source.byteOffset;
+    const srcByteLength = source.byteLength;
+    // In gltf parser it is set as "arrayBuffer" instead of "buffer"
+    // https://github.com/visgl/loaders.gl/blob/1e3a82a0a65d7b6a67b1e60633453e5edda2960a/modules/gltf/src/lib/parse-gltf.js#L85
+    sourceArray = new Uint8Array(source.buffer || source.arrayBuffer, srcByteOffset, srcByteLength);
+  }
+
+  // Pack buffer onto the big target array
+  target.set(sourceArray, targetOffset);
+
+  return targetOffset + padToNBytes(sourceArray.byteLength, 4);
+}

package/src/loader-utils/lib/crs/crs-reprojection.ts

@@ -0,0 +1,52 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ReadonlyCRSDefinition} from '@math.gl/geospatial/crs';
+
+/**
+ * Common opt-in coordinate transformation options for vector loaders.
+ *
+ * Loader geometry and bounds use `xy` order (x/easting or longitude first), regardless of the
+ * authoritative axis order advertised by a CRS. The CRS definition itself is preserved separately
+ * in metadata. When `reproject` is true, `targetCrs` defaults to WGS84 for compatibility.
+ */
+export type CRSReprojectionOptions = {
+  /** Transform decoded coordinates into the requested output CRS. Defaults to `false`. */
+  reproject?: boolean;
+  /** CRS of returned coordinates. Defaults to `WGS84` when reprojection is enabled. */
+  targetCrs?: ReadonlyCRSDefinition;
+};
+
+/** Stable categories for failures while an opt-in CRS transformation is being prepared. */
+export type CRSReprojectionErrorCode =
+  | 'missing-source-crs'
+  | 'unsupported-source-crs'
+  | 'unsupported-target-crs'
+  | 'transformation-failed';
+
+/** Error thrown when a requested CRS transformation cannot be performed. */
+export class CRSReprojectionError extends Error {
+  /** Machine-readable reason for the failed transformation request. */
+  readonly code: CRSReprojectionErrorCode;
+  /** Source CRS when it was discovered or supplied. */
+  readonly sourceCrs?: ReadonlyCRSDefinition;
+  /** Requested output CRS when it was supplied. */
+  readonly targetCrs?: ReadonlyCRSDefinition;
+
+  constructor(
+    code: CRSReprojectionErrorCode,
+    message: string,
+    options: {
+      sourceCrs?: ReadonlyCRSDefinition;
+      targetCrs?: ReadonlyCRSDefinition;
+      cause?: unknown;
+    } = {}
+  ) {
+    super(message, {cause: options.cause});
+    this.name = 'CRSReprojectionError';
+    this.code = code;
+    this.sourceCrs = options.sourceCrs;
+    this.targetCrs = options.targetCrs;
+  }
+}

package/src/loader-utils/lib/env-utils/assert.ts

@@ -0,0 +1,13 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * Throws an `Error` with the optional `message` if `condition` is falsy
+ * @note Replacement for the external assert method to reduce bundle size
+ */
+export function assert(condition: any, message?: string): void {
+  if (!condition) {
+    throw new Error(message || 'loader assertion failed.');
+  }
+}

package/src/loader-utils/lib/env-utils/globals.ts

@@ -0,0 +1,36 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// Purpose: include this in your module to avoid
+// dependencies on micro modules like 'global' and 'is-browser';
+
+/* eslint-disable no-restricted-globals */
+const globals = {
+  self: typeof self !== 'undefined' && self,
+  window: typeof window !== 'undefined' && window,
+  global: typeof global !== 'undefined' && global,
+  document: typeof document !== 'undefined' && document
+};
+
+type obj = {[key: string]: any};
+const self_: obj = globals.self || globals.window || globals.global || {};
+const window_: obj = globals.window || globals.self || globals.global || {};
+const global_: obj = globals.global || globals.self || globals.window || {};
+const document_: obj = globals.document || {};
+
+export {self_ as self, window_ as window, global_ as global, document_ as document};
+
+/** true if running in a browser */
+export const isBrowser: boolean =
+  // @ts-ignore process does not exist on browser
+  Boolean(typeof process !== 'object' || String(process) !== '[object process]' || process.browser);
+
+/** true if running in a worker thread */
+export const isWorker: boolean = typeof importScripts === 'function';
+
+// Extract node major version
+const matches =
+  typeof process !== 'undefined' && process.version && /v([0-9]*)/.exec(process.version);
+/** Major Node version (as a number) */
+export const nodeVersion: number = (matches && parseFloat(matches[1])) || 0;

package/src/loader-utils/lib/file-provider/data-view-file.ts

@@ -0,0 +1,83 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {FileProviderInterface} from './file-provider-interface';
+import {copyToArrayBuffer} from '../binary-utils/memory-conversion-utils';
+
+/**
+ * Checks if bigint can be converted to number and convert it if possible
+ * @param bigint bigint to be converted
+ * @returns number
+ */
+const toNumber = (bigint: bigint) => {
+  if (bigint > Number.MAX_SAFE_INTEGER) {
+    throw new Error('Offset is out of bounds');
+  }
+  return Number(bigint);
+};
+
+/**
+ * Provides file data using DataView
+ * @deprecated - will be replaced with ReadableFile
+ */
+export class DataViewFile implements FileProviderInterface {
+  /** The DataView from which data is provided */
+  private file: DataView;
+
+  constructor(file: DataView) {
+    this.file = file;
+  }
+
+  async destroy(): Promise<void> {}
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: bigint): Promise<number> {
+    return this.file.getUint8(toNumber(offset));
+  }
+
+  /**
+   * Gets an unsigned 16-bit intege at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: bigint): Promise<number> {
+    return this.file.getUint16(toNumber(offset), true);
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: bigint): Promise<number> {
+    return this.file.getUint32(toNumber(offset), true);
+  }
+
+  /**
+   * Gets an unsigned 64-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: bigint): Promise<bigint> {
+    return this.file.getBigUint64(toNumber(offset), true);
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    return copyToArrayBuffer(
+      this.file.buffer,
+      toNumber(startOffset + BigInt(this.file.byteOffset)),
+      toNumber(endOffset - startOffset)
+    );
+  }
+
+  /** the length (in bytes) of the data. */
+  get length() {
+    return BigInt(this.file.byteLength);
+  }
+}

package/src/loader-utils/lib/file-provider/file-handle-file.ts

@@ -0,0 +1,114 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {FileProviderInterface} from './file-provider-interface';
+import {NodeFileFacade as NodeFile} from '../files/node-file-facade';
+
+/**
+ * Provides file data using node fs library
+ * @deprecated - will be replaced with ReadableFile
+ */
+export class FileHandleFile implements FileProviderInterface {
+  /** The FileHandle from which data is provided */
+  private file: NodeFile;
+
+  /** Create a new FileHandleFile */
+  constructor(path: string, append: boolean = false) {
+    this.file = new NodeFile(path, append ? 'a+' : 'r');
+  }
+
+  /**
+   * Truncates the file descriptor.
+   * @param length desired file lenght
+   */
+  async truncate(length: number): Promise<void> {
+    await this.file.truncate(length);
+  }
+
+  /**
+   * Append data to a file.
+   * @param buffer data to append
+   */
+  async append(buffer: Uint8Array): Promise<void> {
+    await this.file.append(buffer);
+  }
+
+  /** Close file */
+  async destroy(): Promise<void> {
+    await this.file.close();
+  }
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 1);
+    const val = new Uint8Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 2);
+    const val = new Uint16Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 4);
+    const val = new Uint32Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: number | bigint): Promise<bigint> {
+    const arrayBuffer = await this.file.read(offset, 8);
+    const val = new BigInt64Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in byte, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    const bigLength = endOffset - startOffset;
+    if (bigLength > Number.MAX_SAFE_INTEGER) {
+      throw new Error('too big slice');
+    }
+    const length = Number(bigLength);
+    return await this.file.read(startOffset, length);
+  }
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  get length(): bigint {
+    return this.file.bigsize;
+  }
+}

package/src/loader-utils/lib/file-provider/file-provider-interface.ts

@@ -0,0 +1,60 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * Interface for providing file data
+ */
+export interface FileProviderInterface {
+  /**
+   * Cleanup class data
+   */
+  destroy(): Promise<void>;
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint8(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint16(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the file of the view where to read the data.
+   */
+  getUint32(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in byte, from the file of the view where to read the data.
+   */
+  getBigUint64(offset: bigint): Promise<bigint>;
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer>;
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  length: bigint;
+}
+
+/**
+ * Check is the object has FileProvider members
+ * @param fileProvider - tested object
+ */
+export const isFileProvider = (fileProvider: unknown) => {
+  return (
+    (fileProvider as FileProviderInterface)?.getUint8 &&
+    (fileProvider as FileProviderInterface)?.slice &&
+    (fileProvider as FileProviderInterface)?.length
+  );
+};

package/src/loader-utils/lib/file-provider/file-provider.ts

@@ -0,0 +1,130 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import {ReadableFile} from '../files/file';
+import {FileProviderInterface} from './file-provider-interface';
+
+/**
+ * Provides file data using range requests to the server
+ * @deprecated - will be replaced with ReadableFile
+ */
+export class FileProvider implements FileProviderInterface {
+  /** The File object from which data is provided */
+  private file: ReadableFile;
+  private size: bigint;
+
+  /** Create a new BrowserFile */
+  private constructor(file: ReadableFile, size: bigint | number) {
+    this.file = file;
+    this.size = BigInt(size);
+  }
+
+  static async create(file: ReadableFile): Promise<FileProvider> {
+    let size: bigint | number = 0n;
+    if (file.bigsize > 0n) {
+      size = file.bigsize;
+    } else if (file.size > 0) {
+      size = file.size;
+    } else {
+      const stats = await file.stat?.();
+      size = stats?.bigsize ?? 0n;
+    }
+
+    return new FileProvider(file, size);
+  }
+
+  /**
+   * Truncates the file descriptor.
+   * @param length desired file lenght
+   */
+  async truncate(length: number): Promise<void> {
+    throw new Error('file loaded via range requests cannot be changed');
+  }
+
+  /**
+   * Append data to a file.
+   * @param buffer data to append
+   */
+  async append(buffer: Uint8Array): Promise<void> {
+    throw new Error('file loaded via range requests cannot be changed');
+  }
+
+  /** Close file */
+  async destroy(): Promise<void> {
+    throw new Error('file loaded via range requests cannot be changed');
+  }
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 1);
+    const val = new Uint8Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 2);
+    const val = new Uint16Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 4);
+    const val = new Uint32Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: number | bigint): Promise<bigint> {
+    const arrayBuffer = await this.file.read(offset, 8);
+    const val = new BigInt64Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in byte, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffset: bigint | number, endOffset: bigint | number): Promise<ArrayBuffer> {
+    const bigLength = BigInt(endOffset) - BigInt(startOffset);
+    if (bigLength > Number.MAX_SAFE_INTEGER) {
+      throw new Error('too big slice');
+    }
+    const length = Number(bigLength);
+
+    return await this.file.read(startOffset, length);
+  }
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  get length(): bigint {
+    return this.size;
+  }
+}