@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/loader-utils/lib/binary-utils/array-buffer-utils.ts

@@ -0,0 +1,114 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray} from '../../types';
+
+/**
+ * compare two binary arrays for equality
+ * @param a
+ * @param b
+ * @param byteLength
+ */
+export function compareArrayBuffers(
+  arrayBuffer1: ArrayBufferLike,
+  arrayBuffer2: ArrayBufferLike,
+  byteLength?: number
+): boolean {
+  byteLength = byteLength || arrayBuffer1.byteLength;
+  if (arrayBuffer1.byteLength < byteLength || arrayBuffer2.byteLength < byteLength) {
+    return false;
+  }
+  const array1 = new Uint8Array(arrayBuffer1);
+  const array2 = new Uint8Array(arrayBuffer2);
+  for (let i = 0; i < array1.length; ++i) {
+    if (array1[i] !== array2[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Concatenate a sequence of ArrayBuffers from arguments
+ * @return A concatenated ArrayBuffer
+ */
+export function concatenateArrayBuffers(...sources: (ArrayBuffer | Uint8Array)[]): ArrayBuffer {
+  return concatenateArrayBuffersFromArray(sources);
+}
+
+/**
+ * Concatenate a sequence of ArrayBuffers from array
+ * @return A concatenated ArrayBuffer
+ */
+export function concatenateArrayBuffersFromArray(
+  sources: (ArrayBuffer | Uint8Array)[]
+): ArrayBuffer {
+  // Make sure all inputs are wrapped in typed arrays
+  const sourceArrays = sources.map(source2 =>
+    source2 instanceof ArrayBuffer ? new Uint8Array(source2) : source2
+  );
+
+  // Get length of all inputs
+  const byteLength = sourceArrays.reduce((length, typedArray) => length + typedArray.byteLength, 0);
+
+  // Allocate array with space for all inputs
+  const result = new Uint8Array(byteLength);
+
+  // Copy the subarrays
+  let offset = 0;
+  for (const sourceArray of sourceArrays) {
+    result.set(sourceArray, offset);
+    offset += sourceArray.byteLength;
+  }
+
+  // We work with ArrayBuffers, discard the typed array wrapper
+  return result.buffer;
+}
+
+/**
+ * Concatenate arbitrary count of typed arrays
+ * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Typed_arrays
+ * @param - list of arrays. All arrays should be the same type
+ * @return A concatenated TypedArray
+ */
+export function concatenateTypedArrays<T>(...typedArrays: T[]): T {
+  // @ts-ignore
+  const arrays = typedArrays as TypedArray[];
+  // @ts-ignore
+  const TypedArrayConstructor = (arrays && arrays.length > 1 && arrays[0].constructor) || null;
+  if (!TypedArrayConstructor) {
+    throw new Error(
+      '"concatenateTypedArrays" - incorrect quantity of arguments or arguments have incompatible data types'
+    );
+  }
+
+  const sumLength = arrays.reduce((acc, value) => acc + value.length, 0);
+  // @ts-ignore typescript does not like dynamic constructors
+  const result = new TypedArrayConstructor(sumLength);
+  let offset = 0;
+  for (const array of arrays) {
+    result.set(array, offset);
+    offset += array.length;
+  }
+  return result;
+}
+
+/**
+ * Copy a view of an ArrayBuffer into new ArrayBuffer with byteOffset = 0
+ * @param arrayBuffer
+ * @param byteOffset
+ * @param byteLength
+ */
+export function sliceArrayBuffer(
+  arrayBuffer: ArrayBufferLike,
+  byteOffset: number,
+  byteLength?: number
+): ArrayBuffer {
+  const subArray =
+    byteLength !== undefined
+      ? new Uint8Array(arrayBuffer).subarray(byteOffset, byteOffset + byteLength)
+      : new Uint8Array(arrayBuffer).subarray(byteOffset);
+  const arrayCopy = new Uint8Array(subArray);
+  return arrayCopy.buffer;
+}

package/src/loader-utils/lib/binary-utils/binary-chunk-reader.ts

@@ -0,0 +1,297 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Checkpoint returned by {@link BinaryChunkReader.checkpoint}. */
+export type BinaryChunkReaderCheckpoint = {
+  /** Reader offset relative to the first retained chunk. */
+  offset: number;
+};
+
+/** Options for {@link BinaryChunkReader}. */
+export type BinaryChunkReaderOptions = {
+  /** Number of consumed bytes to keep so callers can rewind. */
+  maxRewindBytes?: number;
+};
+
+/** Chunked binary reader for streaming parsers. */
+export class BinaryChunkReader {
+  /** Current offset relative to the first retained chunk. */
+  offset: number = 0;
+  /** Whether the input stream has ended. */
+  ended: boolean = false;
+  /** Bytes behind the current offset to keep available for rewind. */
+  maxRewindBytes: number;
+
+  private chunks: Uint8Array[] = [];
+  private scratch = new ArrayBuffer(8);
+  private checkpoints: BinaryChunkReaderCheckpoint[] = [];
+
+  constructor(options?: BinaryChunkReaderOptions) {
+    this.maxRewindBytes = options?.maxRewindBytes || 0;
+  }
+
+  /** Retained source buffers. Exposed for compatibility with existing tests. */
+  get arrayBuffers(): ArrayBufferLike[] {
+    return this.chunks.map(chunk => chunk.buffer);
+  }
+
+  /** Add a binary chunk without copying when the input is an ArrayBuffer or typed-array view. */
+  write(arrayBuffer: ArrayBufferLike | ArrayBufferView): void {
+    this.chunks.push(toUint8ArrayView(arrayBuffer));
+  }
+
+  /** Mark the input as complete and discard retained chunks. */
+  end(): void {
+    this.chunks = [];
+    this.checkpoints = [];
+    this.offset = 0;
+    this.ended = true;
+  }
+
+  /** Returns true when at least `bytes` can be read from the current offset. */
+  hasAvailableBytes(bytes: number): boolean {
+    return this.getAvailableByteLength() >= bytes;
+  }
+
+  /** Returns the number of bytes currently available from the current offset. */
+  getAvailableByteLength(): number {
+    let bytesAvailable = -this.offset;
+    for (const chunk of this.chunks) {
+      bytesAvailable += chunk.byteLength;
+      if (bytesAvailable > 0) {
+        continue;
+      }
+    }
+    return Math.max(0, bytesAvailable);
+  }
+
+  /** Return source chunk ranges needed to read `bytes` from the current offset. */
+  findBufferOffsets(bytes: number): [number, [number, number]][] | null {
+    let offset = -this.offset;
+    const selectedBuffers: [number, [number, number]][] = [];
+
+    for (let chunkIndex = 0; chunkIndex < this.chunks.length; chunkIndex++) {
+      const chunk = this.chunks[chunkIndex];
+
+      if (offset + chunk.byteLength <= 0) {
+        offset += chunk.byteLength;
+        continue;
+      }
+
+      const start = offset <= 0 ? Math.abs(offset) : 0;
+      let end: number;
+
+      if (start + bytes <= chunk.byteLength) {
+        end = start + bytes;
+        selectedBuffers.push([chunkIndex, [start, end]]);
+        return selectedBuffers;
+      }
+
+      end = chunk.byteLength;
+      selectedBuffers.push([chunkIndex, [start, end]]);
+      bytes -= chunk.byteLength - start;
+      offset += chunk.byteLength;
+    }
+
+    return null;
+  }
+
+  /** Read `bytes` into a DataView, returning null when more input is needed. */
+  getDataView(bytes: number): DataView | null {
+    const bufferOffsets = this.findBufferOffsets(bytes);
+    if (!bufferOffsets) {
+      if (this.ended) {
+        throw new Error('binary data exhausted');
+      }
+      return null;
+    }
+
+    let dataView: DataView;
+    if (bufferOffsets.length === 1) {
+      const [chunkIndex, [start, end]] = bufferOffsets[0];
+      const chunk = this.chunks[chunkIndex];
+      dataView = new DataView(chunk.buffer, chunk.byteOffset + start, end - start);
+    } else {
+      const copy = this.combineBufferOffsets(bufferOffsets);
+      dataView = new DataView(copy.buffer, copy.byteOffset, copy.byteLength);
+    }
+
+    this.offset += bytes;
+    this.disposeBuffers();
+    return dataView;
+  }
+
+  /** Read one byte. */
+  readByte(): number {
+    const bufferOffsets = this.findBufferOffsets(1);
+    if (!bufferOffsets) {
+      throw new Error('binary data exhausted');
+    }
+    const [chunkIndex, [start]] = bufferOffsets[0];
+    const value = this.chunks[chunkIndex][start];
+    this.offset++;
+    this.disposeBuffers();
+    return value;
+  }
+
+  /** Read a little-endian unsigned 16-bit integer. */
+  readUint16LE(): number {
+    const byte0 = this.readByte();
+    const byte1 = this.readByte();
+    return byte0 | (byte1 << 8);
+  }
+
+  /** Read a little-endian unsigned 32-bit integer. */
+  readUint32LE(): number {
+    const byte0 = this.readByte();
+    const byte1 = this.readByte();
+    const byte2 = this.readByte();
+    const byte3 = this.readByte();
+    return (byte0 | (byte1 << 8) | (byte2 << 16) | ((byte3 << 24) >>> 0)) >>> 0;
+  }
+
+  /** Read a little-endian signed 32-bit integer. */
+  readInt32LE(): number {
+    return this.readUint32LE() | 0;
+  }
+
+  /** Read a little-endian 64-bit floating point number. */
+  readFloat64LE(): number {
+    const dataView = this.getContiguousDataView(8);
+    if (dataView) {
+      return dataView.getFloat64(0, true);
+    }
+
+    const bytes = new Uint8Array(this.scratch);
+    this.readInto(bytes, 0, 8);
+    return new DataView(this.scratch).getFloat64(0, true);
+  }
+
+  /** Read bytes into a caller-provided target. */
+  readInto(target: Uint8Array, targetOffset: number, length: number): void {
+    const bufferOffsets = this.findBufferOffsets(length);
+    if (!bufferOffsets) {
+      throw new Error('binary data exhausted');
+    }
+
+    let offset = targetOffset;
+    for (const [chunkIndex, [start, end]] of bufferOffsets) {
+      const chunk = this.chunks[chunkIndex];
+      target.set(chunk.subarray(start, end), offset);
+      offset += end - start;
+    }
+    this.offset += length;
+    this.disposeBuffers();
+  }
+
+  /** Read bytes as a Uint8Array, copying only when the requested range spans chunks. */
+  readBytes(length: number): Uint8Array {
+    const bufferOffsets = this.findBufferOffsets(length);
+    if (!bufferOffsets) {
+      throw new Error('binary data exhausted');
+    }
+
+    let bytes: Uint8Array;
+    if (bufferOffsets.length === 1) {
+      const [chunkIndex, [start, end]] = bufferOffsets[0];
+      const chunk = this.chunks[chunkIndex];
+      bytes = chunk.subarray(start, end);
+    } else {
+      bytes = this.combineBufferOffsets(bufferOffsets);
+    }
+
+    this.offset += length;
+    this.disposeBuffers();
+    return bytes;
+  }
+
+  /** Advance the current offset. */
+  skip(bytes: number): void {
+    this.offset += bytes;
+  }
+
+  /** Move the current offset backwards. */
+  rewind(bytes: number): void {
+    this.offset -= bytes;
+  }
+
+  /** Create a checkpoint that can be restored later. */
+  checkpoint(): BinaryChunkReaderCheckpoint {
+    const checkpoint = {offset: this.offset};
+    this.checkpoints.push(checkpoint);
+    return checkpoint;
+  }
+
+  /** Restore a previous checkpoint. */
+  restore(checkpoint: BinaryChunkReaderCheckpoint): void {
+    this.offset = checkpoint.offset;
+    this.checkpoints.pop();
+  }
+
+  /** Dispose chunks that are older than the configured rewind window. */
+  discardConsumed(): void {
+    this.disposeBuffers();
+  }
+
+  /** Copy multiple source ranges into one tightly sized ArrayBuffer. */
+  _combineArrayBuffers(bufferOffsets: [number, [number, number]][]): ArrayBufferLike {
+    return this.combineBufferOffsets(bufferOffsets).buffer;
+  }
+
+  private getContiguousDataView(bytes: number): DataView | null {
+    const bufferOffsets = this.findBufferOffsets(bytes);
+    if (!bufferOffsets || bufferOffsets.length !== 1) {
+      return null;
+    }
+    const [chunkIndex, [start, end]] = bufferOffsets[0];
+    const chunk = this.chunks[chunkIndex];
+    const dataView = new DataView(chunk.buffer, chunk.byteOffset + start, end - start);
+    this.offset += bytes;
+    this.disposeBuffers();
+    return dataView;
+  }
+
+  private disposeBuffers(): void {
+    let protectedOffset =
+      this.checkpoints.length > 0
+        ? Math.min(...this.checkpoints.map(checkpoint => checkpoint.offset))
+        : Number.POSITIVE_INFINITY;
+    while (
+      this.chunks.length > 0 &&
+      this.offset - this.maxRewindBytes >= this.chunks[0].byteLength &&
+      protectedOffset >= this.chunks[0].byteLength
+    ) {
+      this.offset -= this.chunks[0].byteLength;
+      // Checkpoints use offsets relative to the first retained chunk and must move with disposal.
+      for (const checkpoint of this.checkpoints) {
+        checkpoint.offset -= this.chunks[0].byteLength;
+      }
+      protectedOffset -= this.chunks[0].byteLength;
+      this.chunks.shift();
+    }
+  }
+
+  private combineBufferOffsets(bufferOffsets: [number, [number, number]][]): Uint8Array {
+    let byteLength = 0;
+    for (const [, [start, end]] of bufferOffsets) {
+      byteLength += end - start;
+    }
+
+    const result = new Uint8Array(byteLength);
+    let resultOffset = 0;
+    for (const [chunkIndex, [start, end]] of bufferOffsets) {
+      const chunk = this.chunks[chunkIndex];
+      result.set(chunk.subarray(start, end), resultOffset);
+      resultOffset += end - start;
+    }
+    return result;
+  }
+}
+
+function toUint8ArrayView(arrayBuffer: ArrayBufferLike | ArrayBufferView): Uint8Array {
+  if (ArrayBuffer.isView(arrayBuffer)) {
+    return new Uint8Array(arrayBuffer.buffer, arrayBuffer.byteOffset, arrayBuffer.byteLength);
+  }
+  return new Uint8Array(arrayBuffer);
+}

package/src/loader-utils/lib/binary-utils/dataview-copy-utils.ts

@@ -0,0 +1,121 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TypedArray} from '../../types';
+import {padToNBytes} from './memory-copy-utils';
+
+/**
+ * Helper function that pads a string with spaces to fit a certain byte alignment
+ * @param string
+ * @param byteAlignment
+ * @returns
+ *
+ * @todo PERFORMANCE IDEA: No need to copy string twice...
+ */
+export function padStringToByteAlignment(string: string, byteAlignment: number): string {
+  const length = string.length;
+  const paddedLength = Math.ceil(length / byteAlignment) * byteAlignment; // Round up to the required alignment
+  const padding = paddedLength - length;
+  let whitespace = '';
+  for (let i = 0; i < padding; ++i) {
+    whitespace += ' ';
+  }
+  return string + whitespace;
+}
+
+/**
+ *
+ * @param dataView
+ * @param byteOffset
+ * @param string
+ * @param byteLength
+ * @returns
+ */
+export function copyStringToDataView(
+  dataView: DataView,
+  byteOffset: number,
+  string: string,
+  byteLength: number
+): number {
+  if (dataView) {
+    for (let i = 0; i < byteLength; i++) {
+      dataView.setUint8(byteOffset + i, string.charCodeAt(i));
+    }
+  }
+  return byteOffset + byteLength;
+}
+
+export function copyBinaryToDataView(dataView, byteOffset, binary, byteLength) {
+  if (dataView) {
+    for (let i = 0; i < byteLength; i++) {
+      dataView.setUint8(byteOffset + i, binary[i]);
+    }
+  }
+  return byteOffset + byteLength;
+}
+
+/**
+ * Copy sourceBuffer to dataView with some padding
+ *
+ * @param dataView - destination data container. If null - only new offset is calculated
+ * @param byteOffset - destination byte offset to copy to
+ * @param sourceBuffer - source data buffer
+ * @param padding - pad the resulting array to multiple of "padding" bytes. Additional bytes are filled with 0x20 (ASCII space)
+ *
+ * @return new byteOffset of resulting dataView
+ */
+export function copyPaddedArrayBufferToDataView(
+  dataView: DataView | null,
+  byteOffset: number,
+  sourceBuffer: TypedArray,
+  padding: number
+): number {
+  const paddedLength = padToNBytes(sourceBuffer.byteLength, padding);
+  const padLength = paddedLength - sourceBuffer.byteLength;
+
+  if (dataView) {
+    // Copy array
+    const targetArray = new Uint8Array(
+      dataView.buffer,
+      dataView.byteOffset + byteOffset,
+      sourceBuffer.byteLength
+    );
+    const sourceArray = new Uint8Array(sourceBuffer);
+    targetArray.set(sourceArray);
+
+    // Add PADDING
+    for (let i = 0; i < padLength; ++i) {
+      // json chunk is padded with spaces (ASCII 0x20)
+      dataView.setUint8(byteOffset + sourceBuffer.byteLength + i, 0x20);
+    }
+  }
+  byteOffset += paddedLength;
+  return byteOffset;
+}
+
+/**
+ * Copy string to dataView with some padding
+ *
+ * @param {DataView | null} dataView - destination data container. If null - only new offset is calculated
+ * @param {number} byteOffset - destination byte offset to copy to
+ * @param {string} string - source string
+ * @param {number} padding - pad the resulting array to multiple of "padding" bytes. Additional bytes are filled with 0x20 (ASCII space)
+ *
+ * @return new byteOffset of resulting dataView
+ */
+export function copyPaddedStringToDataView(
+  dataView: DataView | null,
+  byteOffset: number,
+  string: string,
+  padding: number
+): number {
+  const textEncoder = new TextEncoder();
+  // PERFORMANCE IDEA: We encode twice, once to get size and once to store
+  // PERFORMANCE IDEA: Use TextEncoder.encodeInto() to avoid temporary copy
+  const stringBuffer = textEncoder.encode(string);
+
+  byteOffset = copyPaddedArrayBufferToDataView(dataView, byteOffset, stringBuffer, padding);
+
+  return byteOffset;
+}

package/src/loader-utils/lib/binary-utils/encode-array-buffer-to-base64.ts

@@ -0,0 +1,17 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Encode binary bytes without exceeding argument limits or inserting padding between chunks. */
+export function encodeArrayBufferToBase64(arrayBuffer: ArrayBuffer): string {
+  const bytes = new Uint8Array(arrayBuffer);
+  const encodedChunks: string[] = [];
+  // Full chunks contain a multiple of three bytes, so only the final chunk can need padding.
+  const chunkByteLength = 3 * 4096;
+  for (let byteOffset = 0; byteOffset < bytes.length; byteOffset += chunkByteLength) {
+    encodedChunks.push(
+      btoa(String.fromCharCode(...bytes.subarray(byteOffset, byteOffset + chunkByteLength)))
+    );
+  }
+  return encodedChunks.join('');
+}

package/src/loader-utils/lib/binary-utils/get-first-characters.ts

@@ -0,0 +1,46 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/**
+ * Get the first characters from a binary file (interpret the first bytes as an ASCII string)
+ * @param data
+ * @param length
+ * @returns
+ */
+export function getFirstCharacters(data: string | ArrayBuffer, length: number = 5): string {
+  if (typeof data === 'string') {
+    return data.slice(0, length);
+  } else if (ArrayBuffer.isView(data)) {
+    // Typed Arrays can have offsets into underlying buffer
+    return getMagicString(data.buffer, data.byteOffset, length);
+  } else if (data instanceof ArrayBuffer) {
+    const byteOffset = 0;
+    return getMagicString(data, byteOffset, length);
+  }
+  return '';
+}
+
+/**
+ * Gets a magic string from a "file"
+ * Typically used to check or detect file format
+ * @param arrayBuffer
+ * @param byteOffset
+ * @param length
+ * @returns
+ */
+export function getMagicString(
+  arrayBuffer: ArrayBufferLike,
+  byteOffset: number,
+  length: number
+): string {
+  if (arrayBuffer.byteLength <= byteOffset + length) {
+    return '';
+  }
+  const dataView = new DataView(arrayBuffer);
+  let magic = '';
+  for (let i = 0; i < length; i++) {
+    magic += String.fromCharCode(dataView.getUint8(byteOffset + i));
+  }
+  return magic;
+}

package/src/loader-utils/lib/binary-utils/hex-transcoder.ts

@@ -0,0 +1,54 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// Forked from https://github.com/jessetane/hex-transcoder under MIT license
+
+const alphabet = '0123456789abcdef';
+const encodeLookup: string[] = [];
+const decodeLookup: number[] = [];
+
+for (let i = 0; i < 256; i++) {
+  encodeLookup[i] = alphabet[(i >> 4) & 0xf] + alphabet[i & 0xf];
+  if (i < 16) {
+    if (i < 10) {
+      decodeLookup[0x30 + i] = i;
+    } else {
+      decodeLookup[0x61 - 10 + i] = i;
+    }
+  }
+}
+
+/**
+ * Encode a Uint8Array to a hex string
+ *
+ * @param  array Bytes to encode to string
+ * @return hex string
+ */
+export function encodeHex(array: Uint8Array): string {
+  const length = array.length;
+  let string = '';
+  let i = 0;
+  while (i < length) {
+    string += encodeLookup[array[i++]];
+  }
+  return string;
+}
+
+/**
+ * Decodes a hex string to a Uint8Array
+ *
+ * @param string hex string to decode to Uint8Array
+ * @return Uint8Array
+ */
+export function decodeHex(string: string): Uint8Array {
+  const sizeof = string.length >> 1;
+  const length = sizeof << 1;
+  const array = new Uint8Array(sizeof);
+  let n = 0;
+  let i = 0;
+  while (i < length) {
+    array[n++] = (decodeLookup[string.charCodeAt(i++)] << 4) | decodeLookup[string.charCodeAt(i++)];
+  }
+  return array;
+}