@loaders.gl/core 5.0.0-alpha.11 → 5.0.0-alpha.13

package/src/loader-utils/lib/files/http-file.ts

@@ -0,0 +1,267 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// SPDX-FileCopyrightText: Copyright (c) vis.gl contributors
+
+import {waitForPromiseWithSignal} from '../request-utils/abort-utils';
+
+import type {ReadableFile, Stat} from './file';
+import type {RangeRequestTransportResult} from '../request-utils/range-request-scheduler';
+import {RangeRequestScheduler} from '../request-utils/range-request-scheduler';
+import type {HttpFileIdentity, HttpFileOptions, HttpFileTelemetry} from './http-file-types';
+import {HttpFileTransport} from './http-file-transport';
+
+export type {
+  HttpFileConsistency,
+  HttpFileFetch,
+  HttpFileIdentity,
+  HttpFileOptions,
+  HttpFileTelemetry
+} from './http-file-types';
+
+/** A reusable, validator-aware, byte-range-addressable HTTP file. */
+export class HttpFile implements ReadableFile {
+  /** URL used as the ReadableFile handle. */
+  readonly handle: string;
+  /** Remote object URL. */
+  readonly url: string;
+
+  /** HTTP transport and response validator for this file. */
+  private readonly transport: HttpFileTransport;
+  /** Scheduler used to coalesce compatible range reads. */
+  private readonly rangeScheduler: RangeRequestScheduler;
+  /** Unique key that prevents coalescing across transport or credential contexts. */
+  private readonly schedulerIsolationKey = {};
+  /** Caller-supplied length available before identity discovery. */
+  private readonly suppliedByteLength?: number;
+
+  /** Pinned remote object identity. */
+  private identity: HttpFileIdentity | null;
+  /** Shared opening probe, while identity discovery is in progress. */
+  private openPromise: Promise<HttpFileIdentity> | null = null;
+  /** Whether the file has stopped accepting operations. */
+  private closed = false;
+
+  /** Creates a lazy HTTP file. Use {@link HttpFile.open} when identity must be pinned immediately. */
+  constructor(url: string, options: HttpFileOptions = {}) {
+    this.handle = url;
+    this.url = url;
+    this.transport = new HttpFileTransport(url, options);
+    this.rangeScheduler =
+      options.rangeScheduler ||
+      new RangeRequestScheduler({batchDelayMs: 0, ...options.rangeSchedulerProps});
+    this.suppliedByteLength = options.byteLength;
+    this.identity = this.transport.getInitialIdentity();
+  }
+
+  /** Opens a file and pins its byte length and available validators. */
+  static async open(
+    url: string,
+    options: HttpFileOptions = {},
+    signal?: AbortSignal
+  ): Promise<HttpFile> {
+    const file = new HttpFile(url, options);
+    await file.open(signal);
+    return file;
+  }
+
+  /** Pins this file's byte length and available validators. */
+  async open(signal?: AbortSignal): Promise<this> {
+    this.assertOpen();
+    if (signal?.aborted) {
+      const error = createAbortError();
+      this.transport.trackFailure(error);
+      throw error;
+    }
+    await this.getIdentity(signal);
+    return this;
+  }
+
+  /** Pinned file length, or zero before a lazy file has opened. */
+  get size(): number {
+    return this.identity?.byteLength ?? this.suppliedByteLength ?? 0;
+  }
+
+  /** Pinned file length as a bigint, or zero before a lazy file has opened. */
+  get bigsize(): bigint {
+    return BigInt(this.size);
+  }
+
+  /** Returns the pinned identity, or null before a lazy file has opened. */
+  getIdentitySnapshot(): HttpFileIdentity | null {
+    return this.identity;
+  }
+
+  /** Returns an immutable point-in-time copy of this file's transport counters. */
+  getTelemetry(): HttpFileTelemetry {
+    return this.transport.getTelemetry();
+  }
+
+  /** Prevents new operations. Active fetches remain controlled by their read signals. */
+  async close(): Promise<void> {
+    this.closed = true;
+  }
+
+  /** Returns pinned file information without issuing a HEAD request. */
+  async stat(): Promise<Stat> {
+    this.assertOpen();
+    const identity = await this.getIdentity();
+    return {
+      size: identity.byteLength,
+      bigsize: BigInt(identity.byteLength),
+      isDirectory: false
+    };
+  }
+
+  /** Reads exactly one byte range, with optional cancellation. */
+  async read(
+    offset: number | bigint = 0,
+    length: number = 0,
+    signal?: AbortSignal
+  ): Promise<ArrayBuffer> {
+    this.assertOpen();
+    if (signal?.aborted) {
+      const error = createAbortError();
+      this.transport.trackFailure(error);
+      throw error;
+    }
+
+    const identity = await this.getIdentity(signal);
+    const numericOffset = normalizeOffset(offset);
+    validateRange(numericOffset, length, identity.byteLength);
+    if (length === 0) {
+      return new ArrayBuffer(0);
+    }
+
+    try {
+      return await this.rangeScheduler.scheduleRequest({
+        sourceId: this.url,
+        isolationKey: this.schedulerIsolationKey,
+        offset: numericOffset,
+        length,
+        signal,
+        fetchRange: this.fetchScheduledRange
+      });
+    } catch (error) {
+      this.transport.trackFailure(error);
+      throw error;
+    }
+  }
+
+  /** Reads a range and returns a Response for legacy ReadableFile consumers. */
+  async fetchRange(
+    offset: number | bigint,
+    length: number,
+    signal?: AbortSignal
+  ): Promise<Response> {
+    const numericOffset = normalizeOffset(offset);
+    const arrayBuffer = await this.read(numericOffset, length, signal);
+    const headers = createIdentityHeaders(this.identity, numericOffset, length);
+    return new Response(arrayBuffer, {status: 206, headers});
+  }
+
+  /** Stable scheduler callback used by every read from this file. */
+  private readonly fetchScheduledRange = async (
+    offset: number,
+    length: number,
+    signal?: AbortSignal
+  ): Promise<RangeRequestTransportResult> => {
+    const identity = await this.getIdentity(signal);
+    const result = await this.transport.requestRange(offset, length, signal, identity);
+    return {
+      arrayBuffer: result.arrayBuffer,
+      status: result.status,
+      transportBytes: result.transportBytes,
+      networkTimeMs: result.networkTimeMs
+    };
+  };
+
+  /** Returns the shared lazy identity promise, retrying after a failed probe. */
+  private getIdentity(signal?: AbortSignal): Promise<HttpFileIdentity> {
+    if (this.identity) {
+      return Promise.resolve(this.identity);
+    }
+    if (!this.openPromise) {
+      // The shared probe must not be owned by the first caller's cancellation signal.
+      // Each caller can independently stop waiting while the probe populates the cache.
+      const nextOpenPromise = this.probeIdentity();
+      const cachedOpenPromise = nextOpenPromise.catch(error => {
+        if (this.openPromise === cachedOpenPromise) {
+          this.openPromise = null;
+        }
+        throw error;
+      });
+      this.openPromise = cachedOpenPromise;
+    }
+    return waitForPromiseWithSignal(this.openPromise, signal).catch(error => {
+      this.transport.trackFailure(error);
+      throw error;
+    });
+  }
+
+  /** Uses a one-byte GET range to discover object length and validators. */
+  private async probeIdentity(): Promise<HttpFileIdentity> {
+    try {
+      const result = await this.transport.requestRange(0, 1, undefined, null);
+      this.identity = result.identity;
+      return result.identity;
+    } catch (error) {
+      this.transport.trackFailure(error);
+      throw error;
+    }
+  }
+
+  /** Throws when the file has been closed. */
+  private assertOpen(): void {
+    if (this.closed) {
+      throw new Error('HttpFile is closed');
+    }
+  }
+}
+
+/** Creates the identity headers returned by the compatibility fetchRange method. */
+function createIdentityHeaders(
+  identity: HttpFileIdentity | null,
+  offset: number,
+  length: number
+): Headers {
+  const headers = new Headers();
+  if (length > 0 && identity) {
+    headers.set('Content-Range', `bytes ${offset}-${offset + length - 1}/${identity.byteLength}`);
+  }
+  if (identity?.etag) {
+    headers.set('ETag', identity.etag);
+  }
+  if (identity?.lastModified) {
+    headers.set('Last-Modified', identity.lastModified);
+  }
+  return headers;
+}
+
+/** Converts and validates a number or bigint offset. */
+function normalizeOffset(offset: number | bigint): number {
+  const numericOffset = Number(offset);
+  if (!Number.isSafeInteger(numericOffset) || numericOffset < 0) {
+    throw new Error('HTTP byte-range offset must be a non-negative safe integer');
+  }
+  return numericOffset;
+}
+
+/** Validates a requested byte range against the pinned object length. */
+function validateRange(offset: number, length: number, byteLength: number): void {
+  if (!Number.isSafeInteger(length) || length < 0) {
+    throw new Error('HTTP byte-range length must be a non-negative safe integer');
+  }
+  if (offset + length > byteLength) {
+    throw new Error('HTTP byte range extends beyond the end of the file');
+  }
+}
+
+/** Creates a conventional abort error without requiring DOMException in every host. */
+function createAbortError(): Error {
+  if (typeof DOMException !== 'undefined') {
+    return new DOMException('Request aborted', 'AbortError');
+  }
+  const error = new Error('Request aborted');
+  error.name = 'AbortError';
+  return error;
+}

package/src/loader-utils/lib/files/node-file-facade.ts

@@ -0,0 +1,57 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {isBrowser} from '../env-utils/globals';
+import {ReadableFile, WritableFile, Stat} from './file';
+
+const NOT_IMPLEMENTED = new Error('Not implemented');
+
+/** This class is a facade that gets replaced with an actual NodeFile instance  */
+export class NodeFileFacade implements ReadableFile, WritableFile {
+  handle: unknown;
+  size: number = 0;
+  bigsize: bigint = 0n;
+  url: string = '';
+
+  constructor(url: string, flags?: 'r' | 'w' | 'wx' | 'a+', mode?: number) {
+    // Return the actual implementation instance
+    if (globalThis.loaders?.NodeFile) {
+      return new globalThis.loaders.NodeFile(url, flags, mode);
+    }
+    if (isBrowser) {
+      throw new Error("Can't instantiate NodeFile in browser.");
+    }
+    throw new Error(
+      "Can't instantiate NodeFile. Make sure to import @loaders.gl/core/polyfills first."
+    );
+  }
+  /** Read data */
+  async read(start?: number | bigint, length?: number): Promise<ArrayBuffer> {
+    throw NOT_IMPLEMENTED;
+  }
+  /** Write to file. The number of bytes written will be returned */
+  async write(
+    arrayBuffer: ArrayBuffer,
+    offset?: number | bigint,
+    length?: number | bigint
+  ): Promise<number> {
+    throw NOT_IMPLEMENTED;
+  }
+  /** Get information about file */
+  async stat(): Promise<Stat> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  /** Truncates the file descriptor. Only available on NodeFile. */
+  async truncate(length: number): Promise<void> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  /** Append data to a file. Only available on NodeFile. */
+  async append(data: Uint8Array): Promise<void> {
+    throw NOT_IMPLEMENTED;
+  }
+  /** Close the file */
+  async close(): Promise<void> {}
+}

package/src/loader-utils/lib/files/sources.ts

@@ -0,0 +1,154 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/*
+import {fetchFile} from '@loaders.gl/core';
+
+import {Source as PMTilesSourceLoader, RangeResponse} from 'pmtiles';
+
+/** @note "source" here is a PMTiles library type, referring to  *
+export function makeSource(data: string | Blob, fetch?) {
+  if (typeof data === 'string') {
+    return new FetchSource(data, fetch);
+  }
+  if (data instanceof Blob) {
+    const url = '';
+    return new BlobSource(data, url);
+  }
+}
+
+export class BlobSource implements PMTilesSourceLoader {
+  blob: Blob;
+  key: string;
+
+  constructor(blob: Blob, key: string) {
+    this.blob = blob;
+    this.key = key;
+  }
+
+  // TODO - how is this used?
+  getKey() {
+    return this.blob.url || '';
+  }
+
+  async getBytes(offset: number, length: number, signal?: AbortSignal): Promise<RangeResponse> {
+    const data = await this.blob.arrayBuffer();
+    return {
+      data
+      // etag: response.headers.get('ETag') || undefined,
+      // cacheControl: response.headers.get('Cache-Control') || undefined,
+      // expires: response.headers.get('Expires') || undefined
+    };
+  }
+}
+
+export class FetchSource implements PMTilesSourceLoader {
+  url: string;
+  fetch;
+
+  constructor(url: string, fetch = fetchFile) {
+    this.url = url;
+    this.fetch = fetch;
+  }
+
+  // TODO - how is this used?
+  getKey() {
+    return this.url;
+  }
+
+  async getBytes(offset: number, length: number, signal?: AbortSignal): Promise<RangeResponse> {
+    let controller;
+    if (!signal) {
+      // ToDO why is it so important to abort in case 200?
+      // TODO check this works or assert 206
+      controller = new AbortController();
+      signal = controller.signal;
+    }
+
+    let response = await fetch(this.url, {
+      signal,
+      headers: {Range: `bytes=${offset}-${offset + length - 1}`}
+    });
+
+    switch (response.status) {
+      case 206: // Partial Content success
+        // This is the expected success code for a range request
+        break;
+
+      case 200:
+        // some well-behaved backends, e.g. DigitalOcean CDN, respond with 200 instead of 206
+        // but we also need to detect no support for Byte Serving which is returning the whole file
+        const content_length = response.headers.get('Content-Length');
+        if (!content_length || Number(content_length) > length) {
+          if (controller) {
+            controller.abort();
+          }
+          throw Error(
+            'content-length header missing or exceeding request. Server must support HTTP Byte Serving.'
+          );
+        }
+
+      case 416: // "Range Not Satisfiable"
+        // some HTTP servers don't accept ranges beyond the end of the resource.
+        // Retry with the exact length
+        // TODO: can return 416 with offset > 0 if content changed, which will have a blank etag.
+        // See https://github.com/protomaps/PMTiles/issues/90
+        if (offset === 0) {
+          const content_range = response.headers.get('Content-Range');
+          if (!content_range || !content_range.startsWith('bytes *')) {
+            throw Error('Missing content-length on 416 response');
+          }
+          const actual_length = Number(content_range.substr(8));
+          response = await fetch(this.url, {
+            signal,
+            headers: {Range: `bytes=0-${actual_length - 1}`}
+          });
+        }
+        break;
+
+      default:
+        if (response.status >= 300) {
+          throw Error(`Bad response code: ${response.status}`);
+        }
+    }
+
+    const data = await response.arrayBuffer();
+    return {
+      data,
+      etag: response.headers.get('ETag') || undefined,
+      cacheControl: response.headers.get('Cache-Control') || undefined,
+      expires: response.headers.get('Expires') || undefined
+    };
+  }
+}
+
+/*
+class TestNodeFileSource implements Source {
+  buffer: ArrayBuffer;
+  path: string;
+  key: string;
+  etag?: string;
+
+  constructor(path: string, key: string) {
+    this.path = path;
+    this.buffer = fs.readFileSync(path);
+    this.key = key;
+  }
+
+  getKey() {
+    return this.key;
+  }
+
+  replaceData(path: string) {
+    this.path = path;
+    this.buffer = fs.readFileSync(path);
+  }
+
+  async getBytes(offset: number, length: number): Promise<RangeResponse> {
+    const slice = new Uint8Array(this.buffer.slice(offset, offset + length))
+      .buffer;
+    return { data: slice, etag: this.etag };
+  }
+}
+*/

package/src/loader-utils/lib/filesystems/filesystem.ts

@@ -0,0 +1,40 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {ReadableFile, WritableFile} from '../files/file';
+
+/**
+ * A FileSystem interface can encapsulate various file sources,
+ * a FileList, a Node.js filesystem, a ZipFile, a GoogleDrive etc.
+ */
+export interface FileSystem {
+  /** Return a list of file names in a "directory" */
+  readdir(dirname?: string, options?: {recursive?: boolean}): Promise<string[]>;
+
+  /** Gets information from a local file from the filesystem */
+  stat(filename: string, options?: object): Promise<{size: number}>;
+
+  /** Removes a file from the file system */
+  unlink?(path: string): Promise<void>;
+
+  /** Fetches the full contents of a file from the filesystem (or a URL) */
+  fetch(path: string, options?: RequestInit): Promise<Response>;
+}
+
+/**
+ * A random access file system, open readable and/or writable files
+ */
+export interface RandomAccessFileSystem extends FileSystem {
+  /** Can open readable files */
+  readonly readable: boolean;
+
+  /** Can open writable files */
+  readonly writable: boolean;
+
+  /** Open a readable file */
+  openReadableFile(path: string, flags?: 'r'): Promise<ReadableFile>;
+
+  /** Open a writable file */
+  openWritableFile(path: string, flags?: 'w' | 'wx', mode?: number): Promise<WritableFile>;
+}

package/src/loader-utils/lib/filesystems/node-filesystem-facade.ts

@@ -0,0 +1,65 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {isBrowser} from '../env-utils/globals';
+import {Stat} from '../files/file';
+import {NodeFileFacade as NodeFile} from '../files/node-file-facade';
+import {RandomAccessFileSystem} from './filesystem';
+
+const NOT_IMPLEMENTED = new Error('Not implemented');
+
+/**
+ * FileSystem pass-through for Node.js
+ * Compatible with BrowserFileSystem.
+ * @note Dummy implementation, not used (constructor returns a real NodeFileSystem instance)
+ * @param options
+ */
+export class NodeFileSystemFacade implements RandomAccessFileSystem {
+  // implements FileSystem
+  constructor(options: {[key: string]: any}) {
+    if (globalThis.loaders?.NodeFileSystem) {
+      return new globalThis.loaders.NodeFileSystem(options);
+    }
+    if (isBrowser) {
+      throw new Error("Can't instantiate NodeFileSystem in browser.");
+    }
+    throw new Error(
+      "Can't instantiate NodeFileSystem. Make sure to import @loaders.gl/core/polyfills first."
+    );
+  }
+
+  // DUMMY IMPLEMENTATION, not used (constructor returns a real NodeFileSystem instance)
+
+  // implements RandomAccessReadFileSystem
+
+  readonly readable = true;
+  readonly writable = true;
+
+  async openReadableFile(path: string, flags): Promise<NodeFile> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  // implements RandomAccessWriteFileSystem
+  async openWritableFile(path: string, flags, mode): Promise<NodeFile> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  // Implements file system
+
+  async readdir(dirname = '.', options?: {}): Promise<string[]> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  async stat(path: string, options?: {}): Promise<Stat> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  async unlink(path: string): Promise<void> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  async fetch(path: RequestInfo, options?: RequestInit): Promise<Response> {
+    throw NOT_IMPLEMENTED;
+  }
+}