@loaders.gl/loader-utils 4.0.0-beta.2 → 4.0.0-beta.3

package/src/lib/files/http-file.ts

@@ -0,0 +1,120 @@
+// loaders.gl, MIT license
+
+import {ReadableFile, Stat} from './file';
+
+export class HttpFile implements ReadableFile {
+  readonly handle: string;
+  readonly size: number = 0;
+  readonly bigsize: bigint = 0n;
+  readonly url: string;
+
+  constructor(url: string) {
+    this.handle = url;
+    this.url = url;
+  }
+
+  async close(): Promise<void> {}
+
+  async stat(): Promise<Stat> {
+    const response = await fetch(this.handle, {method: 'HEAD'});
+    if (!response.ok) {
+      throw new Error(`Failed to fetch HEAD ${this.handle}`);
+    }
+    const size = parseInt(response.headers.get('Content-Length') || '0');
+    return {
+      size,
+      bigsize: BigInt(size),
+      isDirectory: false
+    };
+  }
+
+  async read(offset: number | bigint, length: number): Promise<ArrayBuffer> {
+    const response = await this.fetchRange(offset, length);
+    const arrayBuffer = await response.arrayBuffer();
+    return arrayBuffer;
+  }
+
+  /**
+   *
+   * @param offset
+   * @param length
+   * @param signal
+   * @returns
+   * @see https://github.com/protomaps/PMTiles
+   */
+  // eslint-disable-next-line complexity
+  async fetchRange(
+    offset: number | bigint,
+    length: number,
+    signal?: AbortSignal
+  ): Promise<Response> {
+    const nOffset = Number(offset);
+    const nLength = Number(length);
+
+    let controller: AbortController | undefined;
+    if (!signal) {
+      // ToDO why is it so important to abort in case 200?
+      // TODO check this works or assert 206
+      controller = new AbortController();
+      signal = controller.signal;
+    }
+
+    const url = this.handle;
+    let response = await fetch(url, {
+      signal,
+      headers: {Range: `bytes=${nOffset}-${nOffset + nLength - 1}`}
+    });
+
+    switch (response.status) {
+      case 206: // Partial Content success
+        // This is the expected success code for a range request
+        break;
+
+      case 200:
+        // some well-behaved backends, e.g. DigitalOcean CDN, respond with 200 instead of 206
+        // but we also need to detect no support for Byte Serving which is returning the whole file
+        const contentLength = response.headers.get('Content-Length');
+        if (!contentLength || Number(contentLength) > length) {
+          if (controller) {
+            controller.abort();
+          }
+          throw Error(
+            'content-length header missing or exceeding request. Server must support HTTP Byte Serving.'
+          );
+        }
+
+      // @eslint-disable-next-line no-fallthrough
+      case 416: // "Range Not Satisfiable"
+        // some HTTP servers don't accept ranges beyond the end of the resource.
+        // Retry with the exact length
+        // TODO: can return 416 with offset > 0 if content changed, which will have a blank etag.
+        // See https://github.com/protomaps/PMTiles/issues/90
+        if (offset === 0) {
+          const contentRange = response.headers.get('Content-Range');
+          if (!contentRange || !contentRange.startsWith('bytes *')) {
+            throw Error('Missing content-length on 416 response');
+          }
+          const actualLength = Number(contentRange.substr(8));
+          response = await fetch(this.url, {
+            signal,
+            headers: {Range: `bytes=0-${actualLength - 1}`}
+          });
+        }
+        break;
+
+      default:
+        if (response.status >= 300) {
+          throw Error(`Bad response code: ${response.status}`);
+        }
+    }
+
+    return response;
+    // const data = await response.arrayBuffer();
+    // return {
+    //   data,
+    //   etag: response.headers.get('ETag') || undefined,
+    //   cacheControl: response.headers.get('Cache-Control') || undefined,
+    //   expires: response.headers.get('Expires') || undefined
+    // };
+  }
+}

package/src/lib/files/node-file-facade.ts

@@ -0,0 +1,39 @@
+// loaders.gl, MIT license
+
+import {isBrowser} from '../env-utils/globals';
+import {ReadableFile, WritableFile, Stat} from './file';
+
+const NOT_IMPLEMENTED = new Error('Not implemented');
+
+/** This class is a facade that gets replaced with an actual NodeFile instance  */
+export class NodeFileFacade implements ReadableFile, WritableFile {
+  handle: unknown;
+  size: number = 0;
+  bigsize: bigint = 0n;
+  url: string = '';
+
+  constructor(url: string, flags?: 'r' | 'w' | 'wx', mode?: number) {
+    // Return the actual implementation instance
+    if (globalThis.loaders?.NodeFile) {
+      return new globalThis.loaders.NodeFile(url, flags, mode);
+    }
+    if (isBrowser) {
+      throw new Error('Can\'t instantiate NodeFile in browser.');
+    }
+    throw new Error('Can\'t instantiate NodeFile. Make sure to import @loaders.gl/polyfills first.');
+  }
+  /** Read data */
+  async read(start?: number | bigint, end?: number | bigint): Promise<ArrayBuffer> {
+    throw NOT_IMPLEMENTED;
+  }
+  /** Write to file. The number of bytes written will be returned */
+  async write(arrayBuffer: ArrayBuffer, offset?: number | bigint, length?: number | bigint): Promise<number> {
+    throw NOT_IMPLEMENTED;
+  }
+  /** Get information about file */
+  async stat(): Promise<Stat> {
+    throw NOT_IMPLEMENTED;
+  }
+  /** Close the file */
+  async close(): Promise<void> {}
+}

package/src/lib/files/sources.ts

@@ -0,0 +1,150 @@
+/*
+import {fetchFile} from '@loaders.gl/core';
+
+import {Source as PMTilesSource, RangeResponse} from 'pmtiles';
+
+/** @note "source" here is a PMTiles library type, referring to  *
+export function makeSource(data: string | Blob, fetch?) {
+  if (typeof data === 'string') {
+    return new FetchSource(data, fetch);
+  }
+  if (data instanceof Blob) {
+    const url = '';
+    return new BlobSource(data, url);
+  }
+}
+
+export class BlobSource implements PMTilesSource {
+  blob: Blob;
+  key: string;
+
+  constructor(blob: Blob, key: string) {
+    this.blob = blob;
+    this.key = key;
+  }
+
+  // TODO - how is this used?
+  getKey() {
+    return this.blob.url || '';
+  }
+
+  async getBytes(offset: number, length: number, signal?: AbortSignal): Promise<RangeResponse> {
+    const data = await this.blob.arrayBuffer();
+    return {
+      data
+      // etag: response.headers.get('ETag') || undefined,
+      // cacheControl: response.headers.get('Cache-Control') || undefined,
+      // expires: response.headers.get('Expires') || undefined
+    };
+  }
+}
+
+export class FetchSource implements PMTilesSource {
+  url: string;
+  fetch;
+
+  constructor(url: string, fetch = fetchFile) {
+    this.url = url;
+    this.fetch = fetch;
+  }
+
+  // TODO - how is this used?
+  getKey() {
+    return this.url;
+  }
+
+  async getBytes(offset: number, length: number, signal?: AbortSignal): Promise<RangeResponse> {
+    let controller;
+    if (!signal) {
+      // ToDO why is it so important to abort in case 200?
+      // TODO check this works or assert 206
+      controller = new AbortController();
+      signal = controller.signal;
+    }
+
+    let response = await fetch(this.url, {
+      signal,
+      headers: {Range: `bytes=${offset}-${offset + length - 1}`}
+    });
+
+    switch (response.status) {
+      case 206: // Partial Content success
+        // This is the expected success code for a range request
+        break;
+
+      case 200:
+        // some well-behaved backends, e.g. DigitalOcean CDN, respond with 200 instead of 206
+        // but we also need to detect no support for Byte Serving which is returning the whole file
+        const content_length = response.headers.get('Content-Length');
+        if (!content_length || Number(content_length) > length) {
+          if (controller) {
+            controller.abort();
+          }
+          throw Error(
+            'content-length header missing or exceeding request. Server must support HTTP Byte Serving.'
+          );
+        }
+
+      case 416: // "Range Not Satisfiable"
+        // some HTTP servers don't accept ranges beyond the end of the resource.
+        // Retry with the exact length
+        // TODO: can return 416 with offset > 0 if content changed, which will have a blank etag.
+        // See https://github.com/protomaps/PMTiles/issues/90
+        if (offset === 0) {
+          const content_range = response.headers.get('Content-Range');
+          if (!content_range || !content_range.startsWith('bytes *')) {
+            throw Error('Missing content-length on 416 response');
+          }
+          const actual_length = Number(content_range.substr(8));
+          response = await fetch(this.url, {
+            signal,
+            headers: {Range: `bytes=0-${actual_length - 1}`}
+          });
+        }
+        break;
+
+      default:
+        if (response.status >= 300) {
+          throw Error(`Bad response code: ${response.status}`);
+        }
+    }
+
+    const data = await response.arrayBuffer();
+    return {
+      data,
+      etag: response.headers.get('ETag') || undefined,
+      cacheControl: response.headers.get('Cache-Control') || undefined,
+      expires: response.headers.get('Expires') || undefined
+    };
+  }
+}
+
+/*
+class TestNodeFileSource implements Source {
+  buffer: ArrayBuffer;
+  path: string;
+  key: string;
+  etag?: string;
+
+  constructor(path: string, key: string) {
+    this.path = path;
+    this.buffer = fs.readFileSync(path);
+    this.key = key;
+  }
+
+  getKey() {
+    return this.key;
+  }
+
+  replaceData(path: string) {
+    this.path = path;
+    this.buffer = fs.readFileSync(path);
+  }
+
+  async getBytes(offset: number, length: number): Promise<RangeResponse> {
+    const slice = new Uint8Array(this.buffer.slice(offset, offset + length))
+      .buffer;
+    return { data: slice, etag: this.etag };
+  }
+}
+*/

package/src/lib/filesystems/filesystem.ts

@@ -1,87 +1,38 @@
 // loaders.gl, MIT license
 
-export type ReadOptions = {};
-
-export type Stat = {
-  size: number;
-  isDirectory: () => boolean;
-};
+import {ReadableFile, WritableFile} from '../files/file';
 
 /**
  * A FileSystem interface can encapsulate various file sources,
- * a FileList, a ZipFile, a GoogleDrive etc.
+ * a FileList, a Node.js filesystem, a ZipFile, a GoogleDrive etc.
  */
 export interface FileSystem {
-  /**
-   * Return a list of file names
-   * @param dirname directory name. file system root directory if omitted
-   */
+  /** Return a list of file names in a "directory" */
   readdir(dirname?: string, options?: {recursive?: boolean}): Promise<string[]>;
 
-  /**
-   * Gets information from a local file from the filesystem
-   * @param filename file name to stat
-   * @param options currently unused
-   * @throws if filename is not in local filesystem
-   */
+  /** Gets information from a local file from the filesystem */
   stat(filename: string, options?: object): Promise<{size: number}>;
 
-  /**
-   * Fetches a local file from the filesystem (or a URL)
-   * @param filename
-   * @param options
-   */
-  fetch(filename: RequestInfo, options?: RequestInit): Promise<Response>;
-}
+  /** Removes a file from the file system */
+  unlink?(path: string): Promise<void>;
 
-/**
- * A random access file system
- */
-export interface RandomAccessReadFileSystem extends FileSystem {
-  open(path: string, flags: unknown, mode?: unknown): Promise<any>;
-  close(fd: unknown): Promise<void>;
-  fstat(fd: unknown): Promise<Stat>;
-  read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Uint8Array}>;
-  // read(
-  //   fd: any,
-  //   buffer: ArrayBuffer | ArrayBufferView,
-  //   offset?: number,
-  //   length?: number,
-  //   position?: number
-  // ): Promise<{bytesRead: number; buffer: ArrayBuffer}>;
+  /** Fetches the full contents of a file from the filesystem (or a URL) */
+  fetch(path: string, options?: RequestInit): Promise<Response>;
 }
 
 /**
- * A FileSystem interface can encapsulate a FileList, a ZipFile, a GoogleDrive etc.
- *
-export interface IFileSystem {
-  /**
-   * Return a list of file names
-   * @param dirname directory name. file system root directory if omitted
-   *
-  readdir(dirname?: string, options?: {recursive?: boolean}): Promise<string[]>;
+ * A random access file system, open readable and/or writable files
+ */
+export interface RandomAccessFileSystem extends FileSystem {
+  /** Can open readable files */
+  readonly readable: boolean;
 
-  /**
-   * Gets information from a local file from the filesystem
-   * @param filename file name to stat
-   * @param options currently unused
-   * @throws if filename is not in local filesystem
-   *
-  stat(filename: string, options?: object): Promise<{size: number}>;
+  /** Can open writable files */
+  readonly writable: boolean;
 
-  /**
-   * Fetches a local file from the filesystem (or a URL)
-   * @param filename
-   * @param options
-   *
-  fetch(filename: string, options?: object): Promise<Response>;
-}
+  /** Open a readable file */
+  openReadableFile(path: string, flags?: 'r'): Promise<ReadableFile>;
 
-type ReadOptions = {buffer?: ArrayBuffer; offset?: number; length?: number; position?: number};
-export interface IRandomAccessReadFileSystem extends IFileSystem {
-  open(path: string, flags: string | number, mode?: any): Promise<any>;
-  close(fd: any): Promise<void>;
-  fstat(fd: any): Promise<object>;
-  read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Buffer}>;
+  /** Open a writable file */
+  openWritableFile(path: string, flags?: 'w' | 'wx', mode?: number): Promise<WritableFile>;
 }
-*/

package/src/lib/filesystems/node-filesystem-facade.ts

@@ -0,0 +1,63 @@
+// loaders.gl, MIT license
+
+import {isBrowser} from '../env-utils/globals';
+import {Stat} from '../files/file';
+import {NodeFileFacade as NodeFile} from '../files/node-file-facade';
+import {RandomAccessFileSystem} from './filesystem';
+
+const NOT_IMPLEMENTED = new Error('Not implemented');
+
+/**
+ * FileSystem pass-through for Node.js
+ * Compatible with BrowserFileSystem.
+ * @note Dummy implementation, not used (constructor returns a real NodeFileSystem instance)
+ * @param options
+ */
+export class NodeFileSystemFacade implements RandomAccessFileSystem {
+  // implements FileSystem
+  constructor(options: {[key: string]: any}) {
+    if (globalThis.loaders?.NodeFileSystem) {
+      return new globalThis.loaders.NodeFileSystem(options);
+    }
+    if (isBrowser) {
+      throw new Error('Can\'t instantiate NodeFileSystem in browser.');
+    }
+    throw new Error(
+      'Can\'t instantiate NodeFileSystem. Make sure to import @loaders.gl/polyfills first.'
+    );
+  }
+
+  // DUMMY IMPLEMENTATION, not used (constructor returns a real NodeFileSystem instance)
+
+  // implements RandomAccessReadFileSystem
+
+  readonly readable = true;
+  readonly writable = true;
+
+  async openReadableFile(path: string, flags): Promise<NodeFile> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  // implements RandomAccessWriteFileSystem
+  async openWritableFile(path: string, flags, mode): Promise<NodeFile> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  // Implements file system
+
+  async readdir(dirname = '.', options?: {}): Promise<string[]> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  async stat(path: string, options?: {}): Promise<Stat> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  async unlink(path: string): Promise<void> {
+    throw NOT_IMPLEMENTED;
+  }
+
+  async fetch(path: RequestInfo, options?: RequestInit): Promise<Response> {
+    throw NOT_IMPLEMENTED;
+  }
+}

package/src/lib/node/buffer.browser.ts

@@ -15,6 +15,6 @@ export function toArrayBuffer(buffer) {
 /**
  * Convert (copy) ArrayBuffer to Buffer
  */
-export function toBuffer(binaryData: ArrayBuffer | ArrayBuffer | Buffer): Buffer {
+export function toBuffer(binaryData: ArrayBuffer | Buffer): Buffer {
   throw new Error('Buffer not supported in browser');
 }

package/src/lib/node/buffer.ts

@@ -20,7 +20,7 @@ export function toArrayBuffer(buffer) {
 /**
  * Convert (copy) ArrayBuffer to Buffer
  */
-export function toBuffer(binaryData: ArrayBuffer | ArrayBuffer | Buffer): Buffer {
+export function toBuffer(binaryData: ArrayBuffer | Buffer): Buffer {
   if (Buffer.isBuffer(binaryData)) {
     return binaryData;
   }

package/src/types.ts

@@ -7,8 +7,6 @@ export type TypedIntArray =
   | Int16Array
   | Uint16Array
   | Int32Array
-  | Uint32Array
-  | Int32Array
   | Uint32Array;
 
 export type TypedFloatArray = Uint16Array | Float32Array | Float64Array;
@@ -24,8 +22,6 @@ export type TypedArrayConstructor =
   | Uint16ArrayConstructor
   | Int32ArrayConstructor
   | Uint32ArrayConstructor
-  | Int32ArrayConstructor
-  | Uint32ArrayConstructor
   | Float32ArrayConstructor
   | Float64ArrayConstructor;