@loaders.gl/polyfills 4.0.0-beta.2 → 4.0.0-beta.3

package/src/file/install-file-polyfills.ts

@@ -0,0 +1,20 @@
+// import {ReadableStreamPolyfill} from './readable-stream';
+import {FileReaderPolyfill} from './file-reader';
+import {FilePolyfill} from './file';
+
+export function installFilePolyfills() {
+  if (typeof FileReader === 'undefined' && !globalThis.FileReader) {
+    // @ts-ignore;
+    globalThis.FileReader = FileReaderPolyfill;
+  }
+
+  // Install minimal Node.js File polyfill
+  if (typeof File === 'undefined' && !globalThis.File) {
+    // @ts-ignore;
+    globalThis.File = FilePolyfill;
+  }
+
+  return global;
+}
+
+export const File_ = installFilePolyfills();

package/src/{node/file → file}/readable-stream.ts

@@ -2,7 +2,7 @@ import {ReadableStream as WSPReadableStream} from 'web-streams-polyfill';
 
 // Want a polyfill, but please don't install it
 // @ts-ignore
-delete global.ReadableStream;
+delete globalThis.ReadableStream;
 
 // @ts-ignore
 export class ReadableStreamPolyfill<T> extends WSPReadableStream<T> implements ReadableStream {}

package/src/filesystems/fetch-node.ts

@@ -0,0 +1,96 @@
+// loaders.gl, MIT license
+
+import fs from 'fs';
+import {Readable} from 'stream';
+import {resolvePath} from '@loaders.gl/loader-utils';
+import {decompressReadStream} from './stream-utils.node';
+
+const isBoolean = (x) => typeof x === 'boolean';
+const isFunction = (x) => typeof x === 'function';
+const isObject = (x) => x !== null && typeof x === 'object';
+const isReadableNodeStream = (x) =>
+  isObject(x) && isFunction(x.read) && isFunction(x.pipe) && isBoolean(x.readable);
+
+/**
+ * Enables
+ * @param url
+ * @param options
+ * @returns
+ */
+// eslint-disable-next-line max-statements
+export async function fetchNode(url: string, options?: RequestInit): Promise<Response> {
+  // Support `file://` protocol
+  const FILE_PROTOCOL_REGEX = /^file:\/\//;
+  url.replace(FILE_PROTOCOL_REGEX, '/');
+
+  // Remove any query parameters, as they have no meaning
+  let noqueryUrl = url.split('?')[0];
+  noqueryUrl = resolvePath(noqueryUrl);
+
+  const responseHeaders = new Headers();
+  // Automatically decompress gzipped files with .gz extension
+  if (url.endsWith('.gz')) {
+    // url = url.slice(0, -3);
+    responseHeaders['content-encoding'] = 'gzip';
+  }
+  if (url.endsWith('.br')) {
+    // url = url.slice(0, -3);
+    responseHeaders['content-encoding'] = 'br';
+  }
+
+  try {
+    // Now open the stream
+    const body = await new Promise<fs.ReadStream>((resolve, reject) => {
+      // @ts-ignore
+      const stream = fs.createReadStream(noqueryUrl, {encoding: null});
+      stream.once('readable', () => resolve(stream));
+      stream.on('error', (error) => reject(error));
+    });
+
+    let bodyStream: Readable = body;
+
+    // Check for content-encoding and create a decompression stream
+    if (isReadableNodeStream(body)) {
+      bodyStream = decompressReadStream(body, responseHeaders);
+    } else if (typeof body === 'string') {
+      bodyStream = Readable.from([new TextEncoder().encode(body)]);
+    } else {
+      bodyStream = Readable.from([body || new ArrayBuffer(0)]);
+    }
+
+    const status = 200;
+    const statusText = 'OK';
+    const headers = getHeadersForFile(noqueryUrl);
+    // @ts-expect-error
+    const response = new Response(bodyStream, {headers, status, statusText});
+    Object.defineProperty(response, 'url', {value: url});
+    return response;
+  } catch (error) {
+    // console.error(error);
+    const errorMessage = (error as Error).message;
+    const status = 400;
+    const statusText = errorMessage;
+    const headers = {};
+    const response = new Response(errorMessage, {headers, status, statusText});
+    Object.defineProperty(response, 'url', {value: url});
+    return response;
+  }
+}
+
+function getHeadersForFile(noqueryUrl: string): Headers {
+  const headers = {};
+
+  // Fix up content length if we can for best progress experience
+  if (!headers['content-length']) {
+    const stats = fs.statSync(noqueryUrl);
+    headers['content-length'] = stats.size;
+  }
+
+  // Automatically decompress gzipped files with .gz extension
+  if (noqueryUrl.endsWith('.gz')) {
+    noqueryUrl = noqueryUrl.slice(0, -3);
+    headers['content-encoding'] = 'gzip';
+  }
+
+  return new Headers(headers);
+}

package/src/filesystems/node-file.ts

@@ -0,0 +1,137 @@
+import type {ReadableFile, WritableFile, Stat} from '@loaders.gl/loader-utils';
+import {resolvePath} from '@loaders.gl/loader-utils';
+import fs from 'fs';
+
+export class NodeFile implements ReadableFile, WritableFile {
+  handle: number;
+  size: number;
+  bigsize: bigint;
+  url: string;
+
+  constructor(path: string, flags: 'r' | 'w' | 'wx', mode?: number) {
+    path = resolvePath(path);
+    this.handle = fs.openSync(path, flags, mode);
+    const stats = fs.fstatSync(this.handle, {bigint: true});
+    this.size = Number(stats.size);
+    this.bigsize = stats.size;
+    this.url = path;
+  }
+
+  async close(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      fs.close(this.handle, (err) => (err ? reject(err) : resolve()));
+    });
+  }
+
+  async stat(): Promise<Stat> {
+    return await new Promise((resolve, reject) =>
+      fs.fstat(this.handle, {bigint: true}, (err, info) => {
+        const stats: Stat = {
+          size: Number(info.size),
+          bigsize: info.size,
+          isDirectory: info.isDirectory()
+        };
+        if (err) {
+          reject(err);
+        } else {
+          resolve(stats);
+        }
+      })
+    );
+  }
+
+  async read(offset: number | bigint, length: number): Promise<ArrayBuffer> {
+    const arrayBuffer = new ArrayBuffer(length);
+    let bigOffset = BigInt(offset);
+
+    let totalBytesRead = 0;
+    const uint8Array = new Uint8Array(arrayBuffer);
+
+    let position;
+    // Read in loop until we get required number of bytes
+    while (length > 0) {
+      const bytesRead = await readBytes(this.handle, uint8Array, 0, length, bigOffset);
+
+      // Check if end of file reached
+      if (bytesRead === 0) {
+        break;
+      }
+
+      totalBytesRead += bytesRead;
+      bigOffset += BigInt(bytesRead);
+      length -= bytesRead;
+
+      // Advance position unless we are using built-in position advancement
+      if (position !== undefined) {
+        position += bytesRead;
+      }
+    }
+    return totalBytesRead < length ? arrayBuffer.slice(0, totalBytesRead) : arrayBuffer;
+  }
+
+  async write(
+    arrayBuffer: ArrayBuffer,
+    offset: number | bigint = 0,
+    length: number = arrayBuffer.byteLength
+  ): Promise<number> {
+    return new Promise((resolve, reject) => {
+      // TODO - Node.js doesn't offer write with bigint offsets???
+      const nOffset = Number(offset);
+      const uint8Array = new Uint8Array(arrayBuffer, Number(offset), length);
+      fs.write(this.handle, uint8Array, 0, length, nOffset, (err, bytesWritten) =>
+        err ? reject(err) : resolve(bytesWritten)
+      );
+    });
+  }
+}
+
+async function readBytes(
+  fd: number,
+  uint8Array: Uint8Array,
+  offset: number,
+  length: number,
+  position: number | bigint | null
+): Promise<number> {
+  return await new Promise<number>((resolve, reject) =>
+    fs.read(fd, uint8Array, offset, length, position, (err, bytesRead) =>
+      err ? reject(err) : resolve(bytesRead)
+    )
+  );
+}
+
+// TODO - implement streaming write
+/*
+export interface WriteStreamOptions {
+  flags?: string;
+  encoding?: 'utf8';
+  fd?: number;
+  mode?: number;
+  autoClose?: boolean;
+  start?: number;
+}
+
+export class NodeStreamWritableFile implements WritableFile {
+  outputStream: fs.WriteStream | Writable;
+
+  constructor(pathOrStream: string | Writable, options?: WriteStreamOptions) {
+    this.outputStream =
+      typeof pathOrStream === 'string' ? fs.createWriteStream(pathOrStream, options) : pathOrStream;
+  }
+
+  async write(buffer: ArrayBuffer): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const uint8Array = new Uint8Array(buffer);
+      this.outputStream.write(uint8Array, (err) => (err ? reject(err) : resolve()));
+    });
+  }
+
+  async close(): Promise<void> {
+    if (this.outputStream instanceof fs.WriteStream) {
+      return new Promise((resolve, reject) => {
+        const stream = this.outputStream as fs.WriteStream;
+        stream.close((err) => (err ? reject(err) : resolve()));
+      });
+    }
+  }
+}
+*/

package/src/filesystems/node-filesystem.ts

@@ -1,87 +1,52 @@
-import {FileSystem, RandomAccessReadFileSystem} from '@loaders.gl/loader-utils';
-import fs from 'fs';
+// loaders.gl, MIT license
+
+import {Stat, RandomAccessFileSystem} from '@loaders.gl/loader-utils';
 import fsPromise from 'fs/promises';
+import {NodeFile} from './node-file';
+import {fetchNode} from './fetch-node';
 
 // import {fetchFile} from "../fetch/fetch-file"
 // import {selectLoader} from "../api/select-loader";
 
-type Stat = {
-  size: number;
-  isDirectory: () => boolean;
-  info?: fs.Stats;
-};
-
-type ReadOptions = {
-  buffer?: Buffer;
-  offset?: number;
-  length?: number;
-  position?: number;
-};
-
 /**
  * FileSystem pass-through for Node.js
  * Compatible with BrowserFileSystem.
  * @param options
  */
-export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem {
+export class NodeFileSystem implements RandomAccessFileSystem {
+  readable: boolean = true;
+  writable: boolean = true;
+
   // implements FileSystem
-  constructor(options: {[key: string]: any}) {
-    this.fetch = options._fetch;
-  }
+  constructor() {}
 
   async readdir(dirname = '.', options?: {}): Promise<any[]> {
     return await fsPromise.readdir(dirname, options);
   }
 
-  async stat(path: string, options?: {}): Promise<Stat> {
-    const info = await fsPromise.stat(path, options);
-    return {size: Number(info.size), isDirectory: () => false, info};
+  async stat(path: string): Promise<Stat> {
+    const info = await fsPromise.stat(path, {bigint: true});
+    return {
+      size: Number(info.size),
+      bigsize: info.size,
+      isDirectory: info.isDirectory()
+    };
   }
 
-  async fetch(path: string, options: {[key: string]: any}) {
-    // Falls back to handle https:/http:/data: etc fetches
-    // eslint-disable-next-line
-    const fallbackFetch = options.fetch || this.fetch;
-    return fallbackFetch(path, options);
+  async unlink(path: string): Promise<void> {
+    return await fsPromise.unlink(path);
   }
 
-  // implements IRandomAccessFileSystem
-  async open(path: string, flags: string | number, mode?: any): Promise<number> {
-    return (await fsPromise.open(path, flags)) as unknown as number;
-  }
-
-  async close(fd: number): Promise<void> {
-    fs.close(fd);
+  async fetch(path: string, options: RequestInit): Promise<Response> {
+    return await fetchNode(path, options);
   }
 
-  async fstat(fd: number): Promise<Stat> {
-    return await new Promise((resolve, reject) =>
-      fs.fstat(fd, (err, info) => (err ? reject(err) : resolve(info)))
-    );
+  // implements IRandomAccessFileSystem
+  async openReadableFile(path: string, flags: 'r' = 'r'): Promise<NodeFile> {
+    return new NodeFile(path, flags);
   }
 
-  async read(
-    fd: number,
-    // @ts-ignore Possibly null
-    {buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions
-  ): Promise<{bytesRead: number; buffer: Uint8Array}> {
-    let totalBytesRead = 0;
-    // Read in loop until we get required number of bytes
-    while (totalBytesRead < length) {
-      const {bytesRead} = await new Promise<{bytesRead: number; buffer: Buffer}>(
-        // eslint-disable-next-line no-loop-func
-        (resolve, reject) =>
-          fs.read(
-            fd,
-            buffer,
-            offset + totalBytesRead,
-            length - totalBytesRead,
-            position + totalBytesRead,
-            (err, bytesRead, buffer) => (err ? reject(err) : resolve({bytesRead, buffer}))
-          )
-      );
-      totalBytesRead += bytesRead;
-    }
-    return {bytesRead: totalBytesRead, buffer};
+  async openWritableFile(path: string, flags: 'w' | 'wx' = 'w', mode?: any): Promise<NodeFile> {
+    return new NodeFile(path, flags, mode);
   }
 }

package/src/{node/fetch/utils → filesystems}/stream-utils.node.ts

@@ -1,14 +1,16 @@
 // loaders.gl, MIT license
 
 import zlib from 'zlib';
+import {Readable} from 'stream';
 
-import {toArrayBuffer} from './decode-data-uri.node';
+const isArrayBuffer = (x) => x && x instanceof ArrayBuffer;
+const isBuffer = (x) => x && x instanceof Buffer;
 
 /**
  *
  */
-export function decompressReadStream(readStream, headers) {
-  switch (headers.get('content-encoding')) {
+export function decompressReadStream(readStream: Readable, headers?: Headers) {
+  switch (headers?.get('content-encoding')) {
     case 'br':
       return readStream.pipe(zlib.createBrotliDecompress());
     case 'gzip':
@@ -77,3 +79,40 @@ export function concatenateArrayBuffers(sources: (ArrayBuffer | Uint8Array)[]):
   // We work with ArrayBuffers, discard the typed array wrapper
   return result.buffer;
 }
+
+/**
+ * @param data
+ * @todo Duplicate of core
+ */
+export function toArrayBuffer(data: unknown): ArrayBuffer {
+  if (isArrayBuffer(data)) {
+    return data as ArrayBuffer;
+  }
+
+  // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+  if (isBuffer(data)) {
+    // @ts-expect-error
+    const typedArray = new Uint8Array(data);
+    return typedArray.buffer;
+  }
+
+  // Careful - Node Buffers will look like ArrayBuffers (keep after isBuffer)
+  if (ArrayBuffer.isView(data)) {
+    return data.buffer;
+  }
+
+  if (typeof data === 'string') {
+    const text = data;
+    const uint8Array = new TextEncoder().encode(text);
+    return uint8Array.buffer;
+  }
+
+  // HACK to support Blob polyfill
+  // @ts-expect-error
+  if (data && typeof data === 'object' && data._toArrayBuffer) {
+    // @ts-expect-error
+    return data._toArrayBuffer();
+  }
+
+  throw new Error(`toArrayBuffer(${JSON.stringify(data, null, 2).slice(10)})`);
+}

package/src/index.browser.ts

@@ -1,14 +1,10 @@
 // loaders.gl, MIT License
 
-import {allSettled} from './promise/all-settled';
-
-if (!('allSettled' in Promise)) {
-  // @ts-ignore
-  Promise.allSettled = allSettled;
-}
-
-// eslint-disable-next-line @typescript-eslint/no-empty-function
 export function installFilePolyfills() {}
 
 // Dummy export to avoid import errors in browser tests
 export const NodeFileSystem = null;
+
+export function fetchNode(path: string, options: RequestInit): Promise<Response> {
+  throw new Error('fetchNode not available in browser');
+}

package/src/index.ts

@@ -1,20 +1,29 @@
 /* eslint-disable dot-notation */
-import {isBrowser} from './utils/globals';
+import {isBrowser} from './utils/is-browser';
 
-import {TextDecoder, TextEncoder} from './lib/encoding';
-import {allSettled} from './promise/all-settled';
+import {TextDecoder, TextEncoder} from './text-encoder/text-encoder';
 
 // Node specific
-import * as base64 from './node/buffer/btoa.node';
+import {atob, btoa} from './buffer/btoa.node';
 
-import {encodeImageNode} from './node/images/encode-image.node';
-import {parseImageNode, NODE_FORMAT_SUPPORT} from './node/images/parse-image.node';
+import {encodeImageNode} from './images/encode-image.node';
+import {parseImageNode, NODE_FORMAT_SUPPORT} from './images/parse-image.node';
 
-// export {ReadableStreamPolyfill} from './node/file/readable-stream';
-// export {BlobPolyfill} from './node/file/blob';
-export {FileReaderPolyfill} from './node/file/file-reader';
-export {FilePolyfill} from './node/file/file';
-export {installFilePolyfills} from './node/file/install-file-polyfills';
+// FILESYSTEM POLYFILLS
+import {NodeFile} from './filesystems/node-file';
+import {NodeFileSystem} from './filesystems/node-filesystem';
+import {fetchNode} from './filesystems/fetch-node';
+
+// NODE VERSION
+import {versions} from 'node:process';
+export const nodeVersion = parseInt(versions.node.split('.')[0]);
+
+// STREAM POLYFILLS
+import {makeNodeStream} from './streams/make-node-stream';
+
+// BLOB AND FILE POLYFILLS
+export {Blob_ as Blob} from './file/install-blob-polyfills';
+export {File_ as File} from './file/install-file-polyfills';
 
 if (isBrowser) {
   // eslint-disable-next-line no-console
@@ -23,6 +32,17 @@ if (isBrowser) {
   );
 }
 
+globalThis.loaders = globalThis.loaders || {};
+
+// STREAM POLYFILLS
+export {makeNodeStream} from './streams/make-node-stream';
+globalThis.loaders.makeNodeStream = makeNodeStream;
+
+// FILESYSTEM POLYFILLS
+globalThis.loaders.NodeFile = NodeFile;
+globalThis.loaders.NodeFileSystem = NodeFileSystem;
+globalThis.loaders.fetchNode = fetchNode;
+
 // POLYFILLS: TextEncoder, TextDecoder
 // - Recent Node versions have these classes but virtually no encodings unless special build.
 // - Browser: Edge, IE11 do not have these
@@ -41,58 +61,48 @@ if (!globalThis.TextDecoder) {
 // - Node: Yes
 // - Browser: No
 
-if (!isBrowser && !('atob' in globalThis) && base64.atob) {
-  globalThis['atob'] = base64.atob;
+if (!('atob' in globalThis) && atob) {
+  globalThis['atob'] = atob;
 }
-if (!isBrowser && !('btoa' in globalThis) && base64.btoa) {
-  globalThis['btoa'] = base64.btoa;
+if (!('btoa' in globalThis) && btoa) {
+  globalThis['btoa'] = btoa;
 }
 
-globalThis.loaders = globalThis.loaders || {};
-
-// FILESYSTEM POLYFILLS:
-export {NodeFileSystem} from './filesystems/node-filesystem';
-import {NodeFileSystem} from './filesystems/node-filesystem';
-globalThis.loaders.NodeFileSystem = NodeFileSystem;
-
 // NODE IMAGE FUNCTIONS:
 // These are not official polyfills but used by the @loaders.gl/images module if installed
 // TODO - is there an appropriate Image API we could polyfill using an adapter?
 
-if (!isBrowser && !('_encodeImageNode' in globalThis) && encodeImageNode) {
+if (!('_encodeImageNode' in globalThis) && encodeImageNode) {
   globalThis['_encodeImageNode'] = encodeImageNode;
 }
 
-if (!isBrowser && !('_parseImageNode' in globalThis) && parseImageNode) {
+if (!('_parseImageNode' in globalThis) && parseImageNode) {
   globalThis['_parseImageNode'] = parseImageNode;
   globalThis['_imageFormatsNode'] = NODE_FORMAT_SUPPORT;
 }
 
-if (!('allSettled' in Promise)) {
-  // @ts-ignore
-  Promise.allSettled = allSettled;
-}
-
 // DEPRECATED POLYFILL:
 // - Node v18+: No, not needed
 // - Node v16 and lower: Yes
 // - Browsers (evergreen): Not needed.
 // - IE11: No. This polyfill is node only, install external polyfill
-import {Headers as HeadersNode} from './node/fetch/headers.node';
-import {Response as ResponseNode} from './node/fetch/response.node';
-import {fetchNode as fetchNode} from './node/fetch/fetch.node';
-
-if (!isBrowser && !('Headers' in globalThis) && HeadersNode) {
-  // @ts-expect-error
-  globalThis.Headers = HeadersNode;
-}
-
-if (!isBrowser && !('Response' in globalThis) && ResponseNode) {
-  // @ts-expect-error
-  globalThis.Response = ResponseNode;
-}
-
-if (!isBrowser && !('fetch' in globalThis) && fetchNode) {
-  // @ts-expect-error
-  globalThis.fetch = fetchNode;
+import {Headers as HeadersNode} from './fetch/headers-polyfill';
+import {Response as ResponseNode} from './fetch/response-polyfill';
+import {fetchNode as fetchNodePolyfill} from './fetch/fetch-polyfill';
+
+if (nodeVersion < 18) {
+  if (!('Headers' in globalThis) && HeadersNode) {
+    // @ts-ignore
+    globalThis.Headers = HeadersNode;
+  }
+
+  if (!('Response' in globalThis) && ResponseNode) {
+    // @ts-ignore
+    globalThis.Response = ResponseNode;
+  }
+
+  if (!('fetch' in globalThis) && fetchNodePolyfill) {
+    // @ts-ignore
+    globalThis.fetch = fetchNodePolyfill;
+  }
 }

package/src/streams/make-node-stream.ts

@@ -0,0 +1,74 @@
+import type {ReadableOptions} from 'stream';
+import * as Stream from 'stream';
+
+class _Readable {}
+
+type ReadableType = Stream.Readable | _Readable;
+const Readable = Stream.Readable || _Readable;
+
+export type MakeStreamOptions = ReadableOptions;
+
+/** Builds a node stream from an iterator */
+export function makeNodeStream<ArrayBuffer>(
+  source: Iterable<ArrayBuffer> | AsyncIterable<ArrayBuffer>,
+  options?: ReadableOptions
+): ReadableType {
+  // @ts-ignore AsyncGenerator
+  const iterator = source[Symbol.asyncIterator]
+    ? // @ts-ignore AsyncGenerator
+      source[Symbol.asyncIterator]()
+    : // @ts-ignore AsyncGenerator
+      source[Symbol.iterator]();
+  return new AsyncIterableReadable(iterator, options);
+}
+
+class AsyncIterableReadable extends Readable {
+  private _pulling: boolean;
+  private _bytesMode: boolean;
+  private _iterator: AsyncIterator<ArrayBuffer>;
+
+  constructor(it: AsyncIterator<ArrayBuffer>, options?: ReadableOptions) {
+    super(options);
+    this._iterator = it;
+    this._pulling = false;
+    this._bytesMode = !options || !options.objectMode;
+  }
+
+  async _read(size: number): Promise<void> {
+    if (!this._pulling) {
+      this._pulling = true;
+      this._pulling = await this._pull(size, this._iterator);
+    }
+  }
+
+  async _destroy(error: Error | null, cb: (e: Error | null) => void): Promise<void> {
+    if (!this._iterator) {
+      return;
+    }
+    if (error) {
+      await this._iterator?.throw?.(error);
+    } else {
+      await this._iterator?.return?.(error);
+    }
+    cb?.(null);
+  }
+
+  // eslint-disable-next-line complexity
+  private async _pull(size: number, it: AsyncIterator<ArrayBuffer>): Promise<boolean> {
+    const bm = this._bytesMode;
+    let r: IteratorResult<ArrayBuffer> | null = null;
+    // while (this.readable && !(r = await it.next(bm ? size : null)).done) {
+    while (this.readable && !(r = await it.next()).done) {
+      if (size !== null) {
+        size -= bm && ArrayBuffer.isView(r.value) ? r.value.byteLength : 1;
+      }
+      if (!this.push(new Uint8Array(r.value)) || size <= 0) {
+        break;
+      }
+    }
+    if ((r?.done || !this.readable) && (this.push(null) || true)) {
+      it?.return?.();
+    }
+    return !this.readable;
+  }
+}