@loaders.gl/loader-utils 4.0.0-alpha.22 → 4.0.0-alpha.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/loader-utils",
-  "version": "4.0.0-alpha.22",
+  "version": "4.0.0-alpha.24",
   "description": "Framework-independent loaders for 3D graphics formats",
   "license": "MIT",
   "publishConfig": {
@@ -28,18 +28,22 @@
   ],
   "browser": {
     "./src/lib/node/buffer.ts": "./src/lib/node/buffer.browser.ts",
-    "./src/lib/node/buffer.js": "./src/lib/node/buffer.browser.js",
-    "./dist/lib/node/buffer.js": "./dist/lib/node/buffer.browser.js",
     "./dist/es5/lib/node/buffer.js": "./dist/es5/lib/node/buffer.browser.js",
     "./dist/esm/lib/node/buffer.js": "./dist/esm/lib/node/buffer.browser.js",
+    "./src/lib/node/fs.ts": "./src/lib/node/fs.browser.ts",
+    "./dist/es5/lib/node/fs.js": "./dist/es5/lib/node/fs.browser.js",
+    "./dist/esm/lib/node/fs.js": "./dist/esm/lib/node/fs.browser.js",
+    "./src/lib/node/stream.ts": "./src/lib/node/stream.browser.ts",
+    "./dist/es5/lib/node/stream.js": "./dist/es5/lib/node/stream.browser.js",
+    "./dist/esm/lib/node/stream.js": "./dist/esm/lib/node/stream.browser.js",
     "fs": false,
     "stream": false
   },
   "scripts": {},
   "dependencies": {
     "@babel/runtime": "^7.3.1",
-    "@loaders.gl/worker-utils": "4.0.0-alpha.22",
+    "@loaders.gl/worker-utils": "4.0.0-alpha.24",
     "@probe.gl/stats": "^4.0.2"
   },
-  "gitHead": "0da838c506d1275383f2fd3d244d9c72b25397d2"
+  "gitHead": "97a8990595c132fb14e3445a8768d9f4cb98ff05"
 }

package/src/index.ts

@@ -7,8 +7,6 @@ export type {
   DataType,
   SyncDataType,
   BatchableDataType,
-  IFileSystem,
-  IRandomAccessReadFileSystem,
   // numeric array types
   TypedArray,
   BigTypedArray,
@@ -121,10 +119,18 @@ import * as stream from './lib/node/stream';
 export {stream};
 
 // EXPERIMENTAL
+export type {FileSystem, RandomAccessReadFileSystem} from './lib/filesystems/filesystem';
+export {NodeFileSystem as _NodeFileSystem} from './lib/filesystems/node-filesystem';
+
+export type {FileProvider} from './lib/file-provider/file-provider';
+export {isFileProvider} from './lib/file-provider/file-provider';
+
+export {FileHandle} from './lib/file-provider/file-handle';
+export {FileHandleFile} from './lib/file-provider/file-handle-file';
+export {DataViewFile} from './lib/file-provider/data-view-file';
+
 export type {ReadableFile} from './lib/filesystems/readable-file';
 export {makeReadableFile} from './lib/filesystems/readable-file';
 
 export type {WritableFile} from './lib/filesystems/writable-file';
 export {makeWritableFile} from './lib/filesystems/writable-file';
-
-export {default as _NodeFileSystem} from './lib/filesystems/node-filesystem';

package/src/lib/file-provider/data-view-file.ts

@@ -0,0 +1,72 @@
+import {FileProvider} from './file-provider';
+
+/**
+ * Checks if bigint can be converted to number and convert it if possible
+ * @param bigint bigint to be converted
+ * @returns number
+ */
+const toNumber = (bigint: bigint) => {
+  if (bigint > Number.MAX_SAFE_INTEGER) {
+    throw new Error('Offset is out of bounds');
+  }
+  return Number(bigint);
+};
+
+/** Provides file data using DataView */
+export class DataViewFile implements FileProvider {
+  /** The DataView from which data is provided */
+  private file: DataView;
+
+  constructor(file: DataView) {
+    this.file = file;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  async destroy(): Promise<void> {}
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: bigint): Promise<number> {
+    return this.file.getUint8(toNumber(offset));
+  }
+
+  /**
+   * Gets an unsigned 16-bit intege at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: bigint): Promise<number> {
+    return this.file.getUint16(toNumber(offset), true);
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: bigint): Promise<number> {
+    return this.file.getUint32(toNumber(offset), true);
+  }
+
+  /**
+   * Gets an unsigned 64-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: bigint): Promise<bigint> {
+    return this.file.getBigUint64(toNumber(offset), true);
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    return this.file.buffer.slice(toNumber(startOffset), toNumber(endOffset));
+  }
+
+  /** the length (in bytes) of the data. */
+  get length() {
+    return BigInt(this.file.byteLength);
+  }
+}

package/src/lib/file-provider/file-handle-file.ts

@@ -0,0 +1,116 @@
+import {FileProvider} from './file-provider';
+import {FileHandle} from './file-handle';
+import {resolvePath} from '../path-utils/file-aliases';
+
+/**
+ * Provides file data using node fs library
+ */
+export class FileHandleFile implements FileProvider {
+  /**
+   * Returns a new copy of FileHandleFile
+   * @param path The path to the file in file system
+   */
+  static async from(path: string): Promise<FileHandleFile> {
+    path = resolvePath(path);
+    const fileDescriptor = await FileHandle.open(path);
+    return new FileHandleFile(fileDescriptor, fileDescriptor.stat.size);
+  }
+
+  /**
+   * The FileHandle from which data is provided
+   */
+  private fileDescriptor: FileHandle;
+
+  /**
+   * The file length in bytes
+   */
+  private size: bigint;
+
+  private constructor(fileDescriptor: FileHandle, size: bigint) {
+    this.fileDescriptor = fileDescriptor;
+    this.size = size;
+  }
+
+  /** Close file */
+  async destroy(): Promise<void> {
+    await this.fileDescriptor.close();
+  }
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: bigint): Promise<number> {
+    const val = new Uint8Array(
+      (await this.fileDescriptor.read(Buffer.alloc(1), 0, 1, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: bigint): Promise<number> {
+    const val = new Uint16Array(
+      (await this.fileDescriptor.read(Buffer.alloc(2), 0, 2, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: bigint): Promise<number> {
+    const val = new Uint32Array(
+      (await this.fileDescriptor.read(Buffer.alloc(4), 0, 4, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: bigint): Promise<bigint> {
+    const val = new BigInt64Array(
+      (await this.fileDescriptor.read(Buffer.alloc(8), 0, 8, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffsset The offset, in byte, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffsset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    const bigLength = endOffset - startOffsset;
+    if (bigLength > Number.MAX_SAFE_INTEGER) {
+      throw new Error('too big slice');
+    }
+    const length = Number(bigLength);
+    return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffsset)).buffer
+      .buffer;
+  }
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  get length(): bigint {
+    return this.size;
+  }
+}

package/src/lib/file-provider/file-handle.ts

@@ -0,0 +1,74 @@
+import {read, open, close, stat, BigIntStats} from 'fs';
+
+/** file reading result */
+export type FileReadResult = {
+  /** amount of the bytes read */
+  bytesRead: number;
+  /** the buffer filled with data from file*/
+  buffer: Buffer;
+};
+
+/** Object handling file info */
+export class FileHandle {
+  private fileDescriptor: number;
+  private stats: BigIntStats;
+
+  private constructor(fileDescriptor: number, stats: BigIntStats) {
+    this.fileDescriptor = fileDescriptor;
+    this.stats = stats;
+  }
+  /**
+   * Opens a `FileHandle`.
+   *
+   * @param path path to the file
+   * @return Fulfills with a {FileHandle} object.
+   */
+
+  static async open(path: string): Promise<FileHandle> {
+    const [fd, stats] = await Promise.all([
+      new Promise<number>((resolve, reject) => {
+        open(path, undefined, undefined, (_err, fd) => (_err ? reject(_err) : resolve(fd)));
+      }),
+      new Promise<BigIntStats>((resolve, reject) => {
+        stat(path, {bigint: true}, (_err, stats) => (_err ? reject(_err) : resolve(stats)));
+      })
+    ]);
+    return new FileHandle(fd, stats);
+  }
+
+  /** Close file */
+  async close(): Promise<void> {
+    return new Promise<void>((resolve) => {
+      close(this.fileDescriptor, (_err) => resolve());
+    });
+  }
+
+  /**
+   * Reads data from the file and stores that in the given buffer.
+   *
+   * If the file is not modified concurrently, the end-of-file is reached when the
+   * number of bytes read is zero.
+   * @param buffer A buffer that will be filled with the file data read.
+   * @param offset The location in the buffer at which to start filling.
+   * @param length The number of bytes to read.
+   * @param position The location where to begin reading data from the file. If `null`, data will be read from the current file position, and the position will be updated. If `position` is an
+   * integer, the current file position will remain unchanged.
+   * @return Fulfills upon success with a FileReadResult object
+   */
+  read = (
+    buffer: Buffer,
+    offset: number,
+    length: number,
+    position: number | bigint
+  ): Promise<FileReadResult> => {
+    return new Promise((s) => {
+      read(this.fileDescriptor, buffer, offset, length, position, (_err, bytesRead, buffer) =>
+        s({bytesRead, buffer})
+      );
+    });
+  };
+
+  get stat(): BigIntStats {
+    return this.stats;
+  }
+}

package/src/lib/file-provider/file-provider.ts

@@ -0,0 +1,56 @@
+/**
+ * Interface for providing file data
+ */
+export interface FileProvider {
+  /**
+   * Cleanup class data
+   */
+  destroy(): Promise<void>;
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint8(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint16(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the file of the view where to read the data.
+   */
+  getUint32(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in byte, from the file of the view where to read the data.
+   */
+  getBigUint64(offset: bigint): Promise<bigint>;
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer>;
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  length: bigint;
+}
+
+/**
+ * Check is the object has FileProvider members
+ * @param fileProvider - tested object
+ */
+export const isFileProvider = (fileProvider: unknown) => {
+  return (
+    (fileProvider as FileProvider)?.getUint8 &&
+    (fileProvider as FileProvider)?.slice &&
+    (fileProvider as FileProvider)?.length
+  );
+};

package/src/lib/filesystems/filesystem.ts

@@ -0,0 +1,87 @@
+// loaders.gl, MIT license
+
+export type ReadOptions = {};
+
+export type Stat = {
+  size: number;
+  isDirectory: () => boolean;
+};
+
+/**
+ * A FileSystem interface can encapsulate various file sources,
+ * a FileList, a ZipFile, a GoogleDrive etc.
+ */
+export interface FileSystem {
+  /**
+   * Return a list of file names
+   * @param dirname directory name. file system root directory if omitted
+   */
+  readdir(dirname?: string, options?: {recursive?: boolean}): Promise<string[]>;
+
+  /**
+   * Gets information from a local file from the filesystem
+   * @param filename file name to stat
+   * @param options currently unused
+   * @throws if filename is not in local filesystem
+   */
+  stat(filename: string, options?: object): Promise<{size: number}>;
+
+  /**
+   * Fetches a local file from the filesystem (or a URL)
+   * @param filename
+   * @param options
+   */
+  fetch(filename: RequestInfo, options?: RequestInit): Promise<Response>;
+}
+
+/**
+ * A random access file system
+ */
+export interface RandomAccessReadFileSystem extends FileSystem {
+  open(path: string, flags: unknown, mode?: unknown): Promise<any>;
+  close(fd: unknown): Promise<void>;
+  fstat(fd: unknown): Promise<Stat>;
+  read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Buffer}>;
+  // read(
+  //   fd: any,
+  //   buffer: ArrayBuffer | ArrayBufferView,
+  //   offset?: number,
+  //   length?: number,
+  //   position?: number
+  // ): Promise<{bytesRead: number; buffer: ArrayBuffer}>;
+}
+
+/**
+ * A FileSystem interface can encapsulate a FileList, a ZipFile, a GoogleDrive etc.
+ *
+export interface IFileSystem {
+  /**
+   * Return a list of file names
+   * @param dirname directory name. file system root directory if omitted
+   *
+  readdir(dirname?: string, options?: {recursive?: boolean}): Promise<string[]>;
+
+  /**
+   * Gets information from a local file from the filesystem
+   * @param filename file name to stat
+   * @param options currently unused
+   * @throws if filename is not in local filesystem
+   *
+  stat(filename: string, options?: object): Promise<{size: number}>;
+
+  /**
+   * Fetches a local file from the filesystem (or a URL)
+   * @param filename
+   * @param options
+   *
+  fetch(filename: string, options?: object): Promise<Response>;
+}
+
+type ReadOptions = {buffer?: ArrayBuffer; offset?: number; length?: number; position?: number};
+export interface IRandomAccessReadFileSystem extends IFileSystem {
+  open(path: string, flags: string | number, mode?: any): Promise<any>;
+  close(fd: any): Promise<void>;
+  fstat(fd: any): Promise<object>;
+  read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Buffer}>;
+}
+*/

package/src/lib/filesystems/node-filesystem.ts

@@ -1,5 +1,5 @@
 import * as fs from '../node/fs';
-import {IFileSystem, IRandomAccessReadFileSystem} from '../../types';
+import {FileSystem, RandomAccessReadFileSystem} from './filesystem';
 // import {fetchFile} from "../fetch/fetch-file"
 // import {selectLoader} from "../api/select-loader";
 
@@ -21,8 +21,8 @@ type ReadOptions = {
  * Compatible with BrowserFileSystem.
  * @param options
  */
-export default class NodeFileSystem implements IFileSystem, IRandomAccessReadFileSystem {
-  // implements IFileSystem
+export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem {
+  // implements FileSystem
   constructor(options: {[key: string]: any}) {
     this.fetch = options._fetch;
   }

package/src/lib/node/fs.browser.ts

@@ -0,0 +1 @@
+export const isSupported = false;

package/src/lib/node/fs.ts

@@ -4,74 +4,37 @@ import {toArrayBuffer} from './buffer';
 import {promisify2, promisify3} from './promisify';
 
 export type {Stats, ReadStream, WriteStream} from 'fs';
-
-export let readdir;
 /** Wrapper for Node.js fs method */
-export let stat;
-export let statSync: typeof fs.statSync;
+export const readdir: any = promisify2(fs.readdir);
+/** Wrapper for Node.js fs method */
+export const stat: any = promisify2(fs.stat);
+export const statSync = fs.statSync;
 
 /** Wrapper for Node.js fs method */
-export let readFile;
+export const readFile: any = fs.readFile;
 /** Wrapper for Node.js fs method */
-export let readFileSync: typeof fs.readFileSync;
+export const readFileSync = fs.readFileSync;
 /** Wrapper for Node.js fs method */
-export let writeFile;
+export const writeFile: any = promisify3(fs.writeFile);
 /** Wrapper for Node.js fs method */
-export let writeFileSync: typeof fs.writeFileSync;
+export const writeFileSync = fs.writeFileSync;
 
 // file descriptors
 
 /** Wrapper for Node.js fs method */
-export let open;
+export const open: any = fs.open;
 /** Wrapper for Node.js fs method */
-export let close: (fd: number) => Promise<void>;
+export const close = (fd: number) =>
+  new Promise<void>((resolve, reject) => fs.close(fd, (err) => (err ? reject(err) : resolve())));
 /** Wrapper for Node.js fs method */
-export let read;
+export const read: any = fs.read;
 /** Wrapper for Node.js fs method */
-export let fstat;
-
-export let createReadStream: typeof fs.createReadStream;
-export let createWriteStream: typeof fs.createWriteStream;
-
-export let isSupported = Boolean(fs);
-
-// paths
+export const fstat: any = fs.fstat;
 
-try {
-  /** Wrapper for Node.js fs method */
-  readdir = promisify2(fs.readdir);
-  /** Wrapper for Node.js fs method */
-  stat = promisify2(fs.stat);
-  statSync = fs.statSync;
+export const createReadStream = fs.createReadStream;
+export const createWriteStream = fs.createWriteStream;
 
-  /** Wrapper for Node.js fs method */
-  readFile = fs.readFile;
-  /** Wrapper for Node.js fs method */
-  readFileSync = fs.readFileSync;
-  /** Wrapper for Node.js fs method */
-  writeFile = promisify3(fs.writeFile);
-  /** Wrapper for Node.js fs method */
-  writeFileSync = fs.writeFileSync;
-
-  // file descriptors
-
-  /** Wrapper for Node.js fs method */
-  open = fs.open;
-  /** Wrapper for Node.js fs method */
-  close = (fd: number) =>
-    new Promise((resolve, reject) => fs.close(fd, (err) => (err ? reject(err) : resolve())));
-  /** Wrapper for Node.js fs method */
-  read = fs.read;
-  /** Wrapper for Node.js fs method */
-  fstat = fs.fstat;
-
-  createReadStream = fs.createReadStream;
-  createWriteStream = fs.createWriteStream;
-
-  isSupported = Boolean(fs);
-} catch {
-  // ignore
-}
+export const isSupported = Boolean(fs);
 
 export async function _readToArrayBuffer(fd: number, start: number, length: number) {
   const buffer = Buffer.alloc(length);

package/src/lib/node/stream.browser.ts

@@ -0,0 +1 @@
+export const isSupported = false;

package/src/lib/node/stream.ts

@@ -4,15 +4,7 @@ import stream from 'stream';
 
 export type {Writable} from 'stream';
 
-export let Transform;
+/** Wrapper for Node.js stream method */
+export const Transform = stream.Transform;
 
 export const isSupported = Boolean(stream);
-
-// paths
-
-try {
-  /** Wrapper for Node.js fs method */
-  Transform = stream.Transform;
-} catch {
-  // ignore
-}

package/src/types.ts

@@ -69,37 +69,3 @@ export type BatchableDataType =
   | Iterable<ArrayBuffer>
   | AsyncIterable<ArrayBuffer>
   | Promise<AsyncIterable<ArrayBuffer>>;
-
-/**
- * A FileSystem interface can encapsulate a FileList, a ZipFile, a GoogleDrive etc.
- */
-export interface IFileSystem {
-  /**
-   * Return a list of file names
-   * @param dirname directory name. file system root directory if omitted
-   */
-  readdir(dirname?: string, options?: {recursive?: boolean}): Promise<string[]>;
-
-  /**
-   * Gets information from a local file from the filesystem
-   * @param filename file name to stat
-   * @param options currently unused
-   * @throws if filename is not in local filesystem
-   */
-  stat(filename: string, options?: object): Promise<{size: number}>;
-
-  /**
-   * Fetches a local file from the filesystem (or a URL)
-   * @param filename
-   * @param options
-   */
-  fetch(filename: string, options?: object): Promise<Response>;
-}
-
-type ReadOptions = {buffer?: ArrayBuffer; offset?: number; length?: number; position?: number};
-export interface IRandomAccessReadFileSystem extends IFileSystem {
-  open(path: string, flags: string | number, mode?: any): Promise<any>;
-  close(fd: any): Promise<void>;
-  fstat(fd: any): Promise<object>;
-  read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Buffer}>;
-}