@loaders.gl/loader-utils 4.0.0-alpha.21 → 4.0.0-alpha.23

package/src/index.ts

@@ -1,23 +1,12 @@
+// loaders.gl, MIT license
+
 // TYPES
+
 export type {
-  // loaders
-  Loader,
-  LoaderWithParser,
-  LoaderContext,
-  LoaderOptions,
-  LoaderOptionsType,
-  LoaderReturnType,
-  LoaderBatchType,
-  // writers
-  Writer,
-  WriterOptions,
-  WriterOptionsType,
   // misc
   DataType,
   SyncDataType,
   BatchableDataType,
-  IFileSystem,
-  IRandomAccessReadFileSystem,
   // numeric array types
   TypedArray,
   BigTypedArray,
@@ -29,6 +18,24 @@ export type {
   FetchLike
 } from './types';
 
+// loaders
+
+export type {
+  Loader,
+  LoaderWithParser,
+  LoaderContext,
+  LoaderOptions,
+  LoaderOptionsType,
+  LoaderReturnType,
+  LoaderBatchType
+} from './loader-types';
+
+export {parseFromContext, parseSyncFromContext, parseInBatchesFromContext} from './loader-types';
+
+// writers
+
+export type {Writer, WriterOptions, WriterOptionsType} from './writer-types';
+
 // GENERAL UTILS
 export {assert} from './lib/env-utils/assert';
 export {
@@ -112,10 +119,18 @@ import * as stream from './lib/node/stream';
 export {stream};
 
 // EXPERIMENTAL
+export type {FileSystem, RandomAccessReadFileSystem} from './lib/filesystems/filesystem';
+export {NodeFileSystem as _NodeFileSystem} from './lib/filesystems/node-filesystem';
+
+export type {FileProvider} from './lib/file-provider/file-provider';
+export {isFileProvider} from './lib/file-provider/file-provider';
+
+export {FileHandle} from './lib/file-provider/file-handle';
+export {FileHandleFile} from './lib/file-provider/file-handle-file';
+export {DataViewFile} from './lib/file-provider/data-view-file';
+
 export type {ReadableFile} from './lib/filesystems/readable-file';
 export {makeReadableFile} from './lib/filesystems/readable-file';
 
 export type {WritableFile} from './lib/filesystems/writable-file';
 export {makeWritableFile} from './lib/filesystems/writable-file';
-
-export {default as _NodeFileSystem} from './lib/filesystems/node-filesystem';

package/src/json-loader.ts

@@ -1,4 +1,4 @@
-import type {LoaderWithParser, LoaderOptions} from './types';
+import type {LoaderWithParser, LoaderOptions} from './loader-types';
 import type {Table, TableBatch} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline

package/src/lib/file-provider/data-view-file.ts

@@ -0,0 +1,72 @@
+import {FileProvider} from './file-provider';
+
+/**
+ * Checks if bigint can be converted to number and convert it if possible
+ * @param bigint bigint to be converted
+ * @returns number
+ */
+const toNumber = (bigint: bigint) => {
+  if (bigint > Number.MAX_SAFE_INTEGER) {
+    throw new Error('Offset is out of bounds');
+  }
+  return Number(bigint);
+};
+
+/** Provides file data using DataView */
+export class DataViewFile implements FileProvider {
+  /** The DataView from which data is provided */
+  private file: DataView;
+
+  constructor(file: DataView) {
+    this.file = file;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  async destroy(): Promise<void> {}
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: bigint): Promise<number> {
+    return this.file.getUint8(toNumber(offset));
+  }
+
+  /**
+   * Gets an unsigned 16-bit intege at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: bigint): Promise<number> {
+    return this.file.getUint16(toNumber(offset), true);
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: bigint): Promise<number> {
+    return this.file.getUint32(toNumber(offset), true);
+  }
+
+  /**
+   * Gets an unsigned 64-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: bigint): Promise<bigint> {
+    return this.file.getBigUint64(toNumber(offset), true);
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    return this.file.buffer.slice(toNumber(startOffset), toNumber(endOffset));
+  }
+
+  /** the length (in bytes) of the data. */
+  get length() {
+    return BigInt(this.file.byteLength);
+  }
+}

package/src/lib/file-provider/file-handle-file.ts

@@ -0,0 +1,116 @@
+import {FileProvider} from './file-provider';
+import {FileHandle} from './file-handle';
+import {resolvePath} from '../path-utils/file-aliases';
+
+/**
+ * Provides file data using node fs library
+ */
+export class FileHandleFile implements FileProvider {
+  /**
+   * Returns a new copy of FileHandleFile
+   * @param path The path to the file in file system
+   */
+  static async from(path: string): Promise<FileHandleFile> {
+    path = resolvePath(path);
+    const fileDescriptor = await FileHandle.open(path);
+    return new FileHandleFile(fileDescriptor, fileDescriptor.stat.size);
+  }
+
+  /**
+   * The FileHandle from which data is provided
+   */
+  private fileDescriptor: FileHandle;
+
+  /**
+   * The file length in bytes
+   */
+  private size: bigint;
+
+  private constructor(fileDescriptor: FileHandle, size: bigint) {
+    this.fileDescriptor = fileDescriptor;
+    this.size = size;
+  }
+
+  /** Close file */
+  async destroy(): Promise<void> {
+    await this.fileDescriptor.close();
+  }
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: bigint): Promise<number> {
+    const val = new Uint8Array(
+      (await this.fileDescriptor.read(Buffer.alloc(1), 0, 1, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: bigint): Promise<number> {
+    const val = new Uint16Array(
+      (await this.fileDescriptor.read(Buffer.alloc(2), 0, 2, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: bigint): Promise<number> {
+    const val = new Uint32Array(
+      (await this.fileDescriptor.read(Buffer.alloc(4), 0, 4, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: bigint): Promise<bigint> {
+    const val = new BigInt64Array(
+      (await this.fileDescriptor.read(Buffer.alloc(8), 0, 8, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffsset The offset, in byte, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffsset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    const bigLength = endOffset - startOffsset;
+    if (bigLength > Number.MAX_SAFE_INTEGER) {
+      throw new Error('too big slice');
+    }
+    const length = Number(bigLength);
+    return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffsset)).buffer
+      .buffer;
+  }
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  get length(): bigint {
+    return this.size;
+  }
+}

package/src/lib/file-provider/file-handle.ts

@@ -0,0 +1,74 @@
+import {read, open, close, stat, BigIntStats} from 'fs';
+
+/** file reading result */
+export type FileReadResult = {
+  /** amount of the bytes read */
+  bytesRead: number;
+  /** the buffer filled with data from file*/
+  buffer: Buffer;
+};
+
+/** Object handling file info */
+export class FileHandle {
+  private fileDescriptor: number;
+  private stats: BigIntStats;
+
+  private constructor(fileDescriptor: number, stats: BigIntStats) {
+    this.fileDescriptor = fileDescriptor;
+    this.stats = stats;
+  }
+  /**
+   * Opens a `FileHandle`.
+   *
+   * @param path path to the file
+   * @return Fulfills with a {FileHandle} object.
+   */
+
+  static async open(path: string): Promise<FileHandle> {
+    const [fd, stats] = await Promise.all([
+      new Promise<number>((resolve, reject) => {
+        open(path, undefined, undefined, (_err, fd) => (_err ? reject(_err) : resolve(fd)));
+      }),
+      new Promise<BigIntStats>((resolve, reject) => {
+        stat(path, {bigint: true}, (_err, stats) => (_err ? reject(_err) : resolve(stats)));
+      })
+    ]);
+    return new FileHandle(fd, stats);
+  }
+
+  /** Close file */
+  async close(): Promise<void> {
+    return new Promise<void>((resolve) => {
+      close(this.fileDescriptor, (_err) => resolve());
+    });
+  }
+
+  /**
+   * Reads data from the file and stores that in the given buffer.
+   *
+   * If the file is not modified concurrently, the end-of-file is reached when the
+   * number of bytes read is zero.
+   * @param buffer A buffer that will be filled with the file data read.
+   * @param offset The location in the buffer at which to start filling.
+   * @param length The number of bytes to read.
+   * @param position The location where to begin reading data from the file. If `null`, data will be read from the current file position, and the position will be updated. If `position` is an
+   * integer, the current file position will remain unchanged.
+   * @return Fulfills upon success with a FileReadResult object
+   */
+  read = (
+    buffer: Buffer,
+    offset: number,
+    length: number,
+    position: number | bigint
+  ): Promise<FileReadResult> => {
+    return new Promise((s) => {
+      read(this.fileDescriptor, buffer, offset, length, position, (_err, bytesRead, buffer) =>
+        s({bytesRead, buffer})
+      );
+    });
+  };
+
+  get stat(): BigIntStats {
+    return this.stats;
+  }
+}

package/src/lib/file-provider/file-provider.ts

@@ -0,0 +1,56 @@
+/**
+ * Interface for providing file data
+ */
+export interface FileProvider {
+  /**
+   * Cleanup class data
+   */
+  destroy(): Promise<void>;
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint8(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint16(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the file of the view where to read the data.
+   */
+  getUint32(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in byte, from the file of the view where to read the data.
+   */
+  getBigUint64(offset: bigint): Promise<bigint>;
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer>;
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  length: bigint;
+}
+
+/**
+ * Check is the object has FileProvider members
+ * @param fileProvider - tested object
+ */
+export const isFileProvider = (fileProvider: unknown) => {
+  return (
+    (fileProvider as FileProvider)?.getUint8 &&
+    (fileProvider as FileProvider)?.slice &&
+    (fileProvider as FileProvider)?.length
+  );
+};

package/src/lib/filesystems/filesystem.ts

@@ -0,0 +1,87 @@
+// loaders.gl, MIT license
+
+export type ReadOptions = {};
+
+export type Stat = {
+  size: number;
+  isDirectory: () => boolean;
+};
+
+/**
+ * A FileSystem interface can encapsulate various file sources,
+ * a FileList, a ZipFile, a GoogleDrive etc.
+ */
+export interface FileSystem {
+  /**
+   * Return a list of file names
+   * @param dirname directory name. file system root directory if omitted
+   */
+  readdir(dirname?: string, options?: {recursive?: boolean}): Promise<string[]>;
+
+  /**
+   * Gets information from a local file from the filesystem
+   * @param filename file name to stat
+   * @param options currently unused
+   * @throws if filename is not in local filesystem
+   */
+  stat(filename: string, options?: object): Promise<{size: number}>;
+
+  /**
+   * Fetches a local file from the filesystem (or a URL)
+   * @param filename
+   * @param options
+   */
+  fetch(filename: RequestInfo, options?: RequestInit): Promise<Response>;
+}
+
+/**
+ * A random access file system
+ */
+export interface RandomAccessReadFileSystem extends FileSystem {
+  open(path: string, flags: unknown, mode?: unknown): Promise<any>;
+  close(fd: unknown): Promise<void>;
+  fstat(fd: unknown): Promise<Stat>;
+  read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Buffer}>;
+  // read(
+  //   fd: any,
+  //   buffer: ArrayBuffer | ArrayBufferView,
+  //   offset?: number,
+  //   length?: number,
+  //   position?: number
+  // ): Promise<{bytesRead: number; buffer: ArrayBuffer}>;
+}
+
+/**
+ * A FileSystem interface can encapsulate a FileList, a ZipFile, a GoogleDrive etc.
+ *
+export interface IFileSystem {
+  /**
+   * Return a list of file names
+   * @param dirname directory name. file system root directory if omitted
+   *
+  readdir(dirname?: string, options?: {recursive?: boolean}): Promise<string[]>;
+
+  /**
+   * Gets information from a local file from the filesystem
+   * @param filename file name to stat
+   * @param options currently unused
+   * @throws if filename is not in local filesystem
+   *
+  stat(filename: string, options?: object): Promise<{size: number}>;
+
+  /**
+   * Fetches a local file from the filesystem (or a URL)
+   * @param filename
+   * @param options
+   *
+  fetch(filename: string, options?: object): Promise<Response>;
+}
+
+type ReadOptions = {buffer?: ArrayBuffer; offset?: number; length?: number; position?: number};
+export interface IRandomAccessReadFileSystem extends IFileSystem {
+  open(path: string, flags: string | number, mode?: any): Promise<any>;
+  close(fd: any): Promise<void>;
+  fstat(fd: any): Promise<object>;
+  read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Buffer}>;
+}
+*/

package/src/lib/filesystems/node-filesystem.ts

@@ -1,5 +1,5 @@
 import * as fs from '../node/fs';
-import {IFileSystem, IRandomAccessReadFileSystem} from '../../types';
+import {FileSystem, RandomAccessReadFileSystem} from './filesystem';
 // import {fetchFile} from "../fetch/fetch-file"
 // import {selectLoader} from "../api/select-loader";
 
@@ -21,8 +21,8 @@ type ReadOptions = {
  * Compatible with BrowserFileSystem.
  * @param options
  */
-export default class NodeFileSystem implements IFileSystem, IRandomAccessReadFileSystem {
-  // implements IFileSystem
+export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem {
+  // implements FileSystem
   constructor(options: {[key: string]: any}) {
     this.fetch = options._fetch;
   }

package/src/lib/option-utils/merge-loader-options.ts

@@ -1,6 +1,6 @@
 // loaders.gl, MIT license
 
-import {LoaderOptions} from '../../types';
+import {LoaderOptions} from '../../loader-types';
 
 /**
  *

package/src/lib/worker-loader-utils/create-loader-worker.ts

@@ -1,5 +1,5 @@
 /* eslint-disable no-restricted-globals */
-import type {LoaderWithParser} from '../../types';
+import type {LoaderWithParser, LoaderOptions, LoaderContext} from '../../loader-types';
 import {WorkerBody} from '@loaders.gl/worker-utils';
 // import {validateLoaderVersion} from './validate-loader-version';
 
@@ -27,9 +27,10 @@ export function createLoaderWorker(loader: LoaderWithParser) {
             loader,
             arrayBuffer: input,
             options,
+            // @ts-expect-error fetch missing
             context: {
               ...context,
-              parse: parseOnMainThread
+              _parse: parseOnMainThread
             }
           });
           WorkerBody.postMessage('done', {result});
@@ -43,7 +44,12 @@ export function createLoaderWorker(loader: LoaderWithParser) {
   };
 }
 
-function parseOnMainThread(arrayBuffer: ArrayBuffer, options: {[key: string]: any}): Promise<void> {
+function parseOnMainThread(
+  arrayBuffer: ArrayBuffer,
+  loader: any,
+  options?: LoaderOptions,
+  context?: LoaderContext
+): Promise<void> {
   return new Promise((resolve, reject) => {
     const id = requestId++;
 
@@ -83,7 +89,17 @@ function parseOnMainThread(arrayBuffer: ArrayBuffer, options: {[key: string]: an
 // TODO - Why not support async loader.parse* funcs here?
 // TODO - Why not reuse a common function instead of reimplementing loader.parse* selection logic? Keeping loader small?
 // TODO - Lack of appropriate parser functions can be detected when we create worker, no need to wait until parse
-async function parseData({loader, arrayBuffer, options, context}) {
+async function parseData({
+  loader,
+  arrayBuffer,
+  options,
+  context
+}: {
+  loader: LoaderWithParser;
+  arrayBuffer: ArrayBuffer;
+  options: LoaderOptions;
+  context: LoaderContext;
+}) {
   let data;
   let parser;
   if (loader.parseSync || loader.parse) {

package/src/lib/worker-loader-utils/encode-with-worker.ts

@@ -1,5 +1,5 @@
 import {WorkerFarm} from '@loaders.gl/worker-utils';
-import {Writer, WriterOptions} from '../../types';
+import {Writer, WriterOptions} from '../../writer-types';
 import {isBrowser} from '../env-utils/globals';
 
 /**

package/src/lib/worker-loader-utils/parse-with-worker.ts

@@ -4,7 +4,7 @@ import {
   WorkerMessagePayload,
   isBrowser
 } from '@loaders.gl/worker-utils';
-import type {Loader, LoaderOptions, LoaderContext} from '../../types';
+import type {Loader, LoaderOptions, LoaderContext} from '../../loader-types';
 import {WorkerFarm, getWorkerURL} from '@loaders.gl/worker-utils';
 
 /**