@loaders.gl/loader-utils 4.0.0-alpha.9 → 4.0.0-beta.1

package/src/index.ts

@@ -1,32 +1,41 @@
+// loaders.gl, MIT license
+
 // TYPES
+
 export type {
-  // loaders
-  Loader,
-  LoaderWithParser,
-  LoaderContext,
-  LoaderOptions,
-  LoaderOptionsType,
-  LoaderReturnType,
-  LoaderBatchType,
-  // writers
-  Writer,
-  WriterOptions,
-  WriterOptionsType,
   // misc
   DataType,
   SyncDataType,
   BatchableDataType,
-  IFileSystem,
-  IRandomAccessReadFileSystem,
   // numeric array types
   TypedArray,
   BigTypedArray,
   TypedArrayConstructor,
   BigTypedArrayConstructor,
   NumberArray,
-  NumericArray
+  NumericArray,
+  // fetch
+  FetchLike
 } from './types';
 
+// loaders
+
+export type {
+  Loader,
+  LoaderWithParser,
+  LoaderContext,
+  LoaderOptions,
+  LoaderOptionsType,
+  LoaderReturnType,
+  LoaderBatchType
+} from './loader-types';
+
+export {parseFromContext, parseSyncFromContext, parseInBatchesFromContext} from './loader-types';
+
+// writers
+
+export type {Writer, WriterOptions, WriterOptionsType} from './writer-types';
+
 // GENERAL UTILS
 export {assert} from './lib/env-utils/assert';
 export {
@@ -109,11 +118,42 @@ export {fs};
 import * as stream from './lib/node/stream';
 export {stream};
 
-// EXPERIMENTAL
+// EXPERIMENTAL: FILE SYSTEMS
+
+export type {FileSystem, RandomAccessReadFileSystem} from './lib/filesystems/filesystem';
+export {NodeFileSystem as _NodeFileSystem} from './lib/filesystems/node-filesystem';
+
+export type {FileProvider} from './lib/file-provider/file-provider';
+export {isFileProvider} from './lib/file-provider/file-provider';
+
+export {FileHandle} from './lib/file-provider/file-handle';
+export {FileHandleFile} from './lib/file-provider/file-handle-file';
+export {DataViewFile} from './lib/file-provider/data-view-file';
+
 export type {ReadableFile} from './lib/filesystems/readable-file';
 export {makeReadableFile} from './lib/filesystems/readable-file';
 
 export type {WritableFile} from './lib/filesystems/writable-file';
 export {makeWritableFile} from './lib/filesystems/writable-file';
 
-export {default as _NodeFileSystem} from './lib/filesystems/node-filesystem';
+// EXPERIMENTAL: DATA SOURCES
+export type {Service} from './service-types';
+
+export type {DataSourceProps} from './lib/sources/data-source';
+export {DataSource} from './lib/sources/data-source';
+
+export type {ImageType} from './lib/sources/utils/image-type';
+export type {ImageSourceProps, ImageSourceMetadata} from './lib/sources/image-source';
+export type {GetImageParameters} from './lib/sources/image-source';
+export {ImageSource} from './lib/sources/image-source';
+
+export type {
+  TileSourceProps,
+  TileSourceMetadata,
+  GetTileParameters
+} from './lib/sources/tile-source';
+export type {TileSource} from './lib/sources/tile-source';
+
+export type {ImageTileSource} from './lib/sources/image-tile-source';
+
+export type {VectorTileSource} from './lib/sources/vector-tile-source';

package/src/json-loader.ts

@@ -1,4 +1,4 @@
-import type {LoaderWithParser, LoaderOptions} from './types';
+import type {LoaderWithParser, LoaderOptions} from './loader-types';
 import type {Table, TableBatch} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline

package/src/lib/file-provider/data-view-file.ts

@@ -0,0 +1,72 @@
+import {FileProvider} from './file-provider';
+
+/**
+ * Checks if bigint can be converted to number and convert it if possible
+ * @param bigint bigint to be converted
+ * @returns number
+ */
+const toNumber = (bigint: bigint) => {
+  if (bigint > Number.MAX_SAFE_INTEGER) {
+    throw new Error('Offset is out of bounds');
+  }
+  return Number(bigint);
+};
+
+/** Provides file data using DataView */
+export class DataViewFile implements FileProvider {
+  /** The DataView from which data is provided */
+  private file: DataView;
+
+  constructor(file: DataView) {
+    this.file = file;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  async destroy(): Promise<void> {}
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: bigint): Promise<number> {
+    return this.file.getUint8(toNumber(offset));
+  }
+
+  /**
+   * Gets an unsigned 16-bit intege at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: bigint): Promise<number> {
+    return this.file.getUint16(toNumber(offset), true);
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: bigint): Promise<number> {
+    return this.file.getUint32(toNumber(offset), true);
+  }
+
+  /**
+   * Gets an unsigned 64-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: bigint): Promise<bigint> {
+    return this.file.getBigUint64(toNumber(offset), true);
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    return this.file.buffer.slice(toNumber(startOffset), toNumber(endOffset));
+  }
+
+  /** the length (in bytes) of the data. */
+  get length() {
+    return BigInt(this.file.byteLength);
+  }
+}

package/src/lib/file-provider/file-handle-file.ts

@@ -0,0 +1,116 @@
+import {FileProvider} from './file-provider';
+import {FileHandle} from './file-handle';
+import {resolvePath} from '../path-utils/file-aliases';
+
+/**
+ * Provides file data using node fs library
+ */
+export class FileHandleFile implements FileProvider {
+  /**
+   * Returns a new copy of FileHandleFile
+   * @param path The path to the file in file system
+   */
+  static async from(path: string): Promise<FileHandleFile> {
+    path = resolvePath(path);
+    const fileDescriptor = await FileHandle.open(path);
+    return new FileHandleFile(fileDescriptor, fileDescriptor.stat.size);
+  }
+
+  /**
+   * The FileHandle from which data is provided
+   */
+  private fileDescriptor: FileHandle;
+
+  /**
+   * The file length in bytes
+   */
+  private size: bigint;
+
+  private constructor(fileDescriptor: FileHandle, size: bigint) {
+    this.fileDescriptor = fileDescriptor;
+    this.size = size;
+  }
+
+  /** Close file */
+  async destroy(): Promise<void> {
+    await this.fileDescriptor.close();
+  }
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint8(offset: bigint): Promise<number> {
+    const val = new Uint8Array(
+      (await this.fileDescriptor.read(Buffer.alloc(1), 0, 1, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint16(offset: bigint): Promise<number> {
+    const val = new Uint16Array(
+      (await this.fileDescriptor.read(Buffer.alloc(2), 0, 2, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getUint32(offset: bigint): Promise<number> {
+    const val = new Uint32Array(
+      (await this.fileDescriptor.read(Buffer.alloc(4), 0, 4, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  async getBigUint64(offset: bigint): Promise<bigint> {
+    const val = new BigInt64Array(
+      (await this.fileDescriptor.read(Buffer.alloc(8), 0, 8, offset)).buffer.buffer
+    ).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffsset The offset, in byte, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  async slice(startOffsset: bigint, endOffset: bigint): Promise<ArrayBuffer> {
+    const bigLength = endOffset - startOffsset;
+    if (bigLength > Number.MAX_SAFE_INTEGER) {
+      throw new Error('too big slice');
+    }
+    const length = Number(bigLength);
+    return (await this.fileDescriptor.read(Buffer.alloc(length), 0, length, startOffsset)).buffer
+      .buffer;
+  }
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  get length(): bigint {
+    return this.size;
+  }
+}

package/src/lib/file-provider/file-handle.ts

@@ -0,0 +1,79 @@
+import * as fs from '../node/fs';
+
+/** file reading result */
+export type FileReadResult = {
+  /** amount of the bytes read */
+  bytesRead: number;
+  /** the buffer filled with data from file*/
+  buffer: Buffer;
+};
+
+/** Object handling file info */
+export class FileHandle {
+  private fileDescriptor: number;
+  private stats: fs.BigIntStats;
+
+  private constructor(fileDescriptor: number, stats: fs.BigIntStats) {
+    this.fileDescriptor = fileDescriptor;
+    this.stats = stats;
+  }
+  /**
+   * Opens a `FileHandle`.
+   *
+   * @param path path to the file
+   * @return Fulfills with a {FileHandle} object.
+   */
+
+  static async open(path: string): Promise<FileHandle> {
+    const [fd, stats] = await Promise.all([
+      new Promise<number>((resolve, reject) => {
+        fs.open(path, undefined, undefined, (_err, fd) => (_err ? reject(_err) : resolve(fd)));
+      }),
+      fs.stat(path, {bigint: true})
+      // new Promise<fs.BigIntStats>((resolve, reject) => {
+      //   console.error(fs.stat)
+      //   fs.stat(path, {bigint: true}, (_err, stats) => (_err ? reject(_err) : resolve(stats)));
+      //   console.error(fs.open)
+      // })
+    ]);
+    return new FileHandle(fd, stats);
+  }
+
+  /** Close file */
+  async close(): Promise<void> {
+    return fs.close(this.fileDescriptor);
+    // return new Promise<void>((resolve) => {
+    //   // @ts-expect-error
+    //   fs.close(this.fileDescriptor, (_err) => resolve());
+    // });
+  }
+
+  /**
+   * Reads data from the file and stores that in the given buffer.
+   *
+   * If the file is not modified concurrently, the end-of-file is reached when the
+   * number of bytes read is zero.
+   * @param buffer A buffer that will be filled with the file data read.
+   * @param offset The location in the buffer at which to start filling.
+   * @param length The number of bytes to read.
+   * @param position The location where to begin reading data from the file. If `null`, data will be read from the current file position, and the position will be updated. If `position` is an
+   * integer, the current file position will remain unchanged.
+   * @return Fulfills upon success with a FileReadResult object
+   */
+  read = (
+    buffer: Buffer,
+    offset: number,
+    length: number,
+    position: number | bigint
+  ): Promise<FileReadResult> => {
+    return new Promise((s) => {
+      fs.read(this.fileDescriptor, buffer, offset, length, position, (_err, bytesRead, buffer) =>
+        s({bytesRead, buffer})
+      );
+    });
+  };
+
+  get stat(): fs.BigIntStats {
+    return this.stats;
+  }
+}

package/src/lib/file-provider/file-provider.ts

@@ -0,0 +1,56 @@
+/**
+ * Interface for providing file data
+ */
+export interface FileProvider {
+  /**
+   * Cleanup class data
+   */
+  destroy(): Promise<void>;
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint8(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint16(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the file of the view where to read the data.
+   */
+  getUint32(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in byte, from the file of the view where to read the data.
+   */
+  getBigUint64(offset: bigint): Promise<bigint>;
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer>;
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  length: bigint;
+}
+
+/**
+ * Check is the object has FileProvider members
+ * @param fileProvider - tested object
+ */
+export const isFileProvider = (fileProvider: unknown) => {
+  return (
+    (fileProvider as FileProvider)?.getUint8 &&
+    (fileProvider as FileProvider)?.slice &&
+    (fileProvider as FileProvider)?.length
+  );
+};

package/src/lib/filesystems/filesystem.ts

@@ -0,0 +1,87 @@
+// loaders.gl, MIT license
+
+export type ReadOptions = {};
+
+export type Stat = {
+  size: number;
+  isDirectory: () => boolean;
+};
+
+/**
+ * A FileSystem interface can encapsulate various file sources,
+ * a FileList, a ZipFile, a GoogleDrive etc.
+ */
+export interface FileSystem {
+  /**
+   * Return a list of file names
+   * @param dirname directory name. file system root directory if omitted
+   */
+  readdir(dirname?: string, options?: {recursive?: boolean}): Promise<string[]>;
+
+  /**
+   * Gets information from a local file from the filesystem
+   * @param filename file name to stat
+   * @param options currently unused
+   * @throws if filename is not in local filesystem
+   */
+  stat(filename: string, options?: object): Promise<{size: number}>;
+
+  /**
+   * Fetches a local file from the filesystem (or a URL)
+   * @param filename
+   * @param options
+   */
+  fetch(filename: RequestInfo, options?: RequestInit): Promise<Response>;
+}
+
+/**
+ * A random access file system
+ */
+export interface RandomAccessReadFileSystem extends FileSystem {
+  open(path: string, flags: unknown, mode?: unknown): Promise<any>;
+  close(fd: unknown): Promise<void>;
+  fstat(fd: unknown): Promise<Stat>;
+  read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Uint8Array}>;
+  // read(
+  //   fd: any,
+  //   buffer: ArrayBuffer | ArrayBufferView,
+  //   offset?: number,
+  //   length?: number,
+  //   position?: number
+  // ): Promise<{bytesRead: number; buffer: ArrayBuffer}>;
+}
+
+/**
+ * A FileSystem interface can encapsulate a FileList, a ZipFile, a GoogleDrive etc.
+ *
+export interface IFileSystem {
+  /**
+   * Return a list of file names
+   * @param dirname directory name. file system root directory if omitted
+   *
+  readdir(dirname?: string, options?: {recursive?: boolean}): Promise<string[]>;
+
+  /**
+   * Gets information from a local file from the filesystem
+   * @param filename file name to stat
+   * @param options currently unused
+   * @throws if filename is not in local filesystem
+   *
+  stat(filename: string, options?: object): Promise<{size: number}>;
+
+  /**
+   * Fetches a local file from the filesystem (or a URL)
+   * @param filename
+   * @param options
+   *
+  fetch(filename: string, options?: object): Promise<Response>;
+}
+
+type ReadOptions = {buffer?: ArrayBuffer; offset?: number; length?: number; position?: number};
+export interface IRandomAccessReadFileSystem extends IFileSystem {
+  open(path: string, flags: string | number, mode?: any): Promise<any>;
+  close(fd: any): Promise<void>;
+  fstat(fd: any): Promise<object>;
+  read(fd: any, options?: ReadOptions): Promise<{bytesRead: number; buffer: Buffer}>;
+}
+*/

package/src/lib/filesystems/node-filesystem.browser.ts

@@ -0,0 +1,61 @@
+import * as fs from '../node/fs';
+import {FileSystem, RandomAccessReadFileSystem} from './filesystem';
+// import {fetchFile} from "../fetch/fetch-file"
+// import {selectLoader} from "../api/select-loader";
+
+type Stat = {
+  size: number;
+  isDirectory: () => boolean;
+  info?: fs.Stats;
+};
+
+type ReadOptions = {
+  buffer?: Buffer;
+  offset?: number;
+  length?: number;
+  position?: number;
+};
+
+/**
+ * FileSystem pass-through for Node.js
+ * Compatible with BrowserFileSystem.
+ * @param options
+ */
+export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem {
+  // implements FileSystem
+  constructor(options: {[key: string]: any}) {
+    throw new Error('Can\'t instantiate NodeFileSystem in browser');
+  }
+
+  async readdir(dirname = '.', options?: {}): Promise<any[]> {
+    return [];
+  }
+
+  async stat(path: string, options?: {}): Promise<Stat> {
+    return {size: 0, isDirectory: () => false};
+  }
+
+  async fetch(path: string, options: {[key: string]: any}) {
+    return globalThis.fetch(path, options);
+  }
+
+  // implements IRandomAccessFileSystem
+  async open(path: string, flags: string | number, mode?: any): Promise<number> {
+    return 0;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  async close(fd: number): Promise<void> {}
+
+  async fstat(fd: number): Promise<Stat> {
+    return {size: 0, isDirectory: () => false};
+  }
+
+  async read(
+    fd: number,
+    // @ts-ignore Possibly null
+    {buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions
+  ): Promise<{bytesRead: number; buffer: Uint8Array}> {
+    return {bytesRead: 0, buffer: new Uint8Array(0)};
+  }
+}

package/src/lib/filesystems/node-filesystem.ts

@@ -1,5 +1,5 @@
 import * as fs from '../node/fs';
-import {IFileSystem, IRandomAccessReadFileSystem} from '../../types';
+import {FileSystem, RandomAccessReadFileSystem} from './filesystem';
 // import {fetchFile} from "../fetch/fetch-file"
 // import {selectLoader} from "../api/select-loader";
 
@@ -21,8 +21,8 @@ type ReadOptions = {
  * Compatible with BrowserFileSystem.
  * @param options
  */
-export default class NodeFileSystem implements IFileSystem, IRandomAccessReadFileSystem {
-  // implements IFileSystem
+export class NodeFileSystem implements FileSystem, RandomAccessReadFileSystem {
+  // implements FileSystem
   constructor(options: {[key: string]: any}) {
     this.fetch = options._fetch;
   }
@@ -61,7 +61,7 @@ export default class NodeFileSystem implements IFileSystem, IRandomAccessReadFil
     fd: number,
     // @ts-ignore Possibly null
     {buffer = null, offset = 0, length = buffer.byteLength, position = null}: ReadOptions
-  ): Promise<{bytesRead: number; buffer: Buffer}> {
+  ): Promise<{bytesRead: number; buffer: Uint8Array}> {
     let totalBytesRead = 0;
     // Read in loop until we get required number of bytes
     while (totalBytesRead < length) {

package/src/lib/node/fs.browser.ts

@@ -0,0 +1 @@
+export const isSupported = false;