@loaders.gl/loader-utils 4.0.0-alpha.9 → 4.0.0-beta.1

package/src/lib/node/fs.ts

@@ -1,73 +1,43 @@
 // fs wrapper (promisified fs + avoids bundling fs in browsers)
-import fs from 'fs';
+import * as fs from 'fs';
+import * as fsPromises from 'fs/promises';
 import {toArrayBuffer} from './buffer';
 import {promisify2, promisify3} from './promisify';
 
-export type {Stats, WriteStream} from 'fs';
+export type {BigIntStats, Stats} from 'fs';
+export type {ReadStream, WriteStream} from 'fs';
 
-export let readdir;
 /** Wrapper for Node.js fs method */
-export let stat;
+export const readdir: any = promisify2(fs.readdir);
+/** Wrapper for Node.js fs method */
+export const stat: any = fsPromises.stat;
+export const statSync = fs.statSync;
 
 /** Wrapper for Node.js fs method */
-export let readFile;
+export const readFile: any = fs.readFile;
 /** Wrapper for Node.js fs method */
-export let readFileSync;
+export const readFileSync = fs.readFileSync;
 /** Wrapper for Node.js fs method */
-export let writeFile;
+export const writeFile: any = promisify3(fs.writeFile);
 /** Wrapper for Node.js fs method */
-export let writeFileSync;
+export const writeFileSync = fs.writeFileSync;
 
 // file descriptors
 
 /** Wrapper for Node.js fs method */
-export let open;
+export const open: any = fs.open;
 /** Wrapper for Node.js fs method */
-export let close: (fd: number) => Promise<void>;
+export const close = (fd: number) =>
+  new Promise<void>((resolve, reject) => fs.close(fd, (err) => (err ? reject(err) : resolve())));
 /** Wrapper for Node.js fs method */
-export let read;
+export const read: any = fs.read;
 /** Wrapper for Node.js fs method */
-export let fstat;
-
-export let createWriteStream: typeof fs.createWriteStream;
-
-export let isSupported = Boolean(fs);
-
-// paths
-
-try {
-  /** Wrapper for Node.js fs method */
-  readdir = promisify2(fs.readdir);
-  /** Wrapper for Node.js fs method */
-  stat = promisify2(fs.stat);
+export const fstat: any = fs.fstat;
 
-  /** Wrapper for Node.js fs method */
-  readFile = fs.readFile;
-  /** Wrapper for Node.js fs method */
-  readFileSync = fs.readFileSync;
-  /** Wrapper for Node.js fs method */
-  writeFile = promisify3(fs.writeFile);
-  /** Wrapper for Node.js fs method */
-  writeFileSync = fs.writeFileSync;
+export const createReadStream = fs.createReadStream;
+export const createWriteStream = fs.createWriteStream;
 
-  // file descriptors
-
-  /** Wrapper for Node.js fs method */
-  open = fs.open;
-  /** Wrapper for Node.js fs method */
-  close = (fd: number) =>
-    new Promise((resolve, reject) => fs.close(fd, (err) => (err ? reject(err) : resolve())));
-  /** Wrapper for Node.js fs method */
-  read = fs.read;
-  /** Wrapper for Node.js fs method */
-  fstat = fs.fstat;
-
-  createWriteStream = fs.createWriteStream;
-
-  isSupported = Boolean(fs);
-} catch {
-  // ignore
-}
+export const isSupported = Boolean(fs);
 
 export async function _readToArrayBuffer(fd: number, start: number, length: number) {
   const buffer = Buffer.alloc(length);

package/src/lib/node/stream.browser.ts

@@ -0,0 +1 @@
+export const isSupported = false;

package/src/lib/node/stream.ts

@@ -1,18 +1,10 @@
 // loaders.gl, MIT license
 
-import stream from 'stream';
+import * as stream from 'stream';
 
 export type {Writable} from 'stream';
 
-export let Transform;
+/** Wrapper for Node.js stream method */
+export const Transform = stream.Transform;
 
 export const isSupported = Boolean(stream);
-
-// paths
-
-try {
-  /** Wrapper for Node.js fs method */
-  Transform = stream.Transform;
-} catch {
-  // ignore
-}

package/src/lib/option-utils/merge-loader-options.ts

@@ -1,6 +1,6 @@
 // loaders.gl, MIT license
 
-import {LoaderOptions} from '../../types';
+import {LoaderOptions} from '../../loader-types';
 
 /**
  *
@@ -21,7 +21,7 @@ function mergeOptionsRecursively(
 ): Record<string, unknown> {
   const options = {...baseOptions};
   for (const [key, newValue] of Object.entries(newOptions)) {
-    if (newValue && typeof newValue === 'object') {
+    if (newValue && typeof newValue === 'object' && !Array.isArray(newValue)) {
       options[key] = mergeOptionsRecursively(
         (options[key] as Record<string, unknown>) || {},
         newOptions[key] as Record<string, unknown>

package/src/lib/sources/data-source.ts

@@ -0,0 +1,74 @@
+// loaders.gl, MIT license
+
+import type {LoaderOptions} from '@loaders.gl/loader-utils';
+
+/** Common properties for all data sources */
+export type DataSourceProps = {
+  /** LoaderOptions provide an option to override `fetch`. Will also be passed to any sub loaders */
+  loadOptions?: LoaderOptions;
+};
+
+/** base class of all data sources */
+export abstract class DataSource<PropsT extends DataSourceProps = DataSourceProps> {
+  /** A resolved fetch function extracted from loadOptions prop */
+  fetch: (url: string, options?: RequestInit) => Promise<Response>;
+  /** The actual load options, if calling a loaders.gl loader */
+  loadOptions: LoaderOptions;
+  _needsRefresh: boolean = true;
+
+  props: PropsT;
+
+  constructor(props: PropsT) {
+    this.props = {...props};
+    this.loadOptions = {...props.loadOptions};
+    this.fetch = getFetchFunction(this.loadOptions);
+  }
+
+  setProps(props: PropsT) {
+    this.props = Object.assign(this.props, props);
+    // TODO - add a shallow compare to avoid setting refresh if no change?
+    this.setNeedsRefresh();
+  }
+
+  /** Mark this data source as needing a refresh (redraw) */
+  setNeedsRefresh(): void {
+    this._needsRefresh = true;
+  }
+
+  /**
+   * Does this data source need refreshing?
+   * @note The specifics of the refresh mechanism depends on type of data source
+   */
+  getNeedsRefresh(clear: boolean = true) {
+    const needsRefresh = this._needsRefresh;
+    if (clear) {
+      this._needsRefresh = false;
+    }
+    return needsRefresh;
+  }
+}
+
+/**
+ * Gets the current fetch function from options
+ * @todo - move to loader-utils module
+ * @todo - use in core module counterpart
+ * @param options
+ * @param context
+ */
+export function getFetchFunction(options?: LoaderOptions) {
+  const fetchFunction = options?.fetch;
+
+  // options.fetch can be a function
+  if (fetchFunction && typeof fetchFunction === 'function') {
+    return (url: string, fetchOptions?: RequestInit) => fetchFunction(url, fetchOptions);
+  }
+
+  // options.fetch can be an options object, use global fetch with those options
+  const fetchOptions = options?.fetch;
+  if (fetchOptions && typeof fetchOptions !== 'function') {
+    return (url) => fetch(url, fetchOptions);
+  }
+
+  // else return the global fetch function
+  return (url) => fetch(url);
+}

package/src/lib/sources/image-source.ts

@@ -0,0 +1,95 @@
+// loaders.gl, MIT license
+
+import type {DataSourceProps} from './data-source';
+import {DataSource} from './data-source';
+import {ImageType} from './utils/image-type';
+
+/**
+ * Normalized capabilities of an Image service
+ * @example
+ *  The WMSService will normalize the response to the WMS `GetCapabilities`
+ *  data structure extracted from WMS XML response into an ImageSourceMetadata.
+ */
+export type ImageSourceMetadata = {
+  name: string;
+  title?: string;
+  abstract?: string;
+  keywords: string[];
+  layers: ImageSourceLayer[];
+};
+
+/** Description of one data layer in the image source */
+export type ImageSourceLayer = {
+  /** Name of this layer */
+  name?: string;
+  /** Human readable title of this layer */
+  title?: string;
+  /** Coordinate systems supported by this layer */
+  crs?: string[];
+  /** layer limits in unspecified CRS:84-like lng/lat, for quick access w/o CRS calculations. */
+  geographicBoundingBox?: [min: [x: number, y: number], max: [x: number, y: number]];
+  /** Sub layers of this layer */
+  layers?: ImageSourceLayer[];
+
+  /** @deprecated from v3.4: non-vis.gl style bounding box. Use `.geographicBoundingBox` instead */
+  boundingBox?: [number, number, number, number];
+};
+
+/** Generic parameters for requesting an image from an image source */
+export type GetImageParameters = {
+  /** Layers to render */
+  layers: string | string[];
+  /** Styling */
+  styles?: unknown;
+  /** bounding box of the requested map image */
+  bbox: [number, number, number, number];
+  /** pixel width of returned image */
+  width: number;
+  /** pixels */
+  height: number;
+  /** crs for the image (not the bounding box) */
+  crs?: string;
+  /** requested format for the return image */
+  format?: 'image/png';
+};
+
+// Attempt to break down GetImageParameters
+export type ImageFilters = {
+  /** Layers to render */
+  layers: string | string[];
+  /** Styling */
+  styles?: unknown;
+};
+
+export type ImageRegion = {
+  /** bounding box of the requested map image */
+  bbox: [number, number, number, number];
+};
+
+export type ImageFormat = {
+  /** pixel width of returned image */
+  width: number;
+  /** pixels */
+  height: number;
+  /** crs for the image (not the bounding box) */
+  crs?: string;
+  /** requested format for the return image */
+  format?: 'image/png';
+};
+
+export type ImageSourceProps = DataSourceProps;
+
+/**
+ * MapImageSource - data sources that allow data to be queried by (geospatial) extents
+ * @note
+ * - If geospatial, bounding box is expected to be in web mercator coordinates
+ */
+export abstract class ImageSource<
+  PropsT extends ImageSourceProps = ImageSourceProps
+> extends DataSource<PropsT> {
+  static type: string = 'template';
+  static testURL = (url: string): boolean => false;
+
+  abstract getMetadata(): Promise<ImageSourceMetadata>;
+  abstract getImage(parameters: GetImageParameters): Promise<ImageType>;
+}

package/src/lib/sources/image-tile-source.ts

@@ -0,0 +1,14 @@
+// loaders.gl, MIT license
+
+import type {ImageType} from './utils/image-type';
+import type {TileSource, TileSourceMetadata} from './tile-source';
+import type {GetTileParameters} from './tile-source';
+
+/**
+ * MapTileSource - data sources that allow data to be queried by (geospatial) tile
+ * @note If geospatial, bounding box is expected to be in web mercator coordinates
+ */
+export interface ImageTileSource<MetadataT extends TileSourceMetadata = TileSourceMetadata>
+  extends TileSource<MetadataT> {
+  getImageTile(parameters: GetTileParameters): Promise<ImageType | null>;
+}

package/src/lib/sources/tile-source.ts

@@ -0,0 +1,101 @@
+// loaders.gl, MIT license
+
+/**
+ * Normalized capabilities of an Image service
+ * Sources are expected to normalize the response to capabilities
+ * @example
+ *  A WMS service would normalize the response to the WMS `GetCapabilities` data structure extracted from WMS XML response
+ *  into an TileSourceMetadata.
+ */
+export type TileSourceMetadata = {
+  format?: string;
+  formatHeader?: unknown;
+
+  /** Name of the tileset (extracted from JSON metadata if available) */
+  name?: string;
+  title?: string;
+  abstract?: string;
+  keywords?: string[];
+  /** Attribution string (extracted from JSON metadata if available) */
+  attributions?: string[];
+
+  /** Minimal zoom level of tiles in this tileset */
+  minZoom?: number;
+  /** Maximal zoom level of tiles in this tileset */
+  maxZoom?: number;
+  /** Bounding box of tiles in this tileset `[[w, s], [e, n]]`  */
+  boundingBox?: [min: [x: number, y: number], max: [x: number, y: number]];
+
+  /** Layer information */
+  layer?: {
+    name: string;
+    title?: string;
+    srs?: string[];
+    boundingBox?: [number, number, number, number];
+    layers: TileSourceLayer[];
+  };
+};
+
+/**
+ * Description of one data layer in the image source
+ */
+export type TileSourceLayer = {
+  name: string;
+  title?: string;
+  srs?: string[];
+  boundingBox?: [number, number, number, number];
+  layers: TileSourceLayer[];
+};
+
+/**
+ * Generic parameters for requesting an image from an image source
+ */
+export type GetTileParameters = {
+  /** Layers to render */
+  layers: string | string[];
+  /** Styling */
+  styles?: unknown;
+  /** bounding box of the requested map image */
+  zoom: number;
+  /** tile x coordinate */
+  x: number;
+  /** tile y coordinate */
+  y: number;
+  /** srs for the image (not the bounding box) */
+  srs?: string;
+  /** requested format for the return image */
+  format?: 'image/png';
+};
+
+export type TileLoadParameters = {
+  index: {x: number; y: number; z: number};
+  id: string;
+  bbox: TileBoundingBox;
+  zoom?: number;
+  url?: string | null;
+  signal?: AbortSignal;
+  userData?: Record<string, any>;
+};
+
+/** deck.gl compatible bounding box */
+export type TileBoundingBox = NonGeoBoundingBox | GeoBoundingBox;
+export type GeoBoundingBox = {west: number; north: number; east: number; south: number};
+export type NonGeoBoundingBox = {left: number; top: number; right: number; bottom: number};
+
+/**
+ * Props for a TileSource
+ */
+export type TileSourceProps = {};
+
+/**
+ * MapTileSource - data sources that allow data to be queried by (geospatial) extents
+ * @note
+ * - If geospatial, bounding box is expected to be in web mercator coordinates
+ */
+export interface TileSource<MetadataT extends TileSourceMetadata> {
+  getMetadata(): Promise<MetadataT>;
+  /** Flat parameters */
+  getTile(parameters: GetTileParameters): Promise<unknown>;
+  /** deck.gl style parameters */
+  getTileData(parameters: TileLoadParameters): Promise<unknown | null>;
+}

package/src/lib/sources/utils/image-type.ts

@@ -0,0 +1,10 @@
+/** Supported Image Types @note duplicates definition in images/schema to avoid circular deps */
+export type ImageType = ImageBitmap | ImageDataType | HTMLImageElement;
+
+/** data images @note duplicates definition in images/schema to avoid circular deps */
+export type ImageDataType = {
+  data: Uint8Array;
+  width: number;
+  height: number;
+  compressed?: boolean;
+};

package/src/lib/sources/utils/utils.ts

@@ -0,0 +1,42 @@
+// loaders.gl, MIT license
+
+import type {LoaderOptions} from '@loaders.gl/loader-utils';
+
+/**
+ * Gets the current fetch function from options
+ * @todo - move to loader-utils module
+ * @todo - use in core module counterpart
+ * @param options
+ * @param context
+ */
+export function getFetchFunction(options?: LoaderOptions) {
+  const fetchFunction = options?.fetch;
+
+  // options.fetch can be a function
+  if (fetchFunction && typeof fetchFunction === 'function') {
+    return (url: string, fetchOptions?: RequestInit) => fetchFunction(url, fetchOptions);
+  }
+
+  // options.fetch can be an options object, use global fetch with those options
+  const fetchOptions = options?.fetch;
+  if (fetchOptions && typeof fetchOptions !== 'function') {
+    return (url) => fetch(url, fetchOptions);
+  }
+
+  // else return the global fetch function
+  return (url) => fetch(url);
+}
+
+export function mergeImageServiceProps<Props extends {loadOptions?: any}>(
+  props: Props
+): Required<Props> {
+  // @ts-expect-error
+  return {
+    // Default fetch
+    ...props,
+    loadOptions: {
+      ...props.loadOptions,
+      fetch: getFetchFunction(props.loadOptions)
+    }
+  };
+}

package/src/lib/sources/vector-tile-source.ts

@@ -0,0 +1,13 @@
+// loaders.gl, MIT license
+
+import {TileSource, TileSourceMetadata} from './tile-source';
+import {GetTileParameters} from './tile-source';
+
+/**
+ * MapTileSource - data sources that allow data to be queried by (geospatial) tile
+ * @note If geospatial, bounding box is expected to be in web mercator coordinates
+ */
+export interface VectorTileSource<MetadataT extends TileSourceMetadata = TileSourceMetadata>
+  extends TileSource<MetadataT> {
+  getVectorTile(parameters: GetTileParameters): Promise<unknown | null>;
+}

package/src/lib/worker-loader-utils/create-loader-worker.ts

@@ -1,5 +1,5 @@
 /* eslint-disable no-restricted-globals */
-import type {LoaderWithParser} from '../../types';
+import type {LoaderWithParser, LoaderOptions, LoaderContext} from '../../loader-types';
 import {WorkerBody} from '@loaders.gl/worker-utils';
 // import {validateLoaderVersion} from './validate-loader-version';
 
@@ -27,9 +27,10 @@ export function createLoaderWorker(loader: LoaderWithParser) {
             loader,
             arrayBuffer: input,
             options,
+            // @ts-expect-error fetch missing
             context: {
               ...context,
-              parse: parseOnMainThread
+              _parse: parseOnMainThread
             }
           });
           WorkerBody.postMessage('done', {result});
@@ -43,7 +44,12 @@ export function createLoaderWorker(loader: LoaderWithParser) {
   };
 }
 
-function parseOnMainThread(arrayBuffer: ArrayBuffer, options: {[key: string]: any}): Promise<void> {
+function parseOnMainThread(
+  arrayBuffer: ArrayBuffer,
+  loader: any,
+  options?: LoaderOptions,
+  context?: LoaderContext
+): Promise<void> {
   return new Promise((resolve, reject) => {
     const id = requestId++;
 
@@ -83,7 +89,17 @@ function parseOnMainThread(arrayBuffer: ArrayBuffer, options: {[key: string]: an
 // TODO - Why not support async loader.parse* funcs here?
 // TODO - Why not reuse a common function instead of reimplementing loader.parse* selection logic? Keeping loader small?
 // TODO - Lack of appropriate parser functions can be detected when we create worker, no need to wait until parse
-async function parseData({loader, arrayBuffer, options, context}) {
+async function parseData({
+  loader,
+  arrayBuffer,
+  options,
+  context
+}: {
+  loader: LoaderWithParser;
+  arrayBuffer: ArrayBuffer;
+  options: LoaderOptions;
+  context: LoaderContext;
+}) {
   let data;
   let parser;
   if (loader.parseSync || loader.parse) {

package/src/lib/worker-loader-utils/encode-with-worker.ts

@@ -1,5 +1,5 @@
 import {WorkerFarm} from '@loaders.gl/worker-utils';
-import {Writer, WriterOptions} from '../../types';
+import {Writer, WriterOptions} from '../../writer-types';
 import {isBrowser} from '../env-utils/globals';
 
 /**

package/src/lib/worker-loader-utils/parse-with-worker.ts

@@ -4,7 +4,7 @@ import {
   WorkerMessagePayload,
   isBrowser
 } from '@loaders.gl/worker-utils';
-import type {Loader, LoaderOptions, LoaderContext} from '../../types';
+import type {Loader, LoaderOptions, LoaderContext} from '../../loader-types';
 import {WorkerFarm, getWorkerURL} from '@loaders.gl/worker-utils';
 
 /**
@@ -34,7 +34,7 @@ export async function parseWithWorker(
   data: any,
   options?: LoaderOptions,
   context?: LoaderContext,
-  parseOnMainThread?: (arrayBuffer: ArrayBuffer, options: {[key: string]: any}) => Promise<void>
+  parseOnMainThread?: (arrayBuffer: ArrayBuffer, options: {[key: string]: any}) => Promise<unknown>
 ) {
   const name = loader.id; // TODO
   const url = getWorkerURL(loader, options);