@oceanum/datamesh 0.1.1 → 0.4.2

package/src/lib/datasource.ts

@@ -4,37 +4,59 @@ import duration from "dayjs/plugin/duration";
 
 import { DataVariable } from "./datamodel";
 
-/**
- * Represents a data variable.
- */
-enum Coordinate {
-  "Station" = "s", // locations assumed stationary, datasource multigeometry coordinate indexed by station coordinate
-  "Ensemble" = "e",
-  "Raster band" = "b",
-  "Category" = "c",
-  "Quantile" = "q",
-  "Season" = "n",
-  "Month" = "m",
-  "Time" = "t",
-  "Vertical coordinate" = "z",
-  "Horizontal northerly" = "y",
-  "Horizontal easterly" = "x",
-  "Geometry" = "g", // Abstract coordinate - a 2 or 3D geometry that defines a feature location
-  "Frequency" = "f", // spectra
-  "Direction" = "d", // spectra or stats
-  "Coordinate_i" = "i",
-  "Coordinate_j" = "j",
-  "Coordinate_k" = "k",
-}
-
-type Coordinates = {
+export type Coordinate =
+  | "s" // locations assumed stationary, datasource multigeometry coordinate indexed by station coordinate
+  | "e" // Ensemble
+  | "b" // Raster band
+  | "c" // Category
+  | "q" // Quantile
+  | "n" // Season
+  | "m" // Month
+  | "t" // Time
+  | "z" // Vertical coordinate
+  | "y" // Horizontal northerly
+  | "x" // Horizontal easterly
+  | "g" // Abstract coordinate - a 2 or 3D geometry that defines a feature location
+  | "f" // Frequency - spectra
+  | "d" // Direction - spectra or stats
+  | "i" // Coordinate_i
+  | "j" // Coordinate_j
+  | "k"; // Coordinate_k
+
+export type Coordkeys = {
   [key in Coordinate]?: string;
 };
 
 /**
- * Represents the schema of a data source.
+ * Represents the internal schema of a data source.
  */
 export type Schema = {
+  /**
+   * Attributes of the schema.
+   */
+  attributes?: Record<string, string | number>;
+
+  /**
+   * Dimensions of the schema.
+   */
+  dimensions: Record<string, number>;
+
+  /**
+   * Coordinate map of the schema.
+   */
+  coordkeys?: Coordkeys;
+
+  /**
+   * Data variables of the schema.
+   */
+  variables: Record<string, DataVariable>;
+};
+
+/**
+ * Datamesh schema
+ */
+
+export type DatameshSchema = {
   /**
    * Attributes of the schema.
    */
@@ -46,14 +68,14 @@ export type Schema = {
   dims: Record<string, number>;
 
   /**
-   * Coordinates of the schema.
+   * Coordinate map of the schema.
    */
-  coords?: Record<string, DataVariable>;
+  coords?: Record<string, DatameshSchema>;
 
   /**
    * Data variables of the schema.
    */
-  data_vars: Record<string, DataVariable>;
+  data_vars?: Record<string, DatameshSchema>;
 };
 
 /**
@@ -78,7 +100,7 @@ export type Datasource = {
   /**
    * Parameters associated with the data source.
    */
-  parameters?: Record<string, unknown>;
+  parameters?: Record<string, string | number>;
 
   /**
    * Geometric representation of the data source.
@@ -118,12 +140,12 @@ export type Datasource = {
   /**
    * Schema information for the data source.
    */
-  schema: Schema;
+  schema: DatameshSchema;
 
   /**
-   * Coordinate mappings for the data source.
+   * Coordinate map for the data source.
    */
-  coordinates: Coordinates;
+  coordinates: Coordkeys;
 
   /**
    * Additional details about the data source.

package/src/lib/observe.ts

@@ -0,0 +1,21 @@
+/** @ignore */
+export function measureTime(
+  target: any,
+  propertyKey: string,
+  descriptor: PropertyDescriptor
+) {
+  const originalMethod = descriptor.value;
+
+  descriptor.value = async function (...args: any[]) {
+    const start = Date.now();
+    const result = await originalMethod.apply(this, args);
+    const end = Date.now();
+    const executionTime = end - start;
+
+    console.debug(`${propertyKey} took ${executionTime}ms`);
+
+    return result;
+  };
+
+  return descriptor;
+}

package/src/lib/query.ts

@@ -5,73 +5,47 @@ import duration from "dayjs/plugin/duration";
 dayjs.extend(duration);
 
 /**
- * GeoFilterType enum representing types of geofilters.
+ * GeoFilterType type representing types of geofilters.
  */
-enum GeoFilterType {
-  Feature = "feature",
-  Bbox = "bbox",
-}
+export type GeoFilterType = "feature" | "bbox";
 
 /**
- * GeoFilterInterp enum representing interpolation methods for geofilters.
+ * GeoFilterInterp type representing interpolation methods for geofilters.
  */
-enum GeoFilterInterp {
-  Nearest = "nearest",
-  Linear = "linear",
-}
+export type GeoFilterInterp = "nearest" | "linear";
 
 /**
- * LevelFilterInterp enum representing interpolation methods for level filters.
+ * LevelFilterInterp type representing interpolation methods for level filters.
  */
-enum LevelFilterInterp {
-  Nearest = "nearest",
-  Linear = "linear",
-}
+export type LevelFilterInterp = "nearest" | "linear";
 
 /**
- * TimeFilterType enum representing types of time filters.
+ * TimeFilterType type representing types of time filters.
  */
-enum TimeFilterType {
-  Range = "range",
-  Series = "series",
-  Trajectory = "trajectory",
-}
+export type TimeFilterType = "range" | "series" | "trajectory";
 
 /**
- * LevelFilterType enum representing types of level filters.
+ * LevelFilterType type representing types of level filters.
  */
-enum LevelFilterType {
-  Range = "range",
-  Series = "series",
-}
+export type LevelFilterType = "range" | "series";
 
 /**
- * ResampleType enum representing types of resampling.
+ * ResampleType type representing types of resampling.
  */
-enum ResampleType {
-  Mean = "mean",
-  Nearest = "nearest",
-  Slinear = "linear",
-}
+export type ResampleType = "mean" | "nearest" | "linear";
 
 /**
- * AggregateOps enum representing aggregation operations.
+ * AggregateOps type representing aggregation operations.
  */
-enum AggregateOps {
-  Mean = "mean",
-  Min = "min",
-  Max = "max",
-  Std = "std",
-  Sum = "sum",
-}
+export type AggregateOps = "mean" | "min" | "max" | "std" | "sum";
 
 /**
- * Container enum representing data container types.
+ * Container type representing data container types.
  */
-enum Container {
-  GeoDataFrame = "geodataframe",
-  DataFrame = "dataframe",
-  Dataset = "dataset",
+export type Container = "geodataframe" | "dataframe" | "dataset";
+
+export interface GeoFilterFeature extends Omit<Feature, "properties"> {
+  properties?: Record<string, unknown> | undefined;
 }
 
 /**
@@ -79,7 +53,7 @@ enum Container {
  */
 export type GeoFilter = {
   type: GeoFilterType;
-  geom: Array<number[]> | Feature;
+  geom: Array<number[]> | GeoFilterFeature;
   interp?: GeoFilterInterp;
   resolution?: number;
   alltouched?: boolean;
@@ -88,7 +62,7 @@ export type GeoFilter = {
 /**
  * LevelFilter type representing a vertical subset or interpolation.
  */
-type LevelFilter = {
+export type LevelFilter = {
   type: LevelFilterType;
   levels: Array<number | null>;
   interp?: LevelFilterInterp;
@@ -126,7 +100,7 @@ const timeFilterValidate = (timefilter: TimeFilter): TimeFilter => {
   const times = timefilter.times.map((t) => stringifyTime(t));
 
   return {
-    type: timefilter.type || TimeFilterType.Range,
+    type: timefilter.type || "range",
     times,
     resolution: timefilter.resolution,
     resample: timefilter.resample,
@@ -136,7 +110,7 @@ const timeFilterValidate = (timefilter: TimeFilter): TimeFilter => {
 /**
  * Aggregate type representing aggregation operations.
  */
-type Aggregate = {
+export type Aggregate = {
   operations: AggregateOps[];
   spatial?: boolean;
   temporal?: boolean;
@@ -145,7 +119,7 @@ type Aggregate = {
 /**
  * CoordSelector type representing coordinate selection.
  */
-type CoordSelector = {
+export type CoordSelector = {
   coord: string;
   values: Array<string | number>;
 };
@@ -155,7 +129,7 @@ type CoordSelector = {
  */
 export interface IQuery {
   datasource: string;
-  parameters?: Record<string, number | string | number[] | string[]>;
+  parameters?: Record<string, number | string>;
   description?: string;
   variables?: string[];
   timefilter?: TimeFilter;
@@ -171,6 +145,7 @@ export interface IQuery {
 /**
  * Stage interface representing the result of staging a query.
  */
+/** @ignore */
 export type Stage = {
   query: Query;
   qhash: string;
@@ -178,6 +153,7 @@ export type Stage = {
   size: number;
   dlen: number;
   coordmap: Record<string, string>;
+  coordkeys: Record<string, string>;
   container: Container;
   sig: string;
 };
@@ -187,7 +163,7 @@ export type Stage = {
  */
 export class Query implements IQuery {
   datasource: string;
-  parameters?: Record<string, number | string | number[] | string[]>;
+  parameters?: Record<string, number | string>;
   description?: string;
   variables?: string[];
   timefilter?: TimeFilter;

package/src/lib/session.ts

@@ -0,0 +1,171 @@
+import { measureTime } from "./observe";
+
+/**
+ * Session class for datamesh connections.
+ *
+ * Sessions are used to manage authentication and resource allocation
+ * for datamesh operations.
+ */
+export class Session {
+  id!: string;
+  user!: string;
+  creationTime!: Date;
+  endTime!: Date;
+  write!: boolean;
+  verified: boolean = false;
+  private _connection!: any;
+
+  /**
+   * Acquire a session from the connection.
+   *
+   * @param connection - Connection object to acquire session from.
+   * @param options - Session options.
+   * @param options.duration - The desired length of time for the session in hours. Defaults to 1 hour.
+   * @returns A new session instance.
+   * @throws {Error} - If the session cannot be acquired.
+   */
+  @measureTime
+  static async acquire(
+    connection: any,
+    options: { duration?: number } = {}
+  ): Promise<Session> {
+    // Check if the connection supports sessions (v1 API)
+    if (!connection._isV1) {
+      const session = new Session();
+      session.id = "dummy_session";
+      session.user = "dummy_user";
+      session.creationTime = new Date();
+      session.endTime = new Date(
+        Date.now() + (options.duration || 1) * 60 * 60 * 1000
+      );
+      session.write = false;
+      session.verified = false;
+      session._connection = connection;
+
+      // Register cleanup function for when the process exits
+      if (typeof process !== "undefined" && process.on) {
+        process.on("beforeExit", () => {
+          session.close();
+        });
+      }
+
+      return session;
+    }
+
+    try {
+      const headers = { ...connection._authHeaders };
+      headers["Cache-Control"] = "no-store";
+      const params = { duration: options.duration || 1 };
+      const response = await fetch(
+        `${connection._gateway}/session/?${params}`,
+        { headers }
+      );
+
+      if (response.status !== 200) {
+        throw new Error(`Failed to create session: ${await response.text()}`);
+      }
+
+      const data = await response.json();
+      const session = new Session();
+      session.id = data.id;
+      session.user = data.user;
+      session.creationTime = new Date(data.creation_time);
+      session.endTime = new Date(data.end_time);
+      session.write = data.write;
+      session.verified = data.verified || false;
+      session._connection = connection;
+
+      // Register cleanup function for when the process exits
+      if (typeof process !== "undefined" && process.on) {
+        process.on("beforeExit", () => {
+          session.close();
+        });
+      }
+
+      return session;
+    } catch (error) {
+      throw new Error(`Error when acquiring datamesh session: ${error}`);
+    }
+  }
+
+  /**
+   * Get the session header for requests.
+   *
+   * @returns The session header object.
+   */
+  get header(): Record<string, string> {
+    return { "X-DATAMESH-SESSIONID": this.id };
+  }
+
+  /**
+   * Add session header to an existing headers object.
+   *
+   * @param headers - The headers object to add the session header to.
+   * @returns The updated headers object.
+   */
+  addHeader(headers: Record<string, string>): Record<string, string> {
+    return { ...headers, ...this.header };
+  }
+
+  /**
+   * Close the session.
+   *
+   * @param finaliseWrite - Whether to finalise any write operations. Defaults to false.
+   * @throws {Error} - If the session cannot be closed and finaliseWrite is true.
+   */
+  async close(finaliseWrite: boolean = false): Promise<void> {
+    // Back-compatibility with beta version (ignoring)
+    if (!this._connection._isV1) {
+      return;
+    }
+
+    try {
+      // Remove the beforeExit handler if possible
+      if (typeof process !== "undefined" && process.off) {
+        process.off("beforeExit", this.close);
+      }
+
+      const response = await fetch(
+        `${this._connection._gateway}/session/${this.id}`,
+        {
+          method: "DELETE",
+          headers: this.header,
+          body: JSON.stringify({ finalise_write: finaliseWrite }),
+        }
+      );
+
+      if (response.status !== 204) {
+        if (finaliseWrite) {
+          throw new Error(`Failed to finalise write: ${await response.text()}`);
+        }
+        console.warn(`Failed to close session: ${await response.text()}`);
+      }
+    } catch (error) {
+      if (finaliseWrite) {
+        throw new Error(`Error when closing datamesh session: ${error}`);
+      }
+      console.warn(`Error when closing datamesh session: ${error}`);
+    }
+  }
+
+  /**
+   * Enter a session context.
+   *
+   * @returns The session instance.
+   */
+  async enter(): Promise<Session> {
+    return this;
+  }
+
+  /**
+   * Exit a session context.
+   *
+   * @param error - Any error that occurred during the session.
+   * @returns A promise that resolves when the session is closed.
+   */
+  async exit(error?: any): Promise<void> {
+    // When using context manager, close the session
+    // and finalise the write if no exception was raised
+    await this.close(error === undefined);
+  }
+}

package/src/lib/workers/README.md

@@ -0,0 +1,3 @@
+This is an optional service worker to act as a caching proxy with the datamesh service.
+
+!! Work in progress - not available yet !!!

package/src/lib/zarr.ts

@@ -12,36 +12,73 @@ function delay(t: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, t));
 }
 
+interface CachedHTTPStoreOptions {
+  parameters?: Record<string, string | number>;
+  chunks?: string;
+  downsample?: Record<string, number>;
+  nocache?: boolean;
+  timeout?: number;
+}
+
 export class CachedHTTPStore implements AsyncReadable {
   cache: UseStore | undefined;
   url: string;
+  params: Record<string, string>;
   cache_prefix: string;
   fetchOptions: RequestInit;
+  timeout: number;
+  _pending: Record<string, boolean> = {};
+
   constructor(
     root: string,
     authHeaders: Record<string, string>,
-    parameters?: Record<string, string | number>,
-    chunks?: string,
-    downsample?: Record<string, number>,
-    nocache?: boolean
+    options: CachedHTTPStoreOptions = {}
   ) {
+    // Create a copy of the auth headers to avoid modifying the original
+    // Important: We need to preserve all auth headers, including session headers
     const headers = { ...authHeaders };
-    if (parameters) headers["x-parameters"] = JSON.stringify(parameters);
-    if (chunks) headers["x-chunks"] = chunks;
-    if (downsample) headers["x-downsample"] = JSON.stringify(downsample);
+
+    // Add parameters, chunks, and downsample as headers if provided
+    if (options.parameters)
+      headers["x-parameters"] = JSON.stringify(options.parameters);
+    if (options.chunks) headers["x-chunks"] = options.chunks;
+    if (options.downsample)
+      headers["x-downsample"] = JSON.stringify(options.downsample);
     headers["x-filtered"] = "True";
+
+    this.params = {};
+    if (authHeaders["x-datamesh-auth"]) {
+      this.params["auth"] = authHeaders["x-datamesh-auth"];
+    }
+    if (authHeaders["x-datamesh-sig"]) {
+      this.params["sig"] = authHeaders["x-datamesh-sig"];
+    }
+
     this.fetchOptions = { headers };
+
     this.url = root;
-    if (nocache) {
+    const datasource = root.split("/").pop();
+
+    // Determine if caching should be used
+    if (options.nocache || typeof window === "undefined") {
       this.cache = undefined;
     } else {
       this.cache = createStore("zarr", "cache");
     }
-    this.cache_prefix = hash({ ...parameters, chunks, downsample });
+
+    // Create a cache prefix based on datasource and options
+    // Note: We don't include auth headers in the cache key to avoid leaking sensitive information
+    this.cache_prefix = hash({
+      datasource,
+      ...options.parameters,
+      chunks: options.chunks,
+      downsample: options.downsample,
+    });
+    this.timeout = options.timeout || 60000;
   }
 
   async get(
-    item: string,
+    item: AbsolutePath,
     options?: RequestInit,
     retry = 0
   ): Promise<Uint8Array | undefined> {
@@ -49,36 +86,61 @@ export class CachedHTTPStore implements AsyncReadable {
     let data = null;
     if (this.cache) {
       data = await get_cache(key, this.cache);
-      if (data === "pending") {
+      if (data) delete this._pending[key];
+      if (this._pending[key]) {
         await delay(200);
-        if (retry > 5 * 60) {
+        //console.debug("Zarr pending:" + key);
+        if (retry > this.timeout) {
           await del_cache(key, this.cache);
-          throw new Error("Zarr timeout");
+          delete this._pending[key];
+          console.error("Zarr timeout");
+          return undefined;
         } else {
-          return await this.get(item, options, retry + 1);
+          return await this.get(item, options, retry + 200);
         }
       }
     }
-    if (!data || !this.cache) {
-      if (this.cache) set_cache(key, "pending", this.cache);
-      //console.log(`${this.url}/${item}`);
-      //console.log(this.fetchOptions.headers);
-      const response = await fetch(`${this.url}${item}`, {
-        ...this.fetchOptions,
-        ...options,
-      });
-
-      if (response.status === 404) {
-        // Item is not found
+    if (!data) {
+      this._pending[key] = true;
+      try {
+        // Ensure we're preserving the headers from fetchOptions when making the request
+        const requestOptions = {
+          ...this.fetchOptions,
+          ...options,
+          headers: {
+            ...(this.fetchOptions.headers || {}),
+            ...(options?.headers || {}),
+          },
+          signal: AbortSignal.timeout(this.timeout),
+        };
+        const query = new URLSearchParams(this.params).toString();
+        const response = await fetch(
+          `${this.url}${item}?${query}`,
+          requestOptions
+        );
+
+        if (response.status === 404) {
+          // Item is not found
+          if (this.cache) await del_cache(key, this.cache);
+          return undefined;
+        }
+        if (response.status >= 400) {
+          throw new Error(`HTTP error: ${response.status}`);
+        }
+        data = new Uint8Array(await response.arrayBuffer());
+        if (this.cache) await set_cache(key, data, this.cache);
+      } catch (e) {
+        console.debug("Zarr retry:" + key);
+        if (retry < this.timeout / 200) {
+          delete this._pending[key];
+          return await this.get(item, options, retry + 200);
+        }
         if (this.cache) await del_cache(key, this.cache);
+        console.error(e);
         return undefined;
-      } else if (response.status !== 200) {
-        // Item is found but there was an error
-        if (this.cache) await del_cache(key, this.cache);
-        throw new Error(String(response.status));
+      } finally {
+        delete this._pending[key];
       }
-      data = new Uint8Array(await response.arrayBuffer());
-      if (this.cache) await set_cache(key, data, this.cache);
     }
     return data;
   }