@oceanum/datamesh 0.2.0 → 0.4.2

package/src/lib/datasource.ts

@@ -23,14 +23,40 @@ export type Coordinate =
   | "j" // Coordinate_j
   | "k"; // Coordinate_k
 
-export type Coordinates = {
+export type Coordkeys = {
   [key in Coordinate]?: string;
 };
 
 /**
- * Represents the schema of a data source.
+ * Represents the internal schema of a data source.
  */
 export type Schema = {
+  /**
+   * Attributes of the schema.
+   */
+  attributes?: Record<string, string | number>;
+
+  /**
+   * Dimensions of the schema.
+   */
+  dimensions: Record<string, number>;
+
+  /**
+   * Coordinate map of the schema.
+   */
+  coordkeys?: Coordkeys;
+
+  /**
+   * Data variables of the schema.
+   */
+  variables: Record<string, DataVariable>;
+};
+
+/**
+ * Datamesh schema
+ */
+
+export type DatameshSchema = {
   /**
    * Attributes of the schema.
    */
@@ -42,14 +68,14 @@ export type Schema = {
   dims: Record<string, number>;
 
   /**
-   * Coordinates of the schema.
+   * Coordinate map of the schema.
    */
-  coords?: Record<string, DataVariable>;
+  coords?: Record<string, DatameshSchema>;
 
   /**
    * Data variables of the schema.
    */
-  data_vars: Record<string, DataVariable>;
+  data_vars?: Record<string, DatameshSchema>;
 };
 
 /**
@@ -74,7 +100,7 @@ export type Datasource = {
   /**
    * Parameters associated with the data source.
    */
-  parameters?: Record<string, unknown>;
+  parameters?: Record<string, string | number>;
 
   /**
    * Geometric representation of the data source.
@@ -114,12 +140,12 @@ export type Datasource = {
   /**
    * Schema information for the data source.
    */
-  schema: Schema;
+  schema: DatameshSchema;
 
   /**
-   * Coordinate mappings for the data source.
+   * Coordinate map for the data source.
    */
-  coordinates: Coordinates;
+  coordinates: Coordkeys;
 
   /**
    * Additional details about the data source.

package/src/lib/query.ts

@@ -129,7 +129,7 @@ export type CoordSelector = {
  */
 export interface IQuery {
   datasource: string;
-  parameters?: Record<string, number | string | number[] | string[]>;
+  parameters?: Record<string, number | string>;
   description?: string;
   variables?: string[];
   timefilter?: TimeFilter;
@@ -163,7 +163,7 @@ export type Stage = {
  */
 export class Query implements IQuery {
   datasource: string;
-  parameters?: Record<string, number | string | number[] | string[]>;
+  parameters?: Record<string, number | string>;
   description?: string;
   variables?: string[];
   timefilter?: TimeFilter;

package/src/lib/session.ts

@@ -0,0 +1,171 @@
+import { measureTime } from "./observe";
+
+/**
+ * Session class for datamesh connections.
+ *
+ * Sessions are used to manage authentication and resource allocation
+ * for datamesh operations.
+ */
+export class Session {
+  id!: string;
+  user!: string;
+  creationTime!: Date;
+  endTime!: Date;
+  write!: boolean;
+  verified: boolean = false;
+  private _connection!: any;
+
+  /**
+   * Acquire a session from the connection.
+   *
+   * @param connection - Connection object to acquire session from.
+   * @param options - Session options.
+   * @param options.duration - The desired length of time for the session in hours. Defaults to 1 hour.
+   * @returns A new session instance.
+   * @throws {Error} - If the session cannot be acquired.
+   */
+  @measureTime
+  static async acquire(
+    connection: any,
+    options: { duration?: number } = {}
+  ): Promise<Session> {
+    // Check if the connection supports sessions (v1 API)
+    if (!connection._isV1) {
+      const session = new Session();
+      session.id = "dummy_session";
+      session.user = "dummy_user";
+      session.creationTime = new Date();
+      session.endTime = new Date(
+        Date.now() + (options.duration || 1) * 60 * 60 * 1000
+      );
+      session.write = false;
+      session.verified = false;
+      session._connection = connection;
+
+      // Register cleanup function for when the process exits
+      if (typeof process !== "undefined" && process.on) {
+        process.on("beforeExit", () => {
+          session.close();
+        });
+      }
+
+      return session;
+    }
+
+    try {
+      const headers = { ...connection._authHeaders };
+      headers["Cache-Control"] = "no-store";
+      const params = { duration: options.duration || 1 };
+      const response = await fetch(
+        `${connection._gateway}/session/?${params}`,
+        { headers }
+      );
+
+      if (response.status !== 200) {
+        throw new Error(`Failed to create session: ${await response.text()}`);
+      }
+
+      const data = await response.json();
+      const session = new Session();
+      session.id = data.id;
+      session.user = data.user;
+      session.creationTime = new Date(data.creation_time);
+      session.endTime = new Date(data.end_time);
+      session.write = data.write;
+      session.verified = data.verified || false;
+      session._connection = connection;
+
+      // Register cleanup function for when the process exits
+      if (typeof process !== "undefined" && process.on) {
+        process.on("beforeExit", () => {
+          session.close();
+        });
+      }
+
+      return session;
+    } catch (error) {
+      throw new Error(`Error when acquiring datamesh session: ${error}`);
+    }
+  }
+
+  /**
+   * Get the session header for requests.
+   *
+   * @returns The session header object.
+   */
+  get header(): Record<string, string> {
+    return { "X-DATAMESH-SESSIONID": this.id };
+  }
+
+  /**
+   * Add session header to an existing headers object.
+   *
+   * @param headers - The headers object to add the session header to.
+   * @returns The updated headers object.
+   */
+  addHeader(headers: Record<string, string>): Record<string, string> {
+    return { ...headers, ...this.header };
+  }
+
+  /**
+   * Close the session.
+   *
+   * @param finaliseWrite - Whether to finalise any write operations. Defaults to false.
+   * @throws {Error} - If the session cannot be closed and finaliseWrite is true.
+   */
+  async close(finaliseWrite: boolean = false): Promise<void> {
+    // Back-compatibility with beta version (ignoring)
+    if (!this._connection._isV1) {
+      return;
+    }
+
+    try {
+      // Remove the beforeExit handler if possible
+      if (typeof process !== "undefined" && process.off) {
+        process.off("beforeExit", this.close);
+      }
+
+      const response = await fetch(
+        `${this._connection._gateway}/session/${this.id}`,
+        {
+          method: "DELETE",
+          headers: this.header,
+          body: JSON.stringify({ finalise_write: finaliseWrite }),
+        }
+      );
+
+      if (response.status !== 204) {
+        if (finaliseWrite) {
+          throw new Error(`Failed to finalise write: ${await response.text()}`);
+        }
+        console.warn(`Failed to close session: ${await response.text()}`);
+      }
+    } catch (error) {
+      if (finaliseWrite) {
+        throw new Error(`Error when closing datamesh session: ${error}`);
+      }
+      console.warn(`Error when closing datamesh session: ${error}`);
+    }
+  }
+
+  /**
+   * Enter a session context.
+   *
+   * @returns The session instance.
+   */
+  async enter(): Promise<Session> {
+    return this;
+  }
+
+  /**
+   * Exit a session context.
+   *
+   * @param error - Any error that occurred during the session.
+   * @returns A promise that resolves when the session is closed.
+   */
+  async exit(error?: any): Promise<void> {
+    // When using context manager, close the session
+    // and finalise the write if no exception was raised
+    await this.close(error === undefined);
+  }
+}

package/src/lib/zarr.ts

@@ -12,36 +12,73 @@ function delay(t: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, t));
 }
 
+interface CachedHTTPStoreOptions {
+  parameters?: Record<string, string | number>;
+  chunks?: string;
+  downsample?: Record<string, number>;
+  nocache?: boolean;
+  timeout?: number;
+}
+
 export class CachedHTTPStore implements AsyncReadable {
   cache: UseStore | undefined;
   url: string;
+  params: Record<string, string>;
   cache_prefix: string;
   fetchOptions: RequestInit;
+  timeout: number;
+  _pending: Record<string, boolean> = {};
+
   constructor(
     root: string,
     authHeaders: Record<string, string>,
-    parameters?: Record<string, string | number>,
-    chunks?: string,
-    downsample?: Record<string, number>,
-    nocache?: boolean
+    options: CachedHTTPStoreOptions = {}
   ) {
+    // Create a copy of the auth headers to avoid modifying the original
+    // Important: We need to preserve all auth headers, including session headers
     const headers = { ...authHeaders };
-    if (parameters) headers["x-parameters"] = JSON.stringify(parameters);
-    if (chunks) headers["x-chunks"] = chunks;
-    if (downsample) headers["x-downsample"] = JSON.stringify(downsample);
+
+    // Add parameters, chunks, and downsample as headers if provided
+    if (options.parameters)
+      headers["x-parameters"] = JSON.stringify(options.parameters);
+    if (options.chunks) headers["x-chunks"] = options.chunks;
+    if (options.downsample)
+      headers["x-downsample"] = JSON.stringify(options.downsample);
     headers["x-filtered"] = "True";
+
+    this.params = {};
+    if (authHeaders["x-datamesh-auth"]) {
+      this.params["auth"] = authHeaders["x-datamesh-auth"];
+    }
+    if (authHeaders["x-datamesh-sig"]) {
+      this.params["sig"] = authHeaders["x-datamesh-sig"];
+    }
+
     this.fetchOptions = { headers };
+
     this.url = root;
-    if (nocache) {
+    const datasource = root.split("/").pop();
+
+    // Determine if caching should be used
+    if (options.nocache || typeof window === "undefined") {
       this.cache = undefined;
     } else {
       this.cache = createStore("zarr", "cache");
     }
-    this.cache_prefix = hash({ ...parameters, chunks, downsample });
+
+    // Create a cache prefix based on datasource and options
+    // Note: We don't include auth headers in the cache key to avoid leaking sensitive information
+    this.cache_prefix = hash({
+      datasource,
+      ...options.parameters,
+      chunks: options.chunks,
+      downsample: options.downsample,
+    });
+    this.timeout = options.timeout || 60000;
   }
 
   async get(
-    item: string,
+    item: AbsolutePath,
     options?: RequestInit,
     retry = 0
   ): Promise<Uint8Array | undefined> {
@@ -49,36 +86,61 @@ export class CachedHTTPStore implements AsyncReadable {
     let data = null;
     if (this.cache) {
       data = await get_cache(key, this.cache);
-      if (data === "pending") {
+      if (data) delete this._pending[key];
+      if (this._pending[key]) {
         await delay(200);
-        if (retry > 5 * 60) {
+        //console.debug("Zarr pending:" + key);
+        if (retry > this.timeout) {
           await del_cache(key, this.cache);
-          throw new Error("Zarr timeout");
+          delete this._pending[key];
+          console.error("Zarr timeout");
+          return undefined;
         } else {
-          return await this.get(item, options, retry + 1);
+          return await this.get(item, options, retry + 200);
         }
       }
     }
-    if (!data || !this.cache) {
-      if (this.cache) set_cache(key, "pending", this.cache);
-      //console.log(`${this.url}/${item}`);
-      //console.log(this.fetchOptions.headers);
-      const response = await fetch(`${this.url}${item}`, {
-        ...this.fetchOptions,
-        ...options,
-      });
-
-      if (response.status === 404) {
-        // Item is not found
+    if (!data) {
+      this._pending[key] = true;
+      try {
+        // Ensure we're preserving the headers from fetchOptions when making the request
+        const requestOptions = {
+          ...this.fetchOptions,
+          ...options,
+          headers: {
+            ...(this.fetchOptions.headers || {}),
+            ...(options?.headers || {}),
+          },
+          signal: AbortSignal.timeout(this.timeout),
+        };
+        const query = new URLSearchParams(this.params).toString();
+        const response = await fetch(
+          `${this.url}${item}?${query}`,
+          requestOptions
+        );
+
+        if (response.status === 404) {
+          // Item is not found
+          if (this.cache) await del_cache(key, this.cache);
+          return undefined;
+        }
+        if (response.status >= 400) {
+          throw new Error(`HTTP error: ${response.status}`);
+        }
+        data = new Uint8Array(await response.arrayBuffer());
+        if (this.cache) await set_cache(key, data, this.cache);
+      } catch (e) {
+        console.debug("Zarr retry:" + key);
+        if (retry < this.timeout / 200) {
+          delete this._pending[key];
+          return await this.get(item, options, retry + 200);
+        }
         if (this.cache) await del_cache(key, this.cache);
+        console.error(e);
         return undefined;
-      } else if (response.status !== 200) {
-        // Item is found but there was an error
-        if (this.cache) await del_cache(key, this.cache);
-        throw new Error(String(response.status));
+      } finally {
+        delete this._pending[key];
       }
-      data = new Uint8Array(await response.arrayBuffer());
-      if (this.cache) await set_cache(key, data, this.cache);
     }
     return data;
   }

package/src/test/dataframe.test.ts

@@ -1,4 +1,4 @@
-import { test, expect } from "vitest";
+import { expect } from "vitest";
 import { Connector } from "../lib/connector";
 import { datameshTest } from "./fixtures";
 

package/src/test/dataset.test.ts

@@ -1,13 +1,33 @@
 import { assertType, test, expect } from "vitest";
 import { Dataset, IDataVar } from "../lib/datamodel";
 import { Connector } from "../lib/connector";
-import { dataset, datameshTest, DATAMESH_GATEWAY, HEADERS } from "./fixtures";
+import { datameshTest, DATAMESH_GATEWAY, HEADERS } from "./fixtures";
 
-test("dataset init", async () => {
-  const dstest = await Dataset.init(dataset);
-  assertType<Record<string, unknown>>(dstest.attrs);
-  assertType<Record<string, IDataVar>>(dstest.data_vars);
-  const datatest = await dstest.data_vars.temperature.get();
+datameshTest("dataset init", async ({ dataset }) => {
+  const coordkeys = { t: "time", x: "lon", y: "lat" };
+  const ds = {
+    attributes: dataset.attrs,
+    dimensions: dataset.dims,
+    variables: {},
+  };
+  for (const v in dataset.coords) {
+    ds.variables[v] = {
+      attributes: dataset.coords[v].attrs,
+      dimensions: dataset.coords[v].dims,
+      data: dataset.coords[v].data,
+    };
+  }
+  for (const v in dataset.data_vars) {
+    ds.variables[v] = {
+      attributes: dataset.data_vars[v].attrs,
+      dimensions: dataset.data_vars[v].dims,
+      data: dataset.data_vars[v].data,
+    };
+  }
+  const dstest = await Dataset.init(ds, coordkeys);
+  assertType<Record<string, unknown>>(dstest.attributes);
+  assertType<Record<string, IDataVar>>(dstest.variables);
+  const datatest = await dstest.variables.temperature.get();
   expect(datatest).toBeInstanceOf(Array);
   expect(datatest.length).toBe(10);
   expect(datatest[0].length).toBe(30);
@@ -15,21 +35,22 @@ test("dataset init", async () => {
   expect(datatest[3][4][5]).toEqual(
     dataset.data_vars.temperature.data[3][4][5]
   );
-  const datatest0 = await dstest.data_vars.scalar.get();
+  const datatest0 = await dstest.variables.scalar.get();
   expect(datatest0[0]).closeTo(10.1, 0.0001);
 });
 
 datameshTest(
   "dataset zarr",
-  async ({ dataset: Dataset }) => {
+  async ({ dataset }) => {
     //Test the zarr proxy endpoint directly
     const dstest = await Dataset.zarr(
       DATAMESH_GATEWAY + "/zarr/" + dataset.attrs.id,
-      HEADERS
+      HEADERS,
+      { nocache: true }
     );
-    assertType<Record<string, unknown>>(dstest.attrs);
-    assertType<Record<string, IDataVar>>(dstest.data_vars);
-    let datatest = await dstest.data_vars.temperature.get();
+    assertType<Record<string, unknown>>(dstest.attributes);
+    assertType<Record<string, IDataVar>>(dstest.variables);
+    let datatest = await dstest.variables.temperature.get();
     expect(datatest).toBeInstanceOf(Array);
     expect(datatest.length).toBe(10);
     expect(datatest[0].length).toBe(30);
@@ -37,15 +58,17 @@ datameshTest(
     expect(datatest[3][4][5]).toEqual(
       dataset.data_vars.temperature.data[3][4][5]
     );
-    datatest = await dstest.data_vars.scalar.get();
+    datatest = await dstest.variables.scalar.get();
     expect(datatest[0]).closeTo(10.1, 0.0001);
 
     //Now test with the connector
-    const datamesh = new Connector(process.env.DATAMESH_TOKEN);
+    const datamesh = new Connector(process.env.DATAMESH_TOKEN, {
+      nocache: true,
+    });
     const dstest2 = await datamesh.loadDatasource(dataset.attrs.id);
-    assertType<Record<string, unknown>>(dstest2.attrs);
-    assertType<Record<string, IDataVar>>(dstest2.data_vars);
-    datatest = await dstest.data_vars.temperature.get();
+    assertType<Record<string, unknown>>(dstest2.attributes);
+    assertType<Record<string, IDataVar>>(dstest2.variables);
+    datatest = await dstest2.variables.temperature.get();
     expect(datatest).toBeInstanceOf(Array);
     expect(datatest.length).toBe(10);
     expect(datatest[0].length).toBe(30);
@@ -53,8 +76,105 @@ datameshTest(
     expect(datatest[3][4][5]).toEqual(
       dataset.data_vars.temperature.data[3][4][5]
     );
-    datatest = await dstest.data_vars.scalar.get();
+    datatest = await dstest2.variables.scalar.get();
     expect(datatest[0]).closeTo(10.1, 0.0001);
   },
-  { timeout: 100000 }
+  { timeout: 200000 }
 );
+
+datameshTest("dataset fromGeojson", async () => {
+  // Test invalid FeatureCollection (no features array)
+  const invalidGeoJson = {
+    type: "FeatureCollection",
+  };
+  await expect(Dataset.fromGeojson(invalidGeoJson)).rejects.toThrow(
+    "Invalid FeatureCollection: features array is required"
+  );
+
+  // Test empty FeatureCollection
+  const emptyGeoJson = {
+    type: "FeatureCollection",
+    features: [],
+  };
+  await expect(Dataset.fromGeojson(emptyGeoJson)).rejects.toThrow(
+    "FeatureCollection contains no features"
+  );
+
+  // Test valid GeoJSON with multiple features and property types
+  const validGeoJson = {
+    type: "FeatureCollection",
+    features: [
+      {
+        type: "Feature",
+        geometry: {
+          type: "Point",
+          coordinates: [174.0, -37.0],
+        },
+        properties: {
+          time: "1970-01-01T00:00:00.000Z",
+          temperature: 15.5,
+          elevation: 100,
+          name: "Location A",
+          active: true,
+        },
+      },
+      {
+        type: "Feature",
+        geometry: {
+          type: "LineString",
+          coordinates: [
+            [174.1, -37.1],
+            [174.2, -37.2],
+          ],
+        },
+        properties: {
+          time: "1970-01-02T00:00:00.000Z",
+          temperature: 16.5,
+          elevation: 200,
+          name: "Path B",
+          active: false,
+        },
+      },
+    ],
+  };
+
+  const ds = await Dataset.fromGeojson(validGeoJson);
+
+  // Test that Dataset was created with correct structure
+  expect(ds).toBeInstanceOf(Dataset);
+  assertType<Record<string, unknown>>(ds.attributes);
+  assertType<Record<string, IDataVar>>(ds.variables);
+
+  // Test that all properties were correctly extracted
+  expect(Object.keys(ds.variables)).toContain("temperature");
+  expect(Object.keys(ds.variables)).toContain("elevation");
+  expect(Object.keys(ds.variables)).toContain("name");
+  expect(Object.keys(ds.variables)).toContain("active");
+
+  // Test property values
+  const names = await ds.variables.name.get();
+  expect(names).toBeInstanceOf(Array);
+  expect(names).toHaveLength(2);
+  expect(names[0]).toBe("Location A");
+  expect(names[1]).toBe("Path B");
+
+  const active = await ds.variables.active.get();
+  expect(active).toBeInstanceOf(Array);
+  expect(active).toHaveLength(2);
+  expect(active[0]).toBe(true);
+  expect(active[1]).toBe(false);
+
+  const temp = await ds.variables.temperature.get();
+  expect(temp).toBeInstanceOf(Float32Array);
+  expect(temp).toHaveLength(2);
+  expect(temp[0]).toBe(15.5);
+  expect(temp[1]).toBe(16.5);
+
+  // Test with custom coordkeys
+  const customcoordkeys = { t: "time", g: "geometry" };
+  const dsWithcoordkeys = await Dataset.fromGeojson(
+    validGeoJson,
+    customcoordkeys
+  );
+  expect(dsWithcoordkeys.coordkeys).toEqual(customcoordkeys);
+});