@oceanum/datamesh 0.2.0 → 0.4.3

package/README.md

@@ -16,16 +16,16 @@ npm install @oceanum/datamesh
 import { Connector } from "@oceanum/datamesh";
 
 //Instatiate the Datamesh Connector
-const datamesh=Connector("my_datamesh_token"); //Get your datamesh token from your Oceanum.io account
+const datamesh = Connector("my_datamesh_token"); //Get your datamesh token from your Oceanum.io account
 
 //Define a datamesh query
-const query={
-    "datasource":"oceanum-sizing_giants"
-}
+const query = {
+  datasource: "oceanum-sizing_giants",
+};
 
 //Get the data
-const data=await datamesh.query(query);
+const data = await datamesh.query(query);
 ```
 
 [!WARNING]
-DO NOT put your Datamesh token directly into browser code. For use in an SPA, you should forward your Datamesh request through a reverse proxy to conceal your token. Read the [library documentation](https://oceanum-js.oceanum.io/datamesh/reverse_proxy) to learn more.
+DO NOT put your Datamesh token directly into browser code. For use in an SPA, you should forward your Datamesh request through a reverse proxy to conceal your token. Read the [library documentation](https://oceanum-js.oceanum.io/datamesh) to learn more.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@oceanum/datamesh",
-  "version": "0.2.0",
+  "version": "0.4.3",
   "scripts": {
     "build:docs": "typedoc"
   },
@@ -8,17 +8,15 @@
     "access": "public"
   },
   "dependencies": {
-    "@types/geojson": "^7946.0.15",
+    "@types/geojson": "^7946.0.16",
     "@types/object-hash": "^3.0.6",
-    "@zarrita/core": "^0.1.0-next.15",
-    "@zarrita/indexing": "^0.1.0-next.17",
-    "@zarrita/storage": "^0.1.0-next.7",
-    "apache-arrow": "^18.1.0",
+    "apache-arrow": "^19.0.1",
+    "buffer": "^6.0.3",
     "dayjs": "^1.11.13",
     "idb-keyval": "^6.2.1",
     "object-hash": "^3.0.0",
-    "wkx": "^0.5.0",
-    "zarrita": "^0.4.0-next.17"
+    "wkx-ts": "^1.0.1",
+    "zarrita": "^0.4.0-next.23"
   },
   "type": "module",
   "main": "./dist/index.js",

package/src/lib/connector.ts

@@ -1,8 +1,9 @@
 import { Datasource } from "./datasource";
 import { IQuery, Stage } from "./query";
-import { Dataset, DatameshStore, TempStore } from "./datamodel";
+import { Dataset, HttpZarr, TempZarr } from "./datamodel";
 import { measureTime } from "./observe";
 import { tableFromIPC, Table } from "apache-arrow";
+import { Session } from "./session";
 
 /**
  * Datamesh connector class.
@@ -10,54 +11,110 @@ import { tableFromIPC, Table } from "apache-arrow";
  * All datamesh operations are methods of this class.
  *
  */
+const DATAMESH_SERVICE =
+  process.env.DATAMESH_SERVICE || "https://datamesh.oceanum.io";
 
 export class Connector {
   static LAZY_LOAD_SIZE = 1e8;
   private _token: string;
-  private _proto: string;
   private _host: string;
   private _authHeaders: Record<string, string>;
   private _gateway: string;
+  private _nocache = false;
+  private _isV1 = false;
+  private _sessionParams: Record<string, number> = {};
+  private _currentSession: Session | null = null;
+  service?: string;
+  gateway?: string;
 
   /**
    * Datamesh connector constructor
    *
    * @param token - Your datamesh access token. Defaults to environment variable DATAMESH_TOKEN is defined else as literal string "DATAMESH_TOKEN". DO NOT put your Datamesh token directly into public facing browser code.
-   * @param service - URL of datamesh service. Defaults to environment variable DATAMESH_SERVICE or "https://datamesh.oceanum.io".
-   * @param _gateway - URL of gateway service. Defaults to "https://gateway.datamesh.oceanum.io".
+   * @param options - Constructor options.
+   * @param options.service - URL of datamesh service. Defaults to environment variable DATAMESH_SERVICE or "https://datamesh.oceanum.io".
+   * @param options.gateway - URL of gateway service. Defaults to "https://gateway.<datamesh_service_domain>".
+   * @param options.jwtAuth - JWT for Oceanum service.
+   * @param options.nocache - Disable caching of datamesh results.
+   * @param options.sessionDuration - The desired length of time for acquired datamesh sessions in hours. Will be 1 hour by default.
    *
    * @throws {Error} - If a valid token is not provided.
    */
   constructor(
     token = process.env.DATAMESH_TOKEN || "$DATAMESH_TOKEN",
-    service = process.env.DATAMESH_SERVICE || "https://datamesh.oceanum.io",
-    // @ignore //
-    _gateway = process.env.DATAMESH_GATEWAY ||
-      "https://gateway.datamesh.oceanum.io"
+    options?: {
+      service?: string;
+      gateway?: string;
+      jwtAuth?: string;
+      nocache?: boolean;
+      sessionDuration?: number;
+    }
   ) {
-    if (!token) {
+    if (!token && !options?.jwtAuth) {
       throw new Error(
         "A valid datamesh token must be supplied as a connector constructor argument or defined in environment variables as DATAMESH_TOKEN"
       );
     }
 
     this._token = token;
-    const url = new URL(service);
-    this._proto = url.protocol;
-    this._host = url.hostname;
-    this._authHeaders = {
-      Authorization: `Token ${this._token}`,
-      "X-DATAMESH-TOKEN": this._token,
-    };
+    this._nocache = options?.nocache ?? false;
+    const url = new URL(options?.service || DATAMESH_SERVICE);
+    this._host = `${url.protocol}//${url.hostname}`;
+    this._authHeaders = options?.jwtAuth
+      ? {
+          Authorization: `Bearer ${options.jwtAuth}`,
+        }
+      : {
+          Authorization: `Token ${this._token}`,
+          "X-DATAMESH-TOKEN": this._token,
+        };
 
     /* This is for testing  the gateway service is not always the same as the service domain */
-    this._gateway = _gateway || `${this._proto}//gateway.${this._host}`;
+    this._gateway =
+      options?.gateway || `${url.protocol}//gateway.${url.hostname}`;
 
     if (
       this._host.split(".").slice(-1)[0] !==
       this._gateway.split(".").slice(-1)[0]
     ) {
-      console.warn("Gateway and service domain do not match");
+      console.warn("Datamesh gateway and service domains do not match");
+    }
+
+    // Set session parameters if provided
+    if (
+      options?.sessionDuration &&
+      typeof options.sessionDuration === "number"
+    ) {
+      this._sessionParams = { duration: options.sessionDuration };
+    }
+
+    // Check if the API is v1 (supports sessions)
+    this._checkApiVersion();
+  }
+
+  /**
+   * Check if the API version supports sessions.
+   *
+   * @private
+   */
+  private async _checkApiVersion(): Promise<void> {
+    try {
+      // Simply check to see if we can get a session
+      const response = await fetch(`${this._gateway}/session`, {
+        headers: this._authHeaders,
+      });
+
+      if (response.status === 200) {
+        this._isV1 = true;
+        console.info("Using datamesh API version 1");
+      } else {
+        this._isV1 = false;
+        console.info("Using datamesh API version 0");
+      }
+    } catch {
+      // If we can't connect to the gateway, assume it's not a v1 API
+      this._isV1 = false;
+      console.info("Using datamesh API version 0");
     }
   }
 
@@ -73,10 +130,10 @@ export class Connector {
   /**
    * Check the status of the metadata server.
    *
-   * @returns True if the metadata server is up, false otherwise.
+   * @returns True if the server is up, false otherwise.
    */
   async status(): Promise<boolean> {
-    const response = await fetch(`${this._proto}//${this._host}`, {
+    const response = await fetch(this._host, {
       headers: this._authHeaders,
     });
     return response.status === 200;
@@ -101,6 +158,56 @@ export class Connector {
     }
   }
 
+  /**
+   * Create a new session.
+   *
+   * @param options - Session options.
+   * @param options.duration - The desired length of time for the session in hours. Defaults to the value set in the constructor or 1 hour.
+   * @returns A new session instance.
+   */
+  async createSession(options: { duration?: number } = {}): Promise<Session> {
+    const sessionOptions = {
+      duration: options.duration || this._sessionParams.duration || 1,
+    };
+    this._currentSession = await Session.acquire(this, sessionOptions);
+    return this._currentSession;
+  }
+
+  /**
+   * Get the current session or create a new one if none exists.
+   *
+   * @returns The current session.
+   */
+  async getSession(): Promise<Session> {
+    if (!this._currentSession) {
+      return this.createSession();
+    }
+    return this._currentSession;
+  }
+
+  /**
+   * Get headers with session information if available.
+   *
+   * @param additionalHeaders - Additional headers to include.
+   * @returns Headers with session information.
+   */
+  private async getSessionHeaders(
+    additionalHeaders: Record<string, string> = {}
+  ): Promise<Record<string, string>> {
+    if (this._isV1 && !this._currentSession) {
+      await this.createSession();
+    }
+
+    if (this._currentSession) {
+      return this._currentSession.addHeader({
+        ...this._authHeaders,
+        ...additionalHeaders,
+      });
+    }
+
+    return { ...this._authHeaders, ...additionalHeaders };
+  }
+
   /**
    * Request metadata from datamesh.
    *
@@ -112,20 +219,17 @@ export class Connector {
     datasourceId = "",
     params = {} as Record<string, string>
   ): Promise<Response> {
-    const url = new URL(
-      `${this._proto}//${this._host}/datasource/${datasourceId}`
-    );
+    const url = new URL(`${this._host}/datasource/${datasourceId}`);
     Object.keys(params).forEach((key) =>
       url.searchParams.append(key, params[key])
     );
 
+    const headers = await this.getSessionHeaders();
     const response = await fetch(url.toString(), {
-      headers: this._authHeaders,
+      headers,
     });
 
-    if (response.status === 404) {
-      throw new Error(`Datasource ${datasourceId} not found`);
-    } else if (response.status === 401) {
+    if (response.status === 403) {
       throw new Error(`Datasource ${datasourceId} not authorized`);
     }
 
@@ -144,8 +248,9 @@ export class Connector {
     qhash: string,
     dataFormat = "application/vnd.apache.arrow.file"
   ): Promise<Table> {
+    const headers = await this.getSessionHeaders({ Accept: dataFormat });
     const response = await fetch(`${this._gateway}/oceanql/${qhash}?f=arrow`, {
-      headers: { Accept: dataFormat, ...this._authHeaders },
+      headers,
     });
     await this.validateResponse(response);
     return tableFromIPC(await response.arrayBuffer());
@@ -158,11 +263,15 @@ export class Connector {
    * @returns The staged response.
    */
   @measureTime
-  private async stageRequest(query: IQuery): Promise<Stage | null> {
+  async stageRequest(query: IQuery): Promise<Stage | null> {
     const data = JSON.stringify(query);
+    const headers = await this.getSessionHeaders({
+      "Content-Type": "application/json",
+    });
+
     const response = await fetch(`${this._gateway}/oceanql/stage/`, {
       method: "POST",
-      headers: { "Content-Type": "application/json", ...this._authHeaders },
+      headers,
       body: data,
     });
     if (response.status >= 400) {
@@ -179,12 +288,15 @@ export class Connector {
    * Execute a query to the datamesh.
    *
    * @param query - The query to execute.
+   * @param options.timeout - Additional options for the query.
    * @returns The response from the server.
    */
   @measureTime
   async query(
-    query: IQuery
-  ): Promise<Dataset</** @ignore */ DatameshStore | TempStore> | null> {
+    query: IQuery,
+    options: { timeout?: number } = {}
+  ): Promise<Dataset<HttpZarr | TempZarr> | null> {
+    //Stage the query
     const stage = await this.stageRequest(query);
     if (!stage) {
       console.warn("No data found for query");
@@ -196,8 +308,40 @@ export class Connector {
       const dataset = await Dataset.fromArrow(table, stage.coordkeys);
       return dataset;
     }
-    const url = `${this._gateway}/zarr/${stage.qhash}`;
-    const dataset = await Dataset.zarr(url, this._authHeaders);
+    let url = null;
+    let params = undefined;
+    if (
+      query.timefilter ||
+      query.geofilter ||
+      query.levelfilter ||
+      query.coordfilter
+    ) {
+      url = `${this._gateway}/zarr/${stage.qhash}`;
+    } else {
+      url = `${this._gateway}/zarr/${query.datasource}`;
+      params = query.parameters;
+    }
+
+    // Get headers with session information if available
+    const headers = await this.getSessionHeaders();
+
+    // Pass the headers to the Dataset.zarr method
+    const dataset = await Dataset.zarr(url, headers, {
+      parameters: params,
+      timeout: options.timeout || 60000, // Default timeout value
+      nocache: this._nocache,
+    });
+
+    if (query.variables) {
+      for (const v of Object.keys(dataset.variables)) {
+        if (
+          !query.variables.includes(v) &&
+          !Object.values(dataset.coordkeys).includes(v)
+        ) {
+          delete dataset.variables[v];
+        }
+      }
+    }
     return dataset;
   }
 
@@ -208,7 +352,7 @@ export class Connector {
    * @returns The datasource instance.
    * @throws {Error} - If the datasource cannot be found or is not authorized.
    */
-  @measureTime
+  //@measureTime
   async getDatasource(datasourceId: string): Promise<Datasource> {
     const meta = await this.metadataRequest(datasourceId);
     const metaDict = await meta.json();
@@ -226,13 +370,26 @@ export class Connector {
    * @param parameters - Additional datasource parameters.
    * @returns The dataset.
    */
-  @measureTime
+  //@measureTime
   async loadDatasource(
     datasourceId: string,
     parameters: Record<string, string | number> = {}
-  ): Promise<Dataset</** @ignore */ DatameshStore> | null> {
+  ): Promise<Dataset<HttpZarr | TempZarr> | null> {
     const query = { datasource: datasourceId, parameters };
     const dataset = await this.query(query);
     return dataset;
   }
+
+  /**
+   * Close the current session if one exists.
+   *
+   * @param finaliseWrite - Whether to finalise any write operations. Defaults to false.
+   * @returns A promise that resolves when the session is closed.
+   */
+  async closeSession(finaliseWrite = false): Promise<void> {
+    if (this._currentSession) {
+      await this._currentSession.close(finaliseWrite);
+      this._currentSession = null;
+    }
+  }
 }