@oceanum/datamesh 0.1.1 → 0.4.2

package/README.md

@@ -4,7 +4,7 @@ A typescript library for interacting with the Oceanum.io Datamesh.
 
 ## Installation
 
-You can use this library in Node.js, Deno or browser code
+You can use this library in Node.js, Deno or browser code (with the caveat below)
 
 ```sh
 npm install @oceanum/datamesh
@@ -16,15 +16,16 @@ npm install @oceanum/datamesh
 import { Connector } from "@oceanum/datamesh";
 
 //Instatiate the Datamesh Connector
-const datamesh=Connector("my_datamesh_token"); //Get you datamesh token from your Oceanum.io account
+const datamesh = Connector("my_datamesh_token"); //Get your datamesh token from your Oceanum.io account
 
 //Define a datamesh query
-const query={
-    "datasource":"oceanum-sizing_giants"
-}
+const query = {
+  datasource: "oceanum-sizing_giants",
+};
 
 //Get the data
-const data=await datamesh.query(query);
+const data = await datamesh.query(query);
 ```
 
-DO NOT put your Datamesh token directly into browser code. For use in an SPA, you can either forward your Datamesh request through a proxy or implement a token exchange. Read the [library documentation](https://oceanum-js.oceanum.io/) to learn more.
+[!WARNING]
+DO NOT put your Datamesh token directly into browser code. For use in an SPA, you should forward your Datamesh request through a reverse proxy to conceal your token. Read the [library documentation](https://oceanum-js.oceanum.io/datamesh) to learn more.

package/package.json

@@ -1,22 +1,24 @@
 {
   "name": "@oceanum/datamesh",
-  "version": "0.1.1",
-  "scripts": {},
+  "version": "0.4.2",
+  "scripts": {
+    "build:docs": "typedoc"
+  },
   "publishConfig": {
     "access": "public"
   },
   "dependencies": {
-    "@types/geojson": "^7946.0.14",
+    "@types/geojson": "^7946.0.16",
     "@types/object-hash": "^3.0.6",
-    "@zarrita/core": "^0.1.0-next.15",
-    "@zarrita/indexing": "^0.1.0-next.17",
-    "@zarrita/storage": "^0.1.0-next.7",
+    "apache-arrow": "^19.0.1",
+    "buffer": "^6.0.3",
     "dayjs": "^1.11.13",
     "idb-keyval": "^6.2.1",
     "object-hash": "^3.0.0",
-    "zarrita": "^0.4.0-next.17"
+    "wkx-ts": "^1.0.1",
+    "zarrita": "^0.4.0-next.23"
   },
+  "type": "module",
   "main": "./dist/index.js",
-  "module": "./dist/index.mjs",
   "typings": "./dist/index.d.ts"
 }

package/src/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+exports.__esModule = true;
+__exportStar(require("./lib/connector"), exports);
+__exportStar(require("./lib/datasource"), exports);
+__exportStar(require("./lib/query"), exports);
+__exportStar(require("./lib/datamodel"), exports);

package/src/index.ts

@@ -1,2 +1,4 @@
 export * from "./lib/connector";
 export * from "./lib/datasource";
+export * from "./lib/query";
+export * from "./lib/datamodel";

package/src/lib/connector.ts

@@ -1,56 +1,120 @@
 import { Datasource } from "./datasource";
 import { IQuery, Stage } from "./query";
-import { Dataset, DatameshStore } from "./datamodel";
+import { Dataset, HttpZarr, TempZarr } from "./datamodel";
+import { measureTime } from "./observe";
+import { tableFromIPC, Table } from "apache-arrow";
+import { Session } from "./session";
 
 /**
  * Datamesh connector class.
  *
  * All datamesh operations are methods of this class.
+ *
  */
+const DATAMESH_SERVICE =
+  process.env.DATAMESH_SERVICE || "https://datamesh.oceanum.io";
+
 export class Connector {
+  static LAZY_LOAD_SIZE = 1e8;
   private _token: string;
-  private _proto: string;
   private _host: string;
   private _authHeaders: Record<string, string>;
   private _gateway: string;
+  private _nocache = false;
+  private _isV1 = false;
+  private _sessionParams: Record<string, number> = {};
+  private _currentSession: Session | null = null;
+  service?: string;
+  gateway?: string;
 
   /**
    * Datamesh connector constructor
    *
    * @param token - Your datamesh access token. Defaults to environment variable DATAMESH_TOKEN is defined else as literal string "DATAMESH_TOKEN". DO NOT put your Datamesh token directly into public facing browser code.
-   * @param service - URL of datamesh service. Defaults to environment variable DATAMESH_SERVICE or "https://datamesh.oceanum.io".
-   * @param gateway - URL of gateway service. Defaults to "https://gateway.datamesh.oceanum.io".
+   * @param options - Constructor options.
+   * @param options.service - URL of datamesh service. Defaults to environment variable DATAMESH_SERVICE or "https://datamesh.oceanum.io".
+   * @param options.gateway - URL of gateway service. Defaults to "https://gateway.<datamesh_service_domain>".
+   * @param options.jwtAuth - JWT for Oceanum service.
+   * @param options.nocache - Disable caching of datamesh results.
+   * @param options.sessionDuration - The desired length of time for acquired datamesh sessions in hours. Will be 1 hour by default.
    *
    * @throws {Error} - If a valid token is not provided.
    */
   constructor(
     token = process.env.DATAMESH_TOKEN || "$DATAMESH_TOKEN",
-    service = process.env.DATAMESH_SERVICE || "https://datamesh.oceanum.io",
-    gateway = process.env.DATAMESH_GATEWAY ||
-      "https://gateway.datamesh.oceanum.io"
+    options?: {
+      service?: string;
+      gateway?: string;
+      jwtAuth?: string;
+      nocache?: boolean;
+      sessionDuration?: number;
+    }
   ) {
-    if (!token) {
+    if (!token && !options?.jwtAuth) {
       throw new Error(
         "A valid datamesh token must be supplied as a connector constructor argument or defined in environment variables as DATAMESH_TOKEN"
       );
     }
 
     this._token = token;
-    const url = new URL(service);
-    this._proto = url.protocol;
-    this._host = url.hostname;
-    this._authHeaders = {
-      Authorization: `Token ${this._token}`,
-      "X-DATAMESH-TOKEN": this._token,
-    };
+    this._nocache = options?.nocache ?? false;
+    const url = new URL(options?.service || DATAMESH_SERVICE);
+    this._host = `${url.protocol}//${url.hostname}`;
+    this._authHeaders = options?.jwtAuth
+      ? {
+          Authorization: `Bearer ${options.jwtAuth}`,
+        }
+      : {
+          Authorization: `Token ${this._token}`,
+          "X-DATAMESH-TOKEN": this._token,
+        };
 
-    this._gateway = gateway || `${this._proto}//gateway.${this._host}`;
+    /* This is for testing  the gateway service is not always the same as the service domain */
+    this._gateway =
+      options?.gateway || `${url.protocol}//gateway.${url.hostname}`;
 
     if (
       this._host.split(".").slice(-1)[0] !==
       this._gateway.split(".").slice(-1)[0]
     ) {
-      console.warn("Gateway and service domain do not match");
+      console.warn("Datamesh gateway and service domains do not match");
+    }
+
+    // Set session parameters if provided
+    if (
+      options?.sessionDuration &&
+      typeof options.sessionDuration === "number"
+    ) {
+      this._sessionParams = { duration: options.sessionDuration };
+    }
+
+    // Check if the API is v1 (supports sessions)
+    this._checkApiVersion();
+  }
+
+  /**
+   * Check if the API version supports sessions.
+   *
+   * @private
+   */
+  private async _checkApiVersion(): Promise<void> {
+    try {
+      // Simply check to see if we can get a session
+      const response = await fetch(`${this._gateway}/session`, {
+        headers: this._authHeaders,
+      });
+
+      if (response.status === 200) {
+        this._isV1 = true;
+        console.info("Using datamesh API version 1");
+      } else {
+        this._isV1 = false;
+        console.info("Using datamesh API version 0");
+      }
+    } catch {
+      // If we can't connect to the gateway, assume it's not a v1 API
+      this._isV1 = false;
+      console.info("Using datamesh API version 0");
     }
   }
 
@@ -66,10 +130,10 @@ export class Connector {
   /**
    * Check the status of the metadata server.
    *
-   * @returns True if the metadata server is up, false otherwise.
+   * @returns True if the server is up, false otherwise.
    */
   async status(): Promise<boolean> {
-    const response = await fetch(`${this._proto}//${this._host}`, {
+    const response = await fetch(this._host, {
       headers: this._authHeaders,
     });
     return response.status === 200;
@@ -94,6 +158,56 @@ export class Connector {
     }
   }
 
+  /**
+   * Create a new session.
+   *
+   * @param options - Session options.
+   * @param options.duration - The desired length of time for the session in hours. Defaults to the value set in the constructor or 1 hour.
+   * @returns A new session instance.
+   */
+  async createSession(options: { duration?: number } = {}): Promise<Session> {
+    const sessionOptions = {
+      duration: options.duration || this._sessionParams.duration || 1,
+    };
+    this._currentSession = await Session.acquire(this, sessionOptions);
+    return this._currentSession;
+  }
+
+  /**
+   * Get the current session or create a new one if none exists.
+   *
+   * @returns The current session.
+   */
+  async getSession(): Promise<Session> {
+    if (!this._currentSession) {
+      return this.createSession();
+    }
+    return this._currentSession;
+  }
+
+  /**
+   * Get headers with session information if available.
+   *
+   * @param additionalHeaders - Additional headers to include.
+   * @returns Headers with session information.
+   */
+  private async getSessionHeaders(
+    additionalHeaders: Record<string, string> = {}
+  ): Promise<Record<string, string>> {
+    if (this._isV1 && !this._currentSession) {
+      await this.createSession();
+    }
+
+    if (this._currentSession) {
+      return this._currentSession.addHeader({
+        ...this._authHeaders,
+        ...additionalHeaders,
+      });
+    }
+
+    return { ...this._authHeaders, ...additionalHeaders };
+  }
+
   /**
    * Request metadata from datamesh.
    *
@@ -105,20 +219,17 @@ export class Connector {
     datasourceId = "",
     params = {} as Record<string, string>
   ): Promise<Response> {
-    const url = new URL(
-      `${this._proto}//${this._host}/datasource/${datasourceId}`
-    );
+    const url = new URL(`${this._host}/datasource/${datasourceId}`);
     Object.keys(params).forEach((key) =>
       url.searchParams.append(key, params[key])
     );
 
+    const headers = await this.getSessionHeaders();
     const response = await fetch(url.toString(), {
-      headers: this._authHeaders,
+      headers,
     });
 
-    if (response.status === 404) {
-      throw new Error(`Datasource ${datasourceId} not found`);
-    } else if (response.status === 401) {
+    if (response.status === 403) {
       throw new Error(`Datasource ${datasourceId} not authorized`);
     }
 
@@ -133,15 +244,16 @@ export class Connector {
    * @param dataFormat - The format of the requested data. Defaults to "application/json".
    * @returns The path to the cached file.
    */
-  async dataRequest(
-    datasourceId: string,
-    dataFormat = "application/json"
-  ): Promise<Blob> {
-    const response = await fetch(`${this._gateway}/data/${datasourceId}`, {
-      headers: { Accept: dataFormat, ...this._authHeaders },
+  private async dataRequest(
+    qhash: string,
+    dataFormat = "application/vnd.apache.arrow.file"
+  ): Promise<Table> {
+    const headers = await this.getSessionHeaders({ Accept: dataFormat });
+    const response = await fetch(`${this._gateway}/oceanql/${qhash}?f=arrow`, {
+      headers,
     });
     await this.validateResponse(response);
-    return response.blob();
+    return tableFromIPC(await response.arrayBuffer());
   }
 
   /**
@@ -150,11 +262,16 @@ export class Connector {
    * @param query - The query to stage.
    * @returns The staged response.
    */
-  private async stageRequest(query: IQuery): Promise<Stage | null> {
+  @measureTime
+  async stageRequest(query: IQuery): Promise<Stage | null> {
     const data = JSON.stringify(query);
+    const headers = await this.getSessionHeaders({
+      "Content-Type": "application/json",
+    });
+
     const response = await fetch(`${this._gateway}/oceanql/stage/`, {
       method: "POST",
-      headers: { "Content-Type": "application/json", ...this._authHeaders },
+      headers,
       body: data,
     });
     if (response.status >= 400) {
@@ -171,16 +288,60 @@ export class Connector {
    * Execute a query to the datamesh.
    *
    * @param query - The query to execute.
+   * @param options.timeout - Additional options for the query.
    * @returns The response from the server.
    */
-  async query(query: IQuery): Promise<Dataset<DatameshStore> | null> {
+  @measureTime
+  async query(
+    query: IQuery,
+    options: { timeout?: number } = {}
+  ): Promise<Dataset<HttpZarr | TempZarr> | null> {
+    //Stage the query
     const stage = await this.stageRequest(query);
     if (!stage) {
       console.warn("No data found for query");
       return null;
     }
-    const url = `${this._gateway}/zarr/${stage.qhash}`;
-    const dataset = await Dataset.zarr(url, this._authHeaders);
+    //For smaller dataframes use arrow for transport
+    if (stage.size < Connector.LAZY_LOAD_SIZE && stage.container != "dataset") {
+      const table = await this.dataRequest(stage.qhash);
+      const dataset = await Dataset.fromArrow(table, stage.coordkeys);
+      return dataset;
+    }
+    let url = null;
+    let params = undefined;
+    if (
+      query.timefilter ||
+      query.geofilter ||
+      query.levelfilter ||
+      query.coordfilter
+    ) {
+      url = `${this._gateway}/zarr/${stage.qhash}`;
+    } else {
+      url = `${this._gateway}/zarr/${query.datasource}`;
+      params = query.parameters;
+    }
+
+    // Get headers with session information if available
+    const headers = await this.getSessionHeaders();
+
+    // Pass the headers to the Dataset.zarr method
+    const dataset = await Dataset.zarr(url, headers, {
+      parameters: params,
+      timeout: options.timeout || 60000, // Default timeout value
+      nocache: this._nocache,
+    });
+
+    if (query.variables) {
+      for (const v of Object.keys(dataset.variables)) {
+        if (
+          !query.variables.includes(v) &&
+          !Object.values(dataset.coordkeys).includes(v)
+        ) {
+          delete dataset.variables[v];
+        }
+      }
+    }
     return dataset;
   }
 
@@ -191,6 +352,7 @@ export class Connector {
    * @returns The datasource instance.
    * @throws {Error} - If the datasource cannot be found or is not authorized.
    */
+  //@measureTime
   async getDatasource(datasourceId: string): Promise<Datasource> {
     const meta = await this.metadataRequest(datasourceId);
     const metaDict = await meta.json();
@@ -206,23 +368,28 @@ export class Connector {
    *
    * @param datasourceId - Unique datasource ID.
    * @param parameters - Additional datasource parameters.
-   * @param useDask - Whether to use Dask for loading. Defaults to false.
-   * @returns The datasource container.
+   * @returns The dataset.
    */
+  //@measureTime
   async loadDatasource(
     datasourceId: string,
     parameters: Record<string, string | number> = {}
-  ): Promise<Dataset<DatameshStore> | null> {
+  ): Promise<Dataset<HttpZarr | TempZarr> | null> {
     const query = { datasource: datasourceId, parameters };
-    const stage = await this.stageRequest(query);
-    if (!stage) {
-      console.warn("No data found for query");
-      return null;
-    }
-    const dataset = await Dataset.zarr(
-      `${this._gateway}/zarr/${stage.qhash}`,
-      this._authHeaders
-    );
+    const dataset = await this.query(query);
     return dataset;
   }
+
+  /**
+   * Close the current session if one exists.
+   *
+   * @param finaliseWrite - Whether to finalise any write operations. Defaults to false.
+   * @returns A promise that resolves when the session is closed.
+   */
+  async closeSession(finaliseWrite = false): Promise<void> {
+    if (this._currentSession) {
+      await this._currentSession.close(finaliseWrite);
+      this._currentSession = null;
+    }
+  }
 }