@oceanum/datamesh 0.1.1 → 0.2.0

package/package.json

@@ -1,22 +1,26 @@
 {
   "name": "@oceanum/datamesh",
-  "version": "0.1.1",
-  "scripts": {},
+  "version": "0.2.0",
+  "scripts": {
+    "build:docs": "typedoc"
+  },
   "publishConfig": {
     "access": "public"
   },
   "dependencies": {
-    "@types/geojson": "^7946.0.14",
+    "@types/geojson": "^7946.0.15",
     "@types/object-hash": "^3.0.6",
     "@zarrita/core": "^0.1.0-next.15",
     "@zarrita/indexing": "^0.1.0-next.17",
     "@zarrita/storage": "^0.1.0-next.7",
+    "apache-arrow": "^18.1.0",
     "dayjs": "^1.11.13",
     "idb-keyval": "^6.2.1",
     "object-hash": "^3.0.0",
+    "wkx": "^0.5.0",
     "zarrita": "^0.4.0-next.17"
   },
+  "type": "module",
   "main": "./dist/index.js",
-  "module": "./dist/index.mjs",
   "typings": "./dist/index.d.ts"
 }

package/src/index.js

@@ -0,0 +1,20 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+exports.__esModule = true;
+__exportStar(require("./lib/connector"), exports);
+__exportStar(require("./lib/datasource"), exports);
+__exportStar(require("./lib/query"), exports);
+__exportStar(require("./lib/datamodel"), exports);

package/src/index.ts

@@ -1,2 +1,4 @@
 export * from "./lib/connector";
 export * from "./lib/datasource";
+export * from "./lib/query";
+export * from "./lib/datamodel";

package/src/lib/connector.ts

@@ -1,13 +1,18 @@
 import { Datasource } from "./datasource";
 import { IQuery, Stage } from "./query";
-import { Dataset, DatameshStore } from "./datamodel";
+import { Dataset, DatameshStore, TempStore } from "./datamodel";
+import { measureTime } from "./observe";
+import { tableFromIPC, Table } from "apache-arrow";
 
 /**
  * Datamesh connector class.
  *
  * All datamesh operations are methods of this class.
+ *
  */
+
 export class Connector {
+  static LAZY_LOAD_SIZE = 1e8;
   private _token: string;
   private _proto: string;
   private _host: string;
@@ -19,14 +24,15 @@ export class Connector {
    *
    * @param token - Your datamesh access token. Defaults to environment variable DATAMESH_TOKEN is defined else as literal string "DATAMESH_TOKEN". DO NOT put your Datamesh token directly into public facing browser code.
    * @param service - URL of datamesh service. Defaults to environment variable DATAMESH_SERVICE or "https://datamesh.oceanum.io".
-   * @param gateway - URL of gateway service. Defaults to "https://gateway.datamesh.oceanum.io".
+   * @param _gateway - URL of gateway service. Defaults to "https://gateway.datamesh.oceanum.io".
    *
    * @throws {Error} - If a valid token is not provided.
    */
   constructor(
     token = process.env.DATAMESH_TOKEN || "$DATAMESH_TOKEN",
     service = process.env.DATAMESH_SERVICE || "https://datamesh.oceanum.io",
-    gateway = process.env.DATAMESH_GATEWAY ||
+    // @ignore //
+    _gateway = process.env.DATAMESH_GATEWAY ||
       "https://gateway.datamesh.oceanum.io"
   ) {
     if (!token) {
@@ -44,7 +50,8 @@ export class Connector {
       "X-DATAMESH-TOKEN": this._token,
     };
 
-    this._gateway = gateway || `${this._proto}//gateway.${this._host}`;
+    /* This is for testing  the gateway service is not always the same as the service domain */
+    this._gateway = _gateway || `${this._proto}//gateway.${this._host}`;
 
     if (
       this._host.split(".").slice(-1)[0] !==
@@ -133,15 +140,15 @@ export class Connector {
    * @param dataFormat - The format of the requested data. Defaults to "application/json".
    * @returns The path to the cached file.
    */
-  async dataRequest(
-    datasourceId: string,
-    dataFormat = "application/json"
-  ): Promise<Blob> {
-    const response = await fetch(`${this._gateway}/data/${datasourceId}`, {
+  private async dataRequest(
+    qhash: string,
+    dataFormat = "application/vnd.apache.arrow.file"
+  ): Promise<Table> {
+    const response = await fetch(`${this._gateway}/oceanql/${qhash}?f=arrow`, {
       headers: { Accept: dataFormat, ...this._authHeaders },
     });
     await this.validateResponse(response);
-    return response.blob();
+    return tableFromIPC(await response.arrayBuffer());
   }
 
   /**
@@ -150,6 +157,7 @@ export class Connector {
    * @param query - The query to stage.
    * @returns The staged response.
    */
+  @measureTime
   private async stageRequest(query: IQuery): Promise<Stage | null> {
     const data = JSON.stringify(query);
     const response = await fetch(`${this._gateway}/oceanql/stage/`, {
@@ -173,12 +181,21 @@ export class Connector {
    * @param query - The query to execute.
    * @returns The response from the server.
    */
-  async query(query: IQuery): Promise<Dataset<DatameshStore> | null> {
+  @measureTime
+  async query(
+    query: IQuery
+  ): Promise<Dataset</** @ignore */ DatameshStore | TempStore> | null> {
     const stage = await this.stageRequest(query);
     if (!stage) {
       console.warn("No data found for query");
       return null;
     }
+    //For smaller dataframes use arrow for transport
+    if (stage.size < Connector.LAZY_LOAD_SIZE && stage.container != "dataset") {
+      const table = await this.dataRequest(stage.qhash);
+      const dataset = await Dataset.fromArrow(table, stage.coordkeys);
+      return dataset;
+    }
     const url = `${this._gateway}/zarr/${stage.qhash}`;
     const dataset = await Dataset.zarr(url, this._authHeaders);
     return dataset;
@@ -191,6 +208,7 @@ export class Connector {
    * @returns The datasource instance.
    * @throws {Error} - If the datasource cannot be found or is not authorized.
    */
+  @measureTime
   async getDatasource(datasourceId: string): Promise<Datasource> {
     const meta = await this.metadataRequest(datasourceId);
     const metaDict = await meta.json();
@@ -206,23 +224,15 @@ export class Connector {
    *
    * @param datasourceId - Unique datasource ID.
    * @param parameters - Additional datasource parameters.
-   * @param useDask - Whether to use Dask for loading. Defaults to false.
-   * @returns The datasource container.
+   * @returns The dataset.
    */
+  @measureTime
   async loadDatasource(
     datasourceId: string,
     parameters: Record<string, string | number> = {}
-  ): Promise<Dataset<DatameshStore> | null> {
+  ): Promise<Dataset</** @ignore */ DatameshStore> | null> {
     const query = { datasource: datasourceId, parameters };
-    const stage = await this.stageRequest(query);
-    if (!stage) {
-      console.warn("No data found for query");
-      return null;
-    }
-    const dataset = await Dataset.zarr(
-      `${this._gateway}/zarr/${stage.qhash}`,
-      this._authHeaders
-    );
+    const dataset = await this.query(query);
     return dataset;
   }
 }

package/src/lib/datamodel.ts

@@ -1,22 +1,28 @@
-import { CachedHTTPStore } from "./zarr";
 import * as zarr from "@zarrita/core";
 import { Chunk, DataType, Listable, Location, TypedArray } from "@zarrita/core";
 import { Mutable, AsyncReadable } from "@zarrita/storage";
 import { get, set, Slice } from "@zarrita/indexing";
+import { Table, DataType as ArrowDataType } from "apache-arrow";
+import { Geometry, Feature, FeatureCollection } from "geojson";
+import wkx from "wkx";
 
-import { Schema } from "./datasource";
+import { CachedHTTPStore } from "./zarr";
+import { Schema, Coordinates } from "./datasource";
+import { measureTime } from "./observe";
 
-type ATypedArray =
+export type ATypedArray =
   | Int8Array
   | Int16Array
   | Int32Array
+  | BigInt64Array
   | Uint8Array
   | Uint16Array
   | Uint32Array
+  | BigUint64Array
   | Float32Array
   | Float64Array;
-type Scalar = string | number | boolean;
-type NDArray =
+export type Scalar = string | number | boolean;
+export type NDArray =
   | Scalar[]
   | Scalar[][]
   | Scalar[][][]
@@ -25,7 +31,7 @@ type NDArray =
   | ATypedArray[][]
   | ATypedArray[][][]
   | ATypedArray[][][][];
-type Data = NDArray | ATypedArray | Scalar;
+export type Data = NDArray | ATypedArray | Scalar;
 
 /**
  * Represents a data variable.
@@ -57,11 +63,11 @@ const getShape = (a: Data) => {
   const dim = [] as number[];
   if (!isArray(a)) return dim;
   for (;;) {
-    // @ts-ignore: Scalar already returned
+    // @ts-expect-error: Scalar already returned
     dim.push(a.length);
-    // @ts-ignore: Scalar already returned
+    // @ts-expect-error: Scalar already returned
     if (isArray(a[0])) {
-      // @ts-ignore: Scalar already returned
+      // @ts-expect-error: Scalar already returned
       a = a[0];
     } else {
       break;
@@ -81,7 +87,7 @@ const getDtype = (data: Data): DataType => {
   if (typeof data === "number") {
     return "float32";
   } else if (typeof data === "string") {
-    return "int8";
+    return "v2:object";
   } else {
     switch (data.constructor.name) {
       case "Int8Array":
@@ -90,6 +96,8 @@ const getDtype = (data: Data): DataType => {
         return "int16";
       case "Int32Array":
         return "int32";
+      case "BigInt64Array":
+        return "int64";
       case "Uint8Array":
         return "uint8";
       case "Uint16Array":
@@ -209,15 +217,18 @@ const flatten = (
   return rows;
 };
 
+/** @ignore */
 export type DatameshStore = Location<Listable<AsyncReadable>>;
+/** @ignore */
 export type TempStore = Location<Mutable>;
 
 /**
  * Represents a data variable within a dataset.
  */
 export class DataVar<
+  /** @ignore */
   DType extends DataType,
-  S extends TempStore | DatameshStore
+  S extends TempStore | DatameshStore,
 > {
   /**
    * Creates an instance of DataVar.
@@ -251,7 +262,7 @@ export class DataVar<
    * @param slice - Optional slice parameters to retrieve specific data from the zarr array.
    * @returns A promise that resolves to the data of the zarr array.
    */
-
+  @measureTime
   async get(
     slice?: (null | Slice | number)[] | null | undefined
   ): Promise<Data> {
@@ -259,7 +270,7 @@ export class DataVar<
       this.arr as zarr.Array<DType, AsyncReadable>,
       slice
     );
-    if (_data.shape) {
+    if (this.arr.dtype !== "v2:object" && _data.shape) {
       return unravel(_data.data, _data.shape, _data.stride);
     } else {
       return _data.data as Data;
@@ -271,19 +282,21 @@ export class DataVar<
  * Represents a dataset with dimensions, data variables, and attributes.
  * Implements the DatasetApi interface.
  */
-export class Dataset<S extends DatameshStore | TempStore> {
+export class Dataset</** @ignore */ S extends DatameshStore | TempStore> {
   /**
    * Creates an instance of Dataset.
    * @param dims - The dimensions of the dataset.
    * @param data_vars - The data variables of the dataset.
    * @param attrs - The attributes of the dataset.
    * @param root - The root group of the dataset.
+   * @param coordinates - The coordinates map of the dataset.
    */
   dims: Record<string, number>;
   data_vars: S extends TempStore
     ? Record<string, DataVar<DataType, TempStore>>
     : Record<string, DataVar<DataType, DatameshStore>>;
   attrs: Record<string, unknown>;
+  coordinates: Coordinates;
   root: S;
 
   constructor(
@@ -292,23 +305,26 @@ export class Dataset<S extends DatameshStore | TempStore> {
       ? Record<string, DataVar<DataType, TempStore>>
       : Record<string, DataVar<DataType, DatameshStore>>,
     attrs: Record<string, unknown>,
+    coordinates: Coordinates,
     root: S
   ) {
     this.data_vars = data_vars;
     this.dims = dims;
     this.attrs = attrs;
     this.root = root;
+    this.coordinates = coordinates;
   }
 
   /**
    * Creates a Dataset instance from a Zarr store.
-   * @param gateway - The URL of the datamesh gateway.
+   * @param url - The URL of the datamesh gateway.
    * @param authHeaders - The authentication headers.
    * @param parameters - Optional parameters for the request.
    * @param chunks - Optional chunking strategy.
    * @param downsample - Optional downsampling strategy.
    * @returns A promise that resolves to a Dataset instance.
    */
+  @measureTime
   static async zarr(
     url: string,
     authHeaders: Record<string, string>,
@@ -354,16 +370,67 @@ export class Dataset<S extends DatameshStore | TempStore> {
           });
       }
     }
-    return new Dataset<DatameshStore>(dims, data_vars, group.attrs, root);
+    const coords = JSON.parse(
+      group.attrs["_coordinates"] as string
+    ) as Coordinates;
+    return new Dataset<DatameshStore>(
+      dims,
+      data_vars,
+      group.attrs,
+      coords,
+      root
+    );
+  }
+
+  static async fromArrow(
+    data: Table,
+    coordmap: Coordinates
+  ): Promise<Dataset<TempStore>> {
+    const attrs = {};
+    const dims = { record: data.numRows };
+    const data_vars = {} as Record<string, DataVar<DataType, TempStore>>;
+    data.schema.fields.forEach((field) => {
+      const column = data.getChild(field.name);
+      let array = column?.toArray();
+      if (ArrowDataType.isTimestamp(field.type)) {
+        const carray = new Int32Array(array.length);
+        const m = BigInt(1000 ** (field.type.unit - 1));
+        for (let i = 0; i < array.length; i++) {
+          carray[i] = Number(array[i] / m);
+        }
+        array = carray;
+      } else if (ArrowDataType.isBinary(field.type)) {
+        const carray = [];
+        for (let i = 0; i < array.length; i++) {
+          carray.push(new Buffer(array[i]).toString("base64"));
+        }
+        array = carray;
+      }
+      data_vars[field.name] = {
+        dims: ["record"],
+        attrs: {},
+        data: array,
+      };
+    });
+    return await Dataset.init({ dims, data_vars, attrs }, coordmap);
   }
 
   /**
    * Initializes an in memory Dataset instance from a data object.
    * @param datasource - An object containing id, dimensions, data variables, and attributes.
    */
-  static async init(datasource: Schema): Promise<Dataset<TempStore>> {
+  static async init(
+    datasource: Schema,
+    coordinates?: Coordinates
+  ): Promise<Dataset<TempStore>> {
     const root = zarr.root(new Map());
-    const ds = new Dataset(datasource.dims, {}, datasource.attrs || {}, root);
+    const ds = new Dataset(
+      datasource.dims,
+      {},
+      datasource.attrs || {},
+      coordinates || {},
+      root
+    );
     for (const k in datasource.data_vars) {
       const { dims, attrs, data }: DataVariable = datasource.data_vars[k];
       await ds.assign(k, dims, data as Data, attrs);
@@ -372,7 +439,7 @@ export class Dataset<S extends DatameshStore | TempStore> {
   }
 
   /**
-   * Converts the data variables into a dataframe format.
+   * Converts the dataset into a dataframe format.
    *
    * @returns {Promise<Record<string, unknown>[]>} A promise that resolves to an array of records,
    * where each record represents a row in the dataframe.
@@ -380,23 +447,94 @@ export class Dataset<S extends DatameshStore | TempStore> {
    * @remarks
    * This method iterates over the data variables, retrieves their dimensions and data,
    * and then flattens the data into a dataframe structure.
+   * Time coordinates are converted to IDO8601 format.
+   * BigInt datatypes are coerced to number.
    *
    * @example
    * ```typescript
-   * const dataframe = await instance.to_dataframe();
+   * const dataframe = await instance.asDataframe();
    * console.log(dataframe);
    * ```
    */
-  async to_dataframe(): Promise<Record<string, unknown>[]> {
+  @measureTime
+  async asDataframe(): Promise<Record<string, unknown>[]> {
     const data = {} as Record<string, DataVariable>;
+    const bigint = [];
     for (const k in this.data_vars) {
       data[k] = {
         attrs: this.data_vars[k].attrs,
         dims: this.data_vars[k].dims,
       };
       data[k].data = (await this.data_vars[k].get()) as Data;
+      if (this.data_vars[k].arr.dtype == "int64") {
+        bigint.push(k);
+      }
+    }
+    const df = flatten(data, { ...this.dims }, []);
+    if (this.coordinates.t) {
+      for (let i = 0; i < df.length; i++) {
+        df[i][this.coordinates.t] = new Date(
+          df[i][this.coordinates.t] as number
+        ).toISOString();
+      }
     }
-    return flatten(data, { ...this.dims }, []);
+    //Convert BigInt to number
+    if (bigint.length > 0) {
+      for (const k of bigint) {
+        for (let i = 0; i < df.length; i++) {
+          df[i][k] = Number(df[i][k]);
+        }
+      }
+    }
+    return df;
+  }
+
+  /**
+   * Converts the dataset into a GeoJSON Feature.
+   * @param geometry - Optional GeoJSON geometry to apply to all records, otherwise geometry column is required. Will override geometry column if present.
+   *
+   * @returns {Promise<FeatureCollection>} A promise that resolves to an array of records,
+   * where each record represents a row in the dataframe.
+   *
+   * @throws Will throw an error if no geometry is found in data or as a parameter
+   *
+   * @remarks
+   * This method iterates over the data variables, retrieves their dimensions and data,
+   * and then flattens the data into a dataframe structure.
+   *
+   * @example
+   * ```typescript
+   * const dataframe = await instance.asDataframe();
+   * console.log(dataframe);
+   * ```
+   */
+  async asGeojson(geom?: Geometry): Promise<FeatureCollection> {
+    if (!this.coordinates.g && !geom) {
+      throw new Error("No geometry found");
+    }
+    const features = [] as Feature[];
+    const df = await this.asDataframe();
+    for (let i = 0; i < df.length; i++) {
+      const { ...properties } = df[i];
+      let geometry = geom;
+      if (!geometry && this.coordinates.g) {
+        delete properties[this.coordinates.g];
+        const wkbbuffer = new Buffer(
+          df[i][this.coordinates.g as string],
+          "base64"
+        );
+        geometry = wkx.Geometry.parse(wkbbuffer).toGeoJSON() as Geometry;
+      }
+      features.push({
+        type: "Feature",
+        geometry,
+        properties,
+      });
+    }
+    return {
+      type: "FeatureCollection",
+      features,
+    };
   }
 
   /**
@@ -406,7 +544,7 @@ export class Dataset<S extends DatameshStore | TempStore> {
    * @param dims - An array of dimension names corresponding to the data.
    * @param data - The data to be assigned, which can be a multi-dimensional array.
    * @param attrs - Optional. A record of attributes to be associated with the variable.
-   * @param dtype - The data type of the variable.
+   * @param coordinates - Optional. A record of coordinates to be associated with the variable.
    * @param chunks - Optional. An array specifying the chunk sizes for the data.
    
    * @returns A promise that resolves when the data has been successfully assigned.
@@ -418,6 +556,7 @@ export class Dataset<S extends DatameshStore | TempStore> {
     dims: string[],
     data: Data,
     attrs?: Record<string, unknown>,
+    coordinates?: Coordinates,
     chunks?: number[]
   ): Promise<void> {
     const shape = getShape(data);
@@ -435,13 +574,14 @@ export class Dataset<S extends DatameshStore | TempStore> {
         this.dims[dim] = shape[i];
       }
     });
-
+    const dtype = getDtype(data);
     const arr = await zarr.create(
       this.root.resolve(varid) as Location<Mutable>,
       {
         shape,
-        data_type: getDtype(data),
+        data_type: dtype,
         chunk_shape: chunks || shape,
+        codecs: dtype == "v2:object" ? [{ name: "json2" }] : [],
       }
     );
     await set(

package/src/lib/datasource.ts

@@ -4,30 +4,26 @@ import duration from "dayjs/plugin/duration";
 
 import { DataVariable } from "./datamodel";
 
-/**
- * Represents a data variable.
- */
-enum Coordinate {
-  "Station" = "s", // locations assumed stationary, datasource multigeometry coordinate indexed by station coordinate
-  "Ensemble" = "e",
-  "Raster band" = "b",
-  "Category" = "c",
-  "Quantile" = "q",
-  "Season" = "n",
-  "Month" = "m",
-  "Time" = "t",
-  "Vertical coordinate" = "z",
-  "Horizontal northerly" = "y",
-  "Horizontal easterly" = "x",
-  "Geometry" = "g", // Abstract coordinate - a 2 or 3D geometry that defines a feature location
-  "Frequency" = "f", // spectra
-  "Direction" = "d", // spectra or stats
-  "Coordinate_i" = "i",
-  "Coordinate_j" = "j",
-  "Coordinate_k" = "k",
-}
-
-type Coordinates = {
+export type Coordinate =
+  | "s" // locations assumed stationary, datasource multigeometry coordinate indexed by station coordinate
+  | "e" // Ensemble
+  | "b" // Raster band
+  | "c" // Category
+  | "q" // Quantile
+  | "n" // Season
+  | "m" // Month
+  | "t" // Time
+  | "z" // Vertical coordinate
+  | "y" // Horizontal northerly
+  | "x" // Horizontal easterly
+  | "g" // Abstract coordinate - a 2 or 3D geometry that defines a feature location
+  | "f" // Frequency - spectra
+  | "d" // Direction - spectra or stats
+  | "i" // Coordinate_i
+  | "j" // Coordinate_j
+  | "k"; // Coordinate_k
+
+export type Coordinates = {
   [key in Coordinate]?: string;
 };
 

package/src/lib/observe.ts

@@ -0,0 +1,21 @@
+/** @ignore */
+export function measureTime(
+  target: any,
+  propertyKey: string,
+  descriptor: PropertyDescriptor
+) {
+  const originalMethod = descriptor.value;
+
+  descriptor.value = async function (...args: any[]) {
+    const start = Date.now();
+    const result = await originalMethod.apply(this, args);
+    const end = Date.now();
+    const executionTime = end - start;
+
+    console.debug(`${propertyKey} took ${executionTime}ms`);
+
+    return result;
+  };
+
+  return descriptor;
+}