@oceanum/datamesh 0.1.0

package/src/lib/query.ts

@@ -0,0 +1,236 @@
+import { Feature } from "geojson";
+import dayjs from "dayjs";
+import duration from "dayjs/plugin/duration";
+
+dayjs.extend(duration);
+
+/**
+ * GeoFilterType enum representing types of geofilters.
+ */
+enum GeoFilterType {
+  Feature = "feature",
+  Bbox = "bbox",
+}
+
+/**
+ * GeoFilterInterp enum representing interpolation methods for geofilters.
+ */
+enum GeoFilterInterp {
+  Nearest = "nearest",
+  Linear = "linear",
+}
+
+/**
+ * LevelFilterInterp enum representing interpolation methods for level filters.
+ */
+enum LevelFilterInterp {
+  Nearest = "nearest",
+  Linear = "linear",
+}
+
+/**
+ * TimeFilterType enum representing types of time filters.
+ */
+enum TimeFilterType {
+  Range = "range",
+  Series = "series",
+  Trajectory = "trajectory",
+}
+
+/**
+ * LevelFilterType enum representing types of level filters.
+ */
+enum LevelFilterType {
+  Range = "range",
+  Series = "series",
+}
+
+/**
+ * ResampleType enum representing types of resampling.
+ */
+enum ResampleType {
+  Mean = "mean",
+  Nearest = "nearest",
+  Slinear = "linear",
+}
+
+/**
+ * AggregateOps enum representing aggregation operations.
+ */
+enum AggregateOps {
+  Mean = "mean",
+  Min = "min",
+  Max = "max",
+  Std = "std",
+  Sum = "sum",
+}
+
+/**
+ * Container enum representing data container types.
+ */
+enum Container {
+  GeoDataFrame = "geodataframe",
+  DataFrame = "dataframe",
+  Dataset = "dataset",
+}
+
+/**
+ * GeoFilter type representing a spatial subset or interpolation.
+ */
+export type GeoFilter = {
+  type: GeoFilterType;
+  geom: Array<number[]> | Feature;
+  interp?: GeoFilterInterp;
+  resolution?: number;
+  alltouched?: boolean;
+};
+
+/**
+ * LevelFilter type representing a vertical subset or interpolation.
+ */
+type LevelFilter = {
+  type: LevelFilterType;
+  levels: Array<number | null>;
+  interp?: LevelFilterInterp;
+};
+
+/**
+ * TimeFilter type representing a temporal subset or interpolation.
+ */
+export type TimeFilter = {
+  type?: TimeFilterType;
+  times: Array<Date | dayjs.Dayjs | duration.Duration | string>;
+  resolution?: string;
+  resample?: ResampleType;
+};
+
+const stringifyTime = (
+  t: Date | dayjs.Dayjs | duration.Duration | string
+): string => {
+  if (t instanceof Date) {
+    return dayjs(t as Date).toISOString();
+  } else if (t instanceof dayjs) {
+    return (t as dayjs.Dayjs).toISOString();
+  } else if (t instanceof dayjs.duration) {
+    return (t as duration.Duration).toISOString();
+  } else {
+    try {
+      return dayjs.duration(t as string).toISOString();
+    } catch {
+      return dayjs(t as string).toISOString();
+    }
+  }
+};
+
+const timeFilterValidate = (timefilter: TimeFilter): TimeFilter => {
+  const times = timefilter.times.map((t) => stringifyTime(t));
+
+  return {
+    type: timefilter.type || TimeFilterType.Range,
+    times,
+    resolution: timefilter.resolution,
+    resample: timefilter.resample,
+  };
+};
+
+/**
+ * Aggregate type representing aggregation operations.
+ */
+type Aggregate = {
+  operations: AggregateOps[];
+  spatial?: boolean;
+  temporal?: boolean;
+};
+
+/**
+ * CoordSelector type representing coordinate selection.
+ */
+type CoordSelector = {
+  coord: string;
+  values: Array<string | number>;
+};
+
+/**
+ * Query interface representing a Datamesh query.
+ */
+export interface IQuery {
+  datasource: string;
+  parameters?: Record<string, number | string | number[] | string[]>;
+  description?: string;
+  variables?: string[];
+  timefilter?: TimeFilter;
+  geofilter?: GeoFilter;
+  levelfilter?: LevelFilter;
+  coordfilter?: CoordSelector[];
+  crs?: string | number;
+  aggregate?: Aggregate;
+  limit?: number;
+  id?: string;
+}
+
+/**
+ * Stage interface representing the result of staging a query.
+ */
+export type Stage = {
+  query: Query;
+  qhash: string;
+  formats: string[];
+  size: number;
+  dlen: number;
+  coordmap: Record<string, string>;
+  container: Container;
+  sig: string;
+};
+
+/**
+ * Query class representing a Datamesh query.
+ */
+export class Query implements IQuery {
+  datasource: string;
+  parameters?: Record<string, number | string | number[] | string[]>;
+  description?: string;
+  variables?: string[];
+  timefilter?: TimeFilter;
+  geofilter?: GeoFilter;
+  levelfilter?: LevelFilter;
+  coordfilter?: CoordSelector[];
+  crs?: string | number;
+  aggregate?: Aggregate;
+  limit?: number;
+  id?: string;
+
+  constructor(query: IQuery) {
+    this.datasource = query.datasource;
+    this.parameters = query.parameters;
+    this.description = query.description;
+    this.variables = query.variables;
+    this.timefilter = query.timefilter && timeFilterValidate(query.timefilter);
+    this.geofilter = query.geofilter;
+    this.levelfilter = query.levelfilter;
+    this.coordfilter = query.coordfilter;
+    this.crs = query.crs;
+    this.aggregate = query.aggregate;
+    this.limit = query.limit;
+    this.id = query.id;
+  }
+
+  /**
+   * Returns the query as a JSON object.
+   */
+  toJSON(): Record<string, unknown> {
+    return {
+      datasource: this.datasource,
+      parameters: this.parameters,
+      description: this.description,
+      variables: this.variables,
+      timefilter: this.timefilter,
+      geofilter: this.geofilter,
+      levelfilter: this.levelfilter,
+      coordfilter: this.coordfilter,
+      crs: this.crs,
+      aggregate: this.aggregate,
+      limit: this.limit,
+      id: this.id,
+    };
+  }
+}

package/src/lib/workers/sw.js

@@ -0,0 +1,44 @@
+/*
+Service worker for interacting with the Datamesh metadata server
+*/
+
+const addResourcesToCache = async (resources) => {
+  const cache = await caches.open("datamesh");
+  await cache.addAll(resources);
+};
+
+const putInCache = async (request, response) => {
+  const cache = await caches.open("datamesh");
+  await cache.put(request, response);
+};
+
+const cacheFirst = async ({ request }) => {
+  // First try to get the resource from the cache
+  const responseFromCache = await caches.match(request);
+  if (responseFromCache) {
+    return responseFromCache;
+  }
+
+  // Next try to get the resource from the network
+  try {
+    const responseFromNetwork = await fetch(request.clone());
+    // response may be used only once
+    // we need to save clone to put one copy in cache
+    // and serve second one
+    putInCache(request, responseFromNetwork.clone());
+    return responseFromNetwork;
+  } catch (error) {
+    return new Response("Datamesh error: " + error.message, {
+      status: 408,
+      headers: { "Content-Type": "text/plain" },
+    });
+  }
+};
+
+self.addEventListener("fetch", (event) => {
+  event.respondWith(
+    cacheFirst({
+      request: event.request,
+    })
+  );
+});

package/src/lib/zarr.ts

@@ -0,0 +1,112 @@
+import {
+  set as set_cache,
+  get as get_cache,
+  del as del_cache,
+  createStore,
+  UseStore,
+} from "idb-keyval";
+import hash from "object-hash";
+import { AsyncReadable, AsyncMutable, AbsolutePath } from "@zarrita/storage";
+
+function delay(t: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, t));
+}
+
+export class CachedHTTPStore implements AsyncReadable {
+  cache: UseStore | undefined;
+  url: string;
+  cache_prefix: string;
+  fetchOptions: RequestInit;
+  constructor(
+    root: string,
+    authHeaders: Record<string, string>,
+    parameters?: Record<string, string | number>,
+    chunks?: string,
+    downsample?: Record<string, number>,
+    nocache?: boolean
+  ) {
+    const headers = { ...authHeaders };
+    if (parameters) headers["x-parameters"] = JSON.stringify(parameters);
+    if (chunks) headers["x-chunks"] = chunks;
+    if (downsample) headers["x-downsample"] = JSON.stringify(downsample);
+    headers["x-filtered"] = "True";
+    this.fetchOptions = { headers };
+    this.url = root;
+    if (nocache) {
+      this.cache = undefined;
+    } else {
+      this.cache = createStore("zarr", "cache");
+    }
+    this.cache_prefix = hash({ ...parameters, chunks, downsample });
+  }
+
+  async get(
+    item: string,
+    options?: RequestInit,
+    retry = 0
+  ): Promise<Uint8Array | undefined> {
+    const key = `${this.cache_prefix}${item}`;
+    let data = null;
+    if (this.cache) {
+      data = await get_cache(key, this.cache);
+      if (data === "pending") {
+        await delay(200);
+        if (retry > 5 * 60) {
+          await del_cache(key, this.cache);
+          throw new Error("Zarr timeout");
+        } else {
+          return await this.get(item, options, retry + 1);
+        }
+      }
+    }
+    if (!data || !this.cache) {
+      if (this.cache) set_cache(key, "pending", this.cache);
+      //console.log(`${this.url}/${item}`);
+      //console.log(this.fetchOptions.headers);
+      const response = await fetch(`${this.url}${item}`, {
+        ...this.fetchOptions,
+        ...options,
+      });
+
+      if (response.status === 404) {
+        // Item is not found
+        if (this.cache) await del_cache(key, this.cache);
+        return undefined;
+      } else if (response.status !== 200) {
+        // Item is found but there was an error
+        if (this.cache) await del_cache(key, this.cache);
+        throw new Error(String(response.status));
+      }
+      data = new Uint8Array(await response.arrayBuffer());
+      if (this.cache) await set_cache(key, data, this.cache);
+    }
+    return data;
+  }
+}
+
+export class IDBStore implements AsyncMutable {
+  cache: UseStore;
+  constructor(public root: string) {
+    this.cache = createStore(root, "store");
+  }
+
+  async get(key: AbsolutePath): Promise<Uint8Array | undefined> {
+    try {
+      return await get_cache(key, this.cache);
+    } catch {
+      return undefined;
+    }
+  }
+
+  async has(key: AbsolutePath): Promise<boolean> {
+    return (await get_cache(key, this.cache)) !== undefined;
+  }
+
+  async set(key: AbsolutePath, value: Uint8Array): Promise<void> {
+    await set_cache(key, value, this.cache);
+  }
+
+  async delete(key: AbsolutePath): Promise<void> {
+    await del_cache(key, this.cache);
+  }
+}

package/src/test/dataset.test.ts

@@ -0,0 +1,60 @@
+import { assertType, test, expect } from "vitest";
+import { Dataset, IDataVar } from "../lib/datamodel";
+import { Connector } from "../lib/connector";
+import { dataset, datameshTest, DATAMESH_GATEWAY, HEADERS } from "./fixtures";
+
+test("dataset init", async () => {
+  const dstest = await Dataset.init(dataset);
+  assertType<Record<string, unknown>>(dstest.attrs);
+  assertType<Record<string, IDataVar>>(dstest.data_vars);
+  const datatest = await dstest.data_vars.temperature.get();
+  expect(datatest).toBeInstanceOf(Array);
+  expect(datatest.length).toBe(10);
+  expect(datatest[0].length).toBe(30);
+  expect(datatest[0][0].length).toBe(20);
+  expect(datatest[3][4][5]).toEqual(
+    dataset.data_vars.temperature.data[3][4][5]
+  );
+  const datatest0 = await dstest.data_vars.scalar.get();
+  expect(datatest0[0]).closeTo(10.1, 0.0001);
+});
+
+datameshTest(
+  "dataset zarr",
+  async ({ dataset }) => {
+    //Test the zarr proxy endpoint directly
+    const dstest = await Dataset.zarr(
+      DATAMESH_GATEWAY + "/zarr/" + dataset.attrs.id,
+      HEADERS
+    );
+    assertType<Record<string, unknown>>(dstest.attrs);
+    assertType<Record<string, IDataVar>>(dstest.data_vars);
+    let datatest = await dstest.data_vars.temperature.get();
+    expect(datatest).toBeInstanceOf(Array);
+    expect(datatest.length).toBe(10);
+    expect(datatest[0].length).toBe(30);
+    expect(datatest[0][0].length).toBe(20);
+    expect(datatest[3][4][5]).toEqual(
+      dataset.data_vars.temperature.data[3][4][5]
+    );
+    datatest = await dstest.data_vars.scalar.get();
+    expect(datatest[0]).closeTo(10.1, 0.0001);
+
+    //Now test with the connector
+    const datamesh = new Connector(process.env.DATAMESH_TOKEN);
+    const dstest2 = await datamesh.loadDatasource(dataset.attrs.id);
+    assertType<Record<string, unknown>>(dstest2.attrs);
+    assertType<Record<string, IDataVar>>(dstest2.data_vars);
+    datatest = await dstest.data_vars.temperature.get();
+    expect(datatest).toBeInstanceOf(Array);
+    expect(datatest.length).toBe(10);
+    expect(datatest[0].length).toBe(30);
+    expect(datatest[0][0].length).toBe(20);
+    expect(datatest[3][4][5]).toEqual(
+      dataset.data_vars.temperature.data[3][4][5]
+    );
+    datatest = await dstest.data_vars.scalar.get();
+    expect(datatest[0]).closeTo(10.1, 0.0001);
+  },
+  { timeout: 100000 }
+);

package/src/test/datasource.test.ts

@@ -0,0 +1,28 @@
+import { assertType, test, expect } from "vitest";
+import { Datasource } from "../lib/datasource";
+import Connector from "../lib/connector";
+import { datameshTest } from "./fixtures";
+
+test("datasource type", () => {
+  const datasource: Datasource = {
+    id: "1",
+    name: "test",
+    description: "test",
+    schema: {
+      attrs: {},
+      dims: {},
+      data_vars: {},
+    },
+  };
+  assertType<Record<string, unknown>>(datasource.schema.attrs);
+  assertType<Record<string, unknown>>(datasource.schema.dims);
+  assertType<Record<string, unknown>>(datasource.schema.data_vars);
+});
+
+datameshTest("datasource_metadata", async ({ metadata }) => {
+  const connector = new Connector(process.env.DATAMESH_TOKEN);
+  const datasource = await connector.getDatasource(metadata.id);
+  assertType<Datasource>(datasource);
+  expect(datasource.id).toBe(metadata.id);
+  expect(datasource.name).toBe(metadata.name);
+});

package/src/test/fixtures.ts

@@ -0,0 +1,169 @@
+/// <reference lib="dom" />
+
+import { test } from "vitest";
+import { Datasource, Schema } from "../lib/datasource";
+
+const DATAMESH_TOKEN: string = process.env.DATAMESH_TOKEN || "$DATAMESH_TOKEN";
+export const HEADERS: HeadersInit = {
+  Authorization: `Token ${DATAMESH_TOKEN}`,
+  "X-DATAMESH-TOKEN": DATAMESH_TOKEN,
+  Accept: "application/json",
+  "Content-Type": "application/json",
+};
+export const DATAMESH_SERVICE =
+  process.env.DATAMESH_SERVICE || "https://datamesh.oceanum.io";
+export const DATAMESH_GATEWAY =
+  process.env.DATAMESH_GATEWAY || "https://gateway.datamesh.oceanum.io";
+
+const datasource: Datasource = {
+  id: "datamesh-js-test",
+  name: "test",
+  description: "datamesh-js test registration",
+  schema: {
+    attrs: {},
+    dims: {},
+    data_vars: {},
+  },
+  coordinates: { t: "time" },
+  driver: "onzarr",
+};
+
+//Create multidemensional random array of typed data
+const createFloatArray = (
+  dims: number[],
+  depth = 0,
+  typed = false
+): number[] | Float32Array | Float32Array[] => {
+  const size = dims[depth];
+  const array =
+    typed && depth == dims.length - 1
+      ? new Float32Array(size)
+      : new Array(size);
+  if (depth === dims.length - 1) {
+    for (let i = 0; i < size; i++) {
+      array[i] = Math.random();
+    }
+  } else {
+    for (let i = 0; i < size; i++) {
+      array[i] = createFloatArray(dims, depth + 1, typed);
+    }
+  }
+  return array;
+};
+
+const jsonify = (data: Record<string, unknown>): string => {
+  return JSON.stringify(data, function (key, value) {
+    if (
+      value instanceof Int8Array ||
+      value instanceof Uint8Array ||
+      value instanceof Uint8ClampedArray ||
+      value instanceof Int16Array ||
+      value instanceof Uint16Array ||
+      value instanceof Int32Array ||
+      value instanceof Uint32Array ||
+      value instanceof Float32Array ||
+      value instanceof Float64Array
+    ) {
+      return Array.from(value);
+    }
+    return value;
+  });
+};
+
+const scalar = new Float32Array(1);
+scalar[0] = 10.1;
+export const dataset: Schema = {
+  dims: { one: 1, time: 10, lon: 20, lat: 30 },
+  coords: {
+    time: {
+      dims: ["time"],
+      attrs: { units: "unix timestamp" },
+      data: [...Array(10).keys()].map((i) => 86400000 * i),
+    },
+    lon: {
+      dims: ["lon"],
+      attrs: { units: "degrees east" },
+      data: [...Array(20).keys()].map((i) => i),
+    },
+    lat: {
+      dims: ["lat"],
+      attrs: { units: "degrees north" },
+      data: [...Array(30).keys()].map((i) => -i),
+    },
+  },
+  data_vars: {
+    temperature: {
+      dims: ["time", "lat", "lon"],
+      attrs: { units: "C" },
+      data: createFloatArray([10, 30, 20], 0, true),
+    },
+    elevation: {
+      dims: ["lat", "lon"],
+      attrs: { units: "m" },
+      data: createFloatArray([30, 20], 0, false),
+    },
+    scalar: {
+      dims: ["one"],
+      attrs: { units: "m" },
+      data: scalar,
+    },
+  },
+};
+
+export const datameshTest = test.extend({
+  metadata: async ({}, use: Function) => {
+    // setup the fixture before each test function
+    console.log(HEADERS);
+    const resp = await fetch(DATAMESH_SERVICE + "/datasource/", {
+      method: "POST",
+      headers: HEADERS,
+      body: JSON.stringify(datasource),
+    });
+    if (resp.status !== 201) {
+      const text = await resp.text();
+      throw new Error("Failed to register datasource: " + text);
+    }
+
+    // use the fixture value
+    await use(datasource);
+
+    // cleanup the fixture after each test function
+    await fetch(DATAMESH_SERVICE + "/datasource/" + datasource.id, {
+      method: "DELETE",
+      headers: HEADERS,
+    });
+  },
+  dataset: async ({}, use: Function) => {
+    // setup the fixture before each test function
+
+    let resp = await fetch(DATAMESH_GATEWAY + "/data/oceanum-js-test/", {
+      method: "PUT",
+      headers: HEADERS,
+      body: jsonify(dataset),
+    });
+    if (resp.status !== 200) {
+      throw new Error("Failed to write dataset");
+    }
+    const patch = jsonify({
+      coordinates: { t: "time", x: "lon", y: "lat" },
+    });
+    resp = await fetch(DATAMESH_SERVICE + "/datasource/oceanum-js-test/", {
+      method: "PATCH",
+      headers: HEADERS,
+      body: patch,
+    });
+    if (resp.status !== 200) {
+      throw new Error("Failed to register dataset");
+    }
+    dataset.attrs = { id: "oceanum-js-test" };
+
+    // use the fixture value
+    await use(dataset);
+
+    // cleanup the fixture after each test function
+    await fetch(DATAMESH_GATEWAY + "/data/oceanum-js-test", {
+      method: "DELETE",
+      headers: HEADERS,
+    });
+  },
+});

package/src/test/query.test.ts

@@ -0,0 +1,49 @@
+import { assertType, test, expect } from "vitest";
+import { Dataset } from "../lib/datamodel";
+import { Connector } from "../lib/connector";
+import { dataset, datameshTest } from "./fixtures";
+
+test("datasource type", () => {
+  const query: IQuery = {
+    datasource: "test",
+    geofilter: {
+      type: "feature",
+      geom: {
+        type: "Feature",
+        geometry: {
+          type: "Point",
+          coordinates: [10, -10],
+        },
+      },
+    },
+    variables: ["temperature"],
+  };
+  assertType<Record<string, unknown>>(query.geofilter);
+});
+
+datameshTest(
+  "datamesh query",
+  async ({ dataset }) => {
+    const datamesh = new Connector(process.env.DATAMESH_TOKEN);
+    const query = {
+      datasource: dataset.attrs.id,
+      geofilter: {
+        type: "feature",
+        geom: {
+          type: "Feature",
+          geometry: {
+            type: "Point",
+            coordinates: [10, -10],
+          },
+        },
+      },
+    };
+    const dstest = await datamesh.query(query);
+    assertType<Dataset>(dstest);
+    const datatest = await dstest.data_vars.temperature.get();
+    expect(datatest).toBeInstanceOf(Float64Array);
+    expect(datatest.length).toBe(10);
+    expect(datatest[5]).toEqual(dataset.data_vars.temperature.data[5][10][10]);
+  },
+  { timeout: 100000 }
+);