@oceanum/datamesh 0.4.3 → 0.5.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@oceanum/datamesh",
-  "version": "0.4.3",
+  "version": "0.5.1",
   "scripts": {
     "build:docs": "typedoc"
   },

package/proxy/cloudflare/index.js

@@ -0,0 +1,42 @@
+//Example cloudfalre worker reverse proxy
+//Add your datamesh token as a secret in the cloudflare worker environment
+
+const DATAMESH = "https://datamesh-v1.oceanum.io";
+
+export default {
+  async fetch(request, env, ctx) {
+    const url = new URL(request.url);
+    const DATAMESH_TOKEN = env.DATAMESH_TOKEN;
+
+    // Extract the path and construct the datamesh API URL
+    const datameshUrl = new URL(url.pathname + url.search, DATAMESH);
+
+    // Clone the request to modify headers
+    const modifiedRequest = new Request(datameshUrl.toString(), {
+      method: request.method,
+      headers: request.headers,
+      body: request.body,
+    });
+
+    // Inject the DATAMESH_TOKEN header
+    modifiedRequest.headers.set("x-DATAMESH-TOKEN", `${DATAMESH_TOKEN}`);
+
+    // Forward the request to datamesh
+    const response = await fetch(modifiedRequest);
+
+    // Return the response with CORS headers
+    const modifiedResponse = new Response(response.body, {
+      status: response.status,
+      statusText: response.statusText,
+      headers: {
+        ...response.headers,
+        "Access-Control-Allow-Origin": "*",
+        "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
+        "Access-Control-Allow-Headers":
+          "Content-Type, Authorization, X-DATAMESH-TOKEN",
+      },
+    });
+
+    return modifiedResponse;
+  },
+};

package/proxy/express/README.md

@@ -0,0 +1,54 @@
+# Node Express Proxy (Example)
+
+Example reverse proxy for Oceanum Datamesh using Node.js + Express.
+
+## Prerequisites
+
+- Node.js v18+ (native `fetch` and web streams)
+- Environment variable `DATAMESH_TOKEN` set to your Datamesh token
+- Optional: `DATAMESH_URL` (defaults to `https://datamesh.oceanum.io`)
+- Optional: `PORT` (defaults to `8787`)
+
+## Install and run
+
+Create a small project in this directory and install Express:
+
+```sh
+npm init -y
+npm install express
+```
+
+Run the proxy:
+
+```sh
+DATAMESH_TOKEN=your_token_here node index.js
+# or with a custom upstream and port
+DATAMESH_TOKEN=your_token_here DATAMESH_URL=https://datamesh.oceanum.io PORT=8080 node index.js
+```
+
+## What it does
+
+- Forwards all incoming requests to `DATAMESH_URL`.
+- Injects/overwrites the `x-DATAMESH-TOKEN` header with your secret token.
+- Adds permissive CORS headers for browser apps.
+
+## Use with @oceanum/datamesh
+
+Point both `service` and `gateway` to your proxy origin:
+
+```ts
+import { Connector } from "@oceanum/datamesh";
+
+const PROXY_URL = "http://localhost:8787"; // or your deployed proxy
+
+const connector = new Connector("proxy", {
+  service: PROXY_URL,
+  gateway: PROXY_URL,
+});
+```
+
+## Security notes
+
+- Never commit your real token. Use env vars or a secret store.
+- Consider tightening CORS and limiting allowed origins in production.
+- This is an example; adapt logging/error handling and header allowlists to your needs.

package/proxy/express/index.js

@@ -0,0 +1,126 @@
+/*
+ Example Node Express reverse proxy for Oceanum Datamesh
+
+ Requirements
+ - Node.js v18+ (for native fetch and web streams)
+ - Environment variable DATAMESH_TOKEN set to your Datamesh token
+ - Optional: DATAMESH_URL (defaults to https://datamesh.oceanum.io)
+ - Optional: PORT (defaults to 8787)
+
+ Usage
+   DATAMESH_TOKEN=xxxx npm start
+   # or
+   DATAMESH_TOKEN=xxxx node index.js
+*/
+
+const express = require("express");
+
+const app = express();
+const PORT = process.env.PORT || 8787;
+const TARGET = process.env.DATAMESH_URL || "https://datamesh.oceanum.io";
+
+if (!process.env.DATAMESH_TOKEN) {
+  console.warn(
+    "Warning: DATAMESH_TOKEN is not set. Requests to Datamesh will fail with 401/403."
+  );
+}
+
+// Basic CORS support (adjust for production as needed)
+app.use((req, res, next) => {
+  res.setHeader("Access-Control-Allow-Origin", "*");
+  res.setHeader(
+    "Access-Control-Allow-Methods",
+    "GET, POST, PUT, DELETE, OPTIONS"
+  );
+  res.setHeader(
+    "Access-Control-Allow-Headers",
+    "Content-Type, Authorization, x-requested-with, x-datamesh-token"
+  );
+  if (req.method === "OPTIONS") {
+    return res.sendStatus(204);
+  }
+  next();
+});
+
+// Proxy all routes to Datamesh, injecting the token
+app.use(async (req, res) => {
+  try {
+    const upstreamUrl = new URL(req.originalUrl, TARGET);
+
+    // Rebuild headers to a fresh Headers instance
+    const headers = new Headers();
+    for (const [key, value] of Object.entries(req.headers)) {
+      if (typeof value === "string") headers.set(key, value);
+      else if (Array.isArray(value)) headers.set(key, value.join(", "));
+    }
+
+    // Ensure correct Host and token header (overwrite any client-supplied values)
+    headers.delete("host");
+    headers.set("x-DATAMESH-TOKEN", process.env.DATAMESH_TOKEN || "");
+
+    // Prepare request init
+    const method = req.method.toUpperCase();
+    const hasBody = !["GET", "HEAD"].includes(method);
+    const init = {
+      method,
+      headers,
+      body: hasBody ? req : undefined, // Stream the incoming request body
+    };
+
+    const response = await fetch(upstreamUrl, init);
+
+    // Forward headers from upstream
+    response.headers.forEach((value, key) => {
+      // Skip hop-by-hop or problematic headers if necessary
+      if (key.toLowerCase() === "transfer-encoding") return;
+      res.setHeader(key, value);
+    });
+
+    // We already added CORS above; ensure it's present in case upstream overrides
+    res.setHeader("Access-Control-Allow-Origin", "*");
+    res.setHeader(
+      "Access-Control-Allow-Methods",
+      "GET, POST, PUT, DELETE, OPTIONS"
+    );
+    res.setHeader(
+      "Access-Control-Allow-Headers",
+      "Content-Type, Authorization, x-requested-with"
+    );
+
+    // Status
+    res.status(response.status);
+
+    // Stream body
+    if (response.body) {
+      response.body
+        .pipeTo(
+          new WritableStream({
+            write(chunk) {
+              res.write(Buffer.from(chunk));
+            },
+            close() {
+              res.end();
+            },
+            abort(err) {
+              console.error("Proxy stream aborted:", err);
+              res.end();
+            },
+          })
+        )
+        .catch((err) => {
+          console.error("Proxy piping error:", err);
+          res.end();
+        });
+    } else {
+      res.end();
+    }
+  } catch (err) {
+    console.error("Proxy error:", err);
+    res.status(502).json({ detail: "Proxy error", error: String(err) });
+  }
+});
+
+app.listen(PORT, () => {
+  console.log(`Datamesh proxy listening on http://localhost:${PORT}`);
+  console.log(`Forwarding to ${TARGET}`);
+});

package/proxy/guide.md

@@ -0,0 +1,140 @@
+---
+title: Proxy Guide
+group: Documents
+category: Guides
+---
+
+# Using a Datamesh Proxy with @oceanum/datamesh
+
+A reverse proxy lets you call Datamesh from a public web app without exposing your Datamesh token and helps you avoid CORS issues.
+
+This repo includes example proxies you can deploy quickly, and shows how to point the `Connector` to them.
+
+- Cloudflare Worker: `packages/datamesh/proxy/cloudflare/index.js`
+- Node/Express: `packages/datamesh/proxy/express/index.js`
+- Client package: `@oceanum/datamesh`
+
+## Why use a proxy?
+
+- Protect secrets: keep your Datamesh token server-side.
+- Simpler CORS: proxy can add permissive CORS headers.
+- Stable domain: front your app with your own domain.
+
+## Cloudflare Worker proxy (example)
+
+The example Cloudflare Worker:
+
+- Forwards all requests to the Datamesh API.
+- Overwrites the `x-DATAMESH-TOKEN` header using a Worker Secret.
+- Adds permissive CORS headers for browser apps.
+
+```js
+// Example Cloudflare Worker reverse proxy
+// Add your datamesh token as a secret in the Cloudflare worker environment
+
+const DATAMESH = "https://datamesh.oceanum.io";
+
+export default {
+  async fetch(request, env) {
+    const url = new URL(request.url);
+    const DATAMESH_TOKEN = env.DATAMESH_TOKEN;
+
+    // Build the upstream request URL
+    const datameshUrl = new URL(url.pathname + url.search, DATAMESH);
+
+    // Clone request and forward body/headers/method
+    const modifiedRequest = new Request(datameshUrl.toString(), {
+      method: request.method,
+      headers: request.headers,
+      body: request.body,
+    });
+
+    // Inject/overwrite the token
+    modifiedRequest.headers.set("x-DATAMESH-TOKEN", `${DATAMESH_TOKEN}`);
+
+    // Forward
+    const response = await fetch(modifiedRequest);
+
+    // Add CORS headers for the browser
+    return new Response(response.body, {
+      status: response.status,
+      statusText: response.statusText,
+      headers: {
+        ...response.headers,
+        "Access-Control-Allow-Origin": "*",
+        "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS",
+        "Access-Control-Allow-Headers":
+          "Content-Type, Authorization, x-requested-with, x-datamesh-token",
+      },
+    });
+  },
+};
+```
+
+### Deploy on Cloudflare
+
+1. Create a new Worker in the Cloudflare dashboard.
+2. Paste the contents of `packages/datamesh/proxy/cloudflare/index.js`.
+3. Add a Worker Secret named `DATAMESH_TOKEN` and set it to your Datamesh token.
+4. (Optional) Change `DATAMESH` to point at a different upstream if needed.
+5. Deploy, and note your Worker URL, e.g. `https://your-proxy.workers.dev`.
+
+### Local testing tips
+
+- You can test the Worker locally with `wrangler dev`.
+- If you front the Worker with your own domain, ensure HTTPS is enabled and the domain is added to your app’s allowed origins if you use restrictive CORS elsewhere.
+
+## Node/Express proxy (example)
+
+You can also run a simple Node/Express reverse proxy locally or deploy it to your own infrastructure.
+
+- Example: `packages/datamesh/proxy/express/index.js`
+
+### Run locally
+
+1. Install dependencies in the example folder:
+
+   ```sh
+   cd packages/datamesh/proxy/express
+   npm init -y
+   npm install express
+   ```
+
+2. Start the proxy:
+
+   ```sh
+   DATAMESH_TOKEN=your_token_here node index.js
+   # Optional:
+   # DATAMESH_URL=https://datamesh.oceanum.io PORT=8787 DATAMESH_TOKEN=your_token_here node index.js
+   ```
+
+The proxy will listen on `http://localhost:8787` by default and forward all requests to `DATAMESH_URL`, injecting `x-DATAMESH-TOKEN` and adding permissive CORS headers.
+
+## Configure @oceanum/datamesh to use the proxy
+
+Point both `service` and `gateway` to your proxy origin. The proxy injects the token, so you can pass any non-empty string for the required `token` parameter.
+
+```ts
+import { Connector } from "@oceanum/datamesh";
+
+const PROXY_URL = "https://your-proxy.workers.dev"; // or your custom domain
+
+const connector = new Connector("proxy", {
+  service: PROXY_URL,
+  gateway: PROXY_URL,
+  // Optional: tweak caching and session duration as needed
+  // nocache: true,
+  // sessionDuration: 1,
+});
+```
+
+Notes:
+
+- The constructor requires a `token`. When using the proxy, the token you pass here is ignored by the upstream because the proxy overwrites the `x-DATAMESH-TOKEN` header with the secret.
+- The connector will probe `GET /session` on the `gateway` to detect the API version. Ensure your proxy forwards that path.
+
+## Security considerations
+
+- Never commit your Datamesh token. Store it as a Worker Secret (or equivalent secret store) in your hosting platform.
+- Consider limiting origins or tightening CORS in production if your app does not need broad access.
+- Audit which headers you forward. The example purposely overwrites `x-DATAMESH-TOKEN` to prevent client-supplied values from leaking upstream.

package/src/lib/connector.ts

@@ -12,7 +12,9 @@ import { Session } from "./session";
  *
  */
 const DATAMESH_SERVICE =
-  process.env.DATAMESH_SERVICE || "https://datamesh.oceanum.io";
+  typeof process !== "undefined" && process.env?.DATAMESH_SERVICE
+    ? process.env.DATAMESH_SERVICE
+    : "https://datamesh.oceanum.io";
 
 export class Connector {
   static LAZY_LOAD_SIZE = 1e8;
@@ -41,7 +43,9 @@ export class Connector {
    * @throws {Error} - If a valid token is not provided.
    */
   constructor(
-    token = process.env.DATAMESH_TOKEN || "$DATAMESH_TOKEN",
+    token = typeof process !== "undefined" && process.env?.DATAMESH_TOKEN
+      ? process.env.DATAMESH_TOKEN
+      : "$DATAMESH_TOKEN",
     options?: {
       service?: string;
       gateway?: string;
@@ -70,8 +74,7 @@ export class Connector {
         };
 
     /* This is for testing  the gateway service is not always the same as the service domain */
-    this._gateway =
-      options?.gateway || `${url.protocol}//gateway.${url.hostname}`;
+    this._gateway = options?.gateway || this._host;
 
     if (
       this._host.split(".").slice(-1)[0] !==
@@ -308,19 +311,9 @@ export class Connector {
       const dataset = await Dataset.fromArrow(table, stage.coordkeys);
       return dataset;
     }
-    let url = null;
-    let params = undefined;
-    if (
-      query.timefilter ||
-      query.geofilter ||
-      query.levelfilter ||
-      query.coordfilter
-    ) {
-      url = `${this._gateway}/zarr/${stage.qhash}`;
-    } else {
-      url = `${this._gateway}/zarr/${query.datasource}`;
-      params = query.parameters;
-    }
+
+    const url = `${this._gateway}/zarr/${this._isV1 ? "query/" : ""}${stage.qhash}`;
+    const params = query.parameters;
 
     // Get headers with session information if available
     const headers = await this.getSessionHeaders();

package/src/lib/datamodel.ts

@@ -15,7 +15,7 @@ import { Geometry, Feature, FeatureCollection } from "geojson";
 import { Geometry as WkxGeometry } from "wkx-ts";
 import { Buffer } from "buffer/index";
 
-import { CachedHTTPStore } from "./zarr";
+import { CachedHTTPStore, zarr_open_v2_datetime } from "./zarr";
 import { Schema, Coordkeys } from "./datasource";
 import { measureTime } from "./observe";
 
@@ -163,7 +163,7 @@ const ravel = (data: Data) => {
   }
 };
 
-function get_strides(shape: readonly number[]) {
+const get_strides = (shape: readonly number[]) => {
   const ndim = shape.length;
   const stride: number[] = Array(ndim);
   for (let i = ndim - 1, step = 1; i >= 0; i--) {
@@ -171,14 +171,14 @@ function get_strides(shape: readonly number[]) {
     step *= shape[i];
   }
   return stride;
-}
+};
 
-function unravel<T extends DataType>(
+const unravel = <T extends DataType>(
   data: TypedArray<T>,
   shape: number[],
   stride: number[],
   offset = 0
-): Data {
+): Data => {
   // @ts-expect-error: Is array
   if (shape.length === 0) return data[0];
   if (shape.length === 1) {
@@ -199,7 +199,34 @@ function unravel<T extends DataType>(
     );
   }
   return arr;
-}
+};
+
+const npdatetime_to_posixtime = (data: Chunk<DataType>, dtype: string) => {
+  const [_, unit] = dtype.split("<M8");
+  const _data = new Float64Array(data.data.length);
+  let _divisor = 1n;
+  switch (unit) {
+    case "[ms]":
+      _divisor = 1000n;
+      break;
+    case "[us]":
+      _divisor = 1000000n;
+      break;
+    case "[ns]":
+      _divisor = 1000000000n;
+      break;
+    default:
+      _divisor = 1n;
+      break;
+  }
+  for (let i = 0; i < data.data.length; i++) {
+    // When dtype is numpy datetime (<M8...), underlying storage corresponds to int64
+    // so we can treat the chunk data as a BigInt64Array for conversion.
+    const v = (data.data as unknown as BigInt64Array)[i];
+    _data[i] = Number(v / _divisor);
+  }
+  return unravel(_data, data.shape, data.stride);
+};
 
 const flatten = (
   data: Record<string, DataVariable>,
@@ -232,7 +259,7 @@ const flatten = (
       for (const k in data) {
         if (data[k].dimensions.includes(dim[0])) {
           subdata[k] = {
-            attributes: {},
+            attributes: data[k].attributes,
             // @ts-expect-error: Is array because include dims
             data: data[k].data[i],
             dimensions: data[k].dimensions.slice(1),
@@ -323,6 +350,13 @@ export class DataVar<
     } else if (this.arr.dtype == "bool") {
       return [..._data.data] as Data;
     } else {
+      // Safely inspect potential numpy datetime dtype stored in attrs
+      const dtype = (this.arr.attrs as Record<string, unknown>)._dtype as
+        | string
+        | undefined;
+      if (dtype?.startsWith("<M8")) {
+        return npdatetime_to_posixtime(_data, dtype) as Data;
+      }
       return unravel(_data.data, _data.shape, _data.stride);
     }
   }
@@ -405,9 +439,23 @@ export class Dataset<S extends HttpZarr | TempZarr> {
     const dims = {} as Record<string, number>;
     for (const item of _zarr.contents()) {
       if (item.kind == "array") {
-        const arr = await zarr.open(root.resolve(item.path), {
-          kind: "array",
-        });
+        let arr;
+        try {
+          arr = await zarr.open(root.resolve(item.path), {
+            kind: "array",
+          });
+        } catch (e: unknown) {
+          const message =
+            typeof e === "object" && e && "message" in e
+              ? String((e as { message?: unknown }).message)
+              : undefined;
+          if (message && message.includes("<M8")) {
+            //A python <M8 type fails to load
+            arr = await zarr_open_v2_datetime(root.resolve(item.path));
+          } else {
+            throw e;
+          }
+        }
         const array_dims = arr.attrs._ARRAY_DIMENSIONS as string[] | null;
         const vid = item.path.split("/").pop() as string;
         vars[vid] = new DataVar<DataType, HttpZarr>(

package/src/lib/session.ts

@@ -12,7 +12,7 @@ export class Session {
   creationTime!: Date;
   endTime!: Date;
   write!: boolean;
-  verified: boolean = false;
+  verified = false;
   private _connection!: any;
 
   /**
@@ -36,7 +36,7 @@ export class Session {
       session.user = "dummy_user";
       session.creationTime = new Date();
       session.endTime = new Date(
-        Date.now() + (options.duration || 1) * 60 * 60 * 1000
+        Date.now() + (options.duration || 3600) * 1000
       );
       session.write = false;
       session.verified = false;
@@ -55,9 +55,11 @@ export class Session {
     try {
       const headers = { ...connection._authHeaders };
       headers["Cache-Control"] = "no-store";
-      const params = { duration: options.duration || 1 };
+      const qs = new URLSearchParams({
+        duration: String(options.duration ?? 3600),
+      });
       const response = await fetch(
-        `${connection._gateway}/session/?${params}`,
+        `${connection._gateway}/session/?${qs.toString()}`,
         { headers }
       );
 
@@ -113,7 +115,7 @@ export class Session {
    * @param finaliseWrite - Whether to finalise any write operations. Defaults to false.
    * @throws {Error} - If the session cannot be closed and finaliseWrite is true.
    */
-  async close(finaliseWrite: boolean = false): Promise<void> {
+  async close(finaliseWrite = false): Promise<void> {
     // Back-compatibility with beta version (ignoring)
     if (!this._connection._isV1) {
       return;

package/src/lib/zarr.ts

@@ -6,7 +6,8 @@ import {
   UseStore,
 } from "idb-keyval";
 import hash from "object-hash";
-import { AsyncReadable, AsyncMutable, AbsolutePath } from "@zarrita/storage";
+import { AsyncReadable, AsyncMutable, Readable, AbsolutePath } from "@zarrita/storage";
+import { Array as ZArray, Location, ArrayMetadata, DataType } from "@zarrita/core";
 
 function delay(t: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, t));
@@ -172,3 +173,64 @@ export class IDBStore implements AsyncMutable {
     await del_cache(key, this.cache);
   }
 }
+
+const load_meta = async <S extends Readable>(
+  location: Location<S>,
+  item = ".zarray"
+) => {
+  const { path } = location.resolve(item);
+  const meta = await location.store.get(path);
+  if (!meta) {
+    return {};
+  }
+  return JSON.parse(new TextDecoder().decode(meta));
+};
+
+//This is modified from the zarrita core library to patch for datetime support
+export async function zarr_open_v2_datetime<Store extends Readable>(
+  location: Location<Store>
+) {
+  const attrs = await load_meta(location, ".zattrs");
+  const meta = await load_meta(location);
+  if (meta.dtype.startsWith("<M8")) {
+    attrs._dtype = meta.dtype;
+  }
+  const codecs: any[] = [];
+
+  if (meta.order === "F") {
+    codecs.push({ name: "transpose", configuration: { order: "F" } });
+  }
+  // Detect big-endian from v2 dtype string (e.g., ">i4"). If so, add a bytes codec.
+  if (typeof meta.dtype === "string" && meta.dtype.startsWith(">")) {
+    codecs.push({ name: "bytes", configuration: { endian: "big" } });
+  }
+  for (const { id, ...configuration } of meta.filters ?? []) {
+    codecs.push({ name: id, configuration });
+  }
+  if (meta.compressor) {
+    const { id, ...configuration } = meta.compressor;
+    codecs.push({ name: id, configuration });
+  }
+  const v3_metadata: ArrayMetadata<DataType> = {
+    zarr_format: 3,
+    node_type: "array",
+    shape: meta.shape,
+    data_type: "int64",
+    chunk_grid: {
+      name: "regular",
+      configuration: {
+        chunk_shape: meta.chunks,
+      },
+    },
+    chunk_key_encoding: {
+      name: "v2",
+      configuration: {
+        separator: meta.dimension_separator ?? ".",
+      },
+    },
+    codecs,
+    fill_value: meta.fill_value,
+    attributes: attrs,
+  };
+  return new ZArray(location.store, location.path, v3_metadata);
+}

package/typedoc.json

@@ -3,5 +3,6 @@
   "entryPoints": ["./src/index.ts"],
   "readme": "./README.md",
   "out": "../../docs/datamesh",
-  "mergeReadme": true
+  "mergeReadme": true,
+  "projectDocuments": ["proxy/guide.md"]
 }