@vitessce/config 3.6.3 → 3.6.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vitessce/config",
-  "version": "3.6.3",
+  "version": "3.6.4",
   "author": "HIDIVE Lab at HMS",
   "homepage": "http://vitessce.io",
   "repository": {
@@ -16,9 +16,11 @@
     "dist-tsc"
   ],
   "dependencies": {
-    "@vitessce/constants-internal": "3.6.3",
-    "@vitessce/utils": "3.6.3",
-    "@vitessce/globals": "3.6.3"
+    "zarrita": "0.5.2",
+    "@vitessce/constants-internal": "3.6.4",
+    "@vitessce/utils": "3.6.4",
+    "@vitessce/globals": "3.6.4",
+    "@vitessce/zarr-utils": "3.6.4"
   },
   "scripts": {
     "bundle": "pnpm exec vite build -c ../../scripts/vite.config.js",

package/src/generate-config-anndata.js

@@ -0,0 +1,75 @@
+/* eslint-disable no-unused-vars */
+import { AbstractAutoConfig } from './generate-config-helpers.js';
+
+export class AnnDataAutoConfig extends AbstractAutoConfig {
+  getOptions() {
+    const { zmetadata } = this;
+    const options = {
+      obsEmbedding: [],
+      obsSets: [],
+    };
+
+    zmetadata.forEach(({ path, attrs }) => {
+      const lowerPath = path.toLowerCase();
+      const relPath = path.substring(1);
+      // Gene expression matrix.
+      if (['/x'].includes(lowerPath)) {
+        options.obsFeatureMatrix = {
+          path: relPath,
+
+          // TODO: Also check the shape of X.
+          // If X is very large, try to initialize initial-filtering properties
+          // (will require that /var contains a boolean column however.)
+        };
+      }
+
+      // Spatial coordinates.
+      if (['/obsm/x_spatial', '/obsm/spatial'].includes(lowerPath)) {
+        // TODO: use obsSpots instead of obsLocations here?
+        options.obsLocations = {
+          path: relPath,
+        };
+      }
+
+      // Embedding arrays.
+      if (['/obsm/x_umap', '/obsm/umap'].includes(lowerPath)) {
+        options.obsEmbedding.push({ path: relPath, embeddingType: 'UMAP' });
+      }
+      if (['/obsm/x_tsne', '/obsm/tsne'].includes(lowerPath)) {
+        options.obsEmbedding.push({ path: relPath, embeddingType: 't-SNE' });
+      }
+      if (['/obsm/x_pca', '/obsm/pca'].includes(lowerPath)) {
+        options.obsEmbedding.push({ path: relPath, embeddingType: 'PCA' });
+      }
+
+      // Cell set columns.
+      // TODO: use all categorical/string columns of obs instead of this fixed set?
+      const supportedObsSetsPaths = [
+        'cluster', 'clusters', 'subcluster', 'cell_type', 'celltype',
+        'leiden', 'louvain', 'disease', 'organism', 'self_reported_ethnicity',
+        'tissue', 'sex',
+      ].map(colname => `/obs/${colname}`);
+      if (supportedObsSetsPaths.includes(lowerPath)) {
+        const name = relPath.split('/').at(-1);
+        options.obsSets.push({ path: relPath, name });
+      }
+    });
+
+    return options;
+  }
+
+  addFiles(vc, dataset) {
+    const { url, fileType } = this;
+    dataset.addFile({
+      url,
+      fileType,
+      options: this.getOptions(),
+      // TODO: coordination values?
+    });
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  addViews(vc, layoutOption) {
+    // TODO
+  }
+}

package/src/generate-config-helpers.js

@@ -0,0 +1,19 @@
+/* eslint-disable no-unused-vars */
+export class AbstractAutoConfig {
+  constructor(parsedStore) {
+    const { url, fileType, zmetadata } = parsedStore;
+    this.url = url;
+    this.fileType = fileType;
+    this.zmetadata = zmetadata;
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  addFiles(vc, dataset) {
+    throw new Error('The addFiles() method has not been implemented.');
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  addViews(vc, layoutOption) {
+    throw new Error('The addViews() method has not been implemented.');
+  }
+}

package/src/generate-config-ome.js

@@ -0,0 +1,21 @@
+/* eslint-disable no-unused-vars */
+import { AbstractAutoConfig } from './generate-config-helpers.js';
+
+// TODO: split into separate classes for OME-TIFF and OME-Zarr?
+// TODO: split into separate classes for image and obsSegmentations?
+export class OmeAutoConfig extends AbstractAutoConfig {
+  addFiles(vc, dataset) {
+    const { url, fileType } = this;
+    dataset.addFile({
+      url,
+      fileType,
+      // TODO: options?
+      // TODO: coordination values?
+    });
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  addViews(vc, layoutOption) {
+    // TODO
+  }
+}

package/src/generate-config-spatialdata.js

@@ -0,0 +1,101 @@
+/* eslint-disable no-unused-vars */
+import { AbstractAutoConfig } from './generate-config-helpers.js';
+
+export class SpatialDataAutoConfig extends AbstractAutoConfig {
+  getOptions() {
+    const { zmetadata } = this;
+    const options = {};
+
+    const availableElements = zmetadata.filter(({ path }) => {
+      const relPath = path.substring(1);
+      return relPath.match(/^(tables|table|images|labels|shapes|points)\/([^/]*)$/);
+    });
+
+    availableElements.forEach(({ path, attrs }) => {
+      const relPath = path.substring(1);
+
+      const firstCoordinateSystem = attrs
+        ?.multiscales?.[0]
+        ?.coordinateTransformations?.[0]
+        ?.output?.name;
+
+      // Handle image elements.
+      if (relPath.match(/^(images)\/([^/]*)$/)) {
+        options.image = {
+          path: relPath,
+          coordinateSystem: firstCoordinateSystem,
+          // TODO: support a fileUid property in the schema?
+        };
+      }
+      // Handle labels elements.
+      if (relPath.match(/^(labels)\/([^/]*)$/)) {
+        options.labels = {
+          path: relPath,
+          coordinateSystem: firstCoordinateSystem,
+          // TODO: support a fileUid property in the schema?
+        };
+
+        // TODO: check which table annotates these labels.
+      }
+
+      // Handle shapes elements.
+      if (relPath.match(/^(shapes)\/([^/]*)$/)) {
+        // TODO: check if shapes are circles or polygons
+        // to determine which Vitessce data type to use.
+        options.obsSpots = {
+          path: relPath,
+          coordinateSystem: firstCoordinateSystem,
+        };
+
+        // TODO: check which table annotates these shapes.
+      }
+
+      // Handle table elements.
+      if (relPath.match(/^(tables|table)\/([^/]*)$/)) {
+        // Identify all sub-paths within this table element.
+        const tableEls = zmetadata.filter(({ path: subpath }) => subpath.startsWith(path));
+
+        // Check if the table contains an X array.
+        const hasX = tableEls.find(el => el.path === `${path}/X`);
+        if (hasX) {
+          options.obsFeatureMatrix = {
+            path: hasX.path.substring(1),
+            // region: null,
+          };
+        }
+
+        // Check if the table contains an obs dataframe.
+        const hasObs = tableEls.find(el => el.path === `${path}/obs`);
+        if (hasObs) {
+          const columnOrder = hasObs.attrs?.['column-order'];
+          // Use the columns of this dataframe to configure the obsSets.
+          options.obsSets = {
+            // region: null,
+            tablePath: relPath,
+            obsSets: columnOrder.map(c => ({
+              path: `${hasObs.path.substring(1)}/${c}`,
+              name: c,
+            })),
+          };
+        }
+      }
+    });
+
+    return options;
+  }
+
+  addFiles(vc, dataset) {
+    const { url, fileType } = this;
+    dataset.addFile({
+      url,
+      fileType,
+      options: this.getOptions(),
+      // TODO: coordination values?
+    });
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  addViews(vc, layoutOption) {
+    // TODO
+  }
+}

package/src/generate-config.js

@@ -0,0 +1,265 @@
+// TODO: ts-check
+import { FileType } from '@vitessce/constants-internal';
+import { withConsolidated, FetchStore, ZipFileStore, open as zarrOpen, root as zarrRoot } from 'zarrita';
+import { VitessceConfig } from './VitessceConfig.js';
+// Classes for different types of objects
+import { AnnDataAutoConfig } from './generate-config-anndata.js';
+import { SpatialDataAutoConfig } from './generate-config-spatialdata.js';
+import { OmeAutoConfig } from './generate-config-ome.js';
+
+const fileTypeToExtensions = {
+  [FileType.IMAGE_OME_TIFF]: ['.ome.tif', '.ome.tiff', '.ome.tf2', '.ome.tf8'],
+  [FileType.IMAGE_OME_ZARR]: ['.ome.zarr'],
+  [FileType.IMAGE_OME_ZARR_ZIP]: ['.ome.zarr.zip'],
+  [FileType.ANNDATA_ZARR]: ['.ad.zarr', '.h5ad.zarr', '.adata.zarr', '.anndata.zarr'],
+  [FileType.ANNDATA_ZARR_ZIP]: ['.ad.zarr.zip', '.h5ad.zarr.zip', '.adata.zarr.zip', '.anndata.zarr.zip'],
+  // TODO: how to handle h5ad-based AnnData (since needs reference JSON file).
+  // Perhaps just assume one H5AD+one JSON (or .ref.json) file correspond to each other?
+  [FileType.SPATIALDATA_ZARR]: ['.sd.zarr', '.sdata.zarr', '.spatialdata.zarr'],
+  [FileType.SPATIALDATA_ZARR_ZIP]: ['.sd.zarr.zip', '.sdata.zarr.zip', '.spatialdata.zarr.zip'],
+};
+
+const fileTypeToClass = {
+  // OME-TIFF
+  [FileType.IMAGE_OME_TIFF]: OmeAutoConfig,
+  [FileType.OBS_SEGMENTATIONS_OME_TIFF]: OmeAutoConfig,
+  // OME-Zarr
+  [FileType.IMAGE_OME_ZARR]: OmeAutoConfig,
+  [FileType.IMAGE_OME_ZARR_ZIP]: OmeAutoConfig,
+  [FileType.OBS_SEGMENTATIONS_OME_ZARR]: OmeAutoConfig,
+  [FileType.OBS_SEGMENTATIONS_OME_ZARR_ZIP]: OmeAutoConfig,
+  // AnnData
+  [FileType.ANNDATA_ZARR]: AnnDataAutoConfig,
+  [FileType.ANNDATA_ZARR_ZIP]: AnnDataAutoConfig,
+  // SpatialData
+  [FileType.SPATIALDATA_ZARR]: SpatialDataAutoConfig,
+  [FileType.SPATIALDATA_ZARR_ZIP]: SpatialDataAutoConfig,
+};
+
+// This list contains file types that can be mapped to a regular Zarr store
+// (e.g., FetchStore or ZipStore).
+const ZARR_FILETYPES = [
+  FileType.ANNDATA_ZARR,
+  FileType.ANNDATA_ZARR_ZIP,
+  FileType.SPATIALDATA_ZARR,
+  FileType.SPATIALDATA_ZARR_ZIP,
+  FileType.IMAGE_OME_ZARR,
+  FileType.IMAGE_OME_ZARR_ZIP,
+  FileType.OBS_SEGMENTATIONS_OME_ZARR,
+  FileType.OBS_SEGMENTATIONS_OME_ZARR_ZIP,
+];
+
+function urlToFileType(url) {
+  const match = Object.entries(fileTypeToExtensions).find(
+    // eslint-disable-next-line no-unused-vars
+    ([fileType, extensions]) => extensions.some(ext => url.endsWith(ext)),
+  );
+  if (match) {
+    return match[0];
+  }
+  throw new Error('The file extension contained in the URL did not map to a supported fileType.');
+}
+
+/**
+ *
+ * @param {{ fileType, url }} parsedUrl
+ * @returns {Readable}
+ */
+function getStore(parsedUrl) {
+  const { fileType, url } = parsedUrl;
+  if (!ZARR_FILETYPES.includes(fileType)) {
+    return null;
+  }
+  return fileType.endsWith('.zip')
+    ? ZipFileStore.fromUrl(url)
+    : new FetchStore(url);
+}
+
+/**
+ * Ensure that each object { url, fileType, [store] }
+ * contains a `store`.
+ * @param {object[]} parsedUrls
+ * @returns {object[]}
+ */
+function ensureStores(parsedUrls) {
+  return parsedUrls.map((parsedUrl) => {
+    if (parsedUrl.store) {
+      return parsedUrl;
+    }
+    const store = getStore(parsedUrl);
+    return {
+      ...parsedUrl,
+      store,
+    };
+  });
+}
+
+/**
+ *
+ * @param {string} s A single string, like this
+ * `http://example.com/my_zarr.zarr#anndata.zarr;
+ * http://example.com/my_tiff.ome.tif`
+ * @returns {{ url: string, fileType: string}[]} The URLs with file types.
+ */
+export function parseUrls(s) {
+  const urlsWithHashes = s.split(';');
+  return urlsWithHashes.map((urlWithHash) => {
+    const parts = urlWithHash.split('#');
+    if (parts.length === 1) {
+      const [url] = parts;
+      return {
+        url,
+        fileType: urlToFileType(url),
+      };
+    } if (parts.length === 2) {
+      const [url, fileType] = parts;
+      return {
+        url,
+        fileType,
+      };
+    }
+    throw new Error('Only expected zero or one # character per URL, but received more.');
+  });
+}
+
+
+export async function parsedUrlToZmetadata(parsedUrl) {
+  const { store: initialStore } = parsedUrl;
+
+  if (!initialStore) {
+    return null;
+  }
+
+  let store;
+  let promises = [];
+
+  try {
+    try {
+      store = await withConsolidated(initialStore);
+    } catch {
+      // Try again with `zmetadata` rather than `.zmetadata`.
+      // Reference: https://github.com/zarr-developers/zarr-python/issues/1121
+      store = await withConsolidated(initialStore, { metadataKey: 'zmetadata' });
+    }
+    // Is consolidated.
+    const contents = store.contents();
+    promises = contents.map(async (value) => {
+      const item = await zarrOpen(zarrRoot(store).resolve(value.path));
+      return {
+        ...value,
+        attrs: item.attrs,
+      };
+    });
+  } catch {
+    store = initialStore;
+    // Is not consolidated.
+    const keysToTry = [
+      // Note: OME-NGFF metadata is stored in the root attrs.
+      '/',
+      // AnnData keys
+      '/X',
+      '/layers',
+      '/obs',
+      '/var',
+      '/obsm',
+      '/obsm/spatial',
+      '/obsm/X_spatial',
+      '/obsm/pca',
+      '/obsm/X_pca',
+      '/obsm/tsne',
+      '/obsm/X_tsne',
+      '/obsm/umap',
+      '/obsm/X_umap',
+      // TODO: second round of getting metadata for
+      // columns listed in /obs and /var .attrs['column-order'] ?
+
+      // SpatialData keys
+      // Note: For spatialData, we assume the store is always consolidated.
+      // TODO: throw error if spatialdata + not consolidated?
+    ];
+    promises = keysToTry.map(async (k) => {
+      try {
+        const item = await zarrOpen(zarrRoot(store).resolve(k));
+        return {
+          path: k,
+          kind: item.kind,
+          attrs: item.attrs,
+        };
+      } catch {
+        return null;
+      }
+    });
+  }
+
+  return (await Promise.all(promises))
+    .filter(entry => entry !== null);
+}
+
+/**
+ *
+ * @param {{ url, fileType, store }[]} parsedUrls
+ * @return {string[]} The layoutOptions.
+ */
+export function parsedUrlsToLayoutOptions(parsedUrls) {
+  // eslint-disable-next-line no-unused-vars
+  const parsedStores = ensureStores(parsedUrls);
+
+  // TODO: implement
+}
+
+/**
+ *
+ * @param {{ url, fileType, store }[]} parsedUrls
+ * @param {string|null} layoutOption
+ */
+export async function generateConfig(parsedUrls, layoutOption = null) {
+  // Map each URL to a Zarr store.
+  const parsedStores = ensureStores(parsedUrls);
+
+  // Obtain Zarr consolidated_metadata for each store.
+  const zmetadataStores = await Promise.all(
+    parsedStores.map(async parsedStore => ({
+      ...parsedStore,
+      zmetadata: await parsedUrlToZmetadata(parsedStore),
+    })),
+  );
+
+  // Create configuration instance.
+  const vc = new VitessceConfig({
+    schemaVersion: '1.0.17',
+    name: 'Automatically-generated configuration.',
+    // TODO: write a description based on what is known
+    // (fileType(s) and maybe layoutOption).
+    description: 'Populate with a description of this visualization.',
+  });
+
+  // Create datasets.
+  // TODO: cases in which more than one dataset should be created?
+  const dataset = vc.addDataset('Main dataset');
+
+  zmetadataStores.forEach((parsedStore) => {
+    const { fileType } = parsedStore;
+    const AutoConfigClass = fileTypeToClass[fileType];
+    const autoConfig = new AutoConfigClass(parsedStore);
+
+    autoConfig.addFiles(vc, dataset);
+    // TODO: add all files, then add all views (in two separate loops)?
+    autoConfig.addViews(vc, layoutOption);
+  });
+
+  const stores = Object.fromEntries(
+    // Here, we use `parsedUrls` rather than `parsedStores`
+    // so that we do not provide more stores than intended
+    // (i.e., we do not provide stores which were solely created
+    // to obtain zmetadata).
+    parsedUrls
+      .filter(d => d.store)
+      .map(d => ([d.url, d.store])),
+  );
+
+  // Return both the config and the `stores` url-to-store dict.
+  return {
+    config: vc,
+    stores,
+  };
+}