@sjcrh/proteinpaint-server 2.190.2-0 → 2.191.0

package/routes/termdb.cluster.js

@@ -1,456 +0,0 @@
-import path from "path";
-import { run_R } from "@sjcrh/proteinpaint-r";
-import { run_rust } from "@sjcrh/proteinpaint-rust";
-import * as utils from "#src/utils.js";
-import serverconfig from "#src/serverconfig.js";
-import { gdc_validate_query_geneExpression } from "#src/mds3.gdc.js";
-import { mayLimitSamples } from "#src/mds3.filter.js";
-import { clusterMethodLst, distanceMethodLst } from "#shared/clustering.js";
-import { getData } from "#src/termdb.matrix.js";
-import {
-  GENE_EXPRESSION,
-  METABOLITE_INTENSITY,
-  NUMERIC_DICTIONARY_TERM,
-  termType2label,
-  ISOFORM_EXPRESSION,
-  SSGSEA,
-  PROTEOME_ABUNDANCE
-} from "#shared/terms.js";
-import { formatElapsedTime } from "#shared/time.js";
-function init({ genomes }) {
-  return async (req, res) => {
-    const q = req.query;
-    let result;
-    try {
-      const g = genomes[q.genome];
-      if (!g) throw "invalid genome name";
-      const ds = g.datasets[q.dslabel];
-      if (!ds) throw "invalid dataset name";
-      if (ds.label === "GDC" && !ds.__gdc?.doneCaching)
-        throw "The server has not finished caching the case IDs: try again in about 2 minutes.";
-      if ([GENE_EXPRESSION, SSGSEA, ISOFORM_EXPRESSION, METABOLITE_INTENSITY, NUMERIC_DICTIONARY_TERM].includes(
-        q.dataType
-      )) {
-        if (!ds.queries?.[q.dataType] && q.dataType !== NUMERIC_DICTIONARY_TERM)
-          throw `no ${q.dataType} data on this dataset`;
-        if (!q.terms) throw `missing gene list`;
-        if (!Array.isArray(q.terms)) throw `gene list is not an array`;
-        if (q.terms.length < 3)
-          throw `A minimum of three genes is required for clustering. Please refresh this page to clear this error.`;
-        result = await getResult(q, ds);
-      } else if (PROTEOME_ABUNDANCE == q.dataType) {
-        const proteomeQuery = ds.queries?.proteome;
-        if (!proteomeQuery?.get) throw `no ${PROTEOME_ABUNDANCE} data getter on this dataset`;
-        if (!q.terms) throw `missing gene list`;
-        if (!Array.isArray(q.terms)) throw `gene list is not an array`;
-        if (q.terms.length < 3)
-          throw `A minimum of three genes is required for clustering. Please refresh this page to clear this error.`;
-        result = await getResult(q, ds);
-      } else {
-        throw "unknown q.dataType " + q.dataType;
-      }
-    } catch (e) {
-      if (e.stack) console.log(e.stack);
-      result = {
-        status: e.status || 400,
-        error: e.message || e
-      };
-    }
-    res.send(result);
-  };
-}
-async function getResult(q, ds) {
-  let _q = q;
-  if (q.dataType == GENE_EXPRESSION) {
-    _q = JSON.parse(JSON.stringify(q));
-    _q.forClusteringAnalysis = true;
-    _q.__abortSignal = q.__abortSignal;
-  }
-  let term2sample2value, byTermId, bySampleId, skippedSexChrGenes;
-  if (q.dataType == NUMERIC_DICTIONARY_TERM) {
-    ;
-    ({ term2sample2value, byTermId, bySampleId } = await getNumericDictTermAnnotation(q, ds));
-  } else if (q.dataType == PROTEOME_ABUNDANCE) {
-    ;
-    ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries.proteome.get({
-      ..._q,
-      dataTypeDetails: _q.proteomeDetails
-    }));
-  } else {
-    ;
-    ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries[q.dataType].get(_q, ds));
-  }
-  const noValueTerms = [];
-  for (const [term, obj] of term2sample2value) {
-    if (Object.keys(obj).length === 0) {
-      const tw = q.terms.find((t2) => t2.$id == term);
-      const termName = !tw ? term : tw.term.type == "geneExpression" ? tw.term.gene : tw.term.type == "isoformExpression" ? tw.term.isoform : tw.term.name;
-      noValueTerms.push(termName);
-      term2sample2value.delete(term);
-      delete byTermId[term];
-    }
-  }
-  const removedHierClusterTerms = [];
-  if (noValueTerms.length) {
-    removedHierClusterTerms.push({
-      text: `Skipped ${q.dataType == GENE_EXPRESSION ? "genes" : "items"} with no data`,
-      lst: noValueTerms
-    });
-  }
-  if (skippedSexChrGenes?.length) {
-    removedHierClusterTerms.push({ text: "Skipped sex chromosome genes", lst: skippedSexChrGenes });
-  }
-  if (term2sample2value.size == 0) throw "no data";
-  if (term2sample2value.size == 1) {
-    const g = Array.from(term2sample2value.keys())[0];
-    return { term: { gene: g, type: GENE_EXPRESSION }, data: term2sample2value.get(g) };
-  }
-  const t = Date.now();
-  const clustering = await doClustering(term2sample2value, q, Object.keys(bySampleId).length);
-  console.log("clustering done:", formatElapsedTime(Date.now() - t));
-  const result = { clustering, byTermId, bySampleId };
-  if (removedHierClusterTerms.length) result.removedHierClusterTerms = removedHierClusterTerms;
-  return result;
-}
-async function getNumericDictTermAnnotation(q, ds) {
-  const getDataArgs = {
-    // TODO: figure out when term is not a termwrapper
-    terms: q.terms.map((tw) => tw.term ? tw : { term: tw, q: { mode: "continuous" } }),
-    filter: q.filter,
-    filter0: q.filter0,
-    __protected__: q.__protected__
-  };
-  const data = await getData(getDataArgs, ds);
-  if (data.error) throw data.error;
-  const term2sample2value = /* @__PURE__ */ new Map();
-  for (const [key, sampleData] of Object.entries(data.samples)) {
-    for (const [term, value] of Object.entries(sampleData)) {
-      if (term !== "sample") {
-        if (!term2sample2value.has(term)) {
-          term2sample2value.set(term, {});
-        }
-        term2sample2value.get(term)[key] = value.value;
-      }
-    }
-  }
-  return { term2sample2value, byTermId: data.refs.byTermId, bySampleId: data.refs.bySampleId };
-}
-async function doClustering(data, q, numCases = 1e3) {
-  const sampleSet = /* @__PURE__ */ new Set();
-  let firstTerm = true;
-  for (const o of data.values()) {
-    const currentSampleIds = new Set(Object.keys(o));
-    if (firstTerm) {
-      currentSampleIds.forEach((id) => sampleSet.add(id));
-      firstTerm = false;
-    } else {
-      for (const id of sampleSet) {
-        if (!currentSampleIds.has(id)) {
-          sampleSet.delete(id);
-        }
-      }
-    }
-  }
-  if (sampleSet.size == 0)
-    throw `termdb.cluster: There are no overlapping tested samples shared across the selected ${termType2label(
-      q.dataType
-    )}`;
-  if (!clusterMethodLst.find((i) => i.value == q.clusterMethod)) throw "Invalid cluster method";
-  if (!distanceMethodLst.find((i) => i.value == q.distanceMethod)) throw "Invalid distance method";
-  const inputData = {
-    matrix: [],
-    row_names: [],
-    // genes
-    col_names: [...sampleSet].slice(0, numCases),
-    // samples
-    cluster_method: q.clusterMethod,
-    distance_method: q.distanceMethod,
-    plot_image: false
-    // When true causes cluster.rs to plot the image into a png file (EXPERIMENTAL)
-  };
-  for (const [gene, o] of data) {
-    inputData.row_names.push(gene);
-    const row = [];
-    for (const s of inputData.col_names) {
-      row.push(o[s]);
-    }
-    inputData.matrix.push(q.zScoreTransformation ? getZscore(row) : row);
-  }
-  if (inputData.matrix.length == 0) throw "Clustering matrix is empty";
-  const Routput = JSON.parse(await run_R("hclust.R", JSON.stringify(inputData)));
-  const row_names_index = Routput.RowOrder.map((row) => inputData.row_names.indexOf(row.name));
-  const col_names_index = Routput.ColOrder.map((col) => inputData.col_names.indexOf(col.name));
-  const output_matrix = [];
-  for (const rowI of row_names_index) {
-    const newRow = [];
-    for (const colI of col_names_index) {
-      newRow.push(inputData.matrix[rowI][colI]);
-    }
-    output_matrix.push(newRow);
-  }
-  return {
-    row: {
-      merge: Routput.RowMerge,
-      height: Routput.RowHeight,
-      order: Routput.RowOrder,
-      inputOrder: inputData.row_names
-    },
-    col: {
-      merge: Routput.ColumnMerge,
-      height: Routput.ColumnHeight,
-      order: Routput.ColOrder,
-      inputOrder: inputData.col_names
-    },
-    matrix: output_matrix
-  };
-}
-function getZscore(l) {
-  const mean = l.reduce((sum, v) => sum + v, 0) / l.length;
-  const sd = Math.sqrt(l.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / l.length);
-  if (sd == 0) {
-    return l;
-  }
-  return l.map((v) => (v - mean) / sd);
-}
-async function validate_query_geneExpression(ds, genome) {
-  const q = ds.queries.geneExpression;
-  if (!q) return;
-  q.geneExpression2bins = {};
-  if (typeof q.get == "function") return;
-  if (q.src == "gdcapi") {
-    gdc_validate_query_geneExpression(ds, genome);
-    return;
-  }
-  if (q.src == "native") {
-    await validateNative(q, ds);
-    return;
-  }
-  throw "unknown queries.geneExpression.src";
-}
-async function queryHDF5(hdf5_file, query, read_mode) {
-  const input = {
-    hdf5_file,
-    query
-  };
-  if (read_mode) input.read_mode = read_mode;
-  try {
-    const result = await run_rust("readH5", JSON.stringify(input));
-    if (!result || result.length === 0) {
-      throw new Error("Failed to retrieve expression data: Empty or missing response");
-    }
-    return result;
-  } catch (error) {
-    console.error(`Error querying HDF5 for ${query}`);
-    throw error;
-  }
-}
-async function validateNative(q, ds) {
-  q.file = path.join(serverconfig.tpmasterdir, q.file);
-  q.samples = [];
-  try {
-    await utils.file_is_readable(q.file);
-    const tmp = await run_rust("readH5", JSON.stringify({ hdf5_file: q.file, validate: true }));
-    const vr = JSON.parse(tmp);
-    if (vr.status !== "success") throw vr.message;
-    if (!vr.samples?.length) throw "HDF5 file has no samples, please check file.";
-    const unknownSamples = /* @__PURE__ */ new Set();
-    for (const sn of vr.samples) {
-      const si = ds.cohort.termdb.q.sampleName2id(sn);
-      if (si === void 0) {
-        if (ds.cohort.db) {
-          throw `unknown sample ${sn} from HDF5 ${q.file}`;
-        } else {
-          unknownSamples.add(sn);
-          continue;
-        }
-      }
-      q.samples.push(si);
-    }
-    console.log(`${ds.label}: geneExpression HDF5 file validated. Format: ${vr.format}, Samples:`, q.samples.length);
-    if (unknownSamples.size) {
-      const arr = [...unknownSamples];
-      console.log(`unknown samples from geneExpression HDF5 file (${arr.length}): ${arr.join(", ")}`);
-    }
-  } catch (error) {
-    throw `${ds.label}: Failed to validate geneExpression HDF5 file: ${error}`;
-  }
-  q.get = async (param) => {
-    const limitSamples = await mayLimitSamples(param, q.samples, ds);
-    if (limitSamples?.size == 0) {
-      return { term2sample2value: /* @__PURE__ */ new Map(), byTermId: {}, bySampleId: {} };
-    }
-    const bySampleId = {};
-    const samples = q.samples || [];
-    if (limitSamples) {
-      for (const sid of limitSamples) {
-        if (ds.cohort?.termdb?.q?.id2sampleRefs) {
-          bySampleId[sid] = ds.cohort.termdb.q.id2sampleRefs(sid);
-        } else {
-          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
-        }
-      }
-    } else {
-      for (const sid of samples) {
-        if (ds.cohort?.termdb?.q?.id2sampleRefs) {
-          bySampleId[sid] = ds.cohort.termdb.q.id2sampleRefs(sid);
-        } else {
-          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
-        }
-      }
-    }
-    const term2sample2value = /* @__PURE__ */ new Map();
-    const byTermId = {};
-    const geneNames = [];
-    for (const tw of param.terms) {
-      if (tw.term.gene) {
-        geneNames.push(tw.term.gene);
-      }
-    }
-    if (geneNames.length === 0) {
-      console.log("No genes to query");
-      return { term2sample2value, byTermId };
-    }
-    const time1 = Date.now();
-    const readMode = param.dslabel == "MMRF" ? "bulk" : null;
-    const geneData = JSON.parse(await queryHDF5(q.file, geneNames, readMode));
-    console.log("Time taken to run gene query:", formatElapsedTime(Date.now() - time1));
-    const genesData = geneData.query_output || {};
-    if (!genesData) throw "No expression data returned from HDF5 query";
-    for (const tw of param.terms) {
-      if (!tw.term.gene) continue;
-      const geneResult = genesData[tw.term.gene];
-      if (!geneResult) {
-        console.warn(`No data found for gene ${tw.term.gene} in the response`);
-        continue;
-      }
-      const samplesData = geneResult.samples || {};
-      const s2v = {};
-      for (const sampleName in samplesData) {
-        const sampleId = ds.cohort.termdb.q.sampleName2id(sampleName);
-        if (!sampleId) continue;
-        if (limitSamples && !limitSamples.has(sampleId)) continue;
-        if (!Number.isFinite(samplesData[sampleName])) continue;
-        s2v[sampleId] = samplesData[sampleName];
-      }
-      if (Object.keys(s2v).length) {
-        term2sample2value.set(tw.$id, s2v);
-      }
-    }
-    if (term2sample2value.size == 0) {
-      throw "No data available for the input " + param.terms?.map((tw) => tw.term.gene).join(", ");
-    }
-    return { term2sample2value, byTermId, bySampleId };
-  };
-}
-async function validateQueryIsoformExpression(ds, _genome) {
-  const q = ds.queries.isoformExpression;
-  if (!q) return;
-  q.geneExpression2bins = {};
-  if (typeof q.get == "function") return;
-  if (q.file) {
-    await validateNativeIsoform(q, ds);
-    return;
-  }
-  throw "isoformExpression requires either .get() or .file";
-}
-async function validateNativeIsoform(q, ds) {
-  q.file = path.join(serverconfig.tpmasterdir, q.file);
-  q.samples = [];
-  try {
-    await utils.file_is_readable(q.file);
-    const tmp = await run_rust("readH5", JSON.stringify({ hdf5_file: q.file, validate: true, include_items: true }));
-    const vr = JSON.parse(tmp);
-    if (vr.status !== "success") throw vr.message;
-    if (!vr.samples?.length) throw "HDF5 file has no samples, please check file.";
-    for (const sn of vr.samples) {
-      const si = ds.cohort.termdb.q.sampleName2id(sn);
-      if (si == void 0) {
-        if (ds.cohort.db) {
-          throw `unknown sample ${sn} from HDF5 ${q.file}`;
-        } else {
-          continue;
-        }
-      }
-      q.samples.push(si);
-    }
-    q.availableItems = vr.items || [];
-    console.log(
-      `${ds.label}: isoformExpression HDF5 file validated. Format: ${vr.format}, Samples:`,
-      q.samples.length,
-      "Items:",
-      q.availableItems.length
-    );
-  } catch (error) {
-    throw `${ds.label}: Failed to validate isoformExpression HDF5 file: ${error}`;
-  }
-  q.get = async (param) => {
-    const limitSamples = await mayLimitSamples(param, q.samples, ds);
-    if (limitSamples?.size == 0) {
-      return { term2sample2value: /* @__PURE__ */ new Map(), byTermId: {}, bySampleId: {} };
-    }
-    const bySampleId = {};
-    const samples = q.samples || [];
-    if (limitSamples) {
-      for (const sid of limitSamples) {
-        if (ds.cohort?.termdb?.q?.id2sampleRefs) {
-          bySampleId[sid] = ds.cohort.termdb.q.id2sampleRefs(sid);
-        } else {
-          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
-        }
-      }
-    } else {
-      for (const sid of samples) {
-        if (ds.cohort?.termdb?.q?.id2sampleRefs) {
-          bySampleId[sid] = ds.cohort.termdb.q.id2sampleRefs(sid);
-        } else {
-          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
-        }
-      }
-    }
-    const term2sample2value = /* @__PURE__ */ new Map();
-    const byTermId = {};
-    const isoformIds = [];
-    for (const tw of param.terms) {
-      if (tw.term.isoform) {
-        isoformIds.push(tw.term.isoform);
-      }
-    }
-    if (isoformIds.length === 0) {
-      console.log("No isoforms to query");
-      return { term2sample2value, byTermId };
-    }
-    const time1 = Date.now();
-    const isoformData = JSON.parse(await queryHDF5(q.file, isoformIds, null));
-    console.log("Time taken to run isoform query:", formatElapsedTime(Date.now() - time1));
-    const isoformsData = isoformData.query_output || {};
-    if (!isoformsData) throw "No expression data returned from HDF5 query";
-    for (const tw of param.terms) {
-      if (!tw.term.isoform) continue;
-      const isoformResult = isoformsData[tw.term.isoform];
-      if (!isoformResult) {
-        console.warn(`No data found for isoform ${tw.term.isoform} in the response`);
-        continue;
-      }
-      const samplesData = isoformResult.samples || {};
-      const s2v = {};
-      for (const sampleName in samplesData) {
-        const sampleId = ds.cohort.termdb.q.sampleName2id(sampleName);
-        if (!sampleId) continue;
-        if (limitSamples && !limitSamples.has(sampleId)) continue;
-        s2v[sampleId] = samplesData[sampleName];
-      }
-      if (Object.keys(s2v).length) {
-        term2sample2value.set(tw.$id, s2v);
-      }
-    }
-    if (term2sample2value.size == 0) {
-      throw "No data available for the input " + param.terms?.map((tw) => tw.term.isoform).join(", ");
-    }
-    return { term2sample2value, byTermId, bySampleId };
-  };
-}
-export {
-  init,
-  validateQueryIsoformExpression,
-  validate_query_geneExpression
-};