@sjcrh/proteinpaint-server 2.184.0 → 2.185.0

package/routes/termdb.DE.js

@@ -1,15 +1,11 @@
-import fs from "fs";
 import path from "path";
 import { diffExpPayload } from "#types/checkers";
-import { run_rust } from "@sjcrh/proteinpaint-rust";
-import { getData } from "../src/termdb.matrix.js";
-import { get_ds_tdb } from "../src/termdb.js";
-import { run_R } from "@sjcrh/proteinpaint-r";
 import { mayLog } from "#src/helpers.ts";
 import serverconfig from "../src/serverconfig.js";
-import { imageSize } from "image-size";
 import { get_header_txt } from "#src/utils.js";
-import { formatElapsedTime } from "#shared";
+import { run_rust } from "@sjcrh/proteinpaint-rust";
+import { renderVolcano } from "../src/renderVolcano.ts";
+import { readCacheFileOrRecompute, resolveDeContext, resolveSampleGroups } from "../src/diffAnalysis.ts";
 const api = {
   endpoint: "termdb/DE",
   methods: {
@@ -27,244 +23,41 @@ function init({ genomes }) {
   return async (req, res) => {
     try {
       const q = req.query;
-      const genome = genomes[q.genome];
-      if (!genome) throw new Error("invalid genome");
-      const [ds] = get_ds_tdb(genome, q);
-      let term_results = [];
-      if (q.tw) {
-        const terms = [q.tw];
-        term_results = await getData(
-          {
-            filter: q.filter,
-            filter0: q.filter0,
-            terms
-          },
-          ds
-        );
-        if (term_results.error) throw new Error(term_results.error);
-      }
-      let term_results2 = [];
-      if (q.tw2) {
-        const terms2 = [q.tw2];
-        term_results2 = await getData(
-          {
-            filter: q.filter,
-            filter0: q.filter0,
-            terms: terms2
-          },
-          ds
-        );
-        if (term_results2.error) throw new Error(term_results2.error);
+      if (q.preAnalysis) {
+        const { ds, term_results, term_results2 } = await resolveDeContext(q, genomes);
+        const groups = resolveSampleGroups(q, ds, term_results, term_results2);
+        const group1Name = q.samplelst.groups[0].name;
+        const group2Name = q.samplelst.groups[1].name;
+        res.send({
+          data: {
+            [group1Name]: groups.group1names.length,
+            [group2Name]: groups.group2names.length,
+            ...groups.alerts.length ? { alert: groups.alerts.join(" | ") } : {}
+          }
+        });
+        return;
       }
-      const results = await run_DE(req.query, ds, term_results, term_results2);
-      if (!results || !results.data) throw new Error("No data available");
-      res.send(results);
+      const { cacheId, geneData, sample_size1, sample_size2, method, images, bcv } = await readCacheFileOrRecompute({
+        daRequest: q,
+        genomes
+      });
+      const rendered = await renderVolcano(geneData, q.volcanoRender);
+      rendered.cacheId = cacheId;
+      const output = {
+        data: rendered,
+        sample_size1,
+        sample_size2,
+        method,
+        images
+      };
+      if (bcv != null) output.bcv = bcv;
+      res.send(output);
     } catch (e) {
       res.send({ status: "error", error: e.message || e });
       if (e instanceof Error && e.stack) console.log(e);
     }
   };
 }
-async function run_DE(param, ds, term_results, term_results2) {
-  if (param.samplelst?.groups?.length != 2) throw new Error(".samplelst.groups.length!=2");
-  if (param.samplelst.groups[0].values?.length < 1) throw new Error("samplelst.groups[0].values.length<1");
-  if (param.samplelst.groups[1].values?.length < 1) throw new Error("samplelst.groups[1].values.length<1");
-  const q = ds.queries.rnaseqGeneCount;
-  if (!q) return;
-  if (!q.file) throw new Error("unknown data type for rnaseqGeneCount");
-  if (!q.storage_type) throw new Error("storage_type is not defined");
-  param.storage_type = q.storage_type;
-  const group1names = [];
-  const conf1_group1 = [];
-  const conf2_group1 = [];
-  for (const s of param.samplelst.groups[0].values) {
-    if (!Number.isInteger(s.sampleId)) continue;
-    const n = ds.cohort.termdb.q.id2sampleName(s.sampleId);
-    if (!n) continue;
-    if (q.allSampleSet.has(n)) {
-      if (param.tw && !param.tw2) {
-        if (term_results.samples[s.sampleId]) {
-          if (param.tw.q.mode == "continuous") {
-            conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
-          } else {
-            conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["key"]);
-          }
-          group1names.push(n);
-        }
-      } else if (!param.tw && param.tw2) {
-        if (term_results2.samples[s.sampleId]) {
-          if (param.tw2.q.mode == "continuous") {
-            conf2_group1.push(term_results2.samples[s.sampleId][param.tw2.$id]["value"]);
-          } else {
-            conf2_group1.push(term_results2.samples[s.sampleId][param.tw2.$id]["key"]);
-          }
-          group1names.push(n);
-        }
-      } else if (param.tw && param.tw2) {
-        if (term_results.samples[s.sampleId] && term_results2.samples[s.sampleId]) {
-          if (param.tw.q.mode == "continuous") {
-            conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
-          } else {
-            conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["key"]);
-          }
-          if (param.tw2.q.mode == "continuous") {
-            conf2_group1.push(term_results2.samples[s.sampleId][param.tw2.$id]["value"]);
-          } else {
-            conf2_group1.push(term_results2.samples[s.sampleId][param.tw2.$id]["key"]);
-          }
-          group1names.push(n);
-        }
-      } else {
-        group1names.push(n);
-      }
-    }
-  }
-  const group2names = [];
-  const conf1_group2 = [];
-  const conf2_group2 = [];
-  for (const s of param.samplelst.groups[1].values) {
-    if (!Number.isInteger(s.sampleId)) continue;
-    const n = ds.cohort.termdb.q.id2sampleName(s.sampleId);
-    if (!n) continue;
-    if (q.allSampleSet.has(n)) {
-      if (param.tw && !param.tw2) {
-        if (term_results.samples[s.sampleId]) {
-          if (param.tw.q.mode == "continuous") {
-            conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
-          } else {
-            conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["key"]);
-          }
-          group2names.push(n);
-        }
-      } else if (!param.tw && param.tw2) {
-        if (term_results2.samples[s.sampleId]) {
-          if (param.tw2.q.mode == "continuous") {
-            conf2_group2.push(term_results2.samples[s.sampleId][param.tw2.$id]["value"]);
-          } else {
-            conf2_group2.push(term_results2.samples[s.sampleId][param.tw2.$id]["key"]);
-          }
-          group2names.push(n);
-        }
-      } else if (param.tw && param.tw2) {
-        if (term_results.samples[s.sampleId] && term_results2.samples[s.sampleId]) {
-          if (param.tw.q.mode == "continuous") {
-            conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
-          } else {
-            conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["key"]);
-          }
-          if (param.tw2.q.mode == "continuous") {
-            conf2_group2.push(term_results2.samples[s.sampleId][param.tw2.$id]["value"]);
-          } else {
-            conf2_group2.push(term_results2.samples[s.sampleId][param.tw2.$id]["key"]);
-          }
-          group2names.push(n);
-        }
-      } else {
-        group2names.push(n);
-      }
-    }
-  }
-  const sample_size1 = group1names.length;
-  const sample_size2 = group2names.length;
-  const alerts = validateGroups(sample_size1, sample_size2, group1names, group2names);
-  if (param.preAnalysis) {
-    const group1Name = param.samplelst.groups[0].name;
-    const group2Name = param.samplelst.groups[1].name;
-    return {
-      data: {
-        [group1Name]: sample_size1,
-        [group2Name]: sample_size2,
-        ...alerts.length ? { alert: alerts.join(" | ") } : {}
-      }
-    };
-  }
-  if (alerts.length) throw new Error(alerts.join(" | "));
-  const cases_string = group2names.map((i) => i).join(",");
-  const controls_string = group1names.map((i) => i).join(",");
-  const expression_input = {
-    case: cases_string,
-    control: controls_string,
-    data_type: "do_DE",
-    input_file: q.file,
-    cachedir: serverconfig.cachedir,
-    min_count: param.min_count,
-    min_total_count: param.min_total_count,
-    cpm_cutoff: param.cpm_cutoff,
-    storage_type: param.storage_type,
-    DE_method: param.method,
-    mds_cutoff: 1e4
-    // If the dimensions of the read counts matrix is below this threshold, only then the mds image will be generated as its very compute intensive. Number of genes * Number of samples < mds_cutoff for mds generation
-  };
-  if (param.tw) {
-    expression_input.conf1 = [...conf1_group2, ...conf1_group1];
-    expression_input.conf1_mode = param.tw.q.mode;
-    if (new Set(expression_input.conf1).size === 1) {
-      throw new Error("Confounding variable 1 has only one value");
-    }
-  }
-  if (param.tw2) {
-    expression_input.conf2 = [...conf2_group2, ...conf2_group1];
-    expression_input.conf2_mode = param.tw2.q.mode;
-    if (new Set(expression_input.conf2).size === 1) {
-      throw new Error("Confounding variable 2 has only one value");
-    }
-  }
-  const sample_size_limit = 8;
-  if (group1names.length <= sample_size_limit && group2names.length <= sample_size_limit || param.method == "edgeR" || param.method == "limma") {
-    const time12 = (/* @__PURE__ */ new Date()).valueOf();
-    const result2 = JSON.parse(await run_R("edge_newh5.R", JSON.stringify(expression_input)));
-    mayLog("Time taken to run edgeR:", formatElapsedTime(Date.now() - time12));
-    param.method = "edgeR";
-    const ql_imagePath = path.join(serverconfig.cachedir, result2.edgeR_ql_image_name[0]);
-    mayLog("ql_imagePath:", ql_imagePath);
-    await readFileAndDelete(ql_imagePath, "ql_image", result2);
-    if (result2.edgeR_mds_image_name) {
-      const mds_imagePath = path.join(serverconfig.cachedir, result2.edgeR_mds_image_name[0]);
-      mayLog("mds_imagePath:", mds_imagePath);
-      await readFileAndDelete(mds_imagePath, "mds_image", result2);
-    }
-    const images = [result2.ql_image];
-    if (result2.mds_image) images.push(result2.mds_image);
-    const output = {
-      data: result2.gene_data,
-      sample_size2: result2.num_cases[0],
-      sample_size1: result2.num_controls[0],
-      method: param.method,
-      images
-    };
-    if (result2.bcv && result2.bcv[0] !== null && result2.bcv[0] !== void 0) {
-      output.bcv = result2.bcv[0];
-    }
-    return output;
-  }
-  const time1 = (/* @__PURE__ */ new Date()).valueOf();
-  const result = JSON.parse(await run_rust("DEanalysis", JSON.stringify(expression_input)));
-  mayLog("Time taken to run rust DE pipeline:", formatElapsedTime(Date.now() - time1));
-  param.method = "wilcoxon";
-  return { data: result, sample_size1, sample_size2, method: param.method };
-}
-function validateGroups(sample_size1, sample_size2, group1names, group2names) {
-  const alerts = [];
-  if (sample_size1 < 1) alerts.push("sample size of group1 < 1");
-  if (sample_size2 < 1) alerts.push("sample size of group2 < 1");
-  const commonnames = group1names.filter((x) => group2names.includes(x));
-  if (commonnames.length) alerts.push(`Common elements found between both groups: ${commonnames.join(", ")}`);
-  return alerts;
-}
-async function readFileAndDelete(file, key, response) {
-  const plot = await fs.promises.readFile(file);
-  const plotBuffer = Buffer.from(plot).toString("base64");
-  const { width, height } = imageSize(file);
-  const obj = {
-    src: `data:image/png;base64,${plotBuffer}`,
-    size: `${width}x${height}`,
-    key
-  };
-  response[key] = obj;
-  fs.unlink(file, (err) => {
-    if (err) throw new Error(err.message || String(err));
-  });
-}
 async function validate_query_rnaseqGeneCount(ds) {
   const q = ds.queries.rnaseqGeneCount;
   if (!q) return;

package/routes/termdb.chat3.js

@@ -0,0 +1,191 @@
+import { ChatPayload } from "#types/checkers";
+import { mayLog } from "#src/helpers.ts";
+import { formatElapsedTime } from "#shared";
+import { readJSONFile, parse_geneset_db } from "./chat/utils.ts";
+import { classifyQuery } from "./chat/classify1.ts";
+import { classifyPlotType } from "./chat/plot.ts";
+import { classifyNotPlot } from "./chat/classify2.ts";
+import { inferScaffold } from "./chat/scaffold.ts";
+import serverconfig from "../src/serverconfig.js";
+import { getDsAllowedTermTypes } from "./termdb.config.ts";
+import { phrase2entity } from "./chat/phrase2entity.ts";
+import { inferTermObjFromEntity } from "./chat/entity2termObj.ts";
+import { resolveToTwTvs } from "./chat/entity2twTvs.ts";
+import path from "path";
+import fs from "fs";
+import { resolveToPlotState } from "./chat/scaffold2state.ts";
+const api = {
+  endpoint: "termdb/chat3",
+  methods: {
+    get: {
+      ...ChatPayload,
+      init
+    },
+    post: {
+      ...ChatPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    const q = req.query;
+    try {
+      const g = genomes[q.genome];
+      if (!g) throw "invalid genome";
+      const ds = g.datasets?.[q.dslabel];
+      if (!ds) throw "invalid dslabel";
+      const aiFilesDir = serverconfig.binpath + "/../../dataset/ai/" + q.dslabel;
+      let agentFiles = [];
+      try {
+        agentFiles = await fs.readdirSync(aiFilesDir).filter((file) => file.endsWith(".json"));
+      } catch (err) {
+        if (err.code === "ENOENT") throw new Error(`Directory not found: ${aiFilesDir}`);
+        if (err.code === "ENOTDIR") throw new Error(`Path is not a directory: ${aiFilesDir}`);
+        throw err;
+      }
+      const llm = serverconfig.llm;
+      if (!llm) throw "serverconfig.llm is not configured";
+      if (llm.provider !== "SJ" && llm.provider !== "ollama" && llm.provider !== "huggingface" && llm.provider !== "azure") {
+        throw "llm.provider must be 'SJ', 'ollama', 'huggingface', or 'azure'";
+      }
+      const rawFilter = typeof q.filter === "string" ? JSON.parse(q.filter) : q.filter;
+      const filter = rawFilter && typeof rawFilter === "object" ? rawFilter : {};
+      const lst = Array.isArray(filter.lst) ? filter.lst : [];
+      const cohortFilter = lst.find((item) => item.tag === "cohortFilter");
+      const cohortKey = cohortFilter ? cohortFilter.tvs.values[0].key : "";
+      const supportedChartTypes = ds.cohort.termdb.q?.getSupportedChartTypes(req)?.[cohortKey];
+      const genedb = serverconfig.tpmasterdir + "/" + g.genedb.dbfile;
+      const _allowedTermTypes = getDsAllowedTermTypes(ds);
+      const ai_output_json = await run_chat_pipeline(
+        q.prompt,
+        llm,
+        ds,
+        genedb,
+        agentFiles,
+        aiFilesDir,
+        supportedChartTypes,
+        _allowedTermTypes
+        // 	testing
+      );
+      mayLog("From init: Final AI output JSON:", JSON.stringify(ai_output_json));
+      res.send(ai_output_json);
+    } catch (e) {
+      if (e.stack) mayLog(e.stack);
+      res.send({ error: e?.message || e });
+    }
+  };
+}
+async function run_chat_pipeline(user_prompt, llm, ds, genedb, agentFiles, aiFilesDir, supportedChartTypes, _allowedTermTypes) {
+  if (!fs.existsSync(path.join(aiFilesDir, "main.json")))
+    throw "Main data file is not specified for dataset:" + ds.label;
+  const dataset_json = await readJSONFile(path.join(aiFilesDir, "main.json"));
+  const time1 = (/* @__PURE__ */ new Date()).valueOf();
+  const class_response = await classifyQuery(user_prompt, llm);
+  mayLog("Time taken for classification:", formatElapsedTime(Date.now() - time1));
+  let ai_output_json;
+  if (class_response.type == "notplot") {
+    const time2 = (/* @__PURE__ */ new Date()).valueOf();
+    const notPlotResult = await classifyNotPlot(user_prompt, llm, agentFiles, aiFilesDir);
+    mayLog("Time taken for classify2:", formatElapsedTime(Date.now() - time2));
+    if (notPlotResult.type == "html") {
+      ai_output_json = notPlotResult;
+    } else {
+      ai_output_json = {
+        type: "text",
+        text: "Your query does not appear to be related to the available data visualizations. Please try rephrasing your question."
+      };
+    }
+  } else if (class_response.type == "plot") {
+    let time = (/* @__PURE__ */ new Date()).valueOf();
+    const plotType = await classifyPlotType(user_prompt, llm);
+    mayLog("Time taken to classify plot type:", formatElapsedTime(Date.now() - time));
+    if (!supportedChartTypes) {
+      const errorMsg = "Supported chart types list is undefined. Please check the dataset configuration and ensure 							  that getSupportedChartTypes is implemented correctly. Skipping chart type validation, but this may 							  lead to unsupported chart type errors downstream.";
+      console.warn(errorMsg);
+      const errorResponse = {
+        type: "text",
+        text: errorMsg
+      };
+      return errorResponse;
+    }
+    if (plotType === "summary") {
+      if (!supportedChartTypes.includes("dictionary")) {
+        const log = 'Plot type: "' + plotType + '" is not supported.';
+        ai_output_json = {
+          type: "text",
+          text: log
+        };
+        mayLog(log);
+        return ai_output_json;
+      }
+    } else if (plotType === "dge") {
+      if (!supportedChartTypes.includes("DA")) {
+        const log = 'Plot type: "' + plotType + '" is not supported.';
+        ai_output_json = {
+          type: "text",
+          text: log
+        };
+        mayLog(log);
+        return ai_output_json;
+      }
+    } else {
+      mayLog(`Supported chart types for this cohort: ${supportedChartTypes}`);
+      if (!supportedChartTypes.includes(plotType)) {
+        const log = 'Plot type: "' + plotType + '" is not supported.';
+        ai_output_json = {
+          type: "text",
+          text: log
+        };
+        mayLog(log);
+        return ai_output_json;
+      }
+    }
+    mayLog("####### First phase: Infer Plot Scaffolds #######");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const scaffoldResult = await inferScaffold(user_prompt, plotType, llm);
+    mayLog("ScaffoldResult: ", scaffoldResult);
+    mayLog("Time taken to infer scaffold:", formatElapsedTime(Date.now() - time));
+    if (!scaffoldResult)
+      throw "Scaffold result is empty or undefined, which is unexpected. Please check the inferScaffold agent for potential issues.";
+    const subplotType = scaffoldResult.plotType === "summary" ? scaffoldResult.chartType : void 0;
+    mayLog("####### Second phase: From Scaffolds's phrases infer Entities #######");
+    const genes_list = await parse_geneset_db(genedb);
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const phrase2entityResult = await phrase2entity(scaffoldResult, plotType, llm, genes_list, dataset_json, ds);
+    mayLog("Time taken to phrase 2 entity:", formatElapsedTime(Date.now() - time));
+    if ("type" in phrase2entityResult && phrase2entityResult.type === "text") {
+      return phrase2entityResult;
+    }
+    mayLog(phrase2entityResult);
+    mayLog("####### Third phase: From Entities infer Term Objects #######");
+    const dataset_db = serverconfig.tpmasterdir + "/" + ds.cohort.db.file;
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const termObj = await inferTermObjFromEntity(
+      phrase2entityResult,
+      plotType,
+      llm,
+      dataset_db,
+      genes_list
+    );
+    mayLog("Time taken to infer term objects:", formatElapsedTime(Date.now() - time));
+    mayLog("Inferred termObj from entity:", JSON.stringify(termObj));
+    mayLog("####### Fourth phase: From Term Objects to TwTvs Objects #######");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const twTvsObj = await resolveToTwTvs(termObj, plotType, llm, dataset_db);
+    mayLog("Time taken to resolve to TwTvs object from termObj:", formatElapsedTime(Date.now() - time));
+    if ("type" in twTvsObj && twTvsObj.type === "text") {
+      return twTvsObj;
+    }
+    mayLog("twTvsObj:", twTvsObj);
+    mayLog("####### Fifth/Final phase: From TwTvs Objects to Plot States #######");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    ai_output_json = resolveToPlotState(twTvsObj, plotType, subplotType);
+    mayLog("Time taken to resolve to plot state:", formatElapsedTime(Date.now() - time));
+  }
+  return ai_output_json;
+}
+export {
+  api,
+  run_chat_pipeline
+};

package/routes/termdb.cluster.js

@@ -7,8 +7,15 @@ import serverconfig from "#src/serverconfig.js";
 import { gdc_validate_query_geneExpression } from "#src/mds3.gdc.js";
 import { mayLimitSamples } from "#src/mds3.filter.js";
 import { clusterMethodLst, distanceMethodLst } from "#shared/clustering.js";
-import { TermTypes, ISOFORM_EXPRESSION, PROTEOME_ABUNDANCE } from "#shared/terms.js";
-import { termType2label } from "#shared/terms.js";
+import { getData } from "#src/termdb.matrix.js";
+import {
+  GENE_EXPRESSION,
+  METABOLITE_INTENSITY,
+  NUMERIC_DICTIONARY_TERM,
+  termType2label,
+  ISOFORM_EXPRESSION,
+  PROTEOME_ABUNDANCE
+} from "#shared/terms.js";
 import { formatElapsedTime } from "#shared/time.js";
 const api = {
   endpoint: "termdb/cluster",
@@ -34,8 +41,9 @@ function init({ genomes }) {
       if (!ds) throw "invalid dataset name";
       if (ds.label === "GDC" && !ds.__gdc?.doneCaching)
         throw "The server has not finished caching the case IDs: try again in about 2 minutes.";
-      if ([TermTypes.GENE_EXPRESSION, ISOFORM_EXPRESSION, TermTypes.METABOLITE_INTENSITY].includes(q.dataType)) {
-        if (!ds.queries?.[q.dataType]) throw `no ${q.dataType} data on this dataset`;
+      if ([GENE_EXPRESSION, ISOFORM_EXPRESSION, METABOLITE_INTENSITY, NUMERIC_DICTIONARY_TERM].includes(q.dataType)) {
+        if (!ds.queries?.[q.dataType] && q.dataType !== NUMERIC_DICTIONARY_TERM)
+          throw `no ${q.dataType} data on this dataset`;
         if (!q.terms) throw `missing gene list`;
         if (!Array.isArray(q.terms)) throw `gene list is not an array`;
         if (q.terms.length < 3)
@@ -43,7 +51,7 @@ function init({ genomes }) {
         result = await getResult(q, ds);
       } else if (PROTEOME_ABUNDANCE == q.dataType) {
         const proteomeQuery = ds.queries?.proteome;
-        if (!proteomeQuery?.get) throw `no ${TermTypes.PROTEOME_ABUNDANCE} data getter on this dataset`;
+        if (!proteomeQuery?.get) throw `no ${PROTEOME_ABUNDANCE} data getter on this dataset`;
         if (!q.terms) throw `missing gene list`;
         if (!Array.isArray(q.terms)) throw `gene list is not an array`;
         if (q.terms.length < 3)
@@ -64,13 +72,16 @@ function init({ genomes }) {
 }
 async function getResult(q, ds) {
   let _q = q;
-  if (q.dataType == TermTypes.GENE_EXPRESSION) {
+  if (q.dataType == GENE_EXPRESSION) {
     _q = JSON.parse(JSON.stringify(q));
     _q.forClusteringAnalysis = true;
     _q.__abortSignal = q.__abortSignal;
   }
   let term2sample2value, byTermId, bySampleId, skippedSexChrGenes;
-  if (q.dataType == PROTEOME_ABUNDANCE) {
+  if (q.dataType == NUMERIC_DICTIONARY_TERM) {
+    ;
+    ({ term2sample2value, byTermId, bySampleId } = await getNumericDictTermAnnotation(q, ds));
+  } else if (q.dataType == PROTEOME_ABUNDANCE) {
     ;
     ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries.proteome.get(_q));
   } else {
@@ -90,7 +101,7 @@ async function getResult(q, ds) {
   const removedHierClusterTerms = [];
   if (noValueTerms.length) {
     removedHierClusterTerms.push({
-      text: `Skipped ${q.dataType == TermTypes.GENE_EXPRESSION ? "genes" : "items"} with no data`,
+      text: `Skipped ${q.dataType == GENE_EXPRESSION ? "genes" : "items"} with no data`,
       lst: noValueTerms
     });
   }
@@ -100,7 +111,7 @@ async function getResult(q, ds) {
   if (term2sample2value.size == 0) throw "no data";
   if (term2sample2value.size == 1) {
     const g = Array.from(term2sample2value.keys())[0];
-    return { term: { gene: g, type: TermTypes.GENE_EXPRESSION }, data: term2sample2value.get(g) };
+    return { term: { gene: g, type: GENE_EXPRESSION }, data: term2sample2value.get(g) };
   }
   const t = Date.now();
   const clustering = await doClustering(term2sample2value, q, Object.keys(bySampleId).length);
@@ -109,6 +120,29 @@ async function getResult(q, ds) {
   if (removedHierClusterTerms.length) result.removedHierClusterTerms = removedHierClusterTerms;
   return result;
 }
+async function getNumericDictTermAnnotation(q, ds) {
+  const getDataArgs = {
+    // TODO: figure out when term is not a termwrapper
+    terms: q.terms.map((tw) => tw.term ? tw : { term: tw, q: { mode: "continuous" } }),
+    filter: q.filter,
+    filter0: q.filter0,
+    __protected__: q.__protected__
+  };
+  const data = await getData(getDataArgs, ds);
+  if (data.error) throw data.error;
+  const term2sample2value = /* @__PURE__ */ new Map();
+  for (const [key, sampleData] of Object.entries(data.samples)) {
+    for (const [term, value] of Object.entries(sampleData)) {
+      if (term !== "sample") {
+        if (!term2sample2value.has(term)) {
+          term2sample2value.set(term, {});
+        }
+        term2sample2value.get(term)[key] = value.value;
+      }
+    }
+  }
+  return { term2sample2value, byTermId: data.refs.byTermId, bySampleId: data.refs.bySampleId };
+}
 async function doClustering(data, q, numCases = 1e3) {
   const sampleSet = /* @__PURE__ */ new Set();
   let firstTerm = true;
@@ -297,6 +331,7 @@ async function validateNative(q, ds) {
         const sampleId = ds.cohort.termdb.q.sampleName2id(sampleName);
         if (!sampleId) continue;
         if (limitSamples && !limitSamples.has(sampleId)) continue;
+        if (!Number.isFinite(samplesData[sampleName])) continue;
         s2v[sampleId] = samplesData[sampleName];
       }
       if (Object.keys(s2v).length) {

package/routes/termdb.config.js

@@ -47,11 +47,12 @@ function make(q, req, res, ds, genome) {
     selectCohort: getSelectCohort(ds, req),
     supportedChartTypes: tdb.q?.getSupportedChartTypes(req),
     renamedChartTypes: ds.cohort.renamedChartTypes,
-    allowedTermTypes: getAllowedTermTypes(ds),
+    allowedTermTypes: getDsAllowedTermTypes(ds),
     massSessionDuration: serverconfig.features.massSessionDuration || 30,
     dataDownloadCatch: tdb.dataDownloadCatch,
     matrix: tdb.matrix,
     hierCluster: tdb.hierCluster,
+    numericDictTermCluster: tdb.numericDictTermCluster,
     mclass: tdb.mclass,
     alwaysRefillCategoricalTermValues: tdb.alwaysRefillCategoricalTermValues,
     isGeneSetTermdb: tdb.isGeneSetTermdb,
@@ -293,7 +294,7 @@ function addNonDictionaryQueries(c, ds, genome) {
     q2.images = {};
   }
 }
-function getAllowedTermTypes(ds) {
+function getDsAllowedTermTypes(ds) {
   const typeSet = /* @__PURE__ */ new Set();
   for (const r of ds.cohort.termdb.termtypeByCohort) {
     if (r.termType) typeSet.add(r.termType);
@@ -329,5 +330,6 @@ function getSelectCohort(ds, req) {
   return copy;
 }
 export {
-  api
+  api,
+  getDsAllowedTermTypes
 };

package/routes/termdb.diffMeth.js

@@ -3,6 +3,7 @@ import { getData } from "../src/termdb.matrix.js";
 import { run_R } from "@sjcrh/proteinpaint-r";
 import { mayLog } from "#src/helpers.ts";
 import { formatElapsedTime } from "#shared";
+import { renderVolcano } from "../src/renderVolcano.ts";
 const api = {
   endpoint: "termdb/diffMeth",
   methods: {
@@ -39,7 +40,7 @@ function init({ genomes }) {
         throw new Error(
           "Differential methylation analysis returned no data. Please verify sample selections and try again."
         );
-      if (Array.isArray(results.data) && !results.data.length)
+      if ("totalRows" in results.data && results.data.totalRows === 0)
         throw new Error("No promoters passed filtering. Try relaxing group criteria or selecting more samples.");
       res.send(results);
     } catch (e) {
@@ -168,8 +169,9 @@ async function run_diffMeth(param, ds, term_results, term_results2) {
   const time1 = Date.now();
   const result = JSON.parse(await run_R("diffMeth.R", JSON.stringify(diffMethInput)));
   mayLog("Time taken to run diffMeth:", formatElapsedTime(Date.now() - time1));
+  const rendered = await renderVolcano(result.promoter_data, param.volcanoRender);
   const output = {
-    data: result.promoter_data,
+    data: rendered,
     sample_size1,
     sample_size2
   };