@sjcrh/proteinpaint-server 2.178.0 → 2.179.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.178.0",
+  "version": "2.179.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -62,11 +62,11 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.143.0",
-    "@sjcrh/proteinpaint-python": "2.178.0",
+    "@sjcrh/proteinpaint-python": "2.179.0",
     "@sjcrh/proteinpaint-r": "2.178.0",
-    "@sjcrh/proteinpaint-rust": "2.178.0",
-    "@sjcrh/proteinpaint-shared": "2.178.0",
-    "@sjcrh/proteinpaint-types": "2.178.0",
+    "@sjcrh/proteinpaint-rust": "2.179.0",
+    "@sjcrh/proteinpaint-shared": "2.179.0",
+    "@sjcrh/proteinpaint-types": "2.179.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",

package/routes/termdb.boxplot.js

@@ -22,14 +22,14 @@ const api = {
 function init({ genomes }) {
   return async (req, res) => {
     const q = req.query;
-    const genome = genomes[q.genome];
-    if (!genome) throw new Error("invalid genome name");
-    const ds = genome.datasets?.[q.dslabel];
-    if (!ds) throw new Error("invalid ds");
-    const terms = [q.tw];
-    if (q.overlayTw) terms.push(q.overlayTw);
-    if (q.divideTw) terms.push(q.divideTw);
     try {
+      const genome = genomes[q.genome];
+      if (!genome) throw new Error("invalid genome name");
+      const ds = genome.datasets?.[q.dslabel];
+      if (!ds) throw new Error("invalid dslabel");
+      const terms = [q.tw];
+      if (q.overlayTw) terms.push(q.overlayTw);
+      if (q.divideTw) terms.push(q.divideTw);
       const data = await getData(
         { filter: q.filter, filter0: q.filter0, terms, __protected__: q.__protected__, __abortSignal: q.__abortSignal },
         ds

package/routes/termdb.chat2.js

@@ -7,6 +7,7 @@ import { extract_DE_search_terms_from_query } from "./chat/DEagent.ts";
 import { extract_summary_terms } from "./chat/summaryagent.ts";
 import { extract_matrix_search_terms_from_query } from "./chat/matrixagent.ts";
 import { extract_samplescatter_terms_from_query } from "./chat/samplescatteragent.ts";
+import { extract_hiercluster_terms_from_query } from "./chat/hierclusteragent.ts";
 import { parse_dataset_db, parse_geneset_db, getGenesetNames } from "./chat/utils.ts";
 import serverconfig from "../src/serverconfig.js";
 import { mayLog } from "#src/helpers.ts";
@@ -50,7 +51,6 @@ function init({ genomes }) {
       const ai_output_json = await run_chat_pipeline(
         q.prompt,
         llm,
-        serverconfig.aiRoute,
         dataset_json,
         testing,
         dataset_db,
@@ -65,7 +65,7 @@ function init({ genomes }) {
     }
   };
 }
-async function run_chat_pipeline(user_prompt, llm, aiRoute, dataset_json, testing, dataset_db, genedb, ds, genesetNames = []) {
+async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, dataset_db, genedb, ds, genesetNames = []) {
   const time1 = (/* @__PURE__ */ new Date()).valueOf();
   const class_response = await classifyQuery(user_prompt, llm);
   let ai_output_json;
@@ -84,9 +84,8 @@ async function run_chat_pipeline(user_prompt, llm, aiRoute, dataset_json, testin
     }
   } else if (class_response.type == "plot") {
     const classResult = await classifyPlotType(user_prompt, llm);
-    mayLog("classResult:", classResult);
     const dataset_db_output = await parse_dataset_db(dataset_db);
-    const genes_list = dataset_json.hasGeneExpression ? await parse_geneset_db(genedb) : [];
+    const genes_list = await parse_geneset_db(genedb);
     if (classResult == "summary") {
       const time12 = (/* @__PURE__ */ new Date()).valueOf();
       ai_output_json = await extract_summary_terms(
@@ -139,6 +138,19 @@ async function run_chat_pipeline(user_prompt, llm, aiRoute, dataset_json, testin
         genesetNames
       );
       mayLog("Time taken for sampleScatter agent:", formatElapsedTime(Date.now() - time12));
+    } else if (classResult == "hiercluster") {
+      const time12 = (/* @__PURE__ */ new Date()).valueOf();
+      ai_output_json = await extract_hiercluster_terms_from_query(
+        user_prompt,
+        llm,
+        dataset_db_output,
+        dataset_json,
+        genes_list,
+        ds,
+        testing,
+        genesetNames
+      );
+      mayLog("Time taken for hierCluster agent:", formatElapsedTime(Date.now() - time12));
     } else {
       ai_output_json = { type: "text", text: "Unknown classification value" };
     }

package/routes/termdb.config.js

@@ -1,7 +1,16 @@
 import serverconfig from "#src/serverconfig.js";
 import { authApi } from "#src/auth.js";
 import { get_ds_tdb } from "#src/termdb.js";
-import { TermTypes } from "#shared/terms.js";
+import {
+  TermTypeGroups,
+  SINGLECELL_CELLTYPE,
+  GENE_EXPRESSION,
+  METABOLITE_INTENSITY,
+  WHOLE_PROTEOME_ABUNDANCE,
+  SINGLECELL_GENE_EXPRESSION,
+  DNA_METHYLATION,
+  SSGSEA
+} from "#shared/terms.js";
 const api = {
   endpoint: "termdb/config",
   methods: {
@@ -255,7 +264,8 @@ function addNonDictionaryQueries(c, ds, genome) {
       q2.singleCell.DEgenes = { termId: q.singleCell.DEgenes.termId };
     }
     if (q.singleCell?.terms?.length) {
-      c.scctTerms = q.singleCell.terms;
+      if (!c.termType2terms) c.termType2terms = {};
+      c.termType2terms[TermTypeGroups.SINGLECELL_CELLTYPE] = q.singleCell.terms;
     }
   }
   if (q.images) {
@@ -270,14 +280,14 @@ function getAllowedTermTypes(ds) {
   if (ds.cohort.termdb.allowedTermTypes) {
     for (const t of ds.cohort.termdb.allowedTermTypes) typeSet.add(t);
   }
-  if (ds.queries?.geneExpression) typeSet.add(TermTypes.GENE_EXPRESSION);
-  if (ds.queries?.metaboliteIntensity) typeSet.add(TermTypes.METABOLITE_INTENSITY);
-  if (ds.queries?.proteome?.whole) typeSet.add(TermTypes.WHOLE_PROTEOME_ABUNDANCE);
-  if (ds.queries?.ssGSEA) typeSet.add(TermTypes.SSGSEA);
-  if (ds.queries?.dnaMethylation) typeSet.add(TermTypes.DNA_METHYLATION);
+  if (ds.queries?.geneExpression) typeSet.add(GENE_EXPRESSION);
+  if (ds.queries?.metaboliteIntensity) typeSet.add(METABOLITE_INTENSITY);
+  if (ds.queries?.proteome?.whole) typeSet.add(WHOLE_PROTEOME_ABUNDANCE);
+  if (ds.queries?.ssGSEA) typeSet.add(SSGSEA);
+  if (ds.queries?.dnaMethylation) typeSet.add(DNA_METHYLATION);
   if (ds.queries?.singleCell) {
-    typeSet.add(TermTypes.SINGLECELL_CELLTYPE);
-    if (ds.queries.singleCell?.geneExpression) typeSet.add(TermTypes.SINGLECELL_GENE_EXPRESSION);
+    typeSet.add(SINGLECELL_CELLTYPE);
+    if (ds.queries.singleCell?.geneExpression) typeSet.add(SINGLECELL_GENE_EXPRESSION);
   }
   if (ds.cohort.termdb.termCollections?.length) typeSet.add("termCollection");
   return [...typeSet];

package/routes/termdb.dmr.js

@@ -1,6 +1,8 @@
 import { TermdbDmrPayload } from "#types/checkers";
-import { run_R } from "@sjcrh/proteinpaint-r";
+import { run_python } from "@sjcrh/proteinpaint-python";
 import { invalidcoord } from "#shared/common.js";
+import { mayLog } from "#src/helpers.ts";
+import { formatElapsedTime } from "#shared";
 const api = {
   endpoint: "termdb/dmr",
   methods: {
@@ -22,24 +24,33 @@ function init({ genomes }) {
       if (!genome) throw new Error("invalid genome");
       const ds = genome.datasets?.[q.dslabel];
       if (!ds) throw new Error("invalid ds");
-      if (!ds.queries?.dnaMethylation) throw new Error("not supported");
+      if (!ds.queries?.dnaMethylation) throw new Error("analysis not supported");
       if (!Array.isArray(q.group1) || q.group1.length == 0) throw new Error("group1 not non empty array");
       if (!Array.isArray(q.group2) || q.group2.length == 0) throw new Error("group2 not non empty array");
       if (invalidcoord(genome, q.chr, q.start, q.stop)) throw new Error("invalid chr/start/stop");
-      const arg = {
-        group1: q.group1,
-        group2: q.group2,
-        file: ds.queries.dnaMethylation.file,
-        // todo change file to mValueFile
+      const group1 = q.group1.map((s) => s.sample).filter(Boolean);
+      const group2 = q.group2.map((s) => s.sample).filter(Boolean);
+      if (group1.length < 3) throw new Error(`Need at least 3 samples in group1, got ${group1.length}`);
+      if (group2.length < 3) throw new Error(`Need at least 3 samples in group2, got ${group2.length}`);
+      const gpdmInput = {
+        h5file: ds.queries.dnaMethylation.file,
         chr: q.chr,
         start: q.start,
-        stop: q.stop
+        stop: q.stop,
+        group1,
+        group2,
+        annotations: q.annotations || [],
+        nan_threshold: q.nan_threshold ?? 0.5
       };
-      const result = JSON.parse(await run_R("dmr.R", JSON.stringify(arg)));
+      const time1 = Date.now();
+      const result = JSON.parse(await run_python("gpdm_analysis.py", JSON.stringify(gpdmInput)));
+      mayLog("DMR analysis time:", formatElapsedTime(Date.now() - time1));
       if (result.error) throw new Error(result.error);
-      res.send(result);
+      res.send({ status: "ok", dmrs: result.dmrs });
     } catch (e) {
-      res.send({ error: e.message || e });
+      const msg = e instanceof Error ? e.message : String(e);
+      res.send({ error: msg });
+      if (e instanceof Error && e.stack) console.log(e);
     }
   };
 }

package/routes/termdb.sampleScatter.js

@@ -7,6 +7,7 @@ import { authApi } from "../src/auth.js";
 import { run_R } from "@sjcrh/proteinpaint-r";
 import { read_file } from "../src/utils.js";
 import { getDescrStats } from "#routes/termdb.descrstats.ts";
+import { SINGLECELL_GENE_EXPRESSION, isSingleCellTerm } from "#shared/terms.js";
 const api = {
   endpoint: "termdb/sampleScatter",
   methods: {
@@ -23,15 +24,15 @@ const api = {
 const refColor = "#F5F5DC";
 function init({ genomes }) {
   return async function(req, res) {
-    const q = req.query;
-    if (!q.genome || !q.dslabel) {
-      throw new Error("Genome and dataset label are required for termdb/sampleScatter request.");
-    }
-    const g = genomes[q.genome];
-    const ds = g.datasets[q.dslabel];
-    if (q.singleCellPlot)
-      return getSingleCellScatter(req, res, ds);
     try {
+      const q = req.query;
+      if (!q.genome || !q.dslabel) {
+        throw new Error("Genome and dataset label are required for termdb/sampleScatter request.");
+      }
+      const g = genomes[q.genome];
+      const ds = g.datasets[q.dslabel];
+      if (q.singleCellPlot)
+        return getSingleCellScatter(req, res, ds);
       let refSamples = [], cohortSamples;
       const terms = [];
       if (q.colorTW) terms.push(q.colorTW);
@@ -121,12 +122,11 @@ async function getSingleCellScatter(req, res, ds) {
   const { name, sample } = q.singleCellPlot;
   try {
     const tw = q.colorTW;
+    if (!tw || !isSingleCellTerm(tw.term))
+      throw new Error("colorTW must be provided and be a single cell term for single cell scatter plot");
     const arg = { plots: [name], sample };
-    if (tw) {
-      if (tw.term.type == "singleCellGeneExpression") arg.gene = tw.term.gene;
-      else if (tw.term.type == "singleCellCellType") arg.colorBy = tw.term.name;
-      else throw new Error("unsupported tw");
-    }
+    if (tw.term.type == SINGLECELL_GENE_EXPRESSION) arg.gene = tw.term.gene;
+    else arg.colorBy = tw.term.name;
     const data = await ds.queries.singleCell.data.get(arg);
     const plot = data.plots[0];
     const cells = [...plot.expCells, ...plot.noExpCells];
@@ -142,7 +142,8 @@ async function getSingleCellScatter(req, res, ds) {
         z: 0,
         category: cell.category,
         shape: "Ref",
-        hidden
+        hidden,
+        geneExp: cell.geneExp
       };
     });
     const [xMin, xMax, yMin, yMax] = samples.reduce(
@@ -151,14 +152,16 @@ async function getSingleCellScatter(req, res, ds) {
     );
     const categories = new Set(samples.map((s) => s.category));
     const colorMap = {};
-    const k2c = getColors(categories.size);
-    for (const category of categories) {
-      const color = k2c(category);
-      colorMap[category] = {
-        sampleCount: samples.filter((s) => s.category == category).length,
-        color,
-        key: category
-      };
+    if (tw.term.type != SINGLECELL_GENE_EXPRESSION) {
+      const k2c = getColors(categories.size);
+      for (const category of categories) {
+        const color = k2c(category);
+        colorMap[category] = {
+          sampleCount: samples.filter((s) => s.category == category).length,
+          color,
+          key: category
+        };
+      }
     }
     const shapeLegend = [["Ref", { sampleCount: samples.length, shape: 0, key: "Ref" }]];
     const colorLegend = Object.entries(colorMap);

package/routes/termdb.singlecellSamples.js

@@ -9,7 +9,7 @@ import { termdbSingleCellSamplesPayload } from "#types/checkers";
 import { validate_query_singleCell_DEgenes } from "./termdb.singlecellDEgenes.ts";
 import { gdc_validate_query_singleCell_data } from "#src/mds3.gdc.js";
 import ky from "ky";
-import { TermTypes } from "#shared/terms.js";
+import { SINGLECELL_CELLTYPE } from "#shared/terms.js";
 const api = {
   endpoint: "termdb/singlecellSamples",
   methods: {
@@ -252,7 +252,7 @@ function colorColumn2terms(plots, ds) {
         /** TODO: possible term may apply to multiple plots.
          * May need to change to plots: [] */
         plot: plot.name,
-        type: TermTypes.SINGLECELL_CELLTYPE,
+        type: SINGLECELL_CELLTYPE,
         groupsetting: {},
         values: baseValues.reduce((acc, v) => {
           const alias = c?.aliases?.[v];

package/routes/termdb.topVariablyExpressedGenes.js

@@ -70,7 +70,7 @@ function nativeValidateQuery(ds) {
       }
     } else {
       for (const i of gE.samples) {
-        const n = ds.cohort.termdb.q.id2sampleName(i.id);
+        const n = ds.cohort.termdb.q.id2sampleName(i);
         if (!n) throw "sample id cannot convert to string name";
         samples.push(n);
       }

package/routes/termdb.violin.js

@@ -190,9 +190,10 @@ async function createCanvasImg(q, result, ds) {
     const chart = result.charts[k];
     const plot2Values = {};
     for (const plot of chart.plots) plot2Values[plot.label] = plot.values;
-    const densities = await getDensities(plot2Values);
-    let axisScale;
     const useLog = q.unit == "log";
+    const logBase = ds.cohort.termdb.logscaleBase2 ? 2 : 10;
+    const densities = await getDensities(plot2Values, useLog, logBase);
+    let axisScale;
     if (useLog) {
       axisScale = scaleLog().base(ds.cohort.termdb.logscaleBase2 ? 2 : 10).domain([result.min, result.max]).range(isH ? [0, q.svgw] : [q.svgw, 0]);
     } else {
@@ -241,8 +242,16 @@ async function getDensity(values) {
   const result = await getDensities({ plot: values });
   return result.plot;
 }
-async function getDensities(plot2Values) {
-  const plot2Density = JSON.parse(await run_R("density.R", JSON.stringify({ plot2Values })));
+async function getDensities(plot2Values, useLog = false, logBase = 10) {
+  let transformedPlot2Values = {};
+  if (useLog) {
+    for (const plot in plot2Values) {
+      transformedPlot2Values[plot] = plot2Values[plot].filter((v) => v > 0).map((v) => Math.log(v) / Math.log(logBase));
+    }
+  } else {
+    transformedPlot2Values = plot2Values;
+  }
+  const plot2Density = JSON.parse(await run_R("density.R", JSON.stringify({ plot2Values: transformedPlot2Values })));
   const densities = {};
   for (const plot in plot2Density) {
     const result = plot2Density[plot];
@@ -253,11 +262,12 @@ async function getDensities(plot2Values) {
     let xMax = -Infinity;
     for (const [i, x] of Object.entries(result.x)) {
       const density2 = result.y[i];
-      xMin = Math.min(xMin, x);
-      xMax = Math.max(xMax, x);
+      const x0 = useLog ? Math.pow(logBase, x) : x;
+      xMin = Math.min(xMin, x0);
+      xMax = Math.max(xMax, x0);
       densityMin = Math.min(densityMin, density2);
       densityMax = Math.max(densityMax, density2);
-      bins.push({ x0: x, density: density2 });
+      bins.push({ x0, density: density2 });
     }
     bins.unshift({ x0: xMin, density: densityMin });
     bins.push({ x0: xMax, density: densityMin });