@sjcrh/proteinpaint-server 2.179.0 → 2.180.0

package/dataset/termdb.test.js

@@ -417,7 +417,7 @@ function termdb_test_default() {
               colorColumns: [
                 {
                   index: 3,
-                  name: "CellType"
+                  name: "Cell Type"
                 }
               ],
               coordsColumns: { x: 1, y: 2 },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.179.0",
+  "version": "2.180.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -64,9 +64,9 @@
     "@sjcrh/augen": "2.143.0",
     "@sjcrh/proteinpaint-python": "2.179.0",
     "@sjcrh/proteinpaint-r": "2.178.0",
-    "@sjcrh/proteinpaint-rust": "2.179.0",
-    "@sjcrh/proteinpaint-shared": "2.179.0",
-    "@sjcrh/proteinpaint-types": "2.179.0",
+    "@sjcrh/proteinpaint-rust": "2.180.0",
+    "@sjcrh/proteinpaint-shared": "2.180.0",
+    "@sjcrh/proteinpaint-types": "2.180.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",

package/routes/gdc.maf.js

@@ -26,7 +26,8 @@ function init({ genomes }) {
       if (!g) throw "hg38 missing";
       const ds = g.datasets.GDC;
       if (!ds) throw "hg38 GDC missing";
-      const payload = await listMafFiles(req.query, ds);
+      const q = req.query;
+      const payload = await listMafFiles(q, ds);
       res.send(payload);
     } catch (e) {
       res.send({ status: "error", error: e.message || e });
@@ -48,7 +49,7 @@ async function listMafFiles(q, ds) {
   if (q.filter0) {
     case_filters.content.push(q.filter0);
   }
-  const { host } = ds.getHostHeaders(q);
+  const { host, headers } = ds.getHostHeaders(q);
   const body = {
     filters,
     size: maxFileNumber,
@@ -66,7 +67,7 @@ async function listMafFiles(q, ds) {
     ].join(",")
   };
   if (case_filters.content.length) body.case_filters = case_filters;
-  const response = await ky.post(joinUrl(host.rest, "files"), { timeout: false, json: body });
+  const response = await ky.post(joinUrl(host.rest, "files"), { headers, timeout: false, json: body });
   if (!response.ok) throw `HTTP Error: ${response.status} ${response.statusText}`;
   const re = await response.json();
   if (!Number.isInteger(re.data?.pagination?.total)) throw "re.data.pagination.total is not int";

package/routes/gdc.mafBuild.js

@@ -20,11 +20,11 @@ const api = {
 function init({ genomes }) {
   return async (req, res) => {
     try {
-      const q = req.query;
       const g = genomes.hg38;
       if (!g) throw "hg38 missing";
       const ds = g.datasets.GDC;
       if (!ds) throw "hg38 GDC missing";
+      const q = req.query;
       await buildMaf(q, res, ds);
     } catch (e) {
       if (e.stack) console.log(e.stack);
@@ -34,14 +34,15 @@ function init({ genomes }) {
 }
 async function buildMaf(q, res, ds) {
   const t0 = Date.now();
-  const { host } = ds.getHostHeaders(q);
-  const fileLst2 = await getFileLstUnderSizeLimit(q.fileIdLst, host);
+  const { host, headers } = ds.getHostHeaders(q);
+  const fileLst2 = await getFileLstUnderSizeLimit(q.fileIdLst, host, headers);
   mayLog(`${fileLst2.length} out of ${q.fileIdLst.length} input MAF files accepted by size limit`, Date.now() - t0);
   const arg = {
     fileIdLst: fileLst2,
     columns: q.columns,
-    host: joinUrl(host.rest, "data")
+    host: joinUrl(host.rest, "data"),
     // must use the /data/ endpoint from current host
+    headers
   };
   const boundary = "------------------------GDC-MAF-BUILD";
   res.setHeader("Content-Type", `multipart/form-data; boundary=${boundary}`);
@@ -97,7 +98,7 @@ async function buildMaf(q, res, ds) {
     if (end) res.end();
   }
 }
-async function getFileLstUnderSizeLimit(lst, host) {
+async function getFileLstUnderSizeLimit(lst, host, headers) {
   if (lst.length == 0) throw "fileIdLst[] not array or blank";
   const body = {
     filters: {
@@ -107,7 +108,7 @@ async function getFileLstUnderSizeLimit(lst, host) {
     size: 1e4,
     fields: "file_size"
   };
-  const response = await ky.post(joinUrl(host.rest, "files"), { timeout: false, json: body });
+  const response = await ky.post(joinUrl(host.rest, "files"), { headers, timeout: false, json: body });
   if (!response.ok) throw `HTTP Error: ${response.status} ${response.statusText}`;
   const re = await response.json();
   if (!Array.isArray(re.data?.hits)) throw "re.data.hits[] not array";

package/routes/termdb.categories.js

@@ -48,11 +48,23 @@ async function trigger_getcategories(q, res, tdb, ds) {
   const data = await getData(arg, ds);
   if (data.error) throw data.error;
   const [lst, orderedLabels] = getCategories(data, q, ds, $id);
+  const allowedValues = tdb.getRestrictedValues?.(q.__protected__?.clientAuthResult, q.tw.term.id);
+  const filtered = filterCategoriesByAllowedValues(lst, orderedLabels, allowedValues);
   res.send({
-    lst,
-    orderedLabels
+    lst: filtered.lst,
+    orderedLabels: filtered.orderedLabels
   });
 }
+function filterCategoriesByAllowedValues(lst, orderedLabels, allowedValues) {
+  if (!allowedValues) {
+    return { lst, orderedLabels };
+  }
+  const allowedValuesSet = new Set(allowedValues.map((v) => String(v)));
+  const filteredLst = lst.filter((item) => allowedValuesSet.has(String(item.key)));
+  const allowedLabels = new Set(filteredLst.map((item) => item.label));
+  const filteredOrderedLabels = orderedLabels.filter((label) => allowedLabels.has(label));
+  return { lst: filteredLst, orderedLabels: filteredOrderedLabels };
+}
 function getCategories(data, q, ds, $id) {
   const lst = [];
   if (q.tw.term.type == "geneVariant" && q.tw.q.type != "predefined-groupset" && q.tw.q.type != "custom-groupset") {
@@ -133,5 +145,6 @@ function getCategories(data, q, ds, $id) {
 }
 export {
   api,
+  filterCategoriesByAllowedValues,
   getCategories
 };

package/routes/termdb.chat2.js

@@ -4,11 +4,13 @@ import { classifyNotPlot } from "./chat/classify2.ts";
 import { classifyPlotType } from "./chat/plot.ts";
 import { readJSONFile } from "./chat/utils.ts";
 import { extract_DE_search_terms_from_query } from "./chat/DEagent.ts";
+import { determineAmbiguousGenePrompt } from "./chat/ambiguousgeneagent.ts";
 import { extract_summary_terms } from "./chat/summaryagent.ts";
 import { extract_matrix_search_terms_from_query } from "./chat/matrixagent.ts";
 import { extract_samplescatter_terms_from_query } from "./chat/samplescatteragent.ts";
 import { extract_hiercluster_terms_from_query } from "./chat/hierclusteragent.ts";
-import { parse_dataset_db, parse_geneset_db, getGenesetNames } from "./chat/utils.ts";
+import { classifyGeneDataType } from "./chat/genedatatypeagent.ts";
+import { extractGenesFromPrompt, parse_dataset_db, parse_geneset_db, getGenesetNames } from "./chat/utils.ts";
 import serverconfig from "../src/serverconfig.js";
 import { mayLog } from "#src/helpers.ts";
 import { formatElapsedTime } from "#shared";
@@ -33,8 +35,7 @@ function init({ genomes }) {
       if (!g) throw "invalid genome";
       const ds = g.datasets?.[q.dslabel];
       if (!ds) throw "invalid dslabel";
-      const serverconfig_ds_entries = serverconfig.genomes.find((genome) => genome.name == q.genome).datasets.find((dslabel) => dslabel.name == ds.label);
-      if (!serverconfig_ds_entries.aifiles) {
+      if (!ds?.queries?.chat?.aifiles) {
         throw "aifiles are missing for chatbot to work";
       }
       const llm = serverconfig.llm;
@@ -44,8 +45,7 @@ function init({ genomes }) {
       }
       const dataset_db = serverconfig.tpmasterdir + "/" + ds.cohort.db.file;
       const genedb = serverconfig.tpmasterdir + "/" + g.genedb.dbfile;
-      const aiFilesPath = serverconfig_ds_entries.aifiles;
-      const dataset_json = await readJSONFile(aiFilesPath);
+      const dataset_json = await readJSONFile(ds?.queries?.chat?.aifiles);
       const testing = false;
       const genesetNames = getGenesetNames(g);
       const ai_output_json = await run_chat_pipeline(
@@ -83,9 +83,40 @@ async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, datase
       };
     }
   } else if (class_response.type == "plot") {
+    let geneFeatures = [];
+    const genes_list = await parse_geneset_db(genedb);
+    const relevant_genes = extractGenesFromPrompt(user_prompt, genes_list);
+    if (relevant_genes.length > 0) {
+      const AmbiguousGeneMessage = determineAmbiguousGenePrompt(user_prompt, relevant_genes, dataset_json);
+      if (AmbiguousGeneMessage.length > 0) {
+        return {
+          type: "text",
+          text: AmbiguousGeneMessage
+        };
+      }
+      const geneDataTypeMessage = await classifyGeneDataType(
+        user_prompt,
+        llm,
+        relevant_genes,
+        dataset_json
+      );
+      if (typeof geneDataTypeMessage === "string" || geneDataTypeMessage instanceof String) {
+        if (geneDataTypeMessage.length > 0) {
+          return {
+            type: "text",
+            text: geneDataTypeMessage
+          };
+        } else {
+          throw "classifyGeneDataType agent returned an empty string, which is unexpected.";
+        }
+      } else if (Array.isArray(geneDataTypeMessage)) {
+        geneFeatures = geneDataTypeMessage;
+      } else {
+        throw "geneDataTypeMessage has unknown data type returned from classifyGeneDataType agent";
+      }
+    }
     const classResult = await classifyPlotType(user_prompt, llm);
     const dataset_db_output = await parse_dataset_db(dataset_db);
-    const genes_list = await parse_geneset_db(genedb);
     if (classResult == "summary") {
       const time12 = (/* @__PURE__ */ new Date()).valueOf();
       ai_output_json = await extract_summary_terms(
@@ -93,10 +124,10 @@ async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, datase
         llm,
         dataset_db_output,
         dataset_json,
-        genes_list,
         ds,
         testing,
-        genesetNames
+        genesetNames,
+        geneFeatures
       );
       mayLog("Time taken for summary agent:", formatElapsedTime(Date.now() - time12));
     } else if (classResult == "dge") {
@@ -119,10 +150,10 @@ async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, datase
         llm,
         dataset_db_output,
         dataset_json,
-        genes_list,
         ds,
         testing,
-        genesetNames
+        genesetNames,
+        geneFeatures
       );
       mayLog("Time taken for matrix agent:", formatElapsedTime(Date.now() - time12));
     } else if (classResult == "samplescatter") {
@@ -132,10 +163,10 @@ async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, datase
         llm,
         dataset_db_output,
         dataset_json,
-        genes_list,
         ds,
         testing,
-        genesetNames
+        genesetNames,
+        geneFeatures
       );
       mayLog("Time taken for sampleScatter agent:", formatElapsedTime(Date.now() - time12));
     } else if (classResult == "hiercluster") {
@@ -145,12 +176,17 @@ async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, datase
         llm,
         dataset_db_output,
         dataset_json,
-        genes_list,
         ds,
         testing,
-        genesetNames
+        genesetNames,
+        geneFeatures
       );
       mayLog("Time taken for hierCluster agent:", formatElapsedTime(Date.now() - time12));
+    } else if (classResult == "lollipop") {
+      ai_output_json = {
+        type: "text",
+        text: "This is a gene mutation prompt. But, lollipop agent has not been implemented yet"
+      };
     } else {
       ai_output_json = { type: "text", text: "Unknown classification value" };
     }

package/routes/termdb.cluster.js

@@ -10,7 +10,6 @@ import { clusterMethodLst, distanceMethodLst } from "#shared/clustering.js";
 import { TermTypes, NUMERIC_DICTIONARY_TERM } from "#shared/terms.js";
 import { getData } from "#src/termdb.matrix.js";
 import { termType2label } from "#shared/terms.js";
-import { mayLog } from "#src/helpers.ts";
 import { formatElapsedTime } from "#shared/time.js";
 const api = {
   endpoint: "termdb/cluster",
@@ -109,7 +108,7 @@ async function getResult(q, ds) {
   }
   const t = Date.now();
   const clustering = await doClustering(term2sample2value, q, Object.keys(bySampleId).length);
-  mayLog("clustering done:", formatElapsedTime(Date.now() - t));
+  console.log("clustering done:", formatElapsedTime(Date.now() - t));
   const result = { clustering, byTermId, bySampleId };
   if (removedHierClusterTerms.length) result.removedHierClusterTerms = removedHierClusterTerms;
   return result;
@@ -292,7 +291,7 @@ async function validateNative(q, ds) {
     }
     const time1 = Date.now();
     const geneData = JSON.parse(await queryHDF5(q.file, geneNames));
-    mayLog("Time taken to run gene query:", formatElapsedTime(Date.now() - time1));
+    console.log("Time taken to run gene query:", formatElapsedTime(Date.now() - time1));
     const genesData = geneData.query_output || {};
     if (!genesData) throw "No expression data returned from HDF5 query";
     for (const tw of param.terms) {

package/routes/termdb.config.js

@@ -258,7 +258,9 @@ function addNonDictionaryQueries(c, ds, genome) {
       }
     };
     if (q.singleCell.geneExpression) {
-      q2.singleCell.geneExpression = {};
+      q2.singleCell.geneExpression = {
+        unit: q.singleCell.geneExpression?.unit
+      };
     }
     if (q.singleCell.DEgenes) {
       q2.singleCell.DEgenes = { termId: q.singleCell.DEgenes.termId };

package/routes/termdb.filterTermValues.js

@@ -40,10 +40,20 @@ async function getFilters(query, ds) {
   );
   const tw2List = {};
   for (const tw of query.terms) {
-    tw2List[tw.term.id] = getList(samplesPerFilter, filtersData, tw, query.showAll);
+    let values = getList(samplesPerFilter, filtersData, tw, query.showAll);
+    const allowedValues = ds.cohort.termdb.getRestrictedValues?.(query.__protected__.clientAuthResult, tw.term.id);
+    values = filterByAllowedValues(values, allowedValues);
+    tw2List[tw.term.id] = values;
   }
   return { ...tw2List };
 }
+function filterByAllowedValues(values, allowedValues) {
+  if (!allowedValues) {
+    return values;
+  }
+  const allowedValuesSet = new Set(allowedValues.map((v) => String(v)));
+  return values.filter((v) => v.value === "" || allowedValuesSet.has(String(v.value)));
+}
 async function getSamplesPerFilter(q, ds) {
   q.ds = ds;
   const samples = {};
@@ -84,5 +94,6 @@ function getList(samplesPerFilter, filtersData, tw, showAll) {
   return filteredValues;
 }
 export {
-  api
+  api,
+  filterByAllowedValues
 };

package/routes/termdb.sampleScatter.js

@@ -7,7 +7,7 @@ import { authApi } from "../src/auth.js";
 import { run_R } from "@sjcrh/proteinpaint-r";
 import { read_file } from "../src/utils.js";
 import { getDescrStats } from "#routes/termdb.descrstats.ts";
-import { SINGLECELL_GENE_EXPRESSION, isSingleCellTerm } from "#shared/terms.js";
+import { isSingleCellTerm, SINGLECELL_GENE_EXPRESSION, SINGLECELL_CELLTYPE } from "#shared/terms.js";
 const api = {
   endpoint: "termdb/sampleScatter",
   methods: {
@@ -126,7 +126,8 @@ async function getSingleCellScatter(req, res, ds) {
       throw new Error("colorTW must be provided and be a single cell term for single cell scatter plot");
     const arg = { plots: [name], sample };
     if (tw.term.type == SINGLECELL_GENE_EXPRESSION) arg.gene = tw.term.gene;
-    else arg.colorBy = tw.term.name;
+    else if (tw.term.type == SINGLECELL_CELLTYPE) arg.colorBy = tw.term.name;
+    else throw new Error(`unsupported single cell term type: ${tw.term.type}`);
     const data = await ds.queries.singleCell.data.get(arg);
     const plot = data.plots[0];
     const cells = [...plot.expCells, ...plot.noExpCells];
@@ -153,7 +154,11 @@ async function getSingleCellScatter(req, res, ds) {
     const categories = new Set(samples.map((s) => s.category));
     const colorMap = {};
     if (tw.term.type != SINGLECELL_GENE_EXPRESSION) {
-      const k2c = getColors(categories.size);
+      const defaultK2c = getColors(categories.size);
+      const k2c = (category) => {
+        const dsTerm = ds.queries.singleCell?.terms ? ds.queries.singleCell.terms.find((t) => t.name == tw.term.name) : void 0;
+        return tw.term.values?.[category]?.color || dsTerm?.values?.[category]?.color || defaultK2c(category);
+      };
       for (const category of categories) {
         const color = k2c(category);
         colorMap[category] = {

package/routes/termdb.singlecellSamples.js

@@ -223,9 +223,13 @@ function gdc_validateGeneExpression(G, ds, genome) {
         gene_ids: [gencodeId],
         file_id: hdf5id
       };
-      const { host } = ds.getHostHeaders(q);
+      const { host, headers } = ds.getHostHeaders(q);
       const t = Date.now();
-      const response = await ky.post(joinUrl(host.rest, "scrna_seq/gene_expression"), { timeout: false, json: body });
+      const response = await ky.post(joinUrl(host.rest, "scrna_seq/gene_expression"), {
+        timeout: false,
+        headers,
+        json: body
+      });
       if (!response.ok) throw new Error(`HTTP Error: ${response.status} ${response.statusText}`);
       const out = await response.json();
       mayLog("gdc scrna gene exp", q.gene, Date.now() - t);

package/routes/termdb.topVariablyExpressedGenes.js

@@ -6,7 +6,6 @@ import { makeFilter } from "#src/mds3.gdc.js";
 import { cachedFetch } from "#src/utils.js";
 import { joinUrl } from "#shared/joinUrl.js";
 import { formatElapsedTime } from "#shared/time.js";
-import { mayLog } from "#src/helpers.ts";
 const api = {
   endpoint: "termdb/topVariablyExpressedGenes",
   methods: {
@@ -34,7 +33,7 @@ function init({ genomes }) {
       result = {
         genes: await ds.queries.topVariablyExpressedGenes.getGenes(q)
       };
-      mayLog("topVariablyExpressedGenes", formatElapsedTime(Date.now() - t));
+      console.log("compute top variably expressed genes:", formatElapsedTime(Date.now() - t));
     } catch (e) {
       result = { status: e.status || 400, error: e.message || e };
     }