@sjcrh/proteinpaint-server 2.180.0 → 2.180.1

package/dataset/termdb.test.js

@@ -438,7 +438,7 @@ function termdb_test_default() {
         jsonFile: "files/hg38/TermdbTest/trackLst/facet.json",
         activeTracks: ["bw 1", "bed 1"]
       },
-      chat: { aifiles: "./proteinpaint/server/dataset/ai/termdb.test.json" }
+      chat: {}
     }
   };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.180.0",
+  "version": "2.180.1",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -63,10 +63,10 @@
   "dependencies": {
     "@sjcrh/augen": "2.143.0",
     "@sjcrh/proteinpaint-python": "2.179.0",
-    "@sjcrh/proteinpaint-r": "2.178.0",
-    "@sjcrh/proteinpaint-rust": "2.180.0",
-    "@sjcrh/proteinpaint-shared": "2.180.0",
-    "@sjcrh/proteinpaint-types": "2.180.0",
+    "@sjcrh/proteinpaint-r": "2.180.1-0",
+    "@sjcrh/proteinpaint-rust": "2.180.1-0",
+    "@sjcrh/proteinpaint-shared": "2.180.1",
+    "@sjcrh/proteinpaint-types": "2.180.1",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",

package/routes/termdb.categories.js

@@ -48,23 +48,11 @@ async function trigger_getcategories(q, res, tdb, ds) {
   const data = await getData(arg, ds);
   if (data.error) throw data.error;
   const [lst, orderedLabels] = getCategories(data, q, ds, $id);
-  const allowedValues = tdb.getRestrictedValues?.(q.__protected__?.clientAuthResult, q.tw.term.id);
-  const filtered = filterCategoriesByAllowedValues(lst, orderedLabels, allowedValues);
   res.send({
-    lst: filtered.lst,
-    orderedLabels: filtered.orderedLabels
+    lst,
+    orderedLabels
   });
 }
-function filterCategoriesByAllowedValues(lst, orderedLabels, allowedValues) {
-  if (!allowedValues) {
-    return { lst, orderedLabels };
-  }
-  const allowedValuesSet = new Set(allowedValues.map((v) => String(v)));
-  const filteredLst = lst.filter((item) => allowedValuesSet.has(String(item.key)));
-  const allowedLabels = new Set(filteredLst.map((item) => item.label));
-  const filteredOrderedLabels = orderedLabels.filter((label) => allowedLabels.has(label));
-  return { lst: filteredLst, orderedLabels: filteredOrderedLabels };
-}
 function getCategories(data, q, ds, $id) {
   const lst = [];
   if (q.tw.term.type == "geneVariant" && q.tw.q.type != "predefined-groupset" && q.tw.q.type != "custom-groupset") {
@@ -145,6 +133,5 @@ function getCategories(data, q, ds, $id) {
 }
 export {
   api,
-  filterCategoriesByAllowedValues,
   getCategories
 };

package/routes/termdb.chat2.js

@@ -2,7 +2,6 @@ import { ChatPayload } from "#types/checkers";
 import { classifyQuery } from "./chat/classify1.ts";
 import { classifyNotPlot } from "./chat/classify2.ts";
 import { classifyPlotType } from "./chat/plot.ts";
-import { readJSONFile } from "./chat/utils.ts";
 import { extract_DE_search_terms_from_query } from "./chat/DEagent.ts";
 import { determineAmbiguousGenePrompt } from "./chat/ambiguousgeneagent.ts";
 import { extract_summary_terms } from "./chat/summaryagent.ts";
@@ -14,6 +13,9 @@ import { extractGenesFromPrompt, parse_dataset_db, parse_geneset_db, getGenesetN
 import serverconfig from "../src/serverconfig.js";
 import { mayLog } from "#src/helpers.ts";
 import { formatElapsedTime } from "#shared";
+import path from "path";
+import fs from "fs";
+import { readJSONFile } from "./chat/utils.ts";
 const api = {
   endpoint: "termdb/chat2",
   methods: {
@@ -35,8 +37,14 @@ function init({ genomes }) {
       if (!g) throw "invalid genome";
       const ds = g.datasets?.[q.dslabel];
       if (!ds) throw "invalid dslabel";
-      if (!ds?.queries?.chat?.aifiles) {
-        throw "aifiles are missing for chatbot to work";
+      const aiFilesDir = serverconfig.binpath + "/../../dataset/ai/" + q.dslabel;
+      let agentFiles = [];
+      try {
+        agentFiles = fs.readdirSync(aiFilesDir).filter((file) => file.endsWith(".json"));
+      } catch (err) {
+        if (err.code === "ENOENT") throw new Error(`Directory not found: ${aiFilesDir}`);
+        if (err.code === "ENOTDIR") throw new Error(`Path is not a directory: ${aiFilesDir}`);
+        throw err;
       }
       const llm = serverconfig.llm;
       if (!llm) throw "serverconfig.llm is not configured";
@@ -45,18 +53,18 @@ function init({ genomes }) {
       }
       const dataset_db = serverconfig.tpmasterdir + "/" + ds.cohort.db.file;
       const genedb = serverconfig.tpmasterdir + "/" + g.genedb.dbfile;
-      const dataset_json = await readJSONFile(ds?.queries?.chat?.aifiles);
       const testing = false;
       const genesetNames = getGenesetNames(g);
       const ai_output_json = await run_chat_pipeline(
         q.prompt,
         llm,
-        dataset_json,
         testing,
         dataset_db,
         genedb,
         ds,
-        genesetNames
+        genesetNames,
+        agentFiles,
+        aiFilesDir
       );
       res.send(ai_output_json);
     } catch (e) {
@@ -65,14 +73,17 @@ function init({ genomes }) {
     }
   };
 }
-async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, dataset_db, genedb, ds, genesetNames = []) {
+async function run_chat_pipeline(user_prompt, llm, testing, dataset_db, genedb, ds, genesetNames = [], agentFiles, aiFilesDir) {
+  if (!fs.existsSync(path.join(aiFilesDir, "main.json")))
+    throw "Main data file is not specified for dataset:" + ds.label;
+  const dataset_json = await readJSONFile(path.join(aiFilesDir, "main.json"));
   const time1 = (/* @__PURE__ */ new Date()).valueOf();
   const class_response = await classifyQuery(user_prompt, llm);
   let ai_output_json;
   mayLog("Time taken for classification:", formatElapsedTime(Date.now() - time1));
   if (class_response.type == "notplot") {
     const time2 = (/* @__PURE__ */ new Date()).valueOf();
-    const notPlotResult = await classifyNotPlot(user_prompt, llm, dataset_json);
+    const notPlotResult = await classifyNotPlot(user_prompt, llm, agentFiles, aiFilesDir);
     mayLog("Time taken for classify2:", formatElapsedTime(Date.now() - time2));
     if (notPlotResult.type == "html") {
       ai_output_json = notPlotResult;
@@ -127,7 +138,8 @@ async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, datase
         ds,
         testing,
         genesetNames,
-        geneFeatures
+        geneFeatures,
+        aiFilesDir
       );
       mayLog("Time taken for summary agent:", formatElapsedTime(Date.now() - time12));
     } else if (classResult == "dge") {
@@ -138,7 +150,8 @@ async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, datase
         dataset_db_output,
         dataset_json,
         ds,
-        testing
+        testing,
+        aiFilesDir
       );
       mayLog("Time taken for DE agent:", formatElapsedTime(Date.now() - time12));
     } else if (classResult == "survival") {
@@ -153,7 +166,8 @@ async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, datase
         ds,
         testing,
         genesetNames,
-        geneFeatures
+        geneFeatures,
+        aiFilesDir
       );
       mayLog("Time taken for matrix agent:", formatElapsedTime(Date.now() - time12));
     } else if (classResult == "samplescatter") {
@@ -166,7 +180,8 @@ async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, datase
         ds,
         testing,
         genesetNames,
-        geneFeatures
+        geneFeatures,
+        aiFilesDir
       );
       mayLog("Time taken for sampleScatter agent:", formatElapsedTime(Date.now() - time12));
     } else if (classResult == "hiercluster") {
@@ -179,7 +194,8 @@ async function run_chat_pipeline(user_prompt, llm, dataset_json, testing, datase
         ds,
         testing,
         genesetNames,
-        geneFeatures
+        geneFeatures,
+        aiFilesDir
       );
       mayLog("Time taken for hierCluster agent:", formatElapsedTime(Date.now() - time12));
     } else if (classResult == "lollipop") {

package/routes/termdb.cluster.js

@@ -7,8 +7,7 @@ import serverconfig from "#src/serverconfig.js";
 import { gdc_validate_query_geneExpression } from "#src/mds3.gdc.js";
 import { mayLimitSamples } from "#src/mds3.filter.js";
 import { clusterMethodLst, distanceMethodLst } from "#shared/clustering.js";
-import { TermTypes, NUMERIC_DICTIONARY_TERM } from "#shared/terms.js";
-import { getData } from "#src/termdb.matrix.js";
+import { TermTypes, PROTEOME_ABUNDANCE } from "#shared/terms.js";
 import { termType2label } from "#shared/terms.js";
 import { formatElapsedTime } from "#shared/time.js";
 const api = {
@@ -35,16 +34,16 @@ function init({ genomes }) {
       if (!ds) throw "invalid dataset name";
       if (ds.label === "GDC" && !ds.__gdc?.doneCaching)
         throw "The server has not finished caching the case IDs: try again in about 2 minutes.";
-      if ([TermTypes.GENE_EXPRESSION, TermTypes.METABOLITE_INTENSITY, NUMERIC_DICTIONARY_TERM].includes(q.dataType)) {
-        if (!ds.queries?.[q.dataType] && q.dataType !== NUMERIC_DICTIONARY_TERM)
-          throw `no ${q.dataType} data on this dataset`;
+      if ([TermTypes.GENE_EXPRESSION, TermTypes.METABOLITE_INTENSITY].includes(q.dataType)) {
+        if (!ds.queries?.[q.dataType]) throw `no ${q.dataType} data on this dataset`;
         if (!q.terms) throw `missing gene list`;
         if (!Array.isArray(q.terms)) throw `gene list is not an array`;
         if (q.terms.length < 3)
           throw `A minimum of three genes is required for clustering. Please refresh this page to clear this error.`;
         result = await getResult(q, ds);
-      } else if (TermTypes.WHOLE_PROTEOME_ABUNDANCE == q.dataType) {
-        if (!ds.queries?.proteome?.whole) throw `no ${TermTypes.WHOLE_PROTEOME_ABUNDANCE} data on this dataset`;
+      } else if (PROTEOME_ABUNDANCE == q.dataType) {
+        const proteomeQuery = ds.queries?.proteome;
+        if (!proteomeQuery?.get) throw `no ${TermTypes.PROTEOME_ABUNDANCE} data getter on this dataset`;
         if (!q.terms) throw `missing gene list`;
         if (!Array.isArray(q.terms)) throw `gene list is not an array`;
         if (q.terms.length < 3)
@@ -71,12 +70,9 @@ async function getResult(q, ds) {
     _q.__abortSignal = q.__abortSignal;
   }
   let term2sample2value, byTermId, bySampleId, skippedSexChrGenes;
-  if (q.dataType == NUMERIC_DICTIONARY_TERM) {
+  if (q.dataType == PROTEOME_ABUNDANCE) {
     ;
-    ({ term2sample2value, byTermId, bySampleId } = await getNumericDictTermAnnotation(q, ds));
-  } else if (q.dataType == TermTypes.WHOLE_PROTEOME_ABUNDANCE) {
-    ;
-    ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries.proteome.whole.get(_q, ds));
+    ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries.proteome.get(_q));
   } else {
     ;
     ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries[q.dataType].get(_q, ds));
@@ -113,28 +109,6 @@ async function getResult(q, ds) {
   if (removedHierClusterTerms.length) result.removedHierClusterTerms = removedHierClusterTerms;
   return result;
 }
-async function getNumericDictTermAnnotation(q, ds) {
-  const getDataArgs = {
-    // TODO: figure out when term is not a termwrapper
-    terms: q.terms.map((tw) => tw.term ? tw : { term: tw, q: { mode: "continuous" } }),
-    filter: q.filter,
-    filter0: q.filter0,
-    __protected__: q.__protected__
-  };
-  const data = await getData(getDataArgs, ds);
-  const term2sample2value = /* @__PURE__ */ new Map();
-  for (const [key, sampleData] of Object.entries(data.samples)) {
-    for (const [term, value] of Object.entries(sampleData)) {
-      if (term !== "sample") {
-        if (!term2sample2value.has(term)) {
-          term2sample2value.set(term, {});
-        }
-        term2sample2value.get(term)[key] = value.value;
-      }
-    }
-  }
-  return { term2sample2value, byTermId: data.refs.byTermId, bySampleId: data.refs.bySampleId };
-}
 async function doClustering(data, q, numCases = 1e3) {
   const sampleSet = /* @__PURE__ */ new Set();
   let firstTerm = true;
@@ -227,13 +201,14 @@ async function validate_query_geneExpression(ds, genome) {
   }
   throw "unknown queries.geneExpression.src";
 }
-async function queryHDF5(hdf5_file, query) {
-  const jsonInput = JSON.stringify({
+async function queryHDF5(hdf5_file, query, read_mode) {
+  const input = {
     hdf5_file,
     query
-  });
+  };
+  if (read_mode) input.read_mode = read_mode;
   try {
-    const result = await run_rust("readH5", jsonInput);
+    const result = await run_rust("readH5", JSON.stringify(input));
     if (!result || result.length === 0) {
       throw new Error("Failed to retrieve expression data: Empty or missing response");
     }
@@ -254,10 +229,16 @@ async function validateNative(q, ds) {
     if (!vr.samples?.length) throw "HDF5 file has no samples, please check file.";
     for (const sn of vr.samples) {
       const si = ds.cohort.termdb.q.sampleName2id(sn);
-      if (si == void 0) throw `unknown sample ${sn} from HDF5 ${q.file}`;
+      if (si == void 0) {
+        if (ds.cohort.db) {
+          throw `unknown sample ${sn} from HDF5 ${q.file}`;
+        } else {
+          continue;
+        }
+      }
       q.samples.push(si);
     }
-    console.log(`${ds.label}: geneExpression HDF5 file validated. Format: ${vr.format}, Samples:`, vr.samples.length);
+    console.log(`${ds.label}: geneExpression HDF5 file validated. Format: ${vr.format}, Samples:`, q.samples.length);
   } catch (error) {
     throw `${ds.label}: Failed to validate geneExpression HDF5 file: ${error}`;
   }
@@ -270,11 +251,19 @@ async function validateNative(q, ds) {
     const samples = q.samples || [];
     if (limitSamples) {
       for (const sid of limitSamples) {
-        bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        if (ds.cohort?.termdb?.q?.id2sampleRefs) {
+          bySampleId[sid] = ds.cohort.termdb.q.id2sampleRefs(sid);
+        } else {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        }
       }
     } else {
       for (const sid of samples) {
-        bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        if (ds.cohort?.termdb?.q?.id2sampleRefs) {
+          bySampleId[sid] = ds.cohort.termdb.q.id2sampleRefs(sid);
+        } else {
+          bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) };
+        }
       }
     }
     const term2sample2value = /* @__PURE__ */ new Map();
@@ -290,7 +279,8 @@ async function validateNative(q, ds) {
       return { term2sample2value, byTermId };
     }
     const time1 = Date.now();
-    const geneData = JSON.parse(await queryHDF5(q.file, geneNames));
+    const readMode = param.dslabel == "MMRF" ? "bulk" : null;
+    const geneData = JSON.parse(await queryHDF5(q.file, geneNames, readMode));
     console.log("Time taken to run gene query:", formatElapsedTime(Date.now() - time1));
     const genesData = geneData.query_output || {};
     if (!genesData) throw "No expression data returned from HDF5 query";

package/routes/termdb.config.js

@@ -6,7 +6,7 @@ import {
   SINGLECELL_CELLTYPE,
   GENE_EXPRESSION,
   METABOLITE_INTENSITY,
-  WHOLE_PROTEOME_ABUNDANCE,
+  PROTEOME_ABUNDANCE,
   SINGLECELL_GENE_EXPRESSION,
   DNA_METHYLATION,
   SSGSEA
@@ -51,7 +51,6 @@ function make(q, req, res, ds, genome) {
     dataDownloadCatch: tdb.dataDownloadCatch,
     matrix: tdb.matrix,
     hierCluster: tdb.hierCluster,
-    numericDictTermCluster: tdb.numericDictTermCluster,
     mclass: tdb.mclass,
     alwaysRefillCategoricalTermValues: tdb.alwaysRefillCategoricalTermValues,
     isGeneSetTermdb: tdb.isGeneSetTermdb,
@@ -89,7 +88,6 @@ function make(q, req, res, ds, genome) {
   if (tdb.maxAnnoTermsPerClientRequest) c.maxAnnoTermsPerClientRequest = tdb.maxAnnoTermsPerClientRequest;
   addRestrictAncestries(c, tdb);
   addMatrixplots(c, ds);
-  addMutationSignatureplots(c, ds);
   addNonDictionaryQueries(c, ds, genome);
   c.requiredAuth = authApi.getRequiredCredForDsEmbedder(q.dslabel, q.embedder);
   const info = authApi.getNonsensitiveInfo(req);
@@ -132,15 +130,6 @@ function addMatrixplots(c, ds) {
     return { name: p.name };
   });
 }
-function addMutationSignatureplots(c, ds) {
-  const mutationSignatureplots = ds.cohort.termdb.termCollections?.find(
-    (tc) => tc.name == "Mutation Signature" && tc.type === "numeric"
-  )?.plots;
-  if (!mutationSignatureplots) return;
-  c.mutationSignatureplots = mutationSignatureplots.map((p) => {
-    return { name: p.name };
-  });
-}
 function addNonDictionaryQueries(c, ds, genome) {
   const q = ds.queries;
   if (!q) return;
@@ -198,6 +187,26 @@ function addNonDictionaryQueries(c, ds, genome) {
   if (q.geneExpression) {
     q2.geneExpression = { unit: q.geneExpression.unit };
   }
+  if (q.proteome) {
+    q2.proteome = {};
+    if (q.proteome.assays) {
+      q2.proteome.assays = {};
+      for (const assay in q.proteome.assays) {
+        q2.proteome.assays[assay] = {};
+        if (q.proteome.assays[assay].cohorts) {
+          q2.proteome.assays[assay].cohorts = {};
+          for (const cohort in q.proteome.assays[assay].cohorts) {
+            q2.proteome.assays[assay].cohorts[cohort] = {};
+            if ("filter" in q.proteome.assays[assay].cohorts[cohort]) {
+              q2.proteome.assays[assay].cohorts[cohort].filter = JSON.parse(
+                JSON.stringify(q.proteome.assays[assay].cohorts[cohort].filter)
+              );
+            }
+          }
+        }
+      }
+    }
+  }
   if (q.dnaMethylation) {
     q2.dnaMethylation = { unit: q.dnaMethylation.unit };
     if (q.dnaMethylation.promoter) {
@@ -284,7 +293,7 @@ function getAllowedTermTypes(ds) {
   }
   if (ds.queries?.geneExpression) typeSet.add(GENE_EXPRESSION);
   if (ds.queries?.metaboliteIntensity) typeSet.add(METABOLITE_INTENSITY);
-  if (ds.queries?.proteome?.whole) typeSet.add(WHOLE_PROTEOME_ABUNDANCE);
+  if (ds.queries?.proteome?.assays) typeSet.add(PROTEOME_ABUNDANCE);
   if (ds.queries?.ssGSEA) typeSet.add(SSGSEA);
   if (ds.queries?.dnaMethylation) typeSet.add(DNA_METHYLATION);
   if (ds.queries?.singleCell) {

package/routes/termdb.filterTermValues.js

@@ -40,20 +40,11 @@ async function getFilters(query, ds) {
   );
   const tw2List = {};
   for (const tw of query.terms) {
-    let values = getList(samplesPerFilter, filtersData, tw, query.showAll);
-    const allowedValues = ds.cohort.termdb.getRestrictedValues?.(query.__protected__.clientAuthResult, tw.term.id);
-    values = filterByAllowedValues(values, allowedValues);
+    const values = getList(samplesPerFilter, filtersData, tw, query.showAll);
     tw2List[tw.term.id] = values;
   }
   return { ...tw2List };
 }
-function filterByAllowedValues(values, allowedValues) {
-  if (!allowedValues) {
-    return values;
-  }
-  const allowedValuesSet = new Set(allowedValues.map((v) => String(v)));
-  return values.filter((v) => v.value === "" || allowedValuesSet.has(String(v.value)));
-}
 async function getSamplesPerFilter(q, ds) {
   q.ds = ds;
   const samples = {};
@@ -94,6 +85,5 @@ function getList(samplesPerFilter, filtersData, tw, showAll) {
   return filteredValues;
 }
 export {
-  api,
-  filterByAllowedValues
+  api
 };