@sjcrh/proteinpaint-server 2.175.0 → 2.176.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.175.0",
+  "version": "2.176.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -66,7 +66,7 @@
     "@sjcrh/proteinpaint-r": "2.152.1-0",
     "@sjcrh/proteinpaint-rust": "2.175.0",
     "@sjcrh/proteinpaint-shared": "2.175.0",
-    "@sjcrh/proteinpaint-types": "2.175.0",
+    "@sjcrh/proteinpaint-types": "2.176.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",

package/routes/termdb.boxplot.js

@@ -30,7 +30,10 @@ function init({ genomes }) {
     if (q.overlayTw) terms.push(q.overlayTw);
     if (q.divideTw) terms.push(q.divideTw);
     try {
-      const data = await getData({ filter: q.filter, filter0: q.filter0, terms, __protected__: q.__protected__ }, ds);
+      const data = await getData(
+        { filter: q.filter, filter0: q.filter0, terms, __protected__: q.__protected__, __abortSignal: q.__abortSignal },
+        ds
+      );
       if (data.error) throw new Error(data.error);
       const { absMin, absMax, bins, charts, uncomputableValues, descrStats, outlierMin, outlierMax } = await processData(data, q);
       const returnData = {

package/routes/termdb.categories.js

@@ -42,7 +42,8 @@ async function trigger_getcategories(q, res, tdb, ds) {
     // optional, from mds3 mayAddGetCategoryArgs()
     rglst: q.rglst,
     // optional, from mds3 mayAddGetCategoryArgs()
-    __protected__: q.__protected__
+    __protected__: q.__protected__,
+    __abortSignal: q.__abortSignal
   };
   const data = await getData(arg, ds);
   if (data.error) throw data.error;

package/routes/termdb.chat.js

@@ -188,6 +188,18 @@ async function run_chat_pipeline(user_prompt, llm, aiRoute, dataset_json, testin
         testing
       );
       mayLog("Time taken for matrix agent:", formatElapsedTime(Date.now() - time12));
+    } else if (classResult == "sampleScatter") {
+      const time12 = (/* @__PURE__ */ new Date()).valueOf();
+      ai_output_json = await extract_samplescatter_terms_from_query(
+        user_prompt,
+        llm,
+        dataset_db_output,
+        dataset_json,
+        genes_list,
+        ds,
+        testing
+      );
+      mayLog("Time taken for sampleScatter agent:", formatElapsedTime(Date.now() - time12));
     } else {
       ai_output_json = { type: "html", html: "Unknown classification value" };
     }
@@ -701,6 +713,118 @@ function validate_matrix_response(response, common_genes, dataset_json, ds) {
     return { type: "plot", plot: pp_plot_json };
   }
 }
+async function extract_samplescatter_terms_from_query(prompt, llm, dataset_db_output, dataset_json, genes_list, ds, testing) {
+  if (!dataset_json.prebuiltPlots || dataset_json.prebuiltPlots.length == 0) {
+    return { type: "html", html: "No pre-built scatter plots (t-SNE/UMAP) are available for this dataset" };
+  }
+  const Schema = {
+    $schema: "http://json-schema.org/draft-07/schema#",
+    $ref: "#/definitions/SampleScatterType",
+    definitions: {
+      SampleScatterType: {
+        type: "object",
+        properties: {
+          plotName: {
+            type: "string",
+            description: "Name of the pre-built scatter plot to display"
+          },
+          colorTW: {
+            type: ["string", "null"],
+            description: "Term name or gene name to overlay as color on the scatter plot. Set to null to remove the color overlay."
+          },
+          shapeTW: {
+            type: ["string", "null"],
+            description: "Term name or gene name to overlay as shape on the scatter plot. Set to null to remove the shape overlay."
+          },
+          term0: {
+            type: ["string", "null"],
+            description: "Term name to use for Z/Divide which splits the plot into panels. Set to null to remove the divide overlay."
+          },
+          simpleFilter: {
+            type: "array",
+            items: { $ref: "#/definitions/FilterTerm" },
+            description: "Optional simple filter terms to restrict the sample set"
+          }
+        },
+        required: ["plotName"],
+        additionalProperties: false
+      },
+      ...FILTER_TERM_DEFINITIONS
+    }
+  };
+  const common_genes = extractGenesFromPrompt(prompt, genes_list);
+  const scatter_ds = dataset_json.charts.find((chart) => chart.type == "sampleScatter");
+  if (!scatter_ds) throw "sampleScatter information is not present in the dataset file.";
+  if (scatter_ds.TrainingData.length == 0) throw "No training data is provided for the sampleScatter agent.";
+  const training_data = formatTrainingExamples(scatter_ds.TrainingData);
+  const plotNames = dataset_json.prebuiltPlots.map((p) => p.name).join(", ");
+  let system_prompt = "I am an assistant that extracts overlay parameters for pre-built scatter plots (t-SNE/UMAP). The final output must be in the following JSON format with NO extra comments. The JSON schema is as follows: " + JSON.stringify(Schema) + " The available pre-built plots are: " + plotNames + '. The "plotName" field must match one of these exactly. The "colorTW", "shapeTW", and "term0" fields should contain names of clinical fields from the sqlite db OR gene names. To remove an overlay, set the corresponding field to null explicitly. If the user does not mention a particular overlay, do NOT include that field in the output (omit it entirely). Only include "colorTW", "shapeTW", or "term0" if the user explicitly mentions coloring, shaping, or dividing. ' + FILTER_DESCRIPTION + checkField(dataset_json.DatasetPrompt) + checkField(scatter_ds.SystemPrompt) + "\n The DB content is as follows: " + dataset_db_output.rag_docs.join(",") + " training data is as follows:" + training_data;
+  if (dataset_json.hasGeneExpression && common_genes.length > 0) {
+    system_prompt += "\n List of relevant genes are as follows (separated by comma(,)):" + common_genes.join(",");
+  }
+  system_prompt += " Question: {" + prompt + "} answer:";
+  const response = await route_to_appropriate_llm_provider(system_prompt, llm);
+  if (testing) {
+    return { action: "sampleScatter", response: JSON.parse(response) };
+  } else {
+    return validate_samplescatter_response(response, common_genes, dataset_json, ds);
+  }
+}
+function validate_samplescatter_response(response, common_genes, dataset_json, ds) {
+  const response_type = JSON.parse(response);
+  let html = "";
+  if (response_type.html) html = response_type.html;
+  if (!response_type.plotName) {
+    html += "plotName is required for sample scatter output";
+  } else {
+    const matchedPlot = dataset_json.prebuiltPlots.find(
+      (p) => p.name.toLowerCase() == response_type.plotName.toLowerCase()
+    );
+    if (!matchedPlot) {
+      const availablePlots = dataset_json.prebuiltPlots.map((p) => p.name).join(", ");
+      html += "Unknown plot name: " + response_type.plotName + ". Available plots are: " + availablePlots;
+    }
+  }
+  const pp_plot_json = {
+    chartType: "sampleScatter",
+    name: response_type.plotName
+  };
+  const validateOverlayTerm = (termName, fieldKey) => {
+    if (termName === null) {
+      pp_plot_json[fieldKey] = null;
+      return;
+    }
+    if (termName === void 0) {
+      return;
+    }
+    const termValidation = validate_term(termName, common_genes, dataset_json, ds);
+    if (termValidation.html.length > 0) {
+      html += termValidation.html;
+    } else {
+      const tw = { ...termValidation.term_type };
+      if (termValidation.category == "float" || termValidation.category == "integer") {
+        tw.q = { mode: "continuous" };
+      }
+      pp_plot_json[fieldKey] = tw;
+    }
+  };
+  validateOverlayTerm(response_type.colorTW, "colorTW");
+  validateOverlayTerm(response_type.shapeTW, "shapeTW");
+  validateOverlayTerm(response_type.term0, "term0");
+  if (response_type.simpleFilter && response_type.simpleFilter.length > 0) {
+    const validated_filters = validate_filter(response_type.simpleFilter, ds, "");
+    if (validated_filters.html.length > 0) {
+      html += validated_filters.html;
+    } else {
+      pp_plot_json.filter = validated_filters.simplefilter;
+    }
+  }
+  if (html.length > 0) {
+    return { type: "html", html };
+  } else {
+    return { type: "plot", plot: pp_plot_json };
+  }
+}
 function validate_term(response_term, common_genes, dataset_json, ds) {
   let html = "";
   let term_type;

package/routes/termdb.cluster.js

@@ -62,6 +62,7 @@ async function getResult(q, ds) {
   if (q.dataType == TermTypes.GENE_EXPRESSION) {
     _q = JSON.parse(JSON.stringify(q));
     _q.forClusteringAnalysis = true;
+    _q.__abortSignal = q.__abortSignal;
   }
   let term2sample2value, byTermId, bySampleId, skippedSexChrGenes;
   if (q.dataType == NUMERIC_DICTIONARY_TERM) {

package/routes/termdb.config.js

@@ -80,7 +80,6 @@ function make(q, req, res, ds, genome) {
   if (ds.cohort.boxplots) c.boxplots = ds.cohort.boxplots;
   if (tdb.maxGeneVariantGeneSetSize) c.maxGeneVariantGeneSetSize = tdb.maxGeneVariantGeneSetSize;
   addRestrictAncestries(c, tdb);
-  addScatterplots(c, ds);
   addMatrixplots(c, ds);
   addMutationSignatureplots(c, ds);
   addNonDictionaryQueries(c, ds, genome);
@@ -89,6 +88,7 @@ function make(q, req, res, ds, genome) {
   c.clientAuthResult = info?.clientAuthResult || {};
   if (tdb.displaySampleIds) c.displaySampleIds = tdb.displaySampleIds(c.clientAuthResult);
   c.authFilter = req.query.filter;
+  addScatterplots(c, ds, info);
   res.send({ termdbConfig: c });
 }
 function addRestrictAncestries(c, tdb) {
@@ -97,8 +97,12 @@ function addRestrictAncestries(c, tdb) {
     return { name: i.name, tvs: i.tvs, PCcount: i.PCcount };
   });
 }
-function addScatterplots(c, ds) {
+function addScatterplots(c, ds, info) {
   if (!ds.cohort.scatterplots) return;
+  if (ds.cohort.scatterplots.get) {
+    c.scatterplots = ds.cohort.scatterplots.get(info?.clientAuthResult);
+    return;
+  }
   c.scatterplots = ds.cohort.scatterplots.plots.map((p) => {
     return {
       name: p.name,

package/routes/termdb.descrstats.js

@@ -37,14 +37,20 @@ function init({ genomes }) {
 }
 async function trigger_getDescrStats(q, ds) {
   const data = await getData(
-    { filter: q.filter, filter0: q.filter0, terms: [q.tw], __protected__: q.__protected__ },
+    {
+      filter: q.filter,
+      filter0: q.filter0,
+      terms: [q.tw],
+      __protected__: q.__protected__,
+      __abortSignal: q.__abortSignal
+    },
     ds
   );
   if (data.error) throw data.error;
   const values = [];
   for (const key in data.samples) {
     const sample = data.samples[key];
-    const v = sample[q.tw.$id];
+    const v = sample[q.tw.$id || ""];
     if (!v && v !== 0) {
       continue;
     }

package/routes/termdb.sampleScatter.js

@@ -40,14 +40,14 @@ function init({ genomes }) {
       if (q.scaleDotTW) terms.push(q.scaleDotTW);
       if (q.coordTWs) for (const tw of q.coordTWs) terms.push(tw);
       const data = await getData(
-        { filter: q.filter, filter0: q.filter0, terms, __protected__: q.__protected__ },
+        { filter: q.filter, filter0: q.filter0, terms, __protected__: q.__protected__, __abortSignal: q.__abortSignal },
         ds,
         true
         // FIXME 3rd arg hardcoded to true
       );
       if (data.error) throw new Error(data.error);
       let result;
-      if (q.coordTWs.length > 0) {
+      if (q.coordTWs && q.coordTWs.length > 0) {
         const tmp = await getSampleCoordinatesByTerms(req, q, ds, data);
         cohortSamples = tmp[0];
       } else {
@@ -465,6 +465,8 @@ async function loadFile(p, ds) {
 }
 async function mayInitiateScatterplots(ds) {
   if (!ds.cohort.scatterplots) return;
+  if (typeof ds.cohort.scatterplots.get == "function") {
+  }
   if (!Array.isArray(ds.cohort.scatterplots.plots)) throw new Error("cohort.scatterplots.plots is not array");
   for (const p of ds.cohort.scatterplots.plots) {
     if (!p.name) throw new Error(".name missing from one of scatterplots.plots[]");

package/routes/termdb.violin.js

@@ -52,7 +52,8 @@ async function getViolin(q, ds) {
       filter: q.filter,
       filter0: q.filter0,
       currentGeneNames: q.currentGeneNames,
-      __protected__: q.__protected__
+      __protected__: q.__protected__,
+      __abortSignal: q.__abortSignal
     },
     ds
   );