@sjcrh/proteinpaint-server 2.176.0 → 2.177.0

package/dataset/termdb.test.js

@@ -1,3 +1,36 @@
+const mafFields = [
+  {
+    id: "tumor_DNA",
+    name: "Tumor DNA",
+    parent_id: null,
+    child_ids: ["tumor_DNA_WGS"],
+    isleaf: true,
+    type: "float",
+    default: true,
+    // indicates default maf term (e.g. will be used by default for making maf filters in predefined groupset)
+    min: 0,
+    max: 1,
+    tvs: {
+      ranges: [
+        {
+          start: 0.1,
+          startinclusive: true,
+          stopunbounded: true
+        }
+      ]
+    }
+  }
+];
+const mafFilter = {
+  opts: { joinWith: ["and", "or"] },
+  filter: {
+    type: "tvslst",
+    join: "",
+    in: true,
+    lst: []
+  },
+  terms: mafFields
+};
 function termdb_test_default() {
   return {
     isMds3: true,
@@ -285,6 +318,7 @@ function termdb_test_default() {
             }
           ]
         },
+        mafFilter,
         skewerRim: {
           type: "format",
           formatKey: "origin",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.176.0",
+  "version": "2.177.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -64,9 +64,9 @@
     "@sjcrh/augen": "2.143.0",
     "@sjcrh/proteinpaint-python": "2.174.0",
     "@sjcrh/proteinpaint-r": "2.152.1-0",
-    "@sjcrh/proteinpaint-rust": "2.175.0",
-    "@sjcrh/proteinpaint-shared": "2.175.0",
-    "@sjcrh/proteinpaint-types": "2.176.0",
+    "@sjcrh/proteinpaint-rust": "2.177.0",
+    "@sjcrh/proteinpaint-shared": "2.177.0",
+    "@sjcrh/proteinpaint-types": "2.177.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",

package/routes/grin2.js

@@ -431,17 +431,6 @@ function filterAndConvertSnvIndel(sampleName, entry, options) {
   if (!Number.isInteger(entry.pos)) {
     return null;
   }
-  if (options.minAltAlleleCount !== void 0 && options.minAltAlleleCount > 0) {
-    if (!entry.altCount || entry.altCount < options.minAltAlleleCount) {
-      return null;
-    }
-  }
-  if (options.minTotalDepth !== void 0 && options.minTotalDepth > 0) {
-    const totalDepth = (entry.refCount || 0) + (entry.altCount || 0);
-    if (totalDepth < options.minTotalDepth) {
-      return null;
-    }
-  }
   const start = entry.pos;
   const end = entry.pos;
   return [sampleName, entry.chr, start, end, dt2lesion[dtsnvindel].lesionTypes[0].lesionType];

package/routes/termdb.chat.js

@@ -139,8 +139,7 @@ async function run_chat_pipeline(user_prompt, llm, aiRoute, dataset_json, testin
     user_prompt,
     llm,
     aiRoute,
-    dataset_json,
-    testing
+    dataset_json
   );
   let ai_output_json;
   mayLog("Time taken for classification:", formatElapsedTime(Date.now() - time1));
@@ -302,7 +301,7 @@ async function readJSONFile(file) {
   const json_file = await fs.promises.readFile(file);
   return JSON.parse(json_file.toString());
 }
-async function classify_query_by_dataset_type(user_prompt, llm, aiRoute, dataset_json, testing) {
+async function classify_query_by_dataset_type(user_prompt, llm, aiRoute, dataset_json) {
   const data = await readJSONFile(aiRoute);
   let contents = data["general"];
   for (const key of Object.keys(data)) {
@@ -320,11 +319,7 @@ async function classify_query_by_dataset_type(user_prompt, llm, aiRoute, dataset
   }
   const template = contents + " training data is as follows:" + training_data + " Question: {" + user_prompt + "} Answer: {answer}";
   const response = await route_to_appropriate_llm_provider(template, llm);
-  if (testing) {
-    return { action: "html", response: JSON.parse(response) };
-  } else {
-    return JSON.parse(response);
-  }
+  return JSON.parse(response);
 }
 async function extract_DE_search_terms_from_query(prompt, llm, dataset_db_output, dataset_json, ds, testing) {
   if (dataset_json.hasDE) {
@@ -567,10 +562,14 @@ function validate_summary_response(response, common_genes, dataset_json, ds) {
   const pp_plot_json = { chartType: "summary" };
   let html = "";
   if (response_type.html) html = response_type.html;
-  if (!response_type.term) html += "term type is not present in summary output";
+  if (!response_type.term) {
+    html += "term type is not present in summary output";
+    return { type: "html", html };
+  }
   const term1_validation = validate_term(response_type.term, common_genes, dataset_json, ds);
   if (term1_validation.html.length > 0) {
     html += term1_validation.html;
+    return { type: "html", html };
   } else {
     pp_plot_json.term = term1_validation.term_type;
     if (term1_validation.category == "float" || term1_validation.category == "integer") {
@@ -582,6 +581,7 @@ function validate_summary_response(response, common_genes, dataset_json, ds) {
     const term2_validation = validate_term(response_type.term2, common_genes, dataset_json, ds);
     if (term2_validation.html.length > 0) {
       html += term2_validation.html;
+      return { type: "html", html };
     } else {
       pp_plot_json.term2 = term2_validation.term_type;
       if (term2_validation.category == "float" || term2_validation.category == "integer") {
@@ -594,8 +594,15 @@ function validate_summary_response(response, common_genes, dataset_json, ds) {
   const resolved = resolveChildType(pp_plot_json.category, pp_plot_json.category2, llmChildType);
   if (resolved.error) {
     html += resolved.error;
+    return { type: "html", html };
   } else {
     pp_plot_json.childType = resolved.childType;
+    if (pp_plot_json.childType == "barchart") {
+      pp_plot_json.term.q = { mode: "discrete" };
+      if (pp_plot_json.term2) {
+        pp_plot_json.term2.q = { mode: "discrete" };
+      }
+    }
     if (resolved.bothNumeric && (resolved.childType == "violin" || resolved.childType == "boxplot")) {
       pp_plot_json.term2.q = { mode: "discrete" };
     }
@@ -606,15 +613,12 @@ function validate_summary_response(response, common_genes, dataset_json, ds) {
     const validated_filters = validate_filter(response_type.simpleFilter, ds, "");
     if (validated_filters.html.length > 0) {
       html += validated_filters.html;
+      return { type: "html", html };
     } else {
       pp_plot_json.filter = validated_filters.simplefilter;
     }
   }
-  if (html.length > 0) {
-    return { type: "html", html };
-  } else {
-    return { type: "plot", plot: pp_plot_json };
-  }
+  return { type: "plot", plot: pp_plot_json };
 }
 async function extract_matrix_search_terms_from_query(prompt, llm, dataset_db_output, dataset_json, genes_list, ds, testing) {
   const Schema = {
@@ -647,8 +651,6 @@ async function extract_matrix_search_terms_from_query(prompt, llm, dataset_db_ou
   };
   const common_genes = extractGenesFromPrompt(prompt, genes_list);
   const matrix_ds = dataset_json.charts.filter((chart) => chart.type == "Matrix");
-  console.log("matrix_ds", matrix_ds);
-  console.log("dataset_json.charts", dataset_json.charts);
   if (matrix_ds.length == 0) throw "Matrix information is not present in the dataset file.";
   if (matrix_ds[0].TrainingData.length == 0) throw "No training data is provided for the matrix agent.";
   const training_data = formatTrainingExamples(matrix_ds[0].TrainingData);

package/routes/termdb.cohort.summary.js

@@ -1,6 +1,5 @@
 import { termdbCohortSummaryPayload } from "#types/checkers";
 import { get_ds_tdb } from "#src/termdb.js";
-import { mayCopyFromCookie } from "#src/utils.js";
 import { get_samples } from "#src/termdb.sql.js";
 import { authApi } from "#src/auth.js";
 const api = {
@@ -15,7 +14,6 @@ const api = {
 function init({ genomes }) {
   return async (req, res) => {
     const q = req.query;
-    mayCopyFromCookie(q, req.cookies);
     try {
       const genome = genomes[q.genome];
       if (!genome) throw "invalid genome";

package/routes/termdb.cohorts.js

@@ -1,6 +1,5 @@
 import { termdbCohortsPayload } from "#types/checkers";
 import { get_ds_tdb } from "#src/termdb.js";
-import { mayCopyFromCookie } from "#src/utils.js";
 const api = {
   endpoint: "termdb/cohorts",
   methods: {
@@ -13,7 +12,6 @@ const api = {
 function init({ genomes }) {
   return async (req, res) => {
     const q = req.query;
-    mayCopyFromCookie(q, req.cookies);
     try {
       const genome = genomes[q.genome];
       if (!genome) throw "invalid genome";

package/routes/termdb.config.js

@@ -1,7 +1,6 @@
 import serverconfig from "#src/serverconfig.js";
 import { authApi } from "#src/auth.js";
 import { get_ds_tdb } from "#src/termdb.js";
-import { mayCopyFromCookie } from "#src/utils.js";
 import { TermTypes } from "#shared/terms.js";
 const api = {
   endpoint: "termdb/config",
@@ -20,7 +19,6 @@ const api = {
 function init({ genomes }) {
   return async (req, res) => {
     const q = req.query;
-    mayCopyFromCookie(q, req.cookies);
     try {
       const genome = genomes[q.genome];
       if (!genome) throw "invalid genome";
@@ -79,6 +77,7 @@ function make(q, req, res, ds, genome) {
   if (ds.cohort.correlationVolcano) c.correlationVolcano = ds.cohort.correlationVolcano;
   if (ds.cohort.boxplots) c.boxplots = ds.cohort.boxplots;
   if (tdb.maxGeneVariantGeneSetSize) c.maxGeneVariantGeneSetSize = tdb.maxGeneVariantGeneSetSize;
+  if (tdb.maxAnnoTermsPerClientRequest) c.maxAnnoTermsPerClientRequest = tdb.maxAnnoTermsPerClientRequest;
   addRestrictAncestries(c, tdb);
   addMatrixplots(c, ds);
   addMutationSignatureplots(c, ds);

package/routes/termdb.runChart.js

@@ -1,5 +1,31 @@
 import { runChartPayload } from "#types/checkers";
-import { getNumberFromDate } from "#shared/terms.js";
+import { getDateFromNumber, getNumberFromDate } from "#shared/terms.js";
+function decimalYearToYearMonth(xRaw) {
+  const date = getDateFromNumber(xRaw);
+  const t = date.getTime();
+  if (Number.isFinite(t)) {
+    const yearNum = date.getFullYear();
+    const monthNum = date.getMonth() + 1;
+    if (Number.isFinite(yearNum) && Number.isFinite(monthNum)) return { yearNum, monthNum };
+  }
+  const parts = String(xRaw).split(".");
+  const year = Number(parts[0]);
+  if (!Number.isFinite(year)) return null;
+  let month;
+  if (parts.length > 1) {
+    const decimalPart = parts[1];
+    if (decimalPart.length === 2) {
+      const monthCandidate = Number(decimalPart);
+      month = monthCandidate >= 1 && monthCandidate <= 12 ? monthCandidate : Math.floor((xRaw - year) * 12) + 1;
+    } else {
+      month = Math.floor((xRaw - year) * 12) + 1;
+    }
+  } else {
+    month = 1;
+  }
+  if (!Number.isFinite(month) || month < 1 || month > 12) return null;
+  return { yearNum: year, monthNum: month };
+}
 const api = {
   endpoint: "termdb/runChart",
   methods: {
@@ -14,9 +40,16 @@ const api = {
   }
 };
 async function getRunChart(q, ds) {
-  const terms = [q.xtw, q.ytw];
   const xTermId = q.xtw["$id"] ?? q.xtw.term?.id;
-  const yTermId = q.ytw["$id"] ?? q.ytw.term?.id;
+  const yTermId = q.ytw ? q.ytw["$id"] ?? q.ytw.term?.id : void 0;
+  if (xTermId == null || xTermId === "") {
+    throw new Error("runChart requires xtw with $id or term.id");
+  }
+  const isFrequency = !q.ytw;
+  if (!isFrequency && (yTermId == null || yTermId === "")) {
+    throw new Error("runChart requires ytw with $id or term.id when ytw is provided");
+  }
+  const terms = isFrequency ? [q.xtw] : [q.xtw, q.ytw];
   const { getData } = await import("../src/termdb.matrix.js");
   const data = await getData(
     {
@@ -29,7 +62,13 @@ async function getRunChart(q, ds) {
   );
   if (data.error) throw new Error(data.error);
   const shouldPartition = q.xtw?.q?.mode === "discrete";
-  return buildRunChartFromData(q.aggregation, xTermId, yTermId, data, shouldPartition, xTermId);
+  if (isFrequency) {
+    return buildFrequencyFromData(xTermId, data, shouldPartition, xTermId, q.showCumulativeFrequency === true);
+  }
+  if (yTermId == null || yTermId === "") {
+    throw new Error("runChart requires ytw with $id or term.id when ytw is provided");
+  }
+  return buildRunChartFromData(q.aggregation ?? "median", xTermId, yTermId, data, shouldPartition, xTermId);
 }
 function buildRunChartFromData(aggregation, xTermId, yTermId, data, shouldPartition, partitionTermId) {
   const allSamples = data.samples || {};
@@ -56,6 +95,81 @@ function buildRunChartFromData(aggregation, xTermId, yTermId, data, shouldPartit
   const one = buildOneSeries(aggregation, xTermId, yTermId, data);
   return { status: "ok", series: [{ ...one }] };
 }
+function buildFrequencyFromData(xTermId, data, shouldPartition, partitionTermId, showCumulativeFrequency) {
+  const allSamples = data.samples || {};
+  if (shouldPartition && partitionTermId) {
+    const period2Samples = {};
+    for (const sampleId in allSamples) {
+      const sample = allSamples[sampleId];
+      const partitionTerm = sample?.[partitionTermId];
+      if (partitionTerm?.key == null) continue;
+      const periodKey = partitionTerm.key;
+      if (!period2Samples[periodKey]) period2Samples[periodKey] = {};
+      period2Samples[periodKey][sampleId] = sample;
+    }
+    const periodKeys = Object.keys(period2Samples).sort();
+    const series = periodKeys.map((seriesId) => {
+      const subset = { samples: period2Samples[seriesId] };
+      const one2 = buildOneSeriesFrequency(xTermId, subset, showCumulativeFrequency === true);
+      return { seriesId, ...one2 };
+    });
+    return { status: "ok", series };
+  }
+  const one = buildOneSeriesFrequency(xTermId, data, showCumulativeFrequency === true);
+  return { status: "ok", series: [{ ...one }] };
+}
+function buildOneSeriesFrequency(xTermId, data, showCumulativeFrequency = false) {
+  if (xTermId == null || xTermId === "") {
+    throw new Error("buildOneSeriesFrequency requires a defined xTermId (xtw.$id or xtw.term.id)");
+  }
+  const buckets = {};
+  for (const sampleId in data.samples || {}) {
+    const sample = data.samples[sampleId];
+    const xRaw = sample?.[xTermId]?.value ?? sample?.[xTermId]?.key;
+    if (xRaw == null) continue;
+    if (typeof xRaw !== "number") {
+      throw new Error(
+        `x value must be a number for sample ${sampleId}: xTermId=${xTermId}, received type ${typeof xRaw}, value: ${xRaw}`
+      );
+    }
+    const parsed = decimalYearToYearMonth(xRaw);
+    if (!parsed) continue;
+    const { yearNum, monthNum } = parsed;
+    const bucketKey = `${yearNum}-${String(monthNum).padStart(2, "0")}`;
+    const bucketDate = new Date(yearNum, monthNum - 1, 1);
+    const xName = bucketDate.toLocaleString("en-US", { month: "long", year: "numeric" });
+    if (!buckets[bucketKey]) {
+      const midMonthX = getNumberFromDate(new Date(yearNum, monthNum - 1, 15));
+      buckets[bucketKey] = {
+        x: Math.round(midMonthX * 100) / 100,
+        xName,
+        count: 0,
+        sortKey: yearNum * 100 + monthNum
+      };
+    }
+    buckets[bucketKey].count += 1;
+  }
+  let points = Object.values(buckets).sort((a, b) => a.sortKey - b.sortKey).map((b) => ({
+    x: b.x,
+    xName: b.xName,
+    y: b.count,
+    sampleCount: b.count
+  }));
+  if (showCumulativeFrequency) {
+    let sum = 0;
+    points = points.map((p) => {
+      sum += p.y;
+      return { ...p, y: sum };
+    });
+  }
+  const yValues = points.map((p) => p.y).filter((v) => typeof v === "number" && !Number.isNaN(v));
+  const median = yValues.length > 0 ? (() => {
+    const sorted = [...yValues].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+  })() : 0;
+  return { median, points };
+}
 function buildOneSeries(aggregation, xTermId, yTermId, data) {
   const supportedAggregations = ["proportion", "count", "median"];
   if (!supportedAggregations.includes(aggregation)) {
@@ -76,39 +190,15 @@ function buildOneSeries(aggregation, xTermId, yTermId, data) {
         `x value must be a number for sample ${sampleId}: xTermId=${xTermId}, received type ${typeof xRaw}, value: ${xRaw}`
       );
     }
-    let year = null;
-    let month = null;
-    const parts = String(xRaw).split(".");
-    year = Number(parts[0]);
-    if (parts.length > 1) {
-      const decimalPart = parts[1];
-      if (decimalPart.length === 2) {
-        const monthCandidate = Number(decimalPart);
-        if (monthCandidate >= 1 && monthCandidate <= 12) {
-          month = monthCandidate;
-        } else {
-          const frac = xRaw - year;
-          month = Math.floor(frac * 12) + 1;
-        }
-      } else {
-        const frac = xRaw - year;
-        month = Math.floor(frac * 12) + 1;
-      }
-    } else {
-      month = 1;
-    }
-    if (year == null || month == null || Number.isNaN(year) || Number.isNaN(month)) {
-      continue;
-    }
-    const yearNum = year;
-    const monthNum = month;
+    const parsed = decimalYearToYearMonth(xRaw);
+    if (!parsed) continue;
+    const { yearNum, monthNum } = parsed;
     const bucketKey = `${yearNum}-${String(monthNum).padStart(2, "0")}`;
-    const x = Number(`${yearNum}.${String(monthNum).padStart(2, "0")}`);
-    const date = new Date(yearNum, monthNum - 1, 1);
-    const xName = date.toLocaleString("en-US", { month: "long", year: "numeric" });
+    const bucketDate = new Date(yearNum, monthNum - 1, 1);
+    const xName = bucketDate.toLocaleString("en-US", { month: "long", year: "numeric" });
     if (!buckets[bucketKey]) {
       buckets[bucketKey] = {
-        x,
+        x: getNumberFromDate(new Date(yearNum, monthNum - 1, 15)),
         xName,
         count: 0,
         missingCount: 0,
@@ -214,6 +304,9 @@ function buildOneSeries(aggregation, xTermId, yTermId, data) {
   })() : 0;
   return { median, points };
 }
+function runChartErrorPayload(message) {
+  return { error: String(message), series: [] };
+}
 function init({ genomes }) {
   return async (req, res) => {
     try {
@@ -225,12 +318,15 @@ function init({ genomes }) {
       const result = await getRunChart(q, ds);
       res.send(result);
     } catch (e) {
-      res.send({ error: e.message || e });
+      res.send(runChartErrorPayload(e.message || e));
     }
   };
 }
 export {
   api,
+  buildFrequencyFromData,
   buildRunChartFromData,
-  getRunChart
+  decimalYearToYearMonth,
+  getRunChart,
+  runChartErrorPayload
 };