@sjcrh/proteinpaint-server 2.138.3-5 → 2.139.0

package/dataset/termdb.test.js

@@ -114,10 +114,11 @@ function termdb_test_default() {
                   name: "Demographics",
                   plots: [
                     {
-                      chartType: "barchart",
-                      settings: { barchart: { colorBars: true, showPercent: true } },
+                      chartType: "violin",
+                      settings: { violin: { showStats: false } },
                       term: {
-                        id: "agedx"
+                        id: "agedx",
+                        q: { mode: "continuous" }
                       }
                     },
                     {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.138.3-5",
+  "version": "2.139.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -60,11 +60,11 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.136.0",
-    "@sjcrh/proteinpaint-python": "2.135.2-0",
+    "@sjcrh/proteinpaint-python": "2.139.0",
     "@sjcrh/proteinpaint-r": "2.137.2-0",
-    "@sjcrh/proteinpaint-rust": "2.137.2-0",
-    "@sjcrh/proteinpaint-shared": "2.137.3",
-    "@sjcrh/proteinpaint-types": "2.138.3-5",
+    "@sjcrh/proteinpaint-rust": "2.138.3-7",
+    "@sjcrh/proteinpaint-shared": "2.138.3-7",
+    "@sjcrh/proteinpaint-types": "2.139.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^9.4.1",

package/routes/grin2.js

@@ -23,6 +23,7 @@ function init({ genomes }) {
   return async (req, res) => {
     try {
       const request = req.query;
+      console.log("[GRIN2] request:", request);
       const g = genomes[request.genome];
       if (!g)
         throw new Error("genome missing");
@@ -96,7 +97,7 @@ async function runGrin2(g, ds, request) {
   const grin2AnalysisTime = Date.now() - grin2AnalysisStart;
   const grin2AnalysisTimeToPrint = Math.round(grin2AnalysisTime / 1e3);
   mayLog(`[GRIN2] Python processing took ${grin2AnalysisTimeToPrint} seconds`);
-  const resultData = JSON.parse(pyResult.stdout);
+  const resultData = JSON.parse(pyResult);
   if (!resultData?.png?.[0]) {
     throw new Error("Invalid Python output: missing PNG data");
   }
@@ -125,21 +126,60 @@ async function processSampleData(samples, ds, request) {
     failedSamples: 0,
     failedFiles: []
   };
+  const dataTypeCounts = {
+    [dtsnvindel]: { samples: 0, entries: 0, processedLesions: 0 },
+    [dtcnv]: { samples: 0, entries: 0, processedLesions: 0 },
+    [dtfusionrna]: { samples: 0, entries: 0, processedLesions: 0 },
+    unknown: /* @__PURE__ */ new Map()
+  };
   mayLog(`[GRIN2] Processing JSON files for ${samples.length.toLocaleString()} samples`);
   for (const sample of samples) {
     try {
       const filepath = path.join(serverconfig.tpmasterdir, ds.queries.singleSampleMutation.folder, sample.name);
       await file_is_readable(filepath);
       const mlst = JSON.parse(await read_file(filepath));
-      const sampleLesions = await processSampleMlst(sample.name, mlst, lesionId, request);
+      const { sampleLesions, sampleDataTypeCounts } = await processSampleMlst(sample.name, mlst, lesionId, request);
       lesions.push(...sampleLesions);
       lesionId += sampleLesions.length;
+      for (const [dt, counts] of Object.entries(sampleDataTypeCounts.known)) {
+        if (counts.entries > 0) {
+          dataTypeCounts[dt].samples++;
+          dataTypeCounts[dt].entries += counts.entries;
+          dataTypeCounts[dt].processedLesions += counts.processedLesions;
+        }
+      }
+      for (const [dt, counts] of sampleDataTypeCounts.unknown) {
+        if (!dataTypeCounts.unknown.has(dt)) {
+          dataTypeCounts.unknown.set(dt, { samples: /* @__PURE__ */ new Set(), entries: 0 });
+        }
+        dataTypeCounts.unknown.get(dt).samples.add(sample.name);
+        dataTypeCounts.unknown.get(dt).entries += counts;
+      }
+      processingSummary.successfulSamples++;
     } catch (error) {
+      processingSummary.failedSamples++;
+      processingSummary.failedFiles?.push(sample.name);
       mayLog(
         `[GRIN2] Error processing sample ${sample.name}: ${typeof error === "object" && error !== null && "message" in error ? error.message : String(error)}`
       );
     }
   }
+  mayLog(`[GRIN2] === DATA TYPE ANALYSIS SUMMARY ===`);
+  mayLog(
+    `[GRIN2] SNV/Indel (dt=${dtsnvindel}): ${dataTypeCounts[dtsnvindel].samples} samples, ${dataTypeCounts[dtsnvindel].entries} entries, ${dataTypeCounts[dtsnvindel].processedLesions} lesions`
+  );
+  mayLog(
+    `[GRIN2] CNV (dt=${dtcnv}): ${dataTypeCounts[dtcnv].samples} samples, ${dataTypeCounts[dtcnv].entries} entries, ${dataTypeCounts[dtcnv].processedLesions} lesions`
+  );
+  mayLog(
+    `[GRIN2] Fusion (dt=${dtfusionrna}): ${dataTypeCounts[dtfusionrna].samples} samples, ${dataTypeCounts[dtfusionrna].entries} entries, ${dataTypeCounts[dtfusionrna].processedLesions} lesions`
+  );
+  if (dataTypeCounts.unknown.size > 0) {
+    mayLog(`[GRIN2] Unknown data types:`);
+    for (const [dt, counts] of dataTypeCounts.unknown) {
+      mayLog(`[GRIN2]   dt="${dt}": ${counts.samples.size} samples, ${counts.entries} entries`);
+    }
+  }
   mayLog(`[GRIN2] Total lesions processed: ${lesions.length.toLocaleString()}`);
   return {
     lesionData: lesions,
@@ -148,38 +188,60 @@ async function processSampleData(samples, ds, request) {
 }
 async function processSampleMlst(sampleName, mlst, startId, request) {
   const lesions = [];
+  const sampleDataTypeCounts = {
+    known: {
+      [dtsnvindel]: { entries: 0, processedLesions: 0 },
+      [dtcnv]: { entries: 0, processedLesions: 0 },
+      [dtfusionrna]: { entries: 0, processedLesions: 0 }
+    },
+    unknown: /* @__PURE__ */ new Map()
+  };
   for (const m of mlst) {
     switch (m.dt) {
       case dtsnvindel: {
+        sampleDataTypeCounts.known[dtsnvindel].entries++;
         if (!request.snvindelOptions)
           break;
         const snvIndelLesion = filterAndConvertSnvIndel(sampleName, m, request.snvindelOptions);
-        if (snvIndelLesion)
+        if (snvIndelLesion) {
           lesions.push(snvIndelLesion);
+          sampleDataTypeCounts.known[dtsnvindel].processedLesions++;
+        }
         break;
       }
       case dtcnv: {
+        sampleDataTypeCounts.known[dtcnv].entries++;
         if (!request.cnvOptions)
           break;
         const cnvLesion = filterAndConvertCnv(sampleName, m, request.cnvOptions);
-        if (cnvLesion)
+        if (cnvLesion) {
           lesions.push(cnvLesion);
+          sampleDataTypeCounts.known[dtcnv].processedLesions++;
+        }
         break;
       }
       case dtfusionrna: {
+        sampleDataTypeCounts.known[dtfusionrna].entries++;
         if (!request.fusionOptions)
           break;
         const fusionLesion = filterAndConvertFusion(sampleName, m, request.fusionOptions);
-        if (fusionLesion)
+        if (fusionLesion) {
           lesions.push(fusionLesion);
+          sampleDataTypeCounts.known[dtfusionrna].processedLesions++;
+        }
         break;
       }
-      default:
-        mayLog(`[GRIN2] Unknown data type "${m.dt}" in sample ${sampleName}, skipping entry`);
+      default: {
+        const currentCount = sampleDataTypeCounts.unknown.get(m.dt) || 0;
+        sampleDataTypeCounts.unknown.set(m.dt, currentCount + 1);
         break;
+      }
     }
   }
-  return lesions;
+  return {
+    sampleLesions: lesions,
+    sampleDataTypeCounts
+  };
 }
 function filterAndConvertSnvIndel(sampleName, entry, options) {
   const opts = {
@@ -188,16 +250,27 @@ function filterAndConvertSnvIndel(sampleName, entry, options) {
     consequences: options?.consequences ?? [],
     hyperMutator: options?.hyperMutator ?? 1e3
   };
-  if (opts.consequences.length > 0 && entry.consequence && !opts.consequences.includes(entry.consequence)) {
+  if (opts.consequences.length > 0 && entry.class && !opts.consequences.includes(entry.class)) {
     return null;
   }
-  return [
-    sampleName,
-    normalizeChromosome(entry.chromosome || entry.chr),
-    String(entry.start || entry.position),
-    String(entry.end || entry.position),
-    "mutation"
-  ];
+  const chromosome = entry.chromosome || entry.chr;
+  const position = entry.start || entry.position || entry.pos;
+  if (!chromosome) {
+    return null;
+  }
+  if (position === void 0 || position === null || position === "undefined") {
+    return null;
+  }
+  const numPosition = parseInt(String(position));
+  if (isNaN(numPosition)) {
+    return null;
+  }
+  const endPosition = entry.end || position;
+  const numEndPosition = parseInt(String(endPosition));
+  if (isNaN(numEndPosition)) {
+    return null;
+  }
+  return [sampleName, normalizeChromosome(chromosome), String(numPosition), String(numEndPosition), "mutation"];
 }
 function filterAndConvertCnv(sampleName, entry, options) {
   const opts = {
@@ -207,18 +280,38 @@ function filterAndConvertCnv(sampleName, entry, options) {
     minSegLength: options?.minSegLength ?? 0,
     hyperMutator: options?.hyperMutator ?? 500
   };
-  const isGain = entry.log2Ratio >= opts.gainThreshold;
-  const isLoss = entry.log2Ratio <= opts.lossThreshold;
-  if (!isGain && !isLoss)
+  const log2Ratio = entry.log2Ratio ?? entry.value ?? entry.ratio;
+  if (log2Ratio === void 0 || log2Ratio === null) {
     return null;
-  const lesionType = entry.log2Ratio >= opts.gainThreshold ? "gain" : "loss";
-  return [
-    sampleName,
-    normalizeChromosome(entry.chromosome || entry.chr),
-    String(entry.start || entry.begin),
-    String(entry.end || entry.stop),
-    lesionType
-  ];
+  }
+  const numLog2Ratio = parseFloat(String(log2Ratio));
+  if (isNaN(numLog2Ratio)) {
+    return null;
+  }
+  const isGain = numLog2Ratio >= opts.gainThreshold;
+  const isLoss = numLog2Ratio <= opts.lossThreshold;
+  if (!isGain && !isLoss) {
+    return null;
+  }
+  const chromosome = entry.chromosome || entry.chr;
+  const start = entry.start || entry.begin;
+  const end = entry.end || entry.stop;
+  if (!chromosome) {
+    return null;
+  }
+  if (start === void 0 || start === null || start === "undefined") {
+    return null;
+  }
+  if (end === void 0 || end === null || end === "undefined") {
+    return null;
+  }
+  const numStart = parseInt(String(start));
+  const numEnd = parseInt(String(end));
+  if (isNaN(numStart) || isNaN(numEnd)) {
+    return null;
+  }
+  const lesionType = numLog2Ratio >= opts.gainThreshold ? "gain" : "loss";
+  return [sampleName, normalizeChromosome(chromosome), String(numStart), String(numEnd), lesionType];
 }
 function filterAndConvertFusion(sampleName, entry, options) {
   const opts = {
@@ -238,6 +331,8 @@ function filterAndConvertFusion(sampleName, entry, options) {
   ];
 }
 function normalizeChromosome(chrom) {
+  if (!chrom)
+    return "chr?";
   return chrom.startsWith("chr") ? chrom : `chr${chrom}`;
 }
 export {

package/routes/termdb.config.js

@@ -265,10 +265,12 @@ function getAllowedTermTypes(ds) {
     for (const t of ds.cohort.termdb.allowedTermTypes)
       typeSet.add(t);
   }
-  if (ds?.queries?.geneExpression)
+  if (ds.queries?.geneExpression)
     typeSet.add(TermTypes.GENE_EXPRESSION);
-  if (ds?.queries?.metaboliteIntensity)
+  if (ds.queries?.metaboliteIntensity)
     typeSet.add(TermTypes.METABOLITE_INTENSITY);
+  if (ds.queries?.ssGSEA)
+    typeSet.add(TermTypes.SSGSEA);
   return [...typeSet];
 }
 function getSelectCohort(ds, req) {

package/routes/termdb.filterTermValues.js

@@ -21,34 +21,30 @@ function init({ genomes }) {
       if (!g)
         throw "invalid genome name";
       const ds = g.datasets?.[req.query.dslabel];
-      getFilters(req.query, ds, res);
+      res.send(await getFilters(req.query, ds));
     } catch (e) {
-      console.log(e);
+      if (e.stack)
+        console.log(e.stack);
       res.send({ status: "error", error: e.message || e });
     }
   };
 }
-async function getFilters(query, ds, res) {
+async function getFilters(query, ds) {
   if (!query.filterByUserSites)
     authApi.mayAdjustFilter(query, ds, query.terms);
-  try {
-    const samplesPerFilter = await getSamplesPerFilter(query, ds);
-    const filtersData = await getData(
-      {
-        terms: query.terms,
-        __protected__: query.__protected__
-      },
-      ds
-    );
-    const tw2List = {};
-    for (const tw of query.terms) {
-      tw2List[tw.term.id] = getList(samplesPerFilter, filtersData, tw, query.showAll);
-    }
-    res.send({ ...tw2List });
-  } catch (e) {
-    console.log(e);
-    res.send({ error: e.message || e });
+  const samplesPerFilter = await getSamplesPerFilter(query, ds);
+  const filtersData = await getData(
+    {
+      terms: query.terms,
+      __protected__: query.__protected__
+    },
+    ds
+  );
+  const tw2List = {};
+  for (const tw of query.terms) {
+    tw2List[tw.term.id] = getList(samplesPerFilter, filtersData, tw, query.showAll);
   }
+  return { ...tw2List };
 }
 function getList(samplesPerFilter, filtersData, tw, showAll) {
   const values = Object.values(tw.term.values);

package/routes/termdb.percentile.js

@@ -1,5 +1,6 @@
 import { percentilePayload } from "#types/checkers";
-import * as termdbsql from "#src/termdb.sql.js";
+import { isNumericTerm } from "#shared/terms.js";
+import { getData } from "#src/termdb.matrix.js";
 import computePercentile from "#shared/compute.percentile.js";
 const api = {
   endpoint: "termdb/getpercentile",
@@ -33,20 +34,23 @@ function init({ genomes }) {
   };
 }
 async function trigger_getpercentile(q, res, ds) {
-  const term = ds.cohort.termdb.q.termjsonByOneid(q.tid);
+  const term = q.term;
   if (!term)
-    throw "invalid termid";
-  if (term.type != "float" && term.type != "integer")
-    throw "not numerical term";
-  const percentile_lst = q.getpercentile;
-  const perc_values = [];
+    throw "term is missing";
+  if (!isNumericTerm(term))
+    throw "not numeric term";
+  const tw = { $id: "_", term, q: { mode: "continuous" } };
+  const data = await getData({ filter: q.filter, filter0: q.filter0, terms: [tw], __protected__: q.__protected__ }, ds);
+  if (data.error)
+    throw data.error;
   const values = [];
-  const rows = await termdbsql.get_rows_by_one_key({
-    ds,
-    key: q.tid,
-    filter: q.filter ? typeof q.filter == "string" ? JSON.parse(q.filter) : q.filter : null
-  });
-  for (const { value } of rows) {
+  for (const key in data.samples) {
+    const sample = data.samples[key];
+    const v = sample[tw.$id];
+    if (!v?.value) {
+      continue;
+    }
+    const value = v.value;
     if (term.values && term.values[value] && term.values[value].uncomputable) {
       continue;
     }
@@ -55,6 +59,8 @@ async function trigger_getpercentile(q, res, ds) {
     }
     values.push(Number(value));
   }
+  const percentile_lst = q.getpercentile;
+  const perc_values = [];
   for (const percentile of percentile_lst) {
     const perc_value = computePercentile(values, percentile);
     perc_values.push(perc_value);