@sjcrh/proteinpaint-server 2.185.0 → 2.186.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.185.0",
+  "version": "2.186.0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -64,9 +64,9 @@
     "@sjcrh/augen": "2.181.1",
     "@sjcrh/proteinpaint-python": "2.185.0",
     "@sjcrh/proteinpaint-r": "2.181.0",
-    "@sjcrh/proteinpaint-rust": "2.185.0",
-    "@sjcrh/proteinpaint-shared": "2.185.0",
-    "@sjcrh/proteinpaint-types": "2.185.0",
+    "@sjcrh/proteinpaint-rust": "2.186.0",
+    "@sjcrh/proteinpaint-shared": "2.186.0",
+    "@sjcrh/proteinpaint-types": "2.186.0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",

package/routes/termdb.cluster.js

@@ -261,18 +261,24 @@ async function validateNative(q, ds) {
     const vr = JSON.parse(tmp);
     if (vr.status !== "success") throw vr.message;
     if (!vr.samples?.length) throw "HDF5 file has no samples, please check file.";
+    const unknownSamples = /* @__PURE__ */ new Set();
     for (const sn of vr.samples) {
       const si = ds.cohort.termdb.q.sampleName2id(sn);
-      if (si == void 0) {
+      if (si === void 0) {
         if (ds.cohort.db) {
           throw `unknown sample ${sn} from HDF5 ${q.file}`;
         } else {
+          unknownSamples.add(sn);
           continue;
         }
       }
       q.samples.push(si);
     }
     console.log(`${ds.label}: geneExpression HDF5 file validated. Format: ${vr.format}, Samples:`, q.samples.length);
+    if (unknownSamples.size) {
+      const arr = [...unknownSamples];
+      console.log(`unknown samples from geneExpression HDF5 file (${arr.length}): ${arr.join(", ")}`);
+    }
   } catch (error) {
     throw `${ds.label}: Failed to validate geneExpression HDF5 file: ${error}`;
   }

package/routes/termdb.config.js

@@ -195,23 +195,34 @@ function addNonDictionaryQueries(c, ds, genome) {
   }
   if (q.proteome) {
     q2.proteome = {};
-    if (q.proteome.overlayTerm) {
-      q2.proteome.overlayTerm = JSON.parse(JSON.stringify(q.proteome.overlayTerm));
-    }
-    if (q.proteome.assays) {
-      q2.proteome.assays = {};
-      for (const assay in q.proteome.assays) {
-        q2.proteome.assays[assay] = {};
-        if (q.proteome.assays[assay].cohorts) {
-          q2.proteome.assays[assay].cohorts = {};
-          for (const cohort in q.proteome.assays[assay].cohorts) {
-            q2.proteome.assays[assay].cohorts[cohort] = {};
-            const src = q.proteome.assays[assay].cohorts[cohort];
-            if ("controlFilter" in src) {
-              q2.proteome.assays[assay].cohorts[cohort].controlFilter = JSON.parse(JSON.stringify(src.controlFilter));
-            }
-            if ("caseFilter" in src) {
-              q2.proteome.assays[assay].cohorts[cohort].caseFilter = JSON.parse(JSON.stringify(src.caseFilter));
+    if (q.proteome.organisms) {
+      q2.proteome.organisms = {};
+      for (const organism in q.proteome.organisms) {
+        q2.proteome.organisms[organism] = {};
+        const orgSrc = q.proteome.organisms[organism];
+        if (orgSrc.overlayTerm) {
+          q2.proteome.organisms[organism].overlayTerm = JSON.parse(JSON.stringify(orgSrc.overlayTerm));
+        }
+        if (orgSrc.assays) {
+          q2.proteome.organisms[organism].assays = {};
+          for (const assay in orgSrc.assays) {
+            q2.proteome.organisms[organism].assays[assay] = {};
+            if (orgSrc.assays[assay].cohorts) {
+              q2.proteome.organisms[organism].assays[assay].cohorts = {};
+              for (const cohort in orgSrc.assays[assay].cohorts) {
+                q2.proteome.organisms[organism].assays[assay].cohorts[cohort] = {};
+                const src = orgSrc.assays[assay].cohorts[cohort];
+                if ("controlFilter" in src) {
+                  q2.proteome.organisms[organism].assays[assay].cohorts[cohort].controlFilter = JSON.parse(
+                    JSON.stringify(src.controlFilter)
+                  );
+                }
+                if ("caseFilter" in src) {
+                  q2.proteome.organisms[organism].assays[assay].cohorts[cohort].caseFilter = JSON.parse(
+                    JSON.stringify(src.caseFilter)
+                  );
+                }
+              }
             }
           }
         }
@@ -305,7 +316,7 @@ function getDsAllowedTermTypes(ds) {
   if (ds.queries?.geneExpression) typeSet.add(GENE_EXPRESSION);
   if (ds.queries?.isoformExpression) typeSet.add(ISOFORM_EXPRESSION);
   if (ds.queries?.metaboliteIntensity) typeSet.add(METABOLITE_INTENSITY);
-  if (ds.queries?.proteome?.assays) typeSet.add(PROTEOME_ABUNDANCE);
+  if (ds.queries?.proteome) typeSet.add(PROTEOME_ABUNDANCE);
   if (ds.queries?.ssGSEA) typeSet.add(SSGSEA);
   if (ds.queries?.dnaMethylation) typeSet.add(DNA_METHYLATION);
   if (ds.queries?.singleCell) {

package/routes/termdb.proteome.js

@@ -26,42 +26,46 @@ function init({ genomes }) {
       const term = q.term?.term || q.term;
       if (!term?.name) throw "term.name missing";
       const cohorts = [];
-      for (const assayName in ds.queries.proteome.assays) {
-        const assay = ds.queries.proteome.assays[assayName];
-        for (const cohortName in assay.cohorts || {}) {
-          const details = {
-            dbfile: ds.queries.proteome.dbfile,
-            assay: assayName,
-            cohort: cohortName
-          };
-          const tw = {
-            $id: "_",
-            term: {
-              name: term.name,
-              type: "proteomeAbundance",
-              proteomeDetails: details
+      for (const organismName in ds.queries.proteome.organisms) {
+        const organism = ds.queries.proteome.organisms[organismName];
+        for (const assayName in organism.assays) {
+          const assay = organism.assays[assayName];
+          for (const cohortName in assay.cohorts || {}) {
+            const details = {
+              dbfile: ds.queries.proteome.dbfile,
+              organism: organismName,
+              assay: assayName,
+              cohort: cohortName
+            };
+            const tw = {
+              $id: "_",
+              term: {
+                name: term.name,
+                type: "proteomeAbundance",
+                proteomeDetails: details
+              }
+            };
+            const cohortData = await ds.queries.proteome.get({
+              terms: [tw],
+              proteomeDetails: details,
+              filter: q.filter,
+              filter0: q.filter0,
+              for: "proteinView",
+              __abortSignal: q.__abortSignal
+            });
+            const controlSampleIds = cohortData.controlSampleIds || /* @__PURE__ */ new Set();
+            const prior = assay.cohorts[cohortName].prior;
+            for (const entry of cohortData.allEntries || []) {
+              const s2v = entry.s2v;
+              const stats = getCohortStats(s2v, controlSampleIds, prior);
+              delete entry.s2v;
+              entry.foldChange = stats.foldChange;
+              entry.pValue = stats.pValue;
+              entry.testedN = stats.testedN;
+              entry.controlN = stats.controlN;
+              if (assay.mclassOverride) entry.mclassOverride = assay.mclassOverride;
+              cohorts.push(entry);
             }
-          };
-          const cohortData = await ds.queries.proteome.get({
-            terms: [tw],
-            proteomeDetails: details,
-            filter: q.filter,
-            filter0: q.filter0,
-            for: "proteinView",
-            __abortSignal: q.__abortSignal
-          });
-          const controlSampleIds = cohortData.controlSampleIds || /* @__PURE__ */ new Set();
-          const prior = assay.cohorts[cohortName].prior;
-          for (const entry of cohortData.allEntries || []) {
-            const s2v = entry.s2v;
-            const stats = getCohortStats(s2v, controlSampleIds, prior);
-            delete entry.s2v;
-            entry.foldChange = stats.foldChange;
-            entry.pValue = stats.pValue;
-            entry.testedN = stats.testedN;
-            entry.controlN = stats.controlN;
-            if (assay.mclassOverride) entry.mclassOverride = assay.mclassOverride;
-            cohorts.push(entry);
           }
         }
       }
@@ -202,8 +206,8 @@ function lnGamma(z) {
 async function validate_query_proteome(ds) {
   const q = ds.queries.proteome;
   if (!q) return;
-  if (!q.assays) {
-    throw "queries.proteome.assays is missing";
+  if (!q.organisms) {
+    throw "queries.proteome.organisms is missing";
   }
   if (!q.dbfile) {
     throw "queries.proteome.dbfile is missing";
@@ -213,21 +217,31 @@ async function validate_query_proteome(ds) {
   } catch (e) {
     throw `Cannot connect to proteome db ${q.dbfile}: ${e.message || e}`;
   }
-  for (const assayName in q.assays) {
-    const assay = q.assays[assayName];
-    if (assay.columnIdx == null) throw `queries.proteome.assays.${assayName}.columnIdx missing`;
-    if (assay.columnValue == null) throw `queries.proteome.assays.${assayName}.columnValue missing`;
-    if (assay.cohorts) {
-      for (const cohortName in assay.cohorts) {
-        const cohort = assay.cohorts[cohortName];
-        if (!cohort.controlFilter)
-          throw `Missing controlFilter in queries.proteome.assays.${assayName}.cohorts.${cohortName}`;
-        if (!cohort.caseFilter) throw `Missing caseFilter in queries.proteome.assays.${assayName}.cohorts.${cohortName}`;
-        if (!cohort.prior?.d0 || !cohort.prior?.s0sq)
-          throw `Missing prior.d0 and prior.s0sq in queries.proteome.assays.${assayName}.cohorts.${cohortName}`;
+  for (const organismName in q.organisms) {
+    const organism = q.organisms[organismName];
+    if (organism.columnIdx == null) throw `queries.proteome.organisms.${organismName}.columnIdx missing`;
+    if (organism.columnValue == null) throw `queries.proteome.organisms.${organismName}.columnValue missing`;
+    if (!organism.assays || typeof organism.assays != "object")
+      throw `queries.proteome.organisms.${organismName}.assays missing or invalid`;
+    for (const assayName in organism.assays) {
+      const assay = organism.assays[assayName];
+      if (assay.columnIdx == null)
+        throw `queries.proteome.organisms.${organismName}.assays.${assayName}.columnIdx missing`;
+      if (assay.columnValue == null)
+        throw `queries.proteome.organisms.${organismName}.assays.${assayName}.columnValue missing`;
+      if (assay.cohorts) {
+        for (const cohortName in assay.cohorts) {
+          const cohort = assay.cohorts[cohortName];
+          if (!cohort.controlFilter)
+            throw `Missing controlFilter in queries.proteome.organisms.${organismName}.assays.${assayName}.cohorts.${cohortName}`;
+          if (!cohort.caseFilter)
+            throw `Missing caseFilter in queries.proteome.organisms.${organismName}.assays.${assayName}.cohorts.${cohortName}`;
+          if (!cohort.prior?.d0 || !cohort.prior?.s0sq)
+            throw `Missing prior.d0 and prior.s0sq in queries.proteome.organisms.${organismName}.assays.${assayName}.cohorts.${cohortName}`;
+        }
+      } else {
+        throw `Invalid assay structure for "${assayName}". Must have .cohorts`;
       }
-    } else {
-      throw `Invalid assay structure for "${assayName}". Must have .cohorts`;
     }
   }
   q.find = async (arg) => {
@@ -235,22 +249,27 @@ async function validate_query_proteome(ds) {
     if (!Array.isArray(proteins) || proteins.length == 0) throw "queries.proteome.find arg.proteins[] missing";
     const matches = /* @__PURE__ */ new Set();
     const details = arg?.proteomeDetails || {};
+    const organism = details.organism;
     const assay = details.assay;
     const cohort = details.cohort;
     const MAX_FIND_RESULTS = 500;
     const filters = [];
     if (Object.keys(details).length) {
-      if (!assay || !cohort) throw "queries.proteome.find arg.proteomeDetails.{assay,cohort} missing";
-      const assayConfig = q.assays?.[assay];
+      if (!organism || !assay || !cohort)
+        throw "queries.proteome.find arg.proteomeDetails.{organism,assay,cohort} missing";
+      const organismConfig = q.organisms?.[organism];
+      if (!organismConfig) throw `queries.proteome.find invalid organism: ${organism}`;
+      const assayConfig = organismConfig.assays?.[assay];
       if (!assayConfig) throw `queries.proteome.find invalid assay: ${assay}`;
       const cohortConfig = assayConfig?.cohorts?.[cohort];
       if (!cohortConfig) throw `queries.proteome.find invalid cohort: ${cohort}`;
+      const organismFilter = [{ columnIdx: organismConfig.columnIdx, columnValue: organismConfig.columnValue }];
       const assayFilter = [{ columnIdx: assayConfig.columnIdx, columnValue: assayConfig.columnValue }];
       const cohortFilter = (Array.isArray(cohortConfig.caseFilter) ? cohortConfig.caseFilter : []).filter(
         (filter) => !!filter
       );
       if (!cohortFilter.length) throw `queries.proteome.find invalid cohort caseFilter: ${cohort}`;
-      filters.push(...assayFilter, ...cohortFilter);
+      filters.push(...organismFilter, ...assayFilter, ...cohortFilter);
     }
     for (const p of proteins) {
       if (!p) continue;
@@ -280,8 +299,8 @@ async function validate_query_proteome(ds) {
   };
   q.get = async (param) => {
     if (!param?.terms?.length) throw "queries.proteome.get param.terms[] missing";
-    if (!param.proteomeDetails?.assay || !param.proteomeDetails?.cohort)
-      throw "queries.proteome.get param.proteomeDetails.{assay,cohort} missing";
+    if (!param.proteomeDetails?.assay || !param.proteomeDetails?.cohort || !param.proteomeDetails?.organism)
+      throw "queries.proteome.get param.proteomeDetails.{assay,cohort,organism} missing";
     return await getProteomeValuesFromCohort(ds, param, q);
   };
 }
@@ -312,23 +331,28 @@ function buildFilterClause(filters) {
 function queryDbRows(db, matchColumn, matchValue, filters) {
   const { conditions, params } = buildFilterClause(filters);
   const allConditions = [`${matchColumn} = ? COLLATE NOCASE`, ...conditions];
-  const sql = `SELECT identifier, protein_accession, isoform, modsite, gene, sample, value
+  const sql = `SELECT organism, disease, identifier, protein_accession, isoform, modsite, gene, sample, value
 		FROM proteome_abundance
 		WHERE ${allConditions.join(" AND ")}`;
   return db.prepare(sql).all(matchValue, ...params);
 }
 async function getProteomeValuesFromCohort(ds, param, q) {
   const db = ds.queries.proteome.db;
-  const { assay, cohort } = param.proteomeDetails;
-  const assayConfig = q.assays?.[assay];
+  const { assay, cohort, organism } = param.proteomeDetails;
+  const organismConfig = q.organisms?.[organism];
+  if (!organismConfig) throw `queries.proteome invalid organism: ${organism}`;
+  const organismColumnIdx = organismConfig.columnIdx;
+  const organismColumnValue = organismConfig.columnValue;
+  const assayConfig = organismConfig.assays?.[assay];
   if (!assayConfig) throw `queries.proteome.get invalid assay: ${assay}`;
-  const PTMType = q.assays[assay].PTMType;
+  const PTMType = assayConfig.PTMType;
   const assayColumnIdx = assayConfig.columnIdx;
   const assayColumnValue = assayConfig.columnValue;
   const cohortConfig = assayConfig?.cohorts?.[cohort];
   if (!cohortConfig) throw `queries.proteome.get invalid cohort: ${cohort}`;
   const cohortControlFilter = cohortConfig.controlFilter;
   const cohortCaseFilter = cohortConfig.caseFilter;
+  const organismFilter = [{ columnIdx: organismColumnIdx, columnValue: organismColumnValue }];
   const assayFilter = [{ columnIdx: assayColumnIdx, columnValue: assayColumnValue }];
   const term2sample2value = /* @__PURE__ */ new Map();
   const allEntries = [];
@@ -346,8 +370,12 @@ async function getProteomeValuesFromCohort(ds, param, q) {
     }
     const matchColumn = param.for === "proteinView" ? "gene" : "identifier";
     const matchValue = param.for === "proteinView" ? geneName : identifier;
-    const caseRows = queryDbRows(db, matchColumn, matchValue, [...assayFilter, ...cohortCaseFilter]);
-    const controlRows = queryDbRows(db, matchColumn, matchValue, [...assayFilter, ...cohortControlFilter]);
+    const caseRows = queryDbRows(db, matchColumn, matchValue, [...organismFilter, ...assayFilter, ...cohortCaseFilter]);
+    const controlRows = queryDbRows(db, matchColumn, matchValue, [
+      ...organismFilter,
+      ...assayFilter,
+      ...cohortControlFilter
+    ]);
     for (const row of controlRows) {
       const sid = ds.cohort.termdb.q.sampleName2id(row.sample);
       if (sid !== void 0) controlSampleIds.add(String(sid));
@@ -359,8 +387,8 @@ async function getProteomeValuesFromCohort(ds, param, q) {
       if (sid !== void 0) allSampleIds.push(sid);
     }
     const uniqueSampleIds = [...new Set(allSampleIds)];
-    const limitSamples = await mayLimitSamples(param, uniqueSampleIds, ds);
-    if (limitSamples?.size == 0) {
+    const allowedSampleIds = await mayLimitSamples(param, uniqueSampleIds, ds);
+    if (allowedSampleIds?.size == 0) {
       return { term2sample2value: /* @__PURE__ */ new Map(), byTermId: {}, bySampleId: {} };
     }
     if (param.for === "proteinView") {
@@ -368,9 +396,11 @@ async function getProteomeValuesFromCohort(ds, param, q) {
       for (const row of allRows) {
         const sid = ds.cohort.termdb.q.sampleName2id(row.sample);
         if (sid === void 0) continue;
-        if (limitSamples && !limitSamples.has(sid)) continue;
+        if (allowedSampleIds && !allowedSampleIds.has(sid)) continue;
         if (!entryMap.has(row.identifier)) {
           entryMap.set(row.identifier, {
+            organism: row.organism,
+            disease: row.disease,
             uniqueIdentifier: row.identifier,
             assayName: assay,
             cohortName: cohort,
@@ -391,7 +421,7 @@ async function getProteomeValuesFromCohort(ds, param, q) {
       for (const row of allRows) {
         const sid = ds.cohort.termdb.q.sampleName2id(row.sample);
         if (sid === void 0) continue;
-        if (limitSamples && !limitSamples.has(sid)) continue;
+        if (allowedSampleIds && !allowedSampleIds.has(sid)) continue;
         s2v[sid] = row.value;
       }
       if (Object.keys(s2v).length) {

package/routes/termdb.singlecellSamples.js

@@ -135,23 +135,24 @@ function validateDataNative(D, ds) {
   }
   const file2Lines = {};
   D.get = async (q) => {
+    const sampleId = q.sample?.eID || q.sample?.sID;
     if (q.checkPlotAvailability) {
       const plots2 = [];
       for (const plot of D.plots) {
         if (!q.plots.includes(plot.name)) continue;
-        const tsvfile = path.join(
-          serverconfig.tpmasterdir,
-          plot.folder,
-          (q.sample?.eID || q.sample?.sID) + (plot.fileSuffix || "")
-        );
+        const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, sampleId + (plot.fileSuffix || ""));
         try {
           await file_is_readable(tsvfile);
-          plots2.push({
-            name: plot.name,
-            expCells: [],
-            // FIXME avoid breaking client but shouldn't be needed
-            noExpCells: []
-          });
+          plots2.push({ name: plot.name });
+        } catch (_) {
+        }
+      }
+      const imgs = ds.queries.singleCell?.images;
+      if (imgs) {
+        const imgFile = path.join(serverconfig.tpmasterdir, imgs.folder, sampleId, imgs.fileName);
+        try {
+          await file_is_readable(imgFile);
+          plots2.push({ name: imgs?.label || "Image" });
         } catch (_) {
         }
       }
@@ -166,11 +167,7 @@ function validateDataNative(D, ds) {
     }
     for (const plot of D.plots) {
       if (!q.plots.includes(plot.name)) continue;
-      const tsvfile = path.join(
-        serverconfig.tpmasterdir,
-        plot.folder,
-        (q.sample?.eID || q.sample?.sID) + (plot.fileSuffix || "")
-      );
+      const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, sampleId + (plot.fileSuffix || ""));
       if (!file2Lines[tsvfile]) {
         await file_is_readable(tsvfile);
         const text = await read_file(tsvfile);

package/routes/termdb.violinBox.js

@@ -71,7 +71,6 @@ function expandNumericTermCollection(q, data) {
     throw new Error("overlayTw is not supported with numeric termCollection; member terms are used as the overlay");
   if (q.divideTw) throw new Error("divideTw is not supported with numeric termCollection");
   const termlst = term.termlst || [];
-  mayLog("termlst", termlst);
   mayLog(
     `Expanding numeric termCollection with ${termlst.length} member terms and ${Object.keys(data.samples).length} samples`
   );