@sjcrh/proteinpaint-server 2.184.1-0 → 2.185.0

package/routes/termdb.DE.js

@@ -1,15 +1,11 @@
-import fs from "fs";
 import path from "path";
 import { diffExpPayload } from "#types/checkers";
-import { run_rust } from "@sjcrh/proteinpaint-rust";
-import { getData } from "../src/termdb.matrix.js";
-import { get_ds_tdb } from "../src/termdb.js";
-import { run_R } from "@sjcrh/proteinpaint-r";
 import { mayLog } from "#src/helpers.ts";
 import serverconfig from "../src/serverconfig.js";
-import { imageSize } from "image-size";
 import { get_header_txt } from "#src/utils.js";
-import { formatElapsedTime } from "#shared";
+import { run_rust } from "@sjcrh/proteinpaint-rust";
+import { renderVolcano } from "../src/renderVolcano.ts";
+import { readCacheFileOrRecompute, resolveDeContext, resolveSampleGroups } from "../src/diffAnalysis.ts";
 const api = {
   endpoint: "termdb/DE",
   methods: {
@@ -27,244 +23,41 @@ function init({ genomes }) {
   return async (req, res) => {
     try {
       const q = req.query;
-      const genome = genomes[q.genome];
-      if (!genome) throw new Error("invalid genome");
-      const [ds] = get_ds_tdb(genome, q);
-      let term_results = [];
-      if (q.tw) {
-        const terms = [q.tw];
-        term_results = await getData(
-          {
-            filter: q.filter,
-            filter0: q.filter0,
-            terms
-          },
-          ds
-        );
-        if (term_results.error) throw new Error(term_results.error);
-      }
-      let term_results2 = [];
-      if (q.tw2) {
-        const terms2 = [q.tw2];
-        term_results2 = await getData(
-          {
-            filter: q.filter,
-            filter0: q.filter0,
-            terms: terms2
-          },
-          ds
-        );
-        if (term_results2.error) throw new Error(term_results2.error);
+      if (q.preAnalysis) {
+        const { ds, term_results, term_results2 } = await resolveDeContext(q, genomes);
+        const groups = resolveSampleGroups(q, ds, term_results, term_results2);
+        const group1Name = q.samplelst.groups[0].name;
+        const group2Name = q.samplelst.groups[1].name;
+        res.send({
+          data: {
+            [group1Name]: groups.group1names.length,
+            [group2Name]: groups.group2names.length,
+            ...groups.alerts.length ? { alert: groups.alerts.join(" | ") } : {}
+          }
+        });
+        return;
       }
-      const results = await run_DE(req.query, ds, term_results, term_results2);
-      if (!results || !results.data) throw new Error("No data available");
-      res.send(results);
+      const { cacheId, geneData, sample_size1, sample_size2, method, images, bcv } = await readCacheFileOrRecompute({
+        daRequest: q,
+        genomes
+      });
+      const rendered = await renderVolcano(geneData, q.volcanoRender);
+      rendered.cacheId = cacheId;
+      const output = {
+        data: rendered,
+        sample_size1,
+        sample_size2,
+        method,
+        images
+      };
+      if (bcv != null) output.bcv = bcv;
+      res.send(output);
     } catch (e) {
       res.send({ status: "error", error: e.message || e });
       if (e instanceof Error && e.stack) console.log(e);
     }
   };
 }
-async function run_DE(param, ds, term_results, term_results2) {
-  if (param.samplelst?.groups?.length != 2) throw new Error(".samplelst.groups.length!=2");
-  if (param.samplelst.groups[0].values?.length < 1) throw new Error("samplelst.groups[0].values.length<1");
-  if (param.samplelst.groups[1].values?.length < 1) throw new Error("samplelst.groups[1].values.length<1");
-  const q = ds.queries.rnaseqGeneCount;
-  if (!q) return;
-  if (!q.file) throw new Error("unknown data type for rnaseqGeneCount");
-  if (!q.storage_type) throw new Error("storage_type is not defined");
-  param.storage_type = q.storage_type;
-  const group1names = [];
-  const conf1_group1 = [];
-  const conf2_group1 = [];
-  for (const s of param.samplelst.groups[0].values) {
-    if (!Number.isInteger(s.sampleId)) continue;
-    const n = ds.cohort.termdb.q.id2sampleName(s.sampleId);
-    if (!n) continue;
-    if (q.allSampleSet.has(n)) {
-      if (param.tw && !param.tw2) {
-        if (term_results.samples[s.sampleId]) {
-          if (param.tw.q.mode == "continuous") {
-            conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
-          } else {
-            conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["key"]);
-          }
-          group1names.push(n);
-        }
-      } else if (!param.tw && param.tw2) {
-        if (term_results2.samples[s.sampleId]) {
-          if (param.tw2.q.mode == "continuous") {
-            conf2_group1.push(term_results2.samples[s.sampleId][param.tw2.$id]["value"]);
-          } else {
-            conf2_group1.push(term_results2.samples[s.sampleId][param.tw2.$id]["key"]);
-          }
-          group1names.push(n);
-        }
-      } else if (param.tw && param.tw2) {
-        if (term_results.samples[s.sampleId] && term_results2.samples[s.sampleId]) {
-          if (param.tw.q.mode == "continuous") {
-            conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
-          } else {
-            conf1_group1.push(term_results.samples[s.sampleId][param.tw.$id]["key"]);
-          }
-          if (param.tw2.q.mode == "continuous") {
-            conf2_group1.push(term_results2.samples[s.sampleId][param.tw2.$id]["value"]);
-          } else {
-            conf2_group1.push(term_results2.samples[s.sampleId][param.tw2.$id]["key"]);
-          }
-          group1names.push(n);
-        }
-      } else {
-        group1names.push(n);
-      }
-    }
-  }
-  const group2names = [];
-  const conf1_group2 = [];
-  const conf2_group2 = [];
-  for (const s of param.samplelst.groups[1].values) {
-    if (!Number.isInteger(s.sampleId)) continue;
-    const n = ds.cohort.termdb.q.id2sampleName(s.sampleId);
-    if (!n) continue;
-    if (q.allSampleSet.has(n)) {
-      if (param.tw && !param.tw2) {
-        if (term_results.samples[s.sampleId]) {
-          if (param.tw.q.mode == "continuous") {
-            conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
-          } else {
-            conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["key"]);
-          }
-          group2names.push(n);
-        }
-      } else if (!param.tw && param.tw2) {
-        if (term_results2.samples[s.sampleId]) {
-          if (param.tw2.q.mode == "continuous") {
-            conf2_group2.push(term_results2.samples[s.sampleId][param.tw2.$id]["value"]);
-          } else {
-            conf2_group2.push(term_results2.samples[s.sampleId][param.tw2.$id]["key"]);
-          }
-          group2names.push(n);
-        }
-      } else if (param.tw && param.tw2) {
-        if (term_results.samples[s.sampleId] && term_results2.samples[s.sampleId]) {
-          if (param.tw.q.mode == "continuous") {
-            conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["value"]);
-          } else {
-            conf1_group2.push(term_results.samples[s.sampleId][param.tw.$id]["key"]);
-          }
-          if (param.tw2.q.mode == "continuous") {
-            conf2_group2.push(term_results2.samples[s.sampleId][param.tw2.$id]["value"]);
-          } else {
-            conf2_group2.push(term_results2.samples[s.sampleId][param.tw2.$id]["key"]);
-          }
-          group2names.push(n);
-        }
-      } else {
-        group2names.push(n);
-      }
-    }
-  }
-  const sample_size1 = group1names.length;
-  const sample_size2 = group2names.length;
-  const alerts = validateGroups(sample_size1, sample_size2, group1names, group2names);
-  if (param.preAnalysis) {
-    const group1Name = param.samplelst.groups[0].name;
-    const group2Name = param.samplelst.groups[1].name;
-    return {
-      data: {
-        [group1Name]: sample_size1,
-        [group2Name]: sample_size2,
-        ...alerts.length ? { alert: alerts.join(" | ") } : {}
-      }
-    };
-  }
-  if (alerts.length) throw new Error(alerts.join(" | "));
-  const cases_string = group2names.map((i) => i).join(",");
-  const controls_string = group1names.map((i) => i).join(",");
-  const expression_input = {
-    case: cases_string,
-    control: controls_string,
-    data_type: "do_DE",
-    input_file: q.file,
-    cachedir: serverconfig.cachedir,
-    min_count: param.min_count,
-    min_total_count: param.min_total_count,
-    cpm_cutoff: param.cpm_cutoff,
-    storage_type: param.storage_type,
-    DE_method: param.method,
-    mds_cutoff: 1e4
-    // If the dimensions of the read counts matrix is below this threshold, only then the mds image will be generated as its very compute intensive. Number of genes * Number of samples < mds_cutoff for mds generation
-  };
-  if (param.tw) {
-    expression_input.conf1 = [...conf1_group2, ...conf1_group1];
-    expression_input.conf1_mode = param.tw.q.mode;
-    if (new Set(expression_input.conf1).size === 1) {
-      throw new Error("Confounding variable 1 has only one value");
-    }
-  }
-  if (param.tw2) {
-    expression_input.conf2 = [...conf2_group2, ...conf2_group1];
-    expression_input.conf2_mode = param.tw2.q.mode;
-    if (new Set(expression_input.conf2).size === 1) {
-      throw new Error("Confounding variable 2 has only one value");
-    }
-  }
-  const sample_size_limit = 8;
-  if (group1names.length <= sample_size_limit && group2names.length <= sample_size_limit || param.method == "edgeR" || param.method == "limma") {
-    const time12 = (/* @__PURE__ */ new Date()).valueOf();
-    const result2 = JSON.parse(await run_R("edge_newh5.R", JSON.stringify(expression_input)));
-    mayLog("Time taken to run edgeR:", formatElapsedTime(Date.now() - time12));
-    param.method = "edgeR";
-    const ql_imagePath = path.join(serverconfig.cachedir, result2.edgeR_ql_image_name[0]);
-    mayLog("ql_imagePath:", ql_imagePath);
-    await readFileAndDelete(ql_imagePath, "ql_image", result2);
-    if (result2.edgeR_mds_image_name) {
-      const mds_imagePath = path.join(serverconfig.cachedir, result2.edgeR_mds_image_name[0]);
-      mayLog("mds_imagePath:", mds_imagePath);
-      await readFileAndDelete(mds_imagePath, "mds_image", result2);
-    }
-    const images = [result2.ql_image];
-    if (result2.mds_image) images.push(result2.mds_image);
-    const output = {
-      data: result2.gene_data,
-      sample_size2: result2.num_cases[0],
-      sample_size1: result2.num_controls[0],
-      method: param.method,
-      images
-    };
-    if (result2.bcv && result2.bcv[0] !== null && result2.bcv[0] !== void 0) {
-      output.bcv = result2.bcv[0];
-    }
-    return output;
-  }
-  const time1 = (/* @__PURE__ */ new Date()).valueOf();
-  const result = JSON.parse(await run_rust("DEanalysis", JSON.stringify(expression_input)));
-  mayLog("Time taken to run rust DE pipeline:", formatElapsedTime(Date.now() - time1));
-  param.method = "wilcoxon";
-  return { data: result, sample_size1, sample_size2, method: param.method };
-}
-function validateGroups(sample_size1, sample_size2, group1names, group2names) {
-  const alerts = [];
-  if (sample_size1 < 1) alerts.push("sample size of group1 < 1");
-  if (sample_size2 < 1) alerts.push("sample size of group2 < 1");
-  const commonnames = group1names.filter((x) => group2names.includes(x));
-  if (commonnames.length) alerts.push(`Common elements found between both groups: ${commonnames.join(", ")}`);
-  return alerts;
-}
-async function readFileAndDelete(file, key, response) {
-  const plot = await fs.promises.readFile(file);
-  const plotBuffer = Buffer.from(plot).toString("base64");
-  const { width, height } = imageSize(file);
-  const obj = {
-    src: `data:image/png;base64,${plotBuffer}`,
-    size: `${width}x${height}`,
-    key
-  };
-  response[key] = obj;
-  fs.unlink(file, (err) => {
-    if (err) throw new Error(err.message || String(err));
-  });
-}
 async function validate_query_rnaseqGeneCount(ds) {
   const q = ds.queries.rnaseqGeneCount;
   if (!q) return;

package/routes/termdb.cluster.js

@@ -7,8 +7,15 @@ import serverconfig from "#src/serverconfig.js";
 import { gdc_validate_query_geneExpression } from "#src/mds3.gdc.js";
 import { mayLimitSamples } from "#src/mds3.filter.js";
 import { clusterMethodLst, distanceMethodLst } from "#shared/clustering.js";
-import { TermTypes, ISOFORM_EXPRESSION, PROTEOME_ABUNDANCE } from "#shared/terms.js";
-import { termType2label } from "#shared/terms.js";
+import { getData } from "#src/termdb.matrix.js";
+import {
+  GENE_EXPRESSION,
+  METABOLITE_INTENSITY,
+  NUMERIC_DICTIONARY_TERM,
+  termType2label,
+  ISOFORM_EXPRESSION,
+  PROTEOME_ABUNDANCE
+} from "#shared/terms.js";
 import { formatElapsedTime } from "#shared/time.js";
 const api = {
   endpoint: "termdb/cluster",
@@ -34,8 +41,9 @@ function init({ genomes }) {
       if (!ds) throw "invalid dataset name";
       if (ds.label === "GDC" && !ds.__gdc?.doneCaching)
         throw "The server has not finished caching the case IDs: try again in about 2 minutes.";
-      if ([TermTypes.GENE_EXPRESSION, ISOFORM_EXPRESSION, TermTypes.METABOLITE_INTENSITY].includes(q.dataType)) {
-        if (!ds.queries?.[q.dataType]) throw `no ${q.dataType} data on this dataset`;
+      if ([GENE_EXPRESSION, ISOFORM_EXPRESSION, METABOLITE_INTENSITY, NUMERIC_DICTIONARY_TERM].includes(q.dataType)) {
+        if (!ds.queries?.[q.dataType] && q.dataType !== NUMERIC_DICTIONARY_TERM)
+          throw `no ${q.dataType} data on this dataset`;
         if (!q.terms) throw `missing gene list`;
         if (!Array.isArray(q.terms)) throw `gene list is not an array`;
         if (q.terms.length < 3)
@@ -43,7 +51,7 @@ function init({ genomes }) {
         result = await getResult(q, ds);
       } else if (PROTEOME_ABUNDANCE == q.dataType) {
         const proteomeQuery = ds.queries?.proteome;
-        if (!proteomeQuery?.get) throw `no ${TermTypes.PROTEOME_ABUNDANCE} data getter on this dataset`;
+        if (!proteomeQuery?.get) throw `no ${PROTEOME_ABUNDANCE} data getter on this dataset`;
         if (!q.terms) throw `missing gene list`;
         if (!Array.isArray(q.terms)) throw `gene list is not an array`;
         if (q.terms.length < 3)
@@ -64,13 +72,16 @@ function init({ genomes }) {
 }
 async function getResult(q, ds) {
   let _q = q;
-  if (q.dataType == TermTypes.GENE_EXPRESSION) {
+  if (q.dataType == GENE_EXPRESSION) {
     _q = JSON.parse(JSON.stringify(q));
     _q.forClusteringAnalysis = true;
     _q.__abortSignal = q.__abortSignal;
   }
   let term2sample2value, byTermId, bySampleId, skippedSexChrGenes;
-  if (q.dataType == PROTEOME_ABUNDANCE) {
+  if (q.dataType == NUMERIC_DICTIONARY_TERM) {
+    ;
+    ({ term2sample2value, byTermId, bySampleId } = await getNumericDictTermAnnotation(q, ds));
+  } else if (q.dataType == PROTEOME_ABUNDANCE) {
     ;
     ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries.proteome.get(_q));
   } else {
@@ -90,7 +101,7 @@ async function getResult(q, ds) {
   const removedHierClusterTerms = [];
   if (noValueTerms.length) {
     removedHierClusterTerms.push({
-      text: `Skipped ${q.dataType == TermTypes.GENE_EXPRESSION ? "genes" : "items"} with no data`,
+      text: `Skipped ${q.dataType == GENE_EXPRESSION ? "genes" : "items"} with no data`,
       lst: noValueTerms
     });
   }
@@ -100,7 +111,7 @@ async function getResult(q, ds) {
   if (term2sample2value.size == 0) throw "no data";
   if (term2sample2value.size == 1) {
     const g = Array.from(term2sample2value.keys())[0];
-    return { term: { gene: g, type: TermTypes.GENE_EXPRESSION }, data: term2sample2value.get(g) };
+    return { term: { gene: g, type: GENE_EXPRESSION }, data: term2sample2value.get(g) };
   }
   const t = Date.now();
   const clustering = await doClustering(term2sample2value, q, Object.keys(bySampleId).length);
@@ -109,6 +120,29 @@ async function getResult(q, ds) {
   if (removedHierClusterTerms.length) result.removedHierClusterTerms = removedHierClusterTerms;
   return result;
 }
+async function getNumericDictTermAnnotation(q, ds) {
+  const getDataArgs = {
+    // TODO: figure out when term is not a termwrapper
+    terms: q.terms.map((tw) => tw.term ? tw : { term: tw, q: { mode: "continuous" } }),
+    filter: q.filter,
+    filter0: q.filter0,
+    __protected__: q.__protected__
+  };
+  const data = await getData(getDataArgs, ds);
+  if (data.error) throw data.error;
+  const term2sample2value = /* @__PURE__ */ new Map();
+  for (const [key, sampleData] of Object.entries(data.samples)) {
+    for (const [term, value] of Object.entries(sampleData)) {
+      if (term !== "sample") {
+        if (!term2sample2value.has(term)) {
+          term2sample2value.set(term, {});
+        }
+        term2sample2value.get(term)[key] = value.value;
+      }
+    }
+  }
+  return { term2sample2value, byTermId: data.refs.byTermId, bySampleId: data.refs.bySampleId };
+}
 async function doClustering(data, q, numCases = 1e3) {
   const sampleSet = /* @__PURE__ */ new Set();
   let firstTerm = true;
@@ -297,6 +331,7 @@ async function validateNative(q, ds) {
         const sampleId = ds.cohort.termdb.q.sampleName2id(sampleName);
         if (!sampleId) continue;
         if (limitSamples && !limitSamples.has(sampleId)) continue;
+        if (!Number.isFinite(samplesData[sampleName])) continue;
         s2v[sampleId] = samplesData[sampleName];
       }
       if (Object.keys(s2v).length) {

package/routes/termdb.config.js

@@ -52,6 +52,7 @@ function make(q, req, res, ds, genome) {
     dataDownloadCatch: tdb.dataDownloadCatch,
     matrix: tdb.matrix,
     hierCluster: tdb.hierCluster,
+    numericDictTermCluster: tdb.numericDictTermCluster,
     mclass: tdb.mclass,
     alwaysRefillCategoricalTermValues: tdb.alwaysRefillCategoricalTermValues,
     isGeneSetTermdb: tdb.isGeneSetTermdb,

package/routes/termdb.diffMeth.js

@@ -3,6 +3,7 @@ import { getData } from "../src/termdb.matrix.js";
 import { run_R } from "@sjcrh/proteinpaint-r";
 import { mayLog } from "#src/helpers.ts";
 import { formatElapsedTime } from "#shared";
+import { renderVolcano } from "../src/renderVolcano.ts";
 const api = {
   endpoint: "termdb/diffMeth",
   methods: {
@@ -39,7 +40,7 @@ function init({ genomes }) {
         throw new Error(
           "Differential methylation analysis returned no data. Please verify sample selections and try again."
         );
-      if (Array.isArray(results.data) && !results.data.length)
+      if ("totalRows" in results.data && results.data.totalRows === 0)
         throw new Error("No promoters passed filtering. Try relaxing group criteria or selecting more samples.");
       res.send(results);
     } catch (e) {
@@ -168,8 +169,9 @@ async function run_diffMeth(param, ds, term_results, term_results2) {
   const time1 = Date.now();
   const result = JSON.parse(await run_R("diffMeth.R", JSON.stringify(diffMethInput)));
   mayLog("Time taken to run diffMeth:", formatElapsedTime(Date.now() - time1));
+  const rendered = await renderVolcano(result.promoter_data, param.volcanoRender);
   const output = {
-    data: result.promoter_data,
+    data: rendered,
     sample_size1,
     sample_size2
   };

package/routes/termdb.proteome.js

@@ -51,9 +51,10 @@ function init({ genomes }) {
             __abortSignal: q.__abortSignal
           });
           const controlSampleIds = cohortData.controlSampleIds || /* @__PURE__ */ new Set();
+          const prior = assay.cohorts[cohortName].prior;
           for (const entry of cohortData.allEntries || []) {
             const s2v = entry.s2v;
-            const stats = getCohortStats(s2v, controlSampleIds);
+            const stats = getCohortStats(s2v, controlSampleIds, prior);
             delete entry.s2v;
             entry.foldChange = stats.foldChange;
             entry.pValue = stats.pValue;
@@ -71,7 +72,7 @@ function init({ genomes }) {
     }
   };
 }
-function getCohortStats(allS2v, controlSampleIds) {
+function getCohortStats(allS2v, controlSampleIds, prior) {
   if (!allS2v || typeof allS2v != "object") return { foldChange: null, pValue: null, testedN: 0, controlN: 0 };
   const controlValues = [];
   const testedValues = [];
@@ -84,7 +85,10 @@ function getCohortStats(allS2v, controlSampleIds) {
   const controlMean = controlValues?.length ? controlValues.reduce((sum, v) => sum + v, 0) / controlValues.length : null;
   const testedMean = testedValues?.length ? testedValues.reduce((sum, v) => sum + v, 0) / testedValues.length : null;
   const foldChange = testedMean != null && controlMean != null && Number.isFinite(testedMean) && Number.isFinite(controlMean) && controlMean !== 0 ? testedMean / controlMean : null;
-  const pValue = getWelchPValue(testedValues, controlValues);
+  if (!Number.isFinite(prior?.d0) || prior.d0 <= 0 || !Number.isFinite(prior?.s0sq) || prior.s0sq <= 0) {
+    throw "prior with finite positive d0 and s0sq is required for moderated t-test";
+  }
+  const pValue = getModeratedPValue(testedValues, controlValues, prior);
   return {
     foldChange,
     pValue,
@@ -92,36 +96,38 @@ function getCohortStats(allS2v, controlSampleIds) {
     controlN: controlValues.length
   };
 }
-function getWelchPValue(a, b) {
+function getModeratedPValue(a, b, prior) {
   const n1 = a.length;
   const n2 = b.length;
   if (n1 < 2 || n2 < 2) return null;
   const mean1 = a.reduce((s, v) => s + v, 0) / n1;
   const mean2 = b.reduce((s, v) => s + v, 0) / n2;
-  const var1 = sampleVariance(a, mean1);
-  const var2 = sampleVariance(b, mean2);
-  if (!Number.isFinite(var1) || !Number.isFinite(var2)) return null;
-  const se2 = var1 / n1 + var2 / n2;
-  if (!(se2 > 0)) {
+  let ss1 = 0;
+  for (const v of a) {
+    const d = v - mean1;
+    ss1 += d * d;
+  }
+  let ss2 = 0;
+  for (const v of b) {
+    const d = v - mean2;
+    ss2 += d * d;
+  }
+  const dfResidual = n1 + n2 - 2;
+  const pooledVar = (ss1 + ss2) / dfResidual;
+  const { d0, s0sq } = prior;
+  const sTildeSq = (d0 * s0sq + dfResidual * pooledVar) / (d0 + dfResidual);
+  const se = Math.sqrt(sTildeSq * (1 / n1 + 1 / n2));
+  if (!(se > 0)) {
     if (mean1 === mean2) return 1;
     return 1e-300;
   }
-  const t = (mean1 - mean2) / Math.sqrt(se2);
-  const df = se2 * se2 / ((var1 / n1) ** 2 / (n1 - 1) + (var2 / n2) ** 2 / (n2 - 1));
+  const t = (mean1 - mean2) / se;
+  const df = d0 + dfResidual;
   if (!Number.isFinite(df) || df < 0.1) return null;
   const p = 2 * tCdfTail(Math.abs(t), df);
   if (!Number.isFinite(p)) return null;
   return Math.max(1e-300, Math.min(1, p));
 }
-function sampleVariance(lst, mean) {
-  if (lst.length < 2) return NaN;
-  let sumsq = 0;
-  for (const v of lst) {
-    const d = v - mean;
-    sumsq += d * d;
-  }
-  return sumsq / (lst.length - 1);
-}
 function tCdfTail(t, df) {
   const x = df / (df + t * t);
   return 0.5 * regularizedBetaIncomplete(df / 2, 0.5, x);
@@ -217,6 +223,8 @@ async function validate_query_proteome(ds) {
         if (!cohort.controlFilter)
           throw `Missing controlFilter in queries.proteome.assays.${assayName}.cohorts.${cohortName}`;
         if (!cohort.caseFilter) throw `Missing caseFilter in queries.proteome.assays.${assayName}.cohorts.${cohortName}`;
+        if (!cohort.prior?.d0 || !cohort.prior?.s0sq)
+          throw `Missing prior.d0 and prior.s0sq in queries.proteome.assays.${assayName}.cohorts.${cohortName}`;
       }
     } else {
       throw `Invalid assay structure for "${assayName}". Must have .cohorts`;

package/routes/termdb.singlecellDEgenes.js

@@ -25,7 +25,8 @@ function init({ genomes }) {
       if (!ds.queries?.singleCell?.DEgenes || !ds.queries.singleCell.DEgenes.get)
         throw new Error("DE genes not supported on this dataset.");
       result = await ds.queries.singleCell.DEgenes.get(q);
-      if (!result || !result.data || !result?.data?.length) {
+      const isEmpty = !result || !result.data || (Array.isArray(result.data) ? result.data.length === 0 : !result.data.totalRows);
+      if (isEmpty) {
         result = {
           status: 404,
           error: !result ? "No data found." : "No differentially expressed genes found."

package/routes/termdb.singlecellSamples.js

@@ -48,11 +48,11 @@ async function validate_query_singleCell(ds, genome) {
   const q = ds.queries.singleCell;
   if (!q) return;
   if (typeof q.samples != "object") throw new Error("singleCell.samples{} not object");
+  if (typeof q.data != "object") throw new Error("singleCell.data{} not object");
   if (typeof q.samples.get == "function") {
   } else {
-    await validateSamplesNative(q.samples, q.data, ds);
+    await validateSamples(q, ds);
   }
-  if (typeof q.data != "object") throw new Error("singleCell.data{} not object");
   if (q.data.src == "gdcapi") {
     gdc_validate_query_singleCell_data(ds, genome);
   } else if (q.data.src == "native") {
@@ -85,7 +85,8 @@ function validateImages(images) {
   if (!images.label) images.label = "Images";
   if (!images.fileName) throw new Error("images.fileName missing");
 }
-async function validateSamplesNative(S, D, ds) {
+async function validateSamples(q, ds) {
+  const S = q.samples, D = q.data;
   const samples = /* @__PURE__ */ new Map();
   for (const plot of D.plots) {
     for (const fn of await fs.promises.readdir(path.join(serverconfig.tpmasterdir, plot.folder))) {
@@ -102,6 +103,8 @@ async function validateSamplesNative(S, D, ds) {
     }
     if (!plot.colorColumns || plot.colorColumns.length == 0) continue;
   }
+  if (samples.size == 0) throw new Error("no scrna samples found");
+  console.log(samples.size, "singleCell samples loaded from " + ds.label);
   if (S.sampleColumns) {
     for (const { termid } of S.sampleColumns) {
       const term = ds.cohort.termdb.q.termjsonByOneid(termid);
@@ -114,7 +117,13 @@ async function validateSamplesNative(S, D, ds) {
     }
   }
   S.get = () => {
-    return { samples: [...samples.values()] };
+    const re = { samples: [...samples.values()] };
+    if (q.metaResults) {
+      re.metaResults = q.metaResults.map((i) => {
+        return { name: i.name };
+      });
+    }
+    return re;
   };
 }
 function validateDataNative(D, ds) {
@@ -126,6 +135,28 @@ function validateDataNative(D, ds) {
   }
   const file2Lines = {};
   D.get = async (q) => {
+    if (q.checkPlotAvailability) {
+      const plots2 = [];
+      for (const plot of D.plots) {
+        if (!q.plots.includes(plot.name)) continue;
+        const tsvfile = path.join(
+          serverconfig.tpmasterdir,
+          plot.folder,
+          (q.sample?.eID || q.sample?.sID) + (plot.fileSuffix || "")
+        );
+        try {
+          await file_is_readable(tsvfile);
+          plots2.push({
+            name: plot.name,
+            expCells: [],
+            // FIXME avoid breaking client but shouldn't be needed
+            noExpCells: []
+          });
+        } catch (_) {
+        }
+      }
+      return { plots: plots2 };
+    }
     const plots = [];
     let geneExpMap;
     if (ds.queries.singleCell.geneExpression && q.gene) {
@@ -138,7 +169,7 @@ function validateDataNative(D, ds) {
       const tsvfile = path.join(
         serverconfig.tpmasterdir,
         plot.folder,
-        (q.sample?.eID || q.sample?.sID) + plot.fileSuffix
+        (q.sample?.eID || q.sample?.sID) + (plot.fileSuffix || "")
       );
       if (!file2Lines[tsvfile]) {
         await file_is_readable(tsvfile);