@sjcrh/proteinpaint-server 2.191.5 → 2.192.0

package/routes/termdb.singleCellPlots.js

@@ -1,159 +0,0 @@
-import { getColors, getCoordinate, calculatePadding, xAxisOffSet, yAxisOffSet } from "#shared";
-import { isSingleCellTerm, SINGLECELL_GENE_EXPRESSION, SINGLECELL_CELLTYPE } from "#shared/terms.js";
-import { createCanvas } from "canvas";
-import { scaleLinear } from "d3-scale";
-import { rgb } from "d3-color";
-import { refColor } from "./termdb.sampleScatter.js";
-function init({ genomes }) {
-  return async function(req, res) {
-    try {
-      const q = req.query;
-      if (!q.genome || !q.dslabel) {
-        throw new Error("Genome and dataset label are required for termdb/singleCellPlots request.");
-      }
-      const g = genomes[q.genome];
-      if (!g) throw new Error("Invalid genome name");
-      const ds = g.datasets[q.dslabel];
-      if (!ds) throw new Error("Invalid dataset label");
-      if (!ds.queries?.singleCell) throw new Error("No single cell data on this dataset");
-      return getSingleCellScatter(req, res, ds);
-    } catch (err) {
-      console.error(err);
-      res.status(500).json({ error: err.message || String(err) });
-    }
-  };
-}
-async function getSingleCellScatter(req, res, ds) {
-  const q = req.query;
-  const { name, sample } = q.singleCellPlot;
-  try {
-    const tw = q.colorTW;
-    if (!tw || !isSingleCellTerm(tw.term))
-      throw new Error("colorTW must be provided and be a single cell term for single cell scatter plot");
-    const arg = { plots: [name], sample };
-    if (tw.term.type == SINGLECELL_GENE_EXPRESSION) arg.gene = tw.term.gene;
-    else if (tw.term.type == SINGLECELL_CELLTYPE) arg.colorBy = tw.term.name;
-    else throw new Error(`unsupported single cell term type: ${tw.term.type}`);
-    const data = await ds.queries.singleCell.data.get(arg);
-    const plot = data.plots[0];
-    const cells = [...plot.expCells, ...plot.noExpCells];
-    const groups = tw.q?.customset?.groups;
-    const cat2GrpName = /* @__PURE__ */ new Map();
-    if (groups) {
-      for (const group of groups) {
-        for (const value of Object.values(group.values)) {
-          cat2GrpName.set(value.key, group.name);
-        }
-      }
-    }
-    const samples = [];
-    const categoryCounts = /* @__PURE__ */ new Map();
-    let xMin = Infinity, xMax = -Infinity, yMin = Infinity, yMax = -Infinity, geMin = Infinity, geMax = -Infinity;
-    let totalCellCount = 0;
-    for (const cell of cells) {
-      let category = cell.category;
-      const groupName = cat2GrpName.get(category);
-      if (groupName !== void 0) category = groupName;
-      const isHidden = tw?.q?.hiddenValues ? category in tw.q.hiddenValues : false;
-      totalCellCount++;
-      categoryCounts.set(category, (categoryCounts.get(category) || 0) + 1);
-      if (cell.x < xMin) xMin = cell.x;
-      if (cell.x > xMax) xMax = cell.x;
-      if (cell.y < yMin) yMin = cell.y;
-      if (cell.y > yMax) yMax = cell.y;
-      if (Number.isFinite(cell.geneExp) && cell.geneExp < geMin) geMin = cell.geneExp;
-      if (Number.isFinite(cell.geneExp) && cell.geneExp > geMax) geMax = cell.geneExp;
-      if (isHidden) continue;
-      samples.push({
-        sampleId: cell.cellId,
-        x: cell.x,
-        y: cell.y,
-        z: 0,
-        category,
-        shape: "Ref",
-        hidden: { category: false },
-        geneExp: cell.geneExp
-      });
-    }
-    const colorMap = {};
-    if (tw.term.type == SINGLECELL_CELLTYPE) {
-      const defaultK2c = getColors(categoryCounts.size);
-      const dsTerm = ds.queries.singleCell?.terms ? ds.queries.singleCell.terms.find((t) => t.name == tw.term.name) : void 0;
-      for (const [category, count] of categoryCounts) {
-        const color = tw.term.values?.[category]?.color || dsTerm?.values?.[category]?.color || defaultK2c(category);
-        colorMap[category] = { sampleCount: count, color, key: category };
-      }
-    }
-    const shapeLegend = [["Ref", { sampleCount: totalCellCount, shape: 0, key: "Ref" }]];
-    const colorLegend = Object.entries(colorMap);
-    const output = {
-      range: { xMin, xMax, yMin, yMax, geMin, geMax },
-      //There should only be one chart
-      result: { Default: { colorLegend, shapeLegend } }
-    };
-    if (totalCellCount >= q.canvasSettings.cutoff) {
-      const { src, canvasWidth, canvasHeight } = await makeCanvas(
-        q,
-        samples,
-        colorMap,
-        { xMin, xMax, yMin, yMax, geMin, geMax },
-        tw.term.type
-      );
-      output.result.Default.src = src;
-      output.result.Default.canvasWidth = canvasWidth;
-      output.result.Default.canvasHeight = canvasHeight;
-      output.result.Default.totalSampleCount = totalCellCount;
-    } else {
-      output.result.Default.samples = samples;
-    }
-    res.send(output);
-  } catch (e) {
-    console.log(e);
-    res.send({ error: e.message || e });
-  }
-}
-async function makeCanvas(q, samples, colorMap, range, termType) {
-  const settings = q.canvasSettings;
-  const dpr = settings.devicePixelRatio || 1;
-  const extraSpaceX = calculatePadding(settings.minXScale, settings.maxXScale, range.xMin, range.xMax);
-  const extraSpaceY = calculatePadding(settings.minYScale, settings.maxYScale, range.yMin, range.yMax);
-  const width = settings.width + xAxisOffSet + extraSpaceX + 20;
-  const height = settings.height + yAxisOffSet + extraSpaceY + 20;
-  const canvas = createCanvas(width * dpr, height * dpr);
-  const ctx = canvas.getContext("2d");
-  if (dpr > 1) ctx.scale(dpr, dpr);
-  const xScale = scaleLinear().domain([range.xMin - extraSpaceX, range.xMax + extraSpaceX]).range([xAxisOffSet, settings.width + xAxisOffSet]);
-  const yScale = scaleLinear().domain([range.yMax + extraSpaceY, range.yMin - extraSpaceY]).range([yAxisOffSet, settings.height + yAxisOffSet]);
-  let colorGenerator;
-  if (Number.isFinite(range.geMin) && Number.isFinite(range.geMax)) {
-    colorGenerator = scaleLinear().domain([range.geMin, range.geMax]).range([settings.startColor, settings.stopColor]);
-  }
-  const color = (sample) => {
-    if (termType == SINGLECELL_GENE_EXPRESSION) {
-      if (!Number.isFinite(sample.geneExp)) return settings.startColor;
-      else if (sample.geneExp > range.geMax) return settings.stopColor;
-      else return colorGenerator(sample.geneExp);
-    }
-    return colorMap[sample.category] ? colorMap[sample.category].color : refColor;
-  };
-  const x = (sample) => {
-    const tmp = getCoordinate(sample.x, settings.minXScale, settings.maxXScale);
-    return xScale(tmp);
-  };
-  const y = (sample) => {
-    const tmp = getCoordinate(sample.y, settings.minYScale, settings.maxYScale);
-    return yScale(tmp);
-  };
-  for (const sample of samples) {
-    const c = rgb(color(sample));
-    c.opacity = settings.opacity;
-    ctx.fillStyle = c.toString();
-    ctx.beginPath();
-    ctx.arc(x(sample), y(sample), settings.radius, 0, Math.PI * 2);
-    ctx.fill();
-  }
-  return { src: canvas.toDataURL(), canvasWidth: width, canvasHeight: height };
-}
-export {
-  init
-};

package/routes/termdb.singlecellSamples.js

@@ -1,312 +0,0 @@
-import fs from "fs";
-import path from "path";
-import { read_file, file_is_readable } from "#src/utils.js";
-import { mayLog } from "#src/helpers.ts";
-import { joinUrl } from "#shared/joinUrl.js";
-import { run_rust } from "@sjcrh/proteinpaint-rust";
-import serverconfig from "#src/serverconfig.js";
-import { validate_query_singleCell_DEgenes } from "../src/routes/termdb.singlecellDEgenes.ts";
-import { gdc_validate_query_singleCell_data } from "#src/mds3.gdc.js";
-import ky from "ky";
-import { SINGLECELL_CELLTYPE } from "#shared/terms.js";
-function init({ genomes }) {
-  return async (req, res) => {
-    const q = req.query;
-    let result;
-    try {
-      const g = genomes[q.genome];
-      if (!g) throw new Error("invalid genome name");
-      const ds = g.datasets[q.dslabel];
-      if (!ds) throw new Error("invalid dataset name");
-      if (!ds.queries?.singleCell) throw new Error("no singlecell data on this dataset");
-      result = await ds.queries.singleCell.samples.get(q);
-    } catch (e) {
-      if (e.stack) console.log(e.stack);
-      result = {
-        status: e.status || 400,
-        error: e.message || e
-      };
-    }
-    res.send(result);
-  };
-}
-async function validate_query_singleCell(ds, genome) {
-  const q = ds.queries.singleCell;
-  if (!q) return;
-  if (typeof q.samples != "object") throw new Error("singleCell.samples{} not object");
-  if (typeof q.data != "object") throw new Error("singleCell.data{} not object");
-  if (typeof q.samples.get == "function") {
-  } else {
-    await validateSamples(q, ds);
-  }
-  if (q.data.src == "gdcapi") {
-    gdc_validate_query_singleCell_data(ds, genome);
-  } else if (q.data.src == "native") {
-    validateDataNative(q.data, ds);
-  } else {
-    throw new Error("unknown singleCell.data.src");
-  }
-  colorColumn2terms(ds.queries.singleCell.data.plots, ds);
-  if (q.geneExpression) {
-    if (typeof q.geneExpression != "object") throw new Error("singleCell.geneExpression not object");
-    if (q.geneExpression.src == "native") {
-      validateGeneExpressionNative(q.geneExpression);
-    } else if (q.geneExpression.src == "gdcapi") {
-      gdc_validateGeneExpression(q.geneExpression, ds, genome);
-    } else {
-      throw new Error("unknown singleCell.geneExpression.src");
-    }
-  }
-  if (q.DEgenes) {
-    if (typeof q.DEgenes != "object") throw new Error("singleCell.DEgenes not object");
-    validate_query_singleCell_DEgenes(ds);
-  }
-  if (q.images) {
-    if (typeof q.images != "object") throw new Error("singleCell.images not object");
-    validateImages(q.images);
-  }
-}
-function validateImages(images) {
-  if (!images.folder) throw new Error("images.folder missing");
-  if (!images.label) images.label = "Images";
-  if (!images.fileName) throw new Error("images.fileName missing");
-}
-async function validateSamples(q, ds) {
-  const S = q.samples, D = q.data;
-  const samples = /* @__PURE__ */ new Map();
-  for (const plot of D.plots) {
-    if (plot.isMetaResult) {
-      const sampleName = plot?.sampleId || plot.name.replace(/\s/g, "_");
-      const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, sampleName + (plot.fileSuffix || ""));
-      try {
-        await file_is_readable(tsvfile);
-        samples.set(sampleName, { sample: sampleName, isMetaResult: true });
-      } catch (e) {
-        throw new Error(`meta result data file missing or unreadable: ${sampleName} (${tsvfile}): ${e.message || e}`);
-      }
-      continue;
-    }
-    for (const fn of await fs.promises.readdir(path.join(serverconfig.tpmasterdir, plot.folder))) {
-      let sampleName = fn;
-      if (plot.fileSuffix) {
-        if (!fn.endsWith(plot.fileSuffix))
-          throw new Error(`singlecell.sample file name ${fn} does not end with required suffix ${plot.fileSuffix}`);
-        sampleName = fn.split(plot.fileSuffix)[0];
-      }
-      if (!sampleName) throw new Error(`singlecell.sample: cannot derive sample name from file name ${fn}`);
-      const sid = ds.cohort.termdb.q.sampleName2id(sampleName);
-      if (sid == void 0) throw new Error(`singlecell.sample: unknown sample name ${sampleName}`);
-      samples.set(sid, { sample: sampleName });
-    }
-    if (!plot.colorColumns || plot.colorColumns.length == 0) continue;
-  }
-  if (samples.size == 0) throw new Error("no scrna samples found");
-  console.log(samples.size, "singleCell samples loaded from " + ds.label);
-  if (S.sampleColumns) {
-    for (const { termid } of S.sampleColumns) {
-      const term = ds.cohort.termdb.q.termjsonByOneid(termid);
-      if (!term) throw new Error("unknown termid from singlecell.samples.sampleColumns[]");
-      const s2v = await ds.cohort.termdb.q.getAllValues4term(termid);
-      for (const [sid, v] of s2v.entries()) {
-        if (!samples.has(sid)) continue;
-        samples.get(sid)[termid] = term.values?.[v]?.label || v;
-      }
-    }
-  }
-  S.get = () => {
-    const re = { samples: [...samples.values()] };
-    if (q.metaResults) {
-      re.metaResults = q.metaResults.map((i) => {
-        return { name: i.name };
-      });
-    }
-    return re;
-  };
-}
-function validateDataNative(D, ds) {
-  const nameSet = /* @__PURE__ */ new Set();
-  for (const plot of D.plots) {
-    if (nameSet.has(plot.name)) throw new Error("duplicate plot.name");
-    nameSet.add(plot.name);
-    if (!plot.folder) throw new Error("plot.folder missing");
-  }
-  const file2Lines = {};
-  D.get = async (q) => {
-    const sampleId = q.sample?.eID || q.sample?.sID;
-    if (q.checkPlotAvailability) {
-      const plots2 = [];
-      for (const plot of D.plots) {
-        if (!q.plots.includes(plot.name)) continue;
-        if (plot.isMetaResult) {
-          const sampleName = plot?.sampleId || plot.name.replace(/\s/g, "_");
-          if (sampleName != sampleId) continue;
-        }
-        const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, sampleId + (plot.fileSuffix || ""));
-        try {
-          await file_is_readable(tsvfile);
-          plots2.push({ name: plot.name });
-        } catch (_) {
-        }
-      }
-      const imgs = ds.queries.singleCell?.images;
-      if (imgs) {
-        const imgFile = path.join(serverconfig.tpmasterdir, imgs.folder, sampleId, imgs.fileName);
-        try {
-          await file_is_readable(imgFile);
-          plots2.push({ name: imgs?.label || "Image" });
-        } catch (_) {
-        }
-      }
-      return { plots: plots2 };
-    }
-    const plots = [];
-    let geneExpMap;
-    if (ds.queries.singleCell.geneExpression && q.gene) {
-      const sample = q.sample || q.singleCellPlot.sample;
-      if (!sample) throw new Error("sample is required for gene expression query");
-      geneExpMap = await ds.queries.singleCell.geneExpression.get({ sample, gene: q.gene });
-    }
-    for (const plot of D.plots) {
-      if (!q.plots.includes(plot.name)) continue;
-      const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, sampleId + (plot.fileSuffix || ""));
-      if (!file2Lines[tsvfile]) {
-        await file_is_readable(tsvfile);
-        const text = await read_file(tsvfile);
-        const lines = text.trim().split("\n");
-        let first = true;
-        const lines2 = [];
-        for (const line of lines) {
-          if (first) {
-            first = false;
-            continue;
-          }
-          lines2.push(line.split("	"));
-        }
-        file2Lines[tsvfile] = lines2;
-      }
-      const checkColorBy = typeof q.colorBy == "string" ? q.colorBy : q.colorBy?.[plot.name];
-      const colorColumn = plot.colorColumns.find((c) => c.name == checkColorBy) || plot.colorColumns[0];
-      const expCells = [];
-      const noExpCells = [];
-      for (const l of file2Lines[tsvfile]) {
-        const cellId = l[0], x = Number(l[plot.coordsColumns.x]), y = Number(l[plot.coordsColumns.y]);
-        const category = l[colorColumn?.index] || "";
-        if (!cellId) throw new Error("cell id missing");
-        if (!Number.isFinite(x) || !Number.isFinite(y)) throw new Error("x/y not number");
-        const cell = { cellId, x, y, category };
-        if (geneExpMap) {
-          if (geneExpMap[cellId] !== void 0) {
-            cell.geneExp = geneExpMap[cellId];
-            expCells.push(cell);
-          } else {
-            noExpCells.push(cell);
-          }
-        } else noExpCells.push(cell);
-      }
-      plots.push({
-        name: plot.name,
-        expCells,
-        noExpCells,
-        colorColumns: plot.colorColumns.map((c) => c.name),
-        colorBy: colorColumn?.name,
-        colorMap: colorColumn?.colorMap
-      });
-    }
-    if (plots.length == 0) {
-      return { nodata: true };
-    }
-    return { plots };
-  };
-}
-function validateGeneExpressionNative(G) {
-  G.sample2gene2expressionBins = {};
-  G.get = async (q) => {
-    const h5file = path.join(serverconfig.tpmasterdir, G.folder, (q.sample?.eID || q.sample?.sID) + ".h5");
-    await file_is_readable(h5file);
-    const query_gene = q.gene;
-    if (!query_gene) {
-      throw new Error("Gene parameter is undefined");
-    }
-    const read_hdf5_input_type = { query: [query_gene], hdf5_file: h5file };
-    const time1 = Date.now();
-    const rust_output = await run_rust("readH5", JSON.stringify(read_hdf5_input_type));
-    mayLog("Time taken to query HDF5 file:", Date.now() - time1, "ms");
-    const result = JSON.parse(rust_output);
-    const out = result.query_output[query_gene]?.samples;
-    if (!out) throw new Error(`No expression data for ${query_gene}`);
-    return out;
-  };
-}
-function gdc_validateGeneExpression(G, ds, genome) {
-  G.sample2gene2expressionBins = {};
-  G.get = async (q) => {
-    try {
-      const fileid = q.sample.eID;
-      if (ds.__gdc.scrnaAnalysis2hdf5.size == 0) {
-        throw new Error("GDC scRNA file mapping is not cached");
-      }
-      const hdf5id = ds.__gdc.scrnaAnalysis2hdf5.get(fileid);
-      if (!hdf5id) throw new Error("cannot map eID to hdf5 id");
-      const aliasLst = genome.genedb.getAliasByName.all(q.gene);
-      const gencodeId = aliasLst.find((a) => a?.alias.toUpperCase().startsWith("ENSG"))?.alias;
-      if (!gencodeId) throw new Error("cannot map gene symbol to GENCODE");
-      const body = {
-        gene_ids: [gencodeId],
-        file_id: hdf5id
-      };
-      const { host, headers } = ds.getHostHeaders(q);
-      const t = Date.now();
-      const response = await ky.post(joinUrl(host.rest, "scrna_seq/gene_expression"), {
-        timeout: false,
-        headers,
-        json: body
-      });
-      if (!response.ok) throw new Error(`HTTP Error: ${response.status} ${response.statusText}`);
-      const out = await response.json();
-      mayLog("gdc scrna gene exp", q.gene, Date.now() - t);
-      const result = out.data[0].cells;
-      const data = {};
-      for (const r of result) {
-        data[r.cell_id] = r.value;
-      }
-      return data;
-    } catch (e) {
-      if (e.stack) console.log(e.stack);
-      return { error: "GDC scRNAseq gene expression request failed with error: " + (e.message || e) };
-    }
-  };
-}
-function colorColumn2terms(plots, ds) {
-  const termSet = /* @__PURE__ */ new Set();
-  for (const plot of plots) {
-    const tmpTerms = plot.colorColumns.map((c) => {
-      const baseValues = c.colorMap ? Object.keys(c.colorMap) : [];
-      return {
-        name: c.name,
-        isleaf: true,
-        /** Note, term may apply to multiple plots.
-         * The plot denotes the data file defined in the ds file,
-         * which may be the same or different file paths for
-         * all the plots. */
-        plot: plot.name,
-        type: SINGLECELL_CELLTYPE,
-        groupsetting: {},
-        values: baseValues.reduce((acc, v) => {
-          const alias = c?.aliases?.[v];
-          acc[v] = {
-            key: v,
-            label: alias || v,
-            color: c.colorMap?.[v] || "#000000"
-          };
-          return acc;
-        }, {})
-      };
-    });
-    tmpTerms.forEach((term) => termSet.add(term));
-  }
-  ds.queries.singleCell.terms = [...termSet];
-}
-export {
-  init,
-  validate_query_singleCell
-};