@sjcrh/proteinpaint-server 2.190.2-0 → 2.191.0

package/routes/termdb.DE.js

@@ -1,176 +0,0 @@
-import { mayLog } from "#src/helpers.ts";
-import serverconfig from "../src/serverconfig.js";
-import { run_R } from "@sjcrh/proteinpaint-r";
-import { run_rust } from "@sjcrh/proteinpaint-rust";
-import { formatElapsedTime } from "#shared";
-import { renderVolcano } from "../src/renderVolcano.ts";
-import { cacheOrRecompute } from "#src/utils/cacheOrRecompute.ts";
-import {
-  buildGroupValues,
-  canonicalizeSamplelst,
-  resolveDaContext
-} from "#src/utils/sampleGroups.ts";
-function init({ genomes }) {
-  return async (req, res) => {
-    try {
-      const q = req.query;
-      if (q.preAnalysis) {
-        const { ds, term_results, term_results2 } = await resolveDaContext(q, genomes);
-        const groups = resolveSampleGroups(q, ds, term_results, term_results2);
-        const group1Name = q.samplelst.groups[0].name;
-        const group2Name = q.samplelst.groups[1].name;
-        res.send({
-          data: {
-            [group1Name]: groups.group1names.length,
-            [group2Name]: groups.group2names.length,
-            ...groups.alerts.length ? { alert: groups.alerts.join(" | ") } : {}
-          }
-        });
-        return;
-      }
-      const { result, cacheId, images } = await loadDeForResponse(q, genomes);
-      const rendered = await renderVolcano(result.geneRows, q.volcanoRender);
-      rendered.cacheId = cacheId;
-      const output = {
-        data: rendered,
-        sample_size1: result.sample_size1,
-        sample_size2: result.sample_size2,
-        method: result.method
-      };
-      if (images.length) output.images = images;
-      if (result.bcv != null) output.bcv = result.bcv;
-      res.send(output);
-    } catch (e) {
-      res.status(e.status || 500).send({ status: "error", error: e.message || e, code: e.code });
-      if (e instanceof Error && e.stack) console.log(e);
-    }
-  };
-}
-function deKeyInputs(req) {
-  return {
-    genome: req.genome,
-    dslabel: req.dslabel,
-    samplelst: canonicalizeSamplelst(req.samplelst),
-    min_count: req.min_count,
-    min_total_count: req.min_total_count,
-    cpm_cutoff: req.cpm_cutoff,
-    method: req.method ?? null,
-    tw: req.tw ?? null,
-    tw2: req.tw2 ?? null,
-    filter: req.filter ?? null,
-    filter0: req.filter0 ?? null
-  };
-}
-async function loadDeForResponse(req, genomes) {
-  const { result, cacheId } = await getDeCacheResult(req, genomes);
-  const images = [];
-  if (result.qlImage) images.push(result.qlImage);
-  if (result.mdsImage) images.push(result.mdsImage);
-  return { result, cacheId, images };
-}
-async function getDeCacheResult(req, genomes) {
-  const { result, cacheId } = await cacheOrRecompute({
-    computeArgument: deKeyInputs(req),
-    cacheSubdir: "de",
-    computeFresh: async () => {
-      const { ds, term_results, term_results2 } = await resolveDaContext(req, genomes);
-      return runDeFresh(req, ds, term_results, term_results2);
-    }
-  });
-  return { result, cacheId };
-}
-async function runDeFresh(param, ds, term_results, term_results2) {
-  const groups = resolveSampleGroups(param, ds, term_results, term_results2);
-  if (groups.alerts.length) throw new Error(groups.alerts.join(" | "));
-  const q = ds.queries.rnaseqGeneCount;
-  const expression_input = {
-    case: groups.group2names.join(","),
-    control: groups.group1names.join(","),
-    data_type: "do_DE",
-    input_file: q.file,
-    cachedir: serverconfig.cachedir,
-    storage_type: q.storage_type,
-    DE_method: param.method,
-    mds_cutoff: 1e4,
-    min_count: param.min_count,
-    min_total_count: param.min_total_count,
-    cpm_cutoff: param.cpm_cutoff
-  };
-  if (param.tw) {
-    expression_input.conf1 = [...groups.conf1_group2, ...groups.conf1_group1];
-    expression_input.conf1_mode = param.tw.q.mode;
-    if (new Set(expression_input.conf1).size === 1) throw new Error("Confounding variable 1 has only one value");
-  }
-  if (param.tw2) {
-    expression_input.conf2 = [...groups.conf2_group2, ...groups.conf2_group1];
-    expression_input.conf2_mode = param.tw2.q.mode;
-    if (new Set(expression_input.conf2).size === 1) throw new Error("Confounding variable 2 has only one value");
-  }
-  const small = groups.group1names.length <= 8 && groups.group2names.length <= 8;
-  const engine = small || param.method === "edgeR" || param.method === "limma" ? "edgeR" : "wilcoxon";
-  if (engine === "edgeR") {
-    const time12 = (/* @__PURE__ */ new Date()).valueOf();
-    const result2 = JSON.parse(await run_R("edge_newh5.R", JSON.stringify(expression_input)));
-    mayLog("Time taken to run edgeR:", formatElapsedTime(Date.now() - time12));
-    param.method = "edgeR";
-    const qlImage = deImageFromB64(result2.ql_image_b64, "ql_image");
-    const mdsImage = deImageFromB64(result2.mds_image_b64, "mds_image");
-    const cacheResult2 = {
-      geneRows: result2.gene_data,
-      sample_size1: result2.num_controls[0],
-      sample_size2: result2.num_cases[0],
-      method: param.method,
-      bcv: result2.bcv && result2.bcv[0] != null ? result2.bcv[0] : void 0,
-      ...qlImage ? { qlImage } : {},
-      ...mdsImage ? { mdsImage } : {}
-    };
-    return cacheResult2;
-  }
-  const time1 = (/* @__PURE__ */ new Date()).valueOf();
-  const result = JSON.parse(await run_rust("DEanalysis", JSON.stringify(expression_input)));
-  mayLog("Time taken to run rust DE pipeline:", formatElapsedTime(Date.now() - time1));
-  param.method = "wilcoxon";
-  const cacheResult = {
-    geneRows: result,
-    sample_size1: groups.group1names.length,
-    sample_size2: groups.group2names.length,
-    method: param.method
-  };
-  return cacheResult;
-}
-function resolveSampleGroups(param, ds, term_results, term_results2) {
-  if (param.samplelst?.groups?.length != 2) throw new Error(".samplelst.groups.length!=2");
-  if (param.samplelst.groups[0].values?.length < 1) throw new Error("samplelst.groups[0].values.length<1");
-  if (param.samplelst.groups[1].values?.length < 1) throw new Error("samplelst.groups[1].values.length<1");
-  const q = ds.queries.rnaseqGeneCount;
-  if (!q) throw new Error("rnaseqGeneCount query missing on ds");
-  if (!q.file) throw new Error("unknown data type for rnaseqGeneCount");
-  if (!q.storage_type) throw new Error("storage_type is not defined");
-  const g1 = buildGroupValues(param.samplelst.groups[0].values, q, ds, param.tw, param.tw2, term_results, term_results2);
-  const g2 = buildGroupValues(param.samplelst.groups[1].values, q, ds, param.tw, param.tw2, term_results, term_results2);
-  const alerts = [];
-  if (g1.names.length < 1) alerts.push("sample size of group1 < 1");
-  if (g2.names.length < 1) alerts.push("sample size of group2 < 1");
-  const commonnames = g1.names.filter((x) => g2.names.includes(x));
-  if (commonnames.length) alerts.push(`Common elements found between both groups: ${commonnames.join(", ")}`);
-  return {
-    group1names: g1.names,
-    group2names: g2.names,
-    conf1_group1: g1.conf1,
-    conf1_group2: g2.conf1,
-    conf2_group1: g1.conf2,
-    conf2_group2: g2.conf2,
-    alerts
-  };
-}
-function deImageFromB64(b64, key) {
-  if (!b64) return null;
-  const padding = b64.endsWith("==") ? 2 : b64.endsWith("=") ? 1 : 0;
-  const size = Math.floor(b64.length * 3 / 4) - padding;
-  return { src: `data:image/png;base64,${b64}`, size, key };
-}
-export {
-  getDeCacheResult,
-  init,
-  resolveSampleGroups
-};

package/routes/termdb.categories.js

@@ -1,123 +0,0 @@
-import { getOrderedLabels } from "#src/termdb.barchart.js";
-import { getData } from "#src/termdb.matrix.js";
-function init({ genomes }) {
-  return async (req, res) => {
-    const q = req.query;
-    try {
-      const g = genomes[req.query.genome];
-      if (!g) throw "invalid genome name";
-      const ds = g.datasets[req.query.dslabel];
-      if (!ds) throw "invalid dataset name";
-      const tdb = ds.cohort.termdb;
-      if (!tdb) throw "invalid termdb object";
-      await trigger_getcategories(q, res, tdb, ds);
-    } catch (e) {
-      res.send({ error: e?.message || e });
-      if (e instanceof Error && e.stack) console.log(e);
-    }
-  };
-}
-async function trigger_getcategories(q, res, tdb, ds) {
-  if (!q.tw.$id) q.tw.$id = "_";
-  const $id = q.tw.$id;
-  const arg = {
-    filter: q.filter,
-    filter0: q.filter0,
-    terms: [q.tw],
-    currentGeneNames: q.currentGeneNames,
-    // optional, from mds3 mayAddGetCategoryArgs()
-    rglst: q.rglst,
-    // optional, from mds3 mayAddGetCategoryArgs()
-    __protected__: q.__protected__,
-    __abortSignal: q.__abortSignal
-  };
-  const data = await getData(arg, ds);
-  if (data.error) throw data.error;
-  const [lst, orderedLabels] = getCategories(data, q, ds, $id);
-  res.send({
-    lst,
-    orderedLabels
-  });
-}
-function getCategories(data, q, ds, $id) {
-  const lst = [];
-  if (q.tw.term.type == "geneVariant" && q.tw.q.type != "predefined-groupset" && q.tw.q.type != "custom-groupset") {
-    const samples = data.samples;
-    const dtClassMap = /* @__PURE__ */ new Map();
-    if (ds.assayAvailability?.byDt) {
-      for (const [dtType, _dtValue] of Object.entries(ds.assayAvailability.byDt)) {
-        const dtValue = _dtValue;
-        if (dtValue.byOrigin) {
-          dtClassMap.set(parseInt(dtType), { byOrigin: { germline: {}, somatic: {} } });
-        }
-      }
-    }
-    const sampleCountedFor = /* @__PURE__ */ new Set();
-    for (const sampleData of Object.values(samples)) {
-      const key = $id;
-      if (!Object.keys(sampleData).includes(key)) continue;
-      const values = sampleData[key].values;
-      sampleCountedFor.clear();
-      for (const value of values) {
-        if (!dtClassMap.has(value.dt)) {
-          dtClassMap.set(value.dt, {});
-        }
-        const dtClasses = dtClassMap.get(value.dt);
-        if (dtClasses.byOrigin) {
-          if (!dtClasses.byOrigin[value.origin][value.class]) {
-            dtClasses.byOrigin[value.origin][value.class] = 1;
-            sampleCountedFor.add(`${value.dt} ${value.origin} ${value.class}`);
-          }
-          if (!sampleCountedFor.has(`${value.dt} ${value.origin} ${value.class}`)) {
-            sampleCountedFor.add(`${value.dt} ${value.origin} ${value.class}`);
-            dtClasses.byOrigin[value.origin][value.class] += 1;
-          }
-        } else {
-          if (!dtClasses[value.class]) {
-            sampleCountedFor.add(`${value.dt} ${value.class}`);
-            dtClasses[value.class] = 1;
-          }
-          if (!sampleCountedFor.has(`${value.dt} ${value.class}`)) {
-            sampleCountedFor.add(`${value.dt} ${value.class}`);
-            dtClasses[value.class] += 1;
-          }
-        }
-      }
-    }
-    for (const [dt, classes] of dtClassMap) {
-      lst.push({
-        dt,
-        classes
-      });
-    }
-  } else {
-    const key2count = /* @__PURE__ */ new Map();
-    for (const sid in data.samples) {
-      const v = data.samples[sid][$id];
-      if (!v) continue;
-      if (!("key" in v)) continue;
-      key2count.set(v.key, 1 + (key2count.get(v.key) || 0));
-    }
-    for (const [key, count] of key2count) {
-      lst.push({
-        samplecount: count,
-        key,
-        label: data.refs?.byTermId?.[$id]?.events?.find((e) => e.event === key).label || q.tw.term?.values?.[key]?.label || key
-      });
-    }
-  }
-  const orderedLabels = getOrderedLabels(
-    q.tw.term,
-    data.refs?.byTermId?.[$id]?.bins || [],
-    data.refs?.byTermId?.[$id]?.events,
-    q.tw.q
-  );
-  if (orderedLabels.length) {
-    lst.sort((a, b) => orderedLabels.indexOf(a.label) - orderedLabels.indexOf(b.label));
-  }
-  return [lst, orderedLabels];
-}
-export {
-  getCategories,
-  init
-};

package/routes/termdb.chat.js

@@ -1,203 +0,0 @@
-import { mayLog } from "#src/helpers.ts";
-import { formatElapsedTime } from "#shared";
-import { readJSONFile, parse_geneset_db, getChatRelatedPlotTypes } from "./chat/utils.ts";
-import { classifyQuery } from "./chat/classify1.ts";
-import { classifyPlotType } from "./chat/plot.ts";
-import { classifyNotPlot } from "./chat/classify2.ts";
-import { inferScaffold } from "./chat/scaffold.ts";
-import serverconfig from "../src/serverconfig.js";
-import { getDsAllowedTermTypes } from "./termdb.config.ts";
-import { phrase2entity } from "./chat/phrase2entity.ts";
-import { inferTermObjFromEntity } from "./chat/entity2termObj.ts";
-import { resolveToTwTvs } from "./chat/entity2twTvs.ts";
-import { answerDataQueries } from "./chat/dataQueries.ts";
-import path from "path";
-import fs from "fs";
-import { resolveToPlotState } from "./chat/scaffold2state.ts";
-function init({ genomes }) {
-  return async (req, res) => {
-    const q = req.query;
-    try {
-      const genome = genomes[q.genome];
-      if (!genome) throw "invalid genome";
-      const ds = genome.datasets?.[q.dslabel];
-      if (!ds) throw "invalid dslabel";
-      if (!ds.queries.chat) {
-        return res.send({
-          type: "text",
-          text: "Only search functionality supported for this data. No chat functionality supported."
-        });
-      }
-      const aiFilesDir = serverconfig.binpath + "/../../dataset/ai/" + q.dslabel;
-      let agentFiles = [];
-      try {
-        agentFiles = await fs.readdirSync(aiFilesDir).filter((file) => file.endsWith(".json"));
-      } catch (err) {
-        if (err.code === "ENOENT") throw new Error(`Directory not found: ${aiFilesDir}`);
-        if (err.code === "ENOTDIR") throw new Error(`Path is not a directory: ${aiFilesDir}`);
-        throw err;
-      }
-      const llm = serverconfig.llm;
-      if (!llm) throw "serverconfig.llm is not configured";
-      if (llm.provider !== "SJ" && llm.provider !== "ollama" && llm.provider !== "huggingface" && llm.provider !== "azure") {
-        throw "llm.provider must be 'SJ', 'ollama', 'huggingface', or 'azure'";
-      }
-      let rawFilter;
-      if (typeof q.filter === "string") {
-        try {
-          rawFilter = JSON.parse(q.filter);
-        } catch (e) {
-          throw new Error("Failed to parse filter JSON string: " + e);
-        }
-      } else {
-        rawFilter = q.filter;
-      }
-      const filter = rawFilter && typeof rawFilter === "object" ? rawFilter : {};
-      const lst = Array.isArray(filter.lst) ? filter.lst : [];
-      const cohortFilter = lst.find((item) => item.tag === "cohortFilter");
-      const cohortKey = cohortFilter ? cohortFilter.tvs.values[0].key : "";
-      const supportedPlotTypes = ds.cohort.termdb.q?.getSupportedChartTypes(req)?.[cohortKey];
-      const chatSupportedPlotTypes = getChatRelatedPlotTypes(supportedPlotTypes);
-      const genedb = serverconfig.tpmasterdir + "/" + genome.genedb.dbfile;
-      const allowedTermTypes = getDsAllowedTermTypes(ds);
-      const ai_output_json = await run_chat_pipeline(
-        q.prompt,
-        llm,
-        ds,
-        genedb,
-        agentFiles,
-        aiFilesDir,
-        chatSupportedPlotTypes,
-        allowedTermTypes,
-        genome
-        // 	testing
-      );
-      mayLog("From init: Final AI output JSON:", JSON.stringify(ai_output_json));
-      res.send(ai_output_json);
-    } catch (e) {
-      if (e.stack) mayLog(e.stack);
-      res.send({ error: e?.message || e });
-    }
-  };
-}
-async function run_chat_pipeline(userPrompt, llm, ds, genedb, agentFiles, aiFilesDir, supportedPlotTypes, allowedTermTypes, genome) {
-  if (!fs.existsSync(path.join(aiFilesDir, "main.json")))
-    throw "Main data file is not specified for dataset:" + ds.label;
-  const dataset_json = await readJSONFile(path.join(aiFilesDir, "main.json"));
-  const time1 = (/* @__PURE__ */ new Date()).valueOf();
-  const class_response = await classifyQuery(userPrompt, llm);
-  mayLog("Time taken for classification:", formatElapsedTime(Date.now() - time1));
-  let ai_output_json;
-  if (class_response.type === "notplot") {
-    const time2 = (/* @__PURE__ */ new Date()).valueOf();
-    const notPlotResult = await classifyNotPlot(userPrompt, llm, agentFiles, aiFilesDir);
-    mayLog("Time taken for classify2:", formatElapsedTime(Date.now() - time2));
-    if (notPlotResult.type === "html") {
-      ai_output_json = notPlotResult;
-    } else {
-      ai_output_json = {
-        type: "text",
-        text: "Your query does not appear to be related to the available data visualizations. Please try rephrasing your question."
-      };
-    }
-  } else if (class_response.type === "binaryQuery") {
-    const answer = await answerDataQueries(userPrompt, llm, allowedTermTypes);
-    if (!answer) throw "Couldn't decide if this is data related query!";
-    mayLog("Data Binary Query: ", answer);
-    ai_output_json = answer;
-  } else if (class_response.type === "plot") {
-    let time = (/* @__PURE__ */ new Date()).valueOf();
-    const plotType = await classifyPlotType(userPrompt, llm);
-    mayLog("Time taken to classify plot type:", formatElapsedTime(Date.now() - time));
-    if (!supportedPlotTypes.includes(plotType)) {
-      const log = 'Plot type: "' + plotType + '" is not supported.';
-      ai_output_json = {
-        type: "text",
-        text: log
-      };
-      mayLog(log);
-      return ai_output_json;
-    }
-    const genes_list = await parse_geneset_db(genedb);
-    mayLog("#################################################");
-    mayLog("####### First phase: Infer Plot Scaffolds #######");
-    mayLog("#################################################");
-    time = (/* @__PURE__ */ new Date()).valueOf();
-    const dataset_db = serverconfig.tpmasterdir + "/" + ds.cohort.db.file;
-    const scaffoldResult = await inferScaffold(
-      userPrompt,
-      plotType,
-      llm,
-      genome,
-      genes_list,
-      allowedTermTypes,
-      dataset_json,
-      ds,
-      dataset_db
-    );
-    mayLog("ScaffoldResult: ", scaffoldResult);
-    if (plotType === "hiercluster" && "plot" in scaffoldResult && scaffoldResult.type === "plot" || "text" in scaffoldResult && scaffoldResult.type === "text") {
-      return scaffoldResult;
-    }
-    mayLog("Time taken to infer scaffold:", formatElapsedTime(Date.now() - time));
-    if (!scaffoldResult)
-      throw "Scaffold result is empty or undefined, which is unexpected. Please check the inferScaffold agent for potential issues.";
-    if ("type" in scaffoldResult && scaffoldResult.type === "text") {
-      return scaffoldResult;
-    }
-    const subplotType = scaffoldResult.plotType === "summary" ? scaffoldResult.chartType : void 0;
-    mayLog("#################################################");
-    mayLog("####### Second phase: From Scaffolds's phrases infer Entities #######");
-    mayLog("#################################################");
-    time = (/* @__PURE__ */ new Date()).valueOf();
-    const phrase2entityResult = await phrase2entity(
-      scaffoldResult,
-      plotType,
-      llm,
-      genes_list,
-      dataset_json,
-      ds,
-      genome
-    );
-    mayLog("Time taken to phrase 2 entity:", formatElapsedTime(Date.now() - time));
-    if ("type" in phrase2entityResult && phrase2entityResult.type === "text") {
-      return phrase2entityResult;
-    }
-    mayLog(phrase2entityResult);
-    mayLog("#################################################");
-    mayLog("####### Third phase: From Entities infer Term Objects #######");
-    mayLog("#################################################");
-    time = (/* @__PURE__ */ new Date()).valueOf();
-    const termObj = await inferTermObjFromEntity(
-      phrase2entityResult,
-      plotType,
-      llm,
-      dataset_db,
-      genes_list,
-      genome
-    );
-    mayLog("Time taken to infer term objects:", formatElapsedTime(Date.now() - time));
-    mayLog("Inferred termObj from entity:", JSON.stringify(termObj));
-    mayLog("#################################################");
-    mayLog("####### Fourth phase: From Term Objects to TwTvs Objects #######");
-    mayLog("#################################################");
-    time = (/* @__PURE__ */ new Date()).valueOf();
-    const twTvsObj = await resolveToTwTvs(termObj, plotType, llm, dataset_db, genome);
-    mayLog("Time taken to resolve to TwTvs object from termObj:", formatElapsedTime(Date.now() - time));
-    if ("type" in twTvsObj && twTvsObj.type === "text") {
-      return twTvsObj;
-    }
-    mayLog("twTvsObj:", twTvsObj);
-    mayLog("#################################################");
-    mayLog("####### Fifth/Final phase: From TwTvs Objects to Plot States #######");
-    mayLog("#################################################");
-    time = (/* @__PURE__ */ new Date()).valueOf();
-    ai_output_json = resolveToPlotState(twTvsObj, plotType, subplotType);
-    mayLog("Time taken to resolve to plot state:", formatElapsedTime(Date.now() - time));
-  }
-  return ai_output_json;
-}
-export {
-  init,
-  run_chat_pipeline
-};