@sjcrh/proteinpaint-server 2.184.0 → 2.184.1-0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.184.0",
+  "version": "2.184.1-0",
   "type": "module",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "src/app.js",
@@ -66,7 +66,7 @@
     "@sjcrh/proteinpaint-r": "2.181.0",
     "@sjcrh/proteinpaint-rust": "2.183.0",
     "@sjcrh/proteinpaint-shared": "2.183.0",
-    "@sjcrh/proteinpaint-types": "2.184.0",
+    "@sjcrh/proteinpaint-types": "2.184.1-0",
     "@types/express": "^5.0.0",
     "@types/express-session": "^1.18.1",
     "better-sqlite3": "^12.4.1",

package/routes/termdb.chat3.js

@@ -0,0 +1,191 @@
+import { ChatPayload } from "#types/checkers";
+import { mayLog } from "#src/helpers.ts";
+import { formatElapsedTime } from "#shared";
+import { readJSONFile, parse_geneset_db } from "./chat/utils.ts";
+import { classifyQuery } from "./chat/classify1.ts";
+import { classifyPlotType } from "./chat/plot.ts";
+import { classifyNotPlot } from "./chat/classify2.ts";
+import { inferScaffold } from "./chat/scaffold.ts";
+import serverconfig from "../src/serverconfig.js";
+import { getDsAllowedTermTypes } from "./termdb.config.ts";
+import { phrase2entity } from "./chat/phrase2entity.ts";
+import { inferTermObjFromEntity } from "./chat/entity2termObj.ts";
+import { resolveToTwTvs } from "./chat/entity2twTvs.ts";
+import path from "path";
+import fs from "fs";
+import { resolveToPlotState } from "./chat/scaffold2state.ts";
+const api = {
+  endpoint: "termdb/chat3",
+  methods: {
+    get: {
+      ...ChatPayload,
+      init
+    },
+    post: {
+      ...ChatPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    const q = req.query;
+    try {
+      const g = genomes[q.genome];
+      if (!g) throw "invalid genome";
+      const ds = g.datasets?.[q.dslabel];
+      if (!ds) throw "invalid dslabel";
+      const aiFilesDir = serverconfig.binpath + "/../../dataset/ai/" + q.dslabel;
+      let agentFiles = [];
+      try {
+        agentFiles = await fs.readdirSync(aiFilesDir).filter((file) => file.endsWith(".json"));
+      } catch (err) {
+        if (err.code === "ENOENT") throw new Error(`Directory not found: ${aiFilesDir}`);
+        if (err.code === "ENOTDIR") throw new Error(`Path is not a directory: ${aiFilesDir}`);
+        throw err;
+      }
+      const llm = serverconfig.llm;
+      if (!llm) throw "serverconfig.llm is not configured";
+      if (llm.provider !== "SJ" && llm.provider !== "ollama" && llm.provider !== "huggingface" && llm.provider !== "azure") {
+        throw "llm.provider must be 'SJ', 'ollama', 'huggingface', or 'azure'";
+      }
+      const rawFilter = typeof q.filter === "string" ? JSON.parse(q.filter) : q.filter;
+      const filter = rawFilter && typeof rawFilter === "object" ? rawFilter : {};
+      const lst = Array.isArray(filter.lst) ? filter.lst : [];
+      const cohortFilter = lst.find((item) => item.tag === "cohortFilter");
+      const cohortKey = cohortFilter ? cohortFilter.tvs.values[0].key : "";
+      const supportedChartTypes = ds.cohort.termdb.q?.getSupportedChartTypes(req)?.[cohortKey];
+      const genedb = serverconfig.tpmasterdir + "/" + g.genedb.dbfile;
+      const _allowedTermTypes = getDsAllowedTermTypes(ds);
+      const ai_output_json = await run_chat_pipeline(
+        q.prompt,
+        llm,
+        ds,
+        genedb,
+        agentFiles,
+        aiFilesDir,
+        supportedChartTypes,
+        _allowedTermTypes
+        // 	testing
+      );
+      mayLog("From init: Final AI output JSON:", JSON.stringify(ai_output_json));
+      res.send(ai_output_json);
+    } catch (e) {
+      if (e.stack) mayLog(e.stack);
+      res.send({ error: e?.message || e });
+    }
+  };
+}
+async function run_chat_pipeline(user_prompt, llm, ds, genedb, agentFiles, aiFilesDir, supportedChartTypes, _allowedTermTypes) {
+  if (!fs.existsSync(path.join(aiFilesDir, "main.json")))
+    throw "Main data file is not specified for dataset:" + ds.label;
+  const dataset_json = await readJSONFile(path.join(aiFilesDir, "main.json"));
+  const time1 = (/* @__PURE__ */ new Date()).valueOf();
+  const class_response = await classifyQuery(user_prompt, llm);
+  mayLog("Time taken for classification:", formatElapsedTime(Date.now() - time1));
+  let ai_output_json;
+  if (class_response.type == "notplot") {
+    const time2 = (/* @__PURE__ */ new Date()).valueOf();
+    const notPlotResult = await classifyNotPlot(user_prompt, llm, agentFiles, aiFilesDir);
+    mayLog("Time taken for classify2:", formatElapsedTime(Date.now() - time2));
+    if (notPlotResult.type == "html") {
+      ai_output_json = notPlotResult;
+    } else {
+      ai_output_json = {
+        type: "text",
+        text: "Your query does not appear to be related to the available data visualizations. Please try rephrasing your question."
+      };
+    }
+  } else if (class_response.type == "plot") {
+    let time = (/* @__PURE__ */ new Date()).valueOf();
+    const plotType = await classifyPlotType(user_prompt, llm);
+    mayLog("Time taken to classify plot type:", formatElapsedTime(Date.now() - time));
+    if (!supportedChartTypes) {
+      const errorMsg = "Supported chart types list is undefined. Please check the dataset configuration and ensure 							  that getSupportedChartTypes is implemented correctly. Skipping chart type validation, but this may 							  lead to unsupported chart type errors downstream.";
+      console.warn(errorMsg);
+      const errorResponse = {
+        type: "text",
+        text: errorMsg
+      };
+      return errorResponse;
+    }
+    if (plotType === "summary") {
+      if (!supportedChartTypes.includes("dictionary")) {
+        const log = 'Plot type: "' + plotType + '" is not supported.';
+        ai_output_json = {
+          type: "text",
+          text: log
+        };
+        mayLog(log);
+        return ai_output_json;
+      }
+    } else if (plotType === "dge") {
+      if (!supportedChartTypes.includes("DA")) {
+        const log = 'Plot type: "' + plotType + '" is not supported.';
+        ai_output_json = {
+          type: "text",
+          text: log
+        };
+        mayLog(log);
+        return ai_output_json;
+      }
+    } else {
+      mayLog(`Supported chart types for this cohort: ${supportedChartTypes}`);
+      if (!supportedChartTypes.includes(plotType)) {
+        const log = 'Plot type: "' + plotType + '" is not supported.';
+        ai_output_json = {
+          type: "text",
+          text: log
+        };
+        mayLog(log);
+        return ai_output_json;
+      }
+    }
+    mayLog("####### First phase: Infer Plot Scaffolds #######");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const scaffoldResult = await inferScaffold(user_prompt, plotType, llm);
+    mayLog("ScaffoldResult: ", scaffoldResult);
+    mayLog("Time taken to infer scaffold:", formatElapsedTime(Date.now() - time));
+    if (!scaffoldResult)
+      throw "Scaffold result is empty or undefined, which is unexpected. Please check the inferScaffold agent for potential issues.";
+    const subplotType = scaffoldResult.plotType === "summary" ? scaffoldResult.chartType : void 0;
+    mayLog("####### Second phase: From Scaffolds's phrases infer Entities #######");
+    const genes_list = await parse_geneset_db(genedb);
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const phrase2entityResult = await phrase2entity(scaffoldResult, plotType, llm, genes_list, dataset_json, ds);
+    mayLog("Time taken to phrase 2 entity:", formatElapsedTime(Date.now() - time));
+    if ("type" in phrase2entityResult && phrase2entityResult.type === "text") {
+      return phrase2entityResult;
+    }
+    mayLog(phrase2entityResult);
+    mayLog("####### Third phase: From Entities infer Term Objects #######");
+    const dataset_db = serverconfig.tpmasterdir + "/" + ds.cohort.db.file;
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const termObj = await inferTermObjFromEntity(
+      phrase2entityResult,
+      plotType,
+      llm,
+      dataset_db,
+      genes_list
+    );
+    mayLog("Time taken to infer term objects:", formatElapsedTime(Date.now() - time));
+    mayLog("Inferred termObj from entity:", JSON.stringify(termObj));
+    mayLog("####### Fourth phase: From Term Objects to TwTvs Objects #######");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const twTvsObj = await resolveToTwTvs(termObj, plotType, llm, dataset_db);
+    mayLog("Time taken to resolve to TwTvs object from termObj:", formatElapsedTime(Date.now() - time));
+    if ("type" in twTvsObj && twTvsObj.type === "text") {
+      return twTvsObj;
+    }
+    mayLog("twTvsObj:", twTvsObj);
+    mayLog("####### Fifth/Final phase: From TwTvs Objects to Plot States #######");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    ai_output_json = resolveToPlotState(twTvsObj, plotType, subplotType);
+    mayLog("Time taken to resolve to plot state:", formatElapsedTime(Date.now() - time));
+  }
+  return ai_output_json;
+}
+export {
+  api,
+  run_chat_pipeline
+};

package/routes/termdb.config.js

@@ -47,7 +47,7 @@ function make(q, req, res, ds, genome) {
     selectCohort: getSelectCohort(ds, req),
     supportedChartTypes: tdb.q?.getSupportedChartTypes(req),
     renamedChartTypes: ds.cohort.renamedChartTypes,
-    allowedTermTypes: getAllowedTermTypes(ds),
+    allowedTermTypes: getDsAllowedTermTypes(ds),
     massSessionDuration: serverconfig.features.massSessionDuration || 30,
     dataDownloadCatch: tdb.dataDownloadCatch,
     matrix: tdb.matrix,
@@ -293,7 +293,7 @@ function addNonDictionaryQueries(c, ds, genome) {
     q2.images = {};
   }
 }
-function getAllowedTermTypes(ds) {
+function getDsAllowedTermTypes(ds) {
   const typeSet = /* @__PURE__ */ new Set();
   for (const r of ds.cohort.termdb.termtypeByCohort) {
     if (r.termType) typeSet.add(r.termType);
@@ -329,5 +329,6 @@ function getSelectCohort(ds, req) {
   return copy;
 }
 export {
-  api
+  api,
+  getDsAllowedTermTypes
 };