@sjcrh/proteinpaint-server 2.189.0 → 2.190.1

package/routes/termdb.chat.js

@@ -1,132 +1,78 @@
-import fs from "fs";
-import { ezFetch } from "#shared";
-import { get_samples } from "#src/termdb.sql.js";
-import { ChatPayload } from "#types/checkers";
-import { extractResourceResponse } from "./chat/resource.ts";
-import serverconfig from "../src/serverconfig.js";
 import { mayLog } from "#src/helpers.ts";
-import Database from "better-sqlite3";
 import { formatElapsedTime } from "#shared";
-const num_filter_cutoff = 3;
-const FILTER_TERM_DEFINITIONS = {
-  FilterTerm: {
-    anyOf: [{ $ref: "#/definitions/CategoricalFilterTerm" }, { $ref: "#/definitions/NumericFilterTerm" }]
-  },
-  CategoricalFilterTerm: {
-    type: "object",
-    properties: {
-      term: { type: "string", description: "Name of categorical term" },
-      category: { type: "string", description: "The category of the term" },
-      join: {
-        type: "string",
-        enum: ["and", "or"],
-        description: "join term to be used only when there is more than one filter term and should be placed from the 2nd filter term onwards describing how it connects to the previous term"
-      }
-    },
-    required: ["term", "category"],
-    additionalProperties: false
-  },
-  NumericFilterTerm: {
-    type: "object",
-    properties: {
-      term: { type: "string", description: "Name of numeric term" },
-      start: { type: "number", description: "start position (or lower limit) of numeric term" },
-      stop: { type: "number", description: "stop position (or upper limit) of numeric term" },
-      join: {
-        type: "string",
-        enum: ["and", "or"],
-        description: "join term to be used only when there is more than one filter term and should be placed from the 2nd filter term onwards describing how it connects to the previous term"
-      }
-    },
-    required: ["term"],
-    additionalProperties: false
-  }
-};
-function formatTrainingExamples(trainingData) {
-  return trainingData.map(
-    (td, i) => "Example question" + (i + 1).toString() + ": " + td.question + " Example answer" + (i + 1).toString() + ":" + JSON.stringify(td.answer)
-  ).join(" ");
-}
-const FILTER_DESCRIPTION = 'There are two kinds of filter variables: "Categorical" and "Numeric". "Categorical" variables are those variables which can have a fixed set of values e.g. gender, race. They are defined by the "CategoricalFilterTerm" which consists of "term" (a field from the sqlite3 db) and "category" (a value of the field from the sqlite db). "Numeric" variables are those which can have any numeric value. They are defined by "NumericFilterTerm" and contain the subfields "term" (a field from the sqlite3 db), "start" an optional filter which is defined when a lower cutoff is defined in the user input for the numeric variable and "stop" an optional filter which is defined when a higher cutoff is defined in the user input for the numeric variable. ';
-function extractGenesFromPrompt(prompt, genes_list) {
-  const words = prompt.replace(/[^a-zA-Z0-9\s]/g, "").split(/\s+/).map((str) => str.toLowerCase());
-  return words.filter((item) => genes_list.includes(item));
-}
-const CHILD_TYPE_DEFAULTS = {
-  "categorical:undefined": "barchart",
-  "numeric:undefined": "violin",
-  "categorical:categorical": "barchart",
-  "numeric:categorical": "violin",
-  "categorical:numeric": "violin",
-  "numeric:numeric": "sampleScatter"
-};
-const CHILD_TYPE_INVALID = {
-  "categorical:undefined": /* @__PURE__ */ new Set(["violin", "boxplot", "sampleScatter"]),
-  "categorical:categorical": /* @__PURE__ */ new Set(["violin", "boxplot", "sampleScatter"])
-};
-function resolveChildType(cat1, cat2, llmChildType) {
-  const norm1 = cat1 == "float" || cat1 == "integer" ? "numeric" : cat1 || "undefined";
-  const norm2 = cat2 == "float" || cat2 == "integer" ? "numeric" : cat2 || "undefined";
-  const key = norm1 + ":" + norm2;
-  const defaultType = CHILD_TYPE_DEFAULTS[key];
-  if (!defaultType) {
-    return { childType: "barchart" };
-  }
-  const invalid = CHILD_TYPE_INVALID[key];
-  if (llmChildType && invalid && invalid.has(llmChildType)) {
-    return {
-      error: "Invalid plot type supplied by the user: " + llmChildType + ". For " + key.replace(":", " and ") + " variables the plot type should always be " + defaultType
-    };
-  }
-  return {
-    childType: llmChildType || defaultType,
-    bothNumeric: norm1 == "numeric" && norm2 == "numeric"
-  };
-}
-const api = {
-  endpoint: "termdb/chat",
-  methods: {
-    get: {
-      ...ChatPayload,
-      init
-    },
-    post: {
-      ...ChatPayload,
-      init
-    }
-  }
-};
+import { readJSONFile, parse_geneset_db, getChatRelatedPlotTypes } from "./chat/utils.ts";
+import { classifyQuery } from "./chat/classify1.ts";
+import { classifyPlotType } from "./chat/plot.ts";
+import { classifyNotPlot } from "./chat/classify2.ts";
+import { inferScaffold } from "./chat/scaffold.ts";
+import serverconfig from "../src/serverconfig.js";
+import { getDsAllowedTermTypes } from "./termdb.config.ts";
+import { phrase2entity } from "./chat/phrase2entity.ts";
+import { inferTermObjFromEntity } from "./chat/entity2termObj.ts";
+import { resolveToTwTvs } from "./chat/entity2twTvs.ts";
+import { answerDataQueries } from "./chat/dataQueries.ts";
+import path from "path";
+import fs from "fs";
+import { resolveToPlotState } from "./chat/scaffold2state.ts";
 function init({ genomes }) {
   return async (req, res) => {
     const q = req.query;
     try {
-      const g = genomes[q.genome];
-      if (!g) throw "invalid genome";
-      const ds = g.datasets?.[q.dslabel];
+      const genome = genomes[q.genome];
+      if (!genome) throw "invalid genome";
+      const ds = genome.datasets?.[q.dslabel];
       if (!ds) throw "invalid dslabel";
-      const serverconfig_ds_entries = serverconfig.genomes.find((genome) => genome.name == q.genome).datasets.find((dslabel) => dslabel.name == ds.label);
-      if (!serverconfig_ds_entries.aifiles) {
-        throw "aifiles are missing for chatbot to work";
+      if (!ds.queries.chat) {
+        return res.send({
+          type: "text",
+          text: "Only search functionality supported for this data. No chat functionality supported."
+        });
+      }
+      const aiFilesDir = serverconfig.binpath + "/../../dataset/ai/" + q.dslabel;
+      let agentFiles = [];
+      try {
+        agentFiles = await fs.readdirSync(aiFilesDir).filter((file) => file.endsWith(".json"));
+      } catch (err) {
+        if (err.code === "ENOENT") throw new Error(`Directory not found: ${aiFilesDir}`);
+        if (err.code === "ENOTDIR") throw new Error(`Path is not a directory: ${aiFilesDir}`);
+        throw err;
       }
       const llm = serverconfig.llm;
       if (!llm) throw "serverconfig.llm is not configured";
-      if (llm.provider !== "SJ" && llm.provider !== "ollama") {
-        throw "llm.provider must be 'SJ' or 'ollama'";
-      }
-      const dataset_db = serverconfig.tpmasterdir + "/" + ds.cohort.db.file;
-      const genedb = serverconfig.tpmasterdir + "/" + g.genedb.dbfile;
-      const dataset_json = await readJSONFile(serverconfig_ds_entries.aifiles);
-      const testing = false;
+      if (llm.provider !== "SJ" && llm.provider !== "ollama" && llm.provider !== "huggingface" && llm.provider !== "azure") {
+        throw "llm.provider must be 'SJ', 'ollama', 'huggingface', or 'azure'";
+      }
+      let rawFilter;
+      if (typeof q.filter === "string") {
+        try {
+          rawFilter = JSON.parse(q.filter);
+        } catch (e) {
+          throw new Error("Failed to parse filter JSON string: " + e);
+        }
+      } else {
+        rawFilter = q.filter;
+      }
+      const filter = rawFilter && typeof rawFilter === "object" ? rawFilter : {};
+      const lst = Array.isArray(filter.lst) ? filter.lst : [];
+      const cohortFilter = lst.find((item) => item.tag === "cohortFilter");
+      const cohortKey = cohortFilter ? cohortFilter.tvs.values[0].key : "";
+      const supportedPlotTypes = ds.cohort.termdb.q?.getSupportedChartTypes(req)?.[cohortKey];
+      const chatSupportedPlotTypes = getChatRelatedPlotTypes(supportedPlotTypes);
+      const genedb = serverconfig.tpmasterdir + "/" + genome.genedb.dbfile;
+      const allowedTermTypes = getDsAllowedTermTypes(ds);
       const ai_output_json = await run_chat_pipeline(
         q.prompt,
         llm,
-        serverconfig.aiRoute,
-        dataset_json,
-        testing,
-        dataset_db,
+        ds,
         genedb,
-        ds
+        agentFiles,
+        aiFilesDir,
+        chatSupportedPlotTypes,
+        allowedTermTypes,
+        genome
+        // 	testing
       );
+      mayLog("From init: Final AI output JSON:", JSON.stringify(ai_output_json));
       res.send(ai_output_json);
     } catch (e) {
       if (e.stack) mayLog(e.stack);
@@ -134,977 +80,124 @@ function init({ genomes }) {
     }
   };
 }
-async function run_chat_pipeline(user_prompt, llm, aiRoute, dataset_json, testing, dataset_db, genedb, ds) {
+async function run_chat_pipeline(userPrompt, llm, ds, genedb, agentFiles, aiFilesDir, supportedPlotTypes, allowedTermTypes, genome) {
+  if (!fs.existsSync(path.join(aiFilesDir, "main.json")))
+    throw "Main data file is not specified for dataset:" + ds.label;
+  const dataset_json = await readJSONFile(path.join(aiFilesDir, "main.json"));
   const time1 = (/* @__PURE__ */ new Date()).valueOf();
-  const class_response = await classify_query_by_dataset_type(
-    user_prompt,
-    llm,
-    aiRoute,
-    dataset_json
-  );
-  let ai_output_json;
+  const class_response = await classifyQuery(userPrompt, llm);
   mayLog("Time taken for classification:", formatElapsedTime(Date.now() - time1));
-  if (class_response.type == "none") {
-    ai_output_json = {
-      type: "text",
-      text: "Your query does not appear to be related to the available data visualizations. Please try rephrasing your question."
-    };
-  } else if (class_response.type == "resource") {
-    const time12 = (/* @__PURE__ */ new Date()).valueOf();
-    ai_output_json = await extractResourceResponse(user_prompt, llm, dataset_json);
-    mayLog("Time taken for resource agent:", formatElapsedTime(Date.now() - time12));
-  } else if (class_response.type == "plot") {
-    const classResult = class_response.plot;
-    mayLog("classResult:", classResult);
-    const dataset_db_output = await parse_dataset_db(dataset_db);
-    const genes_list = dataset_json.hasGeneExpression ? await parse_geneset_db(genedb) : [];
-    if (classResult == "summary") {
-      const time12 = (/* @__PURE__ */ new Date()).valueOf();
-      ai_output_json = await extract_summary_terms(
-        user_prompt,
-        llm,
-        dataset_db_output,
-        dataset_json,
-        genes_list,
-        ds,
-        testing
-      );
-      mayLog("Time taken for summary agent:", formatElapsedTime(Date.now() - time12));
-    } else if (classResult == "dge") {
-      const time12 = (/* @__PURE__ */ new Date()).valueOf();
-      ai_output_json = await extract_DE_search_terms_from_query(
-        user_prompt,
-        llm,
-        dataset_db_output,
-        dataset_json,
-        ds,
-        testing
-      );
-      mayLog("Time taken for DE agent:", formatElapsedTime(Date.now() - time12));
-    } else if (classResult == "survival") {
-      ai_output_json = { type: "text", text: "survival agent has not been implemented yet" };
-    } else if (classResult == "matrix") {
-      const time12 = (/* @__PURE__ */ new Date()).valueOf();
-      ai_output_json = await extract_matrix_search_terms_from_query(
-        user_prompt,
-        llm,
-        dataset_db_output,
-        dataset_json,
-        genes_list,
-        ds,
-        testing
-      );
-      mayLog("Time taken for matrix agent:", formatElapsedTime(Date.now() - time12));
-    } else if (classResult == "sampleScatter") {
-      const time12 = (/* @__PURE__ */ new Date()).valueOf();
-      ai_output_json = await extract_samplescatter_terms_from_query(
-        user_prompt,
-        llm,
-        dataset_db_output,
-        dataset_json,
-        genes_list,
-        ds,
-        testing
-      );
-      mayLog("Time taken for sampleScatter agent:", formatElapsedTime(Date.now() - time12));
-    } else {
-      ai_output_json = { type: "text", text: "Unknown classification value" };
-    }
-  } else {
-    ai_output_json = { type: "text", text: "Unknown classification type" };
-  }
-  return ai_output_json;
-}
-async function call_ollama(prompt, model_name, apilink) {
-  const temperature = 0.01;
-  const top_p = 0.95;
-  const timeout = 2e5;
-  const payload = {
-    model: model_name,
-    messages: [{ role: "user", content: prompt }],
-    raw: false,
-    stream: false,
-    keep_alive: 15,
-    //Keep the LLM loaded for 15mins
-    options: {
-      top_p,
-      temperature,
-      num_ctx: 1e4
-    }
-  };
-  try {
-    const result = await ezFetch(apilink + "/api/chat", {
-      method: "POST",
-      body: payload,
-      // ezfetch automatically stringifies objects
-      headers: { "Content-Type": "application/json" },
-      timeout: { request: timeout }
-      // ezfetch accepts milliseconds directly
-    });
-    if (result && result.message && result.message.content && result.message.content.length > 0)
-      return result.message.content;
-    else {
-      throw "Error: Received an unexpected response format:" + result;
-    }
-  } catch (error) {
-    throw "Ollama API request failed:" + error;
-  }
-}
-async function call_sj_llm(prompt, model_name, apilink) {
-  const temperature = 0.01;
-  const top_p = 0.95;
-  const timeout = 2e5;
-  const max_new_tokens = 512;
-  const payload = {
-    inputs: [
-      {
-        model_name,
-        inputs: {
-          text: prompt,
-          max_new_tokens,
-          temperature,
-          top_p
-        }
-      }
-    ]
-  };
-  try {
-    const response = await ezFetch(apilink, {
-      method: "POST",
-      body: payload,
-      // ezfetch automatically stringifies objects
-      headers: { "Content-Type": "application/json" },
-      timeout: { request: timeout }
-      // ezfetch accepts milliseconds directly
-    });
-    if (response.outputs && response.outputs[0] && response.outputs[0].generated_text) {
-      const result = response.outputs[0].generated_text;
-      return result;
-    } else {
-      throw "Error: Received an unexpected response format:" + response;
-    }
-  } catch (error) {
-    throw "SJ API request failed:" + error;
-  }
-}
-async function route_to_appropriate_llm_provider(template, llm) {
-  let response;
-  if (llm.provider == "SJ") {
-    response = await call_sj_llm(template, llm.modelName, llm.api);
-  } else if (llm.provider == "ollama") {
-    response = await call_ollama(template, llm.modelName, llm.api);
-  } else {
-    throw "Unknown LLM provider";
-  }
-  return response;
-}
-function checkField(sentence) {
-  if (!sentence) return "";
-  else return sentence;
-}
-async function readJSONFile(file) {
-  const json_file = await fs.promises.readFile(file);
-  return JSON.parse(json_file.toString());
-}
-async function classify_query_by_dataset_type(user_prompt, llm, aiRoute, dataset_json) {
-  const data = await readJSONFile(aiRoute);
-  let contents = data["general"];
-  for (const key of Object.keys(data)) {
-    if (key != "general") {
-      contents += data[key];
-    }
-  }
-  const classification_ds = dataset_json.charts.find((chart) => chart.type == "Classification");
-  if (!classification_ds) throw "Classification information is not present in the dataset file.";
-  if (classification_ds.TrainingData.length == 0) throw "No training data is provided for the classification agent.";
-  let training_data = "";
-  if (classification_ds && classification_ds.TrainingData.length > 0) {
-    contents += checkField(dataset_json.DatasetPrompt) + checkField(classification_ds.SystemPrompt);
-    training_data = formatTrainingExamples(classification_ds.TrainingData);
-  }
-  const template = contents + " training data is as follows:" + training_data + " Question: {" + user_prompt + "} Answer: {answer}";
-  const response = await route_to_appropriate_llm_provider(template, llm);
-  return JSON.parse(response);
-}
-async function extract_DE_search_terms_from_query(prompt, llm, dataset_db_output, dataset_json, ds, testing) {
-  if (dataset_json.hasDE) {
-    const Schema = {
-      $schema: "http://json-schema.org/draft-07/schema#",
-      $ref: "#/definitions/DEType",
-      definitions: {
-        DEType: {
-          type: "object",
-          properties: {
-            group1: {
-              type: "array",
-              items: { $ref: "#/definitions/FilterTerm" },
-              description: "Name of group1 which is an array of filter terms"
-            },
-            group2: {
-              type: "array",
-              items: { $ref: "#/definitions/FilterTerm" },
-              description: "Name of group2 which is an array of filter terms"
-            },
-            method: {
-              type: "string",
-              enum: ["edgeR", "limma", "wilcoxon"],
-              description: "Method used for carrying out differential gene expression analysis"
-            }
-          },
-          required: ["group1", "group2"],
-          additionalProperties: false
-        },
-        ...FILTER_TERM_DEFINITIONS
-      }
-    };
-    const DE_ds = dataset_json.charts.find((chart) => chart.type == "DE");
-    if (!DE_ds) throw "DE information is not present in the dataset file.";
-    if (DE_ds.TrainingData.length == 0) throw "No training data is provided for the DE agent.";
-    const training_data = formatTrainingExamples(DE_ds.TrainingData);
-    const system_prompt = "I am an assistant that extracts the groups from the user prompt to carry out differential gene expression. The final output must be in the following JSON with NO extra comments. The schema is as follows: " + JSON.stringify(Schema) + ' . "group1" and "group2" fields are compulsory. Both "group1" and "group2" consist of an array of filter variables. ' + FILTER_DESCRIPTION + checkField(dataset_json.DatasetPrompt) + checkField(DE_ds.SystemPrompt) + "The sqlite db in plain language is as follows:\n" + dataset_db_output.rag_docs.join(",") + " training data is as follows:" + training_data + " Question: {" + prompt + "} answer:";
-    const response = await route_to_appropriate_llm_provider(system_prompt, llm);
-    if (testing) {
-      return { action: "dge", response: JSON.parse(response) };
-    } else {
-      return await validate_DE_response(response, ds, dataset_db_output.db_rows);
-    }
-  } else {
-    return { type: "html", html: "Differential gene expression not supported for this dataset" };
-  }
-}
-async function validate_DE_response(response, ds, db_rows) {
-  const response_type = JSON.parse(response);
-  let html = "";
-  let group1;
-  let samples1lst;
-  const name1 = generate_group_name(response_type.group1, db_rows);
-  if (!response_type.group1) {
-    html += "group1 not present in DE output";
-  } else {
-    const validated_filters = validate_filter(response_type.group1, ds, name1);
-    if (validated_filters.html.length > 0) {
-      html += validated_filters.html;
-    } else {
-      const samples1 = await get_samples({ filter: validated_filters.simplefilter }, ds, true);
-      samples1lst = samples1.map((item) => ({
-        sampleId: item.id,
-        sample: item.name
-      }));
-      group1 = {
-        name: name1,
-        in: true,
-        values: samples1lst
+  let ai_output_json;
+  if (class_response.type === "notplot") {
+    const time2 = (/* @__PURE__ */ new Date()).valueOf();
+    const notPlotResult = await classifyNotPlot(userPrompt, llm, agentFiles, aiFilesDir);
+    mayLog("Time taken for classify2:", formatElapsedTime(Date.now() - time2));
+    if (notPlotResult.type === "html") {
+      ai_output_json = notPlotResult;
+    } else {
+      ai_output_json = {
+        type: "text",
+        text: "Your query does not appear to be related to the available data visualizations. Please try rephrasing your question."
       };
     }
-  }
-  let group2;
-  let samples2lst;
-  const name2 = generate_group_name(response_type.group2, db_rows);
-  if (!response_type.group2) {
-    html += "group2 not present in DE output";
-  } else {
-    const validated_filters = validate_filter(response_type.group2, ds, name2);
-    if (validated_filters.html.length > 0) {
-      html += validated_filters.html;
-    } else {
-      const samples2 = await get_samples({ filter: validated_filters.simplefilter }, ds, true);
-      samples2lst = samples2.map((item) => ({
-        sampleId: item.id,
-        sample: item.name
-      }));
-      group2 = {
-        name: name2,
-        in: true,
-        values: samples2lst
+  } else if (class_response.type === "binaryQuery") {
+    const answer = await answerDataQueries(userPrompt, llm, allowedTermTypes);
+    if (!answer) throw "Couldn't decide if this is data related query!";
+    mayLog("Data Binary Query: ", answer);
+    ai_output_json = answer;
+  } else if (class_response.type === "plot") {
+    let time = (/* @__PURE__ */ new Date()).valueOf();
+    const plotType = await classifyPlotType(userPrompt, llm);
+    mayLog("Time taken to classify plot type:", formatElapsedTime(Date.now() - time));
+    if (!supportedPlotTypes.includes(plotType)) {
+      const log = 'Plot type: "' + plotType + '" is not supported.';
+      ai_output_json = {
+        type: "text",
+        text: log
       };
-    }
-  }
-  let settings;
-  if (response_type.method) {
-    if (response_type.method == "edgeR" || response_type.method == "limma" || response_type.method == "wilcoxon") {
-      settings = { volcano: { method: response_type.method } };
-    } else {
-      html += "Unknown DE method: " + response_type.method;
-    }
-  }
-  if (html.length > 0) {
-    html = removeLastOccurrence(
-      html,
-      "For now, the maximum number of filter terms supported through the chatbot is " + num_filter_cutoff
-      // Remove duplicated statements in error message
+      mayLog(log);
+      return ai_output_json;
+    }
+    const genes_list = await parse_geneset_db(genedb);
+    mayLog("#################################################");
+    mayLog("####### First phase: Infer Plot Scaffolds #######");
+    mayLog("#################################################");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const dataset_db = serverconfig.tpmasterdir + "/" + ds.cohort.db.file;
+    const scaffoldResult = await inferScaffold(
+      userPrompt,
+      plotType,
+      llm,
+      genome,
+      genes_list,
+      allowedTermTypes,
+      dataset_json,
+      ds,
+      dataset_db
     );
-    return { type: "html", html };
-  } else {
-    const pp_plot_json = { childType: "volcano", termType: "geneExpression", chartType: "differentialAnalysis" };
-    const groups = [group1, group2];
-    const tw = {
-      q: {
-        groups
-      },
-      term: {
-        name: name1 + " vs " + name2,
-        type: "samplelst",
-        values: {
-          [name1]: {
-            color: "purple",
-            key: name1,
-            label: name1,
-            list: samples1lst
-          },
-          [name2]: {
-            color: "blue",
-            key: name2,
-            label: name2,
-            list: samples2lst
-          }
-        }
-      }
-    };
-    pp_plot_json.state = {
-      customTerms: [
-        {
-          name: name1 + " vs " + name2,
-          tw
-        }
-      ],
-      groups
-    };
-    pp_plot_json.samplelst = { groups };
-    pp_plot_json.tw = tw;
-    pp_plot_json.settings = settings;
-    return { type: "plot", plot: pp_plot_json };
-  }
-}
-function generate_group_name(filters, db_rows) {
-  let name = "";
-  let iter = 0;
-  for (const filter of filters) {
-    if (iter > 0 && !filter.join) {
-      name += "&";
-    }
-    if (filter.join && filter.join == "and") {
-      name += "&";
-    }
-    if (filter.join && filter.join == "or") {
-      name += "|";
-    }
-    if (filter.category) {
-      name += find_label(filter, db_rows);
-    }
-    if (filter.start) {
-      name += filter.term + ">=" + filter.start.toString();
-    }
-    if (filter.stop) {
-      name += filter.term + "<=" + filter.stop.toString();
-    }
-    iter += 1;
-  }
-  return name;
-}
-function find_label(filter, db_rows) {
-  let label = "";
-  for (const row of db_rows) {
-    if (row.name == filter.term) {
-      for (const value of row.values) {
-        if (value.value && value.value.label && filter.category == value.key) {
-          label = value.value.label;
-          break;
-        }
-      }
-      break;
-    }
-  }
-  return label;
-}
-async function extract_summary_terms(prompt, llm, dataset_db_output, dataset_json, genes_list, ds, testing) {
-  const Schema = {
-    $schema: "http://json-schema.org/draft-07/schema#",
-    $ref: "#/definitions/SummaryType",
-    definitions: {
-      SummaryType: {
-        type: "object",
-        properties: {
-          term: { type: "string", description: "Name of 1st term" },
-          term2: { type: "string", description: "Name of 2nd term" },
-          simpleFilter: {
-            type: "array",
-            items: { $ref: "#/definitions/FilterTerm" },
-            description: "Optional simple filter terms"
-          },
-          childType: {
-            type: "string",
-            enum: ["violin", "boxplot", "sampleScatter", "barchart"],
-            description: "Optional explicit child type requested by the user. If omitted, the logic of the data types picks the child type."
-          }
-        },
-        required: ["term", "simpleFilter"],
-        additionalProperties: false
-      },
-      ...FILTER_TERM_DEFINITIONS
-    }
-  };
-  const common_genes = extractGenesFromPrompt(prompt, genes_list);
-  const summary_ds = dataset_json.charts.find((chart) => chart.type == "Summary");
-  if (!summary_ds) throw "Summary information is not present in the dataset file.";
-  if (summary_ds.TrainingData.length == 0) throw "No training data is provided for the summary agent.";
-  const training_data = formatTrainingExamples(summary_ds.TrainingData);
-  let system_prompt = "I am an assistant that extracts the summary terms from user query. The final output must be in the following JSON format with NO extra comments. The JSON schema is as follows: " + JSON.stringify(Schema) + ' term and term2 (if present) should ONLY contain names of the fields from the sqlite db. The "simpleFilter" field is optional and should contain an array of JSON terms with which the dataset will be filtered. ' + FILTER_DESCRIPTION + checkField(dataset_json.DatasetPrompt) + checkField(summary_ds.SystemPrompt) + "\n The DB content is as follows: " + dataset_db_output.rag_docs.join(",") + " training data is as follows:" + training_data;
-  if (dataset_json.hasGeneExpression) {
-    if (common_genes.length > 0) {
-      system_prompt += "\n List of relevant genes are as follows (separated by comma(,)):" + common_genes.join(",");
-    }
-  }
-  system_prompt += " Question: {" + prompt + "} answer:";
-  const response = await route_to_appropriate_llm_provider(system_prompt, llm);
-  if (testing) {
-    return { action: "summary", response: JSON.parse(response) };
-  } else {
-    return validate_summary_response(response, common_genes, dataset_json, ds);
-  }
-}
-function validate_summary_response(response, common_genes, dataset_json, ds) {
-  const response_type = JSON.parse(response);
-  const pp_plot_json = { chartType: "summary" };
-  let html = "";
-  if (response_type.html) html = response_type.html;
-  if (!response_type.term) {
-    html += "term type is not present in summary output";
-    return { type: "html", html };
-  }
-  const term1_validation = validate_term(response_type.term, common_genes, dataset_json, ds);
-  if (term1_validation.html.length > 0) {
-    html += term1_validation.html;
-    return { type: "html", html };
-  } else {
-    pp_plot_json.term = term1_validation.term_type;
-    if (term1_validation.category == "float" || term1_validation.category == "integer") {
-      pp_plot_json.term.q = { mode: "continuous" };
-    }
-    pp_plot_json.category = term1_validation.category;
-  }
-  if (response_type.term2) {
-    const term2_validation = validate_term(response_type.term2, common_genes, dataset_json, ds);
-    if (term2_validation.html.length > 0) {
-      html += term2_validation.html;
-      return { type: "html", html };
-    } else {
-      pp_plot_json.term2 = term2_validation.term_type;
-      if (term2_validation.category == "float" || term2_validation.category == "integer") {
-        pp_plot_json.term2.q = { mode: "continuous" };
-      }
-      pp_plot_json.category2 = term2_validation.category;
-    }
-  }
-  const llmChildType = response_type.childType && ["violin", "boxplot", "sampleScatter", "barchart"].includes(response_type.childType) ? response_type.childType : void 0;
-  const resolved = resolveChildType(pp_plot_json.category, pp_plot_json.category2, llmChildType);
-  if (resolved.error) {
-    html += resolved.error;
-    return { type: "html", html };
-  } else {
-    pp_plot_json.childType = resolved.childType;
-    if (pp_plot_json.childType == "barchart") {
-      pp_plot_json.term.q = { mode: "discrete" };
-      if (pp_plot_json.term2) {
-        pp_plot_json.term2.q = { mode: "discrete" };
-      }
-    }
-    if (resolved.bothNumeric && (resolved.childType == "violin" || resolved.childType == "boxplot")) {
-      pp_plot_json.term2.q = { mode: "discrete" };
-    }
-  }
-  delete pp_plot_json.category;
-  if (pp_plot_json.category2) delete pp_plot_json.category2;
-  if (response_type.simpleFilter && response_type.simpleFilter.length > 0) {
-    const validated_filters = validate_filter(response_type.simpleFilter, ds, "");
-    if (validated_filters.html.length > 0) {
-      html += validated_filters.html;
-      return { type: "html", html };
-    } else {
-      pp_plot_json.filter = validated_filters.simplefilter;
-    }
-  }
-  return { type: "plot", plot: pp_plot_json };
-}
-async function extract_matrix_search_terms_from_query(prompt, llm, dataset_db_output, dataset_json, genes_list, ds, testing) {
-  const Schema = {
-    $schema: "http://json-schema.org/draft-07/schema#",
-    $ref: "#/definitions/MatrixType",
-    definitions: {
-      MatrixType: {
-        type: "object",
-        properties: {
-          terms: {
-            type: "array",
-            items: { type: "string" },
-            description: "Names of dictionary/clinical terms to include as rows in the matrix"
-          },
-          geneNames: {
-            type: "array",
-            items: { type: "string" },
-            description: "Names of genes to include as gene variant rows in the matrix"
-          },
-          simpleFilter: {
-            type: "array",
-            items: { $ref: "#/definitions/FilterTerm" },
-            description: "Optional simple filter terms to restrict the sample set"
-          }
-        },
-        additionalProperties: false
-      },
-      ...FILTER_TERM_DEFINITIONS
-    }
-  };
-  const common_genes = extractGenesFromPrompt(prompt, genes_list);
-  const matrix_ds = dataset_json.charts.filter((chart) => chart.type == "Matrix");
-  if (matrix_ds.length == 0) throw "Matrix information is not present in the dataset file.";
-  if (matrix_ds[0].TrainingData.length == 0) throw "No training data is provided for the matrix agent.";
-  const training_data = formatTrainingExamples(matrix_ds[0].TrainingData);
-  let system_prompt = "I am an assistant that extracts terms and gene names from the user query to create a matrix plot. A matrix plot displays multiple genes and/or clinical variables across samples in a grid layout. The final output must be in the following JSON format with NO extra comments. The JSON schema is as follows: " + JSON.stringify(Schema) + ' The "terms" field should ONLY contain names of clinical/dictionary fields from the sqlite db. The "geneNames" field should ONLY contain gene names. At least one of "terms" or "geneNames" must be provided. The "simpleFilter" field is optional and should contain an array of JSON terms with which the dataset will be filtered. ' + FILTER_DESCRIPTION + checkField(dataset_json.DatasetPrompt) + checkField(matrix_ds[0].SystemPrompt) + "\n The DB content is as follows: " + dataset_db_output.rag_docs.join(",") + " training data is as follows:" + training_data;
-  if (dataset_json.hasGeneExpression && common_genes.length > 0) {
-    system_prompt += "\n List of relevant genes are as follows (separated by comma(,)):" + common_genes.join(",");
-  }
-  system_prompt += " Question: {" + prompt + "} answer:";
-  const response = await route_to_appropriate_llm_provider(system_prompt, llm);
-  if (testing) {
-    return { action: "matrix", response: JSON.parse(response) };
-  } else {
-    return validate_matrix_response(response, common_genes, dataset_json, ds);
-  }
-}
-function validate_matrix_response(response, common_genes, dataset_json, ds) {
-  const response_type = JSON.parse(response);
-  const pp_plot_json = { chartType: "matrix" };
-  let html = "";
-  if (response_type.html) html = response_type.html;
-  if ((!response_type.terms || response_type.terms.length == 0) && (!response_type.geneNames || response_type.geneNames.length == 0)) {
-    html += "At least one clinical term or gene name is required for a matrix plot";
-  }
-  const twLst = [];
-  if (response_type.terms && Array.isArray(response_type.terms)) {
-    for (const t of response_type.terms) {
-      const term = ds.cohort.termdb.q.termjsonByOneid(t);
-      if (!term) {
-        html += "invalid term id:" + t + " ";
-      } else {
-        twLst.push({ id: term.id });
-      }
-    }
-  }
-  if (response_type.geneNames && Array.isArray(response_type.geneNames)) {
-    for (const g of response_type.geneNames) {
-      const gene_hits = common_genes.filter((gene) => gene == g.toLowerCase());
-      if (gene_hits.length == 0) {
-        html += "invalid gene name:" + g + " ";
-      } else {
-        const geneName = g.toUpperCase();
-        if (dataset_json.hasGeneExpression) {
-          twLst.push({ term: { gene: geneName, type: "geneExpression" } });
-        } else {
-          twLst.push({ term: { gene: geneName, name: geneName, type: "geneVariant" } });
-        }
-      }
-    }
-  }
-  if (response_type.simpleFilter && response_type.simpleFilter.length > 0) {
-    const validated_filters = validate_filter(response_type.simpleFilter, ds, "");
-    if (validated_filters.html.length > 0) {
-      html += validated_filters.html;
-    } else {
-      pp_plot_json.filter = validated_filters.simplefilter;
-    }
-  }
-  if (html.length > 0) {
-    return { type: "html", html };
-  } else {
-    pp_plot_json.termgroups = [{ name: "", lst: twLst }];
-    return { type: "plot", plot: pp_plot_json };
-  }
-}
-async function extract_samplescatter_terms_from_query(prompt, llm, dataset_db_output, dataset_json, genes_list, ds, testing) {
-  if (!dataset_json.prebuiltPlots || dataset_json.prebuiltPlots.length == 0) {
-    return { type: "html", html: "No pre-built scatter plots (t-SNE/UMAP) are available for this dataset" };
-  }
-  const Schema = {
-    $schema: "http://json-schema.org/draft-07/schema#",
-    $ref: "#/definitions/SampleScatterType",
-    definitions: {
-      SampleScatterType: {
-        type: "object",
-        properties: {
-          plotName: {
-            type: "string",
-            description: "Name of the pre-built scatter plot to display"
-          },
-          colorTW: {
-            type: ["string", "null"],
-            description: "Term name or gene name to overlay as color on the scatter plot. Set to null to remove the color overlay."
-          },
-          shapeTW: {
-            type: ["string", "null"],
-            description: "Term name or gene name to overlay as shape on the scatter plot. Set to null to remove the shape overlay."
-          },
-          term0: {
-            type: ["string", "null"],
-            description: "Term name to use for Z/Divide which splits the plot into panels. Set to null to remove the divide overlay."
-          },
-          simpleFilter: {
-            type: "array",
-            items: { $ref: "#/definitions/FilterTerm" },
-            description: "Optional simple filter terms to restrict the sample set"
-          }
-        },
-        required: ["plotName"],
-        additionalProperties: false
-      },
-      ...FILTER_TERM_DEFINITIONS
-    }
-  };
-  const common_genes = extractGenesFromPrompt(prompt, genes_list);
-  const scatter_ds = dataset_json.charts.find((chart) => chart.type == "sampleScatter");
-  if (!scatter_ds) throw "sampleScatter information is not present in the dataset file.";
-  if (scatter_ds.TrainingData.length == 0) throw "No training data is provided for the sampleScatter agent.";
-  const training_data = formatTrainingExamples(scatter_ds.TrainingData);
-  const plotNames = dataset_json.prebuiltPlots.map((p) => p.name).join(", ");
-  let system_prompt = "I am an assistant that extracts overlay parameters for pre-built scatter plots (t-SNE/UMAP). The final output must be in the following JSON format with NO extra comments. The JSON schema is as follows: " + JSON.stringify(Schema) + " The available pre-built plots are: " + plotNames + '. The "plotName" field must match one of these exactly. The "colorTW", "shapeTW", and "term0" fields should contain names of clinical fields from the sqlite db OR gene names. To remove an overlay, set the corresponding field to null explicitly. If the user does not mention a particular overlay, do NOT include that field in the output (omit it entirely). Only include "colorTW", "shapeTW", or "term0" if the user explicitly mentions coloring, shaping, or dividing. ' + FILTER_DESCRIPTION + checkField(dataset_json.DatasetPrompt) + checkField(scatter_ds.SystemPrompt) + "\n The DB content is as follows: " + dataset_db_output.rag_docs.join(",") + " training data is as follows:" + training_data;
-  if (dataset_json.hasGeneExpression && common_genes.length > 0) {
-    system_prompt += "\n List of relevant genes are as follows (separated by comma(,)):" + common_genes.join(",");
-  }
-  system_prompt += " Question: {" + prompt + "} answer:";
-  const response = await route_to_appropriate_llm_provider(system_prompt, llm);
-  if (testing) {
-    return { action: "sampleScatter", response: JSON.parse(response) };
-  } else {
-    return validate_samplescatter_response(response, common_genes, dataset_json, ds);
-  }
-}
-function validate_samplescatter_response(response, common_genes, dataset_json, ds) {
-  const response_type = JSON.parse(response);
-  let html = "";
-  if (response_type.html) html = response_type.html;
-  if (!response_type.plotName) {
-    html += "plotName is required for sample scatter output";
-  } else {
-    const matchedPlot = dataset_json.prebuiltPlots.find(
-      (p) => p.name.toLowerCase() == response_type.plotName.toLowerCase()
+    mayLog("ScaffoldResult: ", scaffoldResult);
+    if (plotType === "hiercluster" && "plot" in scaffoldResult && scaffoldResult.type === "plot" || "text" in scaffoldResult && scaffoldResult.type === "text") {
+      return scaffoldResult;
+    }
+    mayLog("Time taken to infer scaffold:", formatElapsedTime(Date.now() - time));
+    if (!scaffoldResult)
+      throw "Scaffold result is empty or undefined, which is unexpected. Please check the inferScaffold agent for potential issues.";
+    if ("type" in scaffoldResult && scaffoldResult.type === "text") {
+      return scaffoldResult;
+    }
+    const subplotType = scaffoldResult.plotType === "summary" ? scaffoldResult.chartType : void 0;
+    mayLog("#################################################");
+    mayLog("####### Second phase: From Scaffolds's phrases infer Entities #######");
+    mayLog("#################################################");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const phrase2entityResult = await phrase2entity(
+      scaffoldResult,
+      plotType,
+      llm,
+      genes_list,
+      dataset_json,
+      ds,
+      genome
     );
-    if (!matchedPlot) {
-      const availablePlots = dataset_json.prebuiltPlots.map((p) => p.name).join(", ");
-      html += "Unknown plot name: " + response_type.plotName + ". Available plots are: " + availablePlots;
-    }
-  }
-  const pp_plot_json = {
-    chartType: "sampleScatter",
-    name: response_type.plotName
-  };
-  const validateOverlayTerm = (termName, fieldKey) => {
-    if (termName === null) {
-      pp_plot_json[fieldKey] = null;
-      return;
-    }
-    if (termName === void 0) {
-      return;
-    }
-    const termValidation = validate_term(termName, common_genes, dataset_json, ds);
-    if (termValidation.html.length > 0) {
-      html += termValidation.html;
-    } else {
-      const tw = { ...termValidation.term_type };
-      if (termValidation.category == "float" || termValidation.category == "integer") {
-        tw.q = { mode: "continuous" };
-      }
-      pp_plot_json[fieldKey] = tw;
-    }
-  };
-  validateOverlayTerm(response_type.colorTW, "colorTW");
-  validateOverlayTerm(response_type.shapeTW, "shapeTW");
-  validateOverlayTerm(response_type.term0, "term0");
-  if (response_type.simpleFilter && response_type.simpleFilter.length > 0) {
-    const validated_filters = validate_filter(response_type.simpleFilter, ds, "");
-    if (validated_filters.html.length > 0) {
-      html += validated_filters.html;
-    } else {
-      pp_plot_json.filter = validated_filters.simplefilter;
-    }
-  }
-  if (html.length > 0) {
-    return { type: "html", html };
-  } else {
-    return { type: "plot", plot: pp_plot_json };
-  }
-}
-function validate_term(response_term, common_genes, dataset_json, ds) {
-  let html = "";
-  let term_type;
-  let category = "";
-  const term = ds.cohort.termdb.q.termjsonByOneid(response_term);
-  if (!term) {
-    const gene_hits = common_genes.filter((gene) => gene == response_term.toLowerCase());
-    if (gene_hits.length == 0) {
-      html += "invalid term id:" + response_term;
-    } else {
-      if (dataset_json.hasGeneExpression) {
-        term_type = { term: { gene: response_term.toUpperCase(), type: "geneExpression" } };
-        category = "float";
-      } else {
-        html += "Dataset does not support gene expression";
-      }
-    }
-  } else {
-    term_type = { id: term.id };
-    category = term.type;
-  }
-  return { term_type, html, category };
-}
-function countOccurrences(str, word) {
-  if (word === "") return 0;
-  let count = 0;
-  let pos = 0;
-  while ((pos = str.indexOf(word, pos)) !== -1) {
-    count++;
-    pos += word.length;
-  }
-  return count;
-}
-function removeLastOccurrence(str, word) {
-  const index = str.lastIndexOf(word);
-  if (index === -1) return str;
-  const occurrences = countOccurrences(str, word);
-  if (occurrences === 1) {
-    return str;
-  } else {
-    return str.slice(0, index) + str.slice(index + word.length);
-  }
-}
-function sortSameCategoricalFilterKeys(filters, ds) {
-  let html = "";
-  const keys = filters.map((f) => f.term);
-  if (new Set(keys).size == keys.length) return { filters, html };
-  const seen = /* @__PURE__ */ new Set();
-  const categorical_filter_terms_with_multiple_fields = /* @__PURE__ */ new Set();
-  for (const item of filters) {
-    if (seen.has(item.term)) categorical_filter_terms_with_multiple_fields.add(item.term);
-    else seen.add(item.term);
-  }
-  const multiple_fields_keys = [];
-  for (const key of categorical_filter_terms_with_multiple_fields) {
-    const term = ds.cohort.termdb.q.termjsonByOneid(key);
-    if (!term) {
-      html += "invalid filter id:" + key;
-    } else {
-      if (term.type == "categorical") {
-        const multiple_fields = filters.filter((x) => x.term == key);
-        multiple_fields_keys.push({ key, categories: multiple_fields.map((f) => f.category) });
-      }
-    }
-  }
-  const sorted_filter = [];
-  const seen2 = /* @__PURE__ */ new Set();
-  for (const f of filters) {
-    const repeated_term = multiple_fields_keys.find((x) => x.key == f.term);
-    if (!repeated_term) {
-      sorted_filter.push(f);
-    } else {
-      if (!seen2.has(f.term)) {
-        const new_filter_term = {
-          term: f.term,
-          category: repeated_term.categories
-        };
-        seen2.add(f.term);
-        sorted_filter.push(new_filter_term);
-      }
-    }
-  }
-  return { filters: sorted_filter, html };
-}
-function validate_filter(filters, ds, group_name) {
-  if (!Array.isArray(filters)) throw "filter is not array";
-  const sorted_filters = sortSameCategoricalFilterKeys(filters, ds);
-  let filter_result = { html: sorted_filters.html };
-  if (sorted_filters.filters.length <= 2) {
-    const generated = generate_filter_term(sorted_filters.filters, ds);
-    filter_result.simplefilter = generated.simplefilter;
-    filter_result.html += generated.html;
-  } else {
-    if (sorted_filters.filters.length > num_filter_cutoff) {
-      filter_result.html += "For now, the maximum number of filter terms supported through the chatbot is " + num_filter_cutoff;
-      if (group_name.length > 0) {
-        filter_result.html += " . The number of filter terms for group " + group_name + " is " + sorted_filters.filters.length + "\n";
-      } else {
-        filter_result.html += "The number of filter terms for this query is " + sorted_filters.filters.length;
-      }
-    } else {
-      for (let i = 0; i < sorted_filters.filters.length - 1; i++) {
-        const filter_lst = [];
-        if (i == 0) {
-          filter_lst.push(sorted_filters.filters[i]);
-        } else {
-          filter_lst.push(filter_result.simplefilter);
-        }
-        filter_lst.push(sorted_filters.filters[i + 1]);
-        filter_result = generate_filter_term(filter_lst, ds);
-      }
-    }
-  }
-  return { simplefilter: filter_result.simplefilter, html: filter_result.html };
-}
-function generate_filter_term(filters, ds) {
-  let invalid_html = "";
-  const localfilter = { type: "tvslst", in: true, lst: [] };
-  for (const f of filters) {
-    if (f.type == "tvslst") {
-      localfilter.lst.push(f);
-    } else {
-      const term = ds.cohort.termdb.q.termjsonByOneid(f.term);
-      if (!term) {
-        invalid_html += "invalid filter id:" + f.term;
-      } else {
-        if (f.join) {
-          localfilter.join = f.join;
-        }
-        if (term.type == "categorical") {
-          if (Array.isArray(f.category)) {
-            const categories = [];
-            for (const category of f.category) {
-              const cat = findCategoryKey(term.values, category);
-              if (!cat) invalid_html += "invalid category from " + JSON.stringify(f);
-              else {
-                categories.push({ key: cat });
-              }
-            }
-            localfilter.lst.push({
-              type: "tvs",
-              tvs: {
-                term,
-                values: categories
-              }
-            });
-          } else {
-            const cat = findCategoryKey(term.values, f.category);
-            if (!cat) invalid_html += "invalid category from " + JSON.stringify(f);
-            else {
-              localfilter.lst.push({
-                type: "tvs",
-                tvs: {
-                  term,
-                  values: [{ key: cat }]
-                }
-              });
-            }
-          }
-        } else if (term.type == "float" || term.type == "integer") {
-          const numeric = {
-            type: "tvs",
-            tvs: {
-              term,
-              ranges: []
-            }
-          };
-          const range = {};
-          if (f.start && !f.stop) {
-            range.start = Number(f.start);
-            range.stopunbounded = true;
-          } else if (f.stop && !f.start) {
-            range.stop = Number(f.stop);
-            range.startunbounded = true;
-          } else if (f.start && f.stop) {
-            range.start = Number(f.start);
-            range.stop = Number(f.stop);
-          } else {
-            invalid_html += "Neither greater or lesser defined";
-          }
-          numeric.tvs.ranges.push(range);
-          localfilter.lst.push(numeric);
-        }
-      }
-    }
-  }
-  if (filters.length > 1 && !localfilter.join) {
-    localfilter.join = "and";
-  }
-  return { simplefilter: localfilter, html: invalid_html };
-}
-function findCategoryKey(termValues, category) {
-  for (const ck in termValues) {
-    if (ck === category || termValues[ck].label === category) return ck;
-  }
-  return void 0;
-}
-async function parse_geneset_db(genedb) {
-  let genes_list = [];
-  const db = new Database(genedb);
-  try {
-    const desc_rows = db.prepare("SELECT name from codingGenes").all();
-    desc_rows.forEach((row) => {
-      genes_list.push(row.name);
-    });
-    genes_list = genes_list.map((str) => str.toLowerCase());
-  } catch (error) {
-    throw "Could not parse geneDB" + error;
-  } finally {
-    db.close();
-  }
-  return genes_list;
-}
-async function parse_dataset_db(dataset_db) {
-  const db = new Database(dataset_db);
-  const rag_docs = [];
-  const db_rows = [];
-  try {
-    const desc_rows = db.prepare("SELECT * from termhtmldef").all();
-    const description_map = [];
-    desc_rows.forEach((row) => {
-      const name = row.id;
-      const jsonhtml = JSON.parse(row.jsonhtml);
-      const description = jsonhtml.description[0].value;
-      description_map.push({ name, description });
-    });
-    const term_db_rows = db.prepare("SELECT * from terms").all();
-    term_db_rows.forEach((row) => {
-      const found = description_map.find((item) => item.name === row.id);
-      if (found) {
-        const jsondata = JSON.parse(row.jsondata);
-        const description = description_map.filter((item) => item.name === row.id);
-        const term_type = row.type;
-        const values = [];
-        if (jsondata.values && Object.keys(jsondata.values).length > 0) {
-          for (const key of Object.keys(jsondata.values)) {
-            const value = jsondata.values[key];
-            const db_val = { key, value };
-            values.push(db_val);
-          }
-        }
-        const db_row = {
-          name: row.id,
-          description: description[0].description,
-          values,
-          term_type
-        };
-        const stringified_db = parse_db_rows(db_row);
-        rag_docs.push(stringified_db);
-        db_rows.push(db_row);
-      }
-    });
-  } catch (error) {
-    throw "Error in parsing dataset DB:" + error;
-  } finally {
-    db.close();
-  }
-  return { db_rows, rag_docs };
-}
-function parse_db_rows(db_row) {
-  let output_string = 'Name of the field is:"' + db_row.name + '". This field is of the type:' + db_row.term_type + ". Description: " + db_row.description;
-  if (db_row.values.length > 0) {
-    output_string += "This field contains the following possible values.";
-    for (const value of db_row.values) {
-      if (value.value && value.value.label) {
-        output_string += 'The key is "' + value.key + '" and the label is "' + value.value.label + '".';
-      }
-    }
+    mayLog("Time taken to phrase 2 entity:", formatElapsedTime(Date.now() - time));
+    if ("type" in phrase2entityResult && phrase2entityResult.type === "text") {
+      return phrase2entityResult;
+    }
+    mayLog(phrase2entityResult);
+    mayLog("#################################################");
+    mayLog("####### Third phase: From Entities infer Term Objects #######");
+    mayLog("#################################################");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const termObj = await inferTermObjFromEntity(
+      phrase2entityResult,
+      plotType,
+      llm,
+      dataset_db,
+      genes_list,
+      genome
+    );
+    mayLog("Time taken to infer term objects:", formatElapsedTime(Date.now() - time));
+    mayLog("Inferred termObj from entity:", JSON.stringify(termObj));
+    mayLog("#################################################");
+    mayLog("####### Fourth phase: From Term Objects to TwTvs Objects #######");
+    mayLog("#################################################");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    const twTvsObj = await resolveToTwTvs(termObj, plotType, llm, dataset_db, genome);
+    mayLog("Time taken to resolve to TwTvs object from termObj:", formatElapsedTime(Date.now() - time));
+    if ("type" in twTvsObj && twTvsObj.type === "text") {
+      return twTvsObj;
+    }
+    mayLog("twTvsObj:", twTvsObj);
+    mayLog("#################################################");
+    mayLog("####### Fifth/Final phase: From TwTvs Objects to Plot States #######");
+    mayLog("#################################################");
+    time = (/* @__PURE__ */ new Date()).valueOf();
+    ai_output_json = resolveToPlotState(twTvsObj, plotType, subplotType);
+    mayLog("Time taken to resolve to plot state:", formatElapsedTime(Date.now() - time));
   }
-  return output_string;
+  return ai_output_json;
 }
 export {
-  api,
-  readJSONFile,
+  init,
   run_chat_pipeline
 };