@sjcrh/proteinpaint-server 2.187.0 → 2.188.1

package/routes/termdb.chat3.js

@@ -1,7 +1,7 @@
 import { ChatPayload } from "#types/checkers";
 import { mayLog } from "#src/helpers.ts";
 import { formatElapsedTime } from "#shared";
-import { readJSONFile, parse_geneset_db } from "./chat/utils.ts";
+import { readJSONFile, parse_geneset_db, getChatRelatedPlotTypes } from "./chat/utils.ts";
 import { classifyQuery } from "./chat/classify1.ts";
 import { classifyPlotType } from "./chat/plot.ts";
 import { classifyNotPlot } from "./chat/classify2.ts";
@@ -11,6 +11,7 @@ import { getDsAllowedTermTypes } from "./termdb.config.ts";
 import { phrase2entity } from "./chat/phrase2entity.ts";
 import { inferTermObjFromEntity } from "./chat/entity2termObj.ts";
 import { resolveToTwTvs } from "./chat/entity2twTvs.ts";
+import { answerDataQueries } from "./chat/dataQueries.ts";
 import path from "path";
 import fs from "fs";
 import { resolveToPlotState } from "./chat/scaffold2state.ts";
@@ -31,9 +32,9 @@ function init({ genomes }) {
   return async (req, res) => {
     const q = req.query;
     try {
-      const g = genomes[q.genome];
-      if (!g) throw "invalid genome";
-      const ds = g.datasets?.[q.dslabel];
+      const genome = genomes[q.genome];
+      if (!genome) throw "invalid genome";
+      const ds = genome.datasets?.[q.dslabel];
       if (!ds) throw "invalid dslabel";
       const aiFilesDir = serverconfig.binpath + "/../../dataset/ai/" + q.dslabel;
       let agentFiles = [];
@@ -49,14 +50,24 @@ function init({ genomes }) {
       if (llm.provider !== "SJ" && llm.provider !== "ollama" && llm.provider !== "huggingface" && llm.provider !== "azure") {
         throw "llm.provider must be 'SJ', 'ollama', 'huggingface', or 'azure'";
       }
-      const rawFilter = typeof q.filter === "string" ? JSON.parse(q.filter) : q.filter;
+      let rawFilter;
+      if (typeof q.filter === "string") {
+        try {
+          rawFilter = JSON.parse(q.filter);
+        } catch (e) {
+          throw new Error("Failed to parse filter JSON string: " + e);
+        }
+      } else {
+        rawFilter = q.filter;
+      }
       const filter = rawFilter && typeof rawFilter === "object" ? rawFilter : {};
       const lst = Array.isArray(filter.lst) ? filter.lst : [];
       const cohortFilter = lst.find((item) => item.tag === "cohortFilter");
       const cohortKey = cohortFilter ? cohortFilter.tvs.values[0].key : "";
-      const supportedChartTypes = ds.cohort.termdb.q?.getSupportedChartTypes(req)?.[cohortKey];
-      const genedb = serverconfig.tpmasterdir + "/" + g.genedb.dbfile;
-      const _allowedTermTypes = getDsAllowedTermTypes(ds);
+      const supportedPlotTypes = ds.cohort.termdb.q?.getSupportedChartTypes(req)?.[cohortKey];
+      const chatSupportedPlotTypes = getChatRelatedPlotTypes(supportedPlotTypes);
+      const genedb = serverconfig.tpmasterdir + "/" + genome.genedb.dbfile;
+      const allowedTermTypes = getDsAllowedTermTypes(ds);
       const ai_output_json = await run_chat_pipeline(
         q.prompt,
         llm,
@@ -64,8 +75,9 @@ function init({ genomes }) {
         genedb,
         agentFiles,
         aiFilesDir,
-        supportedChartTypes,
-        _allowedTermTypes
+        chatSupportedPlotTypes,
+        allowedTermTypes,
+        genome
         // 	testing
       );
       mayLog("From init: Final AI output JSON:", JSON.stringify(ai_output_json));
@@ -76,19 +88,19 @@ function init({ genomes }) {
     }
   };
 }
-async function run_chat_pipeline(user_prompt, llm, ds, genedb, agentFiles, aiFilesDir, supportedChartTypes, _allowedTermTypes) {
+async function run_chat_pipeline(userPrompt, llm, ds, genedb, agentFiles, aiFilesDir, supportedPlotTypes, allowedTermTypes, genome) {
   if (!fs.existsSync(path.join(aiFilesDir, "main.json")))
     throw "Main data file is not specified for dataset:" + ds.label;
   const dataset_json = await readJSONFile(path.join(aiFilesDir, "main.json"));
   const time1 = (/* @__PURE__ */ new Date()).valueOf();
-  const class_response = await classifyQuery(user_prompt, llm);
+  const class_response = await classifyQuery(userPrompt, llm);
   mayLog("Time taken for classification:", formatElapsedTime(Date.now() - time1));
   let ai_output_json;
-  if (class_response.type == "notplot") {
+  if (class_response.type === "notplot") {
     const time2 = (/* @__PURE__ */ new Date()).valueOf();
-    const notPlotResult = await classifyNotPlot(user_prompt, llm, agentFiles, aiFilesDir);
+    const notPlotResult = await classifyNotPlot(userPrompt, llm, agentFiles, aiFilesDir);
     mayLog("Time taken for classify2:", formatElapsedTime(Date.now() - time2));
-    if (notPlotResult.type == "html") {
+    if (notPlotResult.type === "html") {
       ai_output_json = notPlotResult;
     } else {
       ai_output_json = {
@@ -96,70 +108,72 @@ async function run_chat_pipeline(user_prompt, llm, ds, genedb, agentFiles, aiFil
         text: "Your query does not appear to be related to the available data visualizations. Please try rephrasing your question."
       };
     }
-  } else if (class_response.type == "plot") {
+  } else if (class_response.type === "binaryQuery") {
+    const answer = await answerDataQueries(userPrompt, llm, allowedTermTypes);
+    if (!answer) throw "Couldn't decide if this is data related query!";
+    mayLog("Data Binary Query: ", answer);
+    ai_output_json = answer;
+  } else if (class_response.type === "plot") {
     let time = (/* @__PURE__ */ new Date()).valueOf();
-    const plotType = await classifyPlotType(user_prompt, llm);
+    const plotType = await classifyPlotType(userPrompt, llm);
     mayLog("Time taken to classify plot type:", formatElapsedTime(Date.now() - time));
-    if (!supportedChartTypes) {
-      const errorMsg = "Supported chart types list is undefined. Please check the dataset configuration and ensure 							  that getSupportedChartTypes is implemented correctly. Skipping chart type validation, but this may 							  lead to unsupported chart type errors downstream.";
-      console.warn(errorMsg);
-      const errorResponse = {
+    if (!supportedPlotTypes.includes(plotType)) {
+      const log = 'Plot type: "' + plotType + '" is not supported.';
+      ai_output_json = {
         type: "text",
-        text: errorMsg
+        text: log
       };
-      return errorResponse;
-    }
-    if (plotType === "summary") {
-      if (!supportedChartTypes.includes("dictionary")) {
-        const log = 'Plot type: "' + plotType + '" is not supported.';
-        ai_output_json = {
-          type: "text",
-          text: log
-        };
-        mayLog(log);
-        return ai_output_json;
-      }
-    } else if (plotType === "dge") {
-      if (!supportedChartTypes.includes("DA")) {
-        const log = 'Plot type: "' + plotType + '" is not supported.';
-        ai_output_json = {
-          type: "text",
-          text: log
-        };
-        mayLog(log);
-        return ai_output_json;
-      }
-    } else {
-      mayLog(`Supported chart types for this cohort: ${supportedChartTypes}`);
-      if (!supportedChartTypes.includes(plotType)) {
-        const log = 'Plot type: "' + plotType + '" is not supported.';
-        ai_output_json = {
-          type: "text",
-          text: log
-        };
-        mayLog(log);
-        return ai_output_json;
-      }
+      mayLog(log);
+      return ai_output_json;
     }
+    const genes_list = await parse_geneset_db(genedb);
+    mayLog("#################################################");
     mayLog("####### First phase: Infer Plot Scaffolds #######");
+    mayLog("#################################################");
     time = (/* @__PURE__ */ new Date()).valueOf();
-    const scaffoldResult = await inferScaffold(user_prompt, plotType, llm);
+    const dataset_db = serverconfig.tpmasterdir + "/" + ds.cohort.db.file;
+    const scaffoldResult = await inferScaffold(
+      userPrompt,
+      plotType,
+      llm,
+      genome,
+      genes_list,
+      allowedTermTypes,
+      dataset_json,
+      ds,
+      dataset_db
+    );
     mayLog("ScaffoldResult: ", scaffoldResult);
+    if (plotType === "hiercluster" && "plot" in scaffoldResult && scaffoldResult.type === "plot" || "text" in scaffoldResult && scaffoldResult.type === "text") {
+      return scaffoldResult;
+    }
     mayLog("Time taken to infer scaffold:", formatElapsedTime(Date.now() - time));
     if (!scaffoldResult)
       throw "Scaffold result is empty or undefined, which is unexpected. Please check the inferScaffold agent for potential issues.";
+    if ("type" in scaffoldResult && scaffoldResult.type === "text") {
+      return scaffoldResult;
+    }
     const subplotType = scaffoldResult.plotType === "summary" ? scaffoldResult.chartType : void 0;
+    mayLog("#################################################");
     mayLog("####### Second phase: From Scaffolds's phrases infer Entities #######");
-    const genes_list = await parse_geneset_db(genedb);
+    mayLog("#################################################");
     time = (/* @__PURE__ */ new Date()).valueOf();
-    const phrase2entityResult = await phrase2entity(scaffoldResult, plotType, llm, genes_list, dataset_json, ds);
+    const phrase2entityResult = await phrase2entity(
+      scaffoldResult,
+      plotType,
+      llm,
+      genes_list,
+      dataset_json,
+      ds
+    );
     mayLog("Time taken to phrase 2 entity:", formatElapsedTime(Date.now() - time));
     if ("type" in phrase2entityResult && phrase2entityResult.type === "text") {
       return phrase2entityResult;
     }
     mayLog(phrase2entityResult);
+    mayLog("#################################################");
     mayLog("####### Third phase: From Entities infer Term Objects #######");
-    const dataset_db = serverconfig.tpmasterdir + "/" + ds.cohort.db.file;
+    mayLog("#################################################");
     time = (/* @__PURE__ */ new Date()).valueOf();
     const termObj = await inferTermObjFromEntity(
       phrase2entityResult,
@@ -170,7 +184,9 @@ async function run_chat_pipeline(user_prompt, llm, ds, genedb, agentFiles, aiFil
     );
     mayLog("Time taken to infer term objects:", formatElapsedTime(Date.now() - time));
     mayLog("Inferred termObj from entity:", JSON.stringify(termObj));
+    mayLog("#################################################");
     mayLog("####### Fourth phase: From Term Objects to TwTvs Objects #######");
+    mayLog("#################################################");
     time = (/* @__PURE__ */ new Date()).valueOf();
     const twTvsObj = await resolveToTwTvs(termObj, plotType, llm, dataset_db);
     mayLog("Time taken to resolve to TwTvs object from termObj:", formatElapsedTime(Date.now() - time));
@@ -178,7 +194,9 @@ async function run_chat_pipeline(user_prompt, llm, ds, genedb, agentFiles, aiFil
       return twTvsObj;
     }
     mayLog("twTvsObj:", twTvsObj);
+    mayLog("#################################################");
     mayLog("####### Fifth/Final phase: From TwTvs Objects to Plot States #######");
+    mayLog("#################################################");
     time = (/* @__PURE__ */ new Date()).valueOf();
     ai_output_json = resolveToPlotState(twTvsObj, plotType, subplotType);
     mayLog("Time taken to resolve to plot state:", formatElapsedTime(Date.now() - time));

package/routes/termdb.cluster.js

@@ -83,7 +83,10 @@ async function getResult(q, ds) {
     ({ term2sample2value, byTermId, bySampleId } = await getNumericDictTermAnnotation(q, ds));
   } else if (q.dataType == PROTEOME_ABUNDANCE) {
     ;
-    ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries.proteome.get(_q));
+    ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries.proteome.get({
+      ..._q,
+      dataTypeDetails: _q.proteomeDetails
+    }));
   } else {
     ;
     ({ term2sample2value, byTermId, bySampleId, skippedSexChrGenes } = await ds.queries[q.dataType].get(_q, ds));

package/routes/termdb.config.js

@@ -225,6 +225,9 @@ function addNonDictionaryQueries(c, ds, genome) {
                     JSON.stringify(src.caseFilter)
                   );
                 }
+                if (src.DAPfile) {
+                  q2.proteome.organisms[organism].assays[assay].cohorts[cohort].DAPfile = true;
+                }
               }
             }
           }

package/routes/termdb.dapVolcano.js

@@ -0,0 +1,80 @@
+import path from "path";
+import fs from "fs/promises";
+import { dapVolcanoPayload } from "#types/checkers";
+import { get_ds_tdb } from "#src/termdb.js";
+import { renderVolcano } from "../src/renderVolcano.ts";
+import serverconfig from "../src/serverconfig.js";
+import { countDistinctSamples } from "./termdb.proteome.ts";
+const api = {
+  endpoint: "termdb/dapVolcano",
+  methods: {
+    get: {
+      ...dapVolcanoPayload,
+      init
+    },
+    post: {
+      ...dapVolcanoPayload,
+      init
+    }
+  }
+};
+function init({ genomes }) {
+  return async (req, res) => {
+    try {
+      const q = req.query;
+      const genome = genomes[q.genome];
+      if (!genome) throw "invalid genome";
+      const [ds] = get_ds_tdb(genome, q);
+      const proteomeConfig = ds.queries?.proteome;
+      if (!proteomeConfig) throw "proteome not configured for this dataset";
+      const organismConfig = proteomeConfig.organisms?.[q.organism];
+      if (!organismConfig) throw "invalid organism";
+      const assayConfig = organismConfig.assays?.[q.assay];
+      if (!assayConfig) throw "invalid assay";
+      const cohortConfig = assayConfig.cohorts?.[q.cohort];
+      if (!cohortConfig) throw "invalid cohort";
+      if (!cohortConfig.DAPfile) throw "DAP file not configured for this cohort";
+      const organismFilter = [{ columnIdx: organismConfig.columnIdx, columnValue: organismConfig.columnValue }];
+      const assayFilter = [{ columnIdx: assayConfig.columnIdx, columnValue: assayConfig.columnValue }];
+      const db = proteomeConfig.db;
+      const controlCount = countDistinctSamples(db, [...organismFilter, ...assayFilter, ...cohortConfig.controlFilter]);
+      const caseCount = countDistinctSamples(db, [...organismFilter, ...assayFilter, ...cohortConfig.caseFilter]);
+      if (q.countsOnly) {
+        res.send({ sample_size1: controlCount, sample_size2: caseCount });
+        return;
+      }
+      const filePath = path.join(serverconfig.tpmasterdir, cohortConfig.DAPfile);
+      const content = await fs.readFile(filePath, "utf8");
+      const lines = content.trim().split("\n");
+      const rustRows = [];
+      for (let i = 1; i < lines.length; i++) {
+        const parts = lines[i].split("	");
+        if (parts.length < 4) continue;
+        const fc = Number(parts[2]);
+        if (!Number.isFinite(fc)) continue;
+        const pValue = Number(parts[3]);
+        if (!Number.isFinite(pValue)) continue;
+        rustRows.push({
+          gene_name: parts[0],
+          gene: parts[1],
+          fold_change: fc,
+          original_p_value: pValue,
+          adjusted_p_value: pValue
+        });
+      }
+      const rendered = await renderVolcano(rustRows, q.volcanoRender);
+      for (const d of rendered.dots) delete d.adjusted_p_value;
+      res.send({
+        data: rendered,
+        sample_size1: controlCount,
+        sample_size2: caseCount
+      });
+    } catch (e) {
+      res.send({ status: "error", error: e.message || e });
+      if (e instanceof Error && e.stack) console.log(e);
+    }
+  };
+}
+export {
+  api
+};

package/routes/termdb.proteome.js

@@ -42,12 +42,12 @@ function init({ genomes }) {
               term: {
                 name: term.name,
                 type: "proteomeAbundance",
-                proteomeDetails: details
+                dataTypeDetails: details
               }
             };
             const cohortData = await ds.queries.proteome.get({
               terms: [tw],
-              proteomeDetails: details,
+              dataTypeDetails: details,
               filter: q.filter,
               filter0: q.filter0,
               for: "proteinView",
@@ -249,7 +249,7 @@ async function validate_query_proteome(ds) {
     const proteins = arg?.proteins;
     if (!Array.isArray(proteins) || proteins.length == 0) throw "queries.proteome.find arg.proteins[] missing";
     const matches = /* @__PURE__ */ new Set();
-    const details = arg?.proteomeDetails || {};
+    const details = arg?.dataTypeDetails || {};
     const organism = details.organism;
     const assay = details.assay;
     const cohort = details.cohort;
@@ -257,7 +257,7 @@ async function validate_query_proteome(ds) {
     const filters = [];
     if (Object.keys(details).length) {
       if (!organism || !assay || !cohort)
-        throw "queries.proteome.find arg.proteomeDetails.{organism,assay,cohort} missing";
+        throw "queries.proteome.find arg.dataTypeDetails.{organism,assay,cohort} missing";
       const organismConfig = q.organisms?.[organism];
       if (!organismConfig) throw `queries.proteome.find invalid organism: ${organism}`;
       const assayConfig = organismConfig.assays?.[assay];
@@ -300,8 +300,8 @@ async function validate_query_proteome(ds) {
   };
   q.get = async (param) => {
     if (!param?.terms?.length) throw "queries.proteome.get param.terms[] missing";
-    if (!param.proteomeDetails?.assay || !param.proteomeDetails?.cohort || !param.proteomeDetails?.organism)
-      throw "queries.proteome.get param.proteomeDetails.{assay,cohort,organism} missing";
+    if (!param.dataTypeDetails?.assay || !param.dataTypeDetails?.cohort || !param.dataTypeDetails?.organism)
+      throw "queries.proteome.get param.dataTypeDetails.{assay,cohort,organism} missing";
     return await getProteomeValuesFromCohort(ds, param, q);
   };
 }
@@ -329,6 +329,12 @@ function buildFilterClause(filters) {
   }
   return { conditions, params };
 }
+function countDistinctSamples(db, filters) {
+  if (!filters?.length) throw "countDistinctSamples: filters must not be empty";
+  const { conditions, params } = buildFilterClause(filters);
+  const row = db.prepare(`SELECT COUNT(DISTINCT sample) as cnt FROM proteome_abundance WHERE ${conditions.join(" AND ")}`).get(...params);
+  return row?.cnt || 0;
+}
 function queryDbRows(db, matchColumn, matchValue, filters) {
   const { conditions, params } = buildFilterClause(filters);
   const allConditions = [`${matchColumn} = ? COLLATE NOCASE`, ...conditions];
@@ -339,7 +345,7 @@ function queryDbRows(db, matchColumn, matchValue, filters) {
 }
 async function getProteomeValuesFromCohort(ds, param, q) {
   const db = ds.queries.proteome.db;
-  const { assay, cohort, organism } = param.proteomeDetails;
+  const { assay, cohort, organism } = param.dataTypeDetails;
   const organismConfig = q.organisms?.[organism];
   if (!organismConfig) throw `queries.proteome invalid organism: ${organism}`;
   const organismColumnIdx = organismConfig.columnIdx;
@@ -453,5 +459,6 @@ async function getProteomeValuesFromCohort(ds, param, q) {
 }
 export {
   api,
+  countDistinctSamples,
   validate_query_proteome
 };

package/routes/termdb.sampleScatter.js

@@ -53,6 +53,10 @@ function init({ genomes }) {
         cohortSamples = tmp[0];
       } else {
         if (!q.plotName) throw new Error("Neither plot name or coordinates where provided");
+        if (typeof ds.cohort?.scatterplots?.get == "function") {
+          const allowed = ds.cohort.scatterplots.get(q.__protected__?.clientAuthResult);
+          if (!allowed?.find((i) => i.name == q.plotName)) throw new Error("No permission to display plot");
+        }
         if (!Array.isArray(ds.cohort?.scatterplots?.plots)) throw new Error("not supported");
         const plot = ds.cohort.scatterplots.plots.find((p) => p.name == q.plotName);
         if (!plot) throw new Error(`plot not found with plotName ${q.plotName}`);

package/routes/termdb.singlecellSamples.js

@@ -89,6 +89,17 @@ async function validateSamples(q, ds) {
   const S = q.samples, D = q.data;
   const samples = /* @__PURE__ */ new Map();
   for (const plot of D.plots) {
+    if (plot.isMetaResult) {
+      const sampleName = plot?.sampleId || plot.name.replace(/\s/g, "_");
+      const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, sampleName + (plot.fileSuffix || ""));
+      try {
+        await file_is_readable(tsvfile);
+        samples.set(sampleName, { sample: sampleName, isMetaResult: true });
+      } catch (e) {
+        throw new Error(`meta result data file missing or unreadable: ${sampleName} (${tsvfile}): ${e.message || e}`);
+      }
+      continue;
+    }
     for (const fn of await fs.promises.readdir(path.join(serverconfig.tpmasterdir, plot.folder))) {
       let sampleName = fn;
       if (plot.fileSuffix) {

package/routes/termdb.topVariablyExpressedGenes.js

@@ -1,11 +1,7 @@
 import { termdbTopVariablyExpressedGenesPayload } from "#types/checkers";
-import { run_rust } from "@sjcrh/proteinpaint-rust";
-import serverconfig from "#src/serverconfig.js";
 import { mayLimitSamples } from "#src/mds3.filter.js";
-import { makeFilter } from "#src/mds3.gdc.js";
-import { cachedFetch } from "#src/utils.js";
-import { joinUrl } from "#shared/joinUrl.js";
-import { formatElapsedTime } from "#shared/time.js";
+import { run_python } from "@sjcrh/proteinpaint-python";
+import { mayLog } from "#src/helpers.ts";
 const api = {
   endpoint: "termdb/topVariablyExpressedGenes",
   methods: {
@@ -29,27 +25,23 @@ function init({ genomes }) {
       const ds = genome.datasets?.[q.dslabel];
       if (!ds) throw "invalid dslabel";
       if (!ds.queries?.topVariablyExpressedGenes) throw "not supported on dataset";
+      q.ds = ds;
       const t = Date.now();
       result = {
         genes: await ds.queries.topVariablyExpressedGenes.getGenes(q)
       };
-      console.log("compute top variably expressed genes:", formatElapsedTime(Date.now() - t));
+      mayLog("time for top variably expressed genes", Date.now() - t);
     } catch (e) {
       result = { status: e.status || 400, error: e.message || e };
     }
     res.send(result);
   };
 }
-function validate_query_TopVariablyExpressedGenes(ds, genome) {
+function validate_query_TopVariablyExpressedGenes(ds) {
   const q = ds.queries.topVariablyExpressedGenes;
   if (!q) return;
-  if (q.src == "gdcapi") {
-    gdcValidateQuery(ds, genome);
-  } else if (q.src == "native") {
-    nativeValidateQuery(ds);
-  } else {
-    throw "unknown topVariablyExpressedGenes.src";
-  }
+  if (typeof q.getGenes == "function") return;
+  nativeValidateQuery(ds);
 }
 function nativeValidateQuery(ds) {
   const gE = ds.queries.geneExpression;
@@ -83,21 +75,7 @@ function addTopVEarg(q) {
       id: "filter_extreme_values",
       label: "Filter Extreme Values",
       type: "boolean",
-      value: true,
-      options: [
-        {
-          id: "min_count",
-          label: "Min count",
-          type: "number",
-          value: 10
-        },
-        {
-          id: "min_total_count",
-          label: "Min total count",
-          type: "number",
-          value: 15
-        }
-      ]
+      value: true
     },
     {
       id: "rank_type",
@@ -134,124 +112,20 @@ function addTopVEarg(q) {
   q.arguments = arglst;
 }
 async function computeGenes4nativeDs(q, gE, samples) {
+  if (!["number", "boolean"].includes(typeof q.filter_extreme_values) || q.filter_extreme_values === void 0) {
+    q.filter_extreme_values = false;
+  }
   const input_json = {
     input_file: gE.file,
     samples: samples.join(","),
-    filter_extreme_values: q.filter_extreme_values,
-    num_genes: q.maxGenes,
-    rank_type: q.rank_type?.type
+    filter_extreme_values: typeof q.filter_extreme_values === "number" ? Boolean(q.filter_extreme_values) : q.filter_extreme_values,
+    max_genes: q.maxGenes,
+    rank_type: q.rank_type?.type ?? "var"
   };
-  if (q.filter_extreme_values == 1) {
-    input_json["min_count"] = q.min_count;
-    input_json["min_total_count"] = q.min_total_count;
-  }
-  if (gE.newformat) {
-    input_json["newformat"] = true;
-  }
-  const rust_output = await run_rust("topGeneByExpressionVariance", JSON.stringify(input_json));
-  const rust_output_list = rust_output.split("\n");
-  let output_json;
-  for (const item of rust_output_list) {
-    if (item.includes("output_json:")) {
-      output_json = JSON.parse(item.replace("output_json:", ""));
-    } else {
-      console.log(item);
-    }
-  }
-  const varGenes = output_json.map((i) => i.gene_symbol);
+  const python_output = await run_python("topVEgene.py", JSON.stringify(input_json));
+  const varGenes = typeof python_output === "string" ? JSON.parse(python_output) : [];
   return varGenes;
 }
-function gdcValidateQuery(ds, genome) {
-  ds.queries.topVariablyExpressedGenes.getGenes = async (q) => {
-    if (serverconfig.features.gdcGenes) {
-      console.error(
-        "!!GDC!! using serverconfig.features.gdcGenes[] but not live api query. only use this on DEV and never on PROD!"
-      );
-      return serverconfig.features.gdcGenes;
-    }
-    if (ds.label === "GDC" && !ds.__gdc?.doneCaching) {
-      throw "The server has not finished caching the case IDs: try again in about 2 minutes.";
-    }
-    const { host, headers } = ds.getHostHeaders(q);
-    try {
-      const response = await cachedFetch(
-        joinUrl(host.rest, "/gene_expression/gene_selection"),
-        {
-          method: "POST",
-          headers,
-          body: getGeneSelectionArg(q)
-        },
-        {
-          // noCache: true, // !!! for testing only !!!
-          getErrMessage: (response2) => {
-            const body = response2?.body || response2;
-            return Array.isArray(body?.gene_selection) ? "" : body?.message || body?.error || JSON.stringify(body);
-          }
-        }
-      );
-      const re = response.body;
-      const genes = [];
-      if (!Array.isArray(re.gene_selection)) {
-        throw "re.gene_selection[] is not array: " + JSON.stringify(re);
-      }
-      for (const i of re.gene_selection) {
-        if (i.gene_id && typeof i.gene_id == "string") {
-          const t = genome.genedb.getNameByAlias.get(i.gene_id);
-          if (t) genes.push(t.name);
-        } else if (i.symbol && typeof i.symbol == "string") {
-          genes.push(i.symbol);
-        } else {
-          throw "one of re.gene_selection[] is missing both gene_id and symbol";
-        }
-      }
-      return genes;
-    } catch (e) {
-      console.error(e.stack || e);
-      throw e;
-    }
-  };
-  function getGeneSelectionArg(q) {
-    const arg = {
-      // add any to avoid tsc err
-      case_filters: makeFilter(q),
-      selection_size: q.maxGenes,
-      min_median_log2_uqfpkm: q.min_median_log2_uqfpkm
-    };
-    if (q.geneSet) {
-      if (q.geneSet.type == "all") {
-        arg.gene_type = "protein_coding";
-      } else if (q.geneSet.type == "custom" || q.geneSet.type == "msigdb") {
-        if (!Array.isArray(q.geneSet.geneList)) throw "q.geneSet.geneList is not array";
-        arg.gene_ids = map2ensg(q.geneSet.geneList, genome);
-        if (arg.gene_ids.length == 0) throw "no valid genes from custom gene set";
-      } else {
-        throw "unknown q.geneSet.type";
-      }
-    } else {
-      arg.gene_type = "protein_coding";
-    }
-    return arg;
-  }
-}
-function map2ensg(lst, genome) {
-  const ensg = [];
-  for (const name of lst) {
-    if (name.startsWith("ENSG") && name.length == 15) {
-      ensg.push(name);
-      continue;
-    }
-    const tmp = genome.genedb.getAliasByName.all(name);
-    if (Array.isArray(tmp)) {
-      for (const a of tmp) {
-        if (a.alias.startsWith("ENSG")) {
-          ensg.push(a.alias);
-          break;
-        }
-      }
-    }
-  }
-  return ensg;
-}
 export {
   api,
   validate_query_TopVariablyExpressedGenes