@sjcrh/proteinpaint-server 2.38.0 → 2.39.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.38.0",
+  "version": "2.39.0",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "server.js",
   "bin": "start.js",
@@ -57,7 +57,7 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.35.0",
-    "@sjcrh/proteinpaint-rust": "2.38.0",
+    "@sjcrh/proteinpaint-rust": "2.39.0",
     "better-sqlite3": "^7.5.3",
     "body-parser": "^1.15.2",
     "canvas": "~2.9.3",

package/routes/gdc.mafBuild.ts

@@ -43,10 +43,15 @@ async function buildMaf(q: GdcMafBuildRequest, res: any) {
 	const t0 = Date.now()
 
 	const fileLst2 = (await getFileLstUnderSizeLimit(q.fileIdLst)) as string[]
-	console.log('test gdc maf sizes', Date.now() - t0)
+	if (serverconfig.debugmode)
+		console.log(
+			`${fileLst2.length} out of ${q.fileIdLst.length} input MAF files accepted by size limit`,
+			Date.now() - t0
+		)
 
 	const arg = {
 		fileIdLst: fileLst2,
+		columns: q.columns,
 		host: path.join(apihost, 'data') // must use the /data/ endpoint from current host
 	}
 
@@ -55,7 +60,7 @@ async function buildMaf(q: GdcMafBuildRequest, res: any) {
 	res.setHeader('Content-Disposition', 'attachment; filename=cohort.maf.gz')
 	rustStream.pipe(res)
 
-	console.log('rust gdcmaf', Date.now() - t0)
+	if (serverconfig.debugmode) console.log('rust gdcmaf', Date.now() - t0)
 
 	rustStream.on('end', () => {
 		res.end()

package/routes/termdb.categories.ts

@@ -1,4 +1,4 @@
-// import { getcategoriesRequest, getcategoriesResponse } from '#shared/types/routes/termdb.categories'
+import { getcategoriesRequest, getcategoriesResponse } from '#shared/types/routes/termdb.categories.ts'
 import { getOrderedLabels } from '#src/termdb.barchart.js'
 import { getData } from '#src/termdb.matrix.js'
 
@@ -68,7 +68,7 @@ export const api: any = {
 
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
-		const q = req.query // as getcategoriesRequest
+		const q = req.query as getcategoriesRequest
 		try {
 			const g = genomes[req.query.genome]
 			if (!g) throw 'invalid genome name'
@@ -88,8 +88,8 @@ function init({ genomes }) {
 }
 
 async function trigger_getcategories(
-	q: { tid: string | number; type: string; filter: any; term1_q: any; currentGeneNames: any },
-	res: { send: (arg0: { lst: any[]; orderedLabels: any }) => void },
+	q: { tid: string | number; type: string; filter: any; term1_q: any; currentGeneNames?: string[]; rglst?: any },
+	res: { send: (arg0: { lst: any[]; orderedLabels?: any }) => void },
 	tdb: { q: { termjsonByOneid: (arg0: any) => any } },
 	ds: { assayAvailability: { byDt: { [s: string]: any } | ArrayLike<any> } },
 	genome: any
@@ -99,13 +99,15 @@ async function trigger_getcategories(
 	if (!q.tid) throw '.tid missing'
 	const term =
 		q.type == 'geneVariant' ? { name: q.tid, type: 'geneVariant', isleaf: true } : tdb.q.termjsonByOneid(q.tid)
+
 	const arg = {
 		filter: q.filter,
 		terms:
 			q.type == 'geneVariant'
 				? [{ term: term, q: { isAtomic: true } }]
 				: [{ id: q.tid, term, q: q.term1_q || getDefaultQ(term, q) }],
-		currentGeneNames: q.currentGeneNames
+		currentGeneNames: q.currentGeneNames, // optional, from mds3 mayAddGetCategoryArgs()
+		rglst: q.rglst // optional, from mds3 mayAddGetCategoryArgs()
 	}
 
 	const data = await getData(arg, ds, genome)
@@ -200,7 +202,7 @@ async function trigger_getcategories(
 	res.send({
 		lst,
 		orderedLabels
-	})
+	} as getcategoriesResponse)
 }
 
 function getDefaultQ(

package/routes/termdb.cluster.ts

@@ -0,0 +1,181 @@
+import { TermdbClusterRequest, TermdbClusterResponse } from '#shared/types/routes/termdb.cluster.ts'
+import fs from 'fs'
+import path from 'path'
+import * as utils from '#src/utils.js'
+import serverconfig from '#src/serverconfig.js'
+import { GeneExpressionQuery, GeneExpressionQueryNative } from '#shared/types/dataset.ts'
+import { gdc_validate_query_geneExpression } from '#src/mds3.gdc.js'
+import { mayLimitSamples } from '#src/mds3.filter.js'
+import { doClustering } from '#src/doClustering.js' // unable to convert this to ts yet, when converted, move all code here
+import { dtgeneexpression } from '#shared/common.js'
+
+export const api = {
+	endpoint: 'termdb/cluster',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'TermdbClusterRequest'
+			},
+			response: {
+				typeId: 'TermdbClusterResponse'
+			}
+		},
+		post: {
+			alternativeFor: 'get',
+			init
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		const q = req.query as TermdbClusterRequest
+		let result
+		try {
+			const g = genomes[q.genome]
+			if (!g) throw 'invalid genome name'
+			const ds = g.datasets[q.dslabel]
+			if (!ds) throw 'invalid dataset name'
+			if (ds.__gdc && !ds.__gdc.doneCaching)
+				throw 'The server has not finished caching the case IDs: try again in ~2 minutes'
+			if (q.dataType == dtgeneexpression) {
+				if (!ds.queries?.geneExpression) throw 'no geneExpression data on this dataset'
+				result = (await getResult(q, ds)) as TermdbClusterResponse
+			} else {
+				throw 'unknown q.dataType ' + q.dataType
+			}
+		} catch (e: any) {
+			if (e.stack) console.log(e.stack)
+			result = {
+				status: e.status || 400,
+				error: e.message || e
+			} as TermdbClusterResponse
+		}
+		res.send(result)
+	}
+}
+
+async function getResult(q: TermdbClusterRequest, ds: any) {
+	const { gene2sample2value, byTermId, bySampleId } = await ds.queries.geneExpression.get(q)
+	if (gene2sample2value.size == 0) throw 'no data'
+	if (gene2sample2value.size == 1) {
+		// get data for only 1 gene; still return data, may create violin plot later
+		const g = Array.from(gene2sample2value.keys())[0]
+		return { gene: g, data: gene2sample2value.get(g) }
+	}
+
+	// have data for multiple genes, run clustering
+	const t = Date.now() // use "t=new Date()" will lead to tsc error
+	const clustering = await doClustering(gene2sample2value, q, ds)
+	if (serverconfig.debugmode) console.log('clustering done:', Date.now() - t, 'ms')
+	return { clustering, byTermId, bySampleId }
+}
+
+export async function validate_query_geneExpression(ds: any, genome: any) {
+	const q = ds.queries.geneExpression as GeneExpressionQuery
+	if (!q) return
+
+	if (q.src == 'gdcapi') {
+		gdc_validate_query_geneExpression(ds, genome)
+		// q.get() added
+		return
+	}
+	if (q.src == 'native') {
+		validateNative(q, ds, genome)
+		return
+	}
+	throw 'unknown queries.geneExpression.src'
+}
+
+async function validateNative(q: GeneExpressionQueryNative, ds: any, genome: any) {
+	q.file = path.join(serverconfig.tpmasterdir, q.file)
+	await utils.validate_tabixfile(q.file)
+	q.nochr = await utils.tabix_is_nochr(q.file, null, genome)
+	q.samples = [] as number[]
+
+	{
+		// is a gene-by-sample matrix file
+		const lines = await utils.get_header_tabix(q.file)
+		if (!lines[0]) throw 'header line missing from ' + q.file
+		const l = lines[0].split('\t')
+		if (l.slice(0, 4).join('\t') != '#chr\tstart\tstop\tgene') throw 'header line has wrong content for columns 1-4'
+		for (let i = 4; i < l.length; i++) {
+			const id = ds.cohort.termdb.q.sampleName2id(l[i])
+			if (id == undefined) throw 'unknown sample from header'
+			q.samples.push(id)
+		}
+		console.log(q.samples.length, 'samples from geneExpression of', ds.label)
+	}
+
+	/*
+	query exp data one gene at a time
+	param{}
+	.genes[{}]
+		.gene=str
+		.chr=str
+		.start=int
+		.stop=int
+	.filterObj{}
+	*/
+	q.get = async (param: TermdbClusterRequest) => {
+		const limitSamples = await mayLimitSamples(param, q.samples, ds)
+		if (limitSamples?.size == 0) {
+			// got 0 sample after filtering, return blank array for no data
+			return new Set()
+		}
+
+		// has at least 1 sample passing filter and with exp data
+		// TODO what if there's just 1 sample not enough for clustering?
+		const bySampleId = {}
+		if (limitSamples) {
+			for (const sid of limitSamples) {
+				bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) }
+			}
+		} else {
+			// use all samples with exp data
+			for (const sid of q.samples) {
+				bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) }
+			}
+		}
+
+		const gene2sample2value = new Map() // k: gene symbol, v: { sampleId : value }
+
+		for (const g of param.genes) {
+			// FIXME newly added geneVariant terms from client to be changed to {gene} but not {name}
+			if (!g.gene) continue
+
+			if (!g.chr) {
+				// quick fix: newly added gene from client will lack chr/start/stop
+				const lst = genome.genedb.getjsonbyname.all(g.gene)
+				if (lst.length == 0) continue
+				const j = JSON.parse(lst.find(i => i.isdefault).genemodel || lst[0].genemodel)
+				g.start = j.start
+				g.stop = j.stop
+				g.chr = j.chr
+			}
+
+			gene2sample2value.set(g.gene, {})
+			await utils.get_lines_bigfile({
+				args: [q.file, (q.nochr ? g.chr?.replace('chr', '') : g.chr) + ':' + g.start + '-' + g.stop], // must do g.chr?.replace to avoid tsc error
+				callback: line => {
+					const l = line.split('\t')
+					// case-insensitive match! FIXME if g.gene is alias won't work
+					if (l[3].toLowerCase() != g.gene.toLowerCase()) return
+					for (let i = 4; i < l.length; i++) {
+						const sampleId = q.samples[i - 4]
+						if (limitSamples && !limitSamples.has(sampleId)) continue // doing filtering and sample of current column is not used
+						// if l[i] is blank string?
+						const v = Number(l[i])
+						if (Number.isNaN(v)) throw 'exp value not number'
+						gene2sample2value.get(g.gene)[sampleId] = v
+					}
+				}
+			} as any)
+			// Above!! add "as any" to suppress a npx tsc alert
+		}
+		// pass blank byTermId to match with expected output structure
+		const byTermId = {}
+		return { gene2sample2value, byTermId, bySampleId }
+	}
+}

package/routes/termdb.getdescrstats.ts

@@ -1,4 +1,4 @@
-// import { getdescrstatsRequest, getdescrstatsResponse } from '#shared/types/routes/termdb.descrstats'
+import { getdescrstatsRequest, getdescrstatsResponse } from '#shared/types/routes/termdb.getdescrstats.ts'
 import roundValue from '#shared/roundValue.js'
 import computePercentile from '#shared/compute.percentile.js'
 import * as termdbsql from '#src/termdb.sql.js'
@@ -62,7 +62,7 @@ export const api: any = {
 
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
-		const q = req.query // as getdescrstatsRequest
+		const q = req.query as getdescrstatsRequest
 		try {
 			const g = genomes[req.query.genome]
 			if (!g) throw 'invalid genome name'
@@ -141,5 +141,5 @@ async function trigger_getdescrstats(q: any, res: any, ds: any) {
 			{ id: 'max', label: 'Maximum', value: roundValue(max, 2) },
 			{ id: 'sd', label: 'Standard deviation', value: roundValue(sd, 2) }
 		]
-	})
+	} as getdescrstatsResponse)
 }

package/routes/termdb.getnumericcategories.ts

@@ -0,0 +1,99 @@
+import {
+	getnumericcategoriesRequest,
+	getnumericcategoriesResponse
+} from '#shared/types/routes/termdb.getnumericcategories.ts'
+import * as termdbsql from '#src/termdb.sql.js'
+
+export const api: any = {
+	endpoint: 'termdb/numericcategories',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'getnumericcategoriesRequest'
+			},
+			response: {
+				typeId: 'getnumericcategoriesResponse'
+			},
+			examples: [
+				{
+					request: {
+						body: {
+							genome: 'hg38-test',
+							dslabel: 'TermdbTest',
+							embedder: 'localhost',
+							tid: 'aaclassic_5',
+							filter: {
+								type: 'tvslst',
+								in: true,
+								join: '',
+								lst: [
+									{
+										tag: 'cohortFilter',
+										type: 'tvs',
+										tvs: {
+											term: {
+												name: 'Cohort',
+												type: 'categorical',
+												values: { ABC: { label: 'ABC' }, XYZ: { label: 'XYZ' } },
+												id: 'subcohort',
+												isleaf: false,
+												groupsetting: { disabled: true }
+											},
+											values: [{ key: 'ABC', label: 'ABC' }]
+										}
+									}
+								]
+							}
+						}
+					},
+					response: {
+						header: { status: 200 }
+					}
+				}
+			]
+		},
+		post: {
+			alternativeFor: 'get',
+			init
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		const q = req.query as getnumericcategoriesRequest
+		try {
+			const g = genomes[req.query.genome]
+			if (!g) throw 'invalid genome name'
+			const ds = g.datasets[req.query.dslabel]
+			if (!ds) throw 'invalid dataset name'
+			const tdb = ds.cohort.termdb
+			if (!tdb) throw 'invalid termdb object'
+
+			await trigger_getnumericcategories(q, res, tdb, ds) // as getnumericcategoriesResponse
+		} catch (e) {
+			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+			// @ts-ignore
+			res.send({ error: e?.message || e })
+			if (e instanceof Error && e.stack) console.log(e)
+		}
+	}
+}
+
+async function trigger_getnumericcategories(
+	q: { tid: any; filter?: any },
+	res: { send: (arg0: { lst: any }) => void },
+	tdb: { q: { termjsonByOneid: (arg0: any) => any } },
+	ds: any
+) {
+	if (!q.tid) throw '.tid missing'
+	const term = tdb.q.termjsonByOneid(q.tid)
+	const arg = {
+		ds,
+		term_id: q.tid,
+		filter: q.filter
+	}
+	const lst = await termdbsql.get_summary_numericcategories(arg)
+	res.send({ lst } as getnumericcategoriesResponse)
+}

package/routes/termdb.getpercentile.ts

@@ -1,6 +1,7 @@
-// import { getpercentileRequest, getpercentileResponse } from '#shared/types/routes/termdb.getpercentile'
-import * as termdbsql from '#src/termdb.sql.js'
-import computePercentile from '#shared/compute.percentile.js'
+import { getpercentileRequest, getpercentileResponse } from '#shared/types/routes/termdb.getpercentile.ts'
+import * as termdbsql from '../src/termdb.sql.js'
+import computePercentile from '../shared/compute.percentile.js'
+import { Filter } from '../shared/types/filter'
 
 export const api: any = {
 	endpoint: 'termdb/getpercentile',
@@ -61,7 +62,7 @@ export const api: any = {
 
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
-		const q = req.query // as getpercentileRequest
+		const q = req.query as getpercentileRequest
 		try {
 			const g = genomes[req.query.genome]
 			if (!g) throw 'invalid genome name'
@@ -78,7 +79,7 @@ function init({ genomes }) {
 }
 
 async function trigger_getpercentile(
-	q: { tid: string; getpercentile: number[]; filter: string },
+	q: { tid: string; getpercentile: number[]; filter: Filter },
 	res: { send: (arg0: { values: number[] }) => void },
 	ds: { cohort: { termdb: { q: { termjsonByOneid: (arg0: any) => any } } } }
 ) {
@@ -113,5 +114,5 @@ async function trigger_getpercentile(
 		const perc_value = computePercentile(values, percentile)
 		perc_values.push(perc_value)
 	}
-	res.send({ values: perc_values })
+	res.send({ values: perc_values } as getpercentileResponse)
 }

package/routes/termdb.topVariablyExpressedGenes.ts

@@ -4,8 +4,10 @@ import {
 } from '#shared/types/routes/termdb.topVariablyExpressedGenes.ts'
 import { gdcGetCasesWithExressionDataFromCohort, apihost, geneExpHost } from '../src/mds3.gdc.js'
 import path from 'path'
+import { run_rust } from '@sjcrh/proteinpaint-rust'
 import got from 'got'
 import serverconfig from '#src/serverconfig.js'
+import { get_samples } from '#src/termdb.sql.js'
 
 export const api = {
 	endpoint: 'termdb/topVariablyExpressedGenes',
@@ -32,9 +34,9 @@ function init({ genomes }) {
 			if (!ds) throw 'invalid dslabel'
 			if (!ds.queries?.topVariablyExpressedGenes) throw 'not supported on dataset'
 
-			const t: number = new Date().getTime()
+			const t = Date.now()
 			const genes = await ds.queries.topVariablyExpressedGenes.getGenes(q)
-			if (serverconfig.debugmode) console.log('topVariablyExpressedGenes', new Date().getTime() - t, 'ms')
+			if (serverconfig.debugmode) console.log('topVariablyExpressedGenes', Date.now() - t, 'ms')
 
 			res.send({ genes } as TermdbTopVariablyExpressedGenesResponse)
 		} catch (e: any) {
@@ -57,21 +59,68 @@ export function validate_query_TopVariablyExpressedGenes(ds: any, genome: any) {
 }
 
 function nativeValidateQuery(ds: any, genome: any) {
-	ds.queries.topVariablyExpressedGenes.getGenes = async (
-		q: TermdbTopVariablyExpressedGenesRequest,
-		ds: any,
-		genome: any
-	) => {
-		// get list of samples that are used in current analysis
+	const gE = ds.queries.geneExpression // a separate query required to supply data for computing top genes
+	if (!gE) throw 'topVariablyExpressedGenes query given but geneExpression missing'
+	if (gE.src != 'native') throw 'topVariablyExpressedGenes is native but geneExpression.src is not native'
+
+	ds.queries.topVariablyExpressedGenes.getGenes = async (q: TermdbTopVariablyExpressedGenesRequest) => {
+		// get list of samples that are used in current analysis; gE.samples[] contains all sample integer ids with exp data
 		const samples = [] as string[]
+		if (q.filter) {
+			// get all samples pasing pp filter, may contain those without exp data
+			const sidlst = await get_samples(q.filter, ds)
+			// [{id:int}]
+			// filter for those with exp data from q.samples[]
+			for (const i of sidlst) {
+				if (gE.samples.includes(i.id)) {
+					// this sample passing filter also has exp data; convert to string name
+					const n: string = ds.cohort.termdb.q.id2sampleName(i.id)
+					if (!n) throw 'sample id cannot convert to string name'
+					samples.push(n)
+				}
+			}
+		} else {
+			// no filter, use all samples with exp data
+			for (const i of gE.samples) {
+				const n: string = ds.cohort.termdb.q.id2sampleName(i.id)
+				if (!n) throw 'sample id cannot convert to string name'
+				samples.push(n)
+			}
+		}
+
 		// call rust to compute top genes on these samples
-		const genes = await computeGenes4nativeDs(q, ds, samples)
+		const genes = await computeGenes4nativeDs(q, ds, gE.file, samples)
 		return genes
 	}
 }
 
-async function computeGenes4nativeDs(q: TermdbTopVariablyExpressedGenesRequest, ds: any, samples: string[]) {
-	return []
+async function computeGenes4nativeDs(
+	q: TermdbTopVariablyExpressedGenesRequest,
+	ds: any,
+	matrixFile: string,
+	samples: string[]
+) {
+	// The param option in input JSON is very important. It instructs what method will be used to calculate variation in the counts for a particular gene. It supports variance as well as interquartile region. This is based on the recommendation of this article https://www.frontiersin.org/articles/10.3389/fgene.2021.632620/full . This article recommends using interquartile region over variance.
+	const input_json = {
+		input_file: matrixFile,
+		samples: samples.join(','),
+		filter_extreme_values: true,
+		num_genes: Number(q.maxGenes),
+		param: 'var'
+	}
+	const rust_output = await run_rust('topGeneByExpressionVariance', JSON.stringify(input_json))
+	const rust_output_list = rust_output.split('\n')
+
+	let output_json
+	for (const item of rust_output_list) {
+		if (item.includes('output_json:')) {
+			output_json = JSON.parse(item.replace('output_json:', ''))
+		} else {
+			console.log(item)
+		}
+	}
+	const varGenes = output_json.map(i => i.gene_symbol)
+	return varGenes
 }
 
 function gdcValidateQuery(ds: any, genome: any) {

package/routes/termdb.violin.ts

@@ -1,4 +1,4 @@
-// import { getViolinRequest, getViolinResponse } from '#shared/types/routes/termdb.violin'
+import { getViolinRequest, getViolinResponse } from '#shared/types/routes/termdb.violin.ts'
 import { trigger_getViolinPlotData } from '#src/termdb.violin.js'
 
 export const api: any = {
@@ -61,13 +61,13 @@ export const api: any = {
 
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
-		const q = req.query // as getViolinRequest
+		const q = req.query as getViolinRequest
 		try {
 			const g = genomes[req.query.genome]
 			const ds = g.datasets[req.query.dslabel]
 			if (!g) throw 'invalid genome name'
 			const data = await trigger_getViolinPlotData(req.query, null, ds, g) // as getViolinResponse
-			res.send(data)
+			res.send(data as getViolinResponse)
 		} catch (e) {
 			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
 			// @ts-ignore