@sjcrh/proteinpaint-server 2.34.1 → 2.35.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.34.1",
+  "version": "2.35.0",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "server.js",
   "bin": "start.js",
@@ -56,7 +56,7 @@
     "webpack-notifier": "^1.15.0"
   },
   "dependencies": {
-    "@sjcrh/augen": "2.32.2-0",
+    "@sjcrh/augen": "2.35.0",
     "@sjcrh/proteinpaint-rust": "2.34.0",
     "better-sqlite3": "^7.5.3",
     "body-parser": "^1.15.2",

package/routes/termdb.singleSampleMutation.ts

@@ -0,0 +1,87 @@
+import fs from 'fs'
+import path from 'path'
+import { read_file } from '#src/utils.js'
+import serverconfig from '#src/serverconfig.js'
+import {
+	TermdbSingleSampleMutationRequest,
+	TermdbSingleSampleMutationResponse
+} from '#shared/types/routes/termdb.singleSampleMutation.ts'
+import { gdcValidate_query_singleSampleMutation } from '#src/mds3.gdc.js'
+
+export const api: any = {
+	endpoint: 'termdb/singleSampleMutation',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'TermdbSingleSampleMutationRequest'
+			},
+			response: {
+				typeId: 'TermdbSingleSampleMutationResponse'
+			}
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		const q = req.query as TermdbSingleSampleMutationRequest
+		let result
+		try {
+			const g = genomes[q.genome]
+			if (!g) throw 'invalid genome name'
+			const ds = g.datasets[q.dslabel]
+			if (!ds) throw 'invalid dataset name'
+			if (!ds.queries?.singleSampleMutation) throw 'not supported on this dataset'
+			result = (await ds.queries.singleSampleMutation.get(q)) as TermdbSingleSampleMutationResponse
+		} catch (e: any) {
+			if (e.stack) console.log(e.stack)
+			result = {
+				status: e.status || 400,
+				error: e.message || e
+			} as TermdbSingleSampleMutationResponse
+		}
+		res.send(result)
+	}
+}
+
+/////////////////// ds query validator
+export async function validate_query_singleSampleMutation(ds: any, genome: any) {
+	const _q = ds.queries.singleSampleMutation
+	if (!_q) return
+	if (_q.src == 'gdcapi') {
+		gdcValidate_query_singleSampleMutation(ds, genome)
+	} else if (_q.src == 'native') {
+		// using a folder to store text files for individual samples
+		// file names are integer sample id
+		_q.get = async (q: TermdbSingleSampleMutationRequest) => {
+			/* as mds3 client may not be using integer sample id for now,
+			the argument is string id and has to be mapped to integer id
+			*/
+			let fileName = q.sample
+			if (ds.cohort?.termdb?.q?.sampleName2id) {
+				// has name-to-id converter
+				fileName = ds.cohort.termdb.q.sampleName2id(q.sample)
+				if (fileName == undefined) {
+					// unable to convert string id to integer
+					return []
+				}
+			}
+
+			const file = path.join(serverconfig.tpmasterdir, _q.folder, fileName.toString())
+			try {
+				await fs.promises.stat(file)
+			} catch (e: any) {
+				if (e.code == 'EACCES') throw 'cannot read file, permission denied'
+				if (e.code == 'ENOENT') throw 'no data for this sample'
+				throw 'failed to load data'
+			}
+
+			const data = await read_file(file)
+			// object wraps around mlst[] so it's possible to add other attr e.g. total number of mutations that exceeds viewing limit
+			return { mlst: JSON.parse(data) }
+		}
+	} else {
+		throw 'unknown singleSampleMutation.src'
+	}
+}

package/routes/termdb.singlecellData.ts

@@ -29,17 +29,21 @@ export const api: any = {
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
 		const q = req.query as TermdbSinglecellDataRequest
+		let result
 		try {
 			const g = genomes[q.genome]
 			if (!g) throw 'invalid genome name'
 			const ds = g.datasets[q.dslabel]
 			if (!ds) throw 'invalid dataset name'
 			if (!ds.queries?.singleCell) throw 'no singlecell data on this dataset'
-			const result = (await ds.queries.singleCell.data.get(q)) as TermdbSinglecellDataResponse
-			res.send(result)
+			result = (await ds.queries.singleCell.data.get(q)) as TermdbSinglecellDataResponse
 		} catch (e: any) {
-			if (e instanceof Error && e.stack) console.log(e)
-			res.send({ error: e.message || e })
+			if (e.stack) console.log(e)
+			result = {
+				status: e.status || 400,
+				error: e.message || e
+			} as TermdbSinglecellDataResponse
 		}
+		res.send(result)
 	}
 }

package/routes/termdb.singlecellSamples.ts

@@ -14,13 +14,7 @@ import {
 	TermdbSinglecellsamplesRequest,
 	TermdbSinglecellsamplesResponse
 } from '#shared/types/routes/termdb.singlecellSamples.ts'
-import {
-	Cell,
-	Plot,
-	HasdataResponse,
-	NodataResponse,
-	ErrorResponse
-} from '#shared/types/routes/termdb.singlecellData.ts'
+import { Cell, Plot } from '#shared/types/routes/termdb.singlecellData.ts'
 import { gdc_validate_query_singleCell_samples, gdc_validate_query_singleCell_data } from '#src/mds3.gdc.js'
 
 /* route returns list of samples with sc data
@@ -49,18 +43,22 @@ export const api: any = {
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
 		const q = req.query as TermdbSinglecellsamplesRequest
+		let result
 		try {
 			const g = genomes[q.genome]
 			if (!g) throw 'invalid genome name'
 			const ds = g.datasets[q.dslabel]
 			if (!ds) throw 'invalid dataset name'
 			if (!ds.queries?.singleCell) throw 'no singlecell data on this dataset'
-			const samples = (await ds.queries.singleCell.samples.get(q)) as TermdbSinglecellsamplesResponse
-			res.send({ samples })
+			result = (await ds.queries.singleCell.samples.get(q)) as TermdbSinglecellsamplesResponse
 		} catch (e: any) {
-			if (e instanceof Error && e.stack) console.log(e)
-			res.send({ error: e.message || e })
+			if (e.stack) console.log(e.stack)
+			result = {
+				status: e.status || 400,
+				error: e.message || e
+			} as TermdbSinglecellsamplesResponse
 		}
+		res.send(result)
 	}
 }
 
@@ -71,15 +69,19 @@ export async function validate_query_singleCell(ds: any, genome: any) {
 
 	if (q.samples.src == 'gdcapi') {
 		gdc_validate_query_singleCell_samples(ds, genome)
-	} else {
+	} else if (q.samples.src == 'native') {
 		validateSamplesNative(q.samples as SingleCellSamplesNative, ds)
+	} else {
+		throw 'unknown singleCell.samples.src'
 	}
 	// q.samples.get() added
 
 	if (q.data.src == 'gdcapi') {
 		gdc_validate_query_singleCell_data(ds, genome)
-	} else {
+	} else if (q.data.src == 'native') {
 		validateDataNative(q.data as SingleCellDataNative, ds)
+	} else {
+		throw 'unknown singleCell.data.src'
 	}
 	// q.data.get() added
 }
@@ -94,7 +96,9 @@ function validateSamplesNative(S: SingleCellSamplesNative, ds: any) {
 	}
 	if (samples.length == 0) throw 'no sample with sc data'
 	// getter returns array of {sample:<samplename>, files:[]} where files is gdc specific. each sample is an obj and allows to add ds-specific stuff
-	S.get = () => samples
+	S.get = () => {
+		return { samples, fields: ['sample'], columnNames: ['Sample'] }
+	}
 }
 
 function validateDataNative(D: SingleCellDataNative, ds: any) {
@@ -112,17 +116,22 @@ function validateDataNative(D: SingleCellDataNative, ds: any) {
 		const t = ds.cohort.termdb.q.termjsonByOneid(tid)
 		if (!t) throw 'invalid term id from queries.singleCell.data.termIds[]'
 		_terms.push(t)
-		_tid2cellvalue[tid] = ds.cohort.termdb.q.getAllValues4term(tid)
+		// _tid2cellvalue[tid] = {}
+		// const clusterMap = ds.cohort.termdb.q.getAllValues4term(tid)
+		// for(const [id, cluster] of clusterMap)
+		// {
+		// 	const name = ds.cohort.termdb.q.id2sampleName(id)
+		// 	_tid2cellvalue[tid][name] = cluster
+		// }
 	}
-	D.get = async sample => {
+	D.get = async q => {
 		// if sample is int, may convert to string
 		try {
 			const tid2cellvalue = {}
 			for (const tid of D.termIds) tid2cellvalue[tid] = {} // k: cell id, v: cell value for this term
-
 			const plots = [] as Plot[] // given a sample name, collect every plot data for this sample and return
 			for (const plot of D.plots) {
-				const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, sample, plot.fileSuffix)
+				const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, q.sample + plot.fileSuffix)
 				try {
 					await fs.promises.stat(tsvfile)
 				} catch (e: any) {
@@ -147,9 +156,7 @@ function validateDataNative(D: SingleCellDataNative, ds: any) {
 					cells.push({ cellId, x, y })
 
 					for (const tid of D.termIds) {
-						if (_tid2cellvalue[tid].has(cellId)) {
-							tid2cellvalue[tid][cellId] = _tid2cellvalue[tid].get(cellId)
-						}
+						tid2cellvalue[tid][cellId] = l[1]
 					}
 				}
 				plots.push({ name: plot.name, cells })

package/routes/termdb.topVariablyExpressedGenes.ts

@@ -0,0 +1,161 @@
+import {
+	TermdbTopVariablyExpressedGenesRequest,
+	TermdbTopVariablyExpressedGenesResponse
+} from '#shared/types/routes/termdb.topVariablyExpressedGenes.ts'
+import { gdcGetCasesWithExressionDataFromCohort, apihost, geneExpHost } from '../src/mds3.gdc.js'
+import path from 'path'
+import got from 'got'
+import serverconfig from '#src/serverconfig.js'
+
+export const api = {
+	endpoint: 'termdb/topVariablyExpressedGenes',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'TermdbTopVariablyExpressedGenesRequest'
+			},
+			response: {
+				typeId: 'TermdbTopVariablyExpressedGenesResponse'
+			}
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		try {
+			const q = req.query as TermdbTopVariablyExpressedGenesRequest
+			const genome = genomes[q.genome]
+			if (!genome) throw 'invalid genome'
+			const ds = genome.datasets?.[q.dslabel]
+			if (!ds) throw 'invalid dslabel'
+			if (!ds.queries?.topVariablyExpressedGenes) throw 'not supported on dataset'
+			const genes = await ds.queries.topVariablyExpressedGenes.getGenes(q)
+			res.send({ genes } as TermdbTopVariablyExpressedGenesResponse)
+		} catch (e: any) {
+			res.send({ status: 'error', error: e.message || e })
+		}
+	}
+}
+
+export function validate_query_TopVariablyExpressedGenes(ds: any, genome: any) {
+	const q = ds.queries.topVariablyExpressedGenes
+	if (!q) return
+	if (q.src == 'gdcapi') {
+		gdcValidateQuery(ds, genome)
+	} else if (q.src == 'native') {
+		nativeValidateQuery(ds, genome)
+	} else {
+		throw 'unknown topVariablyExpressedGenes.src'
+	}
+	// added getter: q.getGenes()
+}
+
+function nativeValidateQuery(ds: any, genome: any) {
+	ds.queries.topVariablyExpressedGenes.getGenes = async (
+		q: TermdbTopVariablyExpressedGenesRequest,
+		ds: any,
+		genome: any
+	) => {
+		// get list of samples that are used in current analysis
+		const samples = [] as string[]
+		// call rust to compute top genes on these samples
+		const genes = await computeGenes4nativeDs(q, ds, samples)
+		return genes
+	}
+}
+
+async function computeGenes4nativeDs(q: TermdbTopVariablyExpressedGenesRequest, ds: any, samples: string[]) {
+	return []
+}
+
+function gdcValidateQuery(ds: any, genome: any) {
+	ds.queries.topVariablyExpressedGenes.getGenes = async (q: TermdbTopVariablyExpressedGenesRequest) => {
+		if (serverconfig.features.gdcGenes) {
+			// for testing on dev, must not set on prod!! delete to trigger api query
+			console.log('!!GDC!! using serverconfig.features.gdcGenes[]')
+			return serverconfig.features.gdcGenes as string[]
+		}
+
+		// disable when caching is incomplete (particularly cases with gene exp data); to prevent showing wrong data on client
+		if (!ds.__gdc.doneCaching) throw 'The server has not finished caching the case IDs: try again in ~2 minutes'
+
+		// based on current cohort, get list of cases with exp data, as input of next api query
+		const caseLst = await gdcGetCasesWithExressionDataFromCohort(q, ds)
+		if (caseLst.length == 0) {
+			// there are no cases with gene exp data
+			return [] as string[]
+		}
+
+		// change to this when api is available on prod
+		const url = path.join(geneExpHost, '/gene_expression/gene_selection')
+
+		try {
+			const response = await got.post(url, {
+				headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
+				body: JSON.stringify(getGeneSelectionArg(q, caseLst))
+			})
+
+			const re = JSON.parse(response.body)
+			// {"gene_selection":[{"gene_id":"ENSG00000141510","log2_uqfpkm_median":3.103430497010492,"log2_uqfpkm_stddev":0.8692021350485105,"symbol":"TP53"}, ... ]}
+
+			const genes = [] as string[]
+			if (!Array.isArray(re.gene_selection)) throw 're.gene_selection[] is not array'
+			for (const i of re.gene_selection) {
+				if (i.gene_id && typeof i.gene_id == 'string') {
+					// is ensg, convert to symbol
+					const t = genome.genedb.getNameByAlias.get(i.gene_id)
+					if (t) genes.push(t.name) // ensg
+				} else if (i.symbol && typeof i.symbol == 'string') {
+					genes.push(i.symbol)
+				} else {
+					throw 'one of re.gene_selection[] is missing both gene_id and symbol'
+				}
+			}
+			return genes
+		} catch (e: any) {
+			console.log(e.stack || e)
+			throw e
+		}
+	}
+
+	function getGeneSelectionArg(q: any, caseLst: any) {
+		//to hide messy logic during testing phase
+
+		/* when api performance issue is resolved, return this
+		return {
+			case_ids: caseLst,
+			gene_type:'protein_coding',
+			selection_size: Number(q.maxGenes)
+		}
+		*/
+
+		//////////////////////////////////////////////////
+		//
+		// !!!!!!!!!!!!!!!! TEMPORARY !!!!!!!!!!!!!!!!!!!!
+		//
+		//////////////////////////////////////////////////
+		// limit the case_ids length, and restrict pool to CGC genes, otherwise the request times out !!!
+		// must revert asap
+		return {
+			case_ids: caseLst.slice(0, 20),
+			gene_ids: tempGetCGCgenes(genome),
+			selection_size: Number(q.maxGenes)
+		}
+	}
+}
+
+function tempGetCGCgenes(genome: any) {
+	const lst = [] as string[] // list of ENSG ids from cgc genes
+	// don't think there's need to preparse genome.geneset, as this function is only temporary
+	for (const s of genome.geneset[0].lst) {
+		const a = genome.genedb.getAliasByName.all(s)
+		if (a) {
+			for (const b of a) {
+				if (b.alias.startsWith('ENSG')) lst.push(b.alias)
+			}
+		}
+	}
+	return lst
+}