@sjcrh/proteinpaint-server 2.38.1 → 2.39.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.38.1",
+  "version": "2.39.1",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "server.js",
   "bin": "start.js",
@@ -57,7 +57,7 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.35.0",
-    "@sjcrh/proteinpaint-rust": "2.38.1",
+    "@sjcrh/proteinpaint-rust": "2.39.0",
     "better-sqlite3": "^7.5.3",
     "body-parser": "^1.15.2",
     "canvas": "~2.9.3",

package/routes/gdc.maf.ts

@@ -3,15 +3,28 @@ import path from 'path'
 import got from 'got'
 import serverconfig from '#src/serverconfig.js'
 
-const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
-const maxFileNumber = 1000
+/*
+this route lists available gdc MAF files based on user's cohort filter
+and return them to client to be shown in a table for selection
+*/
+
+const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov' // to switch to serverconfig export
+
+const maxFileNumber = 1000 // determines max number of files to return to client
+// preliminary testing:
+// 36s for 1000 (87Mb)
+// 78s for 2000 (177Mb)
+// if safe to increase to 2000, maybe fast when this runs in gdc env
+
 const allowedWorkflowType = 'Aliquot Ensemble Somatic Variant Merging and Masking'
-const maxTotalSizeCompressed = serverconfig.features.gdcMafMaxFileSize || 50000000 // 50Mb
+
+// change to 400 so it won't limit number of files; should keep this setting as a safeguard; also it's fast to check file size (.5s in gdc.mafBuild.ts)
+export const maxTotalSizeCompressed = serverconfig.features.gdcMafMaxFileSize || 400000000 // 400Mb
 
 export const api = {
 	endpoint: 'gdc/maf',
 	methods: {
-		get: {
+		all: {
 			init,
 			request: {
 				typeId: 'GdcMafRequest'

package/routes/gdc.mafBuild.ts

@@ -5,9 +5,9 @@ import { run_rust_stream } from '@sjcrh/proteinpaint-rust'
 import serverconfig from '#src/serverconfig.js'
 import Readable from 'stream'
 import { GdcMafBuildRequest } from '#shared/types/routes/gdc.mafBuild.ts'
+import { maxTotalSizeCompressed } from './gdc.maf.ts'
 
 const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
-const maxTotalSizeCompressed = serverconfig.features.gdcMafMaxFileSize || 50000000 // 50Mb
 
 export const api = {
 	endpoint: 'gdc/mafBuild',
@@ -43,10 +43,16 @@ async function buildMaf(q: GdcMafBuildRequest, res: any) {
 	const t0 = Date.now()
 
 	const fileLst2 = (await getFileLstUnderSizeLimit(q.fileIdLst)) as string[]
-	console.log('test gdc maf sizes', Date.now() - t0)
+
+	if (serverconfig.debugmode)
+		console.log(
+			`${fileLst2.length} out of ${q.fileIdLst.length} input MAF files accepted by size limit`,
+			Date.now() - t0
+		)
 
 	const arg = {
 		fileIdLst: fileLst2,
+		columns: q.columns,
 		host: path.join(apihost, 'data') // must use the /data/ endpoint from current host
 	}
 
@@ -55,11 +61,12 @@ async function buildMaf(q: GdcMafBuildRequest, res: any) {
 	res.setHeader('Content-Disposition', 'attachment; filename=cohort.maf.gz')
 	rustStream.pipe(res)
 
-	console.log('rust gdcmaf', Date.now() - t0)
-
 	rustStream.on('end', () => {
+		// report amount of time taken to run rust
+		if (serverconfig.debugmode) console.log('rust gdcmaf', Date.now() - t0)
 		res.end()
 	})
+
 	rustStream.on('error', err => {
 		console.error(err)
 		res.statusCode = 500
@@ -80,7 +87,7 @@ async function getFileLstUnderSizeLimit(lst: string[]) {
 			op: 'in',
 			content: { field: 'file_id', value: lst }
 		},
-		size: 1000,
+		size: 10000,
 		fields: 'file_size'
 	}
 	const headers = { 'Content-Type': 'application/json', Accept: 'application/json' }

package/routes/hicdata.ts

@@ -40,7 +40,7 @@ export const api: any = {
 	}
 }
 
-function init({ genomes }) {
+function init() {
 	return async (req: any, res: any): Promise<void> => {
 		try {
 			const payload = await handle_hicdata(req.query as HicdataRequest)
@@ -60,7 +60,7 @@ function handle_hicdata(q: HicdataRequest) {
 		if (e) reject({ error: 'illegal file name' })
 
 		const par = [
-			// TODO add option for observed/oe
+			q.matrixType || 'observed',
 			q.nmeth || 'NONE',
 			file,
 			q.pos1,

package/routes/termdb.categories.ts

@@ -1,4 +1,4 @@
-// import { getcategoriesRequest, getcategoriesResponse } from '#shared/types/routes/termdb.categories'
+import { getcategoriesRequest, getcategoriesResponse } from '#shared/types/routes/termdb.categories.ts'
 import { getOrderedLabels } from '#src/termdb.barchart.js'
 import { getData } from '#src/termdb.matrix.js'
 
@@ -68,7 +68,7 @@ export const api: any = {
 
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
-		const q = req.query // as getcategoriesRequest
+		const q = req.query as getcategoriesRequest
 		try {
 			const g = genomes[req.query.genome]
 			if (!g) throw 'invalid genome name'
@@ -88,8 +88,8 @@ function init({ genomes }) {
 }
 
 async function trigger_getcategories(
-	q: { tid: string | number; type: string; filter: any; term1_q: any; currentGeneNames: any },
-	res: { send: (arg0: { lst: any[]; orderedLabels: any }) => void },
+	q: { tid: string | number; type: string; filter: any; term1_q: any; currentGeneNames?: string[]; rglst?: any },
+	res: { send: (arg0: { lst: any[]; orderedLabels?: any }) => void },
 	tdb: { q: { termjsonByOneid: (arg0: any) => any } },
 	ds: { assayAvailability: { byDt: { [s: string]: any } | ArrayLike<any> } },
 	genome: any
@@ -99,13 +99,15 @@ async function trigger_getcategories(
 	if (!q.tid) throw '.tid missing'
 	const term =
 		q.type == 'geneVariant' ? { name: q.tid, type: 'geneVariant', isleaf: true } : tdb.q.termjsonByOneid(q.tid)
+
 	const arg = {
 		filter: q.filter,
 		terms:
 			q.type == 'geneVariant'
 				? [{ term: term, q: { isAtomic: true } }]
 				: [{ id: q.tid, term, q: q.term1_q || getDefaultQ(term, q) }],
-		currentGeneNames: q.currentGeneNames
+		currentGeneNames: q.currentGeneNames, // optional, from mds3 mayAddGetCategoryArgs()
+		rglst: q.rglst // optional, from mds3 mayAddGetCategoryArgs()
 	}
 
 	const data = await getData(arg, ds, genome)
@@ -200,7 +202,7 @@ async function trigger_getcategories(
 	res.send({
 		lst,
 		orderedLabels
-	})
+	} as getcategoriesResponse)
 }
 
 function getDefaultQ(

package/routes/termdb.cluster.ts

@@ -0,0 +1,176 @@
+import { TermdbClusterRequest, TermdbClusterResponse } from '#shared/types/routes/termdb.cluster.ts'
+import path from 'path'
+import * as utils from '#src/utils.js'
+import serverconfig from '#src/serverconfig.js'
+import { GeneExpressionQuery, GeneExpressionQueryNative } from '#shared/types/dataset.ts'
+import { gdc_validate_query_geneExpression } from '#src/mds3.gdc.js'
+import { mayLimitSamples } from '#src/mds3.filter.js'
+import { doClustering } from '#src/doClustering.js' // unable to convert this to ts yet, when converted, move all code here
+import { dtgeneexpression } from '#shared/common.js'
+
+export const api = {
+	endpoint: 'termdb/cluster',
+	methods: {
+		all: {
+			init,
+			request: {
+				typeId: 'TermdbClusterRequest'
+			},
+			response: {
+				typeId: 'TermdbClusterResponse'
+			}
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		const q = req.query as TermdbClusterRequest
+		let result
+		try {
+			const g = genomes[q.genome]
+			if (!g) throw 'invalid genome name'
+			const ds = g.datasets[q.dslabel]
+			if (!ds) throw 'invalid dataset name'
+			if (ds.__gdc && !ds.__gdc.doneCaching)
+				throw 'The server has not finished caching the case IDs: try again in ~2 minutes'
+			if (q.dataType == dtgeneexpression) {
+				if (!ds.queries?.geneExpression) throw 'no geneExpression data on this dataset'
+				result = (await getResult(q, ds)) as TermdbClusterResponse
+			} else {
+				throw 'unknown q.dataType ' + q.dataType
+			}
+		} catch (e: any) {
+			if (e.stack) console.log(e.stack)
+			result = {
+				status: e.status || 400,
+				error: e.message || e
+			} as TermdbClusterResponse
+		}
+		res.send(result)
+	}
+}
+
+async function getResult(q: TermdbClusterRequest, ds: any) {
+	const { gene2sample2value, byTermId, bySampleId } = await ds.queries.geneExpression.get(q)
+	if (gene2sample2value.size == 0) throw 'no data'
+	if (gene2sample2value.size == 1) {
+		// get data for only 1 gene; still return data, may create violin plot later
+		const g = Array.from(gene2sample2value.keys())[0]
+		return { gene: g, data: gene2sample2value.get(g) }
+	}
+
+	// have data for multiple genes, run clustering
+	const t = Date.now() // use "t=new Date()" will lead to tsc error
+	const clustering = await doClustering(gene2sample2value, q, ds)
+	if (serverconfig.debugmode) console.log('clustering done:', Date.now() - t, 'ms')
+	return { clustering, byTermId, bySampleId }
+}
+
+export async function validate_query_geneExpression(ds: any, genome: any) {
+	const q = ds.queries.geneExpression as GeneExpressionQuery
+	if (!q) return
+
+	if (q.src == 'gdcapi') {
+		gdc_validate_query_geneExpression(ds, genome)
+		// q.get() added
+		return
+	}
+	if (q.src == 'native') {
+		validateNative(q, ds, genome)
+		return
+	}
+	throw 'unknown queries.geneExpression.src'
+}
+
+async function validateNative(q: GeneExpressionQueryNative, ds: any, genome: any) {
+	q.file = path.join(serverconfig.tpmasterdir, q.file)
+	await utils.validate_tabixfile(q.file)
+	q.nochr = await utils.tabix_is_nochr(q.file, null, genome)
+	q.samples = [] as number[]
+
+	{
+		// is a gene-by-sample matrix file
+		const lines = await utils.get_header_tabix(q.file)
+		if (!lines[0]) throw 'header line missing from ' + q.file
+		const l = lines[0].split('\t')
+		if (l.slice(0, 4).join('\t') != '#chr\tstart\tstop\tgene') throw 'header line has wrong content for columns 1-4'
+		for (let i = 4; i < l.length; i++) {
+			const id = ds.cohort.termdb.q.sampleName2id(l[i])
+			if (id == undefined) throw 'unknown sample from header'
+			q.samples.push(id)
+		}
+		console.log(q.samples.length, 'samples from geneExpression of', ds.label)
+	}
+
+	/*
+	query exp data one gene at a time
+	param{}
+	.genes[{}]
+		.gene=str
+		.chr=str
+		.start=int
+		.stop=int
+	.filterObj{}
+	*/
+	q.get = async (param: TermdbClusterRequest) => {
+		const limitSamples = await mayLimitSamples(param, q.samples, ds)
+		if (limitSamples?.size == 0) {
+			// got 0 sample after filtering, must still return expected structure with no data
+			return { gene2sample2value: new Set(), byTermId: {}, bySampleId: {} }
+		}
+
+		// has at least 1 sample passing filter and with exp data
+		// TODO what if there's just 1 sample not enough for clustering?
+		const bySampleId = {}
+		if (limitSamples) {
+			for (const sid of limitSamples) {
+				bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) }
+			}
+		} else {
+			// use all samples with exp data
+			for (const sid of q.samples) {
+				bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) }
+			}
+		}
+
+		const gene2sample2value = new Map() // k: gene symbol, v: { sampleId : value }
+
+		for (const g of param.genes) {
+			// FIXME newly added geneVariant terms from client to be changed to {gene} but not {name}
+			if (!g.gene) continue
+
+			if (!g.chr) {
+				// quick fix: newly added gene from client will lack chr/start/stop
+				const lst = genome.genedb.getjsonbyname.all(g.gene)
+				if (lst.length == 0) continue
+				const j = JSON.parse(lst.find(i => i.isdefault).genemodel || lst[0].genemodel)
+				g.start = j.start
+				g.stop = j.stop
+				g.chr = j.chr
+			}
+
+			gene2sample2value.set(g.gene, {})
+			await utils.get_lines_bigfile({
+				args: [q.file, (q.nochr ? g.chr?.replace('chr', '') : g.chr) + ':' + g.start + '-' + g.stop], // must do g.chr?.replace to avoid tsc error
+				callback: line => {
+					const l = line.split('\t')
+					// case-insensitive match! FIXME if g.gene is alias won't work
+					if (l[3].toLowerCase() != g.gene.toLowerCase()) return
+					for (let i = 4; i < l.length; i++) {
+						const sampleId = q.samples[i - 4]
+						if (limitSamples && !limitSamples.has(sampleId)) continue // doing filtering and sample of current column is not used
+						// if l[i] is blank string?
+						const v = Number(l[i])
+						if (Number.isNaN(v)) throw 'exp value not number'
+						gene2sample2value.get(g.gene)[sampleId] = v
+					}
+				}
+			} as any)
+			// Above!! add "as any" to suppress a npx tsc alert
+		}
+		// pass blank byTermId to match with expected output structure
+		const byTermId = {}
+		return { gene2sample2value, byTermId, bySampleId }
+	}
+}

package/routes/termdb.getdescrstats.ts

@@ -1,7 +1,6 @@
-// import { getdescrstatsRequest, getdescrstatsResponse } from '#shared/types/routes/termdb.descrstats'
-import roundValue from '#shared/roundValue.js'
-import computePercentile from '#shared/compute.percentile.js'
-import * as termdbsql from '#src/termdb.sql.js'
+import { getdescrstatsRequest, getdescrstatsResponse } from '#shared/types/routes/termdb.getdescrstats.ts'
+import * as termdbsql from '../src/termdb.sql.js'
+import Summarystats from '../shared/descriptive.stats.js'
 
 export const api: any = {
 	endpoint: 'termdb/descrstats',
@@ -62,7 +61,7 @@ export const api: any = {
 
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
-		const q = req.query // as getdescrstatsRequest
+		const q = req.query as getdescrstatsRequest
 		try {
 			const g = genomes[req.query.genome]
 			if (!g) throw 'invalid genome name'
@@ -104,42 +103,5 @@ async function trigger_getdescrstats(q: any, res: any, ds: any) {
 		}
 		values.push(value)
 	}
-
-	// compute statistics
-	// total
-	const total = values.length
-
-	// mean
-	const sum = values.reduce((a, b) => a + b, 0)
-	const mean = sum / total
-
-	// percentiles
-	const p25 = computePercentile(values, 25)
-	const median = computePercentile(values, 50)
-	const p75 = computePercentile(values, 75)
-
-	// standard deviation
-	// get sum of squared differences from mean
-	const sumSqDiff = values.map(v => (v - mean) ** 2).reduce((a, b) => a + b, 0)
-	// get variance
-	const variance = sumSqDiff / (values.length - 1)
-	// get standard deviation
-	const sd = Math.sqrt(variance)
-
-	// min/max
-	const min = Math.min(...values)
-	const max = Math.max(...values)
-
-	res.send({
-		values: [
-			{ id: 'total', label: 'n', value: total },
-			{ id: 'min', label: 'Minimum', value: roundValue(min, 2) },
-			{ id: 'p25', label: '1st quartile', value: roundValue(p25, 2) },
-			{ id: 'median', label: 'Median', value: roundValue(median, 2) },
-			{ id: 'mean', label: 'Mean', value: roundValue(mean, 2) },
-			{ id: 'p75', label: '3rd quartile', value: roundValue(p75, 2) },
-			{ id: 'max', label: 'Maximum', value: roundValue(max, 2) },
-			{ id: 'sd', label: 'Standard deviation', value: roundValue(sd, 2) }
-		]
-	})
+	res.send(Summarystats(values) as getdescrstatsResponse)
 }

package/routes/termdb.getnumericcategories.ts

@@ -0,0 +1,99 @@
+import {
+	getnumericcategoriesRequest,
+	getnumericcategoriesResponse
+} from '#shared/types/routes/termdb.getnumericcategories.ts'
+import * as termdbsql from '#src/termdb.sql.js'
+
+export const api: any = {
+	endpoint: 'termdb/numericcategories',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'getnumericcategoriesRequest'
+			},
+			response: {
+				typeId: 'getnumericcategoriesResponse'
+			},
+			examples: [
+				{
+					request: {
+						body: {
+							genome: 'hg38-test',
+							dslabel: 'TermdbTest',
+							embedder: 'localhost',
+							tid: 'aaclassic_5',
+							filter: {
+								type: 'tvslst',
+								in: true,
+								join: '',
+								lst: [
+									{
+										tag: 'cohortFilter',
+										type: 'tvs',
+										tvs: {
+											term: {
+												name: 'Cohort',
+												type: 'categorical',
+												values: { ABC: { label: 'ABC' }, XYZ: { label: 'XYZ' } },
+												id: 'subcohort',
+												isleaf: false,
+												groupsetting: { disabled: true }
+											},
+											values: [{ key: 'ABC', label: 'ABC' }]
+										}
+									}
+								]
+							}
+						}
+					},
+					response: {
+						header: { status: 200 }
+					}
+				}
+			]
+		},
+		post: {
+			alternativeFor: 'get',
+			init
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		const q = req.query as getnumericcategoriesRequest
+		try {
+			const g = genomes[req.query.genome]
+			if (!g) throw 'invalid genome name'
+			const ds = g.datasets[req.query.dslabel]
+			if (!ds) throw 'invalid dataset name'
+			const tdb = ds.cohort.termdb
+			if (!tdb) throw 'invalid termdb object'
+
+			await trigger_getnumericcategories(q, res, tdb, ds) // as getnumericcategoriesResponse
+		} catch (e) {
+			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+			// @ts-ignore
+			res.send({ error: e?.message || e })
+			if (e instanceof Error && e.stack) console.log(e)
+		}
+	}
+}
+
+async function trigger_getnumericcategories(
+	q: { tid: any; filter?: any },
+	res: { send: (arg0: { lst: any }) => void },
+	tdb: { q: { termjsonByOneid: (arg0: any) => any } },
+	ds: any
+) {
+	if (!q.tid) throw '.tid missing'
+	const term = tdb.q.termjsonByOneid(q.tid)
+	const arg = {
+		ds,
+		term_id: q.tid,
+		filter: q.filter
+	}
+	const lst = await termdbsql.get_summary_numericcategories(arg)
+	res.send({ lst } as getnumericcategoriesResponse)
+}

package/routes/termdb.getpercentile.ts

@@ -1,6 +1,7 @@
-// import { getpercentileRequest, getpercentileResponse } from '#shared/types/routes/termdb.getpercentile'
-import * as termdbsql from '#src/termdb.sql.js'
-import computePercentile from '#shared/compute.percentile.js'
+import { getpercentileRequest, getpercentileResponse } from '#shared/types/routes/termdb.getpercentile.ts'
+import * as termdbsql from '../src/termdb.sql.js'
+import computePercentile from '../shared/compute.percentile.js'
+import { Filter } from '../shared/types/filter'
 
 export const api: any = {
 	endpoint: 'termdb/getpercentile',
@@ -61,7 +62,7 @@ export const api: any = {
 
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
-		const q = req.query // as getpercentileRequest
+		const q = req.query as getpercentileRequest
 		try {
 			const g = genomes[req.query.genome]
 			if (!g) throw 'invalid genome name'
@@ -78,7 +79,7 @@ function init({ genomes }) {
 }
 
 async function trigger_getpercentile(
-	q: { tid: string; getpercentile: number[]; filter: string },
+	q: { tid: string; getpercentile: number[]; filter: Filter },
 	res: { send: (arg0: { values: number[] }) => void },
 	ds: { cohort: { termdb: { q: { termjsonByOneid: (arg0: any) => any } } } }
 ) {
@@ -113,5 +114,5 @@ async function trigger_getpercentile(
 		const perc_value = computePercentile(values, percentile)
 		perc_values.push(perc_value)
 	}
-	res.send({ values: perc_values })
+	res.send({ values: perc_values } as getpercentileResponse)
 }

package/routes/termdb.getrootterm.ts

@@ -0,0 +1,67 @@
+import { getroottermRequest, getroottermResponse } from '#shared/types/routes/termdb.getrootterm.ts'
+
+export const api: any = {
+	endpoint: 'termdb/rootterm',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'getroottermRequest'
+			},
+			response: {
+				typeId: 'getroottermResponse'
+			},
+			examples: [
+				{
+					request: {
+						body: {
+							genome: 'hg38-test',
+							dslabel: 'TermdbTest',
+							embedder: 'localhost',
+							default_rootterm: 1,
+							cohortValues: 'ABC'
+						}
+					},
+					response: {
+						header: { status: 200 }
+					}
+				}
+			]
+		},
+		post: {
+			alternativeFor: 'get',
+			init
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		const q = req.query as getroottermRequest
+		try {
+			const g = genomes[req.query.genome]
+			if (!g) throw 'invalid genome name'
+			const ds = g.datasets[req.query.dslabel]
+			if (!ds) throw 'invalid dataset name'
+			const tdb = ds.cohort.termdb
+			if (!tdb) throw 'invalid termdb object'
+
+			await trigger_rootterm(q, res, tdb) // as getroottermResponse
+		} catch (e) {
+			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+			// @ts-ignore
+			res.send({ error: e?.message || e })
+			if (e instanceof Error && e.stack) console.log(e)
+		}
+	}
+}
+
+async function trigger_rootterm(
+	q: { cohortValues: any; treeFilter: any },
+	res: { send: (arg0: { lst: any }) => void },
+	tdb: { q: { getRootTerms: (arg0: any, arg1: any) => any } }
+) {
+	const cohortValues = q.cohortValues ? q.cohortValues : ''
+	const treeFilter = q.treeFilter ? q.treeFilter : ''
+	res.send({ lst: await tdb.q.getRootTerms(cohortValues, treeFilter) } as getroottermResponse)
+}