@sjcrh/proteinpaint-server 2.30.4 → 2.31.0

package/genome/hg19.js

@@ -44,9 +44,9 @@ exports.default = {
             vpad: 4,
             categories: {
                 coding: { color: '#004D99', label: 'Coding gene' },
-                nonCoding: { color: '#009933', label: 'Noncoding gene' },
-                problem: { color: '#FF3300', label: 'Problem' },
-                pseudo: { color: '#FF00CC', label: 'Pseudogene' }
+                nonCoding: { color: '#008833', label: 'Noncoding gene' },
+                problem: { color: '#CC3300', label: 'Problem' },
+                pseudo: { color: '#CC00CC', label: 'Pseudogene' }
             }
         },
         {

package/genome/hg38.js

@@ -48,9 +48,9 @@ exports.default = {
             translatecoding: true,
             categories: {
                 coding: { color: '#004D99', label: 'Coding gene' },
-                nonCoding: { color: '#009933', label: 'Noncoding gene' },
-                problem: { color: '#FF3300', label: 'Problem' },
-                pseudo: { color: '#FF00CC', label: 'Pseudogene' }
+                nonCoding: { color: '#008833', label: 'Noncoding gene' },
+                problem: { color: '#CC3300', label: 'Problem' },
+                pseudo: { color: '#CC00CC', label: 'Pseudogene' }
             },
             type: 'bedj',
             stackheight: 16,

package/genome/hg38.test.js

@@ -24,9 +24,9 @@ var genome = {
             translatecoding: true,
             categories: {
                 coding: { color: '#004D99', label: 'Coding gene' },
-                nonCoding: { color: '#009933', label: 'Noncoding gene' },
-                problem: { color: '#FF3300', label: 'Problem' },
-                pseudo: { color: '#FF00CC', label: 'Pseudogene' }
+                nonCoding: { color: '#008833', label: 'Noncoding gene' },
+                problem: { color: '#CC3300', label: 'Problem' },
+                pseudo: { color: '#CC00CC', label: 'Pseudogene' }
             },
             type: 'bedj',
             name: 'GENCODE v41',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.30.4",
+  "version": "2.31.0",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "server.js",
   "bin": "start.js",
@@ -56,7 +56,7 @@
   },
   "dependencies": {
     "@sjcrh/augen": "2.27.0",
-    "@sjcrh/proteinpaint-rust": "2.30.2",
+    "@sjcrh/proteinpaint-rust": "2.31.0",
     "better-sqlite3": "^7.5.3",
     "body-parser": "^1.15.2",
     "canvas": "~2.9.3",

package/routes/burden.ts

@@ -69,7 +69,10 @@ export const api = {
 	}
 }
 
-async function getBurdenEstimates(q, ds) {
+async function getBurdenEstimates(
+	q: { query: { [x: string]: any } },
+	ds: { cohort: { cumburden: { files: { fit: any; surv: any; sample: any } } } }
+) {
 	const infile = path.join(serverconfig.cachedir, Math.random().toString() + '.json')
 	for (const k in q.query) {
 		q.query[k] = Number(q.query[k])
@@ -91,7 +94,7 @@ async function getBurdenEstimates(q, ds) {
 	return estimates
 }
 
-function formatPayload(estimates) {
+function formatPayload(estimates: object[]) {
 	const rawKeys = Object.keys(estimates[0])
 	const outKeys = [] as string[]
 	const keys = [] as string[]

package/routes/gdc.maf.ts

@@ -1,4 +1,4 @@
-import { GdcMafResponse, File } from '#shared/types/routes/gdc.maf.ts'
+import { GdcMafRequest, GdcMafResponse, File } from '#shared/types/routes/gdc.maf.ts'
 import path from 'path'
 import got from 'got'
 import serverconfig from '#src/serverconfig.js'
@@ -6,36 +6,51 @@ import serverconfig from '#src/serverconfig.js'
 const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
 const maxFileNumber = 1000
 const allowedWorkflowType = 'Aliquot Ensemble Somatic Variant Merging and Masking'
-const allowedExpStrategy = new Set(['WXS', 'Targeted Sequencing'])
 const maxTotalSizeCompressed = serverconfig.features.gdcMafMaxFileSize || 50000000 // 50Mb
 
 export const api = {
 	endpoint: 'gdc/maf',
 	methods: {
 		get: {
-			init({ genomes }) {
-				return async (req: any, res: any): Promise<void> => {
-					try {
-						const g = genomes.hg38
-						if (!g) throw 'hg38 missing'
-						const ds = g.datasets.GDC
-						if (!ds) throw 'hg38 GDC missing'
-						const payload = await listMafFiles(req)
-						res.send(payload)
-					} catch (e: any) {
-						res.send({ status: 'error', error: e.message || e })
-					}
-				}
-			},
+			init,
 			request: {
-				typeId: null
-				//valid: default to type checker
+				typeId: 'GdcMafRequest'
 			},
 			response: {
 				typeId: 'GdcMafResponse'
 				// will combine this with type checker
 				//valid: (t) => {}
-			}
+			},
+			examples: [
+				{
+					request: {
+						body: {
+							experimentalStrategy: 'WXS',
+							embedder: 'localhost'
+						}
+					},
+					response: {
+						header: { status: 200 }
+					}
+				}
+			]
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		try {
+			// g and ds are not used right now, but could be later
+			const g = genomes.hg38
+			if (!g) throw 'hg38 missing'
+			const ds = g.datasets.GDC
+			if (!ds) throw 'hg38 GDC missing'
+
+			const payload = await listMafFiles(req.query as GdcMafRequest)
+			res.send(payload)
+		} catch (e: any) {
+			res.send({ status: 'error', error: e.message || e })
 		}
 	}
 }
@@ -52,21 +67,19 @@ ds {
 	}
 }
 */
-async function listMafFiles(req: any) {
-	if (!allowedExpStrategy.has(req.query.experimentalStrategy)) throw 'invalid req.query.experimentalStrategy'
-
+async function listMafFiles(q: GdcMafRequest) {
 	const filters = {
 		op: 'and',
 		content: [
 			{ op: '=', content: { field: 'data_format', value: 'MAF' } },
-			{ op: '=', content: { field: 'experimental_strategy', value: req.query.experimentalStrategy } },
+			{ op: '=', content: { field: 'experimental_strategy', value: q.experimentalStrategy } },
 			{ op: '=', content: { field: 'analysis.workflow_type', value: allowedWorkflowType } },
 			{ op: '=', content: { field: 'access', value: 'open' } } // delete if later to support controlled files
 		]
 	}
 
-	if (req.query.filter0) {
-		filters.content.push(req.query.filter0)
+	if (q.filter0) {
+		filters.content.push(q.filter0)
 	}
 
 	const headers = { 'Content-Type': 'application/json', Accept: 'application/json' }
@@ -152,7 +165,7 @@ async function listMafFiles(req: any) {
 
 		file.case_submitter_id = c.submitter_id
 		if (c.samples) {
-			file.sample_types = c.samples.map(i => i.sample_type).sort()
+			file.sample_types = c.samples.map((i: { sample_type: any }) => i.sample_type).sort()
 			// sort to show sample type names in consistent alphabetical order
 			// otherwise one file shows 'Blood, Primary' and another shows 'Primary, Blood'
 			// FIXME this includes samples not associated with current maf file

package/routes/gdc.mafBuild.ts

@@ -3,6 +3,8 @@ import path from 'path'
 import fs from 'fs'
 import { run_rust } from '@sjcrh/proteinpaint-rust'
 import serverconfig from '#src/serverconfig.js'
+import Readable from 'stream'
+import { GdcMafBuildRequest } from '#shared/types/routes/gdc.mafBuild.ts'
 
 const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
 const maxTotalSizeCompressed = serverconfig.features.gdcMafMaxFileSize || 50000000 // 50Mb
@@ -11,58 +13,50 @@ export const api = {
 	endpoint: 'gdc/mafBuild',
 	methods: {
 		all: {
-			init({ genomes }) {
-				return async (req: any, res: any): Promise<void> => {
-					try {
-						await buildMaf(req, res)
-					} catch (e) {
-						if (e.stack) console.log(e.stack)
-						res.send({ status: 'error', error: e.message || e })
-					}
-				}
-			},
+			init,
 			request: {
-				typeId: null
-				//valid: default to type checker
+				typeId: 'GdcMafBuildRequest'
 			},
 			response: {
-				typeId: 'GdcMafBuildResponse'
-				// will combine this with type checker
-				//valid: (t) => {}
+				typeId: null // 'GdcMafBuildResponse'
 			}
 		}
 	}
 }
 
-/*
-req.query {
-	fileIdLst []  // list of maf file uuids
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		try {
+			await buildMaf(req.query as GdcMafBuildRequest, res)
+		} catch (e: any) {
+			if (e.stack) console.log(e.stack)
+			res.send({ status: 'error', error: e.message || e })
+		}
+	}
 }
 
+/*
+q{}
 res{}
 */
-async function buildMaf(req: any, res: any) {
-	const t0 = new Date()
-
-	const fileLst2 = (await getFileLstUnderSizeLimit(req.query.fileIdLst)) as string[]
-	console.log('test gdc maf sizes', new Date() - t0)
+async function buildMaf(q: GdcMafBuildRequest, res: any) {
+	const t0 = Date.now()
 
-	const outFile = path.join(serverconfig.cachedir, 'gdcMaf.' + Math.random().toString()) // should be a gzipped file. does it need to end with '.gz' or it's auto-added?
+	const fileLst2 = (await getFileLstUnderSizeLimit(q.fileIdLst)) as string[]
+	console.log('test gdc maf sizes', Date.now() - t0)
 
 	const arg = {
 		fileIdLst: fileLst2,
-		host: path.join(apihost, 'data'), // must use the /data/ endpoint from current host
-		outFile
+		host: path.join(apihost, 'data') // must use the /data/ endpoint from current host
 	}
 
-	await run_rust('gdcmaf', JSON.stringify(arg))
+	const dataGzipped = await run_rust('gdcmaf', JSON.stringify(arg))
 
-	console.log('rust gdcmaf', new Date() - t0)
+	console.log('rust gdcmaf', Date.now() - t0)
 
-	const data = await fs.promises.readFile(outFile)
+	const data = JSON.parse(dataGzipped)
 
 	// by directly returning a blob, it won't tell client how many files are used
-
 	res.writeHead(200, {
 		'Content-Type': 'application/octet-stream',
 		'Content-Disposition': 'attachment; filename=cohort.maf.gz',

package/routes/gdc.topVariablyExpressedGenes.ts

@@ -1,12 +1,15 @@
 import { GdcTopVariablyExpressedGenesResponse } from '#shared/types/routes/gdc.topVariablyExpressedGenes.ts'
 import { getCasesWithExressionDataFromCohort } from '../src/mds3.gdc.js'
-//import path from 'path'
+import path from 'path'
 import got from 'got'
 import serverconfig from '#src/serverconfig.js'
 
 // TODO change when api is released to prod
 //const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
 const apihost = 'https://uat-portal.gdc.cancer.gov/auth/api/v0/gene_expression/gene_selection'
+// temporarily hardcode to use the direct API URL,
+// previously hardcoded to use 'https://uat-portal.gdc.cancer.gov/auth/api/v0/'
+const geneExpHost = 'https://uat-api.gdc.cancer.gov'
 
 const gdcGenome = 'hg38'
 const gdcDslabel = 'GDC'
@@ -23,7 +26,7 @@ export const api = {
 						if (!genome) throw 'hg38 genome missing'
 						const ds = genome.datasets?.[gdcDslabel]
 						if (!ds) throw 'gdc dataset missing'
-						const genes = await getGenes(req.query, ds)
+						const genes = await getGenes(req.query, ds, genome)
 						const payload = { genes } as GdcTopVariablyExpressedGenesResponse
 						res.send(payload)
 					} catch (e: any) {
@@ -53,12 +56,16 @@ req.query {
 
 ds { } // server-side ds object
 
+genome {}
 */
-async function getGenes(q: any, ds: any) {
+async function getGenes(q: any, ds: any, genome: any) {
 	if (serverconfig.features.gdcGenes) {
 		// for testing only; delete when api issue is resolved
 		return serverconfig.features.gdcGenes as string[]
 	}
+	if (!ds.__gdc.doneCaching) {
+		throw `The server has not finished caching the case IDs: try again in ~2 minutes`
+	}
 
 	// based on current cohort, get list of cases with exp data, as input of next api query
 	const caseLst = await getCasesWithExressionDataFromCohort(q, ds)
@@ -68,30 +75,39 @@ async function getGenes(q: any, ds: any) {
 	}
 
 	// change to this when api is available on prod
-	// const url = path.join(apihost, '/gene_expression/gene_selection')
+	const url = path.join(geneExpHost, '/gene_expression/gene_selection')
 
-	const response = await got.post(apihost, {
-		headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
-		body: JSON.stringify({
-			case_ids: caseLst,
-			//gene_ids: [] // this should not be a required parameter
-			size: q.maxGenes || 100
+	try {
+		const response = await got.post(url, {
+			headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
+			body: JSON.stringify({
+				// !!! temporarily limit the case_ids length, otherwise the request times out !!!
+				case_ids: caseLst.slice(0, 20),
+				//gene_ids: [] // this should not be a required parameter
+				gene_type: 'protein_coding',
+				selection_size: Number(q.maxGenes || 100)
+			})
 		})
-	})
 
-	const re = JSON.parse(response.body)
-	// {"gene_selection":[{"gene_id":"ENSG00000141510","log2_uqfpkm_median":3.103430497010492,"log2_uqfpkm_stddev":0.8692021350485105,"symbol":"TP53"}, ... ]}
+		const re = JSON.parse(response.body)
+		// {"gene_selection":[{"gene_id":"ENSG00000141510","log2_uqfpkm_median":3.103430497010492,"log2_uqfpkm_stddev":0.8692021350485105,"symbol":"TP53"}, ... ]}
 
-	const genes = [] as string[]
-	if (!Array.isArray(re.gene_selection)) throw 're.gene_selection[] is not array'
-	for (const i of re.gene_selection) {
-		if (i.gene_id && typeof i.gene_id == 'string') {
-			genes.push(i.gene_id) // ensg
-		} else if (i.symbol && typeof i.symbol == 'string') {
-			genes.push(i.symbol)
-		} else {
-			throw 'one of re.gene_selection[] is missing both gene_id and symbol'
+		const genes = [] as string[]
+		if (!Array.isArray(re.gene_selection)) throw 're.gene_selection[] is not array'
+		for (const i of re.gene_selection) {
+			if (i.gene_id && typeof i.gene_id == 'string') {
+				// is ensg, convert to symbol
+				const t = genome.genedb.getNameByAlias.get(i.gene_id)
+				if (t) genes.push(t.name) // ensg
+			} else if (i.symbol && typeof i.symbol == 'string') {
+				genes.push(i.symbol)
+			} else {
+				throw 'one of re.gene_selection[] is missing both gene_id and symbol'
+			}
 		}
+		return genes
+	} catch (e) {
+		console.log(e.stack || e)
+		throw e
 	}
-	return genes
 }

package/routes/hicdata.ts

@@ -0,0 +1,113 @@
+import { HicdataRequest, HicdataResponse, Item } from '#shared/types/routes/hicdata.ts'
+import { fileurl } from '#src/utils.js'
+import { spawn } from 'child_process'
+import readline from 'readline'
+import serverconfig from '#src/serverconfig.js'
+
+export const api: any = {
+	endpoint: 'hicdata',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'HicdataRequest'
+			},
+			response: {
+				typeId: 'HicdataResponse'
+			}
+			/*
+			examples: [
+				{
+					request: {
+						body: {
+							genome: 'hg38-test',
+							dslabel: 'TermdbTest',
+							embedder: 'localhost',
+							gettermbyid: 'subcohort'
+						}
+					},
+					response: {
+						header: { status: 200 }
+					}
+				}
+			]
+			*/
+		},
+		post: {
+			alternativeFor: 'get',
+			init
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		try {
+			const payload = await handle_hicdata(req.query as HicdataRequest)
+			res.send(payload)
+		} catch (e) {
+			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+			// @ts-ignore
+			res.send({ error: e?.message || e })
+			if (e instanceof Error && e.stack) console.log(e)
+		}
+	}
+}
+
+function handle_hicdata(q: HicdataRequest) {
+	return new Promise((resolve, reject) => {
+		const [e, file, isurl] = fileurl({ query: q })
+		if (e) reject({ error: 'illegal file name' })
+
+		const par = [
+			// TODO add option for observed/oe
+			q.nmeth || 'NONE',
+			file,
+			q.pos1,
+			q.pos2,
+			q.isfrag ? 'FRAG' : 'BP',
+			q.resolution
+		]
+
+		const ps = spawn(serverconfig.hicstraw, par)
+		const rl = readline.createInterface({ input: ps.stdout })
+
+		const items = [] as Item[]
+		const erroutput = [] as string[]
+		let linenot3fields = 0
+		let fieldnotnumerical = 0
+
+		rl.on('line', line => {
+			// straw output: pos1 \t pos2 \t value
+			const l = line.split('\t')
+			if (l.length != 3) {
+				linenot3fields++
+				return
+			}
+			const n1 = Number.parseInt(l[0])
+			const n2 = Number.parseInt(l[1])
+			const v = Number.parseFloat(l[2])
+			if (Number.isNaN(n1) || Number.isNaN(n2) || Number.isNaN(v)) {
+				fieldnotnumerical++
+				return
+			}
+			if (q.mincutoff != undefined && v <= q.mincutoff) {
+				return
+			}
+			items.push([n1, n2, v] as Item)
+		})
+
+		ps.stderr.on('data', i => erroutput.push(i))
+		ps.on('close', () => {
+			const err = erroutput.join('')
+			if (err) reject({ error: err })
+
+			if (linenot3fields) reject({ error: linenot3fields + ' lines have other than 3 fields' })
+
+			if (fieldnotnumerical)
+				reject({ error: fieldnotnumerical + ' lines have non-numerical values in any of the 3 fields' })
+
+			resolve({ items })
+		})
+	})
+}

package/routes/hicstat.ts

@@ -0,0 +1,55 @@
+import { fileurl, file_is_readable } from '#src/utils.js'
+import { do_hicstat } from '#src/hicstat.ts'
+import { HicstatRequestWithValidation } from '#shared/types/routes/hicstat.ts'
+
+export const api = {
+	endpoint: 'hicstat',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'HicstatRequest'
+			},
+			response: {
+				typeId: 'HicstatResponse'
+			},
+			examples: [
+				{
+					request: {
+						body: {
+							genome: 'hg19',
+							file: 'proteinpaint_demo/hg19/hic/hic_demo.hic',
+							embedder: 'localhost'
+						}
+					},
+					response: {
+						header: { status: 200 }
+					}
+				}
+			]
+		},
+		post: {
+			alternativeFor: 'get',
+			init
+		}
+	}
+}
+
+function init() {
+	return async (req: HicstatRequestWithValidation, res: any): Promise<void> => {
+		try {
+			const [e, file, isurl] = fileurl(req)
+			if (e) throw 'illegal file name'
+			if (!isurl) {
+				await file_is_readable(file)
+			}
+			const out = await do_hicstat(file, isurl)
+			res.send({ out })
+		} catch (e: any) {
+			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+			// @ts-ignore
+			res.send({ error: e?.message || e })
+			if (e instanceof Error && e.stack) console.log(e)
+		}
+	}
+}

package/routes/termdb.categories.ts

@@ -87,7 +87,13 @@ function init({ genomes }) {
 	}
 }
 
-async function trigger_getcategories(q, res, tdb, ds, genome) {
+async function trigger_getcategories(
+	q: { tid: string | number; type: string; filter: any; term1_q: any; currentGeneNames: any },
+	res: { send: (arg0: { lst: any[]; orderedLabels: any }) => void },
+	tdb: { q: { termjsonByOneid: (arg0: any) => any } },
+	ds: { assayAvailability: { byDt: { [s: string]: any } | ArrayLike<any> } },
+	genome: any
+) {
 	// thin wrapper of get_summary
 	// works for all types of terms
 	if (!q.tid) throw '.tid missing'
@@ -105,9 +111,9 @@ async function trigger_getcategories(q, res, tdb, ds, genome) {
 	const data = await getData(arg, ds, genome)
 	if (data.error) throw data.error
 
-	const lst = []
+	const lst = [] as any[]
 	if (q.type == 'geneVariant') {
-		const samples = data.samples
+		const samples = data.samples as { [sampleId: string]: any }
 		const dtClassMap = new Map()
 		if (ds.assayAvailability?.byDt) {
 			for (const [dtType, dtValue] of Object.entries(ds.assayAvailability.byDt)) {
@@ -116,7 +122,7 @@ async function trigger_getcategories(q, res, tdb, ds, genome) {
 				}
 			}
 		}
-		const sampleCountedFor = new Set() // if the sample is conunted for the
+		const sampleCountedFor = new Set() // if the sample is counted
 		for (const [sampleId, sampleData] of Object.entries(samples)) {
 			const values = sampleData[q.tid].values
 			sampleCountedFor.clear()
@@ -175,7 +181,9 @@ async function trigger_getcategories(q, res, tdb, ds, genome) {
 				samplecount: count,
 				key,
 				label:
-					data.refs?.byTermId?.[q.tid]?.events?.find(e => e.event === key).label || term?.values?.[key]?.label || key
+					data.refs?.byTermId?.[q.tid]?.events?.find((e: { event: any }) => e.event === key).label ||
+					term?.values?.[key]?.label ||
+					key
 			})
 		}
 	}
@@ -195,7 +203,23 @@ async function trigger_getcategories(q, res, tdb, ds, genome) {
 	})
 }
 
-function getDefaultQ(term, q) {
+function getDefaultQ(
+	term: { type: string; bins: { default: any } },
+	q: {
+		mode?: any
+		breaks?: any
+		bar_by_grade?: any
+		bar_by_children?: any
+		value_by_max_grade?: any
+		value_by_most_recent?: any
+		value_by_computable_grade?: any
+		tid?: string | number
+		type?: string
+		filter?: any
+		term1_q?: any
+		currentGeneNames?: any
+	}
+) {
 	if (term.type == 'categorical') return {}
 	if (term.type == 'survival') return {}
 	if (term.type == 'integer' || term.type == 'float') return term.bins.default