@sjcrh/proteinpaint-server 2.39.6 → 2.40.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.39.6",
+  "version": "2.40.1",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "server.js",
   "bin": "start.js",

package/routes/gdc.maf.ts

@@ -8,8 +8,6 @@ this route lists available gdc MAF files based on user's cohort filter
 and return them to client to be shown in a table for selection
 */
 
-const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov' // to switch to serverconfig export
-
 const maxFileNumber = 1000 // determines max number of files to return to client
 // preliminary testing:
 // 36s for 1000 (87Mb)
@@ -60,7 +58,7 @@ function init({ genomes }) {
 			const ds = g.datasets.GDC
 			if (!ds) throw 'hg38 GDC missing'
 
-			const payload = await listMafFiles(req.query as GdcMafRequest)
+			const payload = await listMafFiles(req.query as GdcMafRequest, ds)
 			res.send(payload)
 		} catch (e: any) {
 			res.send({ status: 'error', error: e.message || e })
@@ -80,7 +78,7 @@ ds {
 	}
 }
 */
-async function listMafFiles(q: GdcMafRequest) {
+async function listMafFiles(q: GdcMafRequest, ds) {
 	const filters = {
 		op: 'and',
 		content: [
@@ -90,15 +88,16 @@ async function listMafFiles(q: GdcMafRequest) {
 			{ op: '=', content: { field: 'access', value: 'open' } } // delete if later to support controlled files
 		]
 	}
-
+	const case_filters: any = { op: 'and', content: [] }
 	if (q.filter0) {
-		filters.content.push(q.filter0)
+		case_filters.content.push(q.filter0)
 	}
 
-	const headers = { 'Content-Type': 'application/json', Accept: 'application/json' }
+	const { host, headers } = ds.getHostHeaders(q)
 
 	const data = {
 		filters,
+		case_filters,
 		size: maxFileNumber,
 		fields: [
 			'id',
@@ -110,7 +109,7 @@ async function listMafFiles(q: GdcMafRequest) {
 		].join(',')
 	}
 
-	const response = await got.post(path.join(apihost, 'files'), { headers, body: JSON.stringify(data) })
+	const response = await got.post(path.join(host.rest, 'files'), { headers, body: JSON.stringify(data) })
 
 	let re
 	try {

package/routes/gdc.mafBuild.ts

@@ -7,8 +7,6 @@ import Readable from 'stream'
 import { GdcMafBuildRequest } from '#shared/types/routes/gdc.mafBuild.ts'
 import { maxTotalSizeCompressed } from './gdc.maf.ts'
 
-const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
-
 export const api = {
 	endpoint: 'gdc/mafBuild',
 	methods: {
@@ -27,7 +25,11 @@ export const api = {
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
 		try {
-			await buildMaf(req.query as GdcMafBuildRequest, res)
+			const g = genomes.hg38
+			if (!g) throw 'hg38 missing'
+			const ds = g.datasets.GDC
+			if (!ds) throw 'hg38 GDC missing'
+			await buildMaf(req.query as GdcMafBuildRequest, res, ds)
 		} catch (e: any) {
 			if (e.stack) console.log(e.stack)
 			res.send({ status: 'error', error: e.message || e })
@@ -39,10 +41,10 @@ function init({ genomes }) {
 q{}
 res{}
 */
-async function buildMaf(q: GdcMafBuildRequest, res: any) {
+async function buildMaf(q: GdcMafBuildRequest, res: any, ds) {
 	const t0 = Date.now()
-
-	const fileLst2 = (await getFileLstUnderSizeLimit(q.fileIdLst)) as string[]
+	const { host, headers } = ds.getHostHeaders(q)
+	const fileLst2 = (await getFileLstUnderSizeLimit(q.fileIdLst, host, headers)) as string[]
 
 	if (serverconfig.debugmode)
 		console.log(
@@ -53,7 +55,7 @@ async function buildMaf(q: GdcMafBuildRequest, res: any) {
 	const arg = {
 		fileIdLst: fileLst2,
 		columns: q.columns,
-		host: path.join(apihost, 'data') // must use the /data/ endpoint from current host
+		host: path.join(host.rest, 'data') // must use the /data/ endpoint from current host
 	}
 
 	const rustStream = run_rust_stream('gdcmaf', JSON.stringify(arg))
@@ -80,7 +82,7 @@ excess files are not processed in order not to crash server
 must not rely on file size sent by client, as that can be spoofed and never to be trusted
 it's inexpensive to query api for this
 */
-async function getFileLstUnderSizeLimit(lst: string[]) {
+async function getFileLstUnderSizeLimit(lst: string[], host, headers) {
 	if (lst.length == 0) throw 'fileIdLst[] not array or blank'
 	const data = {
 		filters: {
@@ -90,8 +92,7 @@ async function getFileLstUnderSizeLimit(lst: string[]) {
 		size: 10000,
 		fields: 'file_size'
 	}
-	const headers = { 'Content-Type': 'application/json', Accept: 'application/json' }
-	const response = await got.post(path.join(apihost, 'files'), { headers, body: JSON.stringify(data) })
+	const response = await got.post(path.join(host.rest, 'files'), { headers, body: JSON.stringify(data) })
 	let re
 	try {
 		re = JSON.parse(response.body)

package/routes/gdc.topMutatedGenes.ts

@@ -1,12 +1,10 @@
 import { GdcTopMutatedGeneRequest, GdcTopMutatedGeneResponse, Gene } from '#shared/types/routes/gdc.topMutatedGenes.ts'
 import { mclasscnvgain, mclasscnvloss, dtsnvindel } from '#shared/common.js'
 import got from 'got'
+import serverconfig from '#src/serverconfig.js'
 
 // TODO change to /termdb/topMutatedGenes
 
-const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
-const apihostGraphql = apihost + (apihost.includes('/v0') ? '' : '/v0') + '/graphql'
-
 export const api = {
 	endpoint: 'gdc/topMutatedGenes',
 	methods: {
@@ -22,7 +20,7 @@ export const api = {
 	}
 }
 
-function init() {
+function init({ genomes }) {
 	/*
 	genomes parameter is currently not used
 	could be used later to:
@@ -31,8 +29,12 @@ function init() {
 	*/
 	return async (req: any, res: any): Promise<void> => {
 		const q: GdcTopMutatedGeneRequest = req.query
+		const g = genomes.hg38
+		if (!g) throw 'hg38 missing'
+		const ds = g.datasets.GDC
+		if (!ds) throw 'hg38 GDC missing'
 		try {
-			const genes = await getGenesGraphql(q)
+			const genes = await getGenesGraphql(q, ds)
 			const payload: GdcTopMutatedGeneResponse = { genes }
 			res.send(payload)
 		} catch (e: any) {
@@ -41,7 +43,10 @@ function init() {
 	}
 }
 
-const query = `
+// query string copied from v1, works in v2. but v1 are not "cohort-centric and lacks the case_filters: $caseFilters"
+// delete the v1 after soft launch
+const queryV1: any = {
+	query: `
 query GenesTable_relayQuery(
   $genesTable_filters: FiltersArgument
   $genesTable_size: Int
@@ -111,131 +116,344 @@ query GenesTable_relayQuery(
     }
   }
 }
-`
-
-async function getGenesGraphql(q: GdcTopMutatedGeneRequest) {
-	// set type "any" to avoid complains
-	const variables: any = {
-		genesTable_filters: {
-			op: 'and',
-			content: []
-		},
-		genesTable_size: q.maxGenes || 50,
-		genesTable_offset: 0,
-		score: 'case.project.project_id',
-		ssmCase: {
-			op: 'and',
-			content: [
-				{
-					op: 'in',
-					content: {
-						field: 'cases.available_variation_data',
-						value: ['ssm']
+`,
+	getVariables: (q: any) => {
+		// set type "any" to avoid complains
+		const variables: any = {
+			genesTable_filters: { op: 'and', content: [] },
+			genesTable_size: q.maxGenes || 50,
+			genesTable_offset: 0,
+			score: 'case.project.project_id',
+			ssmCase: {
+				op: 'and',
+				content: [
+					{
+						op: 'in',
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['ssm']
+						}
+					},
+					{
+						op: 'NOT',
+						content: {
+							field: 'genes.case.ssm.observation.observation_id',
+							value: 'MISSING'
+						}
 					}
-				},
-				{
-					op: 'NOT',
-					content: {
-						field: 'genes.case.ssm.observation.observation_id',
-						value: 'MISSING'
+				]
+			},
+			geneCaseFilter: {
+				content: [
+					{
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['ssm']
+						},
+						op: 'in'
 					}
-				}
-			]
-		},
-		geneCaseFilter: {
-			content: [
-				{
-					content: {
-						field: 'cases.available_variation_data',
-						value: ['ssm']
-					},
-					op: 'in'
-				}
-			],
-			op: 'and'
-		},
-		ssmTested: {
-			content: [
-				{
-					content: {
-						field: 'cases.available_variation_data',
-						value: ['ssm']
-					},
-					op: 'in'
-				}
-			],
-			op: 'and'
-		},
-		cnvTested: {
-			op: 'and',
-			content: [
-				{
-					content: {
-						field: 'cases.available_variation_data',
-						value: ['cnv']
+				],
+				op: 'and'
+			},
+			ssmTested: {
+				content: [
+					{
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['ssm']
+						},
+						op: 'in'
+					}
+				],
+				op: 'and'
+			},
+			cnvTested: {
+				op: 'and',
+				content: [
+					{
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['cnv']
+						},
+						op: 'in'
+					}
+				]
+			},
+			cnvGainFilters: {
+				op: 'and',
+				content: [
+					{
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['cnv']
+						},
+						op: 'in'
 					},
-					op: 'in'
-				}
-			]
-		},
-		cnvGainFilters: {
-			op: 'and',
-			content: [
-				{
-					content: {
-						field: 'cases.available_variation_data',
-						value: ['cnv']
+					{
+						content: {
+							field: 'cnvs.cnv_change',
+							value: ['Gain']
+						},
+						op: 'in'
+					}
+				]
+			},
+			cnvLossFilters: {
+				op: 'and',
+				content: [
+					{
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['cnv']
+						},
+						op: 'in'
 					},
-					op: 'in'
-				},
-				{
-					content: {
-						field: 'cnvs.cnv_change',
-						value: ['Gain']
+					{
+						content: {
+							field: 'cnvs.cnv_change',
+							value: ['Loss']
+						},
+						op: 'in'
+					}
+				]
+			}
+		}
+
+		if (q.filter0) {
+			variables.genesTable_filters.content.push(JSON.parse(JSON.stringify(q.filter0)))
+			variables.geneCaseFilter.content.push(JSON.parse(JSON.stringify(q.filter0)))
+			variables.cnvTested.content.push(JSON.parse(JSON.stringify(q.filter0)))
+			variables.cnvGainFilters.content.push(JSON.parse(JSON.stringify(q.filter0)))
+			variables.cnvLossFilters.content.push(JSON.parse(JSON.stringify(q.filter0)))
+		}
+
+		if (q.geneFilter == 'CGC') {
+			variables.genesTable_filters.content.push(geneCGC())
+			variables.geneCaseFilter.content.push(geneCGC())
+			variables.cnvTested.content.push(geneCGC())
+			variables.cnvGainFilters.content.push(geneCGC())
+			variables.cnvLossFilters.content.push(geneCGC())
+		}
+		return variables
+	}
+}
+
+const queryV2: any = {
+	query: `
+          query GenesTable(
+            $caseFilters: FiltersArgument
+            $genesTable_filters: FiltersArgument
+            $genesTable_size: Int
+            $genesTable_offset: Int
+            $score: String
+            $ssmCase: FiltersArgument
+            $geneCaseFilter: FiltersArgument
+            $ssmTested: FiltersArgument
+            $cnvTested: FiltersArgument
+            $cnvGainFilters: FiltersArgument
+            $cnvLossFilters: FiltersArgument
+            $sort: [Sort]
+          ) {
+            genesTableViewer: viewer {
+              explore {
+                cases {
+                  hits(first: 0, case_filters: $ssmTested) {
+                    total
+                  }
+                }
+                filteredCases: cases {
+                  hits(first: 0, case_filters: $geneCaseFilter) {
+                    total
+                  }
+                }
+                cnvCases: cases {
+                  hits(first: 0, case_filters: $cnvTested) {
+                    total
+                  }
+                }
+                genes {
+                  hits(
+                    first: $genesTable_size
+                    offset: $genesTable_offset
+                    filters: $genesTable_filters
+                    case_filters: $caseFilters
+                    score: $score
+                    sort: $sort
+                  ) {
+                    total
+                    edges {
+                      node {
+                        id
+                        numCases: score
+                        symbol
+                        name
+                        cytoband
+                        biotype
+                        gene_id
+                        is_cancer_gene_census
+                        ssm_case: case {
+                          hits(first: 0, filters: $ssmCase) {
+                            total
+                          }
+                        }
+                        cnv_case: case {
+                          hits(first: 0, filters: $cnvTested) {
+                            total
+                          }
+                        }
+                        case_cnv_gain: case {
+                          hits(first: 0, filters: $cnvGainFilters) {
+                            total
+                          }
+                        }
+                        case_cnv_loss: case {
+                          hits(first: 0, filters: $cnvLossFilters) {
+                            total
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }`,
+	getVariables: (q: any) => {
+		const variables: any = {
+			caseFilters: { op: 'and', content: [] },
+			genesTable_filters: { op: 'and', content: [] },
+			genesTable_size: q.maxGenes || 50,
+			genesTable_offset: 0,
+			score: 'case.project.project_id',
+			ssmCase: {
+				op: 'and',
+				content: [
+					{
+						op: 'in',
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['ssm']
+						}
 					},
-					op: 'in'
-				}
-			]
-		},
-		cnvLossFilters: {
-			op: 'and',
-			content: [
-				{
-					content: {
-						field: 'cases.available_variation_data',
-						value: ['cnv']
+					{
+						op: 'NOT',
+						content: {
+							field: 'genes.case.ssm.observation.observation_id',
+							value: 'MISSING'
+						}
+					}
+				]
+			},
+			geneCaseFilter: {
+				content: [
+					{
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['ssm']
+						},
+						op: 'in'
+					}
+				],
+				op: 'and'
+			},
+			ssmTested: {
+				content: [
+					{
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['ssm']
+						},
+						op: 'in'
+					}
+				],
+				op: 'and'
+			},
+			cnvTested: {
+				op: 'and',
+				content: [
+					{
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['cnv']
+						},
+						op: 'in'
+					}
+				]
+			},
+			cnvGainFilters: {
+				op: 'and',
+				content: [
+					{
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['cnv']
+						},
+						op: 'in'
 					},
-					op: 'in'
-				},
-				{
-					content: {
-						field: 'cnvs.cnv_change',
-						value: ['Loss']
+					{
+						content: {
+							field: 'cnvs.cnv_change',
+							value: ['Gain']
+						},
+						op: 'in'
+					}
+				]
+			},
+			cnvLossFilters: {
+				op: 'and',
+				content: [
+					{
+						content: {
+							field: 'cases.available_variation_data',
+							value: ['cnv']
+						},
+						op: 'in'
 					},
-					op: 'in'
-				}
-			]
+					{
+						content: {
+							field: 'cnvs.cnv_change',
+							value: ['Loss']
+						},
+						op: 'in'
+					}
+				]
+			}
 		}
-	}
 
-	if (q.filter0) {
-		variables.genesTable_filters.content.push(JSON.parse(JSON.stringify(q.filter0)))
-		variables.geneCaseFilter.content.push(JSON.parse(JSON.stringify(q.filter0)))
-		variables.cnvTested.content.push(JSON.parse(JSON.stringify(q.filter0)))
-		variables.cnvGainFilters.content.push(JSON.parse(JSON.stringify(q.filter0)))
-		variables.cnvLossFilters.content.push(JSON.parse(JSON.stringify(q.filter0)))
+		if (q.filter0) {
+			variables.caseFilters.content.push(JSON.parse(JSON.stringify(q.filter0)))
+			variables.geneCaseFilter.content.push(JSON.parse(JSON.stringify(q.filter0)))
+			variables.cnvLossFilters.content.push(JSON.parse(JSON.stringify(q.filter0)))
+			variables.cnvGainFilters.content.push(JSON.parse(JSON.stringify(q.filter0)))
+			variables.cnvTested.content.push(JSON.parse(JSON.stringify(q.filter0)))
+		}
+
+		if (q.geneFilter == 'CGC') {
+			variables.genesTable_filters.content.push(geneCGC())
+			variables.cnvLossFilters.content.push(geneCGC())
+			variables.cnvGainFilters.content.push(geneCGC())
+		}
+		return variables
 	}
+}
 
-	if (q.geneFilter == 'CGC') {
-		variables.genesTable_filters.content.push(geneCGC())
-		variables.geneCaseFilter.content.push(geneCGC())
-		variables.cnvTested.content.push(geneCGC())
-		variables.cnvGainFilters.content.push(geneCGC())
-		variables.cnvLossFilters.content.push(geneCGC())
+async function getGenesGraphql(q: GdcTopMutatedGeneRequest, ds) {
+	let query: string, variables: object
+	const { host, headers } = ds.getHostHeaders(q)
+
+	// TODO: change this condition to use host.geneExp != host.rest ???
+	if (serverconfig.features?.geneExpHost) {
+		// quick fix! this is only set on local dev machines, meaning it's using v1 api; delete this after soft launch!!
+		query = queryV1.query
+		variables = queryV1.getVariables(q)
+	} else {
+		// this is not set and should be in qa-orange. it should be like this after softlaunch
+		query = queryV2.query
+		variables = queryV2.getVariables(q)
 	}
 
-	const response = await got.post(apihostGraphql, {
-		headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
+	const response = await got.post(host.graphql, {
+		headers,
 		body: JSON.stringify({ query, variables })
 	})
 
@@ -294,7 +512,7 @@ this api only gets ssm-cases and does not account for cnv cases, will not return
 thus is replaced by getGenesGraphql
 async function getGenes(q: GdcTopMutatedGeneRequest) {
 	const _f = q.filter0 || { op: 'and', content: [] } // allow blank filter to test geneset edit ui (without filter)
-	const response = await got.post(path.join(apihost, '/analysis/top_mutated_genes_by_project'), {
+	const response = await got.post(path.join(host.rest, '/analysis/top_mutated_genes_by_project'), {
 		headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
 		body: JSON.stringify({
 			size: q.maxGenes || 50,

package/routes/hicdata.ts

@@ -86,7 +86,7 @@ function handle_hicdata(q: HicdataRequest) {
 			}
 			const n1 = Number.parseInt(l[0])
 			const n2 = Number.parseInt(l[1])
-			const v = Number.parseFloat(l[2])
+			const v = q.matrixType == 'oe' ? Math.log(Number.parseFloat(l[2])) : Number.parseFloat(l[2])
 			if (Number.isNaN(n1) || Number.isNaN(n2) || Number.isNaN(v)) {
 				fieldnotnumerical++
 				return

package/routes/termdb.singleSampleMutation.ts

@@ -25,7 +25,7 @@ export const api: any = {
 
 function init({ genomes }) {
 	return async (req: any, res: any): Promise<void> => {
-		const q = req.query as TermdbSingleSampleMutationRequest
+		const q: TermdbSingleSampleMutationRequest = req.query
 		let result
 		try {
 			const g = genomes[q.genome]

package/routes/termdb.topVariablyExpressedGenes.ts

@@ -2,7 +2,7 @@ import {
 	TermdbTopVariablyExpressedGenesRequest,
 	TermdbTopVariablyExpressedGenesResponse
 } from '#shared/types/routes/termdb.topVariablyExpressedGenes.ts'
-import { gdcGetCasesWithExressionDataFromCohort, apihost, geneExpHost } from '../src/mds3.gdc.js'
+import { gdcGetCasesWithExressionDataFromCohort } from '../src/mds3.gdc.js'
 import path from 'path'
 import { run_rust } from '@sjcrh/proteinpaint-rust'
 import got from 'got'
@@ -142,11 +142,11 @@ function gdcValidateQuery(ds: any, genome: any) {
 		}
 
 		// change to this when api is available on prod
-		const url = path.join(geneExpHost, '/gene_expression/gene_selection')
-
+		const { host, headers } = ds.getHostHeaders(q)
+		const url = path.join(host.geneExp, '/gene_expression/gene_selection')
 		try {
 			const response = await got.post(url, {
-				headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
+				headers,
 				body: JSON.stringify(getGeneSelectionArg(q, caseLst))
 			})