@sjcrh/proteinpaint-server 2.30.5 → 2.32.2-0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.30.5",
+  "version": "2.32.2-0",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "server.js",
   "bin": "start.js",
@@ -25,6 +25,7 @@
     "pretest:integration": "tsc --esModuleInterop genome/*.ts dataset/*.ts",
     "test:integration": "echo 'TODO: server integration tests'",
     "prepack": "tsc --esModuleInterop genome/*.ts dataset/*.ts && webpack --env NODE_ENV=production",
+    "test:tsc": "tsc --esModuleInterop --noEmit --allowImportingTsExtensions ./shared/types/test/*.type.spec.ts",
     "response": "nodemon modules/test/test.server.js --watch src",
     "getconf": "../build/getConfigProp.js",
     "doc": "../augen/build.sh routes shared/types/routes shared/checkers ../public/docs/server"
@@ -55,8 +56,8 @@
     "webpack-notifier": "^1.15.0"
   },
   "dependencies": {
-    "@sjcrh/augen": "2.27.0",
-    "@sjcrh/proteinpaint-rust": "2.30.2",
+    "@sjcrh/augen": "2.32.2-0",
+    "@sjcrh/proteinpaint-rust": "2.31.0",
     "better-sqlite3": "^7.5.3",
     "body-parser": "^1.15.2",
     "canvas": "~2.9.3",

package/routes/gdc.mafBuild.ts

@@ -3,6 +3,8 @@ import path from 'path'
 import fs from 'fs'
 import { run_rust } from '@sjcrh/proteinpaint-rust'
 import serverconfig from '#src/serverconfig.js'
+import Readable from 'stream'
+import { GdcMafBuildRequest } from '#shared/types/routes/gdc.mafBuild.ts'
 
 const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
 const maxTotalSizeCompressed = serverconfig.features.gdcMafMaxFileSize || 50000000 // 50Mb
@@ -11,64 +13,56 @@ export const api = {
 	endpoint: 'gdc/mafBuild',
 	methods: {
 		all: {
-			init({ genomes }) {
-				return async (req: any, res: any): Promise<void> => {
-					try {
-						await buildMaf(req, res)
-					} catch (e: any) {
-						if (e.stack) console.log(e.stack)
-						res.send({ status: 'error', error: e.message || e })
-					}
-				}
-			},
+			init,
 			request: {
-				typeId: null
-				//valid: default to type checker
+				typeId: 'GdcMafBuildRequest'
 			},
 			response: {
-				typeId: 'GdcMafBuildResponse'
-				// will combine this with type checker
-				//valid: (t) => {}
+				typeId: null // 'GdcMafBuildResponse'
 			}
 		}
 	}
 }
 
-/*
-req.query {
-	fileIdLst []  // list of maf file uuids
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		try {
+			await buildMaf(req.query as GdcMafBuildRequest, res)
+		} catch (e: any) {
+			if (e.stack) console.log(e.stack)
+			res.send({ status: 'error', error: e.message || e })
+		}
+	}
 }
 
+/*
+q{}
 res{}
 */
-async function buildMaf(req: any, res: any) {
+async function buildMaf(q: GdcMafBuildRequest, res: any) {
 	const t0 = Date.now()
 
-	const fileLst2 = (await getFileLstUnderSizeLimit(req.query.fileIdLst)) as string[]
+	const fileLst2 = (await getFileLstUnderSizeLimit(q.fileIdLst)) as string[]
 	console.log('test gdc maf sizes', Date.now() - t0)
 
-	const outFile = path.join(serverconfig.cachedir, 'gdcMaf.' + Math.random().toString()) // should be a gzipped file. does it need to end with '.gz' or it's auto-added?
-
 	const arg = {
 		fileIdLst: fileLst2,
-		host: path.join(apihost, 'data'), // must use the /data/ endpoint from current host
-		outFile
+		host: path.join(apihost, 'data') // must use the /data/ endpoint from current host
 	}
 
-	await run_rust('gdcmaf', JSON.stringify(arg))
+	const dataGzipped = await run_rust('gdcmaf', JSON.stringify(arg))
 
 	console.log('rust gdcmaf', Date.now() - t0)
 
-	const data = await fs.promises.readFile(outFile)
+	const data = JSON.parse(dataGzipped)
 
 	// by directly returning a blob, it won't tell client how many files are used
-
 	res.writeHead(200, {
 		'Content-Type': 'application/octet-stream',
 		'Content-Disposition': 'attachment; filename=cohort.maf.gz',
 		'Content-Length': data.length
 	})
-	res.end(Buffer.from(data as any, 'binary'))
+	res.end(Buffer.from(data, 'binary'))
 }
 
 /*

package/routes/gdc.topVariablyExpressedGenes.ts

@@ -1,12 +1,15 @@
 import { GdcTopVariablyExpressedGenesResponse } from '#shared/types/routes/gdc.topVariablyExpressedGenes.ts'
 import { getCasesWithExressionDataFromCohort } from '../src/mds3.gdc.js'
-//import path from 'path'
+import path from 'path'
 import got from 'got'
 import serverconfig from '#src/serverconfig.js'
 
 // TODO change when api is released to prod
 //const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
 const apihost = 'https://uat-portal.gdc.cancer.gov/auth/api/v0/gene_expression/gene_selection'
+// temporarily hardcode to use the direct API URL,
+// previously hardcoded to use 'https://uat-portal.gdc.cancer.gov/auth/api/v0/'
+const geneExpHost = 'https://uat-api.gdc.cancer.gov'
 
 const gdcGenome = 'hg38'
 const gdcDslabel = 'GDC'
@@ -23,7 +26,7 @@ export const api = {
 						if (!genome) throw 'hg38 genome missing'
 						const ds = genome.datasets?.[gdcDslabel]
 						if (!ds) throw 'gdc dataset missing'
-						const genes = await getGenes(req.query, ds)
+						const genes = await getGenes(req.query, ds, genome)
 						const payload = { genes } as GdcTopVariablyExpressedGenesResponse
 						res.send(payload)
 					} catch (e: any) {
@@ -53,12 +56,16 @@ req.query {
 
 ds { } // server-side ds object
 
+genome {}
 */
-async function getGenes(q: any, ds: any) {
+async function getGenes(q: any, ds: any, genome: any) {
 	if (serverconfig.features.gdcGenes) {
 		// for testing only; delete when api issue is resolved
 		return serverconfig.features.gdcGenes as string[]
 	}
+	if (!ds.__gdc.doneCaching) {
+		throw `The server has not finished caching the case IDs: try again in ~2 minutes`
+	}
 
 	// based on current cohort, get list of cases with exp data, as input of next api query
 	const caseLst = await getCasesWithExressionDataFromCohort(q, ds)
@@ -68,30 +75,39 @@ async function getGenes(q: any, ds: any) {
 	}
 
 	// change to this when api is available on prod
-	// const url = path.join(apihost, '/gene_expression/gene_selection')
+	const url = path.join(geneExpHost, '/gene_expression/gene_selection')
 
-	const response = await got.post(apihost, {
-		headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
-		body: JSON.stringify({
-			case_ids: caseLst,
-			//gene_ids: [] // this should not be a required parameter
-			size: q.maxGenes || 100
+	try {
+		const response = await got.post(url, {
+			headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
+			body: JSON.stringify({
+				// !!! temporarily limit the case_ids length, otherwise the request times out !!!
+				case_ids: caseLst.slice(0, 20),
+				//gene_ids: [] // this should not be a required parameter
+				gene_type: 'protein_coding',
+				selection_size: Number(q.maxGenes || 100)
+			})
 		})
-	})
 
-	const re = JSON.parse(response.body)
-	// {"gene_selection":[{"gene_id":"ENSG00000141510","log2_uqfpkm_median":3.103430497010492,"log2_uqfpkm_stddev":0.8692021350485105,"symbol":"TP53"}, ... ]}
+		const re = JSON.parse(response.body)
+		// {"gene_selection":[{"gene_id":"ENSG00000141510","log2_uqfpkm_median":3.103430497010492,"log2_uqfpkm_stddev":0.8692021350485105,"symbol":"TP53"}, ... ]}
 
-	const genes = [] as string[]
-	if (!Array.isArray(re.gene_selection)) throw 're.gene_selection[] is not array'
-	for (const i of re.gene_selection) {
-		if (i.gene_id && typeof i.gene_id == 'string') {
-			genes.push(i.gene_id) // ensg
-		} else if (i.symbol && typeof i.symbol == 'string') {
-			genes.push(i.symbol)
-		} else {
-			throw 'one of re.gene_selection[] is missing both gene_id and symbol'
+		const genes = [] as string[]
+		if (!Array.isArray(re.gene_selection)) throw 're.gene_selection[] is not array'
+		for (const i of re.gene_selection) {
+			if (i.gene_id && typeof i.gene_id == 'string') {
+				// is ensg, convert to symbol
+				const t = genome.genedb.getNameByAlias.get(i.gene_id)
+				if (t) genes.push(t.name) // ensg
+			} else if (i.symbol && typeof i.symbol == 'string') {
+				genes.push(i.symbol)
+			} else {
+				throw 'one of re.gene_selection[] is missing both gene_id and symbol'
+			}
 		}
+		return genes
+	} catch (e: any) {
+		console.log(e.stack || e)
+		throw e
 	}
-	return genes
 }

package/routes/termdb.singlecellData.ts

@@ -0,0 +1,45 @@
+import {
+	TermdbSinglecellDataRequest,
+	TermdbSinglecellDataResponse
+} from '#shared/types/routes/termdb.singlecellData.ts'
+
+/*
+given a sample, return it's singlecell data from dataset
+*/
+
+export const api: any = {
+	endpoint: 'termdb/singlecellData',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'TermdbSinglecellDataRequest'
+			},
+			response: {
+				typeId: 'TermdbSinglecellDataResponse'
+			}
+		},
+		post: {
+			alternativeFor: 'get',
+			init
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		const q = req.query as TermdbSinglecellDataRequest
+		try {
+			const g = genomes[q.genome]
+			if (!g) throw 'invalid genome name'
+			const ds = g.datasets[q.dslabel]
+			if (!ds) throw 'invalid dataset name'
+			if (!ds.queries?.singleCell) throw 'no singlecell data on this dataset'
+			const result = (await ds.queries.singleCell.data.get(q)) as TermdbSinglecellDataResponse
+			res.send(result)
+		} catch (e: any) {
+			if (e instanceof Error && e.stack) console.log(e)
+			res.send({ error: e.message || e })
+		}
+	}
+}

package/routes/termdb.singlecellSamples.ts

@@ -0,0 +1,167 @@
+import fs from 'fs'
+import path from 'path'
+import { read_file } from '#src/utils.js'
+import serverconfig from '#src/serverconfig.js'
+import {
+	SingleCellQuery,
+	SingleCellSamplesNative,
+	SingleCellSamplesGdc,
+	SingleCellDataNative,
+	SingleCellDataGdc
+} from '#shared/types/dataset.ts'
+import {
+	Sample,
+	TermdbSinglecellsamplesRequest,
+	TermdbSinglecellsamplesResponse
+} from '#shared/types/routes/termdb.singlecellSamples.ts'
+import {
+	Cell,
+	Plot,
+	HasdataResponse,
+	NodataResponse,
+	ErrorResponse
+} from '#shared/types/routes/termdb.singlecellData.ts'
+import { gdc_validate_query_singleCell_samples, gdc_validate_query_singleCell_data } from '#src/mds3.gdc.js'
+
+/* route returns list of samples with sc data
+this is due to the fact that sometimes not all samples in a dataset has sc data
+*/
+
+export const api: any = {
+	endpoint: 'termdb/singlecellSamples',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'TermdbSinglecellsamplesRequest'
+			},
+			response: {
+				typeId: 'TermdbSinglecellsamplesResponse'
+			}
+		},
+		post: {
+			alternativeFor: 'get',
+			init
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		const q = req.query as TermdbSinglecellsamplesRequest
+		try {
+			const g = genomes[q.genome]
+			if (!g) throw 'invalid genome name'
+			const ds = g.datasets[q.dslabel]
+			if (!ds) throw 'invalid dataset name'
+			if (!ds.queries?.singleCell) throw 'no singlecell data on this dataset'
+			const samples = (await ds.queries.singleCell.samples.get(q)) as TermdbSinglecellsamplesResponse
+			res.send({ samples })
+		} catch (e: any) {
+			if (e instanceof Error && e.stack) console.log(e)
+			res.send({ error: e.message || e })
+		}
+	}
+}
+
+/////////////////// ds query validator
+export async function validate_query_singleCell(ds: any, genome: any) {
+	const q = ds.queries.singleCell as SingleCellQuery
+	if (!q) return
+
+	if (q.samples.src == 'gdcapi') {
+		gdc_validate_query_singleCell_samples(ds, genome)
+	} else {
+		validateSamplesNative(q.samples as SingleCellSamplesNative, ds)
+	}
+	// q.samples.get() added
+
+	if (q.data.src == 'gdcapi') {
+		gdc_validate_query_singleCell_data(ds, genome)
+	} else {
+		validateDataNative(q.data as SingleCellDataNative, ds)
+	}
+	// q.data.get() added
+}
+
+function validateSamplesNative(S: SingleCellSamplesNative, ds: any) {
+	// for now use this quick fix method to pull sample ids annotated by this term
+	// to support situation where not all samples from a dataset has sc data
+	const samples = [] as Sample[] // list of sample ids with sc data
+	const s = ds.cohort.termdb.q.getAllValues4term(S.isSampleTerm)
+	for (const id of s.keys()) {
+		samples.push({ sample: ds.cohort.termdb.q.id2sampleName(id) })
+	}
+	if (samples.length == 0) throw 'no sample with sc data'
+	// getter returns array of {name:<samplename>, files:[]} where files is gdc specific. each sample is an obj and allows to add ds-specific stuff
+	S.get = () => samples
+}
+
+function validateDataNative(D: SingleCellDataNative, ds: any) {
+	const nameSet = new Set() // guard against duplicating plot names
+	for (const plot of D.plots) {
+		if (nameSet.has(plot.name)) throw 'duplicate plot.name'
+		nameSet.add(plot.name)
+	}
+
+	// scoped and cached for runtime
+	const _terms = [] as any
+	const _tid2cellvalue = {} as any
+
+	for (const tid of D.termIds) {
+		const t = ds.cohort.termdb.q.termjsonByOneid(tid)
+		if (!t) throw 'invalid term id from queries.singleCell.data.termIds[]'
+		_terms.push(t)
+		_tid2cellvalue[tid] = ds.cohort.termdb.q.getAllValues4term(tid)
+	}
+	D.get = async sample => {
+		// if sample is int, may convert to string
+		try {
+			const tid2cellvalue = {}
+			for (const tid of D.termIds) tid2cellvalue[tid] = {} // k: cell id, v: cell value for this term
+
+			const plots = [] as Plot[] // given a sample name, collect every plot data for this sample and return
+			for (const plot of D.plots) {
+				const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, sample, plot.fileSuffix)
+				try {
+					await fs.promises.stat(tsvfile)
+				} catch (e: any) {
+					if (e.code == 'ENOENT') {
+						// no file found for this sample; allowed because sampleView tests if that sample has sc data or not
+						continue
+					}
+					if (e.code == 'EACCES') throw 'cannot read file, permission denied'
+					throw 'failed to load sc data file'
+				}
+				const lines = (await read_file(tsvfile)).trim().split('\n')
+				// 1st line is header
+				const cells = [] as Cell[]
+				for (let i = 1; i < lines.length; i++) {
+					// each line is a cell
+					const l = lines[i].split('\t')
+					const cellId = l[0],
+						x = Number(l[4]), // FIXME standardize, or define idx in plot
+						y = Number(l[5])
+					if (!cellId) throw 'cell id missing'
+					if (!Number.isFinite(x) || !Number.isFinite(y)) throw 'x/y not number'
+					cells.push({ cellId, x, y })
+
+					for (const tid of D.termIds) {
+						if (_tid2cellvalue[tid].has(cellId)) {
+							tid2cellvalue[tid][cellId] = _tid2cellvalue[tid].get(cellId)
+						}
+					}
+				}
+				plots.push({ name: plot.name, cells })
+			}
+			if (plots.length == 0) {
+				// no data available for this sample
+				return { nodata: true }
+			}
+			return { plots, terms: _terms, tid2cellvalue }
+		} catch (e: any) {
+			if (e.stack) console.log(e.stack)
+			return { error: e.message || e }
+		}
+	}
+}