@sjcrh/proteinpaint-server 2.31.0 → 2.33.0

package/genome/hg38.js

@@ -9,9 +9,9 @@ exports.default = {
     },
     termdbs: {
         msigdb: {
-            label: 'MSigDB',
+            label: 'MSigDB (2023.2.Hs)',
             cohort: {
-                db: { file: 'anno/msigdb/db' },
+                db: { file: 'anno/msigdb/db_2023.2.Hs' },
                 termdb: {
                     isGeneSetTermdb: true
                 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.31.0",
+  "version": "2.33.0",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "server.js",
   "bin": "start.js",
@@ -25,6 +25,7 @@
     "pretest:integration": "tsc --esModuleInterop genome/*.ts dataset/*.ts",
     "test:integration": "echo 'TODO: server integration tests'",
     "prepack": "tsc --esModuleInterop genome/*.ts dataset/*.ts && webpack --env NODE_ENV=production",
+    "test:tsc": "tsc --esModuleInterop --noEmit --allowImportingTsExtensions ./shared/types/test/*.type.spec.ts",
     "response": "nodemon modules/test/test.server.js --watch src",
     "getconf": "../build/getConfigProp.js",
     "doc": "../augen/build.sh routes shared/types/routes shared/checkers ../public/docs/server"
@@ -55,8 +56,8 @@
     "webpack-notifier": "^1.15.0"
   },
   "dependencies": {
-    "@sjcrh/augen": "2.27.0",
-    "@sjcrh/proteinpaint-rust": "2.31.0",
+    "@sjcrh/augen": "2.32.2-0",
+    "@sjcrh/proteinpaint-rust": "2.33.0",
     "better-sqlite3": "^7.5.3",
     "body-parser": "^1.15.2",
     "canvas": "~2.9.3",

package/routes/gdc.mafBuild.ts

@@ -1,7 +1,7 @@
 import got from 'got'
 import path from 'path'
 import fs from 'fs'
-import { run_rust } from '@sjcrh/proteinpaint-rust'
+import { run_rust_stream } from '@sjcrh/proteinpaint-rust'
 import serverconfig from '#src/serverconfig.js'
 import Readable from 'stream'
 import { GdcMafBuildRequest } from '#shared/types/routes/gdc.mafBuild.ts'
@@ -50,19 +50,21 @@ async function buildMaf(q: GdcMafBuildRequest, res: any) {
 		host: path.join(apihost, 'data') // must use the /data/ endpoint from current host
 	}
 
-	const dataGzipped = await run_rust('gdcmaf', JSON.stringify(arg))
+	const rustStream = run_rust_stream('gdcmaf', JSON.stringify(arg))
+	res.setHeader('Content-Type', 'application/octet-stream')
+	res.setHeader('Content-Disposition', 'attachment; filename=cohort.maf.gz')
+	rustStream.pipe(res)
 
 	console.log('rust gdcmaf', Date.now() - t0)
 
-	const data = JSON.parse(dataGzipped)
-
-	// by directly returning a blob, it won't tell client how many files are used
-	res.writeHead(200, {
-		'Content-Type': 'application/octet-stream',
-		'Content-Disposition': 'attachment; filename=cohort.maf.gz',
-		'Content-Length': data.length
+	rustStream.on('end', () => {
+		res.end()
+	})
+	rustStream.on('error', err => {
+		console.error(err)
+		res.statusCode = 500
+		res.end('Internal Server Error')
 	})
-	res.end(Buffer.from(data, 'binary'))
 }
 
 /*

package/routes/gdc.topVariablyExpressedGenes.ts

@@ -1,15 +1,21 @@
-import { GdcTopVariablyExpressedGenesResponse } from '#shared/types/routes/gdc.topVariablyExpressedGenes.ts'
+import {
+	GdcTopVariablyExpressedGenesRequest,
+	GdcTopVariablyExpressedGenesResponse
+} from '#shared/types/routes/gdc.topVariablyExpressedGenes.ts'
 import { getCasesWithExressionDataFromCohort } from '../src/mds3.gdc.js'
 import path from 'path'
 import got from 'got'
 import serverconfig from '#src/serverconfig.js'
 
+// TODO make it general purpose based on ds.queries.geneExpression.topVariablyExpressedGenes{}; wait till case/gene link changes are done
+
 // TODO change when api is released to prod
 //const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
 const apihost = 'https://uat-portal.gdc.cancer.gov/auth/api/v0/gene_expression/gene_selection'
 // temporarily hardcode to use the direct API URL,
 // previously hardcoded to use 'https://uat-portal.gdc.cancer.gov/auth/api/v0/'
 const geneExpHost = 'https://uat-api.gdc.cancer.gov'
+// https://github.com/NCI-GDC/gdcapi/blob/develop/openapi/gene-expression.yaml
 
 const gdcGenome = 'hg38'
 const gdcDslabel = 'GDC'
@@ -18,47 +24,37 @@ export const api = {
 	endpoint: 'gdc/topVariablyExpressedGenes',
 	methods: {
 		get: {
-			init({ genomes }) {
-				return async (req: any, res: any): Promise<void> => {
-					try {
-						// following logic requires hg38 gdc dataset
-						const genome = genomes[gdcGenome]
-						if (!genome) throw 'hg38 genome missing'
-						const ds = genome.datasets?.[gdcDslabel]
-						if (!ds) throw 'gdc dataset missing'
-						const genes = await getGenes(req.query, ds, genome)
-						const payload = { genes } as GdcTopVariablyExpressedGenesResponse
-						res.send(payload)
-					} catch (e: any) {
-						res.send({ status: 'error', error: e.message || e })
-					}
-				}
-			},
+			init,
 			request: {
-				typeId: null
-				//valid: default to type checker
+				typeId: 'GdcTopVariablyExpressedGenesRequest'
 			},
 			response: {
 				typeId: 'GdcTopVariablyExpressedGenesResponse'
-				// will combine this with type checker
-				//valid: (t) => {}
 			}
 		}
 	}
 }
 
-/*
-req.query {
-	filter0 // optional gdc GFF cohort filter, invisible and read only
-		FIXME should there be pp filter too?
-	maxGenes: int
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		try {
+			// following logic requires hg38 gdc dataset
+			const genome = genomes[gdcGenome]
+			if (!genome) throw 'hg38 genome missing'
+			const ds = genome.datasets?.[gdcDslabel]
+			if (!ds) throw 'gdc dataset missing'
+			const genes = await getGenes(req.query as GdcTopVariablyExpressedGenesRequest, ds, genome)
+			const payload = { genes } as GdcTopVariablyExpressedGenesResponse
+			res.send(payload)
+		} catch (e: any) {
+			res.send({ status: 'error', error: e.message || e })
+		}
+	}
 }
 
-ds { } // server-side ds object
-
-genome {}
-*/
-async function getGenes(q: any, ds: any, genome: any) {
+/*
+ */
+async function getGenes(q: GdcTopVariablyExpressedGenesRequest, ds: any, genome: any) {
 	if (serverconfig.features.gdcGenes) {
 		// for testing only; delete when api issue is resolved
 		return serverconfig.features.gdcGenes as string[]
@@ -83,9 +79,14 @@ async function getGenes(q: any, ds: any, genome: any) {
 			body: JSON.stringify({
 				// !!! temporarily limit the case_ids length, otherwise the request times out !!!
 				case_ids: caseLst.slice(0, 20),
-				//gene_ids: [] // this should not be a required parameter
-				gene_type: 'protein_coding',
-				selection_size: Number(q.maxGenes || 100)
+
+				// temporary!! restrict pool to cgc due to slow api. delete when new api is online
+				gene_ids: tempGetCGCgenes(genome),
+
+				// when gene_ids is deleted, enable this
+				//gene_type: 'protein_coding',
+
+				selection_size: Number(q.maxGenes) // FIXME it's defined as number but why it's string??
 			})
 		})
 
@@ -106,8 +107,22 @@ async function getGenes(q: any, ds: any, genome: any) {
 			}
 		}
 		return genes
-	} catch (e) {
+	} catch (e: any) {
 		console.log(e.stack || e)
 		throw e
 	}
 }
+
+function tempGetCGCgenes(genome: any) {
+	const lst = [] as string[] // list of ENSG ids from cgc genes
+	// don't think there's need to preparse genome.geneset, as this function is only temporary
+	for (const s of genome.geneset[0].lst) {
+		const a = genome.genedb.getAliasByName.all(s)
+		if (a) {
+			for (const b of a) {
+				if (b.alias.startsWith('ENSG')) lst.push(b.alias)
+			}
+		}
+	}
+	return lst
+}

package/routes/termdb.singlecellData.ts

@@ -0,0 +1,45 @@
+import {
+	TermdbSinglecellDataRequest,
+	TermdbSinglecellDataResponse
+} from '#shared/types/routes/termdb.singlecellData.ts'
+
+/*
+given a sample, return it's singlecell data from dataset
+*/
+
+export const api: any = {
+	endpoint: 'termdb/singlecellData',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'TermdbSinglecellDataRequest'
+			},
+			response: {
+				typeId: 'TermdbSinglecellDataResponse'
+			}
+		},
+		post: {
+			alternativeFor: 'get',
+			init
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		const q = req.query as TermdbSinglecellDataRequest
+		try {
+			const g = genomes[q.genome]
+			if (!g) throw 'invalid genome name'
+			const ds = g.datasets[q.dslabel]
+			if (!ds) throw 'invalid dataset name'
+			if (!ds.queries?.singleCell) throw 'no singlecell data on this dataset'
+			const result = (await ds.queries.singleCell.data.get(q)) as TermdbSinglecellDataResponse
+			res.send(result)
+		} catch (e: any) {
+			if (e instanceof Error && e.stack) console.log(e)
+			res.send({ error: e.message || e })
+		}
+	}
+}

package/routes/termdb.singlecellSamples.ts

@@ -1,7 +1,27 @@
+import fs from 'fs'
+import path from 'path'
+import { read_file } from '#src/utils.js'
+import serverconfig from '#src/serverconfig.js'
 import {
+	SingleCellQuery,
+	SingleCellSamplesNative,
+	SingleCellSamplesGdc,
+	SingleCellDataNative,
+	SingleCellDataGdc
+} from '#shared/types/dataset.ts'
+import {
+	Sample,
 	TermdbSinglecellsamplesRequest,
 	TermdbSinglecellsamplesResponse
 } from '#shared/types/routes/termdb.singlecellSamples.ts'
+import {
+	Cell,
+	Plot,
+	HasdataResponse,
+	NodataResponse,
+	ErrorResponse
+} from '#shared/types/routes/termdb.singlecellData.ts'
+import { gdc_validate_query_singleCell_samples, gdc_validate_query_singleCell_data } from '#src/mds3.gdc.js'
 
 /* route returns list of samples with sc data
 this is due to the fact that sometimes not all samples in a dataset has sc data
@@ -35,11 +55,113 @@ function init({ genomes }) {
 			const ds = g.datasets[q.dslabel]
 			if (!ds) throw 'invalid dataset name'
 			if (!ds.queries?.singleCell) throw 'no singlecell data on this dataset'
-			const result = (await ds.queries.singleCell.samples.get(q)) as TermdbSinglecellsamplesResponse
-			res.send(result)
+			const samples = (await ds.queries.singleCell.samples.get(q)) as TermdbSinglecellsamplesResponse
+			res.send({ samples })
 		} catch (e: any) {
 			if (e instanceof Error && e.stack) console.log(e)
 			res.send({ error: e.message || e })
 		}
 	}
 }
+
+/////////////////// ds query validator
+export async function validate_query_singleCell(ds: any, genome: any) {
+	const q = ds.queries.singleCell as SingleCellQuery
+	if (!q) return
+
+	if (q.samples.src == 'gdcapi') {
+		gdc_validate_query_singleCell_samples(ds, genome)
+	} else {
+		validateSamplesNative(q.samples as SingleCellSamplesNative, ds)
+	}
+	// q.samples.get() added
+
+	if (q.data.src == 'gdcapi') {
+		gdc_validate_query_singleCell_data(ds, genome)
+	} else {
+		validateDataNative(q.data as SingleCellDataNative, ds)
+	}
+	// q.data.get() added
+}
+
+function validateSamplesNative(S: SingleCellSamplesNative, ds: any) {
+	// for now use this quick fix method to pull sample ids annotated by this term
+	// to support situation where not all samples from a dataset has sc data
+	const samples = [] as Sample[] // list of sample ids with sc data
+	const s = ds.cohort.termdb.q.getAllValues4term(S.isSampleTerm)
+	for (const id of s.keys()) {
+		samples.push({ sample: ds.cohort.termdb.q.id2sampleName(id) })
+	}
+	if (samples.length == 0) throw 'no sample with sc data'
+	// getter returns array of {name:<samplename>, files:[]} where files is gdc specific. each sample is an obj and allows to add ds-specific stuff
+	S.get = () => samples
+}
+
+function validateDataNative(D: SingleCellDataNative, ds: any) {
+	const nameSet = new Set() // guard against duplicating plot names
+	for (const plot of D.plots) {
+		if (nameSet.has(plot.name)) throw 'duplicate plot.name'
+		nameSet.add(plot.name)
+	}
+
+	// scoped and cached for runtime
+	const _terms = [] as any
+	const _tid2cellvalue = {} as any
+
+	for (const tid of D.termIds) {
+		const t = ds.cohort.termdb.q.termjsonByOneid(tid)
+		if (!t) throw 'invalid term id from queries.singleCell.data.termIds[]'
+		_terms.push(t)
+		_tid2cellvalue[tid] = ds.cohort.termdb.q.getAllValues4term(tid)
+	}
+	D.get = async sample => {
+		// if sample is int, may convert to string
+		try {
+			const tid2cellvalue = {}
+			for (const tid of D.termIds) tid2cellvalue[tid] = {} // k: cell id, v: cell value for this term
+
+			const plots = [] as Plot[] // given a sample name, collect every plot data for this sample and return
+			for (const plot of D.plots) {
+				const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, sample, plot.fileSuffix)
+				try {
+					await fs.promises.stat(tsvfile)
+				} catch (e: any) {
+					if (e.code == 'ENOENT') {
+						// no file found for this sample; allowed because sampleView tests if that sample has sc data or not
+						continue
+					}
+					if (e.code == 'EACCES') throw 'cannot read file, permission denied'
+					throw 'failed to load sc data file'
+				}
+				const lines = (await read_file(tsvfile)).trim().split('\n')
+				// 1st line is header
+				const cells = [] as Cell[]
+				for (let i = 1; i < lines.length; i++) {
+					// each line is a cell
+					const l = lines[i].split('\t')
+					const cellId = l[0],
+						x = Number(l[4]), // FIXME standardize, or define idx in plot
+						y = Number(l[5])
+					if (!cellId) throw 'cell id missing'
+					if (!Number.isFinite(x) || !Number.isFinite(y)) throw 'x/y not number'
+					cells.push({ cellId, x, y })
+
+					for (const tid of D.termIds) {
+						if (_tid2cellvalue[tid].has(cellId)) {
+							tid2cellvalue[tid][cellId] = _tid2cellvalue[tid].get(cellId)
+						}
+					}
+				}
+				plots.push({ name: plot.name, cells })
+			}
+			if (plots.length == 0) {
+				// no data available for this sample
+				return { nodata: true }
+			}
+			return { plots, terms: _terms, tid2cellvalue }
+		} catch (e: any) {
+			if (e.stack) console.log(e.stack)
+			return { error: e.message || e }
+		}
+	}
+}