@sjcrh/proteinpaint-server 2.26.0 → 2.27.0

package/genome/hg19.js

@@ -1,6 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-var clinvar_1 = require("../dataset/clinvar");
 exports.default = {
     species: 'human',
     genomefile: 'genomes/hg19.gz',
@@ -22,11 +21,6 @@ exports.default = {
         db: 'utils/meme/motif_databases/HUMAN/HOCOMOCOv11_full_HUMAN_mono_meme_format.meme',
         annotationfile: 'utils/meme/motif_databases/HUMAN/HOCOMOCOv11_full_annotation_HUMAN_mono.tsv'
     },
-    clinvarVCF: {
-        file: 'hg19/clinvar.hg19.hgvs_short.vep.bcf.gz',
-        infokey: 'CLNSIG',
-        categories: clinvar_1.clinsig
-    },
     tracks: [
         {
             __isgene: true,

package/genome/hg38.js

@@ -1,6 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-var clinvar_1 = require("../dataset/clinvar");
 var cgc_js_1 = require("./cgc.js");
 exports.default = {
     species: 'human',
@@ -30,11 +29,6 @@ exports.default = {
         db: 'utils/meme/motif_databases/HUMAN/HOCOMOCOv11_full_HUMAN_mono_meme_format.meme',
         annotationfile: 'utils/meme/motif_databases/HUMAN/HOCOMOCOv11_full_annotation_HUMAN_mono.tsv'
     },
-    clinvarVCF: {
-        file: 'hg38/clinvar.hg38.hgvs_short.vep.bcf.gz',
-        infokey: 'CLNSIG',
-        categories: clinvar_1.clinsig
-    },
     tracks: [
         {
             __isgene: true,

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.26.0",
+  "version": "2.27.0",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "server.js",
   "bin": "start.js",
   "imports": {
-    "#shared/*": "./shared/*.js",
+    "#shared/*": "./shared/*",
+    "#shared/types/*": "./shared/types/*",
     "#src/*": "./src/*",
     "#routes/*": "./routes/*"
   },
@@ -15,17 +16,18 @@
     "start": "tsc --esModuleInterop genome/*.ts dataset/*.ts && nodemon --enable-source-maps server.js --watch ./server*.js* --watch dataset/*.ts --watch genome/*.ts",
     "pretest": "tsc --esModuleInterop genome/*.ts dataset/*.ts && ./test/pretest.js",
     "test": "webpack --env NODE_ENV=test exportsFilename=all-test-context.js && node --enable-source-maps test/serverTests.js",
-    "pretest:unit": "npm run checkers && tsc genome/hg38.test.ts",
-    "test:unit": "webpack --env NODE_ENV=test exportsFilename=unit-test-context.js && tsc --esModuleInterop dataset/*.ts && node --enable-source-maps test/serverTests.js",
+    "precheckers": "tsc --esModuleInterop genome/*.ts dataset/*.ts",
+    "checkers": "webpack --config=./test/webpack.config.js && node test/emitPrepFiles.bundle.js && typia generate --input shared/checkers-raw --output shared/checkers",
+    "pretest:type": "npm run checkers",
+    "test:type": "webpack --env NODE_ENV=test exportsFilename=type-test-context.js && node --enable-source-maps test/serverTests.js",
+    "pretest:unit": "npm run precheckers",
+    "test:unit": "webpack --env NODE_ENV=test exportsFilename=unit-test-context.js && node --enable-source-maps test/serverTests.js",
     "pretest:integration": "tsc --esModuleInterop genome/*.ts dataset/*.ts",
     "test:integration": "echo 'TODO: server integration tests'",
     "prepack": "tsc --esModuleInterop genome/*.ts dataset/*.ts && webpack --env NODE_ENV=production",
     "response": "nodemon modules/test/test.server.js --watch src",
     "getconf": "../build/getConfigProp.js",
-    "predoc": "npm run checkers",
-    "doc": "typedoc --json ../public/docs/server.json",
-    "precheckers": "npx ts-node ./augen/cli.js typeCheckers $PWD/routes ../../../routes > shared/checkers/raw/index.ts",
-    "checkers": "typia generate --input shared/checkers/raw --output shared/checkers/transformed --project ./shared/checkers/tsconfig.json"
+    "doc": "../augen/build.sh routes shared/types/routes shared/checkers ../public/docs/server"
   },
   "author": "",
   "license": "SEE LICENSE IN ./LICENSE",
@@ -45,7 +47,6 @@
     "ts-node": "^10.9.1",
     "ts-patch": "^3.0.2",
     "typedoc": "^0.24.8",
-    "typedoc-plugin-replace-text": "^3.1.0",
     "typescript": "^5.0.3",
     "typia": "^4.1.14",
     "webpack": "^5.76.0",
@@ -54,7 +55,8 @@
     "webpack-notifier": "^1.15.0"
   },
   "dependencies": {
-    "@sjcrh/proteinpaint-rust": "2.26.0",
+    "@sjcrh/augen": "2.27.0",
+    "@sjcrh/proteinpaint-rust": "2.27.0",
     "better-sqlite3": "^7.5.3",
     "body-parser": "^1.15.2",
     "canvas": "~2.9.3",

package/routes/README.md

@@ -0,0 +1,84 @@
+# Server Routes
+
+## Introduction
+
+This directory contains files that specify server route APIs. By following this guidelines,
+the auto-generation of server routes, tests, and API documentation will work as expected.
+
+## Guidelines
+
+### 1. Use Express to do most of the routing logic
+
+- decentralize the route handling code into smaller, independent handler functions
+- common request processing logic, like genome, dataset, termdb set-up should be imported
+from a shared helper module that is common to a group of routes, or for more advanced cases,
+moved to a [router-level middleware](https://expressjs.com/en/guide/using-middleware.html#middleware.router)
+
+### 2. Export an `api` from the route file
+
+Use the code from other files in this directory as examples
+
+TODO: define the `api` type
+
+```ts
+// work-in-progress
+type RouteApi {
+	[key as methods]: RouteApiMethod
+}
+
+type methods = 'get' | 'post'
+
+type initArg = {
+	app?: any // Express app instance
+	genome: any // `Genome` from shared/types/genome.ts 
+}
+
+/**
+	@param
+*/
+type RouteApiMethod = {
+	endpoint: string
+	init: (initArg) => void
+	request: {
+		typeId: string
+		body?: any // specific to the route
+	}
+	response: {
+		typeId: string
+		header?: {
+			status: number
+		}
+		body?: any // specific to the route
+	}
+	examples: RouteExample[]
+}
+
+type RouteExample = {
+	request: {
+		body?: any
+	}
+	response?: {
+		header: {
+			status: number
+		}
+		body?: any 
+	}
+}
+```
+### 3. Use the appropriate [HTTP response code](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status)
+
+This is a best practice especially for error responses. Use `res.status(code)` to set the error code.
+This convention helps with error troubleshooting. Examples:
+- Status `400` 'Bad Request', something is wrong with the http request payload
+- Status `401` 'Unauthorized', the user must authenticate. The `server/src/auth.js` sets this status code
+- Status `403` 'Forbidden', the user is authenticated/signed-in, but is not permitted to access the requested data
+- Status `404` 'Not Found' for genome, dataset, or other data that is not found
+- Status `500` 'Server Error' for errors related to the server process or host machine, such as the GDC API
+not being available. Do not use code=`500` for errors that are related to specific request handler or data processing functions.
+
+### 4. Auto-generate
+
+- the server code will detect the routes in `server/src/run.sh`
+- `npm run doc` to see the documented routes in http://localhost:3000/server.html
+- `./augen/readme.sh > public/docs/readme.json` for content in http://localhost:3000/readme.html
+

package/routes/burden.ts

@@ -0,0 +1,140 @@
+import { BurdenRequest, BurdenResponse } from '#shared/types/routes/burden.ts'
+import lines2R from '#src/lines2R.js'
+import path from 'path'
+import serverconfig from '#src/serverconfig.js'
+import { write_file } from '#src/utils.js'
+
+export const api = {
+	endpoint: 'burden',
+	methods: {
+		get: {
+			init({ genomes }) {
+				return async (req: any, res: any): Promise<void> => {
+					try {
+						const genome = genomes[req.query.genome]
+						if (!genome) throw `invalid q.genome=${req.query.genome}`
+						const q = req.query as BurdenRequest
+						const ds = genome.datasets[q.dslabel]
+						if (!ds) throw `invalid q.genome=${req.query.dslabel}`
+						if (!ds.cohort.cumburden?.files) throw `missing ds.cohort.cumburden.files`
+
+						const estimates = await getBurdenEstimates(req, ds)
+						const { keys, rows } = formatPayload(estimates)
+						res.send({ status: 'ok', keys, rows } as BurdenResponse)
+					} catch (e: any) {
+						res.send({ status: 'error', error: e.message || e })
+					}
+				}
+			},
+			request: {
+				typeId: 'BurdenRequest'
+			},
+			response: {
+				typeId: 'BurdenResponse'
+			},
+			examples: [
+				{
+					request: {
+						body: {
+							genome: 'hg38',
+							// TODO: !!! use hg38-test and TermdbTest !!!
+							dslabel: 'SJLife',
+							diaggrp: 5,
+							sex: 1,
+							white: 1,
+							agedx: 1,
+							bleo: 0,
+							etop: 0,
+							cisp: 0,
+							carbo: 0,
+							steriod: 0,
+							vcr: 0,
+							hdmtx: 0,
+							itmt: 0,
+							ced: 0,
+							dox: 0,
+							heart: 0,
+							brain: 0,
+							abd: 0,
+							pelvis: 0,
+							chest: 0
+						}
+					},
+					response: {
+						header: { status: 200 }
+					}
+				}
+			]
+		}
+	}
+}
+
+async function getBurdenEstimates(q, ds) {
+	const infile = path.join(serverconfig.cachedir, Math.random().toString() + '.json')
+	for (const k in q.query) {
+		q.query[k] = Number(q.query[k])
+	}
+	const data = Object.assign({}, defaults, q.query)
+	//console.log(40, data, JSON.stringify(data))
+	await write_file(infile, JSON.stringify(data))
+	// TODO: use the dataset location
+	const { fit, surv, sample } = ds.cohort.cumburden.files
+	if (!fit || !surv || !sample) throw `missing one or more of ds.cohort.burden.files.{fit, surv, sample}`
+	const args = [
+		infile,
+		`${serverconfig.tpmasterdir}/${fit}`,
+		`${serverconfig.tpmasterdir}/${surv}`,
+		`${serverconfig.tpmasterdir}/${sample}`
+	]
+	const Routput = await lines2R(path.join(serverconfig.binpath, 'utils/burden.R'), [], args)
+	const estimates = JSON.parse(Routput[0])
+	return estimates
+}
+
+function formatPayload(estimates) {
+	const rawKeys = Object.keys(estimates[0])
+	const outKeys = [] as string[]
+	const keys = [] as string[]
+	for (const k of rawKeys) {
+		if (k == 'chc') {
+			keys.push(k)
+			outKeys.push(k)
+		} else {
+			const age = Number(k.slice(1).split(',')[0])
+			if (age <= 60 && age % 2 == 0) {
+				keys.push(k)
+				outKeys.push(`burden${age}`)
+			}
+		}
+	}
+	const rows = [] as number[][]
+	// v = an array of objects with age as keys as cumulative burden as value for a given CHC
+	for (const v of estimates) {
+		rows.push(keys.map(k => v[k]))
+	}
+	return { keys: outKeys, rows }
+}
+
+const defaults = Object.freeze({
+	diaggrp: 5,
+	sex: 0,
+	white: 1,
+	agedx: 1,
+	// chemotherapy
+	steriod: 0,
+	bleo: 0,
+	vcr: 0, //12, // Vincristine
+	etop: 0, //2500, // Etoposide
+	itmt: 0, // Intrathecal methothrexate_grp: 0,
+	ced: 0, //1.6, // Cyclophosphamide, 0.7692 mean 7692.
+	cisp: 0, //300, // Cisplatin
+	dox: 0, // Anthracycline, 3 mean 300 ml/m2
+	carbo: 0, //  Carboplatin
+	hdmtx: 0, // High-Dose Methotrexate
+	// radiation
+	brain: 0, //5.4,
+	chest: 0, //2.4,
+	heart: 0,
+	pelvis: 0,
+	abd: 0 //2.4
+})

package/routes/gdcMaf.ts

@@ -0,0 +1,106 @@
+import { GdcMafResponse, File } from '#shared/types/routes/gdcMaf.ts'
+import { fileSize } from '#shared/fileSize.js'
+import path from 'path'
+import got from 'got'
+
+const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
+
+export const api = {
+	endpoint: 'gdcMaf',
+	methods: {
+		get: {
+			init({ genomes }) {
+				// genomes parameter is not used
+				// could be used later to verify hg38/GDC is on this instance and otherwise disable this route..
+
+				return async (req: any, res: any): Promise<void> => {
+					try {
+						const files = await listMafFiles(req)
+						const payload = { files } as GdcMafResponse
+						res.send(payload)
+					} catch (e: any) {
+						res.send({ status: 'error', error: e.message || e })
+					}
+				}
+			},
+			request: {
+				typeId: null
+				//valid: default to type checker
+			},
+			response: {
+				typeId: 'GdcMafResponse'
+				// will combine this with type checker
+				//valid: (t) => {}
+			}
+		}
+	}
+}
+
+/*
+req.query {
+	filter0 // optional gdc GFF cohort filter, invisible and read only
+}
+*/
+async function listMafFiles(req: any) {
+	const filters = {
+		op: 'and',
+		content: [
+			{
+				op: '=',
+				content: { field: 'data_format', value: 'MAF' }
+			}
+		]
+	}
+
+	if (req.query.filter0) {
+		filters.content.push(req.query.filter0)
+	}
+
+	const headers = { 'Content-Type': 'application/json', Accept: 'application/json' }
+
+	const data = {
+		filters,
+		size: 1000,
+		fields: [
+			'id',
+			'file_size',
+			'experimental_strategy',
+			'cases.submitter_id', // used when listing all cases & files
+			//'associated_entities.entity_submitter_id', // semi human readable
+			//'associated_entities.case_id', // case uuid
+			'cases.samples.sample_type',
+			'analysis.workflow_type' // to drop out those as skip_workflow_type
+		].join(',')
+	}
+
+	const response = await got.post(path.join(apihost, 'files'), { headers, body: JSON.stringify(data) })
+
+	let re
+	try {
+		re = JSON.parse(response.body)
+	} catch (e) {
+		throw 'invalid JSON from ' + api.endpoint
+	}
+	if (!Array.isArray(re.data?.hits)) throw 're.data.hits[] not array'
+
+	// flatten api return to table row objects
+	// it is possible to set a max size limit to limit the number of files passed to client
+	const files = [] as File[]
+	for (const h of re.data.hits) {
+		const file = {
+			id: h.id,
+			workflow_type: h.analysis?.workflow_type,
+			experimental_strategy: h.experimental_strategy,
+			file_size: fileSize(h.file_size)
+		} as File
+		const c = h.cases?.[0]
+		if (c) {
+			file.case_submitter_id = c.submitter_id
+			if (c.samples) {
+				file.sample_types = c.samples.map(i => i.sample_type).join(', ')
+			}
+		}
+		files.push(file)
+	}
+	return files
+}

package/routes/genelookup.ts

@@ -1,11 +1,14 @@
 import { getResult } from '#src/gene.js'
+import { GeneLookupRequest, GeneLookupResponse } from '#shared/types/routes/genelookup.ts'
 
 function init({ genomes }) {
 	return (req: any, res: any): void => {
 		try {
+			const q = req.query as GeneLookupRequest
 			const g = genomes[req.query.genome]
 			if (!g) throw 'invalid genome name'
-			res.send(getResult(g, req.query))
+			const result = getResult(g, req.query) as GeneLookupResponse
+			res.send(result)
 		} catch (e: any) {
 			res.send({ error: e.message || e })
 			if (e.stack) console.log(e.stack)
@@ -45,14 +48,3 @@ export const api: any = {
 		}
 	}
 }
-
-export type GeneLookupRequest = {
-	input: string
-	genome: string
-	deep: boolean
-}
-
-export type GeneLookupResponse = {
-	error?: string
-	hits: string[]
-}

package/routes/healthcheck.ts

@@ -1,13 +1,14 @@
 import { getStat } from '#src/health.ts'
+import { HealthCheckResponse } from '#shared/types/routes/healthcheck.ts'
 
 export const api = {
 	endpoint: 'healthcheck',
 	methods: {
 		get: {
 			init({ genomes }) {
-				return async (req: any, res: any): Promise<void> => {
+				return async (req: undefined, res: any): Promise<void> => {
 					try {
-						const health = await getStat(genomes)
+						const health = (await getStat(genomes)) as HealthCheckResponse
 						res.send(health)
 					} catch (e: any) {
 						res.send({ status: 'error', error: e.message || e })
@@ -26,47 +27,3 @@ export const api = {
 		}
 	}
 }
-
-/**
- * for documentation only, to signify integer: not type-checked statically
- */
-type int = number
-
-/**
- * Information aboute the server build version and dates,
- * including the date when the server was last launched
- */
-export type VersionInfo = {
-	pkgver: string
-	codedate: string
-	launchdate: string
-}
-
-type BuildByGenome = {
-	[index: string]: GenomeBuildInfo
-}
-
-export type GenomeBuildInfo = {
-	genedb: DbInfo
-	termdbs?: {
-		[index: string]: DbInfo
-	}
-}
-
-type DbInfo = {
-	buildDate: string // "unknown" or a Date-convertible string
-	tables?: {
-		[index: string]: int
-	}
-}
-
-/**
- * Server sttus and data related to it's health
- */
-export type HealthCheckResponse = {
-	status: 'ok' | 'error'
-	genomes: BuildByGenome
-	versionInfo: VersionInfo
-	w?: number[]
-	rs?: number
-}

package/routes/termdb.violin.ts

@@ -0,0 +1,78 @@
+import { trigger_getViolinPlotData } from '#src/termdb.violin.js'
+// import { getViolinRequest, getViolinResponse } from '#shared/types/routes/termdb.violin'
+
+export const api: any = {
+	endpoint: 'termdb/violin',
+	methods: {
+		get: {
+			init,
+			request: {
+				typeId: 'getViolinRequest'
+			},
+			response: {
+				typeId: 'getViolinResponse'
+			},
+			examples: [
+				{
+					request: {
+						body: {
+							genome: 'hg38-test',
+							dslabel: 'TermdbTest',
+							embedder: 'localhost',
+							devicePixelRatio: 2.200000047683716,
+							maxThickness: 150,
+							screenThickness: 1218,
+							filter: {
+								type: 'tvslst',
+								in: true,
+								join: '',
+								lst: [
+									{
+										tag: 'cohortFilter',
+										type: 'tvs',
+										tvs: { term: { id: 'subcohort', type: 'categorical' }, values: [{ key: 'ABC', label: 'ABC' }] }
+									}
+								]
+							},
+							svgw: 227.27272234672367,
+							orientation: 'horizontal',
+							datasymbol: 'bean',
+							radius: 5,
+							strokeWidth: 0.2,
+							axisHeight: 60,
+							rightMargin: 50,
+							unit: 'abs',
+							plotThickness: 150,
+							termid: 'agedx'
+						}
+					},
+					response: {
+						header: { status: 200 }
+					}
+				}
+			]
+		},
+		post: {
+			alternativeFor: 'get',
+			init
+		}
+	}
+}
+
+function init({ genomes }) {
+	return async (req: any, res: any): Promise<void> => {
+		const q = req.query // as getViolinRequest
+		try {
+			const g = genomes[req.query.genome]
+			const ds = g.datasets[req.query.dslabel]
+			if (!g) throw 'invalid genome name'
+			const data = await trigger_getViolinPlotData(req.query, null, ds, g) // as getViolinResponse
+			res.send(data)
+		} catch (e) {
+			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+			// @ts-ignore
+			res.send({ error: e?.message || e })
+			if (e instanceof Error && e.stack) console.log(e)
+		}
+	}
+}