@sjcrh/proteinpaint-server 2.26.1 → 2.27.1

package/cards/databrowser.json

@@ -1,5 +1,5 @@
 {
-  "intro": "<b>We're excited to announce our newest app!</b> By uploading a dictionary in the Data Browser, one can explore the structure of their custom data. <br><br>Options to upload sample annotation and other data types are in development.", 
+  "intro": "By uploading a dictionary in the Data Browser, one can explore the structure of their custom data. <br><br>Options to upload sample annotation and other data types are in development.", 
   "ppcalls":[
     {
         "isUi": true,

package/cards/disco.json

@@ -0,0 +1,69 @@
+{
+    "ribbonMessage": "We're excited to announce our newest app, the <b>Disco Plot!</b>",
+    "ppcalls": [
+        {
+            "isUi": true,
+            "runargs": {
+                "noheader": true,
+                "nobox": 1,
+                "parseurl": false,
+                "tkui": "disco"
+            }
+        },
+        {
+            "message": "In this plot, all chromosomes of the genome are arranged in a circle. Multiple rings along the circle indicates example data points of mutation, copy number change, and structural variation/fusion events. Gene labels are based on mutation and sv/fusion events.",
+            "runargs": {
+                "noheader": true,
+                "nobox": true,
+                "genome": "hg38",
+                "disco": {
+                    "mlst": [
+                        {
+                            "alt": "T",
+                            "chr": "chr1",
+                            "class": "M",
+                            "dt": 1,
+                            "gene": "H3F3A",
+                            "isoform": "NM_002107",
+                            "mname": "K28M",
+                            "position": 226252135,
+                            "ref": "A"
+                        },
+                        {
+                            "dt": 4,
+                            "chr": "chr1",
+                            "start": 1,
+                            "stop": 100000000,
+                            "value": 0.5
+                        },
+                        {
+                            "dt": 4,
+                            "chr": "chr1",
+                            "start": 100000000,
+                            "stop": 200000000,
+                            "value": -0.5
+                        },
+                        {
+                            "chrA": "chr6",
+                            "posA": 3067605,
+                            "geneA": "MDC1",
+                            "chrB": "chr12",
+                            "posB": 61521661,
+                            "geneB": "KMT2D",
+                            "dt": 2,
+                            "strandA": "+",
+                            "strandB": "-"
+                        }
+                    ]
+                }
+            },
+            "testSpec": {
+                "button": 1,
+                "expected": {
+                    "svg": 1,
+                    "g": 2
+                }
+            }
+        }
+    ]
+}

package/cards/hic.json

@@ -1,4 +1,5 @@
 {
+    "ribbonMessage": "Versions 7, 8 and 9 Hi-C files are accepted.",
     "ppcalls": [
         {
             "label": "Hi-C",

package/cards/index.json

@@ -106,6 +106,7 @@
             "section": "tracks",
             "description": "Chromatin interaction at a locus",
             "image": "https://proteinpaint.stjude.org/ppdemo/images/hic-square.png",
+            "ribbon": { "text": "updated", "expireDate": "2023-10-31" },
             "sandboxJson": "hic",
             "searchterms": ["hic", "hicstraw", "chromatin interaction", "conformation"]
         },
@@ -285,7 +286,6 @@
             "section": "apps",
             "description": "2D scatter plot of samples & metadata",
             "image": "https://proteinpaint.stjude.org/ppdemo/images/scatterplot-square.png",
-            "ribbon": { "text": "updated", "expireDate": "2023-03-01" },
             "sandboxJson": "scatterplot",
             "searchterms": ["tSNE", "lasso"]
         },
@@ -307,6 +307,16 @@
             "ribbon": {"text":"beta"},
             "sandboxJson": "databrowser",
             "searchterms": ["clinical", "dictionary"]
+        },
+        {
+            "type": "card",
+            "name": "Disco Plot",
+            "section": "apps",
+            "description": "Circos-like plot of genome-wide mutational events",
+            "image": "https://proteinpaint.stjude.org/ppdemo/images/disco-square.png",
+            "ribbon": { "text":"new", "expireDate": "2023-10-31" },
+            "sandboxJson": "disco",
+            "searchterms": ["disco"]
         }
     ]
-}
+}

package/genome/hg19.js

@@ -1,6 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-var clinvar_1 = require("../dataset/clinvar");
 exports.default = {
     species: 'human',
     genomefile: 'genomes/hg19.gz',
@@ -22,11 +21,6 @@ exports.default = {
         db: 'utils/meme/motif_databases/HUMAN/HOCOMOCOv11_full_HUMAN_mono_meme_format.meme',
         annotationfile: 'utils/meme/motif_databases/HUMAN/HOCOMOCOv11_full_annotation_HUMAN_mono.tsv'
     },
-    clinvarVCF: {
-        file: 'hg19/clinvar.hg19.hgvs_short.vep.bcf.gz',
-        infokey: 'CLNSIG',
-        categories: clinvar_1.clinsig
-    },
     tracks: [
         {
             __isgene: true,

package/genome/hg38.js

@@ -1,6 +1,5 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-var clinvar_1 = require("../dataset/clinvar");
 var cgc_js_1 = require("./cgc.js");
 exports.default = {
     species: 'human',
@@ -30,11 +29,6 @@ exports.default = {
         db: 'utils/meme/motif_databases/HUMAN/HOCOMOCOv11_full_HUMAN_mono_meme_format.meme',
         annotationfile: 'utils/meme/motif_databases/HUMAN/HOCOMOCOv11_full_annotation_HUMAN_mono.tsv'
     },
-    clinvarVCF: {
-        file: 'hg38/clinvar.hg38.hgvs_short.vep.bcf.gz',
-        infokey: 'CLNSIG',
-        categories: clinvar_1.clinsig
-    },
     tracks: [
         {
             __isgene: true,

package/package.json

@@ -1,11 +1,12 @@
 {
   "name": "@sjcrh/proteinpaint-server",
-  "version": "2.26.1",
+  "version": "2.27.1",
   "description": "a genomics visualization tool for exploring a cohort's genotype and phenotype data",
   "main": "server.js",
   "bin": "start.js",
   "imports": {
-    "#shared/*": "./shared/*.js",
+    "#shared/*": "./shared/*",
+    "#shared/types/*": "./shared/types/*",
     "#src/*": "./src/*",
     "#routes/*": "./routes/*"
   },
@@ -15,17 +16,18 @@
     "start": "tsc --esModuleInterop genome/*.ts dataset/*.ts && nodemon --enable-source-maps server.js --watch ./server*.js* --watch dataset/*.ts --watch genome/*.ts",
     "pretest": "tsc --esModuleInterop genome/*.ts dataset/*.ts && ./test/pretest.js",
     "test": "webpack --env NODE_ENV=test exportsFilename=all-test-context.js && node --enable-source-maps test/serverTests.js",
-    "pretest:unit": "npm run checkers && tsc genome/hg38.test.ts",
-    "test:unit": "webpack --env NODE_ENV=test exportsFilename=unit-test-context.js && tsc --esModuleInterop dataset/*.ts && node --enable-source-maps test/serverTests.js",
+    "precheckers": "tsc --esModuleInterop genome/*.ts dataset/*.ts",
+    "checkers": "webpack --config=./test/webpack.config.js && node test/emitPrepFiles.bundle.js && typia generate --input shared/checkers-raw --output shared/checkers",
+    "pretest:type": "npm run checkers",
+    "test:type": "webpack --env NODE_ENV=test exportsFilename=type-test-context.js && node --enable-source-maps test/serverTests.js",
+    "pretest:unit": "npm run precheckers",
+    "test:unit": "webpack --env NODE_ENV=test exportsFilename=unit-test-context.js && node --enable-source-maps test/serverTests.js",
     "pretest:integration": "tsc --esModuleInterop genome/*.ts dataset/*.ts",
     "test:integration": "echo 'TODO: server integration tests'",
     "prepack": "tsc --esModuleInterop genome/*.ts dataset/*.ts && webpack --env NODE_ENV=production",
     "response": "nodemon modules/test/test.server.js --watch src",
     "getconf": "../build/getConfigProp.js",
-    "predoc": "npm run checkers",
-    "doc": "typedoc --json ../public/docs/server.json",
-    "precheckers": "npx ts-node ./augen/cli.js typeCheckers $PWD/routes ../../../routes > shared/checkers/raw/index.ts",
-    "checkers": "typia generate --input shared/checkers/raw --output shared/checkers/transformed --project ./shared/checkers/tsconfig.json"
+    "doc": "../augen/build.sh routes shared/types/routes shared/checkers ../public/docs/server"
   },
   "author": "",
   "license": "SEE LICENSE IN ./LICENSE",
@@ -45,7 +47,6 @@
     "ts-node": "^10.9.1",
     "ts-patch": "^3.0.2",
     "typedoc": "^0.24.8",
-    "typedoc-plugin-replace-text": "^3.1.0",
     "typescript": "^5.0.3",
     "typia": "^4.1.14",
     "webpack": "^5.76.0",
@@ -54,7 +55,8 @@
     "webpack-notifier": "^1.15.0"
   },
   "dependencies": {
-    "@sjcrh/proteinpaint-rust": "2.26.1",
+    "@sjcrh/augen": "2.27.0",
+    "@sjcrh/proteinpaint-rust": "2.27.0",
     "better-sqlite3": "^7.5.3",
     "body-parser": "^1.15.2",
     "canvas": "~2.9.3",
@@ -69,6 +71,7 @@
     "jsonwebtoken": "^9.0.0",
     "jstat": "^1.9.3",
     "lazy": "^1.0.11",
+    "micromatch": "^4.0.5",
     "minimatch": "^3.1.2",
     "node-fetch": "^2.6.1",
     "partjson": "^0.58.1",

package/routes/README.md

@@ -0,0 +1,84 @@
+# Server Routes
+
+## Introduction
+
+This directory contains files that specify server route APIs. By following this guidelines,
+the auto-generation of server routes, tests, and API documentation will work as expected.
+
+## Guidelines
+
+### 1. Use Express to do most of the routing logic
+
+- decentralize the route handling code into smaller, independent handler functions
+- common request processing logic, like genome, dataset, termdb set-up should be imported
+from a shared helper module that is common to a group of routes, or for more advanced cases,
+moved to a [router-level middleware](https://expressjs.com/en/guide/using-middleware.html#middleware.router)
+
+### 2. Export an `api` from the route file
+
+Use the code from other files in this directory as examples
+
+TODO: define the `api` type
+
+```ts
+// work-in-progress
+type RouteApi {
+	[key as methods]: RouteApiMethod
+}
+
+type methods = 'get' | 'post'
+
+type initArg = {
+	app?: any // Express app instance
+	genome: any // `Genome` from shared/types/genome.ts 
+}
+
+/**
+	@param
+*/
+type RouteApiMethod = {
+	endpoint: string
+	init: (initArg) => void
+	request: {
+		typeId: string
+		body?: any // specific to the route
+	}
+	response: {
+		typeId: string
+		header?: {
+			status: number
+		}
+		body?: any // specific to the route
+	}
+	examples: RouteExample[]
+}
+
+type RouteExample = {
+	request: {
+		body?: any
+	}
+	response?: {
+		header: {
+			status: number
+		}
+		body?: any 
+	}
+}
+```
+### 3. Use the appropriate [HTTP response code](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status)
+
+This is a best practice especially for error responses. Use `res.status(code)` to set the error code.
+This convention helps with error troubleshooting. Examples:
+- Status `400` 'Bad Request', something is wrong with the http request payload
+- Status `401` 'Unauthorized', the user must authenticate. The `server/src/auth.js` sets this status code
+- Status `403` 'Forbidden', the user is authenticated/signed-in, but is not permitted to access the requested data
+- Status `404` 'Not Found' for genome, dataset, or other data that is not found
+- Status `500` 'Server Error' for errors related to the server process or host machine, such as the GDC API
+not being available. Do not use code=`500` for errors that are related to specific request handler or data processing functions.
+
+### 4. Auto-generate
+
+- the server code will detect the routes in `server/src/run.sh`
+- `npm run doc` to see the documented routes in http://localhost:3000/server.html
+- `./augen/readme.sh > public/docs/readme.json` for content in http://localhost:3000/readme.html
+

package/routes/burden.ts

@@ -0,0 +1,140 @@
+import { BurdenRequest, BurdenResponse } from '#shared/types/routes/burden.ts'
+import lines2R from '#src/lines2R.js'
+import path from 'path'
+import serverconfig from '#src/serverconfig.js'
+import { write_file } from '#src/utils.js'
+
+export const api = {
+	endpoint: 'burden',
+	methods: {
+		get: {
+			init({ genomes }) {
+				return async (req: any, res: any): Promise<void> => {
+					try {
+						const genome = genomes[req.query.genome]
+						if (!genome) throw `invalid q.genome=${req.query.genome}`
+						const q = req.query as BurdenRequest
+						const ds = genome.datasets[q.dslabel]
+						if (!ds) throw `invalid q.genome=${req.query.dslabel}`
+						if (!ds.cohort.cumburden?.files) throw `missing ds.cohort.cumburden.files`
+
+						const estimates = await getBurdenEstimates(req, ds)
+						const { keys, rows } = formatPayload(estimates)
+						res.send({ status: 'ok', keys, rows } as BurdenResponse)
+					} catch (e: any) {
+						res.send({ status: 'error', error: e.message || e })
+					}
+				}
+			},
+			request: {
+				typeId: 'BurdenRequest'
+			},
+			response: {
+				typeId: 'BurdenResponse'
+			},
+			examples: [
+				{
+					request: {
+						body: {
+							genome: 'hg38',
+							// TODO: !!! use hg38-test and TermdbTest !!!
+							dslabel: 'SJLife',
+							diaggrp: 5,
+							sex: 1,
+							white: 1,
+							agedx: 1,
+							bleo: 0,
+							etop: 0,
+							cisp: 0,
+							carbo: 0,
+							steriod: 0,
+							vcr: 0,
+							hdmtx: 0,
+							itmt: 0,
+							ced: 0,
+							dox: 0,
+							heart: 0,
+							brain: 0,
+							abd: 0,
+							pelvis: 0,
+							chest: 0
+						}
+					},
+					response: {
+						header: { status: 200 }
+					}
+				}
+			]
+		}
+	}
+}
+
+async function getBurdenEstimates(q, ds) {
+	const infile = path.join(serverconfig.cachedir, Math.random().toString() + '.json')
+	for (const k in q.query) {
+		q.query[k] = Number(q.query[k])
+	}
+	const data = Object.assign({}, defaults, q.query)
+	//console.log(40, data, JSON.stringify(data))
+	await write_file(infile, JSON.stringify(data))
+	// TODO: use the dataset location
+	const { fit, surv, sample } = ds.cohort.cumburden.files
+	if (!fit || !surv || !sample) throw `missing one or more of ds.cohort.burden.files.{fit, surv, sample}`
+	const args = [
+		infile,
+		`${serverconfig.tpmasterdir}/${fit}`,
+		`${serverconfig.tpmasterdir}/${surv}`,
+		`${serverconfig.tpmasterdir}/${sample}`
+	]
+	const Routput = await lines2R(path.join(serverconfig.binpath, 'utils/burden.R'), [], args)
+	const estimates = JSON.parse(Routput[0])
+	return estimates
+}
+
+function formatPayload(estimates) {
+	const rawKeys = Object.keys(estimates[0])
+	const outKeys = [] as string[]
+	const keys = [] as string[]
+	for (const k of rawKeys) {
+		if (k == 'chc') {
+			keys.push(k)
+			outKeys.push(k)
+		} else {
+			const age = Number(k.slice(1).split(',')[0])
+			if (age <= 60 && age % 2 == 0) {
+				keys.push(k)
+				outKeys.push(`burden${age}`)
+			}
+		}
+	}
+	const rows = [] as number[][]
+	// v = an array of objects with age as keys as cumulative burden as value for a given CHC
+	for (const v of estimates) {
+		rows.push(keys.map(k => v[k]))
+	}
+	return { keys: outKeys, rows }
+}
+
+const defaults = Object.freeze({
+	diaggrp: 5,
+	sex: 0,
+	white: 1,
+	agedx: 1,
+	// chemotherapy
+	steriod: 0,
+	bleo: 0,
+	vcr: 0, //12, // Vincristine
+	etop: 0, //2500, // Etoposide
+	itmt: 0, // Intrathecal methothrexate_grp: 0,
+	ced: 0, //1.6, // Cyclophosphamide, 0.7692 mean 7692.
+	cisp: 0, //300, // Cisplatin
+	dox: 0, // Anthracycline, 3 mean 300 ml/m2
+	carbo: 0, //  Carboplatin
+	hdmtx: 0, // High-Dose Methotrexate
+	// radiation
+	brain: 0, //5.4,
+	chest: 0, //2.4,
+	heart: 0,
+	pelvis: 0,
+	abd: 0 //2.4
+})

package/routes/gdc.maf.ts

@@ -0,0 +1,106 @@
+import { GdcMafResponse, File } from '#shared/types/routes/gdc.maf.ts'
+import { fileSize } from '#shared/fileSize.js'
+import path from 'path'
+import got from 'got'
+
+const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
+
+export const api = {
+	endpoint: 'gdc/maf',
+	methods: {
+		get: {
+			init({ genomes }) {
+				// genomes parameter is not used
+				// could be used later to verify hg38/GDC is on this instance and otherwise disable this route..
+
+				return async (req: any, res: any): Promise<void> => {
+					try {
+						const files = await listMafFiles(req)
+						const payload = { files } as GdcMafResponse
+						res.send(payload)
+					} catch (e: any) {
+						res.send({ status: 'error', error: e.message || e })
+					}
+				}
+			},
+			request: {
+				typeId: null
+				//valid: default to type checker
+			},
+			response: {
+				typeId: 'GdcMafResponse'
+				// will combine this with type checker
+				//valid: (t) => {}
+			}
+		}
+	}
+}
+
+/*
+req.query {
+	filter0 // optional gdc GFF cohort filter, invisible and read only
+}
+*/
+async function listMafFiles(req: any) {
+	const filters = {
+		op: 'and',
+		content: [
+			{
+				op: '=',
+				content: { field: 'data_format', value: 'MAF' }
+			}
+		]
+	}
+
+	if (req.query.filter0) {
+		filters.content.push(req.query.filter0)
+	}
+
+	const headers = { 'Content-Type': 'application/json', Accept: 'application/json' }
+
+	const data = {
+		filters,
+		size: 1000,
+		fields: [
+			'id',
+			'file_size',
+			'experimental_strategy',
+			'cases.submitter_id', // used when listing all cases & files
+			//'associated_entities.entity_submitter_id', // semi human readable
+			//'associated_entities.case_id', // case uuid
+			'cases.samples.sample_type',
+			'analysis.workflow_type' // to drop out those as skip_workflow_type
+		].join(',')
+	}
+
+	const response = await got.post(path.join(apihost, 'files'), { headers, body: JSON.stringify(data) })
+
+	let re
+	try {
+		re = JSON.parse(response.body)
+	} catch (e) {
+		throw 'invalid JSON from ' + api.endpoint
+	}
+	if (!Array.isArray(re.data?.hits)) throw 're.data.hits[] not array'
+
+	// flatten api return to table row objects
+	// it is possible to set a max size limit to limit the number of files passed to client
+	const files = [] as File[]
+	for (const h of re.data.hits) {
+		const file = {
+			id: h.id,
+			workflow_type: h.analysis?.workflow_type,
+			experimental_strategy: h.experimental_strategy,
+			file_size: fileSize(h.file_size)
+		} as File
+		const c = h.cases?.[0]
+		if (c) {
+			file.case_submitter_id = c.submitter_id
+			if (c.samples) {
+				file.sample_types = c.samples.map(i => i.sample_type).join(', ')
+			}
+		}
+		files.push(file)
+	}
+	return files
+}

package/routes/gdc.topMutatedGenes.ts

@@ -0,0 +1,130 @@
+import { GdcTopMutatedGeneResponse } from '#shared/types/routes/gdc.topMutatedGenes.ts'
+import path from 'path'
+import got from 'got'
+
+const apihost = process.env.PP_GDC_HOST || 'https://api.gdc.cancer.gov'
+
+export const api = {
+	endpoint: 'gdc/topMutatedGenes',
+	methods: {
+		get: {
+			init({ genomes }) {
+				/*
+				genomes parameter is currently not used
+				could be used later to:
+				- verify hg38/GDC is on this instance and otherwise disable this route..
+				- perform conversion on gene name/id for future on needs
+				*/
+
+				return async (req: any, res: any): Promise<void> => {
+					try {
+						const genes = await getGenes(req.query)
+						const payload = { genes } as GdcTopMutatedGeneResponse
+						res.send(payload)
+					} catch (e: any) {
+						res.send({ status: 'error', error: e.message || e })
+					}
+				}
+			},
+			request: {
+				typeId: null
+				//valid: default to type checker
+			},
+			response: {
+				typeId: 'GdcTopMutatedGeneResponse'
+				// will combine this with type checker
+				//valid: (t) => {}
+			}
+		}
+	}
+}
+
+/*
+req.query {
+	filter0 // optional gdc GFF cohort filter, invisible and read only
+		FIXME should there be pp filter too?
+	geneFilter?: str
+	maxGenes: int
+}
+
+mayAddCGC2filter() are copied to
+/utils/gdc/topSSMgenes.js
+and hosted on https://proteinpaint.stjude.org/GDC/
+*/
+async function getGenes(q: any) {
+	let _f = { op: 'and', content: [] } // allow blank filter to test geneset edit ui (without filter)
+	if (q.filter0) {
+		if (typeof q.filter0 != 'object') throw 'filter0 not object'
+		_f = q.filter0
+	}
+	const response = await got.post(path.join(apihost, '/analysis/top_mutated_genes_by_project'), {
+		headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
+		body: JSON.stringify({
+			size: q.maxGenes || 50,
+			fields: 'symbol',
+			filters: mayAddCGC2filter(_f, q.geneFilter)
+		})
+	})
+	const re = JSON.parse(response.body)
+	const genes = [] as string[]
+	for (const hit of re.data.hits) {
+		if (!hit.symbol) continue
+		genes.push(hit.symbol)
+	}
+	return genes
+}
+
+/*
+str:
+	stringified gdc filter object, should not include the "genes.is_cancer_gene_census" filter element
+geneFilter: str
+	if = 'CGC', insert following element into the filter and return stringified obj
+
+	{
+		"op":"and",
+		"content":[
+			{
+				"content":{ "field":"genes.is_cancer_gene_census", "value":["true"] },
+				"op":"in"
+			}
+		]
+	}
+*/
+function mayAddCGC2filter(f: any, geneFilter?: string) {
+	// reformulate f into f2
+	// f may be "in" or "and". f2 is always "and", in order to join in additional filters
+	let f2
+
+	if (f.op == 'in') {
+		// wrap f into f2
+		f2 = { op: 'and', content: [f] }
+	} else if (f.op == 'and') {
+		// no need to wrap
+		f2 = f
+	} else {
+		throw 'f.op not "in" or "and"'
+	}
+
+	// per Phil on 12/16/2022, following filters ensure to return IDH1 as 1st gene for gliomas
+	f2.content.push({
+		op: 'NOT',
+		content: {
+			field: 'ssms.consequence.transcript.annotation.vep_impact',
+			value: 'missing'
+		}
+	})
+	f2.content.push({
+		op: 'in',
+		content: {
+			field: 'ssms.consequence.transcript.consequence_type',
+			value: ['missense_variant', 'frameshift_variant', 'start_lost', 'stop_lost', 'stop_gained']
+		}
+	})
+
+	if (geneFilter == 'CGC') {
+		// using CGC genes, add in filter
+		f2.content.push({ op: 'in', content: { field: 'genes.is_cancer_gene_census', value: ['true'] } })
+	}
+
+	return f2
+}