@sjcrh/proteinpaint-server 2.44.0 → 2.46.1

package/src/serverconfig.js

@@ -3,15 +3,15 @@
 	including generating and applying overrides as needed
 */
 
-const fs = require('fs')
-const path = require('path')
+import fs from 'fs'
+import path from 'path'
 
 // do not assume that serverconfig.json is in the same dir as server.js
 // for example, when using proteinpaint as an npm module or binary
 // or when calling a pp utility script from a tp data directory
 const workdirconfig = process.cwd() ? process.cwd() + '/serverconfig.json' : ''
-const serverdirconfig = path.join(__dirname, '../serverconfig.json')
-const pprootdirconfig = path.join(__dirname, '../../serverconfig.json')
+const serverdirconfig = path.join(import.meta.dirname, '../serverconfig.json')
+const pprootdirconfig = path.join(import.meta.dirname, '../../serverconfig.json')
 // check which config file exists in order of usage priority
 const serverconfigfile =
 	workdirconfig && fs.existsSync(workdirconfig)
@@ -26,6 +26,7 @@ const serverconfigfile =
  GET SERVERCONFIG
 ********************/
 let serverconfig
+
 if (!serverconfigfile) {
 	throw 'missing serverconfig.json'
 } else {
@@ -71,12 +72,16 @@ if (!serverconfig.binpath) {
 	} else {
 		const specfile = process.argv.find(n => n.includes('.spec.js'))
 		if (specfile) {
-			serverconfig.binpath = path.dirname(__dirname)
+			serverconfig.binpath = path.dirname(import.meta.dirname)
+		} else if (import.meta.filename.includes('node_modules/@sjcrh/proteinpaint-server')) {
+			const p = import.meta.filename.split('/proteinpaint-server')[0]
+			serverconfig.binpath = `${p}/proteinpaint-server`
 		} else {
 			const jsfile = process.argv.find(
 				n =>
 					n.endsWith('/bin.js') ||
 					n.endsWith('/server.js') ||
+					n.endsWith('/start.js') ||
 					n.endsWith('/proteinpaint') ||
 					n.endsWith('/proteinpaint-server')
 			)
@@ -90,8 +95,9 @@ if (!serverconfig.binpath) {
 			} else {
 				if (fs.existsSync('./server')) serverconfig.binpath = fs.realpathSync('./server')
 				else if (fs.existsSync('./src')) serverconfig.binpath = fs.realpathSync('./src/..')
-				else if (__dirname.includes('/server/')) serverconfig.binpath = __dirname.split('/server/')[0] + '/server'
-				else if (__dirname.includes('/proteinpaint')) serverconfig.binpath = __dirname
+				else if (import.meta.dirname.includes('/server/'))
+					serverconfig.binpath = import.meta.dirname.split('/server/')[0] + '/server'
+				else if (import.meta.dirname.includes('/proteinpaint')) serverconfig.binpath = import.meta.dirname
 				else throw 'unable to determine the serverconfig.binpath'
 			}
 		}
@@ -216,4 +222,4 @@ if (fs.existsSync('./package.json')) {
 	serverconfig.version = JSON.parse(pkg).version
 }
 
-module.exports = serverconfig
+export default serverconfig

package/start.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-console.log('starting the portal demo')
-console.log('using the proteinpaint-front and proteinpaint-server modules ....')
-require('./server.js')
+import { launch } from './src/app.js'
+console.log(`starting the server ...`)
+launch()
 console.log('server code loaded ...')

package/routes/README.md

@@ -1,84 +0,0 @@
-# Server Routes
-
-## Introduction
-
-This directory contains files that specify server route APIs. By following this guidelines,
-the auto-generation of server routes, tests, and API documentation will work as expected.
-
-## Guidelines
-
-### 1. Use Express to do most of the routing logic
-
-- decentralize the route handling code into smaller, independent handler functions
-- common request processing logic, like genome, dataset, termdb set-up should be imported
-from a shared helper module that is common to a group of routes, or for more advanced cases,
-moved to a [router-level middleware](https://expressjs.com/en/guide/using-middleware.html#middleware.router)
-
-### 2. Export an `api` from the route file
-
-Use the code from other files in this directory as examples
-
-TODO: define the `api` type
-
-```ts
-// work-in-progress
-type RouteApi {
-	[key as methods]: RouteApiMethod
-}
-
-type methods = 'get' | 'post'
-
-type initArg = {
-	app?: any // Express app instance
-	genome: any // `Genome` from shared/types/genome.ts 
-}
-
-/**
-	@param
-*/
-type RouteApiMethod = {
-	endpoint: string
-	init: (initArg) => void
-	request: {
-		typeId: string
-		body?: any // specific to the route
-	}
-	response: {
-		typeId: string
-		header?: {
-			status: number
-		}
-		body?: any // specific to the route
-	}
-	examples: RouteExample[]
-}
-
-type RouteExample = {
-	request: {
-		body?: any
-	}
-	response?: {
-		header: {
-			status: number
-		}
-		body?: any 
-	}
-}
-```
-### 3. Use the appropriate [HTTP response code](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status)
-
-This is a best practice especially for error responses. Use `res.status(code)` to set the error code.
-This convention helps with error troubleshooting. Examples:
-- Status `400` 'Bad Request', something is wrong with the http request payload
-- Status `401` 'Unauthorized', the user must authenticate. The `server/src/auth.js` sets this status code
-- Status `403` 'Forbidden', the user is authenticated/signed-in, but is not permitted to access the requested data
-- Status `404` 'Not Found' for genome, dataset, or other data that is not found
-- Status `500` 'Server Error' for errors related to the server process or host machine, such as the GDC API
-not being available. Do not use code=`500` for errors that are related to specific request handler or data processing functions.
-
-### 4. Auto-generate
-
-- the server code will detect the routes in `server/src/run.sh`
-- `npm run doc` to see the documented routes in http://localhost:3000/server.html
-- `./augen/readme.sh > public/docs/readme.json` for content in http://localhost:3000/readme.html
-

package/routes/burden.ts

@@ -1,143 +0,0 @@
-import { BurdenRequest, BurdenResponse } from '#shared/types/routes/burden.ts'
-import lines2R from '#src/lines2R.js'
-import path from 'path'
-import serverconfig from '#src/serverconfig.js'
-import { write_file } from '#src/utils.js'
-
-export const api = {
-	endpoint: 'burden',
-	methods: {
-		get: {
-			init({ genomes }) {
-				return async (req: any, res: any): Promise<void> => {
-					try {
-						const genome = genomes[req.query.genome]
-						if (!genome) throw `invalid q.genome=${req.query.genome}`
-						const q = req.query as BurdenRequest
-						const ds = genome.datasets[q.dslabel]
-						if (!ds) throw `invalid q.genome=${req.query.dslabel}`
-						if (!ds.cohort.cumburden?.files) throw `missing ds.cohort.cumburden.files`
-
-						const estimates = await getBurdenEstimates(req, ds)
-						const { keys, rows } = formatPayload(estimates)
-						res.send({ status: 'ok', keys, rows } as BurdenResponse)
-					} catch (e: any) {
-						res.send({ status: 'error', error: e.message || e })
-					}
-				}
-			},
-			request: {
-				typeId: 'BurdenRequest'
-			},
-			response: {
-				typeId: 'BurdenResponse'
-			},
-			examples: [
-				{
-					request: {
-						body: {
-							genome: 'hg38',
-							// TODO: !!! use hg38-test and TermdbTest !!!
-							dslabel: 'SJLife',
-							diaggrp: 5,
-							sex: 1,
-							white: 1,
-							agedx: 1,
-							bleo: 0,
-							etop: 0,
-							cisp: 0,
-							carbo: 0,
-							steriod: 0,
-							vcr: 0,
-							hdmtx: 0,
-							itmt: 0,
-							ced: 0,
-							dox: 0,
-							heart: 0,
-							brain: 0,
-							abd: 0,
-							pelvis: 0,
-							chest: 0
-						}
-					},
-					response: {
-						header: { status: 200 }
-					}
-				}
-			]
-		}
-	}
-}
-
-async function getBurdenEstimates(
-	q: { query: { [x: string]: any } },
-	ds: { cohort: { cumburden: { files: { fit: any; surv: any; sample: any } } } }
-) {
-	const infile = path.join(serverconfig.cachedir, Math.random().toString() + '.json')
-	for (const k in q.query) {
-		q.query[k] = Number(q.query[k])
-	}
-	const data = Object.assign({}, defaults, q.query)
-	//console.log(40, data, JSON.stringify(data))
-	await write_file(infile, JSON.stringify(data))
-	// TODO: use the dataset location
-	const { fit, surv, sample } = ds.cohort.cumburden.files
-	if (!fit || !surv || !sample) throw `missing one or more of ds.cohort.burden.files.{fit, surv, sample}`
-	const args = [
-		infile,
-		`${serverconfig.tpmasterdir}/${fit}`,
-		`${serverconfig.tpmasterdir}/${surv}`,
-		`${serverconfig.tpmasterdir}/${sample}`
-	]
-	const Routput = await lines2R(path.join(serverconfig.binpath, 'utils/burden.R'), [], args)
-	const estimates = JSON.parse(Routput[0])
-	return estimates
-}
-
-function formatPayload(estimates: object[]) {
-	const rawKeys = Object.keys(estimates[0])
-	const outKeys = [] as string[]
-	const keys = [] as string[]
-	for (const k of rawKeys) {
-		if (k == 'chc') {
-			keys.push(k)
-			outKeys.push(k)
-		} else {
-			const age = Number(k.slice(1).split(',')[0])
-			if (age <= 60 && age % 2 == 0) {
-				keys.push(k)
-				outKeys.push(`burden${age}`)
-			}
-		}
-	}
-	const rows = [] as number[][]
-	// v = an array of objects with age as keys as cumulative burden as value for a given CHC
-	for (const v of estimates) {
-		rows.push(keys.map(k => v[k]))
-	}
-	return { keys: outKeys, rows }
-}
-
-const defaults = Object.freeze({
-	diaggrp: 5,
-	sex: 0,
-	white: 1,
-	agedx: 1,
-	// chemotherapy
-	steriod: 0,
-	bleo: 0,
-	vcr: 0, //12, // Vincristine
-	etop: 0, //2500, // Etoposide
-	itmt: 0, // Intrathecal methothrexate_grp: 0,
-	ced: 0, //1.6, // Cyclophosphamide, 0.7692 mean 7692.
-	cisp: 0, //300, // Cisplatin
-	dox: 0, // Anthracycline, 3 mean 300 ml/m2
-	carbo: 0, //  Carboplatin
-	hdmtx: 0, // High-Dose Methotrexate
-	// radiation
-	brain: 0, //5.4,
-	chest: 0, //2.4,
-	heart: 0,
-	pelvis: 0,
-	abd: 0 //2.4
-})

package/routes/gdc.maf.ts

@@ -1,195 +0,0 @@
-import { GdcMafRequest, GdcMafResponse, File } from '#shared/types/routes/gdc.maf.ts'
-import path from 'path'
-import got from 'got'
-import serverconfig from '#src/serverconfig.js'
-
-/*
-this route lists available gdc MAF files based on user's cohort filter
-and return them to client to be shown in a table for selection
-*/
-
-const maxFileNumber = 1000 // determines max number of files to return to client
-// preliminary testing:
-// 36s for 1000 (87Mb)
-// 78s for 2000 (177Mb)
-// if safe to increase to 2000, maybe fast when this runs in gdc env
-
-const allowedWorkflowType = 'Aliquot Ensemble Somatic Variant Merging and Masking'
-
-// change to 400 so it won't limit number of files; should keep this setting as a safeguard; also it's fast to check file size (.5s in gdc.mafBuild.ts)
-export const maxTotalSizeCompressed = serverconfig.features.gdcMafMaxFileSize || 400000000 // 400Mb
-
-export const api = {
-	endpoint: 'gdc/maf',
-	methods: {
-		all: {
-			init,
-			request: {
-				typeId: 'GdcMafRequest'
-			},
-			response: {
-				typeId: 'GdcMafResponse'
-				// will combine this with type checker
-				//valid: (t) => {}
-			},
-			examples: [
-				{
-					request: {
-						body: {
-							experimentalStrategy: 'WXS',
-							embedder: 'localhost'
-						}
-					},
-					response: {
-						header: { status: 200 }
-					}
-				}
-			]
-		}
-	}
-}
-
-function init({ genomes }) {
-	return async (req: any, res: any): Promise<void> => {
-		try {
-			// g and ds are not used right now, but could be later
-			const g = genomes.hg38
-			if (!g) throw 'hg38 missing'
-			const ds = g.datasets.GDC
-			if (!ds) throw 'hg38 GDC missing'
-
-			const payload = await listMafFiles(req.query as GdcMafRequest, ds)
-			res.send(payload)
-		} catch (e: any) {
-			res.send({ status: 'error', error: e.message || e })
-		}
-	}
-}
-
-/*
-req.query {
-	filter0 // optional gdc GFF cohort filter, invisible and read only
-	experimentalStrategy: WXS/Targeted Sequencing
-}
-
-ds {
-	__gdc {
-		gdcOpenProjects
-	}
-}
-*/
-async function listMafFiles(q: GdcMafRequest, ds) {
-	const filters = {
-		op: 'and',
-		content: [
-			{ op: '=', content: { field: 'data_format', value: 'MAF' } },
-			{ op: '=', content: { field: 'experimental_strategy', value: q.experimentalStrategy } },
-			{ op: '=', content: { field: 'analysis.workflow_type', value: allowedWorkflowType } },
-			{ op: '=', content: { field: 'access', value: 'open' } } // delete if later to support controlled files
-		]
-	}
-	const case_filters: any = { op: 'and', content: [] }
-	if (q.filter0) {
-		case_filters.content.push(q.filter0)
-	}
-
-	const { host, headers } = ds.getHostHeaders(q)
-
-	const data = {
-		filters,
-		case_filters,
-		size: maxFileNumber,
-		fields: [
-			'id',
-			'file_size',
-			'cases.project.project_id', // for display only
-			'cases.submitter_id', // used when listing all cases & files
-			'cases.samples.sample_type'
-			// may add diagnosis and primary site
-		].join(',')
-	}
-
-	const response = await got.post(path.join(host.rest, 'files'), { headers, body: JSON.stringify(data) })
-
-	let re
-	try {
-		re = JSON.parse(response.body)
-	} catch (e) {
-		throw 'invalid JSON from ' + api.endpoint
-	}
-	if (!Number.isInteger(re.data?.pagination?.total)) throw 're.data.pagination.total is not int'
-	if (!Array.isArray(re.data?.hits)) throw 're.data.hits[] not array'
-
-	// flatten api return to table row objects
-	// it is possible to set a max size limit to limit the number of files passed to client
-	const files = [] as File[]
-
-	for (const h of re.data.hits) {
-		/*
-		{
-		  "id": "39768777-fec5-4a79-9515-65712c002b19",
-		  "cases": [
-			{
-			  "submitter_id": "HTMCP-03-06-02104",
-			  "project": {
-			  	"project_id":"xx"
-			  },
-			  "samples": [
-				{
-				  "sample_type": "Blood Derived Normal"
-				},
-				{
-				  "sample_type": "Primary Tumor"
-				}
-			  ]
-			}
-		  ],
-		  "analysis": {
-			"workflow_type": "MuSE Annotation"
-		  },
-		  "experimental_strategy": "Targeted Sequencing",
-		  "file_size": 146038
-		}
-		*/
-
-		const c = h.cases?.[0]
-		if (!c) throw 'h.cases[0] missing'
-
-		// only keep files from open access projects for now
-		/*
-		if (c.project?.project_id) {
-			if (ds.__gdc.gdcOpenProjects.has(c.project.project_id)) {
-				// open-access project, keep
-			} else {
-				// not open access
-				continue
-			}
-		} else {
-			throw 'h.cases[0].project.project_id missing'
-		}
-		*/
-
-		const file = {
-			id: h.id,
-			project_id: c.project.project_id,
-			file_size: h.file_size
-		} as File
-
-		file.case_submitter_id = c.submitter_id
-		if (c.samples) {
-			file.sample_types = c.samples.map((i: { sample_type: any }) => i.sample_type).sort()
-			// sort to show sample type names in consistent alphabetical order
-			// otherwise one file shows 'Blood, Primary' and another shows 'Primary, Blood'
-			// FIXME this includes samples not associated with current maf file
-		}
-		files.push(file)
-	}
-
-	const result = {
-		files,
-		filesTotal: re.data.pagination.total,
-		maxTotalSizeCompressed
-	} as GdcMafResponse
-
-	return result
-}

package/routes/gdc.mafBuild.ts

@@ -1,114 +0,0 @@
-import got from 'got'
-import path from 'path'
-import fs from 'fs'
-import { run_rust_stream } from '@sjcrh/proteinpaint-rust'
-import serverconfig from '#src/serverconfig.js'
-import Readable from 'stream'
-import { GdcMafBuildRequest } from '#shared/types/routes/gdc.mafBuild.ts'
-import { maxTotalSizeCompressed } from './gdc.maf.ts'
-
-export const api = {
-	endpoint: 'gdc/mafBuild',
-	methods: {
-		all: {
-			init,
-			request: {
-				typeId: 'GdcMafBuildRequest'
-			},
-			response: {
-				typeId: null // 'GdcMafBuildResponse'
-			}
-		}
-	}
-}
-
-function init({ genomes }) {
-	return async (req: any, res: any): Promise<void> => {
-		try {
-			const g = genomes.hg38
-			if (!g) throw 'hg38 missing'
-			const ds = g.datasets.GDC
-			if (!ds) throw 'hg38 GDC missing'
-			await buildMaf(req.query as GdcMafBuildRequest, res, ds)
-		} catch (e: any) {
-			if (e.stack) console.log(e.stack)
-			res.send({ status: 'error', error: e.message || e })
-		}
-	}
-}
-
-/*
-q{}
-res{}
-*/
-async function buildMaf(q: GdcMafBuildRequest, res: any, ds) {
-	const t0 = Date.now()
-	const { host, headers } = ds.getHostHeaders(q)
-	const fileLst2 = (await getFileLstUnderSizeLimit(q.fileIdLst, host, headers)) as string[]
-
-	if (serverconfig.debugmode)
-		console.log(
-			`${fileLst2.length} out of ${q.fileIdLst.length} input MAF files accepted by size limit`,
-			Date.now() - t0
-		)
-
-	const arg = {
-		fileIdLst: fileLst2,
-		columns: q.columns,
-		host: path.join(host.rest, 'data') // must use the /data/ endpoint from current host
-	}
-
-	const rustStream = run_rust_stream('gdcmaf', JSON.stringify(arg))
-	res.setHeader('Content-Type', 'application/octet-stream')
-	res.setHeader('Content-Disposition', 'attachment; filename=cohort.maf.gz')
-	rustStream.pipe(res)
-
-	rustStream.on('end', () => {
-		// report amount of time taken to run rust
-		if (serverconfig.debugmode) console.log('rust gdcmaf', Date.now() - t0)
-		res.end()
-	})
-
-	rustStream.on('error', err => {
-		console.error(err)
-		res.statusCode = 500
-		res.end('Internal Server Error')
-	})
-}
-
-/*
-query api get size of each input maf file, and only process those files with total size under a set limit,
-excess files are not processed in order not to crash server
-must not rely on file size sent by client, as that can be spoofed and never to be trusted
-it's inexpensive to query api for this
-*/
-async function getFileLstUnderSizeLimit(lst: string[], host, headers) {
-	if (lst.length == 0) throw 'fileIdLst[] not array or blank'
-	const data = {
-		filters: {
-			op: 'in',
-			content: { field: 'file_id', value: lst }
-		},
-		size: 10000,
-		fields: 'file_size'
-	}
-	const response = await got.post(path.join(host.rest, 'files'), { headers, body: JSON.stringify(data) })
-	let re
-	try {
-		re = JSON.parse(response.body)
-	} catch (e) {
-		throw 'invalid json from getFileLstUnderSizeLimit'
-	}
-	if (!Array.isArray(re.data?.hits)) throw 're.data.hits[] not array'
-	const out = [] as string[]
-	let cumsize = 0
-	for (const h of re.data.hits) {
-		if (cumsize >= maxTotalSizeCompressed) break // maxed out
-		if (!h.id) throw '.id missing'
-		if (!Number.isInteger(h.file_size)) throw '.file_size not integer'
-		cumsize += h.file_size
-		out.push(h.id)
-	}
-	if (out.length == 0) throw 'no file available'
-	return out
-}