@sjcrh/proteinpaint-server 2.44.0 → 2.46.0

package/routes/hicdata.ts

@@ -1,111 +0,0 @@
-import { HicdataRequest, HicdataResponse, Item } from '#shared/types/routes/hicdata.ts'
-import { fileurl } from '#src/utils.js'
-import { spawn } from 'child_process'
-import readline from 'readline'
-import serverconfig from '#src/serverconfig.js'
-
-export const api: any = {
-	endpoint: 'hicdata',
-	methods: {
-		get: {
-			init,
-			request: {
-				typeId: 'HicdataRequest'
-			},
-			response: {
-				typeId: 'HicdataResponse'
-			}
-			/*
-			examples: [
-				{
-					request: {
-						body: {
-							genome: 'hg38-test',
-							dslabel: 'TermdbTest',
-							embedder: 'localhost',
-							gettermbyid: 'subcohort'
-						}
-					},
-					response: {
-						header: { status: 200 }
-					}
-				}
-			]
-			*/
-		},
-		post: {
-			alternativeFor: 'get',
-			init
-		}
-	}
-}
-
-function init() {
-	return async (req: any, res: any): Promise<void> => {
-		try {
-			const payload = await handle_hicdata(req.query as HicdataRequest)
-			res.send(payload)
-		} catch (e) {
-			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-			// @ts-ignore
-			res.send({ error: e?.message || e })
-			if (e instanceof Error && e.stack) console.log(e)
-		}
-	}
-}
-
-function handle_hicdata(q: HicdataRequest) {
-	return new Promise((resolve, reject) => {
-		const [e, file, isurl] = fileurl({ query: q })
-		if (e) reject({ error: 'illegal file name' })
-
-		/*Value passed from client is not the proper straw parameter.
-		Must convert to straw parameter and apply the corresponding maths to the result.
-		Use 'observed' as default if not provided.
-		*/
-		const matrixType = q.matrixType == 'log(oe)' ? 'oe' : q.matrixType ? q.matrixType : 'observed'
-
-		const par = [matrixType, q.nmeth || 'NONE', file, q.pos1, q.pos2, q.isfrag ? 'FRAG' : 'BP', q.resolution]
-
-		const ps = spawn(serverconfig.hicstraw, par)
-		const rl = readline.createInterface({ input: ps.stdout })
-
-		const items = [] as Item[]
-		const erroutput = [] as string[]
-		let linenot3fields = 0
-		let fieldnotnumerical = 0
-
-		rl.on('line', line => {
-			// straw output: pos1 \t pos2 \t value
-			const l = line.split('\t')
-			if (l.length != 3) {
-				linenot3fields++
-				return
-			}
-			const n1 = Number.parseInt(l[0])
-			const n2 = Number.parseInt(l[1])
-			const v = q.matrixType == 'log(oe)' ? Math.log(Number.parseFloat(l[2])) : Number.parseFloat(l[2])
-			if (Number.isNaN(n1) || Number.isNaN(n2) || Number.isNaN(v)) {
-				fieldnotnumerical++
-				return
-			}
-			if (q.mincutoff != undefined && v <= q.mincutoff) {
-				return
-			}
-			items.push([n1, n2, v] as Item)
-		})
-
-		ps.stderr.on('data', i => erroutput.push(i))
-		ps.on('close', () => {
-			const err = erroutput.join('')
-			if (err) reject({ error: err })
-
-			if (linenot3fields) reject({ error: linenot3fields + ' lines have other than 3 fields' })
-
-			if (fieldnotnumerical)
-				reject({ error: fieldnotnumerical + ' lines have non-numerical values in any of the 3 fields' })
-
-			resolve({ items })
-		})
-	})
-}

package/routes/hicstat.ts

@@ -1,55 +0,0 @@
-import { fileurl, file_is_readable } from '#src/utils.js'
-import { do_hicstat } from '#src/hicstat.ts'
-import { HicstatRequestWithValidation } from '#shared/types/routes/hicstat.ts'
-
-export const api = {
-	endpoint: 'hicstat',
-	methods: {
-		get: {
-			init,
-			request: {
-				typeId: 'HicstatRequest'
-			},
-			response: {
-				typeId: 'HicstatResponse'
-			},
-			examples: [
-				{
-					request: {
-						body: {
-							genome: 'hg19',
-							file: 'proteinpaint_demo/hg19/hic/hic_demo.hic',
-							embedder: 'localhost'
-						}
-					},
-					response: {
-						header: { status: 200 }
-					}
-				}
-			]
-		},
-		post: {
-			alternativeFor: 'get',
-			init
-		}
-	}
-}
-
-function init() {
-	return async (req: HicstatRequestWithValidation, res: any): Promise<void> => {
-		try {
-			const [e, file, isurl] = fileurl(req)
-			if (e) throw 'illegal file name'
-			if (!isurl) {
-				await file_is_readable(file)
-			}
-			const out = await do_hicstat(file, isurl)
-			res.send({ out })
-		} catch (e: any) {
-			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-			// @ts-ignore
-			res.send({ error: e?.message || e })
-			if (e instanceof Error && e.stack) console.log(e)
-		}
-	}
-}

package/routes/termdb.categories.ts

@@ -1,245 +0,0 @@
-import { getcategoriesRequest, getcategoriesResponse } from '#shared/types/routes/termdb.categories.ts'
-import { getOrderedLabels } from '#src/termdb.barchart.js'
-import { getData } from '#src/termdb.matrix.js'
-
-export const api: any = {
-	endpoint: 'termdb/categories',
-	methods: {
-		get: {
-			init,
-			request: {
-				typeId: 'getcategoriesRequest'
-			},
-			response: {
-				typeId: 'getcategoriesResponse'
-			},
-			examples: [
-				{
-					request: {
-						body: {
-							genome: 'hg38-test',
-							dslabel: 'TermdbTest',
-							embedder: 'localhost',
-							getcategories: 1,
-							tid: 'diaggrp',
-							term1_q: {
-								isAtomic: true,
-								hiddenValues: {},
-								type: 'values',
-								groupsetting: { disabled: true },
-								mode: 'discrete'
-							},
-							filter: {
-								type: 'tvslst',
-								in: true,
-								join: '',
-								lst: [
-									{
-										tag: 'cohortFilter',
-										type: 'tvs',
-										tvs: {
-											term: {
-												name: 'Cohort',
-												type: 'categorical',
-												values: { ABC: { label: 'ABC' }, XYZ: { label: 'XYZ' } },
-												id: 'subcohort',
-												isleaf: false,
-												groupsetting: { disabled: true }
-											},
-											values: [{ key: 'ABC', label: 'ABC' }]
-										}
-									}
-								]
-							}
-						}
-					},
-					response: {
-						header: { status: 200 }
-					}
-				}
-			]
-		},
-		post: {
-			alternativeFor: 'get',
-			init
-		}
-	}
-}
-
-function init({ genomes }) {
-	return async (req: any, res: any): Promise<void> => {
-		const q = req.query as getcategoriesRequest
-		try {
-			const g = genomes[req.query.genome]
-			if (!g) throw 'invalid genome name'
-			const ds = g.datasets[req.query.dslabel]
-			if (!ds) throw 'invalid dataset name'
-			const tdb = ds.cohort.termdb
-			if (!tdb) throw 'invalid termdb object'
-
-			await trigger_getcategories(q, res, tdb, ds, g) // as getcategoriesResponse
-		} catch (e) {
-			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-			// @ts-ignore
-			res.send({ error: e?.message || e })
-			if (e instanceof Error && e.stack) console.log(e)
-		}
-	}
-}
-
-async function trigger_getcategories(
-	q: { tid: string | number; type: string; filter: any; term1_q: any; currentGeneNames?: string[]; rglst?: any },
-	res: { send: (arg0: { lst: any[]; orderedLabels?: any }) => void },
-	tdb: { q: { termjsonByOneid: (arg0: any) => any } },
-	ds: { assayAvailability: { byDt: { [s: string]: any } | ArrayLike<any> } },
-	genome: any
-) {
-	// thin wrapper of get_summary
-	// works for all types of terms
-	if (!q.tid) throw '.tid missing'
-	const term =
-		q.type == 'geneVariant' ? { name: q.tid, type: 'geneVariant', isleaf: true } : tdb.q.termjsonByOneid(q.tid)
-
-	const arg = {
-		filter: q.filter,
-		terms:
-			q.type == 'geneVariant'
-				? [{ term: term, q: { isAtomic: true } }]
-				: [{ id: q.tid, term, q: q.term1_q || getDefaultQ(term, q) }],
-		currentGeneNames: q.currentGeneNames, // optional, from mds3 mayAddGetCategoryArgs()
-		rglst: q.rglst // optional, from mds3 mayAddGetCategoryArgs()
-	}
-
-	const data = await getData(arg, ds, genome)
-	if (data.error) throw data.error
-
-	const lst = [] as any[]
-	if (q.type == 'geneVariant') {
-		const samples = data.samples as { [sampleId: string]: any }
-		const dtClassMap = new Map()
-		if (ds.assayAvailability?.byDt) {
-			for (const [dtType, dtValue] of Object.entries(ds.assayAvailability.byDt)) {
-				if (dtValue.byOrigin) {
-					dtClassMap.set(parseInt(dtType), { byOrigin: { germline: {}, somatic: {} } })
-				}
-			}
-		}
-		const sampleCountedFor = new Set() // if the sample is counted
-		for (const [sampleId, sampleData] of Object.entries(samples)) {
-			const values = sampleData[q.tid].values
-			sampleCountedFor.clear()
-			/* values here is an array of result entires, one or more entries for each dt. e.g.
-			[
-				{ dt: 1, class: 'Blank', _SAMPLEID_: 1, origin: 'germline' },
-				{ dt: 1, class: 'WT', _SAMPLEID_: 1, origin: 'somatic' },
-				{ dt: 2, class: 'Blank', _SAMPLEID_: 1 },
-				{ dt: 4, class: 'WT', _SAMPLEID_: 1 }
-			]
-			*/
-			for (const value of values) {
-				if (!dtClassMap.has(value.dt)) {
-					dtClassMap.set(value.dt, {})
-				}
-				const dtClasses = dtClassMap.get(value.dt)
-				if (dtClasses.byOrigin) {
-					if (!dtClasses.byOrigin[value.origin][value.class]) {
-						dtClasses.byOrigin[value.origin][value.class] = 1
-						sampleCountedFor.add(`${value.dt} ${value.origin} ${value.class}`)
-					}
-					if (!sampleCountedFor.has(`${value.dt} ${value.origin} ${value.class}`)) {
-						sampleCountedFor.add(`${value.dt} ${value.origin} ${value.class}`)
-						dtClasses.byOrigin[value.origin][value.class] += 1
-					}
-				} else {
-					if (!dtClasses[value.class]) {
-						sampleCountedFor.add(`${value.dt} ${value.class}`)
-						dtClasses[value.class] = 1
-					}
-					if (!sampleCountedFor.has(`${value.dt} ${value.class}`)) {
-						sampleCountedFor.add(`${value.dt} ${value.class}`)
-						dtClasses[value.class] += 1
-					}
-				}
-			}
-		}
-		for (const [dt, classes] of dtClassMap) {
-			lst.push({
-				dt,
-				classes
-			})
-		}
-	} else {
-		const key2count = new Map()
-		// k: category key
-		// v: number of samples
-		for (const sid in data.samples) {
-			const v = data.samples[sid][q.tid]
-			if (!v) continue
-			if (!('key' in v)) continue
-			key2count.set(v.key, 1 + (key2count.get(v.key) || 0))
-		}
-		for (const [key, count] of key2count) {
-			lst.push({
-				samplecount: count,
-				key,
-				label:
-					data.refs?.byTermId?.[q.tid]?.events?.find((e: { event: any }) => e.event === key).label ||
-					term?.values?.[key]?.label ||
-					key
-			})
-		}
-	}
-
-	const orderedLabels = getOrderedLabels(
-		term,
-		data.refs?.byTermId?.[q.tid]?.bins || [],
-		data.refs?.byTermId?.[q.tid]?.events,
-		q.term1_q
-	)
-	if (orderedLabels.length) {
-		lst.sort((a, b) => orderedLabels.indexOf(a.label) - orderedLabels.indexOf(b.label))
-	}
-	res.send({
-		lst,
-		orderedLabels
-	} as getcategoriesResponse)
-}
-
-function getDefaultQ(
-	term: { type: string; bins: { default: any } },
-	q: {
-		mode?: any
-		breaks?: any
-		bar_by_grade?: any
-		bar_by_children?: any
-		value_by_max_grade?: any
-		value_by_most_recent?: any
-		value_by_computable_grade?: any
-		tid?: string | number
-		type?: string
-		filter?: any
-		term1_q?: any
-		currentGeneNames?: any
-	}
-) {
-	if (term.type == 'categorical') return {}
-	if (term.type == 'survival') return {}
-	if (term.type == 'integer' || term.type == 'float') return term.bins.default
-	if (term.type == 'condition') {
-		return {
-			mode: q.mode,
-			breaks: q.breaks,
-			bar_by_grade: q.bar_by_grade,
-			/*Leave this here until bug with term1_q not passing to getCategories is figured out.
-			Commented out b/c tvs condition tests fail.*/
-			//bar_by_children: term.subconditions || q.bar_by_children,
-			bar_by_children: q.bar_by_children,
-			value_by_max_grade: q.value_by_max_grade,
-			value_by_most_recent: q.value_by_most_recent,
-			//value_by_computable_grade: term.subconditions || q.value_by_computable_grade
-			value_by_computable_grade: q.value_by_computable_grade
-		}
-	}
-	if (term.type == 'geneVariant') return {}
-	throw 'unknown term type'
-}

package/routes/termdb.cluster.ts

@@ -1,248 +0,0 @@
-import path from 'path'
-import fs from 'fs'
-import lines2R from '#src/lines2R.js'
-import {
-	TermdbClusterRequest,
-	TermdbClusterResponse,
-	Clustering,
-	ValidResponse,
-	SinglegeneResponse
-} from '#shared/types/routes/termdb.cluster.ts'
-import * as utils from '#src/utils.js'
-import serverconfig from '#src/serverconfig.js'
-import { GeneExpressionQuery, GeneExpressionQueryNative } from '#shared/types/dataset.ts'
-import { gdc_validate_query_geneExpression } from '#src/mds3.gdc.js'
-import { mayLimitSamples } from '#src/mds3.filter.js'
-import { dtgeneexpression } from '#shared/common.js'
-
-export const api = {
-	endpoint: 'termdb/cluster',
-	methods: {
-		all: {
-			init,
-			request: {
-				typeId: 'TermdbClusterRequest'
-			},
-			response: {
-				typeId: 'TermdbClusterResponse'
-			}
-		}
-	}
-}
-
-function init({ genomes }) {
-	return async (req: any, res: any): Promise<void> => {
-		const q = req.query as TermdbClusterRequest
-		let result
-		try {
-			const g = genomes[q.genome]
-			if (!g) throw 'invalid genome name'
-			const ds = g.datasets[q.dslabel]
-			if (!ds) throw 'invalid dataset name'
-			if (ds.__gdc && !ds.__gdc.doneCaching)
-				throw 'The server has not finished caching the case IDs: try again in ~2 minutes'
-			if (q.dataType == dtgeneexpression) {
-				if (!ds.queries?.geneExpression) throw 'no geneExpression data on this dataset'
-				result = (await getResult(q, ds)) as TermdbClusterResponse
-			} else {
-				throw 'unknown q.dataType ' + q.dataType
-			}
-		} catch (e: any) {
-			if (e.stack) console.log(e.stack)
-			result = {
-				status: e.status || 400,
-				error: e.message || e
-			} as TermdbClusterResponse
-		}
-		res.send(result)
-	}
-}
-
-async function getResult(q: TermdbClusterRequest, ds: any) {
-	const { gene2sample2value, byTermId, bySampleId } = await ds.queries.geneExpression.get(q)
-	if (gene2sample2value.size == 0) throw 'no data'
-	if (gene2sample2value.size == 1) {
-		// get data for only 1 gene; still return data, may create violin plot later
-		const g = Array.from(gene2sample2value.keys())[0]
-		return { gene: g, data: gene2sample2value.get(g) } as SinglegeneResponse
-	}
-
-	// have data for multiple genes, run clustering
-	const t = Date.now() // use "t=new Date()" will lead to tsc error
-	const clustering: Clustering = await doClustering(gene2sample2value, q)
-	if (serverconfig.debugmode) console.log('clustering done:', Date.now() - t, 'ms')
-	return { clustering, byTermId, bySampleId } as ValidResponse
-}
-
-async function doClustering(data: any, q: TermdbClusterRequest) {
-	// get set of unique sample names, to generate col_names dimension
-	const sampleSet = new Set()
-	for (const o of data.values()) {
-		// {sampleId: value}
-		for (const s in o) sampleSet.add(s)
-		break
-	}
-
-	const inputData = {
-		matrix: [] as number[][],
-		row_names: [] as string[], // genes
-		col_names: [...sampleSet] as string[], // samples
-		cluster_method: q.clusterMethod as string,
-		plot_image: false // When true causes cluster.rs to plot the image into a png file (EXPERIMENTAL)
-	}
-
-	// compose "data{}" into a matrix
-	for (const [gene, o] of data) {
-		inputData.row_names.push(gene)
-		const row: number[] = []
-		for (const s of inputData.col_names) {
-			row.push(o[s] || 0)
-		}
-		inputData.matrix.push(getZscore(row))
-	}
-
-	const Rinputfile = path.join(serverconfig.cachedir, Math.random().toString() + '.json')
-	await utils.write_file(Rinputfile, JSON.stringify(inputData))
-	const Routput = JSON.parse(await lines2R(path.join(serverconfig.binpath, 'utils/hclust.R'), [], [Rinputfile]))
-	await fs.promises.unlink(Rinputfile)
-
-	const row_names_index: number[] = Routput.RowOrder.map(row => inputData.row_names.indexOf(row.name)) // sorted rows. value is array index in input data
-	const col_names_index: number[] = Routput.ColOrder.map(col => inputData.col_names.indexOf(col.name)) // sorted columns, value is array index from input array
-
-	// generated sorted matrix based on row/col clustering order
-	const output_matrix: number[][] = []
-	for (const rowI of row_names_index) {
-		const newRow: number[] = []
-		for (const colI of col_names_index) {
-			newRow.push(inputData.matrix[rowI][colI])
-		}
-		output_matrix.push(newRow)
-	}
-
-	return {
-		row: {
-			merge: Routput.RowMerge,
-			height: Routput.RowHeight,
-			order: Routput.RowOrder,
-			inputOrder: inputData.row_names
-		},
-		col: {
-			merge: Routput.ColumnMerge,
-			height: Routput.ColumnHeight,
-			order: Routput.ColOrder,
-			inputOrder: inputData.col_names
-		},
-		matrix: output_matrix
-	}
-}
-function getZscore(l: number[]) {
-	const mean: number = l.reduce((sum, v) => sum + v, 0) / l.length
-	const sd: number = Math.sqrt(l.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / l.length)
-
-	if (sd == 0) {
-		return l
-	}
-	return l.map(v => (v - mean) / sd)
-}
-
-export async function validate_query_geneExpression(ds: any, genome: any) {
-	const q = ds.queries.geneExpression as GeneExpressionQuery
-	if (!q) return
-
-	if (q.src == 'gdcapi') {
-		gdc_validate_query_geneExpression(ds, genome)
-		// q.get() added
-		return
-	}
-	if (q.src == 'native') {
-		await validateNative(q, ds, genome)
-		return
-	}
-	throw 'unknown queries.geneExpression.src'
-}
-
-async function validateNative(q: GeneExpressionQueryNative, ds: any, genome: any) {
-	if (!q.file.startsWith(serverconfig.tpmasterdir)) q.file = path.join(serverconfig.tpmasterdir, q.file)
-	if (!q.samples) q.samples = []
-	await utils.validate_tabixfile(q.file)
-	q.nochr = await utils.tabix_is_nochr(q.file, null, genome)
-	q.samples = [] as number[]
-
-	{
-		// is a gene-by-sample matrix file
-		const lines = await utils.get_header_tabix(q.file)
-		if (!lines[0]) throw 'header line missing from ' + q.file
-		const l = lines[0].split('\t')
-		if (l.slice(0, 4).join('\t') != '#chr\tstart\tstop\tgene') throw 'header line has wrong content for columns 1-4'
-		for (let i = 4; i < l.length; i++) {
-			const id = ds.cohort.termdb.q.sampleName2id(l[i])
-			if (id == undefined) throw 'queries.geneExpression: unknown sample from header: ' + l[i]
-			q.samples.push(id)
-		}
-		console.log(q.samples.length, 'samples from geneExpression of', ds.label)
-	}
-
-	q.get = async (param: TermdbClusterRequest) => {
-		const limitSamples = await mayLimitSamples(param, q.samples, ds)
-		if (limitSamples?.size == 0) {
-			// got 0 sample after filtering, must still return expected structure with no data
-			return { gene2sample2value: new Set(), byTermId: {}, bySampleId: {} }
-		}
-
-		// has at least 1 sample passing filter and with exp data
-		// TODO what if there's just 1 sample not enough for clustering?
-		const bySampleId = {}
-		const samples = q.samples || []
-		if (limitSamples) {
-			for (const sid of limitSamples) {
-				bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) }
-			}
-		} else {
-			// use all samples with exp data
-			for (const sid of samples) {
-				bySampleId[sid] = { label: ds.cohort.termdb.q.id2sampleName(sid) }
-			}
-		}
-
-		// only valid genes with data are added. invalid genes or genes missing from data file is not added. backend returned genes is allowed to be fewer than supplied by client
-		const gene2sample2value = new Map() // k: gene symbol, v: { sampleId : value }
-
-		for (const g of param.genes) {
-			// FIXME newly added geneVariant terms from client to be changed to {gene} but not {name}
-			if (!g.gene) continue
-
-			if (!g.chr) {
-				// quick fix: newly added gene from client will lack chr/start/stop
-				const lst = genome.genedb.getjsonbyname.all(g.gene)
-				if (lst.length == 0) continue
-				const j = JSON.parse(lst.find(i => i.isdefault).genemodel || lst[0].genemodel)
-				g.start = j.start
-				g.stop = j.stop
-				g.chr = j.chr
-			}
-
-			const s2v = {}
-			await utils.get_lines_bigfile({
-				args: [q.file, (q.nochr ? g.chr?.replace('chr', '') : g.chr) + ':' + g.start + '-' + g.stop], // must do g.chr?.replace to avoid tsc error
-				callback: line => {
-					const l = line.split('\t')
-					// case-insensitive match! FIXME if g.gene is alias won't work
-					if (l[3].toLowerCase() != g.gene.toLowerCase()) return
-					for (let i = 4; i < l.length; i++) {
-						const sampleId = samples[i - 4]
-						if (limitSamples && !limitSamples.has(sampleId)) continue // doing filtering and sample of current column is not used
-						if (!l[i]) continue // blank string
-						const v = Number(l[i])
-						if (Number.isNaN(v)) throw 'exp value not number'
-						s2v[sampleId] = v
-					}
-				}
-			} as any)
-			// Above!! add "as any" to suppress a npx tsc alert
-			if (Object.keys(s2v).length) gene2sample2value.set(g.gene, s2v) // only add gene if has data
-		}
-		// pass blank byTermId to match with expected output structure
-		const byTermId = {}
-		return { gene2sample2value, byTermId, bySampleId }
-	}
-}