@sjcrh/proteinpaint-server 2.44.0 → 2.46.0

package/routes/termdb.singlecellSamples.ts

@@ -1,175 +0,0 @@
-import fs from 'fs'
-import path from 'path'
-import { read_file } from '#src/utils.js'
-import serverconfig from '#src/serverconfig.js'
-import { SingleCellQuery, SingleCellSamplesNative, SingleCellDataNative } from '#shared/types/dataset.ts'
-import {
-	Sample,
-	TermdbSinglecellsamplesRequest,
-	TermdbSinglecellsamplesResponse
-} from '#shared/types/routes/termdb.singlecellSamples.ts'
-import { Cell, Plot } from '#shared/types/routes/termdb.singlecellData.ts'
-import { gdc_validate_query_singleCell_samples, gdc_validate_query_singleCell_data } from '#src/mds3.gdc.js'
-
-/* route returns list of samples with sc data
-this is due to the fact that sometimes not all samples in a dataset has sc data
-*/
-
-export const api: any = {
-	endpoint: 'termdb/singlecellSamples',
-	methods: {
-		get: {
-			init,
-			request: {
-				typeId: 'TermdbSinglecellsamplesRequest'
-			},
-			response: {
-				typeId: 'TermdbSinglecellsamplesResponse'
-			}
-		},
-		post: {
-			alternativeFor: 'get',
-			init
-		}
-	}
-}
-
-function init({ genomes }) {
-	return async (req: any, res: any): Promise<void> => {
-		const q = req.query as TermdbSinglecellsamplesRequest
-		let result
-		try {
-			const g = genomes[q.genome]
-			if (!g) throw 'invalid genome name'
-			const ds = g.datasets[q.dslabel]
-			if (!ds) throw 'invalid dataset name'
-			if (!ds.queries?.singleCell) throw 'no singlecell data on this dataset'
-			result = (await ds.queries.singleCell.samples.get(q)) as TermdbSinglecellsamplesResponse
-		} catch (e: any) {
-			if (e.stack) console.log(e.stack)
-			result = {
-				status: e.status || 400,
-				error: e.message || e
-			} as TermdbSinglecellsamplesResponse
-		}
-		res.send(result)
-	}
-}
-
-/////////////////// ds query validator
-export async function validate_query_singleCell(ds: any, genome: any) {
-	const q = ds.queries.singleCell as SingleCellQuery
-	if (!q) return
-
-	if (q.samples.src == 'gdcapi') {
-		gdc_validate_query_singleCell_samples(ds, genome)
-	} else if (q.samples.src == 'native') {
-		getSamplesNative(q.samples as SingleCellSamplesNative, ds)
-	} else {
-		throw 'unknown singleCell.samples.src'
-	}
-	// q.samples.get() added
-
-	if (q.data.src == 'gdcapi') {
-		gdc_validate_query_singleCell_data(ds, genome)
-	} else if (q.data.src == 'native') {
-		getDataNative(q.data as SingleCellDataNative, ds)
-	} else {
-		throw 'unknown singleCell.data.src'
-	}
-	// q.data.get() added
-}
-
-async function getSamplesNative(S: SingleCellSamplesNative, ds: any) {
-	// for now use this quick fix method to pull sample ids annotated by this term
-	// to support situation where not all samples from a dataset has sc data
-	const isSamples = ds.cohort.termdb.q.getAllValues4term(S.isSampleTerm)
-	if (isSamples.size == 0) throw 'no samples found that are identified by isSampleTerm'
-	const samples = [] as any // array of samples with sc data to be sent to client and list in table; cannot use Sample type for the use of "sampleid" temp property
-	for (const sampleid of isSamples.keys()) {
-		if (isSamples.get(sampleid) == '1')
-			samples.push({
-				sample: ds.cohort.termdb.q.id2sampleName(sampleid), // string name for display
-				sampleid // temporarily kept to assign term value to each sample
-			})
-	}
-	if (S.sampleColumns) {
-		// has optional terms to show as table columns and annotate samples
-		for (const term of S.sampleColumns) {
-			const s2v = ds.cohort.termdb.q.getAllValues4term(term.termid) // map. k: sampleid, v: term value
-			for (const s of samples) {
-				if (s2v.has(s.sampleid)) s[term.termid] = s2v.get(s.sampleid)
-			}
-		}
-	}
-	for (const s of samples) delete s.sampleid
-
-	S.get = () => {
-		return { samples: samples as Sample[] }
-	}
-}
-
-function getDataNative(D: SingleCellDataNative, ds: any) {
-	const nameSet = new Set() // guard against duplicating plot names
-	for (const plot of D.plots) {
-		if (nameSet.has(plot.name)) throw 'duplicate plot.name'
-		nameSet.add(plot.name)
-	}
-
-	// scoped and cached for runtime
-	const _terms = [] as any
-
-	for (const tid of D.termIds) {
-		const t = ds.cohort.termdb.q.termjsonByOneid(tid)
-		if (!t) throw 'invalid term id from queries.singleCell.data.termIds[]'
-		_terms.push(t)
-	}
-	D.get = async q => {
-		// if sample is int, may convert to string
-		try {
-			const tid2cellvalue = {}
-			for (const tid of D.termIds) tid2cellvalue[tid] = {} // k: cell id, v: cell value for this term
-			const plots = [] as Plot[] // given a sample name, collect every plot data for this sample and return
-			for (const plot of D.plots) {
-				const tsvfile = path.join(serverconfig.tpmasterdir, plot.folder, q.sample + plot.fileSuffix)
-				try {
-					await fs.promises.stat(tsvfile)
-				} catch (e: any) {
-					if (e.code == 'ENOENT') {
-						// no file found for this sample; allowed because sampleView tests if that sample has sc data or not
-						continue
-					}
-					if (e.code == 'EACCES') throw 'cannot read file, permission denied'
-					throw 'failed to load sc data file'
-				}
-				const lines = (await read_file(tsvfile)).trim().split('\n')
-				// 1st line is header
-				const cells = [] as Cell[]
-				for (let i = 1; i < lines.length; i++) {
-					// each line is a cell
-					const l = lines[i].split('\t')
-					const cellId = l[0],
-						x = Number(l[plot.coordsColumns.x]), // FIXME standardize, or define idx in plot
-						y = Number(l[plot.coordsColumns.y])
-					const category = l[plot.colorColumn?.index] || ''
-					if (!cellId) throw 'cell id missing'
-					if (!Number.isFinite(x) || !Number.isFinite(y)) throw 'x/y not number'
-					cells.push({ cellId, x, y, category })
-
-					for (const tid of D.termIds) {
-						tid2cellvalue[tid][cellId] = l[1]
-					}
-				}
-				plots.push({ name: plot.name, cells, colorBy: plot.colorColumn?.name, colorMap: plot.colorMap })
-			}
-			if (plots.length == 0) {
-				// no data available for this sample
-				return { nodata: true }
-			}
-			return { plots, terms: _terms, tid2cellvalue }
-		} catch (e: any) {
-			if (e.stack) console.log(e.stack)
-			return { error: e.message || e }
-		}
-	}
-}

package/routes/termdb.termsbyids.ts

@@ -1,63 +0,0 @@
-import { gettermsbyidsRequest, gettermsbyidsResponse } from '#shared/types/routes/termdb.termsbyids.js'
-import { copy_term } from '#src/termdb.js'
-
-export const api: any = {
-	endpoint: 'termdb/termsbyids',
-	methods: {
-		get: {
-			init,
-			request: {
-				typeId: 'gettermsbyidsRequest'
-			},
-			response: {
-				typeId: 'gettermsbyidsResponse'
-			}
-		},
-		post: {
-			alternativeFor: 'get',
-			init
-		}
-	}
-}
-
-function init({ genomes }) {
-	return async (req: any, res: any): Promise<void> => {
-		const q = req.query // as getcategoriesRequest
-		try {
-			const g = genomes[req.query.genome]
-			if (!g) throw 'invalid genome name'
-			const ds = g.datasets[req.query.dslabel]
-			if (!ds) throw 'invalid dataset name'
-			const tdb = ds.cohort.termdb
-			if (!tdb) throw 'invalid termdb object'
-
-			await trigger_gettermsbyid(q, res, tdb) // as getcategoriesResponse
-		} catch (e) {
-			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-			// @ts-ignore
-			res.send({ error: e?.message || e })
-			if (e instanceof Error && e.stack) console.log(e)
-		}
-	}
-}
-
-async function trigger_gettermsbyid(
-	q: { ids: any },
-	res: { send: (arg0: { terms: any }) => void },
-	tdb: { q: { termjsonByOneid: (arg0: any) => any } }
-) {
-	const terms = {}
-	for (const id of q.ids) {
-		const term = tdb.q.termjsonByOneid(id)
-		if (term) {
-			if (term.type == 'categorical' && !term.values && !term.groupsetting?.inuse) {
-				term.values = {}
-				term.samplecount = {}
-			}
-		}
-		terms[id] = term ? copy_term(term) : undefined
-	}
-	res.send({
-		terms
-	})
-}

package/routes/termdb.topVariablyExpressedGenes.ts

@@ -1,214 +0,0 @@
-import {
-	TermdbTopVariablyExpressedGenesRequest,
-	TermdbTopVariablyExpressedGenesResponse
-} from '#shared/types/routes/termdb.topVariablyExpressedGenes.ts'
-import { gdcGetCasesWithExressionDataFromCohort } from '../src/mds3.gdc.js'
-import path from 'path'
-import { run_rust } from '@sjcrh/proteinpaint-rust'
-import got from 'got'
-import serverconfig from '#src/serverconfig.js'
-import { get_samples } from '#src/termdb.sql.js'
-
-export const api = {
-	endpoint: 'termdb/topVariablyExpressedGenes',
-	methods: {
-		all: {
-			init,
-			request: {
-				typeId: 'TermdbTopVariablyExpressedGenesRequest'
-			},
-			response: {
-				typeId: 'TermdbTopVariablyExpressedGenesResponse'
-			}
-		}
-	}
-}
-
-function init({ genomes }) {
-	return async (req: any, res: any): Promise<void> => {
-		try {
-			const q = req.query as TermdbTopVariablyExpressedGenesRequest
-			const genome = genomes[q.genome]
-			if (!genome) throw 'invalid genome'
-			const ds = genome.datasets?.[q.dslabel]
-			if (!ds) throw 'invalid dslabel'
-			if (!ds.queries?.topVariablyExpressedGenes) throw 'not supported on dataset'
-
-			const t = Date.now()
-			const genes = await ds.queries.topVariablyExpressedGenes.getGenes(q)
-			if (serverconfig.debugmode) console.log('topVariablyExpressedGenes', Date.now() - t, 'ms')
-
-			res.send({ genes } as TermdbTopVariablyExpressedGenesResponse)
-		} catch (e: any) {
-			res.send({ status: 'error', error: e.message || e })
-		}
-	}
-}
-
-export function validate_query_TopVariablyExpressedGenes(ds: any, genome: any) {
-	const q = ds.queries.topVariablyExpressedGenes
-	if (!q) return
-	if (q.src == 'gdcapi') {
-		gdcValidateQuery(ds, genome)
-	} else if (q.src == 'native') {
-		nativeValidateQuery(ds, genome)
-	} else {
-		throw 'unknown topVariablyExpressedGenes.src'
-	}
-	// added getter: q.getGenes()
-}
-
-function nativeValidateQuery(ds: any, genome: any) {
-	const gE = ds.queries.geneExpression // a separate query required to supply data for computing top genes
-	if (!gE) throw 'topVariablyExpressedGenes query given but geneExpression missing'
-	if (gE.src != 'native') throw 'topVariablyExpressedGenes is native but geneExpression.src is not native'
-
-	ds.queries.topVariablyExpressedGenes.getGenes = async (q: TermdbTopVariablyExpressedGenesRequest) => {
-		// get list of samples that are used in current analysis; gE.samples[] contains all sample integer ids with exp data
-		const samples = [] as string[]
-		if (q.filter) {
-			// get all samples pasing pp filter, may contain those without exp data
-			const sidlst = await get_samples(q.filter, ds)
-			// [{id:int}]
-			// filter for those with exp data from q.samples[]
-			for (const i of sidlst) {
-				if (gE.samples.includes(i.id)) {
-					// this sample passing filter also has exp data; convert to string name
-					const n: string = ds.cohort.termdb.q.id2sampleName(i.id)
-					if (!n) throw 'sample id cannot convert to string name'
-					samples.push(n)
-				}
-			}
-		} else {
-			// no filter, use all samples with exp data
-			for (const i of gE.samples) {
-				const n: string = ds.cohort.termdb.q.id2sampleName(i.id)
-				if (!n) throw 'sample id cannot convert to string name'
-				samples.push(n)
-			}
-		}
-
-		// call rust to compute top genes on these samples
-		const genes = await computeGenes4nativeDs(q, ds, gE.file, samples)
-		return genes
-	}
-}
-
-async function computeGenes4nativeDs(
-	q: TermdbTopVariablyExpressedGenesRequest,
-	ds: any,
-	matrixFile: string,
-	samples: string[]
-) {
-	// The param option in input JSON is very important. It instructs what method will be used to calculate variation in the counts for a particular gene. It supports variance as well as interquartile region. This is based on the recommendation of this article https://www.frontiersin.org/articles/10.3389/fgene.2021.632620/full . This article recommends using interquartile region over variance.
-	const input_json = {
-		input_file: matrixFile,
-		samples: samples.join(','),
-		filter_extreme_values: true,
-		num_genes: Number(q.maxGenes),
-		param: 'var'
-	}
-	const rust_output = await run_rust('topGeneByExpressionVariance', JSON.stringify(input_json))
-	const rust_output_list = rust_output.split('\n')
-
-	let output_json
-	for (const item of rust_output_list) {
-		if (item.includes('output_json:')) {
-			output_json = JSON.parse(item.replace('output_json:', ''))
-		} else {
-			console.log(item)
-		}
-	}
-	const varGenes = output_json.map(i => i.gene_symbol)
-	return varGenes
-}
-
-function gdcValidateQuery(ds: any, genome: any) {
-	ds.queries.topVariablyExpressedGenes.getGenes = async (q: TermdbTopVariablyExpressedGenesRequest) => {
-		if (serverconfig.features.gdcGenes) {
-			// for testing on dev, must not set on prod!! delete to trigger api query
-			console.log('!!GDC!! using serverconfig.features.gdcGenes[]')
-			return serverconfig.features.gdcGenes as string[]
-		}
-
-		// disable when caching is incomplete (particularly cases with gene exp data); to prevent showing wrong data on client
-		if (!ds.__gdc.doneCaching) throw 'The server has not finished caching the case IDs: try again in ~2 minutes'
-
-		// based on current cohort, get list of cases with exp data, as input of next api query
-		const caseLst = await gdcGetCasesWithExressionDataFromCohort(q, ds)
-		if (caseLst.length == 0) {
-			// there are no cases with gene exp data
-			return [] as string[]
-		}
-
-		// change to this when api is available on prod
-		const { host, headers } = ds.getHostHeaders(q)
-		const url = path.join(host.geneExp, '/gene_expression/gene_selection')
-		try {
-			const response = await got.post(url, {
-				headers,
-				body: JSON.stringify(getGeneSelectionArg(q, caseLst))
-			})
-
-			const re = JSON.parse(response.body)
-			// {"gene_selection":[{"gene_id":"ENSG00000141510","log2_uqfpkm_median":3.103430497010492,"log2_uqfpkm_stddev":0.8692021350485105,"symbol":"TP53"}, ... ]}
-
-			const genes = [] as string[]
-			if (!Array.isArray(re.gene_selection)) throw 're.gene_selection[] is not array'
-			for (const i of re.gene_selection) {
-				if (i.gene_id && typeof i.gene_id == 'string') {
-					// is ensg, convert to symbol
-					const t = genome.genedb.getNameByAlias.get(i.gene_id)
-					if (t) genes.push(t.name) // ensg
-				} else if (i.symbol && typeof i.symbol == 'string') {
-					genes.push(i.symbol)
-				} else {
-					throw 'one of re.gene_selection[] is missing both gene_id and symbol'
-				}
-			}
-			return genes
-		} catch (e: any) {
-			console.log(e.stack || e)
-			throw e
-		}
-	}
-
-	function getGeneSelectionArg(q: any, caseLst: any) {
-		//to hide messy logic during testing phase
-
-		/* when api performance issue is resolved, return this
-		return {
-			case_ids: caseLst,
-			gene_type:'protein_coding',
-			selection_size: Number(q.maxGenes)
-		}
-		*/
-
-		//////////////////////////////////////////////////
-		//
-		// !!!!!!!!!!!!!!!! TEMPORARY !!!!!!!!!!!!!!!!!!!!
-		//
-		//////////////////////////////////////////////////
-		// limit the case_ids length, and restrict pool to CGC genes, otherwise the request times out !!!
-		// must revert asap
-		return {
-			case_ids: caseLst, //.slice(0, 20),
-			gene_ids: tempGetCGCgenes(genome),
-			selection_size: Number(q.maxGenes)
-		}
-	}
-}
-
-function tempGetCGCgenes(genome: any) {
-	const lst = [] as string[] // list of ENSG ids from cgc genes
-	// don't think there's need to preparse genome.geneset, as this function is only temporary
-	for (const s of genome.geneset[0].lst) {
-		const a = genome.genedb.getAliasByName.all(s)
-		if (a) {
-			for (const b of a) {
-				if (b.alias.startsWith('ENSG')) lst.push(b.alias)
-			}
-		}
-	}
-	return lst
-}

package/routes/termdb.violin.ts

@@ -1,77 +0,0 @@
-import { getViolinRequest, getViolinResponse } from '#shared/types/routes/termdb.violin.ts'
-import { trigger_getViolinPlotData } from '#src/termdb.violin.js'
-
-export const api: any = {
-	endpoint: 'termdb/violin',
-	methods: {
-		get: {
-			init,
-			request: {
-				typeId: 'getViolinRequest'
-			},
-			response: {
-				typeId: 'getViolinResponse'
-			},
-			examples: [
-				{
-					request: {
-						body: {
-							genome: 'hg38-test',
-							dslabel: 'TermdbTest',
-							embedder: 'localhost',
-							devicePixelRatio: 2.200000047683716,
-							maxThickness: 150,
-							screenThickness: 1218,
-							filter: {
-								type: 'tvslst',
-								in: true,
-								join: '',
-								lst: [
-									{
-										tag: 'cohortFilter',
-										type: 'tvs',
-										tvs: { term: { id: 'subcohort', type: 'categorical' }, values: [{ key: 'ABC', label: 'ABC' }] }
-									}
-								]
-							},
-							svgw: 227.27272234672367,
-							orientation: 'horizontal',
-							datasymbol: 'bean',
-							radius: 5,
-							strokeWidth: 0.2,
-							axisHeight: 60,
-							rightMargin: 50,
-							unit: 'abs',
-							termid: 'agedx'
-						}
-					},
-					response: {
-						header: { status: 200 }
-					}
-				}
-			]
-		},
-		post: {
-			alternativeFor: 'get',
-			init
-		}
-	}
-}
-
-function init({ genomes }) {
-	return async (req: any, res: any): Promise<void> => {
-		const q = req.query as getViolinRequest
-		try {
-			const g = genomes[req.query.genome]
-			const ds = g.datasets[req.query.dslabel]
-			if (!g) throw 'invalid genome name'
-			const data = await trigger_getViolinPlotData(req.query, null, ds, g) // as getViolinResponse
-			res.send(data as getViolinResponse)
-		} catch (e) {
-			// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-			// @ts-ignore
-			res.send({ error: e?.message || e })
-			if (e instanceof Error && e.stack) console.log(e)
-		}
-	}
-}