jbrowse-plugin-msaview 3.2.0 → 3.4.0

package/src/utils/pantherOrthologs.ts

@@ -0,0 +1,399 @@
+// The second ortholog source, for the genes NCBI's ortholog sets leave out.
+//
+// NCBI Datasets computes orthologs for vertebrates and insects, so a yeast,
+// worm or plant gene comes back with orthologs only in its own clade, and a fly
+// gene gets insects and nothing else. PANTHER's ortholog sets span its 144
+// reference proteomes, human to yeast to Arabidopsis, and one `matchortho` call
+// answers "this gene's ortholog in every genome" with a UniProt accession per
+// target. Sequences come from one UniProt batch call. Both hosts send
+// `Access-Control-Allow-Origin: *`. The measurements that picked PANTHER over
+// OMA, OrthoDB and Ensembl are in react-msaview's
+// agent-docs/ideas/ortholog-sources-beyond-ncbi.md.
+//
+// Rows come out in the same shape as ncbiOrthologs.ts's, so the launch, the
+// labels, the aligner and the CDD overlay do not know which source ran.
+// `protein` is the UniProt accession; efetch serves UniProt accessions as
+// GenPept records with CDD Region features, so the overlay attaches as it does
+// to a RefSeq accession.
+
+import { jsonfetch } from './fetch'
+import { dedupeLabels, defaultMaxSpecies } from './ncbiOrthologs'
+import { fetchTaxonomyInfo } from './taxonomyNames'
+
+import type { OrthologRow } from './ncbiOrthologs'
+
+const PANTHER = 'https://pantherdb.org/services/oai/pantherdb'
+const UNIPROT = 'https://rest.uniprot.org/uniprotkb'
+
+export interface PantherGenome {
+  /** PANTHER's organism code, e.g. HUMAN, DROME */
+  code: string
+  taxId: number
+  /** short common name, e.g. fruit_fly */
+  name: string
+  /** scientific name */
+  longName: string
+}
+
+interface GenomesResponse {
+  search?: {
+    output?: {
+      genomes?: {
+        genome?: {
+          short_name?: string
+          taxon_id?: number
+          name?: string
+          long_name?: string
+        }[]
+      }
+    }
+  }
+}
+
+/**
+ * `supportedgenomes` -> the code<->taxon map every other parse needs. PANTHER
+ * names organisms by code only in ortholog results.
+ */
+export function parseGenomes(json: unknown): PantherGenome[] {
+  const list = (json as GenomesResponse).search?.output?.genomes?.genome ?? []
+  return list.flatMap(g =>
+    g.short_name && g.taxon_id
+      ? [
+          {
+            code: g.short_name,
+            taxId: g.taxon_id,
+            name: g.name ?? g.short_name,
+            longName: g.long_name ?? g.short_name,
+          },
+        ]
+      : [],
+  )
+}
+
+export interface PantherGene {
+  code: string
+  /** UniProt accession */
+  accession: string
+  /** the source database's own id, e.g. HGNC=1773, FlyBase=FBgn0016131 */
+  geneRef: string
+}
+
+export interface PantherHit extends PantherGene {
+  symbol?: string
+  /**
+   * LDO = least diverged ortholog, PANTHER's pick of the one-to-one; O = any
+   * other ortholog in a one-to-many or many-to-many family
+   */
+  type: 'LDO' | 'O'
+}
+
+interface Mapping {
+  id?: string
+  gene?: string
+  target_gene?: string
+  target_gene_symbol?: string | number
+  ortholog?: string
+}
+
+interface MatchResponse {
+  search?: {
+    mapping?: {
+      /** one object for a single (or empty) match, an array otherwise */
+      mapped?: Mapping | Mapping[]
+      unmapped_ids?: unknown
+    }
+  }
+}
+
+// "HUMAN|HGNC=1773|UniProtKB=P11802" -> { code, geneRef, accession }
+function parseGeneRef(ref: string | undefined): PantherGene | undefined {
+  const [code, ...xrefs] = (ref ?? '').split('|')
+  const accession = xrefs
+    .find(x => x.startsWith('UniProtKB='))
+    ?.slice('UniProtKB='.length)
+  const geneRef = xrefs.find(x => !x.startsWith('UniProtKB=')) ?? accession
+  return code && accession && geneRef ? { code, accession, geneRef } : undefined
+}
+
+/**
+ * `matchortho` -> the query gene (PANTHER names it in every row) and one hit
+ * per target gene. An unknown gene comes back under `unmapped_ids`; a gene with
+ * no ortholog in the target set comes back as a bare `{ id }`.
+ */
+export function parseMatches(json: unknown): {
+  unmapped: boolean
+  query?: PantherGene
+  hits: PantherHit[]
+} {
+  const mapping = (json as MatchResponse).search?.mapping
+  const mapped = mapping?.mapped
+  const rows = Array.isArray(mapped) ? mapped : mapped ? [mapped] : []
+  const hits: PantherHit[] = []
+  let query: PantherGene | undefined
+  for (const row of rows) {
+    query ??= parseGeneRef(row.gene)
+    const target = parseGeneRef(row.target_gene)
+    if (target && (row.ortholog === 'LDO' || row.ortholog === 'O')) {
+      hits.push({
+        ...target,
+        symbol:
+          row.target_gene_symbol === undefined
+            ? undefined
+            : String(row.target_gene_symbol),
+        type: row.ortholog,
+      })
+    }
+  }
+  return { unmapped: !!mapping?.unmapped_ids, query, hits }
+}
+
+/**
+ * One hit per organism, in first-seen order: the LDO where PANTHER named one,
+ * else the first other ortholog it listed. A many-to-many family (the Hox
+ * genes) has no LDO at all, so dropping to "first O" is what keeps those
+ * species in the alignment.
+ */
+export function pickOnePerGenome(hits: PantherHit[]): PantherHit[] {
+  const byCode = new Map<string, PantherHit>()
+  for (const hit of hits) {
+    const current = byCode.get(hit.code)
+    if (!current || (current.type === 'O' && hit.type === 'LDO')) {
+      byCode.set(hit.code, hit)
+    }
+  }
+  return [...byCode.values()]
+}
+
+interface UniProtResponse {
+  results?: {
+    primaryAccession?: string
+    sequence?: { value?: string }
+  }[]
+}
+
+/** `uniprotkb/accessions` -> accession -> sequence */
+export function parseSequences(json: unknown): Map<string, string> {
+  const map = new Map<string, string>()
+  for (const r of (json as UniProtResponse).results ?? []) {
+    if (r.primaryAccession && r.sequence?.value) {
+      map.set(r.primaryAccession, r.sequence.value)
+    }
+  }
+  return map
+}
+
+let genomes: Promise<PantherGenome[]> | undefined
+
+/** The proteome list, fetched once per page and forgotten on failure. */
+export function fetchGenomes() {
+  genomes ??= jsonfetch(`${PANTHER}/supportedgenomes`)
+    .then(parseGenomes)
+    .catch((e: unknown) => {
+      genomes = undefined
+      throw e
+    })
+  return genomes
+}
+
+// UniProt caps one `accessions` call at 100 ids
+const UNIPROT_CHUNK = 100
+
+async function fetchSequences(accessions: string[]) {
+  const map = new Map<string, string>()
+  for (let i = 0; i < accessions.length; i += UNIPROT_CHUNK) {
+    const chunk = accessions.slice(i, i + UNIPROT_CHUNK)
+    const json = await jsonfetch(
+      `${UNIPROT}/accessions?accessions=${chunk.join(',')}&fields=accession,sequence&format=json`,
+    )
+    for (const [acc, seq] of parseSequences(json)) {
+      map.set(acc, seq)
+    }
+  }
+  return map
+}
+
+// strip a version suffix (NM_000546.6) and any GFF ID prefix (gene:TP53), as
+// resolveGeneId does for NCBI
+function cleanCandidate(raw: string) {
+  return raw
+    .trim()
+    .replace(/^\w+:/, '')
+    .replace(/\.\d+$/, '')
+}
+
+/**
+ * One `matchortho` per candidate until PANTHER maps one. A JBrowse feature
+ * carries whatever its GFF/BigBed had — `id()`, `name`, `gene_name` — and only
+ * some of those are names PANTHER knows. `targets` omitted asks for every
+ * genome PANTHER has, which is one call rather than one per genome.
+ */
+async function matchOrthologs(
+  candidates: string[],
+  taxId: number,
+  targets: PantherGenome[] | undefined,
+) {
+  let matched: string | undefined
+  for (const raw of candidates) {
+    const query = cleanCandidate(raw)
+    if (!query) {
+      continue
+    }
+    const params = new URLSearchParams({
+      geneInputList: query,
+      organism: String(taxId),
+      orthologType: 'all',
+    })
+    if (targets) {
+      params.set('targetOrganism', targets.map(t => t.taxId).join(','))
+    }
+    const parsed = parseMatches(
+      await jsonfetch(`${PANTHER}/ortholog/matchortho?${params.toString()}`),
+    )
+    if (!parsed.unmapped) {
+      matched ??= query
+      if (parsed.query) {
+        return { ...parsed, matched: query }
+      }
+    }
+  }
+  return matched === undefined
+    ? undefined
+    : { matched, hits: [], query: undefined }
+}
+
+export interface PantherOrthologs {
+  /** the candidate PANTHER recognised */
+  matched: string
+  /** the query gene as PANTHER knows it, with its UniProt sequence */
+  query?: PantherGene & { sequence: string }
+  rows: OrthologRow[]
+}
+
+/**
+ * The whole PANTHER half of the pipeline: gene -> ortholog rows carrying
+ * labels, accessions and sequences, plus the query gene's own protein for the
+ * query row. Two lookups (genomes, orthologs), one taxonomy batch for the
+ * labels, one UniProt batch for the sequences.
+ *
+ * `taxa`, `exclude` and `limit` mean what they mean for fetchOrthologRows:
+ * `taxa` narrows the targets (omitted, every genome PANTHER has, in its order),
+ * `exclude` drops the query taxon, `limit` caps the rows before their
+ * sequences are fetched.
+ */
+export async function fetchPantherOrthologs({
+  candidates,
+  taxId,
+  taxa,
+  exclude,
+  limit = defaultMaxSpecies,
+  onProgress,
+}: {
+  candidates: string[]
+  taxId: number
+  taxa?: Set<number>
+  exclude?: number
+  limit?: number
+  onProgress: (arg: string) => void
+}): Promise<PantherOrthologs> {
+  const all = await fetchGenomes()
+  const byTaxId = new Map(all.map(g => [g.taxId, g]))
+  const byCode = new Map(all.map(g => [g.code, g]))
+  const queryGenome = byTaxId.get(taxId)
+  if (!queryGenome) {
+    throw new Error(
+      `PANTHER has no reference proteome for taxon ${taxId}. NCBI orthologs cover vertebrates and insects; try that source.`,
+    )
+  }
+  const targets = taxa
+    ? [...taxa].flatMap(t => {
+        const g = byTaxId.get(t)
+        return g && t !== exclude ? [g] : []
+      })
+    : undefined
+
+  onProgress('Matching orthologs at PANTHER...')
+  const match = await matchOrthologs(candidates, taxId, targets)
+  if (!match) {
+    throw new Error(
+      `PANTHER has no entry for ${candidates.join(', ')} in ${queryGenome.longName}. Try the NCBI BLAST tab, which needs no gene identifier.`,
+    )
+  }
+
+  const rank = new Map(targets?.map((t, i) => [t.taxId, i]))
+  const picks = pickOnePerGenome(match.hits)
+    .map(hit => ({ hit, genome: byCode.get(hit.code) }))
+    .filter(
+      (p): p is { hit: PantherHit; genome: PantherGenome } =>
+        !!p.genome &&
+        p.genome.taxId !== exclude &&
+        (targets ? rank.has(p.genome.taxId) : true),
+    )
+    .sort((a, b) =>
+      targets ? rank.get(a.genome.taxId)! - rank.get(b.genome.taxId)! : 0,
+    )
+    .slice(0, limit)
+  if (picks.length < 2) {
+    throw new Error(
+      `Only ${picks.length} PANTHER ortholog(s) found for ${match.matched} — not enough to align`,
+    )
+  }
+
+  onProgress(`Fetching ${picks.length} protein sequences from UniProt...`)
+  const [names, sequences] = await Promise.all([
+    taxonomyNames(picks.map(p => p.genome.taxId)),
+    fetchSequences([
+      ...(match.query ? [match.query.accession] : []),
+      ...picks.map(p => p.hit.accession),
+    ]),
+  ])
+
+  const described = picks.map(({ hit, genome }) => {
+    const info = names.get(genome.taxId)
+    return {
+      hit,
+      genome,
+      name: info ? (info.commonName ?? info.sciname) : genome.name,
+      scientificName: info?.sciname || genome.longName,
+      commonName: info?.commonName,
+    }
+  })
+  const labels = dedupeLabels(described.map(d => d.name))
+  const rows = described
+    .map(({ hit, genome, scientificName, commonName }, i) => ({
+      taxId: genome.taxId,
+      label: labels[i]!,
+      scientificName,
+      commonName,
+      geneId: hit.geneRef,
+      protein: hit.accession,
+      sequence: sequences.get(hit.accession) ?? '',
+    }))
+    .filter(r => r.sequence)
+  if (rows.length < 2) {
+    throw new Error('Could not fetch protein sequences for the orthologs')
+  }
+
+  const querySequence = match.query && sequences.get(match.query.accession)
+  return {
+    matched: match.matched,
+    query:
+      match.query && querySequence
+        ? { ...match.query, sequence: querySequence }
+        : undefined,
+    rows,
+  }
+}
+
+/**
+ * NCBI's names for the taxa, so PANTHER rows are labelled exactly as NCBI rows
+ * are. A failed lookup only costs the labels, which fall back to PANTHER's own
+ * short names, so it is logged rather than thrown.
+ */
+async function taxonomyNames(taxIds: number[]) {
+  try {
+    return await fetchTaxonomyInfo(taxIds)
+  } catch (e) {
+    console.warn('[msaview-orthologs] taxonomy name lookup failed:', e)
+    return new Map<number, { sciname: string; commonName?: string }>()
+  }
+}

package/src/utils/phmmer.ts

@@ -0,0 +1,174 @@
+import { StockholmMSA } from 'msa-parsers'
+
+import { fetchEbiResult, submitEbiJob, waitForEbiJob } from './ebiJobDispatcher'
+
+import type { PhmmerDatabase } from '../LaunchMsaView/components/BlastQuery/consts'
+
+const TOOL = 'hmmer3_phmmer'
+
+export interface PhmmerRow {
+  accession: string
+  id: string
+  sciname: string
+  taxid?: number
+  title?: string
+  /** the matched envelope on the target, e.g. '503-912', absent if unparseable */
+  range?: string
+  /** the row as phmmer aligned it, uppercased with '.' inserts turned into '-' */
+  aligned: string
+}
+
+export interface PhmmerAlignment {
+  rows: PhmmerRow[]
+  /**
+   * the query, placed into the same columns. phmmer does not put the query in
+   * its own output, so this is derived — see buildQueryRow.
+   */
+  queryRow: string
+}
+
+/**
+ * Human-facing link to a job, shown while it runs and on error.
+ *
+ * The category has to be sss: jdispatcher serves its shell with a 200 for any
+ * category, so /pfa/ and /psa/ look fine to a fetch and render "Page Not Found"
+ * in a browser.
+ */
+export function phmmerResultUrl(jobId: string) {
+  return `https://www.ebi.ac.uk/jdispatcher/sss/${TOOL}/summary?jobId=${jobId}`
+}
+
+/** EBI job ids are prefixed with the tool that made them */
+export function isPhmmerJobId(jobId: string) {
+  return jobId.startsWith(`${TOOL}-`)
+}
+
+/**
+ * phmmer aligns every hit to a profile built from the query, one match state
+ * per query residue, and marks those columns 'x' in #=GC RF. So the query's own
+ * row is exactly recoverable: walk RF, consume a query residue at each match
+ * column, gap everywhere else.
+ *
+ * This is the one piece of real logic here rather than a library call, and it
+ * is checked hard: if the match columns do not account for the query exactly,
+ * the columns and the query have drifted apart, and a query row that is off by
+ * even one residue would silently mis-map every column to the genome. Throwing
+ * is much better than drawing that.
+ */
+function buildQueryRow({ rf, query }: { rf: string; query: string }) {
+  let consumed = 0
+  const row = Array.from(rf, c =>
+    c === 'x' ? (query[consumed++] ?? '-') : '-',
+  ).join('')
+  if (consumed !== query.length) {
+    throw new Error(
+      `phmmer alignment has ${consumed} match columns for a query of ${query.length} residues, so the query row cannot be placed`,
+    )
+  }
+  return row
+}
+
+/**
+ * '[subseq from] Albumin OS=Homo sapiens OX=9606 GN=ALB PE=1 SV=2' is what a
+ * UniProt target's #=GS DE looks like. Hits from the non-UniProt databases
+ * phmmer also offers (PDB, AlphaFold, MEROPS...) carry no OS=/OX= at all, so
+ * every field here is optional.
+ */
+function parseDescription(de: string | undefined) {
+  const text = (de ?? '').replace('[subseq from] ', '')
+  const sciname = /OS=(.*?)\s+(?:OX|GN|PE|SV)=/.exec(text)?.[1]
+  const ox = /OX=(\d+)/.exec(text)?.[1]
+  return {
+    sciname: sciname ?? 'unknown',
+    taxid: ox ? Number.parseInt(ox, 10) : undefined,
+    title: text.split(' OS=')[0] || undefined,
+  }
+}
+
+/**
+ * Target names look like 'sp|P02768|ALBU_HUMAN/1-609' for UniProt databases and
+ * like anything at all for the others, so an unrecognized name becomes its own
+ * accession rather than being dropped.
+ */
+function parseName(name: string) {
+  const slash = name.lastIndexOf('/')
+  const range =
+    slash === -1 ? undefined : /^\d+-\d+$/.exec(name.slice(slash + 1))?.[0]
+  const bare = range === undefined ? name : name.slice(0, slash)
+  const parts = bare.split('|')
+  return parts.length === 3
+    ? { accession: parts[1]!, id: parts[2]!, range }
+    : { accession: bare, id: bare, range }
+}
+
+/**
+ * Exported for testing against a captured .sto — the annotation names (RF, the
+ * DE line's OS=/OX=) are the whole risk in this mapping, and nothing else in CI
+ * would notice if HMMER or EBI changed one.
+ */
+export function parsePhmmerAlignment({
+  stockholm,
+  query,
+}: {
+  stockholm: string
+  query: string
+}): PhmmerAlignment {
+  const { gc, gs, seqdata, seqname } = new StockholmMSA(stockholm, 0).getMSA()
+  const rf = gc.RF
+  if (!rf) {
+    throw new Error('phmmer alignment has no #=GC RF line')
+  }
+
+  return {
+    queryRow: buildQueryRow({ rf, query }),
+    rows: seqname.map(name => ({
+      ...parseName(name),
+      ...parseDescription(gs.DE?.[name]?.[0]),
+      // insert columns come back lowercase with '.' for gaps; the MSA renderer
+      // looks colors up by the literal letter, so lowercase would draw
+      // uncolored. The insert columns stay visible as gaps in the query row.
+      aligned: (seqdata[name] ?? '').replaceAll('.', '-').toUpperCase(),
+    })),
+  }
+}
+
+export async function queryPhmmer({
+  query,
+  database,
+  onProgress,
+  onRid,
+}: {
+  query: string
+  database: PhmmerDatabase
+  onProgress: (arg: string) => void
+  onRid: (arg: string) => void
+}) {
+  onProgress('Submitting to EBI phmmer...')
+  const jobId = await submitEbiJob({
+    tool: TOOL,
+    params: {
+      database,
+      sequence: query,
+      // the alignment is the whole point of using phmmer here
+      alignView: 'true',
+    },
+  })
+  onRid(jobId)
+
+  await waitForEbiJob({
+    tool: TOOL,
+    jobId,
+    onCountdown: s => {
+      onProgress(`Re-checking phmmer status in... ${s}`)
+    },
+  })
+
+  const alignment = parsePhmmerAlignment({
+    stockholm: await fetchEbiResult({ tool: TOOL, jobId, type: 'sto' }),
+    query,
+  })
+  if (alignment.rows.length === 0) {
+    throw new Error('No hits found')
+  }
+  return { rid: jobId, ...alignment }
+}

package/src/utils/taxonomyNames.ts

@@ -51,8 +51,13 @@ export interface TaxonomyInfo {
 }
 
 export async function fetchTaxonomyInfo(
-  taxids: number[],
+  taxidsWithRepeats: number[],
 ): Promise<Map<number, TaxonomyInfo>> {
+  // callers pass one taxid per alignment row, and a BLAST hit list is several
+  // rows per species: 100 albumin hits are maybe 50 taxa, and asking as they
+  // came did 100 IndexedDB reads and sent eutils 100 ids for 50 answers. The
+  // result is keyed by taxid, so no caller can tell the difference
+  const taxids = [...new Set(taxidsWithRepeats)]
   const result = new Map<number, TaxonomyInfo>()
   const uncachedTaxids: number[] = []
   const cachedResults = await getCachedTaxonomies(taxids)

package/src/version.ts

@@ -1 +1 @@
-export const version = '3.2.0'
+export const version = '3.4.0'

package/dist/MsaViewPanel/components/LoadingBLAST.d.ts

@@ -1,6 +0,0 @@
-import React from 'react';
-import type { JBrowsePluginMsaViewModel } from '../model';
-declare const LoadingBLAST: ({ model, }: {
-    model: JBrowsePluginMsaViewModel;
-}) => React.JSX.Element;
-export default LoadingBLAST;

package/dist/MsaViewPanel/components/LoadingBLAST.js

@@ -1,26 +0,0 @@
-import React from 'react';
-import { ErrorMessage, LoadingEllipses } from '@jbrowse/core/ui';
-import { Typography } from '@mui/material';
-import { observer } from 'mobx-react';
-import { makeStyles } from 'tss-react/mui';
-import JobLink from './JobLink';
-const useStyles = makeStyles()(theme => ({
-    margin: {
-        padding: 20,
-    },
-    loading: {
-        background: theme.palette.background.paper,
-    },
-}));
-const LoadingBLAST = observer(function LoadingBLAST2({ model, }) {
-    const { progress, rid, error } = model;
-    const { classes } = useStyles();
-    return (React.createElement("div", { className: classes.margin },
-        React.createElement(LoadingEllipses, { message: "Running EBI BLAST", variant: "h5" }),
-        error ? (React.createElement("div", null,
-            rid ? React.createElement(JobLink, { jobId: rid }) : null,
-            React.createElement(ErrorMessage, { error: error }))) : rid ? (React.createElement("div", { className: classes.loading },
-            React.createElement(JobLink, { jobId: rid }),
-            React.createElement(Typography, null, progress))) : (React.createElement(Typography, null, progress || 'Initializing BLAST query'))));
-});
-export default LoadingBLAST;

package/src/MsaViewPanel/components/LoadingBLAST.tsx

@@ -1,48 +0,0 @@
-import React from 'react'
-
-import { ErrorMessage, LoadingEllipses } from '@jbrowse/core/ui'
-import { Typography } from '@mui/material'
-import { observer } from 'mobx-react'
-import { makeStyles } from 'tss-react/mui'
-
-import JobLink from './JobLink'
-
-import type { JBrowsePluginMsaViewModel } from '../model'
-
-const useStyles = makeStyles()(theme => ({
-  margin: {
-    padding: 20,
-  },
-  loading: {
-    background: theme.palette.background.paper,
-  },
-}))
-
-const LoadingBLAST = observer(function LoadingBLAST2({
-  model,
-}: {
-  model: JBrowsePluginMsaViewModel
-}) {
-  const { progress, rid, error } = model
-  const { classes } = useStyles()
-  return (
-    <div className={classes.margin}>
-      <LoadingEllipses message="Running EBI BLAST" variant="h5" />
-      {error ? (
-        <div>
-          {rid ? <JobLink jobId={rid} /> : null}
-          <ErrorMessage error={error} />
-        </div>
-      ) : rid ? (
-        <div className={classes.loading}>
-          <JobLink jobId={rid} />
-          <Typography>{progress}</Typography>
-        </div>
-      ) : (
-        <Typography>{progress || 'Initializing BLAST query'}</Typography>
-      )}
-    </div>
-  )
-})
-
-export default LoadingBLAST