jbrowse-plugin-msaview 2.8.2 → 2.9.0

package/src/utils/ebiBlast.ts

@@ -0,0 +1,114 @@
+import { fetchEbiResult, submitEbiJob, waitForEbiJob } from './ebiJobDispatcher'
+
+import type { BlastHit } from './types'
+import type { BlastDatabase } from '../LaunchMsaView/components/BlastQuery/consts'
+
+const TOOL = 'ncbiblast'
+
+/**
+ * The subset of EBI's ncbiblast JSON result this plugin reads. The service
+ * returns a great deal more per hit (urls, bit scores, e-values, the match
+ * string); only what the MSA rows are built from is typed here.
+ */
+interface EbiBlastJson {
+  hits?: {
+    hit_acc?: string
+    hit_id?: string
+    hit_desc?: string
+    /** the UniProt fields are absent on hits from non-UniProt databases */
+    hit_os?: string
+    hit_uni_de?: string
+    hit_uni_os?: string
+    /** NCBI taxon id, delivered as a string */
+    hit_uni_ox?: string
+    hit_hsps?: { hsp_hseq?: string }[]
+  }[]
+}
+
+/**
+ * Map EBI's hit shape onto the normalized one. Exported for testing against a
+ * captured response — the field names are the whole risk here, and nothing else
+ * in CI would notice if EBI renamed one.
+ */
+export function normalizeEbiBlastHits(result: EbiBlastJson): BlastHit[] {
+  return (result.hits ?? []).map(hit => {
+    const taxid = Number.parseInt(hit.hit_uni_ox ?? '', 10)
+    return {
+      description: [
+        {
+          accession: hit.hit_acc ?? 'unknown',
+          id: hit.hit_id ?? hit.hit_acc ?? 'unknown',
+          sciname: hit.hit_uni_os ?? hit.hit_os ?? 'unknown',
+          taxid: Number.isNaN(taxid) ? undefined : taxid,
+          // hit_uni_de is the bare protein name; hit_desc repeats it with the
+          // OS=/OX=/GN= suffix that makeId already covers with real columns
+          title: hit.hit_uni_de ?? hit.hit_desc,
+        },
+      ],
+      hsps: (hit.hit_hsps ?? []).flatMap(hsp =>
+        hsp.hsp_hseq ? [{ hseq: hsp.hsp_hseq }] : [],
+      ),
+    }
+  })
+}
+
+/**
+ * Human-facing link to a job, shown while it runs and on error — so it has to
+ * be EBI's own results UI, not the REST result endpoint, which does not exist
+ * yet at the moment the link is on screen.
+ */
+export function ebiBlastResultUrl(jobId: string) {
+  return `https://www.ebi.ac.uk/jdispatcher/sss/${TOOL}/summary?jobId=${jobId}`
+}
+
+export async function queryEbiBlastFromJobId({
+  jobId,
+  onProgress,
+}: {
+  jobId: string
+  onProgress: (arg: string) => void
+}) {
+  onProgress(`Checking BLAST status for job: ${jobId}...`)
+  await waitForEbiJob({
+    tool: TOOL,
+    jobId,
+    onCountdown: s => {
+      onProgress(`Re-checking BLAST status in... ${s}`)
+    },
+  })
+
+  const hits = normalizeEbiBlastHits(
+    JSON.parse(
+      await fetchEbiResult({ tool: TOOL, jobId, type: 'json' }),
+    ) as EbiBlastJson,
+  )
+  if (hits.length === 0) {
+    throw new Error('No hits found')
+  }
+  return { rid: jobId, hits }
+}
+
+export async function queryEbiBlast({
+  query,
+  blastDatabase,
+  onProgress,
+  onRid,
+}: {
+  query: string
+  blastDatabase: BlastDatabase
+  onProgress: (arg: string) => void
+  onRid: (arg: string) => void
+}) {
+  onProgress('Submitting to EBI BLAST...')
+  const jobId = await submitEbiJob({
+    tool: TOOL,
+    params: {
+      program: 'blastp',
+      stype: 'protein',
+      database: blastDatabase,
+      sequence: query,
+    },
+  })
+  onRid(jobId)
+  return queryEbiBlastFromJobId({ jobId, onProgress })
+}

package/src/utils/ebiJobDispatcher.ts

@@ -0,0 +1,91 @@
+import { textfetch } from './fetch'
+import { pollLoop } from './poll'
+import { readLocalStorage } from './useLocalStorage'
+
+/**
+ * EBI's Job Dispatcher REST services (clustalo, muscle, ncbiblast, ...) all
+ * speak the same run/status/result protocol, so the transport lives here and
+ * each tool only supplies its own parameters and result types.
+ *
+ * Unlike NCBI's Blast.cgi these endpoints send `Access-Control-Allow-Origin: *`,
+ * which is why the BLAST backend moved here — see docs/blast.md.
+ */
+export const EBI_BASE = 'https://www.ebi.ac.uk/Tools/services/rest'
+
+/**
+ * EBI asks for a contact address on every submission so they can reach whoever
+ * is generating the load. A deployment that sends real volume should point this
+ * at its own maintainer via the BLAST settings dialog — otherwise every
+ * msaview job in the world is attributed to one person.
+ */
+export const EBI_EMAIL_STORAGE_KEY = 'msa-ebiContactEmail'
+export const DEFAULT_EBI_EMAIL = 'colin.diesh@gmail.com'
+
+export function getEbiEmail() {
+  const configured = readLocalStorage(
+    EBI_EMAIL_STORAGE_KEY,
+    DEFAULT_EBI_EMAIL,
+  ).trim()
+  return configured || DEFAULT_EBI_EMAIL
+}
+
+/** Statuses that mean the job is over and produced no result. */
+const FAILED_STATUSES = new Set(['ERROR', 'FAILURE', 'NOT_FOUND'])
+
+export async function submitEbiJob({
+  tool,
+  params,
+}: {
+  tool: string
+  params: Record<string, string>
+}) {
+  const jobId = await textfetch(`${EBI_BASE}/${tool}/run`, {
+    method: 'POST',
+    body: new URLSearchParams({ email: getEbiEmail(), ...params }),
+  })
+  return jobId.trim()
+}
+
+export async function waitForEbiJob({
+  tool,
+  jobId,
+  intervalSeconds = 10,
+  onCountdown,
+}: {
+  tool: string
+  jobId: string
+  intervalSeconds?: number
+  onCountdown: (secondsRemaining: number) => void
+}) {
+  await pollLoop({
+    intervalSeconds,
+    onCountdown,
+    check: async () => {
+      // exact match, not includes(): a job whose status is ERROR must not be
+      // able to poll forever waiting for a FINISHED that will never arrive
+      const status = (
+        await textfetch(`${EBI_BASE}/${tool}/status/${jobId}`)
+      ).trim()
+      if (status === 'FINISHED') {
+        return true
+      }
+      if (FAILED_STATUSES.has(status)) {
+        throw new Error(`EBI ${tool} job ${jobId} returned status ${status}`)
+      }
+      // RUNNING, QUEUED, PENDING and anything else EBI adds later
+      return false
+    },
+  })
+}
+
+export async function fetchEbiResult({
+  tool,
+  jobId,
+  type,
+}: {
+  tool: string
+  jobId: string
+  type: string
+}) {
+  return textfetch(`${EBI_BASE}/${tool}/result/${jobId}/${type}`)
+}

package/src/utils/fetch.ts

@@ -1,5 +1,30 @@
+function hostOf(url: string) {
+  try {
+    return new URL(url).host
+  } catch {
+    // a relative url, e.g. a same-origin proxy path
+    return url
+  }
+}
+
 export async function handleFetch(url: string, args?: RequestInit) {
-  const response = await fetch(url, args)
+  let response: Response
+  try {
+    response = await fetch(url, args)
+  } catch (e) {
+    // fetch rejects with a bare "TypeError: Failed to fetch" for every network
+    // level failure, including a cross-origin response the browser refused to
+    // hand over. That is not a hypothetical: NCBI's Blast.cgi stopped sending
+    // Access-Control-Allow-Origin to third-party origins, and the raw TypeError
+    // told users nothing at all about why. See docs/blast.md.
+    if (e instanceof TypeError) {
+      throw new Error(
+        `Could not reach ${hostOf(url)} — the request failed at the network level. This is usually a CORS restriction (the server refused to let this site read the response), an offline connection, or a blocked request.`,
+        { cause: e },
+      )
+    }
+    throw e
+  }
 
   if (!response.ok) {
     throw new Error(

package/src/utils/msa.ts

@@ -1,10 +1,6 @@
-import { textfetch } from './fetch'
-import { pollLoop } from './poll'
+import { fetchEbiResult, submitEbiJob, waitForEbiJob } from './ebiJobDispatcher'
 
-import type { MsaAlgorithm } from '../LaunchMsaView/components/NCBIBlastQuery/consts'
-
-const base = `https://www.ebi.ac.uk/Tools/services/rest`
-const email = 'colin.diesh@gmail.com'
+import type { MsaAlgorithm } from '../LaunchMsaView/components/BlastQuery/consts'
 
 const algorithms: Record<
   MsaAlgorithm,
@@ -15,54 +11,27 @@ const algorithms: Record<
   }
 > = {
   clustalo: {
-    params: { email },
+    params: {},
     msaResult: 'aln-clustal_num',
     treeResult: 'phylotree',
   },
   muscle: {
-    params: { email, format: 'clw', tree: 'tree1' },
+    params: { format: 'clw', tree: 'tree1' },
     msaResult: 'fa',
     treeResult: 'phylotree',
   },
   kalign: {
-    params: { email, stype: 'protein' },
+    params: { stype: 'protein' },
     msaResult: 'fa',
     treeResult: 'phylotree',
   },
   mafft: {
-    params: { email, stype: 'protein' },
+    params: { stype: 'protein' },
     msaResult: 'fa',
     treeResult: 'phylotree',
   },
 }
 
-async function wait({
-  onProgress,
-  jobId,
-  algorithm,
-}: {
-  jobId: string
-  algorithm: MsaAlgorithm
-  onProgress: (arg: string) => void
-}) {
-  await pollLoop({
-    intervalSeconds: 10,
-    onCountdown: s => {
-      onProgress(`Re-checking MSA status in... ${s}`)
-    },
-    check: async () => {
-      const result = await textfetch(`${base}/${algorithm}/status/${jobId}`)
-      if (result.includes('FINISHED')) {
-        return true
-      }
-      if (result.includes('FAILURE')) {
-        throw new Error(`Failed to run: jobId ${jobId}`)
-      }
-      return false
-    },
-  })
-}
-
 export async function launchMSA({
   algorithm,
   sequence,
@@ -76,17 +45,27 @@ export async function launchMSA({
 
   onProgress(`Launching ${algorithm} MSA...`)
 
-  const jobId = await textfetch(`${base}/${algorithm}/run`, {
-    method: 'POST',
-    body: new URLSearchParams({ ...config.params, sequence }),
+  const jobId = await submitEbiJob({
+    tool: algorithm,
+    params: { ...config.params, sequence },
+  })
+  await waitForEbiJob({
+    tool: algorithm,
+    jobId,
+    onCountdown: s => {
+      onProgress(`Re-checking MSA status in... ${s}`)
+    },
   })
-  await wait({ jobId, algorithm, onProgress })
   return {
-    msa: await textfetch(
-      `${base}/${algorithm}/result/${jobId}/${config.msaResult}`,
-    ),
-    tree: await textfetch(
-      `${base}/${algorithm}/result/${jobId}/${config.treeResult}`,
-    ),
+    msa: await fetchEbiResult({
+      tool: algorithm,
+      jobId,
+      type: config.msaResult,
+    }),
+    tree: await fetchEbiResult({
+      tool: algorithm,
+      jobId,
+      type: config.treeResult,
+    }),
   }
 }

package/src/utils/types.ts

@@ -6,17 +6,12 @@ export interface BlastHitDescription {
   title?: string
 }
 
-export interface BlastResults {
-  BlastOutput2: {
-    report: {
-      results: {
-        search: {
-          hits: {
-            description: BlastHitDescription[]
-            hsps: { hseq: string }[]
-          }[]
-        }
-      }
-    }
-  }[]
+/**
+ * The shape utils/ebiBlast.ts normalizes EBI's hits into. It keeps the field
+ * names NCBI's JSON2 used, so everything downstream of the search — row naming,
+ * taxonomy lookup, the MSA — was left untouched when the backend moved.
+ */
+export interface BlastHit {
+  description: BlastHitDescription[]
+  hsps: { hseq: string }[]
 }

package/src/utils/useLocalStorage.ts

@@ -5,7 +5,7 @@ import { useState } from 'react'
 // throw "(0, PR.useLocalStorage) is not a function" on hosts built during that
 // window. Same failure mode as `defaultCodonTable`; keeping our own copy takes
 // this plugin out of the whack-a-mole.
-function readLocalStorage<T>(key: string, initialValue: T): T {
+export function readLocalStorage<T>(key: string, initialValue: T): T {
   try {
     const item = globalThis.localStorage.getItem(key)
     return item === null ? initialValue : (JSON.parse(item) as T)

package/src/version.ts

@@ -1 +1 @@
-export const version = '2.8.2'
+export const version = '2.9.0'

package/dist/LaunchMsaView/components/NCBIBlastQuery/NCBIBlastAutomaticPanel.d.ts

@@ -1,10 +0,0 @@
-import React from 'react';
-import type { AbstractTrackModel, Feature } from '@jbrowse/core/util';
-declare const NCBIBlastAutomaticPanel: ({ handleClose, feature, model, children, baseUrl, }: {
-    model: AbstractTrackModel;
-    feature: Feature;
-    baseUrl: string;
-    handleClose: () => void;
-    children: React.ReactNode;
-}) => React.JSX.Element;
-export default NCBIBlastAutomaticPanel;

package/dist/LaunchMsaView/components/NCBIBlastQuery/NCBIBlastRIDPanel.js

@@ -1,74 +0,0 @@
-import React, { useState } from 'react';
-import { Typography } from '@mui/material';
-import { observer } from 'mobx-react';
-import { makeStyles } from 'tss-react/mui';
-import MsaAlgorithmSelect from './MsaAlgorithmSelect';
-import { blastLaunchView } from './blastLaunchView';
-import ExternalLink from '../../../components/ExternalLink';
-import TextField2 from '../../../components/TextField2';
-import { getBlastViewTitle, getLinearGenomeView } from '../../util';
-import LaunchPanelContent from '../LaunchPanelContent';
-import SubmitCancelActions from '../SubmitCancelActions';
-import TranscriptSelector from '../TranscriptSelector';
-import { useTranscriptSelection } from '../useTranscriptSelection';
-const useStyles = makeStyles()({
-    marginBottom: {
-        marginBottom: 16,
-    },
-    ridField: {
-        width: 150,
-    },
-    infoText: {
-        marginTop: 20,
-    },
-});
-const NCBIBlastRIDPanel = observer(function ({ handleClose, feature, model, children, baseUrl, }) {
-    const { classes } = useStyles();
-    const view = getLinearGenomeView(model);
-    const [launchViewError, setLaunchViewError] = useState();
-    const [rid, setRid] = useState('');
-    const [selectedMsaAlgorithm, setSelectedMsaAlgorithm] = useState('clustalo');
-    const transcriptSelection = useTranscriptSelection({ feature, view });
-    const { selectedTranscript, proteinSequence } = transcriptSelection;
-    const e = transcriptSelection.error ?? launchViewError;
-    const trimmedRid = rid.trim();
-    return (React.createElement(React.Fragment, null,
-        React.createElement(LaunchPanelContent, { error: e },
-            children,
-            React.createElement(Typography, { variant: "body2", className: classes.marginBottom }, "Enter the RID (Request ID) from a previously submitted NCBI BLAST query. You can find the RID in the BLAST results page URL or at the top of the results page. RIDs are typically valid for 24-36 hours after submission."),
-            React.createElement(TextField2, { variant: "outlined", label: "BLAST RID", placeholder: "e.g., ABC12345", fullWidth: true, className: classes.marginBottom, value: rid, onChange: event => {
-                    setRid(event.target.value);
-                } }),
-            trimmedRid ? (React.createElement(Typography, { variant: "body2", className: classes.marginBottom },
-                React.createElement(ExternalLink, { href: `${baseUrl}?CMD=Get&RID=${trimmedRid}` }, "View BLAST results on NCBI"))) : null,
-            React.createElement(MsaAlgorithmSelect, { className: classes.ridField, value: selectedMsaAlgorithm, onChange: setSelectedMsaAlgorithm }),
-            React.createElement(TranscriptSelector, { feature: feature, ...transcriptSelection }),
-            React.createElement(Typography, { className: classes.infoText }, "This will fetch the BLAST results for the provided RID and run them through a multiple sequence alignment. The protein sequence from the selected transcript will be added as the query sequence in the MSA.")),
-        React.createElement(SubmitCancelActions, { submitDisabled: !proteinSequence || !trimmedRid, onSubmit: () => {
-                try {
-                    if (selectedTranscript && trimmedRid) {
-                        setLaunchViewError(undefined);
-                        blastLaunchView({
-                            feature: selectedTranscript,
-                            view,
-                            newViewTitle: getBlastViewTitle(feature, selectedTranscript),
-                            blastParams: {
-                                baseUrl,
-                                blastProgram: 'blastp',
-                                blastDatabase: 'nr',
-                                msaAlgorithm: selectedMsaAlgorithm,
-                                selectedTranscript,
-                                proteinSequence,
-                                rid: trimmedRid,
-                            },
-                        });
-                        handleClose();
-                    }
-                }
-                catch (e) {
-                    console.error(e);
-                    setLaunchViewError(e);
-                }
-            }, onCancel: handleClose })));
-});
-export default NCBIBlastRIDPanel;

package/dist/LaunchMsaView/components/NCBIBlastQuery/NCBISettingsDialog.d.ts

@@ -1,5 +0,0 @@
-import React from 'react';
-export default function NCBISettingsDialog({ handleClose, baseUrl, }: {
-    handleClose: (arg?: string) => void;
-    baseUrl: string;
-}): React.JSX.Element;

package/dist/LaunchMsaView/components/NCBIBlastQuery/consts.d.ts

@@ -1,7 +0,0 @@
-export declare const BASE_BLAST_URL = "https://blast.ncbi.nlm.nih.gov/Blast.cgi";
-export declare const msaAlgorithms: readonly ["clustalo", "muscle", "kalign", "mafft"];
-export type MsaAlgorithm = (typeof msaAlgorithms)[number];
-export declare const blastDatabaseOptions: readonly ["nr", "nr_cluster_seq"];
-export type BlastDatabase = (typeof blastDatabaseOptions)[number];
-export declare const blastPrograms: readonly ["blastp", "quick-blastp"];
-export type BlastProgram = (typeof blastPrograms)[number];

package/dist/LaunchMsaView/components/NCBIBlastQuery/consts.js

@@ -1,4 +0,0 @@
-export const BASE_BLAST_URL = 'https://blast.ncbi.nlm.nih.gov/Blast.cgi';
-export const msaAlgorithms = ['clustalo', 'muscle', 'kalign', 'mafft'];
-export const blastDatabaseOptions = ['nr', 'nr_cluster_seq'];
-export const blastPrograms = ['blastp', 'quick-blastp'];

package/dist/MsaViewPanel/components/RIDLink.d.ts

@@ -1,6 +0,0 @@
-import React from 'react';
-declare function RIDLink({ baseUrl, rid }: {
-    rid: string;
-    baseUrl: string;
-}): React.JSX.Element;
-export default RIDLink;

package/dist/MsaViewPanel/components/RIDLink.js

@@ -1,12 +0,0 @@
-import React from 'react';
-import { Typography } from '@mui/material';
-import ExternalLink from '../../components/ExternalLink';
-function RIDLink({ baseUrl, rid }) {
-    return (React.createElement(Typography, null,
-        "RID ",
-        rid,
-        " (",
-        React.createElement(ExternalLink, { href: `${baseUrl}?CMD=Get&RID=${rid}` }, "see status"),
-        ")"));
-}
-export default RIDLink;

package/dist/utils/ncbiBlast.d.ts

@@ -1,30 +0,0 @@
-import type { BlastDatabase, BlastProgram } from '../LaunchMsaView/components/NCBIBlastQuery/consts';
-export declare function queryBlastFromRid({ rid, baseUrl, onProgress, }: {
-    rid: string;
-    baseUrl: string;
-    onProgress: (arg: string) => void;
-}): Promise<{
-    rid: string;
-    hits: {
-        description: import("./types").BlastHitDescription[];
-        hsps: {
-            hseq: string;
-        }[];
-    }[];
-}>;
-export declare function queryBlast({ query, blastDatabase, blastProgram, baseUrl, onProgress, onRid, }: {
-    query: string;
-    blastDatabase: BlastDatabase;
-    blastProgram: BlastProgram;
-    baseUrl: string;
-    onProgress: (arg: string) => void;
-    onRid: (arg: string) => void;
-}): Promise<{
-    rid: string;
-    hits: {
-        description: import("./types").BlastHitDescription[];
-        hsps: {
-            hseq: string;
-        }[];
-    }[];
-}>;

package/dist/utils/ncbiBlast.js

@@ -1,84 +0,0 @@
-import { jsonfetch, textfetch } from './fetch';
-import { pollLoop } from './poll';
-export async function queryBlastFromRid({ rid, baseUrl, onProgress, }) {
-    onProgress(`Checking BLAST status for RID: ${rid}...`);
-    await waitForRid({
-        rid,
-        onProgress,
-        baseUrl,
-    });
-    const ret = await jsonfetch(`${baseUrl}?CMD=Get&RID=${rid}&FORMAT_TYPE=JSON2_S&FORMAT_OBJECT=Alignment`);
-    return {
-        rid,
-        hits: ret.BlastOutput2[0]?.report.results.search.hits ?? [],
-    };
-}
-export async function queryBlast({ query, blastDatabase, blastProgram, baseUrl, onProgress, onRid, }) {
-    onProgress('Submitting to NCBI BLAST...');
-    const { rid } = await initialQuery({
-        query,
-        blastDatabase,
-        blastProgram,
-        baseUrl,
-    });
-    onRid(rid);
-    return queryBlastFromRid({ rid, baseUrl, onProgress });
-}
-async function initialQuery({ query, blastProgram, blastDatabase, baseUrl, }) {
-    const res = await textfetch(baseUrl, {
-        method: 'POST',
-        body: new URLSearchParams({
-            CMD: 'Put',
-            PROGRAM: blastProgram === 'quick-blastp' ? 'blastp' : blastProgram,
-            DATABASE: blastDatabase,
-            QUERY: query,
-            ...(blastDatabase === 'nr_cluster_seq'
-                ? {
-                    CLUSTERED_DB: 'on',
-                    DB_TYPE: 'Experimental Databases',
-                }
-                : {}),
-            ...(blastProgram === 'quick-blastp'
-                ? { BLAST_PROGRAMS: 'kmerBlastp' }
-                : {}),
-        }),
-    });
-    // the initial submission/query to the BLAST "REST API" does not return JSON
-    // as a response (e.g. FORMAT_TYPE=JSON does not work), so the RID is
-    // literally parsed from the text of the HTML that is returned
-    const rid = /^ {4}RID = (.*$)/m.exec(res)?.[1];
-    const rtoe = /^ {4}RTOE = (.*$)/m.exec(res)?.[1];
-    if (!rid) {
-        throw new Error('Failed to get RID from BLAST request');
-    }
-    return {
-        rid,
-        rtoe,
-    };
-}
-async function waitForRid({ rid, onProgress, baseUrl, }) {
-    await pollLoop({
-        intervalSeconds: 20,
-        onCountdown: s => {
-            onProgress(`Re-checking BLAST status in... ${s}`);
-        },
-        check: async () => {
-            const res = await textfetch(`${baseUrl}?CMD=Get&FORMAT_OBJECT=SearchInfo&RID=${rid}`);
-            const status = /\s+Status=(\S+)/m.exec(res)?.[1];
-            const hasHits = /\s+ThereAreHits=yes/m.test(res);
-            if (status === 'WAITING') {
-                return false;
-            }
-            if (status === 'FAILED') {
-                throw new Error(`BLAST ${rid} failed`);
-            }
-            if (status === 'READY') {
-                if (hasHits) {
-                    return true;
-                }
-                throw new Error('No hits found');
-            }
-            throw new Error(`BLAST ${rid} returned unexpected status: ${status ?? 'unknown'}`);
-        },
-    });
-}