jbrowse-plugin-graphgenomeviewer 1.0.0 → 3.0.3

package/src/RgfaTabixAdapter/RgfaTabixAdapter.test.ts

@@ -0,0 +1,285 @@
+import { firstValueFrom } from 'rxjs'
+import { toArray } from 'rxjs/operators'
+
+import Adapter from './RgfaTabixAdapter.ts'
+import configSchema from './configSchema.ts'
+
+// Built by scripts/build_rgfa_tabix.sh from the minigraph rGFA of four E. coli
+// strains at jbrowse.org/demos/ecoli_pangenome/ecoli_rgfa_slice.gfa (the
+// pangenome tutorial's figure data): 161 segments across four stable sequences,
+// ranks 0-3. Named rgfa_ecoli rather than ecoli_rgfa because the repo's
+// .gitignore ignores `ecoli_*` (the multi-GB pangenome build dirs).
+const prefix = require
+  .resolve('./test_data/rgfa_ecoli.segs.bed.gz')
+  .replace(/\.segs\.bed\.gz$/, '')
+
+function makeAdapter() {
+  const local = (path: string) => ({
+    localPath: path,
+    locationType: 'LocalPathLocation',
+  })
+  return new Adapter(
+    configSchema.create({
+      segmentsLocation: local(`${prefix}.segs.bed.gz`),
+      segmentsIndex: { location: local(`${prefix}.segs.bed.gz.tbi`) },
+      linksLocation: local(`${prefix}.links.bed.gz`),
+      linksIndex: { location: local(`${prefix}.links.bed.gz.tbi`) },
+    }),
+  )
+}
+
+// The graph names its stable sequences PanSN (`K12#1#chr`); an assembly asking
+// for them names them `chr` in assembly `K12`.
+const k12 = {
+  refName: 'chr',
+  assemblyName: 'K12',
+  start: 993236,
+  end: 997574,
+}
+
+test('getRefNames lists the stable sequences', async () => {
+  expect(await makeAdapter().getRefNames()).toEqual([
+    'CFT073#1#chr',
+    'K12#1#chr',
+    'NCTC86#1#chr',
+    'Sakai#1#chr',
+  ])
+})
+
+test('getFeatures returns the segments on a stable sequence', async () => {
+  const features = await firstValueFrom(
+    makeAdapter().getFeatures(k12).pipe(toArray()),
+  )
+  expect(
+    features.map(f => ({
+      name: f.get('name'),
+      start: f.get('start'),
+      end: f.get('end'),
+      rank: f.get('rank'),
+    })),
+  ).toEqual([
+    { name: 's322', start: 993236, end: 993310, rank: 0 },
+    { name: 's323', start: 993310, end: 997574, rank: 0 },
+  ])
+})
+
+test('getFeatures resolves a PanSN name given unqualified', async () => {
+  const qualified = await firstValueFrom(
+    makeAdapter()
+      .getFeatures({ ...k12, refName: 'K12#1#chr' })
+      .pipe(toArray()),
+  )
+  expect(qualified.map(f => f.get('name'))).toEqual(['s322', 's323'])
+})
+
+test('getFeatures is empty for a sequence the graph does not anchor to', async () => {
+  const features = await firstValueFrom(
+    makeAdapter()
+      .getFeatures({ ...k12, assemblyName: 'volvox', refName: 'ctgA' })
+      .pipe(toArray()),
+  )
+  expect(features).toEqual([])
+})
+
+// The point of the two-file index: a region on the reference backbone reaches
+// the rank>0 segments hanging off it, which live at their own coordinates on
+// another stable sequence (here CFT073) and so can never be found by a
+// coordinate query on this region.
+test('getSubgraph reaches off-reference neighbours', async () => {
+  const gfa = await makeAdapter().getSubgraph(k12)
+  expect(gfa).toBe(
+    [
+      'H\tVN:Z:1.0',
+      'S\ts1727\t*\tLN:i:226\tSN:Z:CFT073#1#chr\tSO:i:1044024\tSR:i:2',
+      'S\ts1728\t*\tLN:i:75\tSN:Z:CFT073#1#chr\tSO:i:1048515\tSR:i:2',
+      'S\ts322\t*\tLN:i:74\tSN:Z:K12#1#chr\tSO:i:993236\tSR:i:0',
+      'S\ts323\t*\tLN:i:4264\tSN:Z:K12#1#chr\tSO:i:993310\tSR:i:0',
+      'S\ts324\t*\tLN:i:7093\tSN:Z:K12#1#chr\tSO:i:997574\tSR:i:0',
+      'L\ts1727\t+\ts323\t+\t0M',
+      'L\ts322\t+\ts323\t+\t0M',
+      'L\ts323\t+\ts1728\t+\t0M',
+      'L\ts323\t+\ts324\t+\t0M',
+    ].join('\n'),
+  )
+})
+
+test('getSubgraph is byte-identical across calls', async () => {
+  const adapter = makeAdapter()
+  expect(await adapter.getSubgraph(k12)).toBe(await adapter.getSubgraph(k12))
+})
+
+// context expands by whole link hops, following each newly reached segment from
+// its own stable coordinates — which every link row carries.
+test('getSubgraph context adds another hop', async () => {
+  const adapter = makeAdapter()
+  const ids = (gfa: string) =>
+    gfa
+      .split('\n')
+      .filter(l => l.startsWith('S'))
+      .map(l => l.split('\t')[1]!)
+
+  const near = ids(await adapter.getSubgraph(k12))
+  const far = ids(await adapter.getSubgraph(k12, { hops: 1 }))
+  expect(far.length).toBeGreaterThan(near.length)
+  expect(far).toEqual(expect.arrayContaining(near))
+})
+
+// A hop is one query per off-reference segment, so a cut the view has replaced
+// has a lot left to read. The signal reaches every query, and one that gave up
+// before it started reads nothing.
+test('getSubgraph stops for its signal', async () => {
+  const gaveUp = new AbortController()
+  gaveUp.abort()
+  await expect(
+    makeAdapter().getSubgraph(k12, { hops: 1, signal: gaveUp.signal }),
+  ).rejects.toThrow(/abort/i)
+})
+
+test('getFeatures stops for its signal', async () => {
+  const gaveUp = new AbortController()
+  gaveUp.abort()
+  await expect(
+    firstValueFrom(
+      makeAdapter().getFeatures(k12, { signal: gaveUp.signal }).pipe(toArray()),
+    ),
+  ).rejects.toThrow(/abort/i)
+})
+
+// A bare header drew as an empty pane with no error, which is what a missing
+// assemblyNameToPanSN entry (hs1 without CHM13) looked like.
+test('getSubgraph refuses a sequence the index does not hold, saying how to map one', async () => {
+  await expect(
+    makeAdapter().getSubgraph({
+      ...k12,
+      assemblyName: 'volvox',
+      refName: 'ctgA',
+    }),
+  ).rejects.toThrow(
+    /volvox ctgA is not in this graph's index.*assemblyNameToPanSN/,
+  )
+})
+
+// The carriage half of the index, which no rGFA has: built by
+// scripts/build_pggb_tabix.sh in jbrowse-components from the same five-strain
+// pggb graph the tutorial uses, sliced to the IS5 element at K12
+// chr:1,299,499-1,300,693. 24 segments spanning every carrier count from 1 to
+// 5, so the fixture covers a strain-private allele and core backbone in one
+// window.
+const pggbPrefix = require
+  .resolve('./test_data/pggb_ecoli.segs.bed.gz')
+  .replace(/\.segs\.bed\.gz$/, '')
+
+function makePggbAdapter() {
+  const local = (path: string) => ({
+    localPath: path,
+    locationType: 'LocalPathLocation',
+  })
+  return new Adapter(
+    configSchema.create({
+      segmentsLocation: local(`${pggbPrefix}.segs.bed.gz`),
+      segmentsIndex: { location: local(`${pggbPrefix}.segs.bed.gz.tbi`) },
+      linksLocation: local(`${pggbPrefix}.links.bed.gz`),
+      linksIndex: { location: local(`${pggbPrefix}.links.bed.gz.tbi`) },
+    }),
+  )
+}
+
+const is5 = {
+  refName: 'chr',
+  assemblyName: 'K12',
+  start: 1299260,
+  end: 1300800,
+}
+
+// The lane this exists for: color a linear track by how many haplotypes carry
+// each segment. Before this the tag column reached the graph view's node popup
+// and stopped there, so a feature could say its rank but not its carriage.
+test('getFeatures carries SM:Z: onto the feature', async () => {
+  const features = await firstValueFrom(
+    makePggbAdapter().getFeatures(is5).pipe(toArray()),
+  )
+  const private1199 = features.find(f => f.get('name') === '196827')!
+  expect(private1199.get('end') - private1199.get('start')).toBe(1199)
+  expect(private1199.get('samples')).toEqual(['K12.1'])
+  expect(private1199.get('carriers')).toBe(1)
+
+  const core = features.find(f => f.get('name') === '196843')!
+  expect(core.get('samples')).toEqual([
+    'K12.1',
+    'Sakai.1',
+    'CFT073.1',
+    'NCTC86.1',
+    'IAI39.1',
+  ])
+  expect(core.get('carriers')).toBe(5)
+})
+
+// The other half of the same fixture: the graph route reads carriage off the
+// synthesized S-line, and rgfaBed.test.ts asserts that from a hand-written row.
+// This is the same claim against a real indexed file.
+test('getSubgraph keeps the tag on the S-line it synthesizes', async () => {
+  const gfa = await makePggbAdapter().getSubgraph(is5)
+  expect(gfa.split('\n')).toContain(
+    'S\t196827\t*\tLN:i:1199\tSN:Z:K12#1#chr\tSO:i:1299498\tSR:i:0\tSM:Z:K12.1',
+  )
+})
+
+// An rGFA has no sixth column, so the attribute is absent rather than 0 — a
+// color expression reading `carriers` on such a track gets undefined, not a
+// claim that nothing carries the segment.
+test('getFeatures omits carriage on a graph that states none', async () => {
+  const features = await firstValueFrom(
+    makeAdapter().getFeatures(k12).pipe(toArray()),
+  )
+  expect(features.length).toBeGreaterThan(0)
+  expect(features.every(f => f.get('samples') === undefined)).toBe(true)
+  expect(features.every(f => f.get('carriers') === undefined)).toBe(true)
+})
+
+// Minigraph-Cactus writes PanSN stable names, so HPRC release 2 calls the
+// reference `GRCh38#0#chr1` while the JBrowse assembly is `hg38`. This reuses
+// the `assemblyNameToPanSN` slot and helper the all-vs-all PAF adapters use.
+// Matching on the bare contig instead is not an option: that graph also carries
+// `CHM13#0#chr1`, so it would silently resolve to the wrong sample.
+function makeMappedAdapter(assemblyNameToPanSN: Record<string, string>) {
+  const local = (path: string) => ({
+    localPath: path,
+    locationType: 'LocalPathLocation',
+  })
+  return new Adapter(
+    configSchema.create({
+      segmentsLocation: local(`${prefix}.segs.bed.gz`),
+      segmentsIndex: { location: local(`${prefix}.segs.bed.gz.tbi`) },
+      linksLocation: local(`${prefix}.links.bed.gz`),
+      linksIndex: { location: local(`${prefix}.links.bed.gz.tbi`) },
+      assemblyNameToPanSN,
+    }),
+  )
+}
+
+// the fixture's sample is `K12`, so an assembly named anything else needs the map
+const renamed = { ...k12, assemblyName: 'ecoli_k12' }
+
+test('maps an assembly name to a differing PanSN sample prefix', async () => {
+  const features = await firstValueFrom(
+    makeMappedAdapter({ ecoli_k12: 'K12' })
+      .getFeatures(renamed)
+      .pipe(toArray()),
+  )
+  expect(features.length).toBeGreaterThan(0)
+  expect(features.every(f => f.get('refName') === 'chr')).toBe(true)
+})
+
+test('without the mapping the same query resolves nothing', async () => {
+  const features = await firstValueFrom(
+    makeMappedAdapter({}).getFeatures(renamed).pipe(toArray()),
+  )
+  expect(features).toEqual([])
+})
+
+test('getSubgraph resolves through the mapping too', async () => {
+  const gfa = await makeMappedAdapter({ ecoli_k12: 'K12' }).getSubgraph(renamed)
+  expect(gfa.split('\n').filter(l => l.startsWith('S')).length).toBeGreaterThan(
+    0,
+  )
+})

package/src/RgfaTabixAdapter/RgfaTabixAdapter.ts

@@ -0,0 +1,194 @@
+import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter'
+import { SimpleFeature, updateStatus } from '@jbrowse/core/util'
+import { ObservableCreate } from '@jbrowse/core/util/rxjs'
+
+import { PanSNRefNames, openTabixSlot } from '../panSNTabix.ts'
+import {
+  formatSubgraph,
+  linkKey,
+  parseLinkLine,
+  parseSegmentLine,
+  segmentSamples,
+} from './rgfaBed.ts'
+
+import type { RgfaTabixAdapterConfig } from './configSchema.ts'
+import type { SubgraphAdapterOptions } from '../GetSubgraph.ts'
+import type { RgfaLink, RgfaSegment } from './rgfaBed.ts'
+import type PluginManager from '@jbrowse/core/PluginManager'
+import type { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter'
+import type { getSubAdapterType } from '@jbrowse/core/data_adapters/dataAdapterCache'
+import type { Feature } from '@jbrowse/core/util'
+import type { Region } from '@jbrowse/core/util/types'
+
+// What a hop follows: alleles, never the backbone. A rank-0 segment reached
+// through a link is flanking backbone, usually outside the window, so querying
+// its links walks the reference away from the region, brings back more backbone,
+// and closes nothing. Every segment that closes a bubble is an allele interior,
+// and an allele interior is rank > 0 in both index formats (rGFA's SR tag, and
+// pggb's rank 0-or-1 against the anchor path). Reached backbone still becomes a
+// node and its edge still draws; it is only not walked through.
+//
+// This is gfabase's rule, arrived at from the other end. `gfabase sub` expands
+// from a seed either to the whole connected component (`--connected`, which its
+// own README calls overkill) or with `--cutpoints 1`, which takes the biconnected
+// component by *stopping at cutpoints*, the segments every end-to-end walk of
+// the chromosome must traverse. In a reference-anchored pangenome the cutpoints
+// are the backbone, and rank is what the index already states about it, so
+// `rank > 0` is that stopping rule without a connectivity index to build. The
+// third tool is the same shape again: `gfatools view -R` maps the region onto its
+// precomputed bubble decomposition and returns whole overlapping bubbles
+// (gfa_query_by_reg), which is complete for the same reason. Bandage's own
+// `--distance` is a node-step radius, but Bandage holds the entire graph in
+// memory, so its scope is a drawing filter rather than a fetch strategy.
+//
+// Measured with this rule: the E. coli paa window converges at one hop (17
+// segments at 1 and at 2), so the cut closes and stays closed. HPRC's amylase
+// window keeps growing (63 at 0, 78 at 1, 92 at 2) because its alleles have
+// alleles, and wall-clock is flat across all three, since a hop's queries go out
+// together and the remote index dominates.
+//
+// The exact version, when a graph ships a bubble index beside its segments:
+// gfatools' bubble rows carry the member segment ids of every bubble, so the
+// bubbles overlapping the region state which segments a complete cut must
+// contain. That turns the hop count into a termination condition rather than a
+// setting, and MinigraphBubbleAdapter already parses those rows.
+function offReference(segments: RgfaSegment[]) {
+  return segments.filter(segment => segment.rank > 0)
+}
+
+export default class RgfaTabixAdapter extends BaseFeatureDataAdapter<RgfaTabixAdapterConfig> {
+  public static capabilities = ['getFeatures', 'getRefNames']
+
+  private readonly segments
+  private readonly links
+  // Off the SEGMENT index: it is the one that names every stable sequence the
+  // graph is anchored to, where the link index only names those a link touches.
+  private readonly refNames
+
+  public constructor(
+    config: RgfaTabixAdapterConfig,
+    getSubAdapter?: getSubAdapterType,
+    pluginManager?: PluginManager,
+  ) {
+    super(config, getSubAdapter, pluginManager)
+    this.segments = openTabixSlot(this, 'segmentsLocation', 'segmentsIndex')
+    this.links = openTabixSlot(this, 'linksLocation', 'linksIndex')
+    this.refNames = new PanSNRefNames(this.segments, this)
+  }
+
+  async getRefNames(opts: BaseOptions = {}) {
+    return this.refNames.assemblyRefNames(opts)
+  }
+
+  public async hasDataForRefName() {
+    return true
+  }
+
+  getFeatures(query: Region, opts: BaseOptions = {}) {
+    const { signal, statusCallback } = opts
+    return ObservableCreate<Feature>(async observer => {
+      const tabixRefName = await this.refNames.resolve(query, opts)
+      if (tabixRefName !== undefined) {
+        await updateStatus('Downloading segments', statusCallback, () =>
+          this.segments.getLines(tabixRefName, query.start, query.end, {
+            signal,
+            lineCallback: line => {
+              const segment = parseSegmentLine(line)
+              // `samples` is who, `carriers` is how many, and the second is not
+              // just a convenience: a lane colored by carriage is a jexl
+              // expression in a config, where counting a list means relying on
+              // jexl resolving `.length` through a member access on an array.
+              // The count is the axis the color reads, so the index states it.
+              const samples = segmentSamples(segment)
+              observer.next(
+                new SimpleFeature({
+                  uniqueId: segment.id,
+                  refName: query.refName,
+                  start: segment.start,
+                  end: segment.end,
+                  name: segment.id,
+                  type: 'segment',
+                  rank: segment.rank,
+                  stableName: segment.refName,
+                  ...(samples && { samples, carriers: samples.length }),
+                }),
+              )
+            },
+          }),
+        )
+      }
+      observer.complete()
+    }, signal)
+  }
+
+  // Extract the graph around a region as GFA text, for GraphGenomeView. The
+  // region's own segments come from segs.bed.gz; links.bed.gz supplies the edges
+  // incident to them *and* the coordinates of the segments on the other end,
+  // which typically sit on a different stable sequence (a rank>0 bubble) and so
+  // are not reachable by any coordinate query on this region.
+  //
+  // `hops` is extra rounds of link-following past the region's own segments
+  // and their immediate neighbours, each costing one tabix query per
+  // off-reference segment newly reached. One round is what closes a bubble, so
+  // it is the default the view asks for; see the frontier below. `signal` goes
+  // to every one of those queries, so a cut the view has replaced stops.
+  async getSubgraph(region: Region, opts: SubgraphAdapterOptions = {}) {
+    const { hops = 0, signal } = opts
+    const segments = new Map<string, RgfaSegment>()
+    const links = new Map<string, RgfaLink>()
+    const tabixRefName = await this.refNames.resolve(region, { signal })
+
+    const addLinksOver = async (
+      refName: string,
+      start: number,
+      end: number,
+    ) => {
+      const reached: RgfaSegment[] = []
+      await this.links.getLines(refName, start, end, {
+        signal,
+        lineCallback: line => {
+          const link = parseLinkLine(line)
+          links.set(linkKey(link), link)
+          for (const segment of [link.sourceSegment, link.targetSegment]) {
+            if (!segments.has(segment.id)) {
+              segments.set(segment.id, segment)
+              reached.push(segment)
+            }
+          }
+        },
+      })
+      return reached
+    }
+
+    if (tabixRefName === undefined) {
+      throw new Error(
+        `${region.assemblyName} ${region.refName} is not in this graph's index; a graph with PanSN names (GRCh38#0#chr1) needs ${region.assemblyName} mapped to its prefix in assemblyNameToPanSN`,
+      )
+    }
+    await this.segments.getLines(tabixRefName, region.start, region.end, {
+      signal,
+      lineCallback: line => {
+        const segment = parseSegmentLine(line)
+        segments.set(segment.id, segment)
+      },
+    })
+    let frontier = offReference(
+      await addLinksOver(tabixRefName, region.start, region.end),
+    )
+    for (let hop = 0; hop < hops; hop++) {
+      // One hop's queries are independent of each other, so they go out
+      // together rather than one round-trip at a time. Their callbacks share
+      // the segment and link maps, which is safe because only one of them runs
+      // at a time, and the output is sorted at the end regardless of the order
+      // they arrive in.
+      const reached = await Promise.all(
+        frontier.map(segment =>
+          addLinksOver(segment.refName, segment.start, segment.end),
+        ),
+      )
+      frontier = offReference(reached.flat())
+    }
+
+    return formatSubgraph(segments, links)
+  }
+}

package/src/RgfaTabixAdapter/bubbleTier.test.ts

@@ -0,0 +1,84 @@
+import Adapter from './RgfaTabixAdapter.ts'
+import configSchema from './configSchema.ts'
+import { parseGFA } from '../gfa-core/index.ts'
+
+// A level-of-detail tier: the same two files the adapter always reads, but one
+// node per bubble instead of one per GFA segment. Built by
+// jbrowse-components/scripts/bubbles_to_tier_bed.py from HPRC release 2's
+// hosted `hprc-v2.0-mc-grch38.bubbles.bed.gz`, filtered to chrY at
+// `--min-content 10000`. The whole chromosome is 57 bubbles and 57 backbone
+// nodes in 5.5 kB, which is the point of the tier.
+const prefix = require
+  .resolve('./test_data/bubble_tier_chrY.segs.bed.gz')
+  .replace(/\.segs\.bed\.gz$/, '')
+
+function makeAdapter() {
+  const local = (path: string) => ({
+    localPath: path,
+    locationType: 'LocalPathLocation',
+  })
+  return new Adapter(
+    configSchema.create({
+      segmentsLocation: local(`${prefix}.segs.bed.gz`),
+      segmentsIndex: { location: local(`${prefix}.segs.bed.gz.tbi`) },
+      linksLocation: local(`${prefix}.links.bed.gz`),
+      linksIndex: { location: local(`${prefix}.links.bed.gz.tbi`) },
+      assemblyNameToPanSN: { hg38: 'GRCh38' },
+    }),
+  )
+}
+
+const wholeChromosome = {
+  refName: 'chrY',
+  assemblyName: 'hg38',
+  start: 0,
+  end: 57_227_415,
+}
+
+// The claim the tier exists to make: a whole human chromosome is drawable. The
+// measured ceiling is a few hundred nodes (reference/PANGENOME_GRAPHS.md, "Force
+// layout does not get better with more nodes"), and the fine tier over the same
+// span would be six figures of segments.
+test('a whole chromosome collapses to a drawable node count', async () => {
+  const graph = parseGFA(await makeAdapter().getSubgraph(wholeChromosome))
+  expect(graph.nodes.length).toBe(114)
+  expect(graph.nodes.length).toBeLessThan(500)
+})
+
+test('the collapse summary reaches GraphNode.tags', async () => {
+  const graph = parseGFA(await makeAdapter().getSubgraph(wholeChromosome))
+  const bubbles = graph.nodes.filter(n => n.tags.ct === 'bubble')
+  const backbone = graph.nodes.filter(n => n.tags.ct === 'backbone')
+  expect(bubbles).toHaveLength(57)
+  expect(backbone).toHaveLength(57)
+
+  // Every bubble states what it collapsed, and the numbers arrive typed.
+  expect(
+    bubbles.every(n => typeof n.tags.cn === 'number' && n.tags.cn >= 1),
+  ).toBe(true)
+  expect(bubbles.every(n => typeof n.tags.cl === 'number')).toBe(true)
+})
+
+// A pure insertion is an alternative to nothing, so it has no reference span.
+// It is drawn 1 bp wide and states its real size in cl, which is why the
+// producer thresholds on content rather than on `end - start`.
+test('a pure insertion survives as a 1 bp node stating its true length', async () => {
+  const graph = parseGFA(await makeAdapter().getSubgraph(wholeChromosome))
+  const insertions = graph.nodes.filter(
+    n => n.tags.ct === 'bubble' && n.length === 1,
+  )
+  expect(insertions.length).toBeGreaterThan(0)
+  expect(insertions.every(n => (n.tags.cl as number) >= 10000)).toBe(true)
+})
+
+// Backbone and bubble strictly alternate, so the tier is one connected chain
+// rather than islands, which is what makes it pan-able as a graph.
+test('the tier is a connected alternating chain', async () => {
+  const gfa = await makeAdapter().getSubgraph(wholeChromosome)
+  const graph = parseGFA(gfa)
+  const rank = new Map(graph.nodes.map(n => [n.id, n.tags.ct]))
+  expect(graph.links.length).toBe(113)
+  expect(
+    graph.links.every(l => rank.get(l.source) !== rank.get(l.target)),
+  ).toBe(true)
+})