jbrowse-plugin-graphgenomeviewer 7.1.0 → 7.1.2

package/src/RgfaTabixAdapter/rgfaBed.ts

@@ -120,6 +120,25 @@ export function linkKey(link: RgfaLink) {
   return `${link.source}${link.sourceStrand}${link.target}${link.targetStrand}`
 }
 
+// gfa-to-tabix writes both endpoints of a link into every row and files the
+// link under each endpoint's stable sequence, so a read covering both returns
+// it twice; linkKey dedupes it. Returns the endpoints the cut lacked.
+export function addLink(
+  segments: Map<string, RgfaSegment>,
+  links: Map<string, RgfaLink>,
+  link: RgfaLink,
+) {
+  links.set(linkKey(link), link)
+  const added: RgfaSegment[] = []
+  for (const segment of [link.sourceSegment, link.targetSegment]) {
+    if (!segments.has(segment.id)) {
+      segments.set(segment.id, segment)
+      added.push(segment)
+    }
+  }
+  return added
+}
+
 // The cut an anchored index gives: `under` is what the segment file returned
 // for the window, every segment of every bubble under it, and `found` is what
 // the link file returned. gfa-to-tabix files a link widely enough that `found`
@@ -148,12 +167,7 @@ export function anchoredCut(
       under.has(link.target) ||
       side(link.sourceSegment) * side(link.targetSegment) === -1
     ) {
-      links.set(linkKey(link), link)
-      for (const segment of [link.sourceSegment, link.targetSegment]) {
-        if (!segments.has(segment.id)) {
-          segments.set(segment.id, segment)
-        }
-      }
+      addLink(segments, links, link)
     }
   }
   for (const link of found) {
@@ -164,10 +178,6 @@ export function anchoredCut(
   return { segments, links }
 }
 
-// Both endpoints of a link are written into every row, so the same link is
-// indexed under each endpoint's stable sequence and arrives twice for a region
-// covering both. Callers dedupe on linkKey.
-
 export interface StableSpan {
   refName: string
   start: number
@@ -195,12 +205,12 @@ export function closingSpans(
   return [
     ...mergedSpans(
       held.filter(segment => segment.rank > 0),
-      unread,
+      segment => unread.has(segment.id),
       window.end - window.start,
     ),
     ...mergedSpans(
       held.filter(segment => segment.rank === 0 && outside(segment)),
-      new Set(held.map(segment => segment.id)),
+      () => true,
       0,
     ),
   ]
@@ -208,7 +218,7 @@ export function closingSpans(
 
 function mergedSpans(
   segments: RgfaSegment[],
-  unread: Set<string>,
+  isUnread: (segment: RgfaSegment) => boolean,
   maxGap: number,
 ) {
   const sorted = [...segments].sort(
@@ -221,14 +231,14 @@ function mergedSpans(
       span?.refName === segment.refName &&
       segment.start - span.end <= maxGap
     ) {
-      span.needed ||= segment.start > span.end || unread.has(segment.id)
+      span.needed ||= segment.start > span.end || isUnread(segment)
       span.end = Math.max(span.end, segment.end)
     } else {
       spans.push({
         refName: segment.refName,
         start: segment.start,
         end: segment.end,
-        needed: unread.has(segment.id),
+        needed: isUnread(segment),
       })
     }
   }

package/src/RgfaTabixAdapter/rgfaCut.ts

@@ -0,0 +1,139 @@
+import {
+  addLink,
+  anchoredCut,
+  closingLinks,
+  closingSpans,
+  formatSubgraph,
+  parseLinkLine,
+  parseSegmentRow,
+} from './rgfaBed.ts'
+import { getLines } from '../tabixRange.ts'
+
+import type { RgfaLink, RgfaSegment, StableSpan } from './rgfaBed.ts'
+import type { TabixIndexedFile } from '@gmod/tabix'
+
+// What a hop follows: alleles, never the backbone. A rank-0 segment reached
+// through a link is flanking backbone, usually outside the window, so querying
+// its links walks the reference away from the region, brings back more backbone,
+// and closes nothing. Every segment that closes a bubble is an allele interior,
+// and an allele interior is rank > 0 in both index formats (rGFA's SR tag, and
+// pggb's rank 0-or-1 against the anchor path). Reached backbone still becomes a
+// node and its edge still draws; it is only not walked through.
+//
+// This is gfabase's rule, arrived at from the other end. `gfabase sub` expands
+// from a seed either to the whole connected component (`--connected`, which its
+// own README calls overkill) or with `--cutpoints 1`, which takes the biconnected
+// component by *stopping at cutpoints*, the segments every end-to-end walk of
+// the chromosome must traverse. In a reference-anchored pangenome the cutpoints
+// are the backbone, and rank is what the index already states about it, so
+// `rank > 0` is that stopping rule without a connectivity index to build. The
+// third tool is the same shape again: `gfatools view -R` maps the region onto its
+// precomputed bubble decomposition and returns whole overlapping bubbles
+// (gfa_query_by_reg), which is complete for the same reason. Bandage's own
+// `--distance` is a node-step radius, but Bandage holds the entire graph in
+// memory, so its scope is a drawing filter rather than a fetch strategy.
+//
+// Measured with this rule: the E. coli paa window converges at one hop (17
+// segments at 1 and at 2), so the cut closes and stays closed. HPRC's amylase
+// window keeps growing (63 at 0, 78 at 1, 92 at 2) because its alleles have
+// alleles, and wall-clock is flat across all three, since a hop's queries go out
+// together and the remote index dominates.
+//
+// The exact version, when a graph ships a bubble index beside its segments:
+// gfatools' bubble rows carry the member segment ids of every bubble, so the
+// bubbles overlapping the region state which segments a complete cut must
+// contain. That turns the hop count into a termination condition rather than a
+// setting, and MinigraphBubbleAdapter already parses those rows.
+function offReference(segments: RgfaSegment[]) {
+  return segments.filter(segment => segment.rank > 0)
+}
+
+export interface RgfaFiles {
+  segments: TabixIndexedFile
+  links: TabixIndexedFile
+}
+
+async function linksOver(
+  file: TabixIndexedFile,
+  span: StableSpan,
+  signal?: AbortSignal,
+) {
+  const found: RgfaLink[] = []
+  await getLines(file, span.refName, span.start, span.end, {
+    signal,
+    lineCallback: line => {
+      found.push(parseLinkLine(line))
+    },
+  })
+  return found
+}
+
+/**
+ * The graph around a window of one stable sequence, as GFA text. The window's
+ * own segments come from the segment file; the link file supplies the edges
+ * incident to them *and* the coordinates of the segments on the other end,
+ * which typically sit on a different stable sequence (a rank>0 bubble) and so
+ * are not reachable by any coordinate query on this window.
+ *
+ * `hops` is extra rounds of link-following past the window's own segments and
+ * their immediate neighbours, each costing one tabix query per off-reference
+ * segment newly reached. After the last round, one more read per stretch of
+ * stable sequence the cut holds closes it (closingSpans), so an allele reached
+ * from both ends draws whole. The view asks for one hop; a hopless cut stays
+ * as cheap as it was. `signal` goes to every one of those queries, so a cut
+ * the view has replaced stops.
+ */
+export async function rgfaCut(
+  files: RgfaFiles,
+  window: StableSpan,
+  { hops = 0, signal }: { hops?: number; signal?: AbortSignal } = {},
+) {
+  const segments = new Map<string, RgfaSegment>()
+  const links = new Map<string, RgfaLink>()
+  const layout = { anchored: false }
+  await getLines(files.segments, window.refName, window.start, window.end, {
+    signal,
+    lineCallback: line => {
+      const row = parseSegmentRow(line)
+      layout.anchored ||= row.anchored
+      segments.set(row.segment.id, row.segment)
+    },
+  })
+  // An anchored index files every allele and every link under the reference
+  // interval its bubble hangs from, so one read of each file holds the whole
+  // graph under the window and there is nothing to hop to or close.
+  if (layout.anchored) {
+    const cut = anchoredCut(
+      segments,
+      await linksOver(files.links, window, signal),
+      window,
+    )
+    return formatSubgraph(cut.segments, cut.links)
+  }
+  const addLinksOver = async (span: StableSpan) =>
+    (await linksOver(files.links, span, signal)).flatMap(link =>
+      addLink(segments, links, link),
+    )
+  let frontier = offReference(await addLinksOver(window))
+  for (let hop = 0; hop < hops; hop++) {
+    // One hop's queries are independent of each other, so they go out
+    // together rather than one round-trip at a time, and the output is sorted
+    // at the end regardless of the order they arrive in.
+    const reached = await Promise.all(frontier.map(addLinksOver))
+    frontier = offReference(reached.flat())
+  }
+  if (hops > 0) {
+    const spans = closingSpans(
+      [...segments.values()],
+      new Set(frontier.map(segment => segment.id)),
+      window,
+    )
+    const found = await Promise.all(
+      spans.map(span => linksOver(files.links, span, signal)),
+    )
+    for (const link of closingLinks(found.flat(), segments, spans)) {
+      addLink(segments, links, link)
+    }
+  }
+  return formatSubgraph(segments, links)
+}

package/src/RgfaTabixAdapter/walkFiles.test.ts

@@ -33,6 +33,10 @@ const fixture = (name: string) =>
 //   W HG00097 1 chr1 >1>2>3>4>5>6
 const headerPrefix = fixture('walks2_header')
 
+// walks2_header's walks with the reference named as pggb names a P-line,
+// GRCh38#chr1
+const twoPartPrefix = fixture('walks2_twopart')
+
 // An unchopped graph, node 1 spanning 0-2600 and filed under chunk 0 alone:
 //   W GRCh38 0 chr1 >1>2>4
 //   W HG002  1 chr1 >1>3>4   3 replaces 2
@@ -193,6 +197,13 @@ test("the haplotypes a walk file's header names", async () => {
   expect(gfa).not.toMatch(/^S\t8\t/m)
 })
 
+test('a cut for some haplotypes keeps a reference walk named without a haplotype', async () => {
+  const cut = (await makeAdapter(twoPartPrefix).getSubgraph(chr1(1000, 2600), {
+    haplotypes: ['HG002#1'],
+  })) as GraphTables
+  expect(cut.walks.names).toEqual(['GRCh38#chr1', 'HG002#1#chr1'])
+})
+
 test('a format 1 walk file is refused, with what to rebuild it with', async () => {
   const cut = makeAdapter(fixture('walks1_header')).getSubgraph(chr1(0, 600))
   await expect(cut).rejects.toThrow(/rebuild it with gfa-to-tabix 1.0/)
@@ -228,6 +239,9 @@ test('a cut or a feature read over a range that is not finite queries nothing',
   await expect(
     firstValueFrom(adapter.getFeatures(region).pipe(toArray())),
   ).rejects.toThrow(/not a finite range/)
+  await expect(adapter.getRegionByteSize([region])).rejects.toThrow(
+    /not a finite range/,
+  )
   expect(getLines).not.toHaveBeenCalled()
   expect(bytes).not.toHaveBeenCalled()
   getLines.mockRestore()
@@ -247,6 +261,25 @@ test("a node longer than a chunk is read back as far as the header's maxnode rea
   expect(walkNames(gfa)).toEqual(['GRCh38#0#chr1', 'HG002#1#chr1'])
 })
 
+test("a walk-indexed graph's features are the reference nodes overlapping the window", async () => {
+  const features = await firstValueFrom(
+    makeAdapter(headerPrefix).getFeatures(chr1(1000, 2300)).pipe(toArray()),
+  )
+  expect(features.map(f => f.get('name'))).toEqual(['2', '3', '4'])
+})
+
+test('the byte gate measures a walk file from the chunk before the window', async () => {
+  const bytes = vi.spyOn(TabixIndexedFile.prototype, 'bytesForRegions')
+  const size = await makeAdapter(headerPrefix).getRegionByteSize([
+    chr1(2100, 2600),
+  ])
+  expect(size).toBeGreaterThan(0)
+  expect(bytes.mock.calls.map(call => call[0])).toEqual([
+    [{ refName: 'GRCh38#0#chr1', start: 1000, end: 2600 }],
+  ])
+  bytes.mockRestore()
+})
+
 test('getFeatures reads from the chunk before the window', async () => {
   const getLines = vi.spyOn(TabixIndexedFile.prototype, 'getLines')
   await firstValueFrom(

package/src/RgfaTabixAdapter/walkReader.ts

@@ -1,9 +1,15 @@
-import { panSNHaplotype } from '@jbrowse/bandage-core/pansn'
-import { walkNameFilter } from '@jbrowse/graph-walks'
+import { resolvePanSNPrefix } from '@jbrowse/bandage-core/pansn'
+import { WalkReader, referenceNodes, walksKept } from '@jbrowse/graph-walks'
 
+import { getLines } from '../tabixRange.ts'
 import { walkCutFor } from './walkRowRuns.ts'
 
-import type { WalkReader } from '@jbrowse/graph-walks'
+import type { SubgraphAdapterOptions } from '../GetSubgraph.ts'
+import type { TabixIndexedFile } from '@gmod/tabix'
+import type {
+  BaseFeatureDataAdapter,
+  BaseOptions,
+} from '@jbrowse/core/data_adapters/BaseAdapter'
 
 // The zoom-in notices name the config slots that set the budgets
 export const BUDGET_SLOTS = {
@@ -12,29 +18,99 @@ export const BUDGET_SLOTS = {
 }
 
 /**
- * The graph display's cut of a walk-indexed window: the reference walk, the
- * walks of `prefixes` (every walk when undefined), and the nodes and links
- * they visit. Walk rows draw runs, so the step budget spares them.
- * @jbrowse/graph-walks reads and cuts the window.
+ * A walk file and the answers both walk-indexed adapters give from it,
+ * RgfaTabixAdapter and WalkTabixSyntenyAdapter, whose schemas declare the
+ * budget slots. @jbrowse/graph-walks reads and cuts the file.
  */
-export async function walkSubgraph(
-  reader: WalkReader,
-  refName: string,
-  region: { start: number; end: number },
-  opts: {
-    signal?: AbortSignal
-    prefixes?: string[]
-    walkRows?: boolean
-    stepBudget: number
-    byteBudget: number
-  },
-) {
-  const { graph, kept, fragments } = await reader.cut(refName, region, {
-    signal: opts.signal,
-    keep: walkNameFilter(opts.prefixes, panSNHaplotype(refName)),
-    stepBudget: opts.walkRows ? Infinity : opts.stepBudget,
-    byteBudget: opts.byteBudget,
-    budgetNames: BUDGET_SLOTS,
-  })
-  return walkCutFor(graph.tables(kept, fragments), region, opts.walkRows)
+export class WalkIndex {
+  readonly reader: WalkReader
+
+  constructor(
+    readonly file: TabixIndexedFile,
+    private adapter: BaseFeatureDataAdapter,
+  ) {
+    this.reader = new WalkReader(file)
+  }
+
+  /**
+   * The haplotypes the header names, `sample#haplotype` in byte order, or
+   * undefined when it names none
+   */
+  async haplotypeNames(opts: BaseOptions = {}) {
+    const { haplotypes } = await this.reader.header(opts)
+    return haplotypes.length > 0 ? haplotypes : undefined
+  }
+
+  budgets() {
+    return {
+      stepBudget: this.adapter.getConf('walkStepBudget') as number,
+      byteBudget: this.adapter.getConf('walkByteBudget') as number,
+      budgetNames: BUDGET_SLOTS,
+    }
+  }
+
+  /**
+   * The graph display's cut of a window: the reference walk, the walks of
+   * `haplotypes` (every walk when unset), and the nodes and links they visit.
+   * Walk rows draw runs, so the step budget spares them.
+   */
+  async subgraph(
+    refName: string,
+    region: { start: number; end: number },
+    { signal, haplotypes, walkRows }: SubgraphAdapterOptions,
+  ) {
+    const budgets = this.budgets()
+    const { graph, kept, fragments } = await this.reader.cut(refName, region, {
+      ...budgets,
+      stepBudget: walkRows ? Infinity : budgets.stepBudget,
+      signal,
+      keep: walksKept(
+        haplotypes?.map(lane => resolvePanSNPrefix(this.adapter, lane)),
+        refName,
+      ),
+    })
+    return walkCutFor(graph.tables(kept, fragments), region, walkRows)
+  }
+
+  /**
+   * The reference nodes overlapping a window, from the ref rows of its
+   * chunks. A row's tabix interval is its chunk's first base, and gfa-to-tabix
+   * files a node crossing in from a chunk before under that chunk, so the read
+   * starts at queryStart.
+   */
+  async referenceNodes(
+    refName: string,
+    region: { start: number; end: number },
+    opts: BaseOptions,
+  ) {
+    const refRows: string[][] = []
+    await getLines(
+      this.file,
+      refName,
+      await this.reader.queryStart(region.start, opts),
+      region.end,
+      {
+        signal: opts.signal,
+        lineCallback: line => {
+          const cols = line.split('\t')
+          if (cols[3] === 'ref') {
+            refRows.push(cols)
+          }
+        },
+      },
+    )
+    const seen = new Set<number>()
+    return referenceNodes(refRows).filter(node => {
+      if (
+        node.refName !== refName ||
+        node.end <= region.start ||
+        node.start >= region.end ||
+        seen.has(node.id)
+      ) {
+        return false
+      }
+      seen.add(node.id)
+      return true
+    })
+  }
 }

package/src/RgfaTabixAdapter/walkRowRuns.test.ts

@@ -99,3 +99,56 @@ test('walkRows measures and stops the same walks', () => {
   ).rows.find(r => r.label === 'E#1')!
   expect(short.stop).toEqual({ contigEnds: true, shortBp: 500 })
 })
+
+test('walks named without a haplotype field are rows of their own, on their contigs', () => {
+  const twoPart: GraphTables = {
+    ...tables,
+    walks: {
+      ...tables.walks,
+      names: ['GRCh38#0#chr1', 'A#a', 'A#b', 'C#1#c', 'D#1#d', 'E#1#e'],
+    },
+  }
+  const rows = walkRowsOf(walkRowRuns(twoPart, repeat)!).rows
+  expect(
+    rows
+      .filter(r => r.sample === 'A')
+      .map(({ name, label, haplotype, axis }) => ({
+        name,
+        label,
+        haplotype,
+        contig: axis?.contig,
+      }))
+      .sort((x, y) => x.name.localeCompare(y.name)),
+  ).toEqual([
+    { name: 'A#a', label: 'A', haplotype: undefined, contig: 'a' },
+    { name: 'A#b', label: 'A', haplotype: undefined, contig: 'b' },
+  ])
+})
+
+test('a reference cut into pieces is one reference row, as walkRows reads it', () => {
+  const split: GraphTables = {
+    ...tables,
+    walks: {
+      names: ['GRCh38#0#chr1', 'GRCh38#0#chr1', 'A#1#a'],
+      starts: Float64Array.of(0, 3000, 0),
+      ends: Float64Array.of(2000, 5000, 5000),
+      offsets: Int32Array.of(0, 2, 4, 9),
+      steps: Int32Array.of(...[0, 1], ...[3, 4], ...[0, 1, 2, 3, 4]),
+      reversed: new Uint8Array(9),
+    },
+  }
+  const region = { start: 2200, end: 2800 }
+  const fields = ({ label, bp, offReferenceBp, complete }: WalkRow) => ({
+    label,
+    bp,
+    offReferenceBp,
+    complete,
+  })
+  const ours = walkRowsOf(walkRowRuns(split, region)!)
+  const graph = loadGraph(split, 'cut')
+  const theirs = walkRows({ ...graph, referencePath: 'GRCh38#0#chr1' }, region)!
+  expect(ours.rows.map(fields)).toEqual(theirs.rows.map(fields))
+  expect(fields(ours.reference)).toEqual(fields(theirs.reference))
+  expect(ours.rows.map(r => r.label)).toEqual(['A#1'])
+  expect(ours.rows[0]).toMatchObject({ bp: 1000, complete: true })
+})

package/src/RgfaTabixAdapter/walkRowRuns.ts

@@ -1,3 +1,9 @@
+import {
+  panSNContig,
+  panSNHaplotype,
+  panSNSample,
+} from '@jbrowse/bandage-core/pansn'
+
 import type { GraphTables } from '@jbrowse/bandage-core/gfa/graphTables'
 import type {
   WalkAxis,
@@ -44,16 +50,23 @@ export interface WalkRowRuns {
   flags: Uint8Array
 }
 
-function walkFields(panSN: string) {
-  const [sample = '', haplotype = '', ...contig] = panSN.split('#')
-  return { sample, haplotype: +haplotype, contig: contig.join('#') }
+/**
+ * A walk's PanSN fields: `HG002#1#chr1`, or a pggb P-line's `chm13#chr1`,
+ * which has no haplotype
+ */
+export function walkPath(name: string) {
+  const sample = panSNSample(name)
+  const haplotype = panSNHaplotype(name)
+  return {
+    name,
+    sample,
+    haplotype:
+      haplotype === undefined ? undefined : +haplotype.slice(sample.length + 1),
+    contig: panSNContig(name),
+  }
 }
 
-interface Path {
-  name: string
-  sample: string
-  haplotype: number
-  contig: string
+type Path = ReturnType<typeof walkPath> & {
   start: number
   from: number
   to: number
@@ -84,11 +97,13 @@ class RunWriter {
 
 /**
  * `walkRows(graphFromTables(tables), region)` for a cut whose first walk is
- * the reference, which every walk-file and GBZ cut is; a step is a node index. A gap between two pieces of one haplotype is a
- * step of `-bp`. One difference: a walk reaching only one of the region's
- * flanks measures from that flank, where walkRows takes the whole walk the
- * cut holds, so its length is a lower bound that no longer grows with the
- * cut's context.
+ * the reference, which every walk-file and GBZ cut is, in as many pieces as
+ * the cut breaks it into; a step is a node index. A gap between two pieces of
+ * one haplotype is a step of `-bp`. Two differences: a walk reaching only one of the region's flanks measures from
+ * that flank, where walkRows takes the whole walk the cut holds, so its length
+ * is a lower bound that no longer grows with the cut's context; and a row
+ * keeps its walk's name whole, where graphFromTables reads a two-part name
+ * (`chm13#chr1`) as a haplotype `chr1` with no contig.
  */
 export function walkRowRuns(
   { nodes: n, walks: w }: GraphTables,
@@ -99,19 +114,19 @@ export function walkRowRuns(
   const paths: Path[] = []
   for (let p = 0; p < w.names.length; p++) {
     if (w.offsets[p + 1]! > w.offsets[p]!) {
-      const { sample, haplotype, contig } = walkFields(w.names[p]!)
       paths.push({
-        name: `${sample}#${haplotype}#${contig}`,
-        sample,
-        haplotype,
-        contig,
+        ...walkPath(w.names[p]!),
         start: w.starts[p]!,
         from: w.offsets[p]!,
         to: w.offsets[p + 1]!,
       })
     }
   }
-  const [first, ...others] = paths
+  const referencePieces = paths
+    .filter(p => p.name === paths[0]?.name)
+    .sort((a, b) => a.start - b.start)
+  const first = referencePieces[0]
+  const others = paths.filter(p => p.name !== first?.name)
   if (!first || others.length === 0) {
     return undefined
   }
@@ -130,9 +145,9 @@ export function walkRowRuns(
   let flankedBefore = false
   let flankedAfter = false
   const cut = region && region.end > region.start ? region : undefined
-  {
-    let pos = first.start
-    for (let s = first.from; s < first.to; s++) {
+  for (const piece of referencePieces) {
+    let pos = piece.start
+    for (let s = piece.from; s < piece.to; s++) {
       const node = w.steps[s]!
       const len = lengthOf(node)
       if (Number.isNaN(spanStart[node]!)) {
@@ -345,7 +360,10 @@ export function walkRowRuns(
     out.endRow()
     return {
       name: path.name,
-      label: `${path.sample}#${path.haplotype}`,
+      label:
+        path.haplotype === undefined
+          ? path.sample
+          : `${path.sample}#${path.haplotype}`,
       sample: path.sample,
       haplotype: path.haplotype,
       bp,
@@ -357,7 +375,7 @@ export function walkRowRuns(
     }
   }
 
-  const reference = rowOf([first])
+  const reference = rowOf(referencePieces)
   const rowList = [...walks.values()].map(pieces => ({
     meta: rowOf(pieces),
     row: out.offsets.length - 2,