jbrowse-plugin-graphgenomeviewer 7.4.2 → 7.4.4

package/src/WalkFormsAdapter/WalkFormsAdapter.ts

@@ -0,0 +1,232 @@
+import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter'
+import { SimpleFeature, updateStatus } from '@jbrowse/core/util'
+import { ObservableCreate } from '@jbrowse/core/util/rxjs'
+import {
+  ExpansionTooLargeError,
+  RegionTooLargeError,
+  WalkReader,
+  bpString,
+  formsCut,
+  panSNContig,
+  panSNMatchesPrefix,
+  structuralForms,
+} from '@jbrowse/graph-walks'
+
+import {
+  formSources,
+  formVariants,
+  formWarnings,
+  referenceGenotypes,
+} from './formRows.ts'
+import { floorOf, pinnedLocus, pinnedOf, warningPrefix } from './pinned.ts'
+import { openTabixSlot } from '../panSNTabix.ts'
+
+import type { WalkFormsAdapterConfig } from './configSchema.ts'
+import type { Pinned } from './pinned.ts'
+import type { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter'
+import type { Feature } from '@jbrowse/core/util'
+import type { Region } from '@jbrowse/core/util/types'
+import type { HaplotypeDepartures, StructuralForms } from '@jbrowse/graph-walks'
+
+const megabytes = (bytes: number) => `${(bytes / 1_000_000).toFixed(1)} MB`
+
+const millions = (n: number) => `${(n / 1_000_000).toFixed(1)} M`
+
+// A zoom-in notice would mislead, since the view's zoom does not move the
+// pinned window
+function pinnedTooLarge(error: unknown, pinned: Pinned) {
+  if (!(error instanceof RegionTooLargeError)) {
+    return error
+  }
+  const window = pinnedLocus(pinned)
+  if (error instanceof ExpansionTooLargeError) {
+    return new Error(
+      `The pinned window ${window} is too large to read every walk in full for structural forms: its walks take ${millions(error.steps)} steps, over the ${millions(error.limit)} step budget, and about ${bpString(error.fitsBp)} of it fits. Pin a narrower window`,
+    )
+  }
+  const budget = `walkByteBudget, ${megabytes(error.limit)}`
+  return new Error(
+    `The pinned window ${window} is too large to cut for structural forms: ${
+      error.zoomHelps
+        ? `about ${bpString(error.fitsBp)} of it fits in ${budget}. Pin a narrower window or raise walkByteBudget`
+        : `its cut fetches ${megabytes(error.bytes ?? 0)}, over ${budget}. Raise walkByteBudget`
+    }`,
+  )
+}
+
+// The departures of each pinned window, shared across adapter instances:
+// dataAdapterCache keys an adapter on its whole config, so a new floor or
+// exclude list makes a new adapter, which regroups these without a new cut.
+// A cut at the MHC takes 8 s.
+const CUTS_KEPT = 8
+const cuts = new Map<string, Promise<HaplotypeDepartures[]>>()
+
+function cachedCut(key: string, cut: () => Promise<HaplotypeDepartures[]>) {
+  const kept = cuts.get(key)
+  const read = kept ?? cut()
+  cuts.delete(key)
+  cuts.set(key, read)
+  if (!kept) {
+    read.catch(() => {
+      if (cuts.get(key) === read) {
+        cuts.delete(key)
+      }
+    })
+    for (const old of cuts.keys()) {
+      if (cuts.size <= CUTS_KEPT) {
+        break
+      }
+      cuts.delete(old)
+    }
+  }
+  return read
+}
+
+export default class WalkFormsAdapter extends BaseFeatureDataAdapter<WalkFormsAdapterConfig> {
+  public static capabilities = ['getFeatures', 'getRefNames']
+
+  private readonly reader = new WalkReader(
+    openTabixSlot(this, ['walksLocation'], ['walksIndex']),
+  )
+
+  private formsRead: Promise<StructuralForms> | undefined
+
+  private async referenceSample(opts: BaseOptions) {
+    const { references } = await this.reader.header(opts)
+    return references[0] ?? ''
+  }
+
+  // The reference sample's contigs, as the assembly names them
+  async getRefNames(opts: BaseOptions = {}) {
+    const names = await this.reader.referenceSequences(opts)
+    const sample = await this.referenceSample(opts)
+    const contigs = names
+      .filter(n => panSNMatchesPrefix(n, sample))
+      .map(n => panSNContig(n))
+    return contigs.length > 0 ? [...new Set(contigs)] : names
+  }
+
+  private pinned() {
+    const pinned = pinnedOf(this.getConf('pinned'))
+    if (!pinned) {
+      throw new Error(
+        'WalkFormsAdapter needs `pinned`, the window its forms come from: { refName, start, end }',
+      )
+    }
+    return pinned
+  }
+
+  // Read once per adapter, and so once per floor and exclude list. The cut
+  // takes no caller's signal, since every later caller shares it. A partial
+  // expansion would leave some walks as net changes over a whole chunk, so a
+  // window too large to read them all errors instead.
+  private forms(opts: BaseOptions) {
+    if (!this.formsRead) {
+      const pinned = this.pinned()
+      const key = `${JSON.stringify(this.getConf('walksLocation'))}\t${pinned.refName}:${pinned.start}-${pinned.end}`
+      const byteBudget = this.getConf('walkByteBudget')
+      const read = cachedCut(`${key}\t${byteBudget}`, () =>
+        updateStatus('Cutting walks', opts.statusCallback, () =>
+          formsCut(this.reader, pinned, { byteBudget, partial: false }).catch(
+            (error: unknown) => {
+              throw pinnedTooLarge(error, pinned)
+            },
+          ),
+        ),
+      ).then(departures =>
+        structuralForms(departures, {
+          window: pinned,
+          floorBp: floorOf(this.getConf('floorBp')),
+          exclude: this.getConf('exclude'),
+        }),
+      )
+      read.catch(() => {
+        if (this.formsRead === read) {
+          this.formsRead = undefined
+        }
+      })
+      this.formsRead = read
+    }
+    return this.formsRead
+  }
+
+  private maxForms() {
+    return this.getConf('maxForms')
+  }
+
+  /**
+   * One source per form up to maxForms, which core's multi-sample variant
+   * display draws as a row, and warnings naming the haplotypes left
+   * abridged, unplaced or in the "other forms" row
+   */
+  async getSourcesAndWarnings(opts: BaseOptions = {}) {
+    const result = await this.forms(opts)
+    const prefix = warningPrefix(this.pinned())
+    return {
+      sources: formSources(result, this.maxForms()),
+      warnings: formWarnings(result, this.maxForms()).map(w => prefix + w),
+    }
+  }
+
+  async getSources(_regions: Region[], opts: BaseOptions = {}) {
+    return formSources(await this.forms(opts), this.maxForms())
+  }
+
+  private async onPinnedSequence(refName: string, opts: BaseOptions) {
+    const sample = await this.referenceSample(opts)
+    const [query, pinned] = await Promise.all([
+      this.reader.sequenceFor(sample, refName, opts),
+      this.reader.sequenceFor(sample, this.pinned().refName, opts),
+    ])
+    return query !== undefined && query === pinned
+  }
+
+  getFeatures(query: Region, opts: BaseOptions = {}) {
+    return ObservableCreate<Feature>(async observer => {
+      const result = await this.forms(opts)
+      const variants = (await this.onPinnedSequence(query.refName, opts))
+        ? formVariants(result, this.maxForms()).filter(
+            v => v.start < query.end && v.end > query.start,
+          )
+        : []
+      for (const v of variants) {
+        observer.next(
+          new SimpleFeature({
+            uniqueId: `forms-${result.floorBp}-${v.id}`,
+            refName: query.refName,
+            start: v.start,
+            end: v.end,
+            name: v.name,
+            description: v.description,
+            type: v.type,
+            svType: v.svType,
+            consequence: v.consequence,
+            REF: 'N',
+            ALT: [v.alt],
+            INFO: { SVTYPE: [v.svType], SVLEN: [v.mark.delta] },
+            mark: v.mark,
+            genotypes: v.genotypes,
+          }),
+        )
+      }
+      // Core before 5.0.0-beta.14 (1845c280a9) sends no payload for a block
+      // with no records, which leaves the display on "Loading". The reference
+      // placeholder draws nothing; drop it once the host floor reaches beta.14
+      if (variants.length === 0) {
+        const start = Math.floor(query.start)
+        observer.next(
+          new SimpleFeature({
+            uniqueId: `forms-reference-${query.refName}-${start}`,
+            refName: query.refName,
+            start,
+            end: start + 1,
+            REF: 'N',
+            ALT: ['.'],
+            genotypes: referenceGenotypes(result, this.maxForms()),
+          }),
+        )
+      }
+      observer.complete()
+    }, opts.signal)
+  }
+}

package/src/WalkFormsAdapter/configSchema.ts

@@ -0,0 +1,127 @@
+import { ConfigurationSchema } from '@jbrowse/core/configuration'
+
+import { DEFAULT_MAX_FORMS } from './formRows.ts'
+import {
+  fileLocationSlot,
+  tabixIndexSchema,
+  walksLocations,
+} from '../panSNTabix.ts'
+
+import type { Instance } from '@jbrowse/mobx-state-tree'
+
+/**
+ * #config WalkFormsAdapter
+ * #trackType VariantTrack
+ * #fileFormat variant | Walk-file structural forms | Built by `gfa-to-tabix --walks`; one row per group of haplotypes making the same structural changes
+ * Groups the haplotypes of a
+ * [gfa-to-tabix](https://github.com/GMOD/gfa-to-tabix) `--walks` file by the
+ * structural changes their walks make over one pinned window, through
+ * @jbrowse/graph-walks' `formsCut` and `structuralForms`. Each group is a
+ * form, which `LinearMultiSampleVariantDisplay` draws as one row, and each
+ * change a variant whose genotype is `1` in the forms that make it.
+ *
+ * - The adapter cuts the pinned window once, whatever the view shows; outside
+ *   it the rows stay and draw empty. A window whose abridged walks are too
+ *   many to read in full errors instead
+ * - The largest `maxForms` forms get a row each, labelled by share and largest
+ *   change ("66% · del 18.4kb"); the rest share one empty "other forms" row
+ * - A variant's `consequence` is `loss`, `gain`, `shift`, `inversion` or
+ *   `net`, for colouring the cells; `svType` and `type` follow core's names
+ * - `walksUri` takes the prefix of one reference's walk file and resolves
+ *   `<walksUri>.walks.bed.gz` and its `.tbi`, as the other walk adapters do
+ *
+ * #example
+ * ```js
+ * {
+ *   type: 'WalkFormsAdapter',
+ *   walksUri: 'https://jbrowse.org/demos/hprc/hprc-v2.1-mc-grch38-f5.GRCh38',
+ *   pinned: { refName: 'chr1', start: 109680000, end: 109715000 },
+ *   exclude: ['CHM13'],
+ * }
+ * ```
+ */
+const WalkFormsAdapter = ConfigurationSchema(
+  'WalkFormsAdapter',
+  {
+    /**
+     * #slot
+     * a `gfa-to-tabix --walks` file in walk format 5
+     */
+    walksLocation: fileLocationSlot('/path/to/graph.walks.bed.gz'),
+    /**
+     * #slot
+     */
+    walksIndex: tabixIndexSchema('WalkFormsWalksIndex'),
+    /**
+     * #slot
+     * The window the forms come from, `{ refName, start, end }`, 0-based and
+     * half-open. `refName` is the contig as the walk file's reference sample
+     * names it (`chr1`), or its full PanSN name (`GRCh38#0#chr1`).
+     */
+    pinned: {
+      type: 'frozen',
+      defaultValue: {},
+    },
+    /**
+     * #slot
+     * The smallest change that tells two forms apart: `auto`, which follows
+     * the pinned window's width (50 bp under 25 kb, 1 kb to 300 kb, 10 kb
+     * past it), or 50, 1000 or 10000. A change of floor regroups the same
+     * cut.
+     */
+    floorBp: {
+      type: 'frozen',
+      defaultValue: 'auto',
+    },
+    /**
+     * #slot
+     * Samples or `sample#haplotype`s left out of every form, such as a second
+     * reference the walk file carries as a haplotype (`CHM13` in HPRC's
+     * GRCh38 file)
+     */
+    exclude: {
+      type: 'stringArray',
+      defaultValue: [],
+    },
+    /**
+     * #slot
+     * The most forms drawn as rows of their own, largest first. The rest
+     * share one empty "other forms" row, and a warning counts them.
+     */
+    maxForms: {
+      type: 'integer',
+      defaultValue: DEFAULT_MAX_FORMS,
+    },
+    /**
+     * #slot
+     * The most compressed bytes the forms cut fetches from the walk file, as
+     * its Tabix index estimates it before any row is read. A pinned window
+     * over it shows an error instead. 1 MB holds about 1 Mb of the MHC, whose
+     * cut takes 13 s and 0.8 GB.
+     */
+    walkByteBudget: {
+      type: 'integer',
+      defaultValue: 1_000_000,
+    },
+  },
+  {
+    explicitlyTyped: true,
+
+    /**
+     * #preProcessSnapshot
+     * preprocessor to allow the minimal config
+     * ```json
+     * {
+     *   "type": "WalkFormsAdapter",
+     *   "walksUri": "chr22.GRCh38",
+     *   "pinned": { "refName": "chr22", "start": 0, "end": 50000 }
+     * }
+     * ```
+     */
+    preProcessSnapshot: walksLocations,
+  },
+)
+
+export type WalkFormsAdapterConfig = Instance<typeof WalkFormsAdapter>
+
+export default WalkFormsAdapter

package/src/WalkFormsAdapter/formRows.test.ts

@@ -0,0 +1,258 @@
+import { structuralForms } from '@jbrowse/graph-walks'
+
+import {
+  OTHER_FORMS,
+  formName,
+  formSources,
+  formVariants,
+  formWarnings,
+  referenceGenotypes,
+} from './formRows.ts'
+
+import type { Departure, HaplotypeDepartures } from '@jbrowse/graph-walks'
+
+const window = { start: 0, end: 24_000 }
+
+const departure = (
+  refStart: number,
+  refEnd: number,
+  altLength: number,
+  extra: Partial<Departure> = {},
+): Departure => ({
+  refStart,
+  refEnd,
+  altLength,
+  invertedBp: 0,
+  repeatedBp: 0,
+  ...extra,
+})
+
+const haplotype = (
+  name: string,
+  departures: Departure[],
+  covered: [number, number][] = [[0, 30_000]],
+): HaplotypeDepartures => ({
+  haplotype: name,
+  covered,
+  departures,
+  open: false,
+})
+
+const deletion = departure(1000, 3000, 0)
+const insertion = departure(5000, 5000, 800)
+
+// One haplotype or a few per mark kind, so every state is common and every
+// site tells forms apart
+const departures = [
+  haplotype('A#1', [deletion, insertion]),
+  haplotype('A#2', [deletion, insertion]),
+  haplotype('B#1', [
+    deletion,
+    departure(10_000, 11_000, 1000, { invertedBp: 1000 }),
+  ]),
+  haplotype('B#2', [deletion]),
+  haplotype('C#1', [insertion]),
+  haplotype('C#2', [
+    departure(8000, 8000, 1500, { repeatedBp: 1500, copied: [6500, 8000] }),
+  ]),
+  haplotype('D#1', [
+    departure(8000, 8000, 1500, { repeatedBp: 1500, copied: [6500, 8000] }),
+  ]),
+  haplotype('D#2', [departure(10_000, 11_000, 1000, { invertedBp: 1000 })]),
+  haplotype('E#1', [departure(13_000, 13_600, 2000)]),
+  haplotype('E#2', [departure(15_000, 17_000, 500)]),
+  haplotype('F#1', [
+    departure(18_000, 19_500, 0),
+    departure(19_600, 19_600, 1480),
+  ]),
+  haplotype('F#2', [departure(21_000, 21_500, 3000, { abridged: true })]),
+  haplotype('G#1', []),
+  haplotype('G#2', [], [[0, 12_000]]),
+  haplotype('H#1', [], []),
+]
+
+const result = structuralForms(departures, { window })
+const variants = formVariants(result, Infinity)
+
+// The form a haplotype fell into
+const formOf = (name: string) =>
+  formName(result.forms.findIndex(f => f.members.includes(name)))
+
+const variantAt = (start: number) => {
+  const v = variants.find(v => v.start === start)
+  if (!v) {
+    throw new Error(`no variant at ${start}`)
+  }
+  return v
+}
+
+test('each mark kind names its consequence, svType and SO type', () => {
+  expect(
+    variants.map(v => [v.start, v.end, v.consequence, v.svType, v.type, v.alt]),
+  ).toEqual([
+    [1000, 3000, 'loss', 'DEL', 'deletion', '<DEL>'],
+    [5000, 5001, 'gain', 'INS', 'insertion', '<INS>'],
+    [6500, 8000, 'gain', 'DUP', 'tandem_duplication', '<DUP:TANDEM>'],
+    [10_000, 11_000, 'inversion', 'INV', 'inversion', '<INV>'],
+    [13_000, 13_600, 'gain', 'CPX', 'complex_substitution', '<CPX>'],
+    [15_000, 17_000, 'loss', 'CPX', 'complex_substitution', '<CPX>'],
+    [18_000, 19_600, 'shift', 'CPX', 'complex_structural_alteration', '<CPX>'],
+    [21_000, 21_500, 'net', 'CPX', 'complex_structural_alteration', '<CPX>'],
+  ])
+})
+
+test('a replacement is one variant over its REF span', () => {
+  const v = variantAt(13_000)
+  expect(v.mark).toEqual(
+    expect.objectContaining({
+      kind: 'replacement',
+      refEnd: 13_600,
+      delta: 1400,
+    }),
+  )
+  expect(v.name).toBe('repl 600bp->2kb')
+})
+
+test('a lopsided replacement colours as the side it nearly is', () => {
+  const forms = structuralForms(
+    [
+      haplotype('A#1', [departure(2000, 8460, 94)]),
+      haplotype('B#1', [departure(12_000, 12_178, 13_900)]),
+      haplotype('C#1', [departure(16_000, 16_600, 500)]),
+      haplotype('D#1', []),
+    ],
+    { window },
+  )
+  expect(
+    formVariants(forms).map(v => [
+      v.name,
+      v.mark.kind,
+      v.mark.lopsided,
+      v.consequence,
+    ]),
+  ).toEqual([
+    ['del 6.46kb->94bp', 'replacement', 'deletion', 'loss'],
+    ['ins 178bp->13.9kb', 'replacement', 'insertion', 'gain'],
+    ['repl 600bp->500bp', 'replacement', undefined, 'loss'],
+  ])
+})
+
+test('genotypes are 1 in the forms carrying a state, 0 in the rest', () => {
+  const { genotypes, description } = variantAt(1000)
+  const carriers = ['A#1', 'A#2', 'B#1', 'B#2'].map(formOf)
+  for (const [form, genotype] of Object.entries(genotypes)) {
+    expect([form, genotype]).toEqual([
+      form,
+      carriers.includes(form) ? '1' : '0',
+    ])
+  }
+  expect(description).toBe('4 of 14 haplotypes')
+})
+
+test('a form whose walks do not span a site has no call there', () => {
+  expect(variantAt(1000).genotypes[formOf('G#2')]).toBe('0')
+  expect(variantAt(13_000).genotypes[formOf('G#2')]).toBe('.')
+  expect(variantAt(13_000).genotypes[formOf('E#1')]).toBe('1')
+})
+
+test('sources are the forms, labelled by share and largest change', () => {
+  const sources = formSources(result, Infinity)
+  expect(sources).toHaveLength(result.forms.length)
+  expect(sources[0]).toEqual({
+    name: 'F1',
+    label: '14% · del 2kb +1',
+    haplotypes: 2,
+    forms: 1,
+  })
+  expect(sources.find(s => s.name === formOf('G#1'))?.label).toBe('7% · ref')
+})
+
+test('past maxForms the smaller forms share one empty row', () => {
+  expect(result.forms).toHaveLength(12)
+  const sources = formSources(result, 4)
+  expect(sources.map(s => s.name)).toEqual(['F1', 'F2', 'F3', 'F4', 'other'])
+  expect(sources.at(-1)).toEqual({
+    name: OTHER_FORMS,
+    label: 'other forms · 57%',
+    haplotypes: 8,
+    forms: 8,
+  })
+  expect(formWarnings(result, 4)[0]).toBe(
+    '8 smaller forms, 8 haplotypes, share the empty "other forms" row; maxForms sets how many forms get a row of their own',
+  )
+  const capped = formVariants(result, 4)
+  expect(capped.every(v => Object.keys(v.genotypes).length === 4)).toBe(true)
+  expect(capped.every(v => Object.values(v.genotypes).includes('1'))).toBe(true)
+  expect(capped.length).toBeLessThan(variants.length)
+  expect(variantAt(1000).description).toBe(
+    capped.find(v => v.start === 1000)?.description,
+  )
+})
+
+test('reference genotypes cover the forms with a row of their own', () => {
+  expect(referenceGenotypes(result, 2)).toEqual({ F1: '0', F2: '0' })
+  expect(Object.keys(referenceGenotypes(result, Infinity))).toHaveLength(12)
+})
+
+test('one form past maxForms keeps its own row', () => {
+  expect(formSources(result, 11).map(s => s.name)).not.toContain(OTHER_FORMS)
+  expect(formSources(result, 10).at(-1)?.name).toBe(OTHER_FORMS)
+})
+
+test('warnings name the abridged and unplaced haplotypes', () => {
+  expect(formWarnings(result, Infinity)).toEqual([
+    "1 haplotype kept an abridged walk past the forms cut's step budget, which shows as a net change over its span: F#2",
+    '1 haplotype has no walk over the pinned window, so no form: H#1',
+  ])
+})
+
+test('a change every form shares is a variant carried by all of them', () => {
+  const shared = departure(2000, 2500, 0)
+  const forms = structuralForms(
+    [
+      haplotype('A#1', [shared, insertion]),
+      haplotype('A#2', [shared, insertion]),
+      haplotype('B#1', [shared]),
+    ],
+    { window },
+  )
+  expect(formSources(forms).map(s => s.label)).toEqual([
+    '67% · ins 800bp',
+    '33% · shared',
+  ])
+  expect(
+    formVariants(forms).map(v => [v.start, v.consequence, v.genotypes]),
+  ).toEqual([
+    [2000, 'loss', { F1: '1', F2: '1' }],
+    [5000, 'gain', { F1: '1', F2: '0' }],
+  ])
+})
+
+// a site over 1 kb wide keeps each mark apart; graph-walks sums the marks of a
+// narrower one into a single change
+test('a state of several marks is a variant per mark, carried alike', () => {
+  const forms = structuralForms(
+    [
+      haplotype('A#1', [departure(2000, 2600, 0), departure(2660, 2660, 900)]),
+      haplotype('B#1', [departure(1900, 3200, 0)]),
+      haplotype('C#1', []),
+    ],
+    { window },
+  )
+  const carriedBy = (name: string) =>
+    Object.fromEntries(
+      forms.forms.map((f, i) => [
+        formName(i),
+        f.members.includes(name) ? '1' : '0',
+      ]),
+    )
+  expect(
+    formVariants(forms)
+      .map(v => [v.start, v.end, v.name, v.genotypes])
+      .sort((a, b) => Number(a[0]) - Number(b[0])),
+  ).toEqual([
+    [1900, 3200, 'del 1.3kb', carriedBy('B#1')],
+    [2000, 2600, 'del 600bp', carriedBy('A#1')],
+    [2660, 2661, 'ins 900bp', carriedBy('A#1')],
+  ])
+})