jbrowse-plugin-mafviewer 1.1.4 → 1.2.1

package/package.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.1.4",
+  "version": "1.2.1",
   "license": "MIT",
   "name": "jbrowse-plugin-mafviewer",
   "keywords": [
@@ -41,7 +41,7 @@
     "@typescript-eslint/eslint-plugin": "^8.18.0",
     "@typescript-eslint/parser": "^8.18.0",
     "chalk": "^5.3.0",
-    "esbuild": "^0.24.0",
+    "esbuild": "^0.25.0",
     "eslint": "^9.17.0",
     "eslint-plugin-import": "^2.31.0",
     "eslint-plugin-react": "^7.20.3",
@@ -66,6 +66,7 @@
     "buffer": "^6.0.3",
     "d3-array": "^3.2.4",
     "d3-hierarchy": "^3.1.2",
+    "fast-deep-equal": "^3.1.3",
     "generic-filehandle2": "^1.0.0",
     "long": "^5.2.3"
   }

package/src/BgzipTaffyAdapter/BgzipTaffyAdapter.ts

@@ -1,6 +1,14 @@
 import { unzip } from '@gmod/bgzf-filehandle'
-import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter'
-import { Feature, Region, SimpleFeature } from '@jbrowse/core/util'
+import {
+  BaseFeatureDataAdapter,
+  BaseOptions,
+} from '@jbrowse/core/data_adapters/BaseAdapter'
+import {
+  Feature,
+  Region,
+  SimpleFeature,
+  updateStatus,
+} from '@jbrowse/core/util'
 import { openLocation } from '@jbrowse/core/util/io'
 import { ObservableCreate } from '@jbrowse/core/util/rxjs'
 import Long from 'long'
@@ -9,6 +17,7 @@ import VirtualOffset from './virtualOffset'
 import parseNewick from '../parseNewick'
 import { normalize } from '../util'
 import { parseRowInstructions } from './rowInstructions'
+import { parseLineByLine } from './util'
 
 import type { IndexData, OrganismRecord } from './types'
 interface Entry {
@@ -20,6 +29,9 @@ interface Entry {
   strand: number
   length: number
 }
+
+const toP = (s = 0) => +(+s).toFixed(1)
+
 export default class BgzipTaffyAdapter extends BaseFeatureDataAdapter {
   public setupP?: Promise<IndexData>
 
@@ -28,7 +40,7 @@ export default class BgzipTaffyAdapter extends BaseFeatureDataAdapter {
     return Object.keys(data)
   }
 
-  setup() {
+  setupPre() {
     if (!this.setupP) {
       this.setupP = this.readTaiFile().catch((e: unknown) => {
         this.setupP = undefined
@@ -37,6 +49,12 @@ export default class BgzipTaffyAdapter extends BaseFeatureDataAdapter {
     }
     return this.setupP
   }
+  setup(opts?: BaseOptions) {
+    const { statusCallback = () => {} } = opts || {}
+    return updateStatus('Downloading index', statusCallback, () =>
+      this.setupPre(),
+    )
+  }
 
   async readTaiFile() {
     const text = await openLocation(this.getConf('taiLocation')).readFile(
@@ -81,91 +99,109 @@ export default class BgzipTaffyAdapter extends BaseFeatureDataAdapter {
     return entries
   }
 
-  getFeatures(query: Region) {
+  getFeatures(query: Region, opts?: BaseOptions) {
+    const { statusCallback = () => {} } = opts || {}
     return ObservableCreate<Feature>(async observer => {
       try {
-        const lines = await this.getLines(query)
-        const alignments = {} as Record<string, OrganismRecord>
-
-        const k = lines.length
-        const data = [] as Entry[]
-        let a0: any
-        for (let j = 0; j < k; j++) {
-          const line = lines[j]!
-          if (line) {
-            const [lineData, rowInstructions] = line.split(' ; ')
-            if (rowInstructions) {
-              for (const ins of parseRowInstructions(rowInstructions)) {
-                if (ins.type === 'i') {
-                  data.splice(ins.row, 0, ins)
-                  if (!alignments[ins.asm]) {
-                    alignments[ins.asm] = {
-                      start: ins.start,
-                      strand: ins.strand,
-                      srcSize: ins.length,
-                      chr: ins.ref,
-                      data: '',
+        const byteRanges = await this.setup()
+        const buffer = await updateStatus(
+          'Downloading alignments',
+          statusCallback,
+          () => this.getLines(query, byteRanges),
+        )
+        if (buffer) {
+          const alignments = {} as Record<string, OrganismRecord>
+          const data = [] as Entry[]
+          let a0: any
+          let j = 0
+          let b = 0
+          parseLineByLine(buffer, line => {
+            if (j++ % 100 === 0) {
+              statusCallback(
+                `Processing ${toP(b / 1_000_000)}/${toP(buffer.length / 1_000_000)}Mb`,
+              )
+            }
+            b += line.length
+            if (line) {
+              const [lineData, rowInstructions] = line.split(' ; ')
+              if (rowInstructions) {
+                for (const ins of parseRowInstructions(rowInstructions)) {
+                  if (ins.type === 'i') {
+                    data.splice(ins.row, 0, ins)
+                    if (!alignments[ins.asm]) {
+                      alignments[ins.asm] = {
+                        start: ins.start,
+                        strand: ins.strand,
+                        srcSize: ins.length,
+                        chr: ins.ref,
+                        data: '',
+                      }
                     }
-                  }
-                  const e = alignments[ins.asm]!
-                  e.data += ' '.repeat(Math.max(0, j - e.data.length)) // catch it up
-                } else if (ins.type === 's') {
-                  if (!alignments[ins.asm]) {
-                    alignments[ins.asm] = {
-                      start: ins.start,
-                      strand: ins.strand,
-                      srcSize: ins.length,
-                      chr: ins.ref,
-                      data: '',
+                    const e = alignments[ins.asm]!
+                    e.data += ' '.repeat(Math.max(0, j - e.data.length)) // catch it up
+                  } else if (ins.type === 's') {
+                    if (!alignments[ins.asm]) {
+                      alignments[ins.asm] = {
+                        start: ins.start,
+                        strand: ins.strand,
+                        srcSize: ins.length,
+                        chr: ins.ref,
+                        data: '',
+                      }
                     }
+                    const e = alignments[ins.asm]!
+                    e.data += ' '.repeat(Math.max(0, j - e.data.length)) // catch it up
+                    data[ins.row] = ins
+                  } else if (ins.type === 'd') {
+                    data.splice(ins.row, 1)
                   }
-                  const e = alignments[ins.asm]!
-                  e.data += ' '.repeat(Math.max(0, j - e.data.length)) // catch it up
-                  data[ins.row] = ins
-                } else if (ins.type === 'd') {
-                  data.splice(ins.row, 1)
-                }
 
-                // no gaps for now(?)
-                // else if (ins.type === 'g') {
-                // }
-                // else if (ins.type === 'G') {
-                // }
+                  // no gaps for now(?)
+                  // else if (ins.type === 'g') {
+                  // }
+                  // else if (ins.type === 'G') {
+                  // }
+                }
+                if (!a0) {
+                  a0 = data[0]
+                }
               }
-              if (!a0) {
-                a0 = data[0]
+              const lineLen = lineData!.length
+              for (let i = 0; i < lineLen; i++) {
+                const letter = lineData![i]
+                const r = data[i]
+                if (r) {
+                  alignments[r.asm]!.data += letter
+                } else {
+                  // not sure why but chr22_KI270731v1_random.taf.gz ends up here
+                }
               }
             }
-            const lineLen = lineData!.length
-            for (let i = 0; i < lineLen; i++) {
-              const letter = lineData![i]
-              const r = data[i]!
-              alignments[r.asm]!.data += letter
-            }
+          })
+          if (a0) {
+            const row0 = alignments[a0.asm]!
+
+            // see
+            // https://github.com/ComparativeGenomicsToolkit/taffy/blob/f5a5354/docs/taffy_utilities.md#referenced-based-maftaf-and-indexing
+            // for the significance of row[0]:
+            //
+            // "An anchor line in TAF is a column from which all sequence
+            // coordinates can be deduced without scanning backwards to previous
+            // lines "
+            observer.next(
+              new SimpleFeature({
+                uniqueId: `${row0.start}-${row0.data.length}`,
+                refName: query.refName,
+                start: row0.start,
+                end: row0.start + row0.data.length,
+                strand: row0.strand,
+                alignments,
+                seq: row0.data,
+              }),
+            )
           }
         }
-        if (a0) {
-          const row0 = alignments[a0.asm]!
-
-          // see
-          // https://github.com/ComparativeGenomicsToolkit/taffy/blob/f5a5354/docs/taffy_utilities.md#referenced-based-maftaf-and-indexing
-          // for the significance of row[0]:
-          //
-          // "An anchor line in TAF is a column from which all sequence
-          // coordinates can be deduced without scanning backwards to previous
-          // lines "
-          observer.next(
-            new SimpleFeature({
-              uniqueId: `${row0.start}-${row0.data.length}`,
-              refName: query.refName,
-              start: row0.start,
-              end: row0.start + row0.data.length,
-              strand: row0.strand,
-              alignments,
-              seq: row0.data,
-            }),
-          )
-        }
+        statusCallback('')
         observer.complete()
       } catch (e) {
         observer.error(e)
@@ -188,15 +224,14 @@ export default class BgzipTaffyAdapter extends BaseFeatureDataAdapter {
     }
   }
 
-  async getLines(query: Region) {
-    const byteRanges = await this.setup()
+  // TODO: cache processed large chunks
+  async getLines(query: Region, byteRanges: IndexData) {
     const file = openLocation(this.getConf('tafGzLocation'))
-
-    const decoder = new TextDecoder('utf8')
     const records = byteRanges[query.refName]
     if (records) {
-      let firstEntry = records[0]
+      let firstEntry
       let nextEntry
+
       // two pass:
       // first pass: find first block greater than query start, then -1 from
       // that
@@ -216,6 +251,9 @@ export default class BgzipTaffyAdapter extends BaseFeatureDataAdapter {
       }
 
       nextEntry = nextEntry ?? records.at(-1)
+      // we NEED at least a firstEntry (validate behavior?) because othrwise it fetches whole
+      // file whn you request e.g. out of range region (e.g. taf in chr22:1-100
+      // and you are at chr22:200-300)
       if (firstEntry && nextEntry) {
         const response = await file.read(
           nextEntry.virtualOffset.blockPosition -
@@ -223,12 +261,10 @@ export default class BgzipTaffyAdapter extends BaseFeatureDataAdapter {
           firstEntry.virtualOffset.blockPosition,
         )
         const buffer = await unzip(response)
-        return decoder
-          .decode(buffer.slice(firstEntry.virtualOffset.dataPosition))
-          .split('\n')
+        return buffer.slice(firstEntry.virtualOffset.dataPosition)
       }
     }
-    return []
+    return undefined
   }
 
   freeResources(): void {}

package/src/BgzipTaffyAdapter/util.ts

@@ -0,0 +1,25 @@
+export function parseLineByLine<T>(
+  buffer: Uint8Array,
+  cb: (line: string) => T | undefined,
+): T[] {
+  let blockStart = 0
+  const entries: T[] = []
+  const decoder = new TextDecoder('utf8')
+  while (blockStart < buffer.length) {
+    const n = buffer.indexOf(10, blockStart)
+    if (n === -1) {
+      break
+    }
+    const b = buffer.subarray(blockStart, n)
+    const line = decoder.decode(b).trim()
+    if (line) {
+      const entry = cb(line)
+      if (entry) {
+        entries.push(entry)
+      }
+    }
+
+    blockStart = n + 1
+  }
+  return entries
+}

package/src/BigMafAdapter/BigMafAdapter.ts

@@ -1,5 +1,5 @@
 import { BaseFeatureDataAdapter } from '@jbrowse/core/data_adapters/BaseAdapter'
-import { Feature, Region, SimpleFeature } from '@jbrowse/core/util'
+import { SimpleFeature, updateStatus } from '@jbrowse/core/util'
 import { openLocation } from '@jbrowse/core/util/io'
 import { ObservableCreate } from '@jbrowse/core/util/rxjs'
 import { getSnapshot } from 'mobx-state-tree'
@@ -8,6 +8,9 @@ import { firstValueFrom, toArray } from 'rxjs'
 import parseNewick from '../parseNewick'
 import { normalize } from '../util'
 
+import type { BaseOptions } from '@jbrowse/core/data_adapters/BaseAdapter'
+import type { Feature, Region } from '@jbrowse/core/util'
+
 interface OrganismRecord {
   chr: string
   start: number
@@ -51,11 +54,14 @@ export default class BigMafAdapter extends BaseFeatureDataAdapter {
     return adapter.getHeader()
   }
 
-  getFeatures(query: Region) {
+  getFeatures(query: Region, opts?: BaseOptions) {
+    const { statusCallback = () => {} } = opts || {}
     return ObservableCreate<Feature>(async observer => {
       const { adapter } = await this.setup()
-      const features = await firstValueFrom(
-        adapter.getFeatures(query).pipe(toArray()),
+      const features = await updateStatus(
+        'Downloading alignment',
+        statusCallback,
+        () => firstValueFrom(adapter.getFeatures(query).pipe(toArray())),
       )
       for (const feature of features) {
         const maf = feature.get('mafBlock') as string

package/src/LinearMafDisplay/components/ColorLegend.tsx

@@ -2,7 +2,6 @@ import React from 'react'
 
 import { observer } from 'mobx-react'
 
-// locals
 import { LinearMafDisplayModel } from '../stateModel'
 import RectBg from './RectBg'
 import Tree from './Tree'
@@ -16,7 +15,7 @@ const ColorLegend = observer(function ({
   svgFontSize: number
   labelWidth: number
 }) {
-  const { totalHeight, treeWidth, samples, rowHeight } = model
+  const { totalHeight, treeWidth, samples = [], rowHeight } = model
   const canDisplayLabel = rowHeight >= 8
   const boxHeight = Math.min(20, rowHeight)
 

package/src/LinearMafDisplay/components/ReactComponent.tsx

@@ -52,7 +52,7 @@ const LinearMafDisplay = observer(function (props: {
     >
       <BaseLinearDisplayComponent {...props} />
       <YScaleBars model={model} />
-      {mouseY ? (
+      {mouseY && sources ? (
         <div style={{ position: 'relative' }}>
           <svg
             className={classes.cursor}

package/src/LinearMafDisplay/components/SvgWrapper.tsx

@@ -18,7 +18,7 @@ const SvgWrapper = observer(function ({
   if (exportSVG) {
     return <>{children}</>
   } else {
-    const { rowHeight, samples } = model
+    const { totalHeight } = model
     return (
       <svg
         style={{
@@ -26,7 +26,7 @@ const SvgWrapper = observer(function ({
           top: 0,
           left: 0,
           pointerEvents: 'none',
-          height: samples.length * rowHeight,
+          height: totalHeight,
           width: getContainingView(model).width,
         }}
       >

package/src/LinearMafDisplay/components/YScaleBars.tsx

@@ -22,8 +22,9 @@ export const YScaleBars = observer(function (props: {
 
   const labelWidth = max(
     samples
-      .map(s => measureText(s.label, svgFontSize))
-      .map(width => (canDisplayLabel ? width : minWidth)),
+      ?.map(s => measureText(s.label, svgFontSize))
+      .map(width => (canDisplayLabel ? width : minWidth)) || [],
+    0,
   )
 
   return (

package/src/LinearMafDisplay/renderSvg.tsx

@@ -1,15 +1,15 @@
 import React from 'react'
 
 import { getContainingView } from '@jbrowse/core/util'
-import {
+
+import YScaleBars from './components/YScaleBars'
+
+import type { LinearMafDisplayModel } from './stateModel'
+import type {
   ExportSvgDisplayOptions,
   LinearGenomeViewModel,
 } from '@jbrowse/plugin-linear-genome-view'
 
-// locals
-import YScaleBars from './components/YScaleBars'
-import { LinearMafDisplayModel } from './stateModel'
-
 export async function renderSvg(
   self: LinearMafDisplayModel,
   opts: ExportSvgDisplayOptions,

package/src/LinearMafDisplay/stateModel.ts

@@ -1,27 +1,26 @@
-import PluginManager from '@jbrowse/core/PluginManager'
-import {
-  AnyConfigurationModel,
-  AnyConfigurationSchemaType,
-  ConfigurationReference,
-  getConf,
-} from '@jbrowse/core/configuration'
+import { ConfigurationReference, getConf } from '@jbrowse/core/configuration'
 import { getEnv, getSession } from '@jbrowse/core/util'
 import { getRpcSessionId } from '@jbrowse/core/util/tracks'
-import { ExportSvgDisplayOptions } from '@jbrowse/plugin-linear-genome-view'
 import { ascending } from 'd3-array'
-import { type HierarchyNode, cluster, hierarchy } from 'd3-hierarchy'
+import { cluster, hierarchy } from 'd3-hierarchy'
+import deepEqual from 'fast-deep-equal'
 import { autorun } from 'mobx'
-import { Instance, addDisposer, isAlive, types } from 'mobx-state-tree'
+import { addDisposer, isAlive, types } from 'mobx-state-tree'
 
 import SetRowHeightDialog from './components/SetRowHeight'
-import {
-  NodeWithIds,
-  NodeWithIdsAndLength,
-  maxLength,
-  setBrLength,
-} from './types'
+import { maxLength, setBrLength } from './types'
 import { normalize } from '../util'
 
+import type { NodeWithIds, NodeWithIdsAndLength } from './types'
+import type PluginManager from '@jbrowse/core/PluginManager'
+import type {
+  AnyConfigurationModel,
+  AnyConfigurationSchemaType,
+} from '@jbrowse/core/configuration'
+import type { ExportSvgDisplayOptions } from '@jbrowse/plugin-linear-genome-view'
+import type { HierarchyNode } from 'd3-hierarchy'
+import type { Instance } from 'mobx-state-tree'
+
 interface Sample {
   id: string
   label: string
@@ -90,11 +89,11 @@ export default function stateModelFactory(
       /**
        * #volatile
        */
-      volatileSamples: [] as Sample[],
+      volatileSamples: undefined as Sample[] | undefined,
       /**
        * #volatile
        */
-      tree: undefined as any,
+      volatileTree: undefined as any,
     }))
     .actions(self => ({
       /**
@@ -125,8 +124,12 @@ export default function stateModelFactory(
        * #action
        */
       setSamples({ samples, tree }: { samples: Sample[]; tree: unknown }) {
-        self.volatileSamples = samples
-        self.tree = tree
+        if (!deepEqual(samples, self.volatileSamples)) {
+          self.volatileSamples = samples
+        }
+        if (!deepEqual(tree, self.volatileTree)) {
+          self.volatileTree = tree
+        }
       },
     }))
     .views(self => ({
@@ -159,8 +162,8 @@ export default function stateModelFactory(
        * #getter
        */
       get root() {
-        return self.tree
-          ? hierarchy(self.tree, d => d.children)
+        return self.volatileTree
+          ? hierarchy(self.volatileTree, d => d.children)
               // todo: investigate whether needed, typescript says children always true
               .sum(d => (d.children ? 0 : 1))
               .sort((a, b) => ascending(a.data.length || 1, b.data.length || 1))
@@ -196,7 +199,7 @@ export default function stateModelFactory(
        * #getter
        */
       get totalHeight() {
-        return this.samples.length * self.rowHeight
+        return this.samples ? this.samples.length * self.rowHeight : 1
       },
       /**
        * #getter
@@ -237,8 +240,11 @@ export default function stateModelFactory(
             rowProportion,
             mismatchRendering,
           } = self
+          const s = superRenderProps()
           return {
-            ...superRenderProps(),
+            ...s,
+            notReady:
+              !self.volatileSamples || !self.volatileTree || super.notReady,
             config: rendererConfig,
             samples,
             rowHeight,