@jbrowse/plugin-alignments 1.5.8 → 1.6.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jbrowse/plugin-alignments",
-  "version": "1.5.8",
+  "version": "1.6.2",
   "description": "JBrowse 2 alignments adapters, tracks, etc.",
   "keywords": [
     "jbrowse",
@@ -35,7 +35,7 @@
     "useSrc": "node ../../scripts/useSrc.js"
   },
   "dependencies": {
-    "@gmod/bam": "^1.1.9",
+    "@gmod/bam": "^1.1.11",
     "@gmod/cram": "^1.5.9",
     "@material-ui/icons": "^4.9.1",
     "abortable-promise-cache": "^1.1.3",
@@ -61,5 +61,5 @@
   "publishConfig": {
     "access": "public"
   },
-  "gitHead": "f16afc382629a4a830f9040410a9ff1a3a328918"
+  "gitHead": "92455c6021abd69548496a450983d89f8837860d"
 }

package/src/BamAdapter/BamAdapter.ts

@@ -4,7 +4,11 @@ import {
   BaseOptions,
 } from '@jbrowse/core/data_adapters/BaseAdapter'
 import { Region } from '@jbrowse/core/util/types'
-import { checkAbortSignal } from '@jbrowse/core/util'
+import {
+  checkAbortSignal,
+  bytesForRegions,
+  updateStatus,
+} from '@jbrowse/core/util'
 import { openLocation } from '@jbrowse/core/util/io'
 import { ObservableCreate } from '@jbrowse/core/util/rxjs'
 import { Feature } from '@jbrowse/core/util/simpleFeature'
@@ -35,7 +39,6 @@ export default class BamAdapter extends BaseFeatureDataAdapter {
       const location = readConfObject(this.config, ['index', 'location'])
       const indexType = readConfObject(this.config, ['index', 'indexType'])
       const chunkSizeLimit = readConfObject(this.config, 'chunkSizeLimit')
-      const fetchSizeLimit = readConfObject(this.config, 'fetchSizeLimit')
       const bam = new BamFile({
         bamFilehandle: openLocation(bamLocation, this.pluginManager),
         csiFilehandle:
@@ -47,7 +50,7 @@ export default class BamAdapter extends BaseFeatureDataAdapter {
             ? openLocation(location, this.pluginManager)
             : undefined,
         chunkSizeLimit,
-        fetchSizeLimit,
+        fetchSizeLimit: 100_000_000,
       })
 
       const adapterConfig = readConfObject(this.config, 'sequenceAdapter')
@@ -70,40 +73,44 @@ export default class BamAdapter extends BaseFeatureDataAdapter {
     return bam.getHeaderText(opts)
   }
 
-  private async setup(opts?: BaseOptions) {
-    // note that derived classes may not provide a BAM directly so this is
-    // conditional
+  private async setupPre(opts?: BaseOptions) {
     const { statusCallback = () => {} } = opts || {}
-    if (!this.setupP) {
-      this.setupP = this.configure()
-        .then(async ({ bam }) => {
-          statusCallback('Downloading index')
-          const samHeader = await bam.getHeader(opts)
-
-          // use the @SQ lines in the header to figure out the
-          // mapping between ref ref ID numbers and names
-          const idToName: string[] = []
-          const nameToId: Record<string, number> = {}
-          samHeader
-            .filter(l => l.tag === 'SQ')
-            .forEach((sqLine, refId) => {
-              sqLine.data.forEach(item => {
-                if (item.tag === 'SN') {
-                  // this is the ref name
-                  const refName = item.value
-                  nameToId[refName] = refId
-                  idToName[refId] = refName
-                }
-              })
+    const { bam } = await this.configure()
+    this.samHeader = await updateStatus(
+      'Downloading index',
+      statusCallback,
+      async () => {
+        const samHeader = await bam.getHeader(opts)
+
+        // use the @SQ lines in the header to figure out the
+        // mapping between ref ref ID numbers and names
+        const idToName: string[] = []
+        const nameToId: Record<string, number> = {}
+        samHeader
+          .filter(l => l.tag === 'SQ')
+          .forEach((sqLine, refId) => {
+            sqLine.data.forEach(item => {
+              if (item.tag === 'SN') {
+                // this is the ref name
+                const refName = item.value
+                nameToId[refName] = refId
+                idToName[refId] = refName
+              }
             })
-          statusCallback('')
-          this.samHeader = { idToName, nameToId }
-          return this.samHeader
-        })
-        .catch(e => {
-          this.setupP = undefined
-          throw e
-        })
+          })
+
+        return { idToName, nameToId }
+      },
+    )
+    return this.samHeader
+  }
+
+  private async setup(opts?: BaseOptions) {
+    if (!this.setupP) {
+      this.setupP = this.setupPre(opts).catch(e => {
+        this.setupP = undefined
+        throw e
+      })
     }
     return this.setupP
   }
@@ -188,6 +195,20 @@ export default class BamAdapter extends BaseFeatureDataAdapter {
     }, signal)
   }
 
+  async estimateRegionsStats(regions: Region[], opts?: BaseOptions) {
+    const { bam } = await this.configure()
+    // @ts-ignore
+    const index = bam.index
+    // this is a method to avoid calling on htsget adapters
+    if (index.filehandle !== '?') {
+      const bytes = await bytesForRegions(regions, index)
+      const fetchSizeLimit = readConfObject(this.config, 'fetchSizeLimit')
+      return { bytes, fetchSizeLimit }
+    } else {
+      return super.estimateRegionsStats(regions, opts)
+    }
+  }
+
   freeResources(/* { region } */): void {}
 
   // depends on setup being called before the BAM constructor

package/src/BamAdapter/BamSlightlyLazyFeature.ts

@@ -34,7 +34,6 @@ export default class BamSlightlyLazyFeature implements Feature {
   }
 
   _get_flags(): string {
-    // @ts-ignore
     return this.record.flags
   }
 
@@ -42,11 +41,6 @@ export default class BamSlightlyLazyFeature implements Feature {
     return this.record.isReverseComplemented() ? -1 : 1
   }
 
-  _get_read_group_id(): number {
-    // @ts-ignore
-    return this.record.readGroupId
-  }
-
   _get_pair_orientation() {
     return this.record.isPaired() ? this.record.getPairOrientation() : undefined
   }
@@ -60,36 +54,35 @@ export default class BamSlightlyLazyFeature implements Feature {
     return this.record._refID
   }
 
-  _get_next_refName(): string | undefined {
+  _get_next_refName() {
     return this.adapter.refIdToName(this.record._next_refid())
   }
 
-  _get_next_segment_position(): string | undefined {
-    return this.record.isPaired()
-      ? `${this.adapter.refIdToName(this.record._next_refid())}:${
-          this.record._next_pos() + 1
-        }`
+  _get_next_segment_position() {
+    const { record, adapter } = this
+    return record.isPaired()
+      ? `${adapter.refIdToName(record._next_refid())}:${record._next_pos() + 1}`
       : undefined
   }
 
-  _get_seq(): string {
+  _get_seq() {
     return this.record.getReadBases()
   }
 
-  _get_MD(): string | undefined {
-    const md = this.record.get('MD')
-    const seq = this.get('seq')
+  _get_MD() {
+    const md = this.record.get('MD') as string | undefined
+    const seq = this.get('seq') as string
     if (!md && seq && this.ref) {
       return generateMD(this.ref, this.record.getReadBases(), this.get('CIGAR'))
     }
     return md
   }
 
-  qualRaw(): Buffer | undefined {
+  qualRaw() {
     return this.record.qualRaw()
   }
 
-  set(): void {}
+  set() {}
 
   tags() {
     const properties = Object.getOwnPropertyNames(
@@ -115,7 +108,7 @@ export default class BamSlightlyLazyFeature implements Feature {
     ]
   }
 
-  id(): string {
+  id() {
     return `${this.adapter.id}-${this.record.id()}`
   }
 
@@ -130,19 +123,19 @@ export default class BamSlightlyLazyFeature implements Feature {
     return this.record.get(field)
   }
 
-  _get_refName(): string | undefined {
+  _get_refName() {
     return this.adapter.refIdToName(this.record.seq_id())
   }
 
-  parent(): undefined {
+  parent() {
     return undefined
   }
 
-  children(): undefined {
+  children() {
     return undefined
   }
 
-  pairedFeature(): boolean {
+  pairedFeature() {
     return false
   }
 
@@ -163,7 +156,7 @@ export default class BamSlightlyLazyFeature implements Feature {
     } = {
       cigarAttributeName: 'CIGAR',
     },
-  ): Mismatch[] {
+  ) {
     const { cigarAttributeName } = opts
     let mismatches: Mismatch[] = []
     let cigarOps: string[] = []
@@ -185,7 +178,7 @@ export default class BamSlightlyLazyFeature implements Feature {
   }: {
     cigarAttributeName?: string
     mdAttributeName?: string
-  } = {}): Mismatch[] {
+  } = {}) {
     let mismatches: Mismatch[] = []
     let cigarOps: string[] = []
 

package/src/BamAdapter/configSchema.ts

@@ -25,11 +25,11 @@ export default types.late(() =>
       }),
       chunkSizeLimit: {
         type: 'number',
-        defaultValue: 100000000,
+        defaultValue: 100_000_000,
       },
       fetchSizeLimit: {
         type: 'number',
-        defaultValue: 500000000,
+        defaultValue: 5_000_000,
       },
       sequenceAdapter: {
         type: 'frozen',

package/src/CramAdapter/CramAdapter.ts

@@ -24,14 +24,13 @@ interface Header {
 }
 
 export default class CramAdapter extends BaseFeatureDataAdapter {
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  private cram: any
+  samHeader: Header = {}
 
-  private setupP?: Promise<Header>
-
-  private sequenceAdapter?: BaseFeatureDataAdapter
-
-  public samHeader: Header = {}
+  private setupP?: Promise<{
+    samHeader: Header
+    cram: any // eslint-disable-line @typescript-eslint/no-explicit-any
+    sequenceAdapter: any // eslint-disable-line @typescript-eslint/no-explicit-any
+  }>
 
   // maps a refname to an id
   private seqIdToRefName: string[] | undefined
@@ -48,14 +47,16 @@ export default class CramAdapter extends BaseFeatureDataAdapter {
     if (!craiLocation) {
       throw new Error('missing craiLocation argument')
     }
-    this.cram = new IndexedCramFile({
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const cram: any = new IndexedCramFile({
       cramFilehandle: openLocation(cramLocation, this.pluginManager),
       index: new CraiIndex({
         filehandle: openLocation(craiLocation, this.pluginManager),
       }),
       seqFetch: this.seqFetch.bind(this),
       checkSequenceMD5: false,
-      fetchSizeLimit: this.config.fetchSizeLimit || 600000000,
+      fetchSizeLimit: 200_000_000, // just make this a large size to avoid hitting it
     })
     // instantiate the sequence adapter
     const sequenceAdapterType = readConfObject(this.config, [
@@ -67,62 +68,55 @@ export default class CramAdapter extends BaseFeatureDataAdapter {
       throw new Error('Error getting subadapter')
     }
 
-    const { dataAdapter } = await this.getSubAdapter(
-      readConfObject(this.config, 'sequenceAdapter'),
-    )
-    if (dataAdapter instanceof BaseFeatureDataAdapter) {
-      this.sequenceAdapter = dataAdapter
-    } else {
+    const seqConf = readConfObject(this.config, 'sequenceAdapter')
+    const { dataAdapter: sequenceAdapter } = await this.getSubAdapter(seqConf)
+
+    if (!(sequenceAdapter instanceof BaseFeatureDataAdapter)) {
       throw new Error(
         `CRAM feature adapters cannot use sequence adapters of type '${sequenceAdapterType}'`,
       )
     }
-    return { sequenceAdapter: this.sequenceAdapter }
+
+    return { cram, sequenceAdapter }
   }
 
   async getHeader(opts?: BaseOptions) {
-    await this.configure()
-    return this.cram.cram.getHeaderText(opts)
+    const { cram } = await this.configure()
+    return cram.cram.getHeaderText(opts)
   }
 
   private async seqFetch(seqId: number, start: number, end: number) {
     start -= 1 // convert from 1-based closed to interbase
 
-    const refSeqStore = this.sequenceAdapter
-    if (!refSeqStore) {
-      return undefined
-    }
+    const { sequenceAdapter } = await this.configure()
     const refName = this.refIdToOriginalName(seqId) || this.refIdToName(seqId)
     if (!refName) {
       return undefined
     }
 
-    const features = refSeqStore.getFeatures(
-      {
+    const seqChunks = await sequenceAdapter
+      .getFeatures({
         refName,
         start,
         end,
         assemblyName: '',
-      },
-      {},
-    )
-
-    const seqChunks = await features.pipe(toArray()).toPromise()
+      })
+      .pipe(toArray())
+      .toPromise()
 
-    const trimmed: string[] = []
-    seqChunks
+    const sequence = seqChunks
       .sort((a, b) => a.get('start') - b.get('start'))
-      .forEach((chunk: Feature) => {
+      .map(chunk => {
         const chunkStart = chunk.get('start')
         const chunkEnd = chunk.get('end')
         const trimStart = Math.max(start - chunkStart, 0)
         const trimEnd = Math.min(end - chunkStart, chunkEnd - chunkStart)
         const trimLength = trimEnd - trimStart
         const chunkSeq = chunk.get('seq') || chunk.get('residues')
-        trimmed.push(chunkSeq.substr(trimStart, trimLength))
+        return chunkSeq.substr(trimStart, trimLength)
       })
+      .join('')
 
-    const sequence = trimmed.join('')
     if (sequence.length !== end - start) {
       throw new Error(
         `sequence fetch failed: fetching ${refName}:${(
@@ -135,60 +129,56 @@ export default class CramAdapter extends BaseFeatureDataAdapter {
     return sequence
   }
 
-  private async setup(opts?: BaseOptions) {
+  private async setupPre(opts?: BaseOptions) {
     const { statusCallback = () => {} } = opts || {}
-    if (!this.setupP) {
-      this.setupP = this.configure()
-        .then(async () => {
-          statusCallback('Downloading index')
-          const samHeader: HeaderLine[] = await this.cram.cram.getSamHeader(
-            opts?.signal,
-          )
-
-          // use the @SQ lines in the header to figure out the
-          // mapping between ref ID numbers and names
-          const idToName: string[] = []
-          const nameToId: Record<string, number> = {}
-          samHeader
-            .filter(l => l.tag === 'SQ')
-            .forEach((sqLine, refId) => {
-              sqLine.data.forEach(item => {
-                if (item.tag === 'SN') {
-                  // this is the ref name
-                  const refName = item.value
-                  nameToId[refName] = refId
-                  idToName[refId] = refName
-                }
-              })
-            })
-
-          const readGroups = samHeader
-            .filter(l => l.tag === 'RG')
-            .map(rgLine => rgLine.data.find(item => item.tag === 'ID')?.value)
-
-          if (idToName.length) {
-            this.samHeader = { idToName, nameToId, readGroups }
+    const configured = await this.configure()
+    statusCallback('Downloading index')
+    const { cram } = configured
+    const samHeader: HeaderLine[] = await cram.cram.getSamHeader(opts?.signal)
+
+    // use the @SQ lines in the header to figure out the
+    // mapping between ref ID numbers and names
+    const idToName: string[] = []
+    const nameToId: Record<string, number> = {}
+    samHeader
+      .filter(l => l.tag === 'SQ')
+      .forEach((sqLine, refId) => {
+        sqLine.data.forEach(item => {
+          if (item.tag === 'SN') {
+            // this is the ref name
+            const refName = item.value
+            nameToId[refName] = refId
+            idToName[refId] = refName
           }
-          statusCallback('')
-          return this.samHeader
-        })
-        .catch(e => {
-          this.setupP = undefined
-          throw e
         })
+      })
+
+    const readGroups = samHeader
+      .filter(l => l.tag === 'RG')
+      .map(rgLine => rgLine.data.find(item => item.tag === 'ID')?.value)
+
+    const data = { idToName, nameToId, readGroups }
+    statusCallback('')
+    this.samHeader = data
+    return { samHeader: data, ...configured }
+  }
+
+  private async setup(opts?: BaseOptions) {
+    if (!this.setupP) {
+      this.setupP = this.setupPre(opts).catch(e => {
+        this.setupP = undefined
+        throw e
+      })
     }
     return this.setupP
   }
 
   async getRefNames(opts?: BaseOptions) {
-    await this.setup(opts)
-    if (this.samHeader.idToName) {
-      return this.samHeader.idToName
-    }
-    if (this.sequenceAdapter) {
-      return this.sequenceAdapter.getRefNames()
+    const { samHeader } = await this.setup(opts)
+    if (!samHeader.idToName) {
+      throw new Error('CRAM file has no header lines')
     }
-    throw new Error('unable to get refnames')
+    return samHeader.idToName
   }
 
   // use info from the SAM header if possible, but fall back to using
@@ -227,27 +217,24 @@ export default class CramAdapter extends BaseFeatureDataAdapter {
     const { refName, start, end, originalRefName } = region
 
     return ObservableCreate<Feature>(async observer => {
-      await this.setup(opts)
-      if (this.sequenceAdapter && !this.seqIdToRefName) {
-        this.seqIdToRefName = await this.sequenceAdapter.getRefNames(opts)
+      const { cram, sequenceAdapter } = await this.setup(opts)
+      statusCallback('Downloading alignments')
+      if (!this.seqIdToRefName) {
+        this.seqIdToRefName = await sequenceAdapter.getRefNames(opts)
       }
       const refId = this.refNameToId(refName)
       if (refId !== undefined) {
         if (originalRefName) {
           this.seqIdToOriginalRefName[refId] = originalRefName
         }
-        statusCallback('Downloading alignments')
-        const records = await this.cram.getRecordsForRange(
-          refId,
-          start,
-          end,
-          opts,
-        )
+        const records = await cram.getRecordsForRange(refId, start, end, opts)
         checkAbortSignal(signal)
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         records.forEach((record: any) => {
           observer.next(this.cramRecordToFeature(record))
         })
+      } else {
+        console.warn('Unknown refName', refName)
       }
       statusCallback('')
       observer.complete()
@@ -256,8 +243,35 @@ export default class CramAdapter extends BaseFeatureDataAdapter {
 
   freeResources(/* { region } */): void {}
 
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  cramRecordToFeature(record: any): Feature {
+  cramRecordToFeature(record: unknown) {
     return new CramSlightlyLazyFeature(record, this)
   }
+
+  // we return the configured fetchSizeLimit, and the bytes for the region
+  async estimateRegionsStats(regions: Region[], opts?: BaseOptions) {
+    const bytes = await this.bytesForRegions(regions, opts)
+    const fetchSizeLimit = readConfObject(this.config, 'fetchSizeLimit')
+    return {
+      bytes,
+      fetchSizeLimit,
+    }
+  }
+
+  /**
+   * get the approximate number of bytes queried from the file for the given
+   * query regions
+   * @param regions - list of query regions
+   */
+  private async bytesForRegions(regions: Region[], _opts?: BaseOptions) {
+    const { cram } = await this.configure()
+    const blockResults = await Promise.all(
+      regions.map(region => {
+        const { refName, start, end } = region
+        const chrId = this.refNameToId(refName)
+        return cram.index.getEntriesForRange(chrId, start, end)
+      }),
+    )
+
+    return blockResults.flat().reduce((a, b) => a + b.sliceBytes, 0)
+  }
 }

package/src/CramAdapter/configSchema.ts

@@ -1,12 +1,16 @@
 import PluginManager from '@jbrowse/core/PluginManager'
 import { ConfigurationSchema } from '@jbrowse/core/configuration'
+import { types } from 'mobx-state-tree'
 
 export default (pluginManager: PluginManager) => {
-  const { types } = pluginManager.lib['mobx-state-tree']
   return types.late(() =>
     ConfigurationSchema(
       'CramAdapter',
       {
+        fetchSizeLimit: {
+          type: 'number',
+          defaultValue: 3_000_000,
+        },
         cramLocation: {
           type: 'fileLocation',
           defaultValue: {

package/src/LinearAlignmentsDisplay/models/model.tsx

@@ -32,6 +32,7 @@ const stateModelFactory = (
         height: 250,
         showCoverage: true,
         showPileup: true,
+        userFeatureScreenDensity: types.maybe(types.number),
       }),
     )
     .volatile(() => ({
@@ -126,9 +127,9 @@ const stateModelFactory = (
           height: self.snpCovHeight,
         }
       },
-      setUserBpPerPxLimit(limit: number) {
-        self.PileupDisplay.setUserBpPerPxLimit(limit)
-        self.SNPCoverageDisplay.setUserBpPerPxLimit(limit)
+      setUserFeatureScreenDensity(limit: number) {
+        self.PileupDisplay.setUserFeatureScreenDensity(limit)
+        self.SNPCoverageDisplay.setUserFeatureScreenDensity(limit)
       },
       setPileupDisplay(displayConfig: AnyConfigurationModel) {
         self.PileupDisplay = {

package/src/LinearPileupDisplay/configSchema.ts

@@ -23,10 +23,10 @@ function PileupConfigFactory(pluginManager: PluginManager) {
         SvgFeatureRenderer: SvgFeatureRendererConfigSchema,
       }),
       renderer: '',
-      maxDisplayedBpPerPx: {
+      maxFeatureScreenDensity: {
         type: 'number',
-        description: 'maximum bpPerPx that is displayed in the view',
-        defaultValue: 100,
+        description: 'maximum features per pixel that is displayed in the view',
+        defaultValue: 5,
       },
       colorScheme: {
         type: 'stringEnum',

package/src/LinearPileupDisplay/model.ts

@@ -95,15 +95,11 @@ const stateModelFactory = (configSchema: LinearPileupDisplayConfigModel) =>
       colorTagMap: observable.map<string, string>({}),
       modificationTagMap: observable.map<string, string>({}),
       ready: false,
-      currBpPerPx: 0,
     }))
     .actions(self => ({
       setReady(flag: boolean) {
         self.ready = flag
       },
-      setCurrBpPerPx(n: number) {
-        self.currBpPerPx = n
-      },
       setMaxHeight(n: number) {
         self.trackMaxHeight = n
       },
@@ -404,16 +400,21 @@ const stateModelFactory = (configSchema: LinearPileupDisplayConfigModel) =>
         renderProps() {
           const view = getContainingView(self) as LGV
           const {
-            ready,
             colorTagMap,
             modificationTagMap,
             sortedBy,
             colorBy,
             rpcDriverName,
           } = self
+
+          const superProps = superRenderProps()
+
           return {
-            ...superRenderProps(),
-            notReady: !ready || (sortedBy && self.currBpPerPx !== view.bpPerPx),
+            ...superProps,
+            notReady:
+              superProps.notReady ||
+              !self.ready ||
+              (sortedBy && self.currBpPerPx !== view.bpPerPx),
             rpcDriverName,
             displayModel: self,
             sortedBy,