@gmod/bam 9.0.1 → 10.0.0

package/src/bamFile.ts

@@ -12,7 +12,10 @@ import {
   BAM_MAGIC,
   MAX_CONCURRENT_CHUNK_READS,
   appendInRange,
+  decodeHeaderText,
+  optimizeChunks,
   parseRefSeqs,
+  readBlockSize,
   resolveFilehandle,
   throwIfAborted,
 } from './util.ts'
@@ -464,9 +467,7 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     const parsed = parseRefSeqs(uncba, headLen + 8, this.renameRefSeq)
     let samHeader
     if (parsed) {
-      const headerText = new TextDecoder('utf8').decode(
-        uncba.subarray(8, 8 + headLen),
-      )
+      const headerText = decodeHeaderText(uncba, headLen)
       this.header = headerText
       this.chrToIndex = parsed.chrToIndex
       this.indexToChr = parsed.indexToChr
@@ -539,7 +540,7 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     if (chrId === undefined || !this.index) {
       return []
     }
-    const chunks = await this.index.blocksForRange(chrId, min - 1, max, opts)
+    const chunks = await this.index.blocksForRange(chrId, min, max, opts)
     return this._fetchChunkFeatures(chunks, chrId, chr, min, max, opts)
   }
 
@@ -818,10 +819,12 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     // make the count NaN.
     const readNameCounts = new Map<string, number>()
     const readIds = new Set<number>()
+    const names = new Array<string>(records.length)
 
     for (let i = 0, l = records.length; i < l; i++) {
       const r = records[i]!
       const name = r.name
+      names[i] = name
       readNameCounts.set(name, (readNameCounts.get(name) ?? 0) + 1)
       readIds.add(r.fileOffset)
     }
@@ -829,10 +832,9 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     const matePromises: Promise<Chunk[]>[] = []
     for (let i = 0, l = records.length; i < l; i++) {
       const f = records[i]!
-      const name = f.name
       if (
         this.index &&
-        readNameCounts.get(name) === 1 &&
+        readNameCounts.get(names[i]!) === 1 &&
         (pairAcrossChr ||
           (f.next_refid === chrId &&
             Math.abs(f.start - f.next_pos) < maxInsertSize))
@@ -848,20 +850,11 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
       }
     }
 
-    const map = new Map<string, Chunk>()
-    const res = await Promise.all(matePromises)
-    for (let i = 0, l = res.length; i < l; i++) {
-      const chunks = res[i]!
-      for (let j = 0, jl = chunks.length; j < jl; j++) {
-        const m = chunks[j]!
-        // Key on the virtual-offset span — the same key _cachedChunkFeatures
-        // uses. Chunk.toString() also folds in `bin` and fetchedSize(), which
-        // keeps two chunks covering an identical span apart here even though
-        // the cache below collapses them, so their records came back twice.
-        map.set(chunkCacheKey(m), m)
-      }
-    }
-    const mateChunks = [...map.values()]
+    // Each mate's lookup is merged on its own, so two of them can resolve to
+    // different spans over the same records. Merging the union again makes
+    // them disjoint, which is what keeps a mate in the overlap from coming
+    // back once per span.
+    const mateChunks = optimizeChunks((await Promise.all(matePromises)).flat())
 
     // Bounded for the reason ADR 0008 bounds the main query path: a viewAsPairs
     // query over a busy region resolves to many distinct mate chunks, and an
@@ -961,7 +954,7 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     const hasCpositions = cpositions.length > 0
 
     while (blockStart + 4 < ba.length) {
-      const blockSize = dataView.getInt32(blockStart, true)
+      const blockSize = readBlockSize(dataView, blockStart)
       const blockEnd = blockStart + 4 + blockSize - 1
 
       if (hasDpositions) {

package/src/csi.ts

@@ -10,6 +10,7 @@ import {
 } from './util.ts'
 import { VirtualOffset, fromBytes } from './virtualOffset.ts'
 
+import type { ParsedIndexBase, RefIndex } from './indexFile.ts'
 import type { BaseOpts } from './util.ts'
 
 const CSI1_MAGIC = 21582659 // CSI\1
@@ -24,10 +25,18 @@ function rshift(num: number, bits: number) {
   return Math.floor(num / 2 ** bits)
 }
 
-export default class CSI extends IndexFile {
+interface CsiRefIndex extends RefIndex {
+  loffsets: Map<number, VirtualOffset>
+}
+
+interface CsiParsed extends ParsedIndexBase<CsiRefIndex> {
+  csi: true
+}
+
+export default class CSI extends IndexFile<CsiParsed> {
   private maxBinNumber = 0
-  private depth = 0
-  private minShift = 0
+  protected depth = 0
+  protected minShift = 0
 
   // CSI omits the linear index that BAI's indexCov derives coverage from
   // (CSIv1.tex §3, hts-specs), so there's no equivalent to return.
@@ -74,7 +83,7 @@ export default class CSI extends IndexFile {
   }
 
   // fetch and parse the index
-  async _parse(opts: BaseOpts) {
+  async _parse(opts: BaseOpts): Promise<CsiParsed> {
     const buffer = await this.filehandle.readFile(opts)
     const bytes = await unzip(buffer)
 
@@ -93,7 +102,7 @@ export default class CSI extends IndexFile {
 
     this.minShift = dataView.getInt32(4, true)
     this.depth = dataView.getInt32(8, true)
-    this.maxBinNumber = ((1 << ((this.depth + 1) * 3)) - 1) / 7
+    this.maxBinNumber = (8 ** (this.depth + 1) - 1) / 7
     const maxBinNumber = this.maxBinNumber
     const auxLength = dataView.getInt32(12, true)
     // A tabix-only branch, which is why parseAuxData and the parseNameBytes it
@@ -128,10 +137,10 @@ export default class CSI extends IndexFile {
         if (bin > this.maxBinNumber) {
           curr += 28 + 16
         } else {
-          // A bin's loffset is the smallest virtual offset of any record in it,
-          // so the minimum over loffsets is already the minimum over the bin's
-          // chunks — one read per bin instead of one per chunk. Checked against
-          // every .csi in test/data: same answer on all 19.
+          // A bin's loffset is the linear-index entry at its first window, so
+          // the smallest over every bin is the first record's offset — one
+          // read per bin instead of one per chunk. Checked against every .csi
+          // in test/data: same answer on all 19.
           firstDataLine = minVirtualOffset(bytes, curr, 1, firstDataLine)
           curr += 8 // loffset
           const chunkCount = dataView.getInt32(curr, true)
@@ -149,6 +158,7 @@ export default class CSI extends IndexFile {
       const binCount = dataView.getInt32(curr, true)
       curr += 4
       const binIndex: Record<number, Chunk[]> = {}
+      const loffsets = new Map<number, VirtualOffset>()
       let pseudoBinStats
       for (let j = 0; j < binCount; j++) {
         const bin = dataView.getUint32(curr, true)
@@ -157,7 +167,8 @@ export default class CSI extends IndexFile {
           pseudoBinStats = parsePseudoBin(bytes, curr + 28)
           curr += 28 + 16
         } else {
-          curr += 8 // skip loffset; firstDataLine was computed in the first pass
+          loffsets.set(bin, fromBytes(bytes, curr))
+          curr += 8
           const chunkCount = dataView.getInt32(curr, true)
           curr += 4
           const chunks = new Array<Chunk>(chunkCount)
@@ -175,6 +186,7 @@ export default class CSI extends IndexFile {
       clampChunkEnds(Object.values(binIndex).flat())
       return {
         binIndex,
+        loffsets,
         stats: pseudoBinStats,
       }
     }
@@ -188,12 +200,27 @@ export default class CSI extends IndexFile {
     }
   }
 
-  // CSI has no linear index — every refId starts from the beginning of file.
-  protected getLowestChunk() {
-    return ZERO_OFFSET
+  /**
+   * CSI has no linear index, but each bin's `loffset` is the linear-index entry
+   * at the bin's first window (htslib's `update_loff`), so the finest bin at or
+   * left of `min` bounds the query from below the way BAI's linear index does.
+   * Walks left through siblings and then up to the parent, as `hts_itr_query`
+   * does for CSI.
+   */
+  protected getLowestChunk(refIndex: CsiRefIndex, min: number) {
+    const { loffsets } = refIndex
+    const leaves = 8 ** this.depth
+    let bin =
+      (leaves - 1) / 7 +
+      Math.min(Math.floor(min / 2 ** this.minShift), leaves - 1)
+    while (bin > 0 && !loffsets.has(bin)) {
+      const parent = Math.floor((bin - 1) / 8)
+      bin = bin > parent * 8 + 1 ? bin - 1 : parent
+    }
+    return loffsets.get(bin) ?? ZERO_OFFSET
   }
 
-  // ...and so there is nothing to bound the far end of a query with either, so
+  // No linear index means nothing to forecast the far end of a query with, so
   // estimatedBytesForRegions keeps summing every chunk on a CSI-indexed file.
   protected getHighestChunk() {
     return undefined

package/src/indexFile.ts

@@ -2,6 +2,7 @@ import { SharedReadCache } from '@gmod/shared-read-cache'
 import QuickLRU from '@jbrowse/quick-lru'
 
 import { chunksLikelyRead, optimizeChunks } from './util.ts'
+import { compareOffsets } from './virtualOffset.ts'
 
 import type Chunk from './chunk.ts'
 import type { BaseOpts } from './util.ts'
@@ -48,6 +49,58 @@ export function memoizeByRefId<T>(
   }
 }
 
+/**
+ * The virtual offset past which a coordinate-sorted file holds nothing
+ * overlapping `[.., end)`, from the binning index alone: htslib's `max_off`
+ * (`hts_itr_query` in hts.c).
+ *
+ * Walk right from the finest bin after the one holding `end - 1`, stepping up
+ * to the parent at every first child, so each bin visited begins at or past
+ * `end` and never overlaps the query. Every record in such a bin starts at or
+ * past `end`, so the first chunk of the first bin that exists is a record past
+ * the query, and in a sorted file so is every record after it.
+ *
+ * A bound, unlike the linear-index forecast in `chunksLikelyRead`: it rests on
+ * the same sort order `appendInRange` and the early stop already assume, and
+ * cannot drop a record they would keep. See ADR 0023 for why the caller drops
+ * whole merged chunks with it rather than trimming them.
+ */
+export function maxOffset(
+  binIndex: Record<number, Chunk[]>,
+  end: number,
+  minShift: number,
+  depth: number,
+) {
+  if (end > 2 ** (minShift + depth * 3)) {
+    return undefined
+  }
+  const binCount = (8 ** (depth + 1) - 1) / 7
+  let bin = (8 ** depth - 1) / 7 + Math.floor((end - 1) / 2 ** minShift) + 1
+  if (bin >= binCount) {
+    bin = 0
+  }
+  for (;;) {
+    while (bin % 8 === 1) {
+      bin = (bin - 1) / 8
+    }
+    if (bin === 0) {
+      return undefined
+    }
+    const chunks = binIndex[bin]
+    if (chunks?.length) {
+      let lowest = chunks[0]!.minv
+      for (let i = 1; i < chunks.length; i++) {
+        const minv = chunks[i]!.minv
+        if (compareOffsets(minv, lowest) < 0) {
+          lowest = minv
+        }
+      }
+      return lowest
+    }
+    bin++
+  }
+}
+
 export default abstract class IndexFile<
   TParsed extends ParsedIndexBase = ParsedIndexBase,
 > {
@@ -79,6 +132,11 @@ export default abstract class IndexFile<
     end?: number,
   ): Promise<{ start: number; end: number; score: number }[]>
 
+  // The binning scheme: the finest bins are 2^minShift wide, and there are
+  // depth levels below bin 0. BAI is CSI with minShift 14 and depth 5.
+  protected abstract minShift: number
+  protected abstract depth: number
+
   // Bin numbers that overlap [min, max). Subclasses implement BAI's fixed
   // 5-level scheme or CSI's configurable scheme (SAMv1.pdf §5.1.1, CSIv1.tex §2).
   protected abstract reg2bins(
@@ -87,7 +145,8 @@ export default abstract class IndexFile<
   ): readonly (readonly [number, number])[]
 
   // Lower-bound virtual offset for chunks that could contain alignments in
-  // [min, ...). BAI uses its linear index; CSI has none and returns 0:0.
+  // [min, ...). BAI uses its linear index, CSI the loffset of a bin at or left
+  // of min.
   protected abstract getLowestChunk(
     refIndex: RefIndex,
     min: number,
@@ -133,7 +192,16 @@ export default abstract class IndexFile<
         }
       }
     }
-    return optimizeChunks(chunks, this.getLowestChunk(ba, min))
+    const merged = optimizeChunks(chunks, this.getLowestChunk(ba, min))
+    const past = maxOffset(binIndex, max, this.minShift, this.depth)
+    if (past) {
+      let n = merged.length
+      while (n > 0 && compareOffsets(merged[n - 1]!.minv, past) >= 0) {
+        n--
+      }
+      merged.length = n
+    }
+    return merged
   }
 
   // SYNC: ~/src/gmod/tabix-js/src/indexFile.ts parse — same shape and the same
@@ -197,7 +265,8 @@ export default abstract class IndexFile<
    * exists — was being told 5.6x the truth on exactly the windows a reader
    * spends their time in, and cannot answer it by zooming: every window narrower
    * than a linear-index interval resolves to the same chunks and so to the same
-   * number.
+   * number. The table predates `max_off` (ADR 0023), which now drops most of
+   * the gap between the first two columns from `blocksForRange` itself.
    *
    * Still summed over merged chunks rather than per region, so two regions
    * sharing a chunk are charged for it once.

package/src/record.ts

@@ -440,16 +440,18 @@ export default class BamRecord {
   }
 
   // QUAL is present whenever the record has bases — independent of the unmapped
-  // flag (unmapped reads routinely carry SEQ/QUAL). A zero-length SEQ means
-  // there is no quality to return.
+  // flag (unmapped reads routinely carry SEQ/QUAL). null for a zero-length SEQ,
+  // and for a QUAL of `*`, which BAM stores as 0xFF in every byte (SAMv1
+  // §4.2.3). htslib decides on the first byte alone, and so does this.
   get qual() {
     const seqLen = this.seq_length
     if (seqLen === 0) {
       return null
-    } else {
-      const p = this.seqStart + ((seqLen + 1) >> 1)
-      return this._byteArray.subarray(p, p + seqLen)
     }
+    const p = this.seqStart + ((seqLen + 1) >> 1)
+    return this._byteArray[p] === 0xff
+      ? null
+      : this._byteArray.subarray(p, p + seqLen)
   }
 
   get strand() {
@@ -475,7 +477,7 @@ export default class BamRecord {
     return this.seqStart + ((seqLen + 1) >> 1) + seqLen
   }
 
-  // batch fromCharCode: fastest for typical name lengths (see benchmarks/string-building.bench.ts)
+  // batch fromCharCode: fastest for typical name lengths (benchmarks/string-building.bench.ts, removed in 36a3968)
   //
   // Deliberately NOT memoized, unlike end/tags/length_on_ref. Consumers read a
   // read name about once — jbrowse-components' buildBaseFeatureData copies it
@@ -510,8 +512,8 @@ export default class BamRecord {
   }
 
   getTag(tagName: string) {
-    if (this._cachedTags !== undefined) {
-      return this._cachedTags[tagName]
+    if (this._cachedTags !== undefined || tagName === 'CG') {
+      return this.tags[tagName]
     }
     return this._findTag(tagName, false)
   }
@@ -621,6 +623,9 @@ export default class BamRecord {
       )
       p = end
     }
+    if (this._hasCGPlaceholder() && isNumericCigar(tags.CG)) {
+      delete tags.CG
+    }
     return tags
   }
 
@@ -672,7 +677,7 @@ export default class BamRecord {
     return !!(this.flags & Constants.BAM_FSUPPLEMENTARY)
   }
 
-  // Benchmark results for CIGAR parsing strategies (see benchmarks/cigar-lifecycle.bench.ts):
+  // Benchmark results for CIGAR parsing strategies (benchmarks/cigar-strategies.bench.ts, removed in 36a3968):
   //
   // Aligned data:
   //   - Plain array is 1.6-1.8x faster than Uint32Array for small CIGARs (≤50 ops)
@@ -696,12 +701,23 @@ export default class BamRecord {
     // htslib stores the placeholder as exactly two ops: <seqlen>S<reflen>N.
     if (numCigarOps === 2) {
       const cigop = this._dataView.getInt32(p, true)
-      return (cigop & 0xf) === CIGAR_SOFT_CLIP && cigop >> 4 === this.seq_length
+      return (
+        (cigop & 0xf) === CIGAR_SOFT_CLIP && cigop >>> 4 === this.seq_length
+      )
     } else {
       return false
     }
   }
 
+  // SAMv1 §4.2.2: with the placeholder and a CG tag, the tag is the CIGAR and
+  // a reader removes it from the tags, as htslib does
+  private _hasCGPlaceholder() {
+    return this._isCGTagPattern(
+      this.b0 + this.read_name_length,
+      this.flag_nc & 0xffff,
+    )
+  }
+
   private _computeLengthOnRef(): number {
     const flag_nc = this._dataView.getInt32(this._start + 16, true)
     if (flag_nc & (Constants.BAM_FUNMAP << 16)) {
@@ -716,7 +732,7 @@ export default class BamRecord {
       if ((cigop2 & 0xf) !== CIGAR_REF_SKIP) {
         console.warn('CG tag with no N tag')
       }
-      return cigop2 >> 4
+      return cigop2 >>> 4
     }
 
     const absOffset = this._byteArray.byteOffset + p
@@ -732,7 +748,7 @@ export default class BamRecord {
       let lref = 0
       for (let c = 0; c < numCigarOps; ++c) {
         const co = cigarView[c]!
-        lref += (co >> 4) * ((CIGAR_CONSUMES_REF_MASK >> (co & 0xf)) & 1)
+        lref += (co >>> 4) * ((CIGAR_CONSUMES_REF_MASK >> (co & 0xf)) & 1)
       }
       return lref
     }
@@ -740,7 +756,7 @@ export default class BamRecord {
     let lref = 0
     for (let c = 0; c < numCigarOps; ++c) {
       const co = this._dataView.getInt32(p + c * 4, true)
-      lref += (co >> 4) * ((CIGAR_CONSUMES_REF_MASK >> (co & 0xf)) & 1)
+      lref += (co >>> 4) * ((CIGAR_CONSUMES_REF_MASK >> (co & 0xf)) & 1)
     }
     return lref
   }
@@ -761,10 +777,10 @@ export default class BamRecord {
     const p = this.b0 + this.read_name_length
 
     if (this._isCGTagPattern(p, numCigarOps)) {
-      // getTag, not this.tags: the real CIGAR lives in one tag, so there's no
-      // reason to decode every other tag on the record to reach it
-      const cg = this.getTag('CG')
-      return isNumericCigar(cg) ? cg : new Uint32Array(0)
+      const cg = this._findTag('CG', false)
+      if (isNumericCigar(cg)) {
+        return cg
+      }
     }
 
     const absOffset = this._byteArray.byteOffset + p
@@ -811,18 +827,21 @@ export default class BamRecord {
     let result = ''
     for (let i = 0, l = numeric.length; i < l; i++) {
       const packed = numeric[i]!
-      result += packed >> 4
+      result += packed >>> 4
       result += String.fromCharCode(ASCII_CIGAR_CODES[packed & 0xf]!)
     }
     return result
   }
 
   get num_cigar_ops() {
-    return this.flag_nc & 0xffff
+    return this._hasCGPlaceholder()
+      ? this.NUMERIC_CIGAR.length
+      : this.flag_nc & 0xffff
   }
 
+  // the stored CIGAR field, which for a long CIGAR is the two-op placeholder
   get num_cigar_bytes() {
-    return this.num_cigar_ops << 2
+    return (this.flag_nc & 0xffff) << 2
   }
 
   get read_name_length() {

package/src/streamBam.ts

@@ -9,7 +9,9 @@ import { parseHeaderText } from './sam.ts'
 import {
   BAM_MAGIC,
   concatUint8Array,
+  decodeHeaderText,
   parseRefSeqs,
+  readBlockSize,
   resolveFilehandle,
   throwIfAborted,
 } from './util.ts'
@@ -273,9 +275,7 @@ export async function* streamBamRecords<T extends BamRecordLike = BAMFeature>({
           recordCarry = bytes
           continue
         }
-        const headerText = new TextDecoder('utf8').decode(
-          bytes.subarray(8, 8 + lText),
-        )
+        const headerText = decodeHeaderText(bytes, lText)
         onHeader?.({
           headerText,
           samHeader: parseHeaderText(headerText),
@@ -288,7 +288,7 @@ export async function* streamBamRecords<T extends BamRecordLike = BAMFeature>({
 
       const sink: T[] = []
       while (blockStart + 4 <= bytes.length) {
-        const blockSize = dataView.getInt32(blockStart, true)
+        const blockSize = readBlockSize(dataView, blockStart)
         const blockEnd = blockStart + 4 + blockSize - 1
         if (blockEnd >= bytes.length) {
           break

package/src/util.ts

@@ -9,6 +9,22 @@ import type { GenericFilehandle } from 'generic-filehandle2'
 /** 'BAM\1' read as a little-endian int32 */
 export const BAM_MAGIC = 21840194
 
+/**
+ * The `block_size` of the record at `offset`, rejecting one too small to hold a
+ * record's 32 bytes of fixed fields, as htslib's `bam_read1` does. A negative
+ * one would otherwise leave the loops that advance by it standing still,
+ * pushing a record per turn until the process runs out of memory.
+ */
+export function readBlockSize(dataView: DataView, offset: number) {
+  const blockSize = dataView.getInt32(offset, true)
+  if (blockSize < 32) {
+    throw new Error(
+      `corrupt BAM record: block_size ${blockSize} at byte ${offset}`,
+    )
+  }
+  return blockSize
+}
+
 export function resolveFilehandle(
   filehandle?: GenericFilehandle,
   path?: string,
@@ -66,10 +82,13 @@ export const MAX_CONCURRENT_CHUNK_READS = 6
  *
  * `blocksForRange` returns every chunk of every bin overlapping the query, at
  * every level of the binning scheme, minus the ones the linear index puts
- * entirely before it. On a long-read file that is wildly more than the query
- * reads: a coarse bin's chunks run to the end of the bin's span, so a 380bp
- * window on a deep ONT BAM resolves to 90 chunks / 43.5MB, of which
- * `getRecordsForRange` reads 6 / 7.8MB before the early stop fires.
+ * entirely before it. Until `max_off` (ADR 0023) also dropped the ones past it,
+ * that was wildly more than a long-read query reads: a coarse bin's chunks run
+ * to the end of the bin's span, so a 380bp window on a deep ONT BAM resolved to
+ * 90 chunks / 43.5MB, of which `getRecordsForRange` read 6 / 7.8MB before the
+ * early stop fired. Since then this changes the forecast in 4 of 320 fixture
+ * windows, undershooting in all 4; ADR 0023 says what to check before
+ * removing it.
  *
  * Two bounds, and the answer is the larger:
  *
@@ -325,6 +344,16 @@ export function clampChunkEnds(
   }
 }
 
+// SAMv1 §4.2 counts NUL padding in l_text, and htslib reads the text as a C
+// string, so the header ends at the first NUL.
+export function decodeHeaderText(bytes: Uint8Array, lText: number) {
+  const text = bytes.subarray(8, 8 + lText)
+  const nul = text.indexOf(0)
+  return new TextDecoder('utf8').decode(
+    nul === -1 ? text : text.subarray(0, nul),
+  )
+}
+
 // Parse the BAM reference-sequence table (SAMv1.pdf §4.2). Returns undefined
 // if `uncba` doesn't yet contain the full table — caller fetches more bytes
 // and retries.

package/src/virtualOffset.ts

@@ -35,3 +35,7 @@ export function fromBytes(bytes: Uint8Array, offset = 0) {
     (bytes[offset + 1]! << 8) | bytes[offset]!,
   )
 }
+
+export function compareOffsets(a: OffsetCoords, b: OffsetCoords) {
+  return a.blockPosition - b.blockPosition || a.dataPosition - b.dataPosition
+}