@gmod/bam 7.8.2 → 7.9.0

package/src/record.ts

@@ -31,6 +31,41 @@ for (let hi = 0; hi < 16; hi++) {
 // read lengths.
 const SEQ_DECODER_THRESHOLD = 300
 
+// Four bases per entry, indexed by a PAIR of SEQ bytes, so the sub-threshold
+// path halves its concat count: 1.5-1.6x on a short-read query end to end
+// (shortreads_300x 46.5 -> 30.5 ms over 53.6k reads, volvox 4.8 -> 2.9 ms).
+//
+// Built lazily, and only once the short path has been taken enough times to pay
+// for it. Filling 65536 entries costs ~6 ms and retains ~2 MB, so building on
+// first use is a LOSS on a file that decodes only a handful of short reads:
+// long-read fixtures with a short-read tail measured 1.4x slower that way
+// (ecoli_nanopore has 27 sub-300bp reads out of 480, chm1 has 5 of 204). The
+// counter is module-global on purpose — the table is shared, so what has to
+// amortize is the total number of short decodes in the process, not per file.
+const SEQRET_QUAD_WARMUP = 1024
+let seqretShortCalls = 0
+let SEQRET_QUAD_STRINGS: string[] | undefined
+
+// The 4-base table, or undefined while still warming up (caller falls back to
+// the 2-base table).
+function seqretQuads() {
+  if (SEQRET_QUAD_STRINGS === undefined) {
+    if (++seqretShortCalls < SEQRET_QUAD_WARMUP) {
+      return undefined
+    }
+    const quads = new Array<string>(65536)
+    for (let a = 0; a < 256; a++) {
+      const sa = SEQRET_PAIR_STRINGS[a]!
+      const base = a << 8
+      for (let b = 0; b < 256; b++) {
+        quads[base | b] = sa + SEQRET_PAIR_STRINGS[b]!
+      }
+    }
+    SEQRET_QUAD_STRINGS = quads
+  }
+  return SEQRET_QUAD_STRINGS
+}
+
 // Precomputed pair orientation strings, indexed by
 //   ((flags >> 4) & 0x7) | (selfIsLeft ? 8 : 0)
 // bits 0-2 are flag bits 0x10 (self reverse), 0x20 (mate reverse), 0x40 (read1);
@@ -115,12 +150,6 @@ function isNumericCigar(value: unknown): value is NumericCigar {
   )
 }
 
-export interface Bytes {
-  start: number
-  end: number
-  byteArray: Uint8Array
-}
-
 type BArrayValue =
   | Int8Array
   | Uint8Array
@@ -213,13 +242,20 @@ function decodeBArrayTag(
 }
 
 // Byte span of a 'B' tag's element payload, for advancing the cursor past it.
+// Returns -1 for a subtype outside the spec's cCsSiIf, whose element width is
+// unknowable. Guessing one byte per element (which is what falling through to
+// `limit` did) doesn't skip the tag, it lands the cursor mid-value and decodes
+// the whole rest of the record's tag list out of garbage — where an unknown
+// top-level type has always stopped the walk instead. Same choice here.
 function bArrayByteLength(Btype: number, limit: number) {
   if (Btype === 0x69 || Btype === 0x49 || Btype === 0x66) {
     return limit << 2
   } else if (Btype === 0x73 || Btype === 0x53) {
     return limit << 1
-  } else {
+  } else if (Btype === 0x63 || Btype === 0x43) {
     return limit
+  } else {
+    return -1
   }
 }
 
@@ -253,6 +289,11 @@ function tagValueEnd(
     case 0x5a: // 'Z'
     case 0x48: {
       // 'H'
+      // Stays a plain byte loop. Swapping in Uint8Array.indexOf past a short
+      // inline probe is 2.7x faster on long-read MD (mean 9083 bytes on
+      // jb2bench's 200x.longread) but ~1.13x slower on short-read Z values,
+      // which are 4-13 bytes and are what the dominant case is made of. Measured
+      // both ways against the realistic corpus — see ADR 0012.
       let q = p
       while (q < blockEnd && ba[q] !== 0) {
         q++
@@ -263,7 +304,12 @@ function tagValueEnd(
       // 'B'
       const Btype = ba[p]!
       const limit = dataView.getInt32(p + 1, true)
-      return p + 5 + bArrayByteLength(Btype, limit)
+      const payload = bArrayByteLength(Btype, limit)
+      if (payload < 0) {
+        console.error('Unknown BAM B tag subtype', Btype)
+        return 0
+      }
+      return p + 5 + payload
     }
     default:
       console.error('Unknown BAM tag type', type)
@@ -462,6 +508,60 @@ export default class BamRecord {
     return this._findTag(tagName, true)
   }
 
+  /**
+   * The value of `tagName`, or of `altName` if the record carries no `tagName` —
+   * resolved in ONE pass over the tag block instead of two.
+   *
+   * For the MM/Mm and ML/Ml alias pairs that modified-base callers emit, where
+   * `getTag(a) ?? getTag(b)` walks every tag on the record TWICE whenever
+   * neither is present, which is every read in a file without base
+   * modifications. jbrowse-components issues exactly that lookup per record on
+   * every render (extractModifications runs unconditionally), and on
+   * jb2bench's 1000x.shortread it was 12.9% of the whole query — more than the
+   * CIGAR, SEQ and MD reads the pileup actually uses, spent proving absence.
+   *
+   * `tagName` wins wherever it appears, so the result matches the two-lookup
+   * form even for a (malformed) record carrying both.
+   */
+  getTagAlt(tagName: string, altName: string) {
+    if (this._cachedTags !== undefined) {
+      return this._cachedTags[tagName] ?? this._cachedTags[altName]
+    }
+    const a0 = tagName.charCodeAt(0)
+    const a1 = tagName.charCodeAt(1)
+    const b0 = altName.charCodeAt(0)
+    const b1 = altName.charCodeAt(1)
+    const blockEnd = this._end
+    const ba = this._byteArray
+    let p = this.tagsStart
+    // where the alternate landed, if it turns up before the primary does
+    let altType = -1
+    let altStart = 0
+    let altEnd = 0
+    while (p < blockEnd) {
+      const c0 = ba[p]
+      const c1 = ba[p + 1]
+      const type = ba[p + 2]!
+      const valueStart = p + 3
+      const end = tagValueEnd(ba, this._dataView, type, valueStart, blockEnd)
+      if (end === 0) {
+        break // unknown type: can't compute how far to advance
+      }
+      if (c0 === a0 && c1 === a1) {
+        return decodeTagValue(ba, this._dataView, type, valueStart, end, false)
+      }
+      if (altType < 0 && c0 === b0 && c1 === b1) {
+        altType = type
+        altStart = valueStart
+        altEnd = end
+      }
+      p = end
+    }
+    return altType < 0
+      ? undefined
+      : decodeTagValue(ba, this._dataView, altType, altStart, altEnd, false)
+  }
+
   private _findTag(tagName: string, raw: boolean) {
     const tag1 = tagName.charCodeAt(0)
     const tag2 = tagName.charCodeAt(1)
@@ -633,12 +733,18 @@ export default class BamRecord {
     return lref
   }
 
+  // Unmapped records are not special-cased here, unlike in
+  // _computeLengthOnRef. The spec says no assumptions can be made about an
+  // unmapped read's CIGAR, but "no assumptions" is not "no CIGAR": aligners
+  // emit placed unmapped mates that carry a real one, and htslib prints
+  // whatever is stored. Dropping it lost data the file had — paired.bam's
+  // SRR062635.1831187 at 20:74230 is FLAG 133 with 35M65S.
+  //
+  // Reference span stays 0 for them regardless, since that is a claim about
+  // alignment rather than about the stored bytes, and the query filter reads
+  // it.
   private _computeNumericCigar(): NumericCigar {
     const flag_nc = this._dataView.getInt32(this._start + 16, true)
-    if (flag_nc & (Constants.BAM_FUNMAP << 16)) {
-      return new Uint32Array(0)
-    }
-
     const numCigarOps = flag_nc & 0xffff
     const p = this.b0 + this.read_name_length
 
@@ -675,14 +781,26 @@ export default class BamRecord {
     return this._cachedNumericCigar
   }
 
+  // Two appends per op, NOT `result += length + String.fromCharCode(op)`. The
+  // one-append form builds an intermediate cons string per op just to append it
+  // and drop it; appending each piece straight onto the rope is 1.23-1.35x
+  // faster on long reads, which is where this accessor dominates
+  // (chr22_nanopore_subset 51.2 -> 38.6 ms for 757 reads averaging 2171 ops,
+  // ultra-long-ont 13.2 -> 9.9 ms). Short reads carry 1-3 ops and land inside
+  // this box's noise band either way.
+  //
+  // ADR 0003 rejected two other rewrites of this loop — a precomputed 16-entry
+  // op-char table, and digits into a Uint8Array with one TextDecoder.decode —
+  // and both are still losers. Re-measured here: the op-char table is 1.13x
+  // SLOWER than String.fromCharCode even on top of this change, because V8
+  // already hands back an interned single-character string.
   get CIGAR() {
     const numeric = this.NUMERIC_CIGAR
     let result = ''
     for (let i = 0, l = numeric.length; i < l; i++) {
       const packed = numeric[i]!
-      const length = packed >> 4
-      const opCode = ASCII_CIGAR_CODES[packed & 0xf]!
-      result += length + String.fromCharCode(opCode)
+      result += packed >> 4
+      result += String.fromCharCode(ASCII_CIGAR_CODES[packed & 0xf]!)
     }
     return result
   }
@@ -708,9 +826,10 @@ export default class BamRecord {
     return this._byteArray.subarray(p, p + this.num_seq_bytes)
   }
 
-  // Decode two bases per iteration off a 256-entry table. Building an array of
-  // 1-char strings and join()ing it — the obvious approach — is 3x slower at
-  // 100bp and 35x slower at 15kb.
+  // Decode four bases per iteration off the 65536-entry table (two off the
+  // 256-entry one until that table has warmed up). Building an array of 1-char
+  // strings and join()ing it — the obvious approach — is 3x slower at 100bp and
+  // 35x slower at 15kb.
   get seq() {
     const len = this.seq_length
     const ba = this._byteArray
@@ -718,9 +837,22 @@ export default class BamRecord {
     const nPairs = len >> 1
     let seq: string
     if (len < SEQ_DECODER_THRESHOLD) {
+      const quads = seqretQuads()
       seq = ''
-      for (let j = 0; j < nPairs; j++) {
-        seq += SEQRET_PAIR_STRINGS[ba[p + j]!]!
+      if (quads === undefined) {
+        for (let j = 0; j < nPairs; j++) {
+          seq += SEQRET_PAIR_STRINGS[ba[p + j]!]!
+        }
+      } else {
+        const nQuads = nPairs >> 1
+        for (let j = 0; j < nQuads; j++) {
+          const q = p + 2 * j
+          seq += quads[(ba[q]! << 8) | ba[q + 1]!]!
+        }
+        // odd byte count: two more bases off the 2-base table
+        if (nPairs & 1) {
+          seq += SEQRET_PAIR_STRINGS[ba[p + nPairs - 1]!]!
+        }
       }
       if (len & 1) {
         seq += SEQRET_DECODER[(ba[p + nPairs]! & 0xf0) >> 4]!

package/src/util.ts

@@ -29,6 +29,16 @@ export interface BaseOpts {
   onProgress?: (bytesDownloaded: number, totalBytes?: number) => void
 }
 
+/**
+ * Merge and order the chunks a query resolved to.
+ *
+ * Takes ownership of `chunks`: with no `lowest` to pre-filter against it sorts
+ * the array IN PLACE rather than copying. Every caller builds a fresh array to
+ * hand over, which is what makes that safe — passing something you still hold,
+ * or anything reachable from the index's per-refId cache, would reorder it
+ * underneath you. (The Chunk objects themselves are never mutated; a merged
+ * span produces a new instance.)
+ */
 export function optimizeChunks(chunks: Chunk[], lowest?: OffsetCoords) {
   const n = chunks.length
   if (n === 0) {
@@ -185,19 +195,14 @@ export function clampChunkEnds(
   if (chunks.length === 0) {
     return
   }
-  // A plain array, sized once and filled by index. Not a Float64Array: this
-  // runs once per reference, and an assembly with tens of thousands of unplaced
-  // scaffolds (cho.bam.bai has 28751 references) pays the allocation that many
-  // times over, where each individual boundary list is a handful of entries.
-  // The small-allocation cost dominates the faster typed sort by a wide margin
-  // at that shape. Filling by index still avoids the intermediate array that
-  // mapping the linear index to block positions used to build.
-  // Pre-sized and filled by index. Measured against both alternatives on five
-  // real .bai shapes (min of 21, interleaved, sign-stable over 3 runs):
-  // building with push instead is 5-8% slower here, and a Float64Array is
-  // faster to sort but allocates per reference, which costs 1.7x on an
-  // assembly with 28751 scaffolds. Filling by index also avoids the
-  // intermediate array that mapping the linear index used to build.
+  // A plain array, pre-sized and filled by index. Measured against both
+  // alternatives on five real .bai shapes (min of 21, interleaved, sign-stable
+  // over 3 runs): building with push instead is 5-8% slower, and a
+  // Float64Array is faster to sort but allocates per reference, which costs
+  // 1.7x on an assembly with tens of thousands of unplaced scaffolds
+  // (cho.bam.bai has 28751 references, each with a handful of boundaries).
+  // Filling by index also avoids the intermediate array that mapping the
+  // linear index to block positions used to build.
   const boundaries = new Array<number>(
     extraBoundaries.length + chunks.length * 2,
   )