@gmod/bam 7.3.3 → 7.4.0

package/src/record.ts

@@ -1,16 +1,45 @@
 import { CIGAR_REF_SKIP, CIGAR_SOFT_CLIP } from './cigar.ts'
 import Constants from './constants.ts'
 
-const SEQRET_DECODER = '=ACMGRSVTWYHKDBN'.split('')
+const SEQRET = '=ACMGRSVTWYHKDBN'
+const SEQRET_DECODER = SEQRET.split('')
+const SEQRET_CODES = Uint8Array.from(SEQRET, c => c.charCodeAt(0))
+
+// Both bases of a SEQ byte, precomputed for all 256 bytes so decoding advances a
+// byte at a time. Two forms because `seq` has two strategies (see below): packed
+// ASCII codes for a single Uint16Array store, and the 2-char string to append.
+// Byte order within the u16 depends on host endianness.
+const LITTLE_ENDIAN = new Uint16Array(Uint8Array.of(1, 0).buffer)[0] === 1
+const SEQRET_PAIR_CODES = new Uint16Array(256)
+const SEQRET_PAIR_STRINGS = new Array<string>(256)
+for (let hi = 0; hi < 16; hi++) {
+  for (let lo = 0; lo < 16; lo++) {
+    const h = SEQRET_CODES[hi]!
+    const l = SEQRET_CODES[lo]!
+    SEQRET_PAIR_CODES[(hi << 4) | lo] = LITTLE_ENDIAN
+      ? h | (l << 8)
+      : (h << 8) | l
+    SEQRET_PAIR_STRINGS[(hi << 4) | lo] =
+      SEQRET_DECODER[hi]! + SEQRET_DECODER[lo]!
+  }
+}
 
-// precomputed pair orientation strings indexed by ((flags >> 4) & 0xF) | (isize > 0 ? 16 : 0)
-// bits 0-3 encode flag bits 0x10(reverse),0x20(mate reverse),0x40(read1),0x80(read2)
-// bit 4 encodes whether isize > 0
+// Read length at which building a byte buffer and calling TextDecoder once
+// overtakes plain string concatenation. Below it concat wins by 2-4x (rope
+// building is cheap and the decode has fixed overhead); above it the decoder
+// wins by 5-8x. Measured crossover is ~300bp, i.e. just past typical Illumina
+// read lengths.
+const SEQ_DECODER_THRESHOLD = 300
+
+// Precomputed pair orientation strings, indexed by
+//   ((flags >> 4) & 0x7) | (selfIsLeft ? 8 : 0)
+// bits 0-2 are flag bits 0x10 (self reverse), 0x20 (mate reverse), 0x40 (read1);
+// bit 3 is whether this read is the leftmost of the pair. The read2 flag (0x80)
+// is deliberately not consulted — "not read1" is what decides the numbering, so
+// a record with neither flag set reads the same as read2.
 // prettier-ignore
 const PAIR_ORIENTATION_TABLE = [
-  'F F ','F R ','R F ','R R ','F2F1','F2R1','R2F1','R2R1',
   'F1F2','F1R2','R1F2','R1R2','F2F1','F2R1','R2F1','R2R1',
-  'F F ','R F ','F R ','R R ','F1F2','R1F2','F1R2','R1R2',
   'F2F1','R2F1','F2R1','R2R1','F1F2','R1F2','F1R2','R1R2',
 ]
 const ASCII_CIGAR_CODES = [
@@ -23,6 +52,19 @@ const textDecoder = new TextDecoder()
 // Binary: 0b111001101 = 0x1CD
 const CIGAR_CONSUMES_REF_MASK = 0x1cd
 
+// A CIGAR as packed op words. Either a view over (or copy of) the record's own
+// CIGAR field, or the CG tag's array for long-CIGAR records — hence Int32Array
+// too, since a writer may encode CG as B:i rather than the usual B:I.
+export type NumericCigar = Uint32Array | Int32Array | number[]
+
+function isNumericCigar(value: unknown): value is NumericCigar {
+  return (
+    value instanceof Uint32Array ||
+    value instanceof Int32Array ||
+    Array.isArray(value)
+  )
+}
+
 export interface Bytes {
   start: number
   end: number
@@ -42,7 +84,11 @@ type BArrayValue =
 // Decode a 'B' (array) tag value starting at `p` (the byte after type+subtype+
 // count). When the data is naturally aligned we return a typed-array view over
 // the underlying buffer (zero-copy); otherwise we copy element-by-element since
-// typed-array views require alignment. Shared by getTag and the full-tag parse.
+// typed-array views require alignment. The copy target is a plain array, not a
+// typed one: benchmarking the unaligned path found number[] fills faster below
+// ~10k elements and reads at least as fast at every size, so the only thing a
+// typed copy would buy is half the retained bytes. Shared by getTag and the
+// full-tag parse.
 function decodeBArrayTag(
   ba: Uint8Array,
   dataView: DataView,
@@ -127,6 +173,97 @@ function bArrayByteLength(Btype: number, limit: number) {
   }
 }
 
+// Cursor position just past a tag's value — i.e. the start of the next tag —
+// given the value starts at `p`. For null-terminated Z/H this steps over the
+// terminator. Returns 0 for an unknown type, whose value width is unknowable so
+// the caller must stop. Shared by _findTag and _computeTags so the two loops
+// can't drift in how they walk the tag layout. (0 is an impossible real end:
+// `p` is always past the fixed record prefix.)
+function tagValueEnd(
+  ba: Uint8Array,
+  dataView: DataView,
+  type: number,
+  p: number,
+  blockEnd: number,
+) {
+  switch (type) {
+    case 0x41: // 'A'
+    case 0x63: // 'c'
+    case 0x43: // 'C'
+      return p + 1
+    case 0x73: // 's'
+    case 0x53: // 'S'
+      return p + 2
+    case 0x69: // 'i'
+    case 0x49: // 'I'
+    case 0x66: {
+      // 'f'
+      return p + 4
+    }
+    case 0x5a: // 'Z'
+    case 0x48: {
+      // 'H'
+      let q = p
+      while (q < blockEnd && ba[q] !== 0) {
+        q++
+      }
+      return q + 1 // step past the null terminator
+    }
+    case 0x42: {
+      // 'B'
+      const Btype = ba[p]!
+      const limit = dataView.getInt32(p + 1, true)
+      return p + 5 + bArrayByteLength(Btype, limit)
+    }
+    default:
+      console.error('Unknown BAM tag type', type)
+      return 0
+  }
+}
+
+// Decode the value of a tag of `type` whose bytes start at `p`; `end` is the
+// next-tag cursor from tagValueEnd (used only to bound Z/H strings, whose bytes
+// run up to the null terminator at end-1). When `raw`, Z/H return the undecoded
+// byte subarray. Assumes `type` is a known scalar/string/B type.
+function decodeTagValue(
+  ba: Uint8Array,
+  dataView: DataView,
+  type: number,
+  p: number,
+  end: number,
+  raw: boolean,
+): unknown {
+  switch (type) {
+    case 0x41: // 'A'
+      return String.fromCharCode(ba[p]!)
+    case 0x69: // 'i'
+      return dataView.getInt32(p, true)
+    case 0x49: // 'I'
+      return dataView.getUint32(p, true)
+    case 0x63: // 'c'
+      return dataView.getInt8(p)
+    case 0x43: // 'C'
+      return dataView.getUint8(p)
+    case 0x73: // 's'
+      return dataView.getInt16(p, true)
+    case 0x53: // 'S'
+      return dataView.getUint16(p, true)
+    case 0x66: // 'f'
+      return dataView.getFloat32(p, true)
+    case 0x5a: // 'Z'
+    case 0x48: // 'H'
+      return raw
+        ? ba.subarray(p, end - 1)
+        : textDecoder.decode(ba.subarray(p, end - 1))
+    default: {
+      // 'B'
+      const Btype = ba[p]!
+      const limit = dataView.getInt32(p + 1, true)
+      return decodeBArrayTag(ba, dataView, p + 5, Btype, limit)
+    }
+  }
+}
+
 export default class BamRecord {
   public fileOffset: number
   private _byteArray: Uint8Array
@@ -137,7 +274,7 @@ export default class BamRecord {
   private _cachedEnd?: number
   private _cachedTags?: Record<string, unknown>
   private _cachedLengthOnRef?: number
-  private _cachedNumericCigar?: Uint32Array | number[]
+  private _cachedNumericCigar?: NumericCigar
   private _cachedNUMERIC_MD?: Uint8Array | null
   private _cachedSeqStart?: number
 
@@ -261,172 +398,49 @@ export default class BamRecord {
   private _findTag(tagName: string, raw: boolean) {
     const tag1 = tagName.charCodeAt(0)
     const tag2 = tagName.charCodeAt(1)
-
-    let p = this.tagsStart
-
     const blockEnd = this._end
     const ba = this._byteArray
+    let p = this.tagsStart
     while (p < blockEnd) {
-      const currentTag1 = ba[p]
-      const currentTag2 = ba[p + 1]
-      const type = ba[p + 2]
-      p += 3
-
-      const isMatch = currentTag1 === tag1 && currentTag2 === tag2
-
-      switch (type) {
-        case 0x41: // 'A'
-          if (isMatch) {
-            return String.fromCharCode(ba[p]!)
-          }
-          p += 1
-          break
-        case 0x69: // 'i'
-          if (isMatch) {
-            return this._dataView.getInt32(p, true)
-          }
-          p += 4
-          break
-        case 0x49: // 'I'
-          if (isMatch) {
-            return this._dataView.getUint32(p, true)
-          }
-          p += 4
-          break
-        case 0x63: // 'c'
-          if (isMatch) {
-            return this._dataView.getInt8(p)
-          }
-          p += 1
-          break
-        case 0x43: // 'C'
-          if (isMatch) {
-            return this._dataView.getUint8(p)
-          }
-          p += 1
-          break
-        case 0x73: // 's'
-          if (isMatch) {
-            return this._dataView.getInt16(p, true)
-          }
-          p += 2
-          break
-        case 0x53: // 'S'
-          if (isMatch) {
-            return this._dataView.getUint16(p, true)
-          }
-          p += 2
-          break
-        case 0x66: // 'f'
-          if (isMatch) {
-            return this._dataView.getFloat32(p, true)
-          }
-          p += 4
-          break
-        case 0x5a: // 'Z'
-        case 0x48: {
-          // 'H'
-          const start = p
-          while (p < blockEnd && ba[p] !== 0) {
-            p++
-          }
-          if (isMatch) {
-            return raw
-              ? ba.subarray(start, p)
-              : textDecoder.decode(ba.subarray(start, p))
-          }
-          p++ // advance past null terminator
-          break
-        }
-        case 0x42: {
-          // 'B'
-          const Btype = ba[p++]!
-          const limit = this._dataView.getInt32(p, true)
-          p += 4
-          if (isMatch) {
-            return decodeBArrayTag(ba, this._dataView, p, Btype, limit)
-          }
-          p += bArrayByteLength(Btype, limit)
-          break
-        }
-        default:
-          if (type !== undefined) {
-            console.error('Unknown BAM tag type', type)
-          }
-          break
+      const isMatch = ba[p] === tag1 && ba[p + 1] === tag2
+      const type = ba[p + 2]!
+      const valueStart = p + 3
+      const end = tagValueEnd(ba, this._dataView, type, valueStart, blockEnd)
+      if (end === 0) {
+        break // unknown type: can't compute how far to advance
+      }
+      if (isMatch) {
+        return decodeTagValue(ba, this._dataView, type, valueStart, end, raw)
       }
+      p = end
     }
     return undefined
   }
 
   private _computeTags() {
-    let p = this.tagsStart
-
     const blockEnd = this._end
     const ba = this._byteArray
-    const tags: Record<string, unknown> = {}
+    // null prototype: tag names come from the file, so a read carrying a
+    // "constructor" or "toString" tag must not resolve to Object.prototype's
+    const tags: Record<string, unknown> = Object.create(null)
+    let p = this.tagsStart
     while (p < blockEnd) {
       const tag = String.fromCharCode(ba[p]!, ba[p + 1]!)
       const type = ba[p + 2]!
-      p += 3
-
-      switch (type) {
-        case 0x41: // 'A'
-          tags[tag] = String.fromCharCode(ba[p]!)
-          p += 1
-          break
-        case 0x69: // 'i'
-          tags[tag] = this._dataView.getInt32(p, true)
-          p += 4
-          break
-        case 0x49: // 'I'
-          tags[tag] = this._dataView.getUint32(p, true)
-          p += 4
-          break
-        case 0x63: // 'c'
-          tags[tag] = this._dataView.getInt8(p)
-          p += 1
-          break
-        case 0x43: // 'C'
-          tags[tag] = this._dataView.getUint8(p)
-          p += 1
-          break
-        case 0x73: // 's'
-          tags[tag] = this._dataView.getInt16(p, true)
-          p += 2
-          break
-        case 0x53: // 'S'
-          tags[tag] = this._dataView.getUint16(p, true)
-          p += 2
-          break
-        case 0x66: // 'f'
-          tags[tag] = this._dataView.getFloat32(p, true)
-          p += 4
-          break
-        case 0x5a: // 'Z'
-        case 0x48: {
-          // 'H'
-          const start = p
-          while (p < blockEnd && ba[p] !== 0) {
-            p++
-          }
-          tags[tag] = textDecoder.decode(ba.subarray(start, p))
-          p++ // advance past null terminator
-          break
-        }
-        case 0x42: {
-          // 'B'
-          const Btype = ba[p++]!
-          const limit = this._dataView.getInt32(p, true)
-          p += 4
-          tags[tag] = decodeBArrayTag(ba, this._dataView, p, Btype, limit)
-          p += bArrayByteLength(Btype, limit)
-          break
-        }
-        default:
-          console.error('Unknown BAM tag type', type)
-          break
+      const valueStart = p + 3
+      const end = tagValueEnd(ba, this._dataView, type, valueStart, blockEnd)
+      if (end === 0) {
+        break // unknown type: can't compute how far to advance
       }
+      tags[tag] = decodeTagValue(
+        ba,
+        this._dataView,
+        type,
+        valueStart,
+        end,
+        false,
+      )
+      p = end
     }
     return tags
   }
@@ -533,6 +547,8 @@ export default class BamRecord {
         absOffset,
         numCigarOps,
       )
+      // the view we need to sum is exactly what NUMERIC_CIGAR would build, so
+      // seed its cache rather than making it construct a second one
       this._cachedNumericCigar = cigarView
       let lref = 0
       for (let c = 0; c < numCigarOps; ++c) {
@@ -550,7 +566,7 @@ export default class BamRecord {
     return lref
   }
 
-  private _computeNumericCigar(): Uint32Array | number[] {
+  private _computeNumericCigar(): NumericCigar {
     const flag_nc = this._dataView.getInt32(this._start + 16, true)
     if (flag_nc & (Constants.BAM_FUNMAP << 16)) {
       return new Uint32Array(0)
@@ -560,10 +576,10 @@ export default class BamRecord {
     const p = this.b0 + this.read_name_length
 
     if (this._isCGTagPattern(p, numCigarOps)) {
-      return (
-        (this.tags.CG as Uint32Array | number[] | undefined) ??
-        new Uint32Array(0)
-      )
+      // getTag, not this.tags: the real CIGAR lives in one tag, so there's no
+      // reason to decode every other tag on the record to reach it
+      const cg = this.getTag('CG')
+      return isNumericCigar(cg) ? cg : new Uint32Array(0)
     }
 
     const absOffset = this._byteArray.byteOffset + p
@@ -625,43 +641,66 @@ export default class BamRecord {
     return this._byteArray.subarray(p, p + this.num_seq_bytes)
   }
 
+  // Decode two bases per iteration off a 256-entry table. Building an array of
+  // 1-char strings and join()ing it — the obvious approach — is 3x slower at
+  // 100bp and 35x slower at 15kb.
   get seq() {
     const len = this.seq_length
-    const seqStart = this.seqStart
-    const numeric = this._byteArray
-    const buf = new Array(len)
-    let i = 0
-    const fullBytes = len >> 1
-
-    for (let j = 0; j < fullBytes; ++j) {
-      const sb = numeric[seqStart + j]!
-      buf[i++] = SEQRET_DECODER[(sb & 0xf0) >> 4]!
-      buf[i++] = SEQRET_DECODER[sb & 0x0f]!
-    }
-
-    if (i < len) {
-      const sb = numeric[seqStart + fullBytes]!
-      buf[i] = SEQRET_DECODER[(sb & 0xf0) >> 4]!
+    const ba = this._byteArray
+    const p = this.seqStart
+    const nPairs = len >> 1
+    let seq: string
+    if (len < SEQ_DECODER_THRESHOLD) {
+      seq = ''
+      for (let j = 0; j < nPairs; j++) {
+        seq += SEQRET_PAIR_STRINGS[ba[p + j]!]!
+      }
+      if (len & 1) {
+        seq += SEQRET_DECODER[(ba[p + nPairs]! & 0xf0) >> 4]!
+      }
+    } else {
+      // round up to an even length so the Uint16Array view spans every pair; a
+      // trailing odd base is written as a single byte and trimmed on decode
+      const out = new Uint8Array((len + 1) & ~1)
+      const pairs = new Uint16Array(out.buffer)
+      for (let j = 0; j < nPairs; j++) {
+        pairs[j] = SEQRET_PAIR_CODES[ba[p + j]!]!
+      }
+      if (len & 1) {
+        out[len - 1] = SEQRET_CODES[(ba[p + nPairs]! & 0xf0) >> 4]!
+      }
+      seq = textDecoder.decode(out.subarray(0, len))
     }
-
-    return buf.join('')
+    return seq
   }
 
-  // adapted from igv.js
-  // uses precomputed lookup table indexed by flag bits + isize sign.
-  // the BAM spec defines tlen as positive for the leftmost segment and
-  // negative for the rightmost, so tlen > 0 reliably indicates which
-  // read comes first without needing position-based correction
+  // Must come out identical from either mate, or the two halves of one normal
+  // pair render as different orientations. The leftmost mate is therefore picked
+  // by a total order on (refId, pos) that both mates evaluate the same way, with
+  // a read1-first tie-break for equal loci.
+  //
+  // Deriving "leftmost" from template_length looks tempting — the spec makes
+  // tlen positive for the leftmost segment and negative for the rightmost — but
+  // aligners leave tlen at 0 whenever the insert size is unavailable, which
+  // includes every cross-reference pair. Both mates then read as "not leftmost"
+  // and disagree with each other.
   // (see also: gmod/cram-js src/cramFile/record.ts getPairOrientation)
   get pair_orientation() {
     const f = this.flags
-    // unmapped (0x4) or mate unmapped (0x8) -> undefined
-    if (f & 0xc || this.ref_id !== this.next_refid) {
+    if (!(f & Constants.BAM_FPAIRED)) {
       return undefined
     }
-    return PAIR_ORIENTATION_TABLE[
-      ((f >> 4) & 0xf) | (this.template_length > 0 ? 16 : 0)
-    ]
+    const refId = this.ref_id
+    const mateRefId = this.next_refid
+    const pos = this.start
+    const matePos = this.next_pos
+    const selfIsLeft =
+      refId !== mateRefId
+        ? refId < mateRefId
+        : pos !== matePos
+          ? pos < matePos
+          : !!(f & Constants.BAM_FREAD1)
+    return PAIR_ORIENTATION_TABLE[((f >> 4) & 0x7) | (selfIsLeft ? 8 : 0)]
   }
 
   get bin_mq_nl() {
@@ -692,7 +731,7 @@ export default class BamRecord {
     if (idx < this.seq_length) {
       const sb = this._byteArray[this.seqStart + (idx >> 1)]!
 
-      return idx % 2 === 0
+      return (idx & 1) === 0
         ? SEQRET_DECODER[(sb & 0xf0) >> 4]!
         : SEQRET_DECODER[sb & 0x0f]!
     } else {

package/src/util.ts

@@ -126,7 +126,10 @@ export function parsePseudoBin(bytes: Uint8Array, offset: number) {
 // maxv.blockPosition is an upper bound on where that final block ends — always at
 // least the true block end, so the clamped fetch still contains the whole block.
 // Shrinks both the byte estimate and the actual fetch with no extra I/O.
-export function clampChunkEnds(chunks: Chunk[], extraBoundaries: number[] = []) {
+export function clampChunkEnds(
+  chunks: Chunk[],
+  extraBoundaries: number[] = [],
+) {
   const boundaries = [...extraBoundaries]
   for (const c of chunks) {
     boundaries.push(c.minv.blockPosition, c.maxv.blockPosition)
@@ -163,9 +166,15 @@ export function parseRefSeqs(
   if (start + 4 > uncba.length) {
     return undefined
   }
-  const dataView = new DataView(uncba.buffer)
+  const dataView = new DataView(
+    uncba.buffer,
+    uncba.byteOffset,
+    uncba.byteLength,
+  )
   const nRef = dataView.getInt32(start, true)
-  const chrToIndex: Record<string, number> = {}
+  // null prototype: ref names come from the file, so a contig named
+  // "constructor" must not resolve to Object.prototype's
+  const chrToIndex: Record<string, number> = Object.create(null)
   const indexToChr: { refName: string; length: number }[] = []
   const decoder = new TextDecoder('utf8')
 
@@ -209,7 +218,7 @@ export function parseNameBytes(
   let currRefId = 0
   let currNameStart = 0
   const refIdToName: string[] = []
-  const refNameToId: Record<string, number> = {}
+  const refNameToId: Record<string, number> = Object.create(null)
   for (let i = 0; i < namesBytes.length; i++) {
     if (!namesBytes[i]) {
       if (currNameStart < i) {
@@ -254,18 +263,19 @@ export function filterTagValue(readVal: unknown, filterVal?: string) {
     : `${readVal}` !== `${filterVal}`
 }
 
-export function filterCacheKey(filterBy?: FilterBy) {
-  if (!filterBy) {
-    return ''
-  }
-  const { flagInclude = 0, flagExclude = 0, tagFilter } = filterBy
-  const tagPart = tagFilter ? `:${tagFilter.tag}=${tagFilter.value ?? '*'}` : ''
-  return `:f${flagInclude}x${flagExclude}${tagPart}`
-}
-
 interface Filterable {
   flags: number
   tags: Record<string, unknown>
+  // BamRecord decodes one tag by walking the tag block, without building the
+  // whole tags object. Optional because a custom recordClass need not have it.
+  getTag?(tag: string): unknown
+}
+
+// Read a single tag, preferring the targeted accessor. Reaching for `tags`
+// instead would decode every unrelated tag on the record (NM/AS/ms/de/… — often
+// ~10 per read) just to test one, which measured 2.6x the cost of getTag.
+function readTag(record: Filterable, tag: string) {
+  return record.getTag ? record.getTag(tag) : record.tags[tag]
 }
 
 // Apply flagInclude/flagExclude/tagFilter to a list of records.
@@ -279,7 +289,7 @@ export function applyFilters<T extends Filterable>(
     const r = records[i]!
     if (
       !filterReadFlag(r.flags, flagInclude, flagExclude) &&
-      !(tagFilter && filterTagValue(r.tags[tagFilter.tag], tagFilter.value))
+      !(tagFilter && filterTagValue(readTag(r, tagFilter.tag), tagFilter.value))
     ) {
       out.push(r)
     }

package/src/virtualOffset.ts

@@ -23,11 +23,7 @@ export class VirtualOffset {
     )
   }
 }
-export function fromBytes(bytes: Uint8Array, offset = 0, bigendian = false) {
-  if (bigendian) {
-    throw new Error('big-endian virtual file offsets not implemented')
-  }
-
+export function fromBytes(bytes: Uint8Array, offset = 0) {
   return new VirtualOffset(
     bytes[offset + 7]! * 0x10000000000 +
       bytes[offset + 6]! * 0x100000000 +