@gmod/bam 9.0.1 → 10.0.1

package/src/streamBam.ts

@@ -9,7 +9,9 @@ import { parseHeaderText } from './sam.ts'
 import {
   BAM_MAGIC,
   concatUint8Array,
+  decodeHeaderText,
   parseRefSeqs,
+  readBlockSize,
   resolveFilehandle,
   throwIfAborted,
 } from './util.ts'
@@ -273,9 +275,7 @@ export async function* streamBamRecords<T extends BamRecordLike = BAMFeature>({
           recordCarry = bytes
           continue
         }
-        const headerText = new TextDecoder('utf8').decode(
-          bytes.subarray(8, 8 + lText),
-        )
+        const headerText = decodeHeaderText(bytes, lText)
         onHeader?.({
           headerText,
           samHeader: parseHeaderText(headerText),
@@ -288,7 +288,7 @@ export async function* streamBamRecords<T extends BamRecordLike = BAMFeature>({
 
       const sink: T[] = []
       while (blockStart + 4 <= bytes.length) {
-        const blockSize = dataView.getInt32(blockStart, true)
+        const blockSize = readBlockSize(dataView, blockStart)
         const blockEnd = blockStart + 4 + blockSize - 1
         if (blockEnd >= bytes.length) {
           break
@@ -305,12 +305,13 @@ export async function* streamBamRecords<T extends BamRecordLike = BAMFeature>({
             // the window size, so it is as stable as a virtual offset without
             // pretending to be one.
             //
-            // NOT the crc32 of the record bytes that readBamFeatures falls back
-            // to when it has no positions. That is a content hash, so the two
-            // byte-identical records in exact_duplicate.bam collide on it, and
-            // hashing every record costs ~40% of the walk (135ms of 340ms over
-            // out.bam) where a counter costs nothing. The fallback only fires on
-            // an unusual path there; here it would fire on every record.
+            // NOT the hash of the record bytes that readBamFeatures falls back
+            // to when it has no positions: the two byte-identical records in
+            // exact_duplicate.bam collide on any content hash, and hashing
+            // every record cost ~40% of the walk (135ms of 340ms over out.bam,
+            // measured with crc32) where a counter costs nothing. The fallback
+            // only fires on an unusual path there; here it would fire on every
+            // record.
             recordIndex++,
             dataView,
           ),

package/src/util.ts

@@ -9,6 +9,45 @@ import type { GenericFilehandle } from 'generic-filehandle2'
 /** 'BAM\1' read as a little-endian int32 */
 export const BAM_MAGIC = 21840194
 
+/**
+ * The `block_size` of the record at `offset`, rejecting one too small to hold a
+ * record's 32 bytes of fixed fields, as htslib's `bam_read1` does. A negative
+ * one would otherwise leave the loops that advance by it standing still,
+ * pushing a record per turn until the process runs out of memory.
+ */
+export function readBlockSize(dataView: DataView, offset: number) {
+  const blockSize = dataView.getInt32(offset, true)
+  if (blockSize < 32) {
+    throw new Error(
+      `corrupt BAM record: block_size ${blockSize} at byte ${offset}`,
+    )
+  }
+  return blockSize
+}
+
+/**
+ * A 53-bit hash of `bytes[start, end)`: cyrb53 (public domain), taken over
+ * bytes. The id of a record read with no file positions, which has to be the
+ * same for the same record in every query, so it hashes content. It is 53 bits
+ * rather than crc32's 32 because ids are deduplicated: at 32 bits a fetch of
+ * 150,000 records expects about 2.6 distinct pairs to collide, each silently
+ * dropping a read, where 53 bits expect about 1e-6.
+ */
+export function contentHash53(bytes: Uint8Array, start: number, end: number) {
+  let h1 = 0xdeadbeef
+  let h2 = 0x41c6ce57
+  for (let i = start; i < end; i++) {
+    const b = bytes[i]!
+    h1 = Math.imul(h1 ^ b, 2654435761)
+    h2 = Math.imul(h2 ^ b, 1597334677)
+  }
+  h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507)
+  h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909)
+  h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507)
+  h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909)
+  return 4294967296 * (2097151 & h2) + (h1 >>> 0)
+}
+
 export function resolveFilehandle(
   filehandle?: GenericFilehandle,
   path?: string,
@@ -66,10 +105,13 @@ export const MAX_CONCURRENT_CHUNK_READS = 6
  *
  * `blocksForRange` returns every chunk of every bin overlapping the query, at
  * every level of the binning scheme, minus the ones the linear index puts
- * entirely before it. On a long-read file that is wildly more than the query
- * reads: a coarse bin's chunks run to the end of the bin's span, so a 380bp
- * window on a deep ONT BAM resolves to 90 chunks / 43.5MB, of which
- * `getRecordsForRange` reads 6 / 7.8MB before the early stop fires.
+ * entirely before it. Until `max_off` (ADR 0023) also dropped the ones past it,
+ * that was wildly more than a long-read query reads: a coarse bin's chunks run
+ * to the end of the bin's span, so a 380bp window on a deep ONT BAM resolved to
+ * 90 chunks / 43.5MB, of which `getRecordsForRange` read 6 / 7.8MB before the
+ * early stop fired. Since then this changes the forecast in 4 of 320 fixture
+ * windows, undershooting in all 4; ADR 0023 says what to check before
+ * removing it.
  *
  * Two bounds, and the answer is the larger:
  *
@@ -325,6 +367,16 @@ export function clampChunkEnds(
   }
 }
 
+// SAMv1 §4.2 counts NUL padding in l_text, and htslib reads the text as a C
+// string, so the header ends at the first NUL.
+export function decodeHeaderText(bytes: Uint8Array, lText: number) {
+  const text = bytes.subarray(8, 8 + lText)
+  const nul = text.indexOf(0)
+  return new TextDecoder('utf8').decode(
+    nul === -1 ? text : text.subarray(0, nul),
+  )
+}
+
 // Parse the BAM reference-sequence table (SAMv1.pdf §4.2). Returns undefined
 // if `uncba` doesn't yet contain the full table — caller fetches more bytes
 // and retries.

package/src/virtualOffset.ts

@@ -35,3 +35,7 @@ export function fromBytes(bytes: Uint8Array, offset = 0) {
     (bytes[offset + 1]! << 8) | bytes[offset]!,
   )
 }
+
+export function compareOffsets(a: OffsetCoords, b: OffsetCoords) {
+  return a.blockPosition - b.blockPosition || a.dataPosition - b.dataPosition
+}