@gmod/tabix 3.4.1 → 3.4.3

package/src/tabixIndexedFile.ts

@@ -14,6 +14,7 @@ import type { GenericFilehandle } from 'generic-filehandle2'
 
 const TAB = 9
 const NEWLINE = 10
+const CARRIAGE_RETURN = 13
 const SEMICOLON = 59
 
 type GetLinesCallback = (
@@ -59,6 +60,56 @@ function resolveFilehandle(
   throw new TypeError('must provide either filehandle, path, or url')
 }
 
+// An explicitly supplied index always wins over the `<path>.tbi` /
+// `<url>.tbi` names derived from the data file, whichever kind it is.
+function resolveIndex({
+  tbiFilehandle,
+  csiFilehandle,
+  tbiPath,
+  csiPath,
+  tbiUrl,
+  csiUrl,
+  path,
+  url,
+}: {
+  tbiFilehandle?: GenericFilehandle
+  csiFilehandle?: GenericFilehandle
+  tbiPath?: string
+  csiPath?: string
+  tbiUrl?: string
+  csiUrl?: string
+  path?: string
+  url?: string
+}) {
+  if (tbiFilehandle) {
+    return new TBI({ filehandle: tbiFilehandle })
+  }
+  if (csiFilehandle) {
+    return new CSI({ filehandle: csiFilehandle })
+  }
+  if (tbiPath) {
+    return new TBI({ filehandle: new LocalFile(tbiPath) })
+  }
+  if (csiPath) {
+    return new CSI({ filehandle: new LocalFile(csiPath) })
+  }
+  if (csiUrl) {
+    return new CSI({ filehandle: new RemoteFile(csiUrl) })
+  }
+  if (tbiUrl) {
+    return new TBI({ filehandle: new RemoteFile(tbiUrl) })
+  }
+  if (path) {
+    return new TBI({ filehandle: new LocalFile(`${path}.tbi`) })
+  }
+  if (url) {
+    return new TBI({ filehandle: new RemoteFile(`${url}.tbi`) })
+  }
+  throw new TypeError(
+    'must provide one of tbiFilehandle, tbiPath, csiFilehandle, csiPath, tbiUrl, csiUrl',
+  )
+}
+
 function calculateFileOffset(
   cpositions: number[],
   dpositions: number[],
@@ -172,28 +223,16 @@ export default class TabixIndexedFile {
     chunkCacheSize?: number
   }) {
     this.filehandle = resolveFilehandle(filehandle, path, url)
-
-    if (tbiFilehandle) {
-      this.index = new TBI({ filehandle: tbiFilehandle })
-    } else if (csiFilehandle) {
-      this.index = new CSI({ filehandle: csiFilehandle })
-    } else if (tbiPath) {
-      this.index = new TBI({ filehandle: new LocalFile(tbiPath) })
-    } else if (csiPath) {
-      this.index = new CSI({ filehandle: new LocalFile(csiPath) })
-    } else if (path) {
-      this.index = new TBI({ filehandle: new LocalFile(`${path}.tbi`) })
-    } else if (csiUrl) {
-      this.index = new CSI({ filehandle: new RemoteFile(csiUrl) })
-    } else if (tbiUrl) {
-      this.index = new TBI({ filehandle: new RemoteFile(tbiUrl) })
-    } else if (url) {
-      this.index = new TBI({ filehandle: new RemoteFile(`${url}.tbi`) })
-    } else {
-      throw new TypeError(
-        'must provide one of tbiFilehandle, tbiPath, csiFilehandle, csiPath, tbiUrl, csiUrl',
-      )
-    }
+    this.index = resolveIndex({
+      tbiFilehandle,
+      csiFilehandle,
+      tbiPath,
+      csiPath,
+      tbiUrl,
+      csiUrl,
+      path,
+      url,
+    })
 
     this.chunkCache = new AbortablePromiseCache<Chunk, ReadChunk>({
       cache: new LRU({ maxSize: Math.floor(chunkCacheSize / (1 << 16)) }),
@@ -318,15 +357,14 @@ export default class TabixIndexedFile {
           continue
         }
 
-        // find tab positions
+        // find tab positions. Columns past the end of the line all get `n`
+        // rather than breaking out, which would leave stale offsets from the
+        // previous line in the tail of the array.
         tabs[0] = blockStart - 1
         for (let i = 0; i < maxColumn; i++) {
-          const tabPos = buffer.indexOf(TAB, tabs[i]! + 1)
-          if (tabPos === -1 || tabPos >= n) {
-            tabs[i + 1] = n
-            break
-          }
-          tabs[i + 1] = tabPos
+          const prev = tabs[i]!
+          const tabPos = prev < n ? buffer.indexOf(TAB, prev + 1) : -1
+          tabs[i + 1] = tabPos === -1 || tabPos >= n ? n : tabPos
         }
 
         // compare ref name bytes directly
@@ -380,7 +418,9 @@ export default class TabixIndexedFile {
         }
 
         if (endCoordinate > start) {
-          const line = decoder.decode(buffer.subarray(blockStart, n))
+          // trim a CRLF terminator, matching htslib's line reader
+          const lineEnd = buffer[n - 1] === CARRIAGE_RETURN ? n - 1 : n
+          const line = decoder.decode(buffer.subarray(blockStart, lineEnd))
           callback(
             line,
             calculateFileOffset(

package/src/tbi.ts

@@ -3,6 +3,7 @@ import { unzip } from '@gmod/bgzf-filehandle'
 import Chunk from './chunk.ts'
 import IndexFile from './indexFile.ts'
 import {
+  clampChunkEnds,
   findFirstData,
   memoizeByRefId,
   optimizeChunks,
@@ -16,21 +17,31 @@ import type VirtualOffset from './virtualOffset.ts'
 
 const TBI_MAGIC = 21_578_324 // TBI\1
 
+// TBI is always the fixed depth-5, 14-bit-leaf binning scheme
+const TBI_DEPTH = 5
+const TBI_MAX_REF_LENGTH = 2 ** (14 + TBI_DEPTH * 3)
+
 /**
  * calculate the list of bins that may overlap with region [beg,end)
  * (zero-based half-open)
  */
 function reg2bins(beg: number, end: number) {
-  beg += 1 // < convert to 1-based closed
-  end -= 1
-  return [
-    [0, 0],
-    [1 + (beg >> 26), 1 + (end >> 26)],
-    [9 + (beg >> 23), 9 + (end >> 23)],
-    [73 + (beg >> 20), 73 + (end >> 20)],
-    [585 + (beg >> 17), 585 + (end >> 17)],
-    [4681 + (beg >> 14), 4681 + (end >> 14)],
-  ] as const
+  // Clamp into the binning scheme's range. Past 2**31 the shifts below
+  // truncate to int32 and go negative, which would silently yield no bins.
+  const b = Math.min(Math.max(beg, 0), TBI_MAX_REF_LENGTH)
+  const e = Math.min(end, TBI_MAX_REF_LENGTH) - 1
+  const bins: (readonly [number, number])[] = []
+  if (b <= e) {
+    bins.push(
+      [0, 0],
+      [1 + (b >> 26), 1 + (e >> 26)],
+      [9 + (b >> 23), 9 + (e >> 23)],
+      [73 + (b >> 20), 73 + (e >> 20)],
+      [585 + (b >> 17), 585 + (e >> 17)],
+      [4681 + (b >> 14), 4681 + (e >> 14)],
+    )
+  }
+  return bins
 }
 
 export default class TabixIndex extends IndexFile {
@@ -41,16 +52,18 @@ export default class TabixIndex extends IndexFile {
       onProgress: opts.onProgress,
     })
     const bytes = (await unzip(buf)) as Uint8Array
-    const dataView = new DataView(bytes.buffer)
+    const dataView = new DataView(
+      bytes.buffer,
+      bytes.byteOffset,
+      bytes.byteLength,
+    )
 
     if (dataView.getUint32(0, true) !== TBI_MAGIC) {
       throw new Error('Not a TBI file')
     }
 
     const refCount = dataView.getUint32(4, true)
-    const depth = 5
-    const maxBinNumber = ((1 << ((depth + 1) * 3)) - 1) / 7
-    const maxRefLength = 2 ** (14 + depth * 3)
+    const maxBinNumber = ((1 << ((TBI_DEPTH + 1) * 3)) - 1) / 7
 
     // TBI header layout matches CSI aux data; parseAuxData handles both
     const {
@@ -147,6 +160,10 @@ export default class TabixIndex extends IndexFile {
         linearIndex[k] = fromBytes(bytes, pos)
         pos += 8
       }
+      clampChunkEnds(
+        Object.values(binIndex).flat(),
+        linearIndex.map(v => v.blockPosition),
+      )
       return { binIndex, linearIndex, stats }
     }
 
@@ -154,7 +171,7 @@ export default class TabixIndex extends IndexFile {
       indices: memoizeByRefId(getIndices),
       metaChar,
       maxBinNumber,
-      maxRefLength,
+      maxRefLength: TBI_MAX_REF_LENGTH,
       skipLines,
       firstDataLine,
       columnNumbers,
@@ -172,10 +189,6 @@ export default class TabixIndex extends IndexFile {
     max: number,
     opts: Options = {},
   ) {
-    if (min < 0) {
-      min = 0
-    }
-
     const indexData = await this.parse(opts)
     const refId = indexData.refNameToId[refName]
     if (refId === undefined) {
@@ -186,13 +199,12 @@ export default class TabixIndex extends IndexFile {
       return []
     }
 
-    const linearIndex = ba.linearIndex ?? []
-    // min >= 2**31 overflows int32 and produces a negative >> 14 result —
-    // query is well beyond the indexed range
-    if (linearIndex.length > 0 && min >= 2 ** 31) {
+    if (min > TBI_MAX_REF_LENGTH) {
       console.warn('querying outside of possible tabix range')
     }
-
+    // clamp before the >> 14 below, which would truncate to int32 and go
+    // negative past 2**31
+    const beg = Math.min(Math.max(min, 0), TBI_MAX_REF_LENGTH)
     const overlappingBins = reg2bins(min, max) // List of bin #s that overlap min, max
     const chunks: Chunk[] = []
 
@@ -211,7 +223,8 @@ export default class TabixIndex extends IndexFile {
     // The linear index is monotonically non-decreasing, so the minimum virtual
     // offset for chunks that could overlap [min, ...) is at index minLin.
     // SYNC: ~/src/gmod/bam-js/src/bai.ts getLowestChunk
-    const lowest = linearIndex[Math.min(min >> 14, linearIndex.length - 1)]
+    const linearIndex = ba.linearIndex
+    const lowest = linearIndex?.[Math.min(beg >> 14, linearIndex.length - 1)]
 
     return optimizeChunks(chunks, lowest)
   }

package/src/util.ts

@@ -65,6 +65,7 @@ export function optimizeChunks(chunks: Chunk[], lowest?: VirtualOffset) {
           lastChunk.minv,
           chunk.maxv,
           lastChunk.bin,
+          chunk.endPosition,
         )
         lastMaxBlock = chunkMaxBlock
       }
@@ -78,6 +79,45 @@ export function optimizeChunks(chunks: Chunk[], lowest?: VirtualOffset) {
   return mergedChunks
 }
 
+// Tighten each chunk's endPosition (default: a full max-size BGZF block past
+// maxv) down to the next known block boundary. Every chunk min/maxv blockPosition
+// and every extraBoundary (e.g. linear-index entries) is a real BGZF block start;
+// since blocks don't overlap, the first boundary strictly greater than a chunk's
+// maxv.blockPosition is an upper bound on where that final block ends — always at
+// least the true block end, so the clamped fetch still contains the whole block.
+// Shrinks both the byte estimate and the actual fetch with no extra I/O.
+export function clampChunkEnds(
+  chunks: Chunk[],
+  extraBoundaries: number[] = [],
+) {
+  // Array#sort with a comparator beats a Float64Array numeric sort here:
+  // extraBoundaries (the linear index) arrives already sorted and TimSort
+  // exploits the existing run, which introsort cannot.
+  const boundaries = [...extraBoundaries]
+  for (const c of chunks) {
+    boundaries.push(c.minv.blockPosition, c.maxv.blockPosition)
+  }
+  boundaries.sort((a, b) => a - b)
+
+  for (const c of chunks) {
+    const max = c.maxv.blockPosition
+    // first boundary strictly greater than max
+    let lo = 0
+    let hi = boundaries.length
+    while (lo < hi) {
+      const mid = (lo + hi) >>> 1
+      if (boundaries[mid]! > max) {
+        hi = mid
+      } else {
+        lo = mid + 1
+      }
+    }
+    if (lo < boundaries.length) {
+      c.endPosition = Math.min(c.endPosition, boundaries[lo]!)
+    }
+  }
+}
+
 export function findFirstData(
   currentFdl: VirtualOffset | undefined,
   virtualOffset: VirtualOffset,
@@ -116,7 +156,11 @@ const tabixFormats: Record<number, string> = {
 }
 
 export function parseAuxData(bytes: Uint8Array, offset: number) {
-  const dataView = new DataView(bytes.buffer)
+  const dataView = new DataView(
+    bytes.buffer,
+    bytes.byteOffset,
+    bytes.byteLength,
+  )
   const formatFlags = dataView.getInt32(offset, true)
   const coordinateType =
     formatFlags & 0x1_00_00 ? 'zero-based-half-open' : '1-based-closed'