@gmod/bam 7.8.2 → 7.9.1

package/src/bamFile.ts

@@ -7,7 +7,7 @@ import CSI from './csi.ts'
 import NullFilehandle from './nullFilehandle.ts'
 import BAMFeature from './record.ts'
 import { parseHeaderText } from './sam.ts'
-import { appendInRange, parseRefSeqs } from './util.ts'
+import { appendInRange, parseRefSeqs, throwIfAborted } from './util.ts'
 
 import type Chunk from './chunk.ts'
 import type { BamOpts, BaseOpts } from './util.ts'
@@ -37,10 +37,13 @@ export const BAM_MAGIC = 21840194
 
 const blockLen = 1 << 16
 
-// Ceiling on the header read. A million contigs is roughly 10MB of compressed
-// @SQ lines and ref-seq table, so anything past this is a corrupt header
-// claiming a huge n_ref rather than a real one, and growing the read further
-// just downloads the file.
+// Ceiling on GROWING the header read. A million contigs is roughly 10MB of
+// compressed @SQ lines and ref-seq table, so a read that has doubled past this
+// is chasing a corrupt header claiming a huge n_ref rather than a real one, and
+// growing further just downloads the file. It does not cap the FIRST read: that
+// length comes from the index's firstDataLine, which is a real offset rather
+// than a guess, so a header that genuinely runs past this still gets its one
+// exact read. See getHeaderPre.
 const maxHeaderReadLen = 32 * 1024 * 1024
 
 function resolveFilehandle(
@@ -62,9 +65,16 @@ interface ChunkEntry<T> {
 
 interface InFlightChunk<T> {
   promise: Promise<ChunkEntry<T>>
-  // the signal the read was started with, so a waiter can tell "the owner
-  // aborted" apart from "the read genuinely failed"
-  signal?: AbortSignal
+  // Signals of the callers still waiting on this read. The read is cancelled
+  // only once every one of them has given up — see joinChunkRead and ADR 0007.
+  signals: Set<AbortSignal>
+  // true once a caller joins without a signal, which pins the read
+  pinned: boolean
+  // aborts when every caller has given up. what the read actually runs under
+  controller: AbortController
+  // aborted to take this read's listeners back off its callers' signals
+  dispose: AbortController
+  settled: boolean
 }
 
 /**
@@ -118,12 +128,24 @@ export const DEFAULT_MAX_CACHE_BYTES = 100 * 1024 * 1024
 const MAX_CONCURRENT_CHUNK_READS = 6
 
 class ChunkFeatureCache<T> {
-  public maxBytes: number
+  private _maxBytes: number
   private entries = new Map<string, ChunkEntry<T>>()
   private bytes = 0
 
   constructor(maxBytes: number) {
-    this.maxBytes = maxBytes
+    this._maxBytes = maxBytes
+  }
+
+  get maxBytes() {
+    return this._maxBytes
+  }
+
+  // Accessor rather than a plain field so lowering the budget frees memory now.
+  // As a field, a caller trimming the cache under memory pressure got nothing
+  // back until the next chunk read happened to call set().
+  set maxBytes(maxBytes: number) {
+    this._maxBytes = maxBytes
+    this.evict()
   }
 
   get size() {
@@ -148,11 +170,15 @@ class ChunkFeatureCache<T> {
     this.delete(key)
     this.entries.set(key, entry)
     this.bytes += entry.bytes
-    // Evict from the least-recently-used end. The size > 1 guard means a single
-    // chunk larger than the whole budget is still kept: the caller needs it for
-    // the query in flight, and dropping it would only force a re-decompress.
+    this.evict()
+  }
+
+  // Evict from the least-recently-used end. The size > 1 guard means a single
+  // chunk larger than the whole budget is still kept: the caller needs it for
+  // the query in flight, and dropping it would only force a re-decompress.
+  private evict() {
     const lru = this.entries.keys()
-    while (this.bytes > this.maxBytes && this.entries.size > 1) {
+    while (this.bytes > this._maxBytes && this.entries.size > 1) {
       this.delete(lru.next().value!)
     }
   }
@@ -285,16 +311,21 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
         ? blockLen
         : indexData.firstDataLine.blockPosition + blockLen
 
+    // do/while, so the index-derived length is always read once and only the
+    // doubling is bounded. Testing readLen before the first read instead meant
+    // a BAM whose header genuinely exceeds maxHeaderReadLen — millions of
+    // contigs — was rejected without a single byte being fetched, and reported
+    // as 'Insufficient data for reference sequences' when the data was there.
     let samHeader
-    let atEof = false
-    while (samHeader === undefined && !atEof && readLen <= maxHeaderReadLen) {
+    let atEof: boolean
+    do {
       const buffer = await this.bam.read(readLen, 0, { signal: opts.signal })
       // a short read means readLen ran past the end of the file, so there are
       // no more bytes to grow into
       atEof = buffer.length < readLen
       samHeader = this.applyHeader(await unzip(buffer))
       readLen *= 2
-    }
+    } while (samHeader === undefined && !atEof && readLen <= maxHeaderReadLen)
     if (samHeader === undefined) {
       throw new Error('Insufficient data for reference sequences')
     }
@@ -368,6 +399,7 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     max: number,
     opts?: BamOpts,
   ) {
+    throwIfAborted(opts?.signal)
     const chrId = await this.getSeqId(chr, opts)
     if (chrId === undefined || !this.index) {
       return []
@@ -378,23 +410,85 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
 
   // Read a chunk, publish it to the cache, and keep the in-flight promise
   // discoverable while it runs.
-  private _startChunkRead(cacheKey: string, chunk: Chunk, opts: BaseOpts) {
-    const promise = this._readChunkFeatures(chunk, opts).then(entry => {
+  //
+  // The read runs under this entry's own controller rather than any one
+  // caller's signal, because the read is shared: it must survive until every
+  // caller waiting on it has given up. joinChunkRead is what registers them.
+  private _startChunkRead(cacheKey: string, chunk: Chunk) {
+    const controller = new AbortController()
+    const promise = this._readChunkFeatures(chunk, {
+      signal: controller.signal,
+    }).then(entry => {
       this.chunkFeatureCache.set(cacheKey, entry)
       return entry
     })
-    const inFlight: InFlightChunk<T> = { promise, signal: opts.signal }
+    const inFlight: InFlightChunk<T> = {
+      promise,
+      signals: new Set(),
+      pinned: false,
+      controller,
+      dispose: new AbortController(),
+      settled: false,
+    }
     this.inFlightChunks.set(cacheKey, inFlight)
-    // Only clear our own entry: a retry may already have replaced it. `.then(f,
-    // f)` rather than `.finally(f)` so the handler's own promise never carries
-    // an unhandled rejection.
+    // `.then(f, f)` rather than `.finally(f)` so the handler's own promise never
+    // carries an unhandled rejection.
     const clear = () => {
+      inFlight.settled = true
+      // nothing reads these once the read has settled, and holding them would
+      // pin each caller's AbortController behind this entry
+      inFlight.dispose.abort()
+      inFlight.signals.clear()
+      // only clear our own entry: a later read for this chunk may have replaced it
       if (this.inFlightChunks.get(cacheKey) === inFlight) {
         this.inFlightChunks.delete(cacheKey)
       }
     }
     promise.then(clear, clear)
-    return promise
+    return inFlight
+  }
+
+  // Register a caller's interest, so the read survives until that caller has
+  // given up too.
+  //
+  // A caller with no signal cannot give up, so it pins the read: there is no
+  // longer any set of aborts that should stop it. That is the honest reading of
+  // a caller that never asked to be cancellable, and it means one signal-free
+  // query makes that chunk's read uncancellable for everyone joined to it.
+  private joinChunkRead(inFlight: InFlightChunk<T>, signal?: AbortSignal) {
+    if (signal === undefined) {
+      inFlight.pinned = true
+    } else if (signal.aborted) {
+      // A caller that has already given up is not a waiter, and must not be
+      // counted as one: an `abort` listener never fires on a signal that
+      // aborted before it was added, so nothing would ever take this signal
+      // back out of the set. The count would never reach zero and the read
+      // would be uncancellable for everyone joined to it, silently.
+      //
+      // `_cachedChunkFeatures` rejects such a caller before it reaches here,
+      // with no `await` in between, so this is unreachable today. It is here
+      // because this is the bug that shipped, and an invariant that fails this
+      // quietly should not rest on a check twenty lines away. `@gmod/cram`
+      // guards the same spot for the same reason; see its ADR 0003.
+      if (!inFlight.pinned && inFlight.signals.size === 0) {
+        inFlight.controller.abort(signal.reason)
+      }
+    } else if (!inFlight.signals.has(signal)) {
+      // guarded so one signal joining twice — a viewAsPairs query reaching the
+      // same chunk for a read and for its mate — does not add two listeners
+      inFlight.signals.add(signal)
+      signal.addEventListener(
+        'abort',
+        () => {
+          inFlight.signals.delete(signal)
+          if (!inFlight.pinned && inFlight.signals.size === 0) {
+            inFlight.controller.abort(signal.reason)
+          }
+        },
+        // `once` covers the abort firing; `dispose` covers it never firing
+        { once: true, signal: inFlight.dispose.signal },
+      )
+    }
   }
 
   // Parsed records for a chunk, reading and decompressing it only on a miss.
@@ -411,6 +505,15 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     chunk: Chunk,
     opts: BaseOpts,
   ): Promise<T[]> {
+    // Before anything else, including the cache hit. A caller reaches here with
+    // a signal that has already fired on the ordinary pan — the abort lands
+    // while blocksForRange is still reading the index, and nothing between
+    // there and here looks at it, since bai.ts and csi.ts never read the
+    // signal. Such a caller must not start a read it has no interest in, and
+    // must not be registered as a waiter on someone else's: see joinChunkRead.
+    // @gmod/cram checks in exactly this position, in SliceRecordCache.getOrFill.
+    throwIfAborted(opts.signal)
+
     const cacheKey = chunkCacheKey(chunk)
     const cached = this.chunkFeatureCache.get(cacheKey)
     if (cached) {
@@ -422,23 +525,35 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     // they collapse onto very few chunk keys, so without this a query pays for
     // the same inflate several times over — the dominant cost of a cold query
     // (ADR 0003).
-    const pending = this.inFlightChunks.get(cacheKey)
-    if (!pending) {
-      return (await this._startChunkRead(cacheKey, chunk, opts)).features
+    let pending = this.inFlightChunks.get(cacheKey)
+    // A read every caller has abandoned is on its way out but may not have
+    // noticed yet. Start a fresh one rather than joining one already doomed.
+    if (pending?.controller.signal.aborted && !pending.settled) {
+      if (this.inFlightChunks.get(cacheKey) === pending) {
+        this.inFlightChunks.delete(cacheKey)
+      }
+      pending = undefined
+    }
+    pending ??= this._startChunkRead(cacheKey, chunk)
+    // Only a read still running has anything to cancel. Joining a settled one
+    // would add this caller to a set nothing will ever take it out of, since
+    // the entry drops its abort listeners when it settles.
+    if (!pending.settled) {
+      this.joinChunkRead(pending, opts.signal)
     }
 
     try {
-      return (await pending.promise).features
+      const entry = await pending.promise
+      // the read finished, but this caller gave up while waiting for it
+      throwIfAborted(opts.signal)
+      return entry.features
     } catch (e) {
-      // The read we joined was started by another caller. If that caller
-      // aborted and we did not, the failure is theirs and says nothing about
-      // our query, so start over — which picks up the cache, joins a sibling's
-      // retry, or reads under our own signal. Any other failure (and our own
-      // abort) propagates as it would have without sharing.
-      if (!pending.signal?.aborted || opts.signal?.aborted) {
-        throw e
-      }
-      return this._cachedChunkFeatures(chunk, opts)
+      // Prefer this caller's own cancellation to whatever the shared read
+      // reported. If we asked to stop, that is the answer we want — and when
+      // the read itself was cancelled it is because we, and everyone else,
+      // asked it to.
+      throwIfAborted(opts.signal)
+      throw e
     }
   }
 
@@ -499,9 +614,7 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
       // unbarriered pool exactly as before. That caps the cost of being wrong
       // at 0.92x-0.95x, against 0.82x-0.88x for barriering every wave.
       const batch = Math.min(MAX_CONCURRENT_CHUNK_READS, chunks.length)
-      await Promise.all(
-        Array.from({ length: batch }, (_, ci) => readOne(ci)),
-      )
+      await Promise.all(Array.from({ length: batch }, (_, ci) => readOne(ci)))
       let stopped = false
       for (let ci = 0; ci < batch; ci++) {
         if (isPastQuery(featureLists[ci], chrId, max)) {
@@ -615,9 +728,13 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
         const mateRecs = [] as T[]
         for (let i = 0, l = features.length; i < l; i++) {
           const feature = features[i]!
+          // fileOffset first: it is a number already on the record, where
+          // `name` decodes a string per record. A mate chunk usually overlaps
+          // the query region, so this skips the decode for every record the
+          // caller is already holding.
           if (
-            readNameCounts.get(feature.name) === 1 &&
-            !readIds.has(feature.fileOffset)
+            !readIds.has(feature.fileOffset) &&
+            readNameCounts.get(feature.name) === 1
           ) {
             mateRecs.push(feature)
           }
@@ -643,6 +760,21 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
         signal: opts.signal,
       },
     )
+    // The last chance to bail before the expensive part, and the reason it is
+    // worth having: honouring the signal is optional in the filehandle.
+    // `RemoteFile` hands it to `fetch`, but `LocalFile.read(length, position)`
+    // does not even take an options argument, so every read under it runs to
+    // completion and arrives here with the cancellation unnoticed. Without this
+    // a fully abandoned chunk still gets inflated and decoded — measured at 6
+    // chunks for one 20kb query on out.bam — which is the dominant cost of a
+    // cold query (ADR 0003), spent on records nobody will ever look at.
+    //
+    // Safe precisely because this is the SHARED signal, not a caller's: it
+    // fires only once every waiter has given up, so there is never a live
+    // caller left to be denied the result. @gmod/cram checks in the same place
+    // and for the same reason, before the decode loop in `_fetchRecords`.
+    throwIfAborted(opts.signal)
+
     const {
       buffer: data,
       cpositions,
@@ -691,9 +823,9 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
           blockEnd,
           hasCpositions
             ? cpositions[pos]! * (1 << 8) +
-              (blockStart - dpositions[pos]!) +
-              chunk.minv.dataPosition +
-              1
+                (blockStart - dpositions[pos]!) +
+                chunk.minv.dataPosition +
+                1
             : crc32(ba.subarray(blockStart, blockEnd)) >>> 0,
           dataView,
         )

package/src/csi.ts

@@ -96,6 +96,20 @@ export default class CSI extends IndexFile {
     this.maxBinNumber = ((1 << ((this.depth + 1) * 3)) - 1) / 7
     const maxBinNumber = this.maxBinNumber
     const auxLength = dataView.getInt32(12, true)
+    // A tabix-only branch, which is why parseAuxData and the parseNameBytes it
+    // calls show as uncovered. CSI is shared between `samtools index -c` and
+    // `tabix -C`, and the aux block is how a tabix index carries what a reader
+    // of bgzipped TEXT needs: which columns hold ref/start/end, the comment
+    // character, lines to skip, and the reference names — which a BAM takes
+    // from its own header instead. So a BAM .csi sets l_aux to 0 and never
+    // reaches here (checked: 0 of the 19 .csi fixtures carry one).
+    //
+    // Kept rather than deleted for two reasons. CSI is exported from
+    // index.ts, so pointing it at a tabix index is reachable (@gmod/tabix is
+    // the right tool, but this would half-work and then silently not).
+    // More importantly parseNameBytes is marked SYNC: with its tabix-js
+    // counterpart, and dropping one side of a deliberately-paired pair of
+    // implementations is exactly what that marker exists to prevent.
     const aux = auxLength >= 30 ? this.parseAuxData(bytes, 16) : undefined
     const refCount = dataView.getInt32(16 + auxLength, true)
 
@@ -200,6 +214,12 @@ export default class CSI extends IndexFile {
     for (; l <= this.depth; s -= 3, t += lshift(1, l * 3), l += 1) {
       const b = t + rshift(beg, s)
       const e = t + rshift(end, s)
+      // Unreachable as long as the clamp above stands, which is why it shows
+      // as uncovered: with end bounded to 2^(minShift + depth*3), level l
+      // spans at most 8^l - 1 bins, so the worst case is 8^depth - 1 + depth
+      // against a maxBinNumber of (8^(depth+1) - 1)/7, i.e. about 1.14 * 8^depth.
+      // Kept as a guard on the arithmetic rather than deleted — reg2bins is
+      // shared with tabix-js, whose callers reach it by other routes.
       if (e - b + bins.length > this.maxBinNumber) {
         throw new Error(
           `query ${beg}-${end} is too large for current binning scheme (shift ${this.minShift}, depth ${this.depth}), try a smaller query or a coarser index binning scheme`,

package/src/index.ts

@@ -4,7 +4,12 @@ export { default as CSI } from './csi.ts'
 export { default as BamRecord } from './record.ts'
 export { default as HtsgetFile } from './htsget.ts'
 
-export type { Bytes } from './record.ts'
+export type { NumericCigar } from './record.ts'
 export type { BamRecordClass, BamRecordLike } from './bamFile.ts'
+// the options every query method takes, and the shapes they hand back. The
+// package has no subpath exports, so a consumer typing a wrapper around
+// getRecordsForRange/indexCov can only name these if they come out of here.
+export type { BamOpts, BaseOpts } from './util.ts'
+export type { IndexCovEntry } from './bai.ts'
 // for typing the HtsgetFile `fetch` option
 export type { Fetcher } from 'generic-filehandle2'

package/src/indexFile.ts

@@ -1,6 +1,6 @@
 import QuickLRU from '@jbrowse/quick-lru'
 
-import { optimizeChunks } from './util.ts'
+import { optimizeChunks, throwIfAborted } from './util.ts'
 
 import type Chunk from './chunk.ts'
 import type { BaseOpts } from './util.ts'
@@ -33,8 +33,11 @@ export function memoizeByRefId<T>(
 ) {
   const cache = new QuickLRU<number, T>({ maxSize })
   return (refId: number) => {
-    if (cache.has(refId)) {
-      return cache.get(refId)
+    // one lookup, not has()+get(): only truthy results are ever cached, so a
+    // miss and a cached value are already distinguishable
+    const cached = cache.get(refId)
+    if (cached !== undefined) {
+      return cached
     }
     const result = getIndices(refId)
     if (result) {
@@ -51,6 +54,13 @@ export default abstract class IndexFile<
   public renameRefSeq: (s: string) => string
 
   private setupP?: Promise<TParsed>
+  /**
+   * The signal `setupP` was started under, while it is still in flight. The
+   * index is parsed once and shared by every query against the file, so without
+   * this the first query to arrive would own a read all the others depend on —
+   * see {@link parse}.
+   */
+  private setupSignal?: AbortSignal
 
   constructor({
     filehandle,
@@ -118,14 +128,70 @@ export default abstract class IndexFile<
     return optimizeChunks(chunks, this.getLowestChunk(ba, min))
   }
 
-  parse(opts: BaseOpts = {}): Promise<TParsed> {
-    if (!this.setupP) {
-      this.setupP = this._parse(opts).catch((e: unknown) => {
-        this.setupP = undefined
+  /**
+   * Parse the index, or join the parse already running.
+   *
+   * The index is downloaded and parsed once for the life of this object, so it
+   * is the one read here that is shared between queries — and therefore the one
+   * place a cancellation can leak from the query that asked for it to a query
+   * that did not. `_parse` hands `opts` straight to `filehandle.readFile`, so
+   * without this the first query to arrive owns a read every other query
+   * depends on: when it pans away, every concurrent query fails with its abort.
+   *
+   * A caller that joined someone else's parse and saw it fail because *they*
+   * aborted starts over rather than inheriting the failure — once, then
+   * propagates. Bounding it at one attempt is what jbrowse's
+   * `RemoteFileWithRangeCache.joinChunk` does with the same retry one layer
+   * down, and for the reason it gives: the pathological case becomes one
+   * duplicate parse rather than a recursion whose depth depends on how the
+   * aborts interleave.
+   *
+   * A retry rather than the reference count `_cachedChunkFeatures` uses, for
+   * the reason `@gmod/cram` gives for the same split in `CraiIndex`: the index
+   * is parsed once for the life of the object, so there is no repeated waste to
+   * recover, and this is a dozen lines against restructuring the memo.
+   */
+  async parse(opts: BaseOpts = {}, retried = false): Promise<TParsed> {
+    throwIfAborted(opts.signal)
+    const pending = this.setupP
+    if (!pending) {
+      return this.startParse(opts)
+    }
+
+    // read before awaiting: the owner is forgotten as soon as the parse settles
+    const ownerSignal = this.setupSignal
+    try {
+      return await pending
+    } catch (e) {
+      if (retried || !ownerSignal?.aborted || opts.signal?.aborted) {
         throw e
-      })
+      }
+      return this.parse(opts, true)
     }
-    return this.setupP
+  }
+
+  private startParse(opts: BaseOpts) {
+    const pending = this._parse(opts)
+    this.setupP = pending
+    this.setupSignal = opts.signal
+    // Drop a rejection rather than keeping it, so one transient failure does not
+    // poison the index for the lifetime of the file. Both branches are
+    // identity-checked so a retry started after this settles is not cleared by
+    // the attempt it already replaced.
+    pending.then(
+      () => {
+        if (this.setupP === pending) {
+          this.setupSignal = undefined
+        }
+      },
+      () => {
+        if (this.setupP === pending) {
+          this.setupP = undefined
+          this.setupSignal = undefined
+        }
+      },
+    )
+    return pending
   }
 
   async lineCount(refId: number, opts?: BaseOpts) {