@gmod/bam 7.9.0 → 7.9.1

package/src/bamFile.ts

@@ -7,7 +7,7 @@ import CSI from './csi.ts'
 import NullFilehandle from './nullFilehandle.ts'
 import BAMFeature from './record.ts'
 import { parseHeaderText } from './sam.ts'
-import { appendInRange, parseRefSeqs } from './util.ts'
+import { appendInRange, parseRefSeqs, throwIfAborted } from './util.ts'
 
 import type Chunk from './chunk.ts'
 import type { BamOpts, BaseOpts } from './util.ts'
@@ -65,9 +65,16 @@ interface ChunkEntry<T> {
 
 interface InFlightChunk<T> {
   promise: Promise<ChunkEntry<T>>
-  // the signal the read was started with, so a waiter can tell "the owner
-  // aborted" apart from "the read genuinely failed"
-  signal?: AbortSignal
+  // Signals of the callers still waiting on this read. The read is cancelled
+  // only once every one of them has given up — see joinChunkRead and ADR 0007.
+  signals: Set<AbortSignal>
+  // true once a caller joins without a signal, which pins the read
+  pinned: boolean
+  // aborts when every caller has given up. what the read actually runs under
+  controller: AbortController
+  // aborted to take this read's listeners back off its callers' signals
+  dispose: AbortController
+  settled: boolean
 }
 
 /**
@@ -392,6 +399,7 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     max: number,
     opts?: BamOpts,
   ) {
+    throwIfAborted(opts?.signal)
     const chrId = await this.getSeqId(chr, opts)
     if (chrId === undefined || !this.index) {
       return []
@@ -402,23 +410,85 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
 
   // Read a chunk, publish it to the cache, and keep the in-flight promise
   // discoverable while it runs.
-  private _startChunkRead(cacheKey: string, chunk: Chunk, opts: BaseOpts) {
-    const promise = this._readChunkFeatures(chunk, opts).then(entry => {
+  //
+  // The read runs under this entry's own controller rather than any one
+  // caller's signal, because the read is shared: it must survive until every
+  // caller waiting on it has given up. joinChunkRead is what registers them.
+  private _startChunkRead(cacheKey: string, chunk: Chunk) {
+    const controller = new AbortController()
+    const promise = this._readChunkFeatures(chunk, {
+      signal: controller.signal,
+    }).then(entry => {
       this.chunkFeatureCache.set(cacheKey, entry)
       return entry
     })
-    const inFlight: InFlightChunk<T> = { promise, signal: opts.signal }
+    const inFlight: InFlightChunk<T> = {
+      promise,
+      signals: new Set(),
+      pinned: false,
+      controller,
+      dispose: new AbortController(),
+      settled: false,
+    }
     this.inFlightChunks.set(cacheKey, inFlight)
-    // Only clear our own entry: a retry may already have replaced it. `.then(f,
-    // f)` rather than `.finally(f)` so the handler's own promise never carries
-    // an unhandled rejection.
+    // `.then(f, f)` rather than `.finally(f)` so the handler's own promise never
+    // carries an unhandled rejection.
     const clear = () => {
+      inFlight.settled = true
+      // nothing reads these once the read has settled, and holding them would
+      // pin each caller's AbortController behind this entry
+      inFlight.dispose.abort()
+      inFlight.signals.clear()
+      // only clear our own entry: a later read for this chunk may have replaced it
       if (this.inFlightChunks.get(cacheKey) === inFlight) {
         this.inFlightChunks.delete(cacheKey)
       }
     }
     promise.then(clear, clear)
-    return promise
+    return inFlight
+  }
+
+  // Register a caller's interest, so the read survives until that caller has
+  // given up too.
+  //
+  // A caller with no signal cannot give up, so it pins the read: there is no
+  // longer any set of aborts that should stop it. That is the honest reading of
+  // a caller that never asked to be cancellable, and it means one signal-free
+  // query makes that chunk's read uncancellable for everyone joined to it.
+  private joinChunkRead(inFlight: InFlightChunk<T>, signal?: AbortSignal) {
+    if (signal === undefined) {
+      inFlight.pinned = true
+    } else if (signal.aborted) {
+      // A caller that has already given up is not a waiter, and must not be
+      // counted as one: an `abort` listener never fires on a signal that
+      // aborted before it was added, so nothing would ever take this signal
+      // back out of the set. The count would never reach zero and the read
+      // would be uncancellable for everyone joined to it, silently.
+      //
+      // `_cachedChunkFeatures` rejects such a caller before it reaches here,
+      // with no `await` in between, so this is unreachable today. It is here
+      // because this is the bug that shipped, and an invariant that fails this
+      // quietly should not rest on a check twenty lines away. `@gmod/cram`
+      // guards the same spot for the same reason; see its ADR 0003.
+      if (!inFlight.pinned && inFlight.signals.size === 0) {
+        inFlight.controller.abort(signal.reason)
+      }
+    } else if (!inFlight.signals.has(signal)) {
+      // guarded so one signal joining twice — a viewAsPairs query reaching the
+      // same chunk for a read and for its mate — does not add two listeners
+      inFlight.signals.add(signal)
+      signal.addEventListener(
+        'abort',
+        () => {
+          inFlight.signals.delete(signal)
+          if (!inFlight.pinned && inFlight.signals.size === 0) {
+            inFlight.controller.abort(signal.reason)
+          }
+        },
+        // `once` covers the abort firing; `dispose` covers it never firing
+        { once: true, signal: inFlight.dispose.signal },
+      )
+    }
   }
 
   // Parsed records for a chunk, reading and decompressing it only on a miss.
@@ -435,6 +505,15 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     chunk: Chunk,
     opts: BaseOpts,
   ): Promise<T[]> {
+    // Before anything else, including the cache hit. A caller reaches here with
+    // a signal that has already fired on the ordinary pan — the abort lands
+    // while blocksForRange is still reading the index, and nothing between
+    // there and here looks at it, since bai.ts and csi.ts never read the
+    // signal. Such a caller must not start a read it has no interest in, and
+    // must not be registered as a waiter on someone else's: see joinChunkRead.
+    // @gmod/cram checks in exactly this position, in SliceRecordCache.getOrFill.
+    throwIfAborted(opts.signal)
+
     const cacheKey = chunkCacheKey(chunk)
     const cached = this.chunkFeatureCache.get(cacheKey)
     if (cached) {
@@ -446,23 +525,35 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
     // they collapse onto very few chunk keys, so without this a query pays for
     // the same inflate several times over — the dominant cost of a cold query
     // (ADR 0003).
-    const pending = this.inFlightChunks.get(cacheKey)
-    if (!pending) {
-      return (await this._startChunkRead(cacheKey, chunk, opts)).features
+    let pending = this.inFlightChunks.get(cacheKey)
+    // A read every caller has abandoned is on its way out but may not have
+    // noticed yet. Start a fresh one rather than joining one already doomed.
+    if (pending?.controller.signal.aborted && !pending.settled) {
+      if (this.inFlightChunks.get(cacheKey) === pending) {
+        this.inFlightChunks.delete(cacheKey)
+      }
+      pending = undefined
+    }
+    pending ??= this._startChunkRead(cacheKey, chunk)
+    // Only a read still running has anything to cancel. Joining a settled one
+    // would add this caller to a set nothing will ever take it out of, since
+    // the entry drops its abort listeners when it settles.
+    if (!pending.settled) {
+      this.joinChunkRead(pending, opts.signal)
     }
 
     try {
-      return (await pending.promise).features
+      const entry = await pending.promise
+      // the read finished, but this caller gave up while waiting for it
+      throwIfAborted(opts.signal)
+      return entry.features
     } catch (e) {
-      // The read we joined was started by another caller. If that caller
-      // aborted and we did not, the failure is theirs and says nothing about
-      // our query, so start over — which picks up the cache, joins a sibling's
-      // retry, or reads under our own signal. Any other failure (and our own
-      // abort) propagates as it would have without sharing.
-      if (!pending.signal?.aborted || opts.signal?.aborted) {
-        throw e
-      }
-      return this._cachedChunkFeatures(chunk, opts)
+      // Prefer this caller's own cancellation to whatever the shared read
+      // reported. If we asked to stop, that is the answer we want — and when
+      // the read itself was cancelled it is because we, and everyone else,
+      // asked it to.
+      throwIfAborted(opts.signal)
+      throw e
     }
   }
 
@@ -669,6 +760,21 @@ export default class BamFile<T extends BamRecordLike = BAMFeature> {
         signal: opts.signal,
       },
     )
+    // The last chance to bail before the expensive part, and the reason it is
+    // worth having: honouring the signal is optional in the filehandle.
+    // `RemoteFile` hands it to `fetch`, but `LocalFile.read(length, position)`
+    // does not even take an options argument, so every read under it runs to
+    // completion and arrives here with the cancellation unnoticed. Without this
+    // a fully abandoned chunk still gets inflated and decoded — measured at 6
+    // chunks for one 20kb query on out.bam — which is the dominant cost of a
+    // cold query (ADR 0003), spent on records nobody will ever look at.
+    //
+    // Safe precisely because this is the SHARED signal, not a caller's: it
+    // fires only once every waiter has given up, so there is never a live
+    // caller left to be denied the result. @gmod/cram checks in the same place
+    // and for the same reason, before the decode loop in `_fetchRecords`.
+    throwIfAborted(opts.signal)
+
     const {
       buffer: data,
       cpositions,

package/src/indexFile.ts

@@ -1,6 +1,6 @@
 import QuickLRU from '@jbrowse/quick-lru'
 
-import { optimizeChunks } from './util.ts'
+import { optimizeChunks, throwIfAborted } from './util.ts'
 
 import type Chunk from './chunk.ts'
 import type { BaseOpts } from './util.ts'
@@ -54,6 +54,13 @@ export default abstract class IndexFile<
   public renameRefSeq: (s: string) => string
 
   private setupP?: Promise<TParsed>
+  /**
+   * The signal `setupP` was started under, while it is still in flight. The
+   * index is parsed once and shared by every query against the file, so without
+   * this the first query to arrive would own a read all the others depend on —
+   * see {@link parse}.
+   */
+  private setupSignal?: AbortSignal
 
   constructor({
     filehandle,
@@ -121,14 +128,70 @@ export default abstract class IndexFile<
     return optimizeChunks(chunks, this.getLowestChunk(ba, min))
   }
 
-  parse(opts: BaseOpts = {}): Promise<TParsed> {
-    if (!this.setupP) {
-      this.setupP = this._parse(opts).catch((e: unknown) => {
-        this.setupP = undefined
+  /**
+   * Parse the index, or join the parse already running.
+   *
+   * The index is downloaded and parsed once for the life of this object, so it
+   * is the one read here that is shared between queries — and therefore the one
+   * place a cancellation can leak from the query that asked for it to a query
+   * that did not. `_parse` hands `opts` straight to `filehandle.readFile`, so
+   * without this the first query to arrive owns a read every other query
+   * depends on: when it pans away, every concurrent query fails with its abort.
+   *
+   * A caller that joined someone else's parse and saw it fail because *they*
+   * aborted starts over rather than inheriting the failure — once, then
+   * propagates. Bounding it at one attempt is what jbrowse's
+   * `RemoteFileWithRangeCache.joinChunk` does with the same retry one layer
+   * down, and for the reason it gives: the pathological case becomes one
+   * duplicate parse rather than a recursion whose depth depends on how the
+   * aborts interleave.
+   *
+   * A retry rather than the reference count `_cachedChunkFeatures` uses, for
+   * the reason `@gmod/cram` gives for the same split in `CraiIndex`: the index
+   * is parsed once for the life of the object, so there is no repeated waste to
+   * recover, and this is a dozen lines against restructuring the memo.
+   */
+  async parse(opts: BaseOpts = {}, retried = false): Promise<TParsed> {
+    throwIfAborted(opts.signal)
+    const pending = this.setupP
+    if (!pending) {
+      return this.startParse(opts)
+    }
+
+    // read before awaiting: the owner is forgotten as soon as the parse settles
+    const ownerSignal = this.setupSignal
+    try {
+      return await pending
+    } catch (e) {
+      if (retried || !ownerSignal?.aborted || opts.signal?.aborted) {
         throw e
-      })
+      }
+      return this.parse(opts, true)
     }
-    return this.setupP
+  }
+
+  private startParse(opts: BaseOpts) {
+    const pending = this._parse(opts)
+    this.setupP = pending
+    this.setupSignal = opts.signal
+    // Drop a rejection rather than keeping it, so one transient failure does not
+    // poison the index for the lifetime of the file. Both branches are
+    // identity-checked so a retry started after this settles is not cleared by
+    // the attempt it already replaced.
+    pending.then(
+      () => {
+        if (this.setupP === pending) {
+          this.setupSignal = undefined
+        }
+      },
+      () => {
+        if (this.setupP === pending) {
+          this.setupP = undefined
+          this.setupSignal = undefined
+        }
+      },
+    )
+    return pending
   }
 
   async lineCount(refId: number, opts?: BaseOpts) {

package/src/util.ts

@@ -39,6 +39,34 @@ export interface BaseOpts {
  * underneath you. (The Chunk objects themselves are never mutated; a merged
  * span produces a new instance.)
  */
+/**
+ * `signal.throwIfAborted()`, without requiring either that method or `reason`.
+ *
+ * Two reasons not to call the built-in directly. It assumes a *real*
+ * `AbortSignal`, and callers pass duck-typed ones — `test/csi.test.ts` casts a
+ * bare `{ aborted }` through `as AbortSignal`, which is a fair model of what
+ * consumers do; calling a missing method there is a `TypeError` rather than the
+ * cancellation the caller asked for, which is a strictly worse failure.
+ *
+ * And it sets a browser floor. `AbortSignal.prototype.throwIfAborted` and
+ * `AbortSignal.reason` are Safari 15.4 / Chrome 100 / Firefox 97 (March 2022) —
+ * higher than anything else in this dependency tree needs, since
+ * `generic-filehandle2` only ever forwards a signal to `fetch`, and higher than
+ * every sibling gmod library, which use plain `.aborted`. Three lines here
+ * keeps that floor where it was.
+ *
+ * Faithful to the spec otherwise: an aborted signal throws its `reason`
+ * whatever that is, and only synthesizes an `AbortError` when there is none.
+ */
+export function throwIfAborted(signal?: AbortSignal) {
+  if (signal?.aborted) {
+    const { reason } = signal
+    throw reason === undefined
+      ? new DOMException('This operation was aborted', 'AbortError')
+      : reason
+  }
+}
+
 export function optimizeChunks(chunks: Chunk[], lowest?: OffsetCoords) {
   const n = chunks.length
   if (n === 0) {