@gmod/bam 8.3.0 → 8.4.0

package/src/indexFile.ts

@@ -1,6 +1,7 @@
+import { SharedReadCache } from '@gmod/shared-read-cache'
 import QuickLRU from '@jbrowse/quick-lru'
 
-import { optimizeChunks, throwIfAborted } from './util.ts'
+import { chunksLikelyRead, optimizeChunks } from './util.ts'
 
 import type Chunk from './chunk.ts'
 import type { BaseOpts } from './util.ts'
@@ -53,14 +54,11 @@ export default abstract class IndexFile<
   public filehandle: GenericFilehandle
   public renameRefSeq: (s: string) => string
 
-  private parseP?: Promise<TParsed>
   /**
-   * The signal `parseP` was started under, while it is still in flight. The
-   * index is parsed once and shared by every query against the file, so without
-   * this the first query to arrive would own a read all the others depend on —
-   * see {@link parse}.
+   * The parsed index, as a shared read — see {@link parse}. One entry, never
+   * evicted, which is what a memo is.
    */
-  private parseSignal?: AbortSignal
+  private parseCache = new SharedReadCache<string, TParsed>({})
 
   constructor({
     filehandle,
@@ -95,6 +93,16 @@ export default abstract class IndexFile<
     min: number,
   ): OffsetCoords | undefined
 
+  // Block position past which a chunk is EXPECTED to hold nothing overlapping
+  // [..., max] — the counterpart of getLowestChunk, and unlike it an estimate
+  // rather than a bound (see chunksLikelyRead). Only `estimatedBytesForRegions`
+  // may use it. CSI has no linear index and returns undefined, which reads as
+  // "no opinion" and leaves that estimate summing every chunk.
+  protected abstract getHighestChunk(
+    refIndex: RefIndex,
+    max: number,
+  ): number | undefined
+
   async blocksForRange(
     refId: number,
     min: number,
@@ -128,72 +136,34 @@ export default abstract class IndexFile<
     return optimizeChunks(chunks, this.getLowestChunk(ba, min))
   }
 
-  // SYNC: ~/src/gmod/tabix-js/src/indexFile.ts parse — same owner-signal
-  // tracking and one-attempt retry, and the same reasoning below.
+  // SYNC: ~/src/gmod/tabix-js/src/indexFile.ts parse — same shape and the same
+  // reasoning below.
   /**
    * Parse the index, or join the parse already running.
    *
    * The index is downloaded and parsed once for the life of this object, so it
    * is the one read here that is shared between queries — and therefore the one
    * place a cancellation can leak from the query that asked for it to a query
-   * that did not. `_parse` hands `opts` straight to `filehandle.readFile`, so
-   * without this the first query to arrive owns a read every other query
-   * depends on: when it pans away, every concurrent query fails with its abort.
+   * that did not. `_parse` hands `opts` straight to `filehandle.readFile`, so a
+   * bare memoized promise makes the first query to arrive the owner of a read
+   * every other query depends on: when it pans away, every concurrent query
+   * fails with its abort.
    *
-   * A caller that joined someone else's parse and saw it fail because *they*
-   * aborted starts over rather than inheriting the failure — once, then
-   * propagates. Bounding it at one attempt is what jbrowse's
-   * `RemoteFileWithRangeCache.joinChunk` does with the same retry one layer
-   * down, and for the reason it gives: the pathological case becomes one
-   * duplicate parse rather than a recursion whose depth depends on how the
-   * aborts interleave.
+   * The same cache the chunk reads use, for the same reason and with the same
+   * rule: the parse runs under a signal of its own and is cancelled only once
+   * every caller waiting on it has given up, so one query's abort is reported
+   * to that query alone. A rejection is dropped rather than cached, so a
+   * transient failure does not poison the index for the life of the file.
    *
-   * A retry rather than the reference count `_cachedChunkFeatures` uses, for
-   * the reason `@gmod/cram` gives for the same split in `CraiIndex`: the index
-   * is parsed once for the life of the object, so there is no repeated waste to
-   * recover, and this is a dozen lines against restructuring the memo.
+   * The fill is per call rather than on the cache so that the caller who starts
+   * the parse has its `onProgress` reach `filehandle.readFile` — the index is a
+   * whole-file read, and a determinate "downloading index" bar is what that
+   * callback exists for.
    */
-  async parse(opts: BaseOpts = {}, retried = false): Promise<TParsed> {
-    throwIfAborted(opts.signal)
-    const pending = this.parseP
-    if (!pending) {
-      return this.startParse(opts)
-    }
-
-    // read before awaiting: the owner is forgotten as soon as the parse settles
-    const ownerSignal = this.parseSignal
-    try {
-      return await pending
-    } catch (e) {
-      if (retried || !ownerSignal?.aborted || opts.signal?.aborted) {
-        throw e
-      }
-      return this.parse(opts, true)
-    }
-  }
-
-  private startParse(opts: BaseOpts) {
-    const pending = this._parse(opts)
-    this.parseP = pending
-    this.parseSignal = opts.signal
-    // Drop a rejection rather than keeping it, so one transient failure does not
-    // poison the index for the lifetime of the file. Both branches are
-    // identity-checked so a retry started after this settles is not cleared by
-    // the attempt it already replaced.
-    pending.then(
-      () => {
-        if (this.parseP === pending) {
-          this.parseSignal = undefined
-        }
-      },
-      () => {
-        if (this.parseP === pending) {
-          this.parseP = undefined
-          this.parseSignal = undefined
-        }
-      },
+  parse(opts: BaseOpts = {}): Promise<TParsed> {
+    return this.parseCache.get('index', opts.signal, signal =>
+      this._parse({ ...opts, signal }),
     )
-    return pending
   }
 
   async lineCount(refId: number, opts?: BaseOpts) {
@@ -206,9 +176,42 @@ export default abstract class IndexFile<
     return !!indexData.indices(seqId)
   }
 
+  /**
+   * Compressed bytes a `getRecordsForRange` over these regions is expected to
+   * download, from the index alone.
+   *
+   * Per region this is `chunksLikelyRead`, not every chunk `blocksForRange`
+   * returns: the difference is the whole point on a long-read file, where a
+   * narrow window inherits every chunk of every overlapping bin and reads a
+   * handful of them. Measured on a 40x ONT BAM (COLO829BL, chr3), summing all
+   * chunks against the bytes the same query really pulls:
+   *
+   * | window | all chunks | actually read | this estimate |
+   * | ------ | ---------- | ------------- | ------------- |
+   * | 380bp  | 43.5MB     | 7.8MB         | 7.8MB         |
+   * | 3.4kb  | 43.5MB     | 7.8MB         | 7.8MB         |
+   * | 100kb  | 46.6MB     | 10.4MB        | 10.4MB        |
+   * | 2Mb    | 155.9MB    | 155.9MB       | 123.8MB       |
+   *
+   * A caller gating on this — jbrowse's "too much data" banner is the one that
+   * exists — was being told 5.6x the truth on exactly the windows a reader
+   * spends their time in, and cannot answer it by zooming: every window narrower
+   * than a linear-index interval resolves to the same chunks and so to the same
+   * number.
+   *
+   * Still summed over merged chunks rather than per region, so two regions
+   * sharing a chunk are charged for it once.
+   */
   async estimatedBytesForRegions(regions: Region[], opts?: BaseOpts) {
+    const indexData = await this.parse(opts)
     const blockResults = await Promise.all(
-      regions.map(r => this.blocksForRange(r.refId, r.start, r.end, opts)),
+      regions.map(async r => {
+        const chunks = await this.blocksForRange(r.refId, r.start, r.end, opts)
+        const refIndex = indexData.indices(r.refId)
+        return refIndex
+          ? chunksLikelyRead(chunks, this.getHighestChunk(refIndex, r.end))
+          : chunks
+      }),
     )
 
     // Deduplicate and merge overlapping blocks across all regions

package/src/util.ts

@@ -29,6 +29,79 @@ export interface BaseOpts {
   onProgress?: (bytesDownloaded: number, totalBytes?: number) => void
 }
 
+// Re-exported so the internal import path stays './util.ts'. The
+// implementation moved to @gmod/shared-read-cache, which needs it anyway --
+// every consumer of that package was carrying an identical copy.
+export { throwIfAborted } from '@gmod/shared-read-cache'
+
+// How many of a query's chunks to read at once. Six is the HTTP/1.1 per-host
+// connection cap browsers enforce, so going much above it buys nothing on the
+// transport that matters and only widens peak memory.
+//
+// It is also the SMALLEST number of chunks any query reads, because
+// `_fetchChunkFeatures` checks its early stop once, after this first batch
+// (ADR 0010) — which is what `chunksLikelyRead` below is built on.
+export const MAX_CONCURRENT_CHUNK_READS = 6
+
+/**
+ * The prefix of `chunks` a query is expected to actually read, for callers that
+ * want to forecast a query's cost without running it.
+ *
+ * `blocksForRange` returns every chunk of every bin overlapping the query, at
+ * every level of the binning scheme, minus the ones the linear index puts
+ * entirely before it. On a long-read file that is wildly more than the query
+ * reads: a coarse bin's chunks run to the end of the bin's span, so a 380bp
+ * window on a deep ONT BAM resolves to 90 chunks / 43.5MB, of which
+ * `getRecordsForRange` reads 6 / 7.8MB before the early stop fires.
+ *
+ * Two bounds, and the answer is the larger:
+ *
+ * - `upperBoundBlockPosition` — the file offset past which a chunk holds no
+ *   record the query wants, from the linear index one window beyond the query
+ *   end. Chunks come back sorted by `minv`, so the ones under it are a prefix.
+ * - `MAX_CONCURRENT_CHUNK_READS` — the first batch is read unconditionally, so
+ *   no query ever costs less than that however tight the upper bound is.
+ *
+ * **This is a forecast, not a bound the reader obeys**, and it is deliberately
+ * only used to describe a query rather than to answer one. The upper bound is
+ * an approximation: a record starting before the query end can sit at a higher
+ * offset than the linear-index entry past it (a long read reaching into the
+ * next window pins that entry low), so pruning a fetch this way would drop
+ * records. A forecast that is occasionally 20% under warns slightly early;
+ * a fetch that is occasionally short returns wrong data.
+ *
+ * An EMPTY prefix is that pin at its worst — the bound has landed at or before
+ * the query's own first chunk, so it orders nothing — and the answer there is
+ * every chunk rather than the batch floor. Without it this returns the floor on
+ * a file whose features are long against the linear-index interval, which is
+ * where the same forecast really does go wrong: ported to tabix-js and measured
+ * on the 1000 Genomes SV callset, whose 1.4Mb deletions pin the entry at the
+ * data start, it forecast 0.04MB against the 0.22MB the query read. On this
+ * corpus the fallback costs one fixture's win (chr22_nanopore, whose 10kb
+ * window keeps summing all 22 chunks) and no accuracy anywhere else.
+ */
+export function chunksLikelyRead(
+  chunks: Chunk[],
+  upperBoundBlockPosition?: number,
+) {
+  if (
+    upperBoundBlockPosition === undefined ||
+    chunks.length <= MAX_CONCURRENT_CHUNK_READS
+  ) {
+    return chunks
+  }
+  let n = 0
+  while (
+    n < chunks.length &&
+    chunks[n]!.minv.blockPosition < upperBoundBlockPosition
+  ) {
+    n++
+  }
+  return n === 0 || n >= chunks.length
+    ? chunks
+    : chunks.slice(0, Math.max(n, MAX_CONCURRENT_CHUNK_READS))
+}
+
 /**
  * Merge and order the chunks a query resolved to.
  *
@@ -39,11 +112,6 @@ export interface BaseOpts {
  * underneath you. (The Chunk objects themselves are never mutated; a merged
  * span produces a new instance.)
  */
-// Re-exported so the internal import path stays './util.ts'. The
-// implementation moved to @gmod/shared-read-cache, which needs it anyway --
-// every consumer of that package was carrying an identical copy.
-export { throwIfAborted } from '@gmod/shared-read-cache'
-
 export function optimizeChunks(chunks: Chunk[], lowest?: OffsetCoords) {
   const n = chunks.length
   if (n === 0) {