@gmod/tabix 3.5.7 → 3.6.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gmod/tabix",
-  "version": "3.5.7",
+  "version": "3.6.1",
   "packageManager": "pnpm@11.15.1",
   "description": "Read Tabix-indexed files, supports both .tbi and .csi indexes",
   "type": "module",
@@ -13,7 +13,7 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/GMOD/tabix-js.git"
+    "url": "git+https://github.com/GMOD/tabix-js.git"
   },
   "author": {
     "name": "Robert Buels",
@@ -29,7 +29,9 @@
     "test": "vitest",
     "clean": "rimraf dist esm",
     "format": "prettier --write .",
+    "format:check": "prettier --check .",
     "lint": "eslint --report-unused-disable-directives --max-warnings 0",
+    "typecheck": "tsc --noEmit -p tsconfig.lint.json",
     "prebuild": "pnpm clean",
     "build:esm": "tsc --outDir esm",
     "build:es5": "tsc --module commonjs --moduleResolution bundler --outDir dist",
@@ -49,25 +51,25 @@
     "genomics"
   ],
   "dependencies": {
-    "@gmod/abortable-promise-cache": "^3.0.4",
     "@gmod/bgzf-filehandle": "^6.3.2",
+    "@gmod/shared-read-cache": "^1.4.4",
     "@jbrowse/quick-lru": "^7.3.5",
-    "generic-filehandle2": "^2.2.0"
+    "generic-filehandle2": "^2.2.1"
   },
   "devDependencies": {
     "@eslint/js": "^10.0.1",
-    "@types/node": "^26.1.1",
+    "@types/node": "^26.1.2",
     "@vitest/coverage-v8": "^4.1.10",
-    "eslint": "^10.7.0",
+    "eslint": "^10.8.0",
     "eslint-plugin-import-x": "^4.17.1",
     "git-cliff": "^2.13.1",
     "prettier": "^3.9.6",
     "rimraf": "^6.1.3",
-    "typescript": "^6.0.0",
+    "typescript": "^6.0.3",
     "typescript-eslint": "^8.65.0",
     "vitest": "^4.1.10",
-    "webpack": "^5.108.4",
-    "webpack-cli": "^7.2.1"
+    "webpack": "^5.109.2",
+    "webpack-cli": "^7.2.2"
   },
   "publishConfig": {
     "access": "public"

package/src/indexFile.ts

@@ -1,6 +1,7 @@
 import { unzip } from '@gmod/bgzf-filehandle'
+import { SharedReadCache } from '@gmod/shared-read-cache'
 
-import { optimizeChunks, throwIfAborted } from './util.ts'
+import { optimizeChunks } from './util.ts'
 
 import type Chunk from './chunk.ts'
 import type VirtualOffset from './virtualOffset.ts'
@@ -51,14 +52,11 @@ export interface IndexData {
 
 export default abstract class IndexFile {
   public filehandle: GenericFilehandle
-  private parseP?: Promise<IndexData>
   /**
-   * The signal `parseP` was started under, while it is still in flight. The
-   * index is parsed once and shared by every query against the file, so without
-   * this the first query to arrive would own a read all the others depend on —
-   * see {@link parse}.
+   * The parsed index, as a shared read — see {@link parse}. One entry, never
+   * evicted, which is what a memo is.
    */
-  private parseSignal?: AbortSignal
+  private parseCache = new SharedReadCache<string, IndexData>({})
 
   constructor({ filehandle }: { filehandle: GenericFilehandle }) {
     this.filehandle = filehandle
@@ -158,78 +156,38 @@ export default abstract class IndexFile {
     return optimizeChunks(chunks, this.lowestOffset(ba, min, indexData))
   }
 
+  // SYNC: ~/src/gmod/bam-js/src/indexFile.ts parse — same shape and the same
+  // reasoning below.
   /**
    * Parse the index, or join the parse already running.
    *
    * The index is downloaded and parsed once for the life of this object, so it
    * is the one read here that is shared between queries — and therefore the one
    * place a cancellation can leak from the query that asked for it to a query
-   * that did not. `_parse` hands `opts` straight to `readIndexBytes`, so
-   * without this the first query to arrive owns a read every other query
-   * depends on: when it pans away, every concurrent query fails with its abort.
+   * that did not. `_parse` hands `opts` straight to `readIndexBytes`, so a bare
+   * memoized promise makes the first query to arrive the owner of a read every
+   * other query depends on: when it pans away, every concurrent query fails
+   * with its abort.
    *
-   * A caller that joined someone else's parse and saw it fail because *they*
-   * aborted starts over rather than inheriting the failure — once, then
-   * propagates. Bounding it at one attempt is what jbrowse's
-   * `RemoteFileWithRangeCache.joinChunk` does with the same retry one layer
-   * down, and for the reason it gives: the pathological case becomes one
-   * duplicate parse rather than a recursion whose depth depends on how the
-   * aborts interleave.
+   * The same cache `ChunkCache` uses, for the same reason and with the same
+   * rule: the parse runs under a signal of its own and is cancelled only once
+   * every caller waiting on it has given up, so one query's abort is reported
+   * to that query alone and a bystander gets the parse already in flight rather
+   * than having to re-read the index. A rejection is dropped rather than
+   * cached, so a transient failure does not poison the index for the life of
+   * the file.
    *
-   * A retry rather than the reference count `chunkCache` gets from
-   * `@gmod/abortable-promise-cache`, because the index is parsed once for the
-   * life of the object: there is no repeated waste to recover, and this is a
-   * dozen lines against restructuring the memo. `@gmod/bam`'s `IndexFile` and
-   * `@gmod/cram`'s `CraiIndex` make the same split for the same reason.
+   * The fill is per call rather than on the cache so that the caller who starts
+   * the parse has its `onProgress` reach `readIndexBytes` — the index is a
+   * whole-file read, and a determinate "downloading index" bar is what that
+   * callback exists for.
    *
    * @internal
    */
-  async parse(opts: Options = {}, retried = false): Promise<IndexData> {
-    throwIfAborted(opts.signal)
-    const pending = this.parseP
-    if (!pending) {
-      return this.startParse(opts)
-    }
-
-    // read before awaiting: the owner is forgotten as soon as the parse settles
-    const ownerSignal = this.parseSignal
-    try {
-      return await pending
-    } catch (e) {
-      if (retried || !ownerSignal?.aborted || opts.signal?.aborted) {
-        throw e
-      }
-      return this.parse(opts, true)
-    }
-  }
-
-  private startParse(opts: Options) {
-    const pending = this._parse(opts)
-    this.parseP = pending
-    this.parseSignal = opts.signal
-    // Drop a rejection rather than keeping it, so one transient failure does not
-    // poison the index for the lifetime of the file. Identity-checked so a retry
-    // started after this settles is not cleared by the attempt it replaced.
-    //
-    // Written as one try/catch rather than `.then(onFulfilled, onRejected)`
-    // because `unicorn/prefer-then-catch` rewrites the two-argument form to
-    // `.then(...).catch(...)`, which is not the same thing — that catch would
-    // also swallow anything the fulfilment handler threw.
-    void (async () => {
-      let failed = false
-      try {
-        await pending
-      } catch {
-        failed = true
-      }
-      if (this.parseP === pending) {
-        if (failed) {
-          this.parseP = undefined
-        }
-        this.parseSignal = undefined
-      }
-    })()
-    return pending
+  parse(opts: Options = {}): Promise<IndexData> {
+    return this.parseCache.get('index', opts.signal, signal =>
+      this._parse({ ...opts, signal }),
+    )
   }
 
   /** @internal */

package/src/tabixIndexedFile.ts

@@ -1,5 +1,5 @@
-import AbortablePromiseCache from '@gmod/abortable-promise-cache'
 import { unzip, unzipChunkSlice } from '@gmod/bgzf-filehandle'
+import { SharedReadCache } from '@gmod/shared-read-cache'
 import { LocalFile, RemoteFile } from 'generic-filehandle2'
 
 import CSI from './csi.ts'
@@ -29,105 +29,28 @@ const MAX_READ_AHEAD_CHUNKS = 6
 // a dense VCF (test/data/1kg.chr1.subset.vcf.gz — 213MB over 600kb of chr1)
 // has single index bins of 17MB compressed, 120MB decompressed. Panning it
 // under the old 80-entry cache peaked at 2GB RSS.
-const DEFAULT_CHUNK_CACHE_BYTES = 100 * 2 ** 20
-
-// The entry type AbortablePromiseCache stores in its backing cache. Not exported
-// by the package, so it is recovered from the constructor signature rather than
-// restated (and left to drift) here.
-type CacheEntry = NonNullable<
-  ReturnType<
-    ConstructorParameters<
-      typeof AbortablePromiseCache<Chunk, ReadChunk>
-    >[0]['cache']['get']
-  >
->
-
-interface CachedChunk {
-  entry: CacheEntry
-  /** 0 until the read settles and the decompressed size is known */
-  bytes: number
-}
-
-/**
- * Backing store for the chunk cache, bounded by the decompressed size of the
- * chunks it holds rather than by entry count.
- *
- * A chunk's size is only known once its read settles, so `set` records the
- * entry immediately and charges it to the budget later. Unsettled entries are
- * therefore free, which is what we want: they are reads a query is waiting on.
- */
-class ByteBoundedChunkCache {
-  private entries = new Map<string, CachedChunk>()
-  private bytes = 0
-  private maxBytes: number
-
-  constructor(maxBytes: number) {
-    this.maxBytes = maxBytes
-  }
-
-  get byteSize() {
-    return this.bytes
-  }
-
-  get size() {
-    return this.entries.size
-  }
-
-  has(key: string) {
-    return this.entries.has(key)
-  }
-
-  get(key: string) {
-    const cached = this.entries.get(key)
-    if (cached) {
-      // re-insert so Map iteration order stays least-recently-used first
-      this.entries.delete(key)
-      this.entries.set(key, cached)
-    }
-    return cached?.entry
-  }
-
-  set(key: string, entry: CacheEntry) {
-    this.delete(key)
-    const cached = { entry, bytes: 0 }
-    this.entries.set(key, cached)
-    void entry.promise
-      .then(chunk => {
-        // a later set() may have replaced this key while the read was in
-        // flight; charging these bytes to it would then double-count
-        if (this.entries.get(key) === cached) {
-          cached.bytes = chunk.buffer.byteLength
-          this.bytes += cached.bytes
-          this.evict()
-        }
-      })
-      .catch(() => {
-        // a failed or aborted read caches nothing and costs nothing
-      })
-  }
-
-  delete(key: string) {
-    const cached = this.entries.get(key)
-    if (cached) {
-      this.entries.delete(key)
-      this.bytes -= cached.bytes
-    }
-  }
-
-  keys() {
-    return this.entries.keys()
-  }
+//
+// That 120MB figure is also why this is no longer 100MB. A budget below one
+// query's working set does not cache less, it caches NOTHING: each entry is
+// evicted before the next pan can reuse it, so the hit rate is zero and the
+// decompress is paid again every time. On that same fixture, a six-window 50kb
+// pan measured 17 refills out of 17 — a total miss — at 100MB, against 0 at
+// 800MB, and 2596ms against 600ms. The working set plateaus at 497MB held, so
+// 1GB clears it with headroom and nothing above 800MB buys anything.
+//
+// Affordable as a ceiling only because of the idle timeout below: it is a peak
+// under panning, not a level a parked consumer holds. A small file is
+// unaffected either way, this being a ceiling and not an allocation.
+const DEFAULT_CHUNK_CACHE_BYTES = 1024 * 2 ** 20
 
-  // Evict from the least-recently-used end. The size > 1 guard means a single
-  // chunk larger than the whole budget is still kept: the caller needs it for
-  // the query in flight, and dropping it would only force a re-decompress.
-  private evict() {
-    const lru = this.entries.keys()
-    while (this.bytes > this.maxBytes && this.entries.size > 1) {
-      this.delete(lru.next().value!)
-    }
-  }
-}
+// SYNC: ~/src/gmod/bam-js/src/bamFile.ts DEFAULT_CACHE_IDLE_TIMEOUT_MS
+//
+// Drop a chunk nothing has looked at for three minutes. The budget above is
+// only applied when a read settles, so it does nothing at all for a consumer
+// sitting still — and a genome browser holds one of these per track for as long
+// as the track is open. Timed from the last read, not the fetch, so panning
+// back and forth over one region never expires it.
+const DEFAULT_CHUNK_CACHE_IDLE_TIMEOUT_MS = 3 * 60 * 1000
 
 type GetLinesCallback = (
   line: string,
@@ -353,8 +276,15 @@ function parseIntFromBytes(buffer: Uint8Array, start: number, end: number) {
 export default class TabixIndexedFile {
   private filehandle: GenericFilehandle
   private index: IndexFile
-  private chunkCache: AbortablePromiseCache<Chunk, ReadChunk>
-  private headerP?: Promise<{ header: string; skippedLines: string[] }>
+  public chunkCache: SharedReadCache<Chunk, ReadChunk>
+  /**
+   * The parsed header, as a shared read — see {@link getParsedHeader}. One
+   * entry, never evicted, which is what a memo is.
+   */
+  private headerCache = new SharedReadCache<
+    string,
+    { header: string; skippedLines: string[] }
+  >({})
 
   constructor({
     path,
@@ -367,6 +297,7 @@ export default class TabixIndexedFile {
     csiUrl,
     csiFilehandle,
     chunkCacheSize = DEFAULT_CHUNK_CACHE_BYTES,
+    chunkCacheIdleTimeoutMs = DEFAULT_CHUNK_CACHE_IDLE_TIMEOUT_MS,
   }: {
     path?: string
     filehandle?: GenericFilehandle
@@ -377,8 +308,24 @@ export default class TabixIndexedFile {
     csiPath?: string
     csiUrl?: string
     csiFilehandle?: GenericFilehandle
-    /** budget for the decompressed chunk cache, in bytes */
+    /**
+     * Budget for the decompressed chunk cache, in bytes. Default 1GB.
+     *
+     * A retention bound, not a bound on peak memory: reads in flight are never
+     * evicted and the last settled entry is kept whatever the budget. Size it
+     * to hold several queries — below one query's working set the hit rate
+     * drops to zero while the memory is retained anyway, so a number between
+     * the two is the worst available choice.
+     */
     chunkCacheSize?: number
+    /**
+     * Drop a cached chunk once nothing has read it for this many milliseconds.
+     * Default 3 minutes; `0` keeps chunks until `chunkCacheSize` evicts them.
+     *
+     * The only thing that lowers the cache while nothing is happening, and what
+     * makes the budget above a peak rather than a resting level.
+     */
+    chunkCacheIdleTimeoutMs?: number
   }) {
     this.filehandle = resolveFilehandle(filehandle, path, url)
     this.index = resolveIndex({
@@ -392,13 +339,29 @@ export default class TabixIndexedFile {
       url,
     })
 
-    this.chunkCache = new AbortablePromiseCache<Chunk, ReadChunk>({
-      cache: new ByteBoundedChunkCache(chunkCacheSize),
-      fill: (args: Chunk, signal?: AbortSignal) =>
-        this.readChunk(args, { signal }),
+    this.chunkCache = new SharedReadCache<Chunk, ReadChunk>({
+      maxSize: chunkCacheSize,
+      // decompressed bytes, not entry count: we fetch compressed and cache
+      // decompressed, and an entry is a whole chunk
+      sizeOf: read => read.buffer.byteLength,
+      cacheKey: chunk => chunk.toString(),
+      idleTimeoutMs: chunkCacheIdleTimeoutMs,
+      fill: (chunk, signal) => this.readChunk(chunk, { signal }),
     })
   }
 
+  /**
+   * Drops every decompressed chunk held by the cache, and stops the idle sweep
+   * until something is cached again.
+   *
+   * `chunkCacheIdleTimeoutMs` reclaims a view the user has wandered away from;
+   * this is for a consumer that knows it is finished — a closed track, a
+   * changed assembly — and should not have to wait it out.
+   */
+  clearChunkCache() {
+    this.chunkCache.clear()
+  }
+
   /**
    * Estimates the compressed byte size of the index chunks covering the given
    * regions. Useful for byte budgeting before issuing a `getLines` call to
@@ -505,7 +468,7 @@ export default class TabixIndexedFile {
     const ensureReadsStarted = (count: number) => {
       while (reads.length < Math.min(count, chunks.length)) {
         const c = chunks[reads.length]!
-        const read = this.chunkCache.get(c.toString(), c, signal)
+        const read = this.chunkCache.get(c, signal)
         void read.catch(() => {
           // a prefetch the early return skips is never awaited, so swallow its
           // rejection here rather than let it surface unhandled
@@ -670,12 +633,32 @@ export default class TabixIndexedFile {
     }
   }
 
-  private async getParsedHeader(opts: Options = {}) {
-    this.headerP ??= this.parseHeader(opts).catch((error: unknown) => {
-      this.headerP = undefined
-      throw error
-    })
-    return this.headerP
+  /**
+   * Parse the header, or join the parse already running.
+   *
+   * `parseHeader` threads `opts` into both `getMetadata` and the header read,
+   * so memoizing it on the first caller's opts put that caller's signal in
+   * charge of a read every later caller joins: when it aborted, they failed
+   * with its cancellation, their own signals untouched.
+   *
+   * The same cache the chunk reads use, and the same rule: the parse runs under
+   * a signal of its own and is cancelled only once every caller waiting on it
+   * has given up, so one caller's abort is reported to that caller alone and a
+   * bystander gets the parse already in flight. There is no retry here because
+   * there is nothing to retry — the parse a bystander joined is not cancelled by
+   * someone else's abort. A rejection is dropped rather than cached, so a
+   * transient failure does not poison the header for the life of the file.
+   *
+   * The fill is per call rather than on the cache, so the caller that starts the
+   * parse has its `onProgress` reach the index download inside `getMetadata`.
+   *
+   * SYNC: ~/src/gmod/bam-js/src/bamFile.ts getHeader — same shape for the same
+   * reason, on the header rather than the index.
+   */
+  private getParsedHeader(opts: Options = {}) {
+    return this.headerCache.get('header', opts.signal, signal =>
+      this.parseHeader({ ...opts, signal }),
+    )
   }
 
   /**

package/src/util.ts

@@ -5,36 +5,10 @@ import { longFromBytesToUnsigned } from './long.ts'
 import VirtualOffset from './virtualOffset.ts'
 
 // SYNC: ~/src/gmod/bam-js/src/util.ts optimizeChunks
-/**
- * `signal.throwIfAborted()`, without requiring either that method or `reason`.
- *
- * Two reasons not to call the built-in directly. It assumes a *real*
- * `AbortSignal`, and callers pass duck-typed ones, where calling a missing
- * method is a `TypeError` rather than the cancellation the caller asked for —
- * a strictly worse failure.
- *
- * And it sets a browser floor. `AbortSignal.prototype.throwIfAborted` and
- * `AbortSignal.reason` are Safari 15.4 / Chrome 100 / Firefox 97 (March 2022),
- * higher than anything else here needs: this package otherwise touches only
- * `.aborted`, and `generic-filehandle2` only forwards a signal to `fetch`.
- *
- * Faithful to the spec otherwise: an aborted signal throws its `reason`
- * whatever that is, and only synthesizes an `AbortError` when there is none.
- * Kept in sync with the copy in `@gmod/bam`'s `src/util.ts`.
- */
-export function throwIfAborted(signal?: AbortSignal) {
-  if (signal?.aborted) {
-    const reason: unknown = signal.reason
-    // Spec-faithful: throwIfAborted throws `reason` verbatim, and `reason` is
-    // whatever the caller passed to abort() — `controller.abort('too slow')`
-    // makes it a string. Coercing it to an Error here would hide that from a
-    // consumer who set it deliberately.
-    // eslint-disable-next-line @typescript-eslint/only-throw-error
-    throw reason === undefined
-      ? new DOMException('This operation was aborted', 'AbortError')
-      : reason
-  }
-}
+// Re-exported so the internal import path stays './util.ts'. The
+// implementation moved to @gmod/shared-read-cache, which needs it anyway --
+// every consumer of that package was carrying an identical copy.
+export { throwIfAborted } from '@gmod/shared-read-cache'
 
 export function optimizeChunks(chunks: Chunk[], lowest?: VirtualOffset) {
   const n = chunks.length