@gmod/hic 1.0.0 → 1.2.0

package/src/hicFile.ts

@@ -1,5 +1,5 @@
+import { inflateRawUnknownSize } from '@gmod/inflate'
 import { LocalFile } from 'generic-filehandle2'
-import { inflate } from 'pako-esm2'
 
 import { binWindow } from './binWindow.ts'
 import BinaryParser from './binary.ts'
@@ -16,6 +16,8 @@ import type {
   Chromosome,
   HicMetadata,
   HicRegion,
+  ProgressCallback,
+  ProgressOpts,
   Reader,
 } from './types.ts'
 import type { GenericFilehandle } from 'generic-filehandle2'
@@ -51,6 +53,13 @@ export interface HicConfig {
    * costs on a pre-v9 file.
    */
   nvi?: string
+  /**
+   * Ceiling on the decompressed contacts the block cache holds, in bytes.
+   * Defaults to 128 MB. Lower it where the budget is tight and re-reading is
+   * cheaper than the memory; raising it buys nothing once a fetch's working set
+   * fits, since the entry cap bounds the cache too.
+   */
+  blockCacheMaxBytes?: number
 }
 
 // Cached blocks are the decompressed records and nothing else. hic-straw also
@@ -112,6 +121,11 @@ const BLOCK_CACHE_MAX_ENTRIES = 1024
 // binsize. Measured on the file above: 0.05 MB at 2.5 Mb, 0.23 MB at 100 kb. So
 // the entry cap now tracks the working set and this tracks the memory, instead
 // of one number answering both badly. See LRU's WeightOpts.
+//
+// This is the one capacity here a caller can override (`blockCacheMaxBytes`),
+// because it is the only one that bounds MEMORY. The two entry caps below are
+// working-set sizes derived from how many region pairs a fetch runs at once,
+// which the library knows and the caller would only have to rederive.
 const BLOCK_CACHE_MAX_BYTES = 128 * 1024 * 1024
 // One entry per (chrIdx1, chrIdx2), so the working set is the pair count.
 // 512 covers a 31-region view; human whole-genome is 300. A matrix holds the
@@ -159,10 +173,9 @@ export class HicFile {
   private matrixCache = new LRU<string, Promise<Matrix | undefined>>(
     MATRIX_CACHE_SIZE,
   )
-  private blockCache = new LRU<string, Block>(BLOCK_CACHE_MAX_ENTRIES, {
-    maxBytes: BLOCK_CACHE_MAX_BYTES,
-    weigh: blockCacheWeight,
-  })
+  // Assigned in the constructor rather than here, since its byte budget is
+  // configurable and a field initializer cannot see `config`.
+  private blockCache: LRU<string, Block>
   private normVectorIndexPosition = -1
   private normVectorIndexSize = -1
 
@@ -185,8 +198,12 @@ export class HicFile {
   private meta: HicMetadata | undefined
 
   constructor(config: HicConfig) {
-    const { filehandle, path, reader } = config
+    const { filehandle, path, reader, blockCacheMaxBytes } = config
     this.config = config
+    this.blockCache = new LRU<string, Block>(BLOCK_CACHE_MAX_ENTRIES, {
+      maxBytes: blockCacheMaxBytes ?? BLOCK_CACHE_MAX_BYTES,
+      weigh: blockCacheWeight,
+    })
     if (reader) {
       this.file = reader
     } else if (filehandle) {
@@ -215,7 +232,7 @@ export class HicFile {
     return this.meta!
   }
 
-  async readHeaderAndFooter() {
+  private async readHeaderAndFooter() {
     // `init` clears its memoized promise on failure so a transient read error
     // retries, and this parse appends as it goes — so a retry has to start from
     // empty or a failure part-way through the chromosome loop leaves the second
@@ -301,7 +318,7 @@ export class HicFile {
     }
   }
 
-  async readFooter() {
+  private async readFooter() {
     const skip = this.version < 9 ? 8 : 12
     let data = await this.file.read(this.footerPosition, skip)
 
@@ -347,7 +364,7 @@ export class HicFile {
     return p
   }
 
-  async readMatrix(chrIdx1: number, chrIdx2: number) {
+  private async readMatrix(chrIdx1: number, chrIdx2: number) {
     await this.init()
 
     const idx = this.masterIndex[Matrix.getKey(chrIdx1, chrIdx2)]
@@ -365,6 +382,7 @@ export class HicFile {
     region2: HicRegion,
     units: string,
     binsize: number,
+    opts?: ProgressOpts,
   ) {
     await this.init()
 
@@ -410,7 +428,10 @@ export class HicFile {
     // offset its values start at.
     const [norm, blocks] = await Promise.all([
       this.getNormVectors(normalization, r1, r2, units, binsize),
-      this.getBlocks(r1, r2, binsize),
+      // The measurable half of the pair: blocks are counted and the two norm
+      // vectors are not, because the vector chain is two hops whatever the
+      // query and the block chain is the one that grows with the region.
+      this.getBlocks(r1, r2, binsize, opts),
     ])
 
     // Sum of the blocks' record counts bounds the survivors, so the output is
@@ -540,7 +561,12 @@ export class HicFile {
     return result
   }
 
-  async getBlocks(region1: HicRegion, region2: HicRegion, binSize: number) {
+  async getBlocks(
+    region1: HicRegion,
+    region2: HicRegion,
+    binSize: number,
+    opts?: ProgressOpts,
+  ) {
     const blockKey = (blockNumber: number, zd: MatrixZoomData) =>
       `${zd.getKey()}_${blockNumber}`
 
@@ -580,10 +606,28 @@ export class HicFile {
           }
         }
 
+        // Cached blocks are already done, not absent: a pan that reuses most
+        // of its blocks reads most of the way along rather than restarting at
+        // zero, which is what the bar should say — the work left is the reads.
+        //
+        // A query covering no blocks at all reports nothing rather than `0, 0`:
+        // there is no fraction in it, and a caller dividing would get NaN.
+        const { onProgress } = opts ?? {}
+        let done = blocks.length
+        if (blockNumbers.length > 0) {
+          onProgress?.(done, blockNumbers.length)
+        }
+        // Still one `Promise.all`, so every block is still issued in one wave:
+        // what a remote file pays is round-trip DEPTH, and counting the
+        // completions must not turn one wave into a queue. `readChainDepth`
+        // pins that.
         const newBlocks = await Promise.all(
-          blockNumbersToQuery.map(blockNumber =>
-            this.readBlock(blockNumber, zd),
-          ),
+          blockNumbersToQuery.map(async blockNumber => {
+            const block = await this.readBlock(blockNumber, zd)
+            done++
+            onProgress?.(done, blockNumbers.length)
+            return block
+          }),
         )
         for (const block of newBlocks) {
           if (block) {
@@ -602,7 +646,14 @@ export class HicFile {
     let block: Block | undefined
     if (idx) {
       const data = await this.file.read(idx.filePosition, idx.size)
-      const plain = inflate(new Uint8Array(data), {}) as Uint8Array
+      // `.subarray(2)` drops the zlib header: libdeflate's raw-deflate path is
+      // the fast one, and a `.hic` records only a block's *compressed* size, so
+      // there is no known output size to hand the exact-size entry point.
+      // `inflateRawUnknownSize` guesses 4x the input, which clears this
+      // format's ~2.3x ratio in one pass.
+      const plain = await inflateRawUnknownSize(
+        new Uint8Array(data).subarray(2),
+      )
       const parser = new BinaryParser(
         new DataView(plain.buffer, plain.byteOffset, plain.byteLength),
       )
@@ -756,7 +807,7 @@ export class HicFile {
     return new NormalizationVector(this.file, filePosition, nValues, dataType)
   }
 
-  async getNormVectorIndex() {
+  async getNormVectorIndex(opts?: ProgressOpts) {
     // Before the version gate, not after it: `version` is 0 until the header is
     // parsed, so calling this first — which `getNormalizationOptions` on its own
     // does — answered "no index" for every file rather than for a v5 one. A
@@ -771,18 +822,21 @@ export class HicFile {
       // sequential range reads, only to rediscover there is nothing there.
       // Cleared on failure (like `init`) so a transient read error retries
       // rather than caching a rejection forever.
-      this.normVectorIndexP ??= this.loadNormVectorIndex().catch(
-        (e: unknown) => {
-          this.normVectorIndexP = undefined
-          throw e
-        },
-      )
+      // The walk runs once, so its progress belongs to the call that performs
+      // it: a caller joining an in-flight or finished load is not waiting on
+      // reads and is told nothing, which is the truth rather than a silence.
+      this.normVectorIndexP ??= this.loadNormVectorIndex(
+        opts?.onProgress,
+      ).catch((e: unknown) => {
+        this.normVectorIndexP = undefined
+        throw e
+      })
       await this.normVectorIndexP
     }
     return this.normVectorIndex
   }
 
-  private async loadNormVectorIndex() {
+  private async loadNormVectorIndex(onProgress?: ProgressCallback) {
     // If we know the position of the norm vector index, read it directly.
     // This is the case for hic v9 files.
     if (this.normVectorIndexPosition > 0 && this.normVectorIndexSize > 0) {
@@ -798,7 +852,7 @@ export class HicFile {
       })
     } else {
       try {
-        await this.readNormExpectedValuesAndNormVectorIndex()
+        await this.readNormExpectedValuesAndNormVectorIndex(onProgress)
       } catch (e) {
         // Not "expected if the file has no norm vectors" — that case never
         // arrives here. hic-straw's own IO threw a 416 for a read past EOF and
@@ -812,14 +866,14 @@ export class HicFile {
     }
   }
 
-  async getNormalizationOptions() {
+  async getNormalizationOptions(opts?: ProgressOpts) {
     // Normalization options are computed as a side effect of loading the
     // index. A bit ugly but alternatives are worse.
-    await this.getNormVectorIndex()
+    await this.getNormVectorIndex(opts)
     return this.normalizationTypes
   }
 
-  async readNormVectorIndex(range: { start: number; size: number }) {
+  private async readNormVectorIndex(range: { start: number; size: number }) {
     await this.init()
     const data = await this.file.read(range.start, range.size)
     const binaryParser = new BinaryParser(new DataView(data))
@@ -833,11 +887,14 @@ export class HicFile {
 
   // Used when the position of the norm vector index is unknown: read through
   // the expected values to find the index.
-  async readNormExpectedValuesAndNormVectorIndex() {
+  private async readNormExpectedValuesAndNormVectorIndex(
+    onProgress?: ProgressCallback,
+  ) {
     await this.init()
     if (this.normExpectedValueVectorsPosition !== undefined) {
       const nviStart = await this.skipExpectedValues(
         this.normExpectedValueVectorsPosition,
+        onProgress,
       )
       let byteCount = INT
 
@@ -875,7 +932,10 @@ export class HicFile {
 
   // Used when the position of the norm vector index is unknown: skip the
   // normalized expected values to find the index.
-  async skipExpectedValues(start: number) {
+  private async skipExpectedValues(
+    start: number,
+    onProgress?: ProgressCallback,
+  ) {
     const version = this.version
     const file = new BufferedFile({ file: this.file, size: 256000 })
     const data = await file.read(start, INT)
@@ -892,12 +952,17 @@ export class HicFile {
     const binaryParser = new BinaryParser(new DataView(data))
     const nEntries = binaryParser.getInt() // Total # of expected value chunks
 
-    const parseNext = async (
-      chunkStart: number,
-      remaining: number,
-    ): Promise<number> => {
+    // Skip one chunk, answering where the next one starts.
+    //
+    // Its two reads land a whole expected-value vector apart — one per bin of
+    // the genome, so tens of MB at a fine binsize — which is further than the
+    // buffer reaches. On a remote file that is two round trips per chunk that
+    // nothing can merge, and it is why this walk, not the index it is looking
+    // for, is the slow part of opening a pre-v9 file. It is also why the chunk
+    // is the unit worth counting: `nEntries` of them, known before the first
+    // read, each costing about the same.
+    const skipChunk = async (chunkStart: number) => {
       let chunkSize = 0
-      const p0 = chunkStart
 
       let buf = await file.read(chunkStart, 500)
       let parser = new BinaryParser(new DataView(buf))
@@ -913,15 +978,26 @@ export class HicFile {
       chunkSize +=
         INT + nChrScaleFactors * (INT + (version < 9 ? DOUBLE : FLOAT))
 
-      return remaining - 1 === 0
-        ? p0 + chunkSize
-        : parseNext(p0 + chunkSize, remaining - 1)
+      return chunkStart + chunkSize
     }
 
-    return nEntries === 0 ? start + INT : parseNext(start + INT, nEntries)
+    // A loop rather than the tail recursion this was, because the position is
+    // the only thing carried between chunks and a counter now rides along with
+    // it. Chunks must be walked in order — each one's size is only known once
+    // the one before it has been parsed — so the awaits are sequential by
+    // nature, not by oversight.
+    if (nEntries > 0) {
+      onProgress?.(0, nEntries)
+    }
+    let position = start + INT
+    for (let done = 0; done < nEntries; done++) {
+      position = await skipChunk(position)
+      onProgress?.(done + 1, nEntries)
+    }
+    return position
   }
 
-  parseNormVectorEntry(binaryParser: BinaryParser) {
+  private parseNormVectorEntry(binaryParser: BinaryParser) {
     const type = binaryParser.getString() // 15
     const chrIdx = binaryParser.getInt() // 4
     const unit = binaryParser.getString() // 3

package/src/index.ts

@@ -7,6 +7,8 @@ export type {
   Chromosome,
   HicMetadata,
   HicRegion,
+  ProgressCallback,
+  ProgressOpts,
   Reader,
   Zoom,
 } from './types.ts'

package/src/types.ts

@@ -9,6 +9,31 @@ export interface Reader {
   read: (position: number, length: number) => Promise<ArrayBuffer>
 }
 
+/**
+ * Progress within one phase: `current` of `total` units done, called as the
+ * phase advances and once with `current` 0 before it starts, so a caller can
+ * show a determinate bar from the first moment rather than after the first
+ * unit lands.
+ *
+ * The unit is whatever that phase measures — expected-value chunks for the
+ * normalization-index walk, blocks for a contact fetch — and a caller renders
+ * `current / total` without needing to know which. `total` never changes within
+ * a call, and `current` never goes backwards.
+ *
+ * A phase with nothing to do does not call this at all, rather than calling it
+ * with a `total` of 0 that a caller could only divide by.
+ */
+export type ProgressCallback = (current: number, total: number) => void
+
+/**
+ * Progress reporting for one call. Per call rather than per file: progress
+ * belongs to the operation a caller is waiting on, and a file-wide callback
+ * could not say which of several concurrent fetches it was describing.
+ */
+export interface ProgressOpts {
+  onProgress?: ProgressCallback
+}
+
 export interface Chromosome {
   index: number
   name: string