@gmod/hic 1.3.2 → 2.1.0

package/src/index.ts

@@ -6,6 +6,7 @@ export type { ContactRecords } from './contactRecords.ts'
 export type {
   BaseOpts,
   Chromosome,
+  ExpectedValues,
   HicMetadata,
   HicRegion,
   ProgressCallback,

package/src/matrixZoomData.ts

@@ -7,6 +7,8 @@ export default class MatrixZoomData {
   chr1: Chromosome
   chr2: Chromosome
   zoom: Zoom
+  /** the raw contacts summed over the whole map at this binsize */
+  sumCounts: number
   blockBinCount: number
   blockColumnCount: number
   blockIndex: Record<number, BlockIndexEntry>
@@ -15,6 +17,7 @@ export default class MatrixZoomData {
     chr1: Chromosome,
     chr2: Chromosome,
     zoom: Zoom,
+    sumCounts: number,
     blockBinCount: number,
     blockColumnCount: number,
     blockIndex: Record<number, BlockIndexEntry>,
@@ -22,6 +25,7 @@ export default class MatrixZoomData {
     this.chr1 = chr1
     this.chr2 = chr2
     this.zoom = zoom
+    this.sumCounts = sumCounts
     this.blockBinCount = blockBinCount
     this.blockColumnCount = blockColumnCount
     this.blockIndex = blockIndex
@@ -104,7 +108,7 @@ export default class MatrixZoomData {
   ) {
     const unit = dis.getString()
     const zoomIndex = dis.getInt()
-    dis.getFloat() // sumCounts
+    const sumCounts = dis.getFloat()
     dis.getFloat() // occupiedCellCount
     dis.getFloat() // stdDev
     dis.getFloat() // percent95
@@ -125,6 +129,7 @@ export default class MatrixZoomData {
       chr1,
       chr2,
       { index: zoomIndex, unit, binSize },
+      sumCounts,
       blockBinCount,
       blockColumnCount,
       blockIndex,

package/src/normalizationVector.ts

@@ -4,9 +4,16 @@ import type { BaseOpts, Reader } from './types.ts'
 
 const DOUBLE = 8
 
+interface CachedValues {
+  start: number
+  end: number
+  values: Promise<Float64Array>
+}
+
 export default class NormalizationVector {
-  private cache:
-    { start: number; end: number; values: Float64Array } | undefined
+  // Holds the in-flight read, so the two concurrent `getValues` calls of a
+  // same-chromosome pair share one read instead of both missing.
+  private cache: CachedValues | undefined
 
   private file: Reader
   private filePosition: number
@@ -25,50 +32,77 @@ export default class NormalizationVector {
     this.dataType = dataType
   }
 
-  async getValues(start: number, end: number, opts?: BaseOpts) {
-    // The read below clamps to `nValues`, so the cache can never cover past it.
-    // Compare against the clamped bound or a request that runs off the end of
-    // the vector — reachable when the assembly's refseq is longer than the size
-    // the `.hic` recorded for that chromosome — leaves `end > cache.end` true
-    // forever, and every call re-reads the same range from the file.
+  async getValues(
+    start: number,
+    end: number,
+    opts?: BaseOpts,
+  ): Promise<Float64Array> {
+    // The read clamps to `nValues`, so the cache can never cover past it.
+    // Comparing against the clamped bound keeps a request running off the end
+    // of the vector (an assembly refseq longer than the `.hic` recorded) from
+    // missing forever.
     const wanted = Math.min(end, this.nValues)
-    if (!this.cache || start < this.cache.start || wanted > this.cache.end) {
-      const adjustedEnd = Math.min(this.nValues, end + 1000)
-      // Clamped against `adjustedEnd`, and not merely against zero, or a window
-      // starting past the vector produces a NEGATIVE `n` — and a negative read
-      // length reaches the reader as `new Uint8Array(-8)`, a RangeError that
-      // fails the whole fetch instead of the empty answer the caller below
-      // already handles. The padding is what sets how far past is far enough:
-      // more than 1000 bins, which at a fine binsize is not far at all.
-      //
-      // Reachable exactly where the comment above says: the assembly's refseq
-      // is longer than the size the `.hic` recorded for that chromosome, so a
-      // view of the tail of it asks for bins the vector never had.
-      const adjustedStart = Math.min(Math.max(0, start - 1000), adjustedEnd)
-      const startPosition = this.filePosition + adjustedStart * this.dataType
-      const n = adjustedEnd - adjustedStart
-      // Nothing to read when the window is entirely past the vector, and on a
-      // remote file a zero-length range request is still a round trip — one per
-      // region pair, on every fetch, for as long as the view stays there.
-      const data =
-        n > 0
-          ? await this.file.read(startPosition, n * this.dataType, opts)
-          : new ArrayBuffer(0)
-      const parser = new BinaryParser(new DataView(data))
-      const values = new Float64Array(n)
-      for (let i = 0; i < n; i++) {
-        values[i] =
-          this.dataType === DOUBLE ? parser.getDouble() : parser.getFloat()
+    let cache = this.cache
+    const joined = !!cache && start >= cache.start && wanted <= cache.end
+    if (!cache || !joined) {
+      cache = this.readRange(start, end, opts)
+      this.cache = cache
+      const pending = cache
+      pending.values.catch(() => {
+        if (this.cache === pending) {
+          this.cache = undefined
+        }
+      })
+    }
+    let values: Float64Array
+    try {
+      values = await cache.values
+    } catch (e) {
+      // The pair that started a joined read passed its own signal, so that
+      // pair aborting says nothing about this caller.
+      if (!joined || opts?.signal?.aborted) {
+        throw e
       }
-      this.cache = { start: adjustedStart, end: adjustedEnd, values }
+      return this.getValues(start, end, opts)
+    }
+
+    // A view, not a copy. Both this and `subarray` clamp a request running past
+    // the end; an out-of-range index reads undefined, the product goes NaN, and
+    // the record filter drops it.
+    const sliceStart = start - cache.start
+    return values.subarray(sliceStart, sliceStart + (end - start))
+  }
+
+  private readRange(start: number, end: number, opts?: BaseOpts): CachedValues {
+    const adjustedEnd = Math.min(this.nValues, end + 1000)
+    // Clamped against `adjustedEnd` too, or a window starting past the vector
+    // gives a negative read length, which the reader turns into a RangeError
+    // instead of the empty answer the caller handles.
+    const adjustedStart = Math.min(Math.max(0, start - 1000), adjustedEnd)
+    const n = adjustedEnd - adjustedStart
+    return {
+      start: adjustedStart,
+      end: adjustedEnd,
+      values: this.readValues(adjustedStart, n, opts),
     }
+  }
 
-    // A view, not a copy: the caller only reads it, and a fetch asks for one of
-    // these per chromosome per region pair. Both this and `slice` clamp a
-    // request running past the end to what exists, which the caller already
-    // handles — an out-of-range index reads undefined, making the product NaN,
-    // which the record filter drops.
-    const sliceStart = start - this.cache.start
-    return this.cache.values.subarray(sliceStart, sliceStart + (end - start))
+  private async readValues(start: number, n: number, opts?: BaseOpts) {
+    // A zero-length range request on a remote file is still a round trip.
+    if (n <= 0) {
+      return new Float64Array(0)
+    }
+    const data = await this.file.read(
+      this.filePosition + start * this.dataType,
+      n * this.dataType,
+      opts,
+    )
+    const parser = new BinaryParser(new DataView(data))
+    const values = new Float64Array(n)
+    for (let i = 0; i < n; i++) {
+      values[i] =
+        this.dataType === DOUBLE ? parser.getDouble() : parser.getFloat()
+    }
+    return values
   }
 }

package/src/types.ts

@@ -68,6 +68,18 @@ export interface BlockIndexEntry {
   size: number
 }
 
+/**
+ * The expected contact count for one region pair, the denominator of
+ * observed/expected. Within a chromosome it depends only on the distance in
+ * bins, so it is the expected-value vector's entries over the distances the
+ * pair spans: `values[|bin2 - bin1| - firstDistance]`, already divided by the
+ * chromosome's scale factor and holding the vector's last value past its end.
+ * Between chromosomes it is one number, the map's average contact.
+ */
+export type ExpectedValues =
+  | { kind: 'intra'; firstDistance: number; values: Float64Array }
+  | { kind: 'inter'; average: number }
+
 export interface HicMetadata {
   version: number
   genome: string

package/dist/lru.d.ts

@@ -1,29 +0,0 @@
-/**
- * Optional second bound, for a cache whose entries differ wildly in size.
- *
- * An entry cap is a memory bound only while entries are interchangeable, and the
- * block cache's are not: a block holds every contact in its bin square, so one
- * can be a few thousand contacts or a few hundred thousand depending on binsize
- * and distance from the diagonal. Capping entries therefore forced one number to
- * answer two questions — how many blocks a fetch needs at once, and how much
- * memory the biggest of them may hold — and the memory question won, which is
- * what left the cache too small to serve a multi-region fetch (see hicFile.ts).
- *
- * Splitting them lets the entry cap track the working set while `maxBytes`
- * stays the backstop it was standing in for.
- */
-interface WeightOpts<V> {
-    maxBytes: number;
-    weigh: (value: V) => number;
-}
-export default class LRU<K, V> {
-    private max;
-    private map;
-    private weight;
-    private weightOpts;
-    constructor(max?: number, weightOpts?: WeightOpts<V>);
-    get(key: K): V | undefined;
-    set(key: K, val: V): void;
-    delete(key: K): void;
-}
-export {};

package/dist/lru.js

@@ -1,53 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-class LRU {
-    max;
-    map = new Map();
-    weight = 0;
-    weightOpts;
-    constructor(max = 10, weightOpts) {
-        this.max = max;
-        this.weightOpts = weightOpts;
-    }
-    get(key) {
-        const item = this.map.get(key);
-        if (item !== undefined) {
-            // refresh key
-            this.map.delete(key);
-            this.map.set(key, item);
-        }
-        return item;
-    }
-    set(key, val) {
-        if (this.map.has(key)) {
-            // refresh key, and drop its old weight — `set` on an existing key
-            // replaces the value, so the weight it contributed goes with it
-            this.delete(key);
-        }
-        this.map.set(key, val);
-        this.weight += this.weightOpts?.weigh(val) ?? 0;
-        // Entry cap first, then the byte budget. `size > 1` keeps a single entry
-        // larger than the whole budget rather than evicting it and looping forever
-        // on an empty map — a cache that can't hold the thing just asked for should
-        // still answer the caller that asked.
-        while (this.map.size > this.max ||
-            (this.weightOpts !== undefined &&
-                this.weight > this.weightOpts.maxBytes &&
-                this.map.size > 1)) {
-            const oldest = this.map.keys().next().value;
-            if (oldest === undefined) {
-                break;
-            }
-            this.delete(oldest);
-        }
-    }
-    delete(key) {
-        const existing = this.map.get(key);
-        if (existing !== undefined) {
-            this.weight -= this.weightOpts?.weigh(existing) ?? 0;
-        }
-        this.map.delete(key);
-    }
-}
-exports.default = LRU;
-//# sourceMappingURL=lru.js.map

package/dist/lru.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"lru.js","sourceRoot":"","sources":["../src/lru.ts"],"names":[],"mappings":";;AAmBA,MAAqB,GAAG;IACd,GAAG,CAAQ;IACX,GAAG,GAAG,IAAI,GAAG,EAAQ,CAAA;IACrB,MAAM,GAAG,CAAC,CAAA;IACV,UAAU,CAA2B;IAE7C,YAAY,GAAG,GAAG,EAAE,EAAE,UAA0B;QAC9C,IAAI,CAAC,GAAG,GAAG,GAAG,CAAA;QACd,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;IAC9B,CAAC;IAED,GAAG,CAAC,GAAM;QACR,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC9B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,cAAc;YACd,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACpB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QACzB,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,GAAG,CAAC,GAAM,EAAE,GAAM;QAChB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,kEAAkE;YAClE,gEAAgE;YAChE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QACtB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAE/C,yEAAyE;QACzE,2EAA2E;QAC3E,4EAA4E;QAC5E,sCAAsC;QACtC,OACE,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG;YACxB,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS;gBAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ;gBACtC,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,EACpB,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAA;YAC3C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,MAAK;YACP,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QACrB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,GAAM;QACX,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAClC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACtD,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC;CACF;AAvDD,sBAuDC"}

package/esm/lru.d.ts

@@ -1,29 +0,0 @@
-/**
- * Optional second bound, for a cache whose entries differ wildly in size.
- *
- * An entry cap is a memory bound only while entries are interchangeable, and the
- * block cache's are not: a block holds every contact in its bin square, so one
- * can be a few thousand contacts or a few hundred thousand depending on binsize
- * and distance from the diagonal. Capping entries therefore forced one number to
- * answer two questions — how many blocks a fetch needs at once, and how much
- * memory the biggest of them may hold — and the memory question won, which is
- * what left the cache too small to serve a multi-region fetch (see hicFile.ts).
- *
- * Splitting them lets the entry cap track the working set while `maxBytes`
- * stays the backstop it was standing in for.
- */
-interface WeightOpts<V> {
-    maxBytes: number;
-    weigh: (value: V) => number;
-}
-export default class LRU<K, V> {
-    private max;
-    private map;
-    private weight;
-    private weightOpts;
-    constructor(max?: number, weightOpts?: WeightOpts<V>);
-    get(key: K): V | undefined;
-    set(key: K, val: V): void;
-    delete(key: K): void;
-}
-export {};

package/esm/lru.js

@@ -1,50 +0,0 @@
-export default class LRU {
-    max;
-    map = new Map();
-    weight = 0;
-    weightOpts;
-    constructor(max = 10, weightOpts) {
-        this.max = max;
-        this.weightOpts = weightOpts;
-    }
-    get(key) {
-        const item = this.map.get(key);
-        if (item !== undefined) {
-            // refresh key
-            this.map.delete(key);
-            this.map.set(key, item);
-        }
-        return item;
-    }
-    set(key, val) {
-        if (this.map.has(key)) {
-            // refresh key, and drop its old weight — `set` on an existing key
-            // replaces the value, so the weight it contributed goes with it
-            this.delete(key);
-        }
-        this.map.set(key, val);
-        this.weight += this.weightOpts?.weigh(val) ?? 0;
-        // Entry cap first, then the byte budget. `size > 1` keeps a single entry
-        // larger than the whole budget rather than evicting it and looping forever
-        // on an empty map — a cache that can't hold the thing just asked for should
-        // still answer the caller that asked.
-        while (this.map.size > this.max ||
-            (this.weightOpts !== undefined &&
-                this.weight > this.weightOpts.maxBytes &&
-                this.map.size > 1)) {
-            const oldest = this.map.keys().next().value;
-            if (oldest === undefined) {
-                break;
-            }
-            this.delete(oldest);
-        }
-    }
-    delete(key) {
-        const existing = this.map.get(key);
-        if (existing !== undefined) {
-            this.weight -= this.weightOpts?.weigh(existing) ?? 0;
-        }
-        this.map.delete(key);
-    }
-}
-//# sourceMappingURL=lru.js.map

package/esm/lru.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"lru.js","sourceRoot":"","sources":["../src/lru.ts"],"names":[],"mappings":"AAmBA,MAAM,CAAC,OAAO,OAAO,GAAG;IACd,GAAG,CAAQ;IACX,GAAG,GAAG,IAAI,GAAG,EAAQ,CAAA;IACrB,MAAM,GAAG,CAAC,CAAA;IACV,UAAU,CAA2B;IAE7C,YAAY,GAAG,GAAG,EAAE,EAAE,UAA0B;QAC9C,IAAI,CAAC,GAAG,GAAG,GAAG,CAAA;QACd,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;IAC9B,CAAC;IAED,GAAG,CAAC,GAAM;QACR,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC9B,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,cAAc;YACd,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACpB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QACzB,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED,GAAG,CAAC,GAAM,EAAE,GAAM;QAChB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,kEAAkE;YAClE,gEAAgE;YAChE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;QACtB,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAE/C,yEAAyE;QACzE,2EAA2E;QAC3E,4EAA4E;QAC5E,sCAAsC;QACtC,OACE,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG;YACxB,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS;gBAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ;gBACtC,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,EACpB,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAA;YAC3C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,MAAK;YACP,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QACrB,CAAC;IACH,CAAC;IAED,MAAM,CAAC,GAAM;QACX,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAClC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACtD,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC;CACF"}

package/src/lru.ts

@@ -1,75 +0,0 @@
-/**
- * Optional second bound, for a cache whose entries differ wildly in size.
- *
- * An entry cap is a memory bound only while entries are interchangeable, and the
- * block cache's are not: a block holds every contact in its bin square, so one
- * can be a few thousand contacts or a few hundred thousand depending on binsize
- * and distance from the diagonal. Capping entries therefore forced one number to
- * answer two questions — how many blocks a fetch needs at once, and how much
- * memory the biggest of them may hold — and the memory question won, which is
- * what left the cache too small to serve a multi-region fetch (see hicFile.ts).
- *
- * Splitting them lets the entry cap track the working set while `maxBytes`
- * stays the backstop it was standing in for.
- */
-interface WeightOpts<V> {
-  maxBytes: number
-  weigh: (value: V) => number
-}
-
-export default class LRU<K, V> {
-  private max: number
-  private map = new Map<K, V>()
-  private weight = 0
-  private weightOpts: WeightOpts<V> | undefined
-
-  constructor(max = 10, weightOpts?: WeightOpts<V>) {
-    this.max = max
-    this.weightOpts = weightOpts
-  }
-
-  get(key: K) {
-    const item = this.map.get(key)
-    if (item !== undefined) {
-      // refresh key
-      this.map.delete(key)
-      this.map.set(key, item)
-    }
-    return item
-  }
-
-  set(key: K, val: V) {
-    if (this.map.has(key)) {
-      // refresh key, and drop its old weight — `set` on an existing key
-      // replaces the value, so the weight it contributed goes with it
-      this.delete(key)
-    }
-    this.map.set(key, val)
-    this.weight += this.weightOpts?.weigh(val) ?? 0
-
-    // Entry cap first, then the byte budget. `size > 1` keeps a single entry
-    // larger than the whole budget rather than evicting it and looping forever
-    // on an empty map — a cache that can't hold the thing just asked for should
-    // still answer the caller that asked.
-    while (
-      this.map.size > this.max ||
-      (this.weightOpts !== undefined &&
-        this.weight > this.weightOpts.maxBytes &&
-        this.map.size > 1)
-    ) {
-      const oldest = this.map.keys().next().value
-      if (oldest === undefined) {
-        break
-      }
-      this.delete(oldest)
-    }
-  }
-
-  delete(key: K) {
-    const existing = this.map.get(key)
-    if (existing !== undefined) {
-      this.weight -= this.weightOpts?.weigh(existing) ?? 0
-    }
-    this.map.delete(key)
-  }
-}