@gmod/hic 1.0.0

package/src/index.ts

@@ -0,0 +1,12 @@
+export { HicFile, NO_DATA_FOR_RESOLUTION } from './hicFile.ts'
+export { readerFromFilehandle } from './reader.ts'
+
+export type { HicConfig } from './hicFile.ts'
+export type { ContactRecords } from './contactRecords.ts'
+export type {
+  Chromosome,
+  HicMetadata,
+  HicRegion,
+  Reader,
+  Zoom,
+} from './types.ts'

package/src/lru.ts

@@ -0,0 +1,77 @@
+/**
+ * Optional second bound, for a cache whose entries differ wildly in size.
+ *
+ * An entry cap is a memory bound only while entries are interchangeable, and the
+ * block cache's are not: a block holds every contact in its bin square, so one
+ * can be a few thousand contacts or a few hundred thousand depending on binsize
+ * and distance from the diagonal. Capping entries therefore forced one number to
+ * answer two questions — how many blocks a fetch needs at once, and how much
+ * memory the biggest of them may hold — and the memory question won, which is
+ * what left the cache too small to serve a multi-region fetch (see hicFile.ts).
+ *
+ * Splitting them lets the entry cap track the working set while `maxBytes`
+ * stays the backstop it was standing in for.
+ */
+interface WeightOpts<V> {
+  maxBytes: number
+  weigh: (value: V) => number
+}
+
+export default class LRU<K, V> {
+  private max: number
+  private map = new Map<K, V>()
+  private weight = 0
+  private weightOpts: WeightOpts<V> | undefined
+
+  constructor(max = 10, weightOpts?: WeightOpts<V>) {
+    this.max = max
+    this.weightOpts = weightOpts
+  }
+
+  get(key: K) {
+    const item = this.map.get(key)
+    if (item !== undefined) {
+      // refresh key
+      this.map.delete(key)
+      this.map.set(key, item)
+    }
+    return item
+  }
+
+  set(key: K, val: V) {
+    if (this.map.has(key)) {
+      // refresh key, and drop its old weight — `set` on an existing key
+      // replaces the value, so the weight it contributed goes with it
+      this.delete(key)
+    }
+    this.map.set(key, val)
+    this.weight += this.weightOpts?.weigh(val) ?? 0
+
+    // Entry cap first, then the byte budget. `size > 1` keeps a single entry
+    // larger than the whole budget rather than evicting it and looping forever
+    // on an empty map — a cache that can't hold the thing just asked for should
+    // still answer the caller that asked.
+    while (
+      this.map.size > this.max ||
+      (this.weightOpts !== undefined &&
+        this.weight > this.weightOpts.maxBytes &&
+        this.map.size > 1)
+    ) {
+      const oldest = this.map.keys().next().value
+      if (oldest === undefined) {
+        break
+      }
+      this.delete(oldest)
+    }
+  }
+
+  // Used to evict a rejected in-flight promise so the next caller retries
+  // rather than resolving against a cached failure forever.
+  delete(key: K) {
+    const existing = this.map.get(key)
+    if (existing !== undefined) {
+      this.weight -= this.weightOpts?.weigh(existing) ?? 0
+    }
+    this.map.delete(key)
+  }
+}

package/src/matrix.ts

@@ -0,0 +1,43 @@
+import BinaryParser from './binary.ts'
+import MatrixZoomData from './matrixZoomData.ts'
+
+import type { Chromosome } from './types.ts'
+
+export default class Matrix {
+  // BP levels only. The FRAG code paths were dropped in this fork, so a
+  // matrix's FRAG zoom levels are parsed off the wire and discarded rather than
+  // kept in a second list nothing can reach.
+  private bpZoomData: MatrixZoomData[]
+
+  constructor(zoomDataList: MatrixZoomData[]) {
+    this.bpZoomData = zoomDataList.filter(zd => zd.zoom.unit === 'BP')
+  }
+
+  // Fetch zoom data by bin size. If no matching level exists return undefined.
+  getZoomData(binSize: number) {
+    return this.bpZoomData.find(zd => binSize === zd.zoom.binSize)
+  }
+
+  static getKey(chrIdx1: number, chrIdx2: number) {
+    const lo = Math.min(chrIdx1, chrIdx2)
+    const hi = Math.max(chrIdx1, chrIdx2)
+    return `${lo}_${hi}`
+  }
+
+  static parseMatrix(data: ArrayBuffer, chromosomes: Chromosome[]) {
+    const dis = new BinaryParser(new DataView(data))
+    const c1 = dis.getInt() // Should equal chrIdx1
+    const c2 = dis.getInt() // Should equal chrIdx2
+
+    const chr1 = chromosomes[c1]!
+    const chr2 = chromosomes[c2]!
+
+    // # of resolution levels (bp and frags)
+    let nResolutions = dis.getInt()
+    const zdList: MatrixZoomData[] = []
+    while (nResolutions-- > 0) {
+      zdList.push(MatrixZoomData.parseMatrixZoomData(chr1, chr2, dis))
+    }
+    return new Matrix(zdList)
+  }
+}

package/src/matrixZoomData.ts

@@ -0,0 +1,133 @@
+import { binWindow } from './binWindow.ts'
+
+import type BinaryParser from './binary.ts'
+import type { BlockIndexEntry, Chromosome, HicRegion, Zoom } from './types.ts'
+
+export default class MatrixZoomData {
+  chr1: Chromosome
+  chr2: Chromosome
+  zoom: Zoom
+  blockBinCount: number
+  blockColumnCount: number
+  blockIndex: Record<number, BlockIndexEntry>
+
+  constructor(
+    chr1: Chromosome,
+    chr2: Chromosome,
+    zoom: Zoom,
+    blockBinCount: number,
+    blockColumnCount: number,
+    blockIndex: Record<number, BlockIndexEntry>,
+  ) {
+    this.chr1 = chr1
+    this.chr2 = chr2
+    this.zoom = zoom
+    this.blockBinCount = blockBinCount
+    this.blockColumnCount = blockColumnCount
+    this.blockIndex = blockIndex
+  }
+
+  getKey() {
+    return `${this.chr1.name}_${this.chr2.name}_${this.zoom.unit}_${this.zoom.binSize}`
+  }
+
+  getBlockNumbers(region1: HicRegion, region2: HicRegion, version: number) {
+    const sameChr = this.chr1 === this.chr2
+    const binsize = this.zoom.binSize
+    const blockBinCount = this.blockBinCount
+    const blockColumnCount = this.blockColumnCount
+
+    // Integer bin bounds, not the raw fractional quotients upstream used: the
+    // blocks selected here must cover every bin the record filter will accept,
+    // and `binWindow` widens that to any bin overlapping the region. Selecting
+    // too many blocks only costs a read the filter then discards; selecting too
+    // few silently drops records (see binWindow.ts).
+    const [x1, x2] = binWindow(region1, binsize)
+    const [y1, y2] = binWindow(region2, binsize)
+
+    // A Set dedups the block numbers that transposition can collide on the
+    // same-chr diagonal, and keeps both branches uniform.
+    const blockNumbers = new Set<number>()
+    if (version < 9 || !sameChr) {
+      const col1 = Math.floor(x1 / blockBinCount)
+      const col2 = Math.floor((x2 - 1) / blockBinCount)
+      const row1 = Math.floor(y1 / blockBinCount)
+      const row2 = Math.floor((y2 - 1) / blockBinCount)
+
+      for (let row = row1; row <= row2; row++) {
+        for (let column = col1; column <= col2; column++) {
+          blockNumbers.add(
+            sameChr && row < column
+              ? column * blockColumnCount + row
+              : row * blockColumnCount + column,
+          )
+        }
+      }
+    } else {
+      // PAD = positionAlongDiagonal (~projected). Depth is the axis
+      // perpendicular to the diagonal; nearer means closer to the diagonal.
+      // Widening stays safe here too: overlapping regions take the
+      // `containsDiagonal` branch (nearerDepth 0) and disjoint ones only see
+      // the near-corner gap shrink, so no depth level is lost.
+      const translatedLowerPAD = Math.floor((x1 + y1) / 2 / blockBinCount)
+      const translatedHigherPAD = Math.floor((x2 + y2) / 2 / blockBinCount)
+      const translatedNearerDepth = Math.floor(
+        Math.log2(1 + Math.abs(x1 - y2) / Math.sqrt(2) / blockBinCount),
+      )
+      const translatedFurtherDepth = Math.floor(
+        Math.log2(1 + Math.abs(x2 - y1) / Math.sqrt(2) / blockBinCount),
+      )
+
+      // code above assumes above diagonal, but we could be below it
+      const containsDiagonal = (x2 - y1) * (x1 - y2) < 0
+      const nearerDepth = containsDiagonal
+        ? 0
+        : Math.min(translatedNearerDepth, translatedFurtherDepth)
+      const furtherDepth = Math.max(
+        translatedNearerDepth,
+        translatedFurtherDepth,
+      )
+
+      for (let depth = nearerDepth; depth <= furtherDepth; depth++) {
+        for (let pad = translatedLowerPAD; pad <= translatedHigherPAD; pad++) {
+          blockNumbers.add(depth * blockColumnCount + pad)
+        }
+      }
+    }
+    return [...blockNumbers]
+  }
+
+  static parseMatrixZoomData(
+    chr1: Chromosome,
+    chr2: Chromosome,
+    dis: BinaryParser,
+  ) {
+    const unit = dis.getString()
+    const zoomIndex = dis.getInt()
+    dis.getFloat() // sumCounts
+    dis.getFloat() // occupiedCellCount
+    dis.getFloat() // stdDev
+    dis.getFloat() // percent95
+    const binSize = dis.getInt()
+    const blockBinCount = dis.getInt()
+    const blockColumnCount = dis.getInt()
+    let nBlocks = dis.getInt()
+
+    const blockIndex: Record<number, BlockIndexEntry> = {}
+    while (nBlocks-- > 0) {
+      const blockNumber = dis.getInt()
+      const filePosition = dis.getLong()
+      const size = dis.getInt()
+      blockIndex[blockNumber] = { filePosition, size }
+    }
+
+    return new MatrixZoomData(
+      chr1,
+      chr2,
+      { index: zoomIndex, unit, binSize },
+      blockBinCount,
+      blockColumnCount,
+      blockIndex,
+    )
+  }
+}

package/src/normalizationVector.ts

@@ -0,0 +1,58 @@
+import BinaryParser from './binary.ts'
+
+import type { Reader } from './types.ts'
+
+const DOUBLE = 8
+
+export default class NormalizationVector {
+  private cache:
+    { start: number; end: number; values: Float64Array } | undefined
+
+  private file: Reader
+  private filePosition: number
+  private nValues: number
+  private dataType: number
+
+  constructor(
+    file: Reader,
+    filePosition: number,
+    nValues: number,
+    dataType: number,
+  ) {
+    this.file = file
+    this.filePosition = filePosition
+    this.nValues = nValues
+    this.dataType = dataType
+  }
+
+  async getValues(start: number, end: number) {
+    // The read below clamps to `nValues`, so the cache can never cover past it.
+    // Compare against the clamped bound or a request that runs off the end of
+    // the vector — reachable when the assembly's refseq is longer than the size
+    // the `.hic` recorded for that chromosome — leaves `end > cache.end` true
+    // forever, and every call re-reads the same range from the file.
+    const wanted = Math.min(end, this.nValues)
+    if (!this.cache || start < this.cache.start || wanted > this.cache.end) {
+      const adjustedStart = Math.max(0, start - 1000)
+      const adjustedEnd = Math.min(this.nValues, end + 1000)
+      const startPosition = this.filePosition + adjustedStart * this.dataType
+      const n = adjustedEnd - adjustedStart
+      const data = await this.file.read(startPosition, n * this.dataType)
+      const parser = new BinaryParser(new DataView(data))
+      const values = new Float64Array(n)
+      for (let i = 0; i < n; i++) {
+        values[i] =
+          this.dataType === DOUBLE ? parser.getDouble() : parser.getFloat()
+      }
+      this.cache = { start: adjustedStart, end: adjustedEnd, values }
+    }
+
+    // A view, not a copy: the caller only reads it, and a fetch asks for one of
+    // these per chromosome per region pair. Both this and `slice` clamp a
+    // request running past the end to what exists, which the caller already
+    // handles — an out-of-range index reads undefined, making the product NaN,
+    // which the record filter drops.
+    const sliceStart = start - this.cache.start
+    return this.cache.values.subarray(sliceStart, sliceStart + (end - start))
+  }
+}

package/src/reader.ts

@@ -0,0 +1,11 @@
+import type { Reader } from './types.ts'
+import type { GenericFilehandle } from 'generic-filehandle2'
+
+export function readerFromFilehandle(filehandle: GenericFilehandle): Reader {
+  return {
+    async read(position: number, length: number) {
+      const buf = await filehandle.read(length, position)
+      return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength)
+    },
+  }
+}

package/src/types.ts

@@ -0,0 +1,40 @@
+/**
+ * The only read this parser needs: `length` bytes at `position`, coming back
+ * SHORT at end-of-file rather than throwing — the header walk deliberately
+ * reads past the end to discover whether a file carries normalization data at
+ * all. `generic-filehandle2` handles behave this way (a remote 416 included),
+ * and `readerFromFilehandle` adapts one to this shape.
+ */
+export interface Reader {
+  read: (position: number, length: number) => Promise<ArrayBuffer>
+}
+
+export interface Chromosome {
+  index: number
+  name: string
+  size: number
+}
+
+export interface HicRegion {
+  chr: string
+  start: number
+  end: number
+}
+
+export interface Zoom {
+  index: number
+  unit: string
+  binSize: number
+}
+
+export interface BlockIndexEntry {
+  filePosition: number
+  size: number
+}
+
+export interface HicMetadata {
+  version: number
+  genome: string
+  chromosomes: Chromosome[]
+  resolutions: number[]
+}