@gmod/hic 1.3.2 → 2.1.0

package/README.md

@@ -5,11 +5,9 @@
 
 Read `.hic` contact matrix files, in Node.js or the browser.
 
-A fork of [hic-straw](https://github.com/aidenlab/hic-straw), rewritten in
-TypeScript for a genome browser: many region pairs per fetch, arbitrary
-viewport windows, and contacts handed to a GPU renderer. See
-[docs/optimizations.md](docs/optimizations.md) and
-[docs/correctness.md](docs/correctness.md) for what changed.
+`@gmod/hic` is a fork of [hic-straw](https://github.com/aidenlab/hic-straw),
+rewritten in TypeScript for a genome browser that fetches many region pairs at
+once over arbitrary viewport windows.
 
 ## Install
 
@@ -29,7 +27,7 @@ await hic.getNormalizationOptions() // ['NONE', 'VC', 'VC_SQRT', 'KR', 'SCALE']
 
 const region = { chr: '1', start: 0, end: 20_000_000 }
 const { records, appliedNormalization, transposed } =
-  await hic.getContactRecords('KR', region, region, 'BP', 2_500_000)
+  await hic.getContactRecords('KR', region, region, 2_500_000)
 
 // records is struct-of-arrays: three parallel typed arrays
 records.bin1 // Int32Array of bin indices along region1
@@ -38,16 +36,11 @@ records.counts // Float32Array of contact counts
 ```
 
 `getNormalizationOptions()` and `getContactRecords()` take an optional trailing
-`{ onProgress }`, called as `(current, total)`:
-
-- The unit is the block for a contact fetch.
-- The unit is the expected-value chunk for the normalization-index walk, the
-  slow part of opening a pre-v9 file.
-
-See [docs/api.md](docs/api.md#progress).
+`{ onProgress }`, called as `(current, total)`; see
+[docs/api.md](docs/api.md#progress).
 
 ```js
-await hic.getContactRecords('KR', region, region, 'BP', 2_500_000, {
+await hic.getContactRecords('KR', region, region, 2_500_000, {
   onProgress: (current, total) => showBar(current / total),
 })
 ```
@@ -67,17 +60,11 @@ const hic = new HicFile({
 })
 ```
 
-**Over HTTP, put a byte-range cache underneath. It is worth more than every
-other optimization in this package combined.**
-
-- A contact-matrix query reads many small blocks scattered through the file, and
-  a browser runs about six requests per origin at a time, so the number of
-  requests sets the load time.
-- [`@gmod/range-cache-filehandle`](https://github.com/GMOD/range-cache-filehandle)
-  is a drop-in for `RemoteFile` that reads 256 KiB chunks, so neighboring blocks
-  share a request.
-- Whole chr1 at 5 kb of a real 69 GB ENCODE file, in headless Chrome: **24.0 s
-  and 225 requests becomes 1.8 s and 45**, with byte-identical output.
+**Over HTTP, put a byte-range cache underneath.** A query reads many small
+scattered blocks, so the request count sets the load time.
+[`@gmod/range-cache-filehandle`](https://github.com/GMOD/range-cache-filehandle)
+cut a whole-chr1 fetch from a 69 GB ENCODE file from 24.0 s to 1.8 s
+([docs/optimizations.md](docs/optimizations.md#round-trips-not-cpu-dominate-a-remote-fetch)).
 
 ```js
 import { RemoteFileWithRangeCache } from '@gmod/range-cache-filehandle'
@@ -105,10 +92,12 @@ See [docs/api.md](docs/api.md) for the full API reference.
 - A `.hic` stores only the `bin1 <= bin2` half of the matrix, so a query whose x
   window sits right of its y window gets swapped. `transposed` marks the swap,
   and `bin1` then runs along `region2`.
-- `appliedNormalization` names the normalization the file applied, which may
+- `appliedNormalization` names the normalization the reader applied, which may
   differ from the one you asked for. The file stores vectors per (type,
   chromosome, unit, binsize), so it can offer KR at 5 kb and nothing at 2.5 Mb.
-- `BP` is the only unit this fork supports; it drops the FRAG code paths.
+- Bins are base pairs only; this fork drops the FRAG (restriction-fragment)
+  code paths.
+- Chromosome names match the file's own exactly, with no `chr` aliasing.
 - Each `HicFile` caches up to 128 MB of decompressed contacts. Pass
   `blockCacheMaxBytes` to change that ceiling.
 - Record order is unspecified — the result concatenates whole blocks, cached
@@ -135,10 +124,6 @@ be linked from [jbrowse.org](http://jbrowse.org).
 If you use `.hic` files, please also cite the Juicer/Juicebox papers from the
 [Aiden Lab](https://github.com/aidenlab), whose hic-straw this is derived from.
 
-## Contributing
-
-See [CONTRIBUTING.md](CONTRIBUTING.md) for development and release workflow.
-
 ## License
 
-MIT, as is upstream hic-straw. See [LICENSE](LICENSE).
+MIT, as is upstream hic-straw; see [LICENSE](LICENSE).

package/dist/bufferedFile.js

@@ -35,33 +35,16 @@ class BufferedFile {
             const sliceStart = start - bufferStart;
             result = buf.slice(sliceStart, sliceStart + length);
         }
-        else if (buf && start < bufferStart && end > bufferStart) {
-            // Overlap left (unexpected in straw); we don't adjust the buffer
-            const l1 = bufferStart - start;
-            const a1 = await this.file.read(position, l1, opts);
-            const l2 = length - l1;
-            if (l2 > 0) {
-                result = concatBuffers(a1, buf.slice(0, l2));
-            }
-            else {
-                result = a1;
-            }
-        }
-        else if (buf && start < bufferEnd && end > bufferEnd) {
-            // Overlap right
+        else if (buf && start >= bufferStart && start < bufferEnd) {
+            // Overlap right. The expected-values walk only moves forward, so a
+            // request starting before the buffer takes the fresh read below.
             const l1 = bufferEnd - start;
             const a1 = buf.slice(this.bufferLength - l1, this.bufferLength);
-            const l2 = length - l1;
-            if (l2 > 0) {
-                const next = await this.file.read(bufferEnd, this.size, opts);
-                this.buffer = next;
-                this.bufferStart = bufferEnd;
-                this.bufferLength = next.byteLength;
-                result = concatBuffers(a1, next.slice(0, l2));
-            }
-            else {
-                result = a1;
-            }
+            const next = await this.file.read(bufferEnd, this.size, opts);
+            this.buffer = next;
+            this.bufferStart = bufferEnd;
+            this.bufferLength = next.byteLength;
+            result = concatBuffers(a1, next.slice(0, length - l1));
         }
         else {
             // No overlap with buffer

package/dist/bufferedFile.js.map

@@ -1 +1 @@
-{"version":3,"file":"bufferedFile.js","sourceRoot":"","sources":["../src/bufferedFile.ts"],"names":[],"mappings":";;AAEA,SAAS,aAAa,CAAC,OAAoB,EAAE,OAAoB;IAC/D,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACnE,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;IACnC,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;IACpD,OAAO,GAAG,CAAC,MAAM,CAAA;AACnB,CAAC;AAED,MAAqB,YAAY;IACvB,IAAI,CAAQ;IACZ,IAAI,CAAQ;IACZ,WAAW,GAAG,CAAC,CAAA;IACf,YAAY,GAAG,CAAC,CAAA;IAChB,MAAM,CAAyB;IAEvC,YAAY,IAAqC;QAC/C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CACR,QAAgB,EAChB,MAAc,EACd,IAAe;QAEf,MAAM,KAAK,GAAG,QAAQ,CAAA;QACtB,MAAM,GAAG,GAAG,QAAQ,GAAG,MAAM,CAAA;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;QAEvB,IAAI,MAAmB,CAAA;QACvB,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YACvB,uEAAuE;YACvE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;YACvB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;YACpB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAA;YACrB,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAA;QACvD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,IAAI,WAAW,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC3D,+BAA+B;YAC/B,MAAM,UAAU,GAAG,KAAK,GAAG,WAAW,CAAA;YACtC,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAAC,CAAA;QACrD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,GAAG,WAAW,IAAI,GAAG,GAAG,WAAW,EAAE,CAAC;YAC3D,iEAAiE;YACjE,MAAM,EAAE,GAAG,WAAW,GAAG,KAAK,CAAA;YAC9B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;YACnD,MAAM,EAAE,GAAG,MAAM,GAAG,EAAE,CAAA;YACtB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACX,MAAM,GAAG,aAAa,CAAC,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;YAC9C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,GAAG,SAAS,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;YACvD,gBAAgB;YAChB,MAAM,EAAE,GAAG,SAAS,GAAG,KAAK,CAAA;YAC5B,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/D,MAAM,EAAE,GAAG,MAAM,GAAG,EAAE,CAAA;YACtB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACX,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;gBAC7D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;gBAClB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAA;gBAC5B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAA;gBACnC,MAAM,GAAG,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;YAC/C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YAC5D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;YAClB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAA;YAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAA;YACnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;QAChC,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;CACF;AApED,+BAoEC"}
+{"version":3,"file":"bufferedFile.js","sourceRoot":"","sources":["../src/bufferedFile.ts"],"names":[],"mappings":";;AAEA,SAAS,aAAa,CAAC,OAAoB,EAAE,OAAoB;IAC/D,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACnE,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;IACnC,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;IACpD,OAAO,GAAG,CAAC,MAAM,CAAA;AACnB,CAAC;AAED,MAAqB,YAAY;IACvB,IAAI,CAAQ;IACZ,IAAI,CAAQ;IACZ,WAAW,GAAG,CAAC,CAAA;IACf,YAAY,GAAG,CAAC,CAAA;IAChB,MAAM,CAAyB;IAEvC,YAAY,IAAqC;QAC/C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CACR,QAAgB,EAChB,MAAc,EACd,IAAe;QAEf,MAAM,KAAK,GAAG,QAAQ,CAAA;QACtB,MAAM,GAAG,GAAG,QAAQ,GAAG,MAAM,CAAA;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;QAEvB,IAAI,MAAmB,CAAA;QACvB,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YACvB,uEAAuE;YACvE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;YACvB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;YACpB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAA;YACrB,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAA;QACvD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,IAAI,WAAW,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC3D,+BAA+B;YAC/B,MAAM,UAAU,GAAG,KAAK,GAAG,WAAW,CAAA;YACtC,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAAC,CAAA;QACrD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,IAAI,WAAW,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YAC5D,mEAAmE;YACnE,iEAAiE;YACjE,MAAM,EAAE,GAAG,SAAS,GAAG,KAAK,CAAA;YAC5B,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YAC7D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;YAClB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAA;YAC5B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAA;YACnC,MAAM,GAAG,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,CAAC,CAAA;QACxD,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YAC5D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;YAClB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAA;YAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAA;YACnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;QAChC,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;CACF;AAtDD,+BAsDC"}

package/dist/hicFile.d.ts

@@ -2,7 +2,7 @@ import Matrix from './matrix.ts';
 import NormalizationVector from './normalizationVector.ts';
 import type { ContactRecords } from './contactRecords.ts';
 import type MatrixZoomData from './matrixZoomData.ts';
-import type { BaseOpts, BlockIndexEntry, HicMetadata, HicRegion, ProgressOpts, Reader } from './types.ts';
+import type { BaseOpts, BlockIndexEntry, ExpectedValues, HicMetadata, HicRegion, ProgressOpts, Reader } from './types.ts';
 import type { GenericFilehandle } from 'generic-filehandle2';
 /**
  * Prefix of the error thrown when a region pair has no matrix at the requested
@@ -27,8 +27,7 @@ export interface HicConfig {
     /**
      * Ceiling on the decompressed contacts the block cache holds, in bytes.
      * Defaults to 128 MB. Lower it where the budget is tight and re-reading is
-     * cheaper than the memory; raising it buys nothing once a fetch's working set
-     * fits, since the entry cap bounds the cache too.
+     * cheaper than the memory.
      */
     blockCacheMaxBytes?: number;
 }
@@ -37,10 +36,11 @@ interface Block {
     records: ContactRecords;
 }
 export declare class HicFile {
-    private config;
+    private nvi;
     private file;
     private headerCache;
     private normVectorIndexCache;
+    private expectedSectionCache;
     private normVectorCache;
     private matrixCache;
     private blockCache;
@@ -49,10 +49,10 @@ export declare class HicFile {
     private version;
     private genomeId;
     private footerPosition;
+    private expectedValuesPosition;
     normExpectedValueVectorsPosition: number | undefined;
     private chromosomes;
     private chromosomeIndexMap;
-    private chrAliasTable;
     private bpResolutions;
     private masterIndex;
     private meta;
@@ -63,7 +63,7 @@ export declare class HicFile {
     private readFooter;
     getMatrix(chrIdx1: number, chrIdx2: number, opts?: BaseOpts): Promise<Matrix | undefined>;
     private readMatrix;
-    getContactRecords(normalization: string, region1: HicRegion, region2: HicRegion, units: string, binsize: number, opts?: ProgressOpts): Promise<{
+    getContactRecords(normalization: string, region1: HicRegion, region2: HicRegion, binsize: number, opts?: ProgressOpts): Promise<{
         records: ContactRecords;
         appliedNormalization: string;
         transposed: boolean;
@@ -81,7 +81,7 @@ export declare class HicFile {
      * of the session, so it is worth one memcpy not to pin an oversized buffer.
      */
     private parseBlockRecords;
-    getNormalizationVector(type: string, chr: string, unit: string, binSize: number, opts?: BaseOpts): Promise<NormalizationVector | undefined>;
+    getNormalizationVector(type: string, chr: string, binSize: number, opts?: BaseOpts): Promise<NormalizationVector | undefined>;
     private readNormalizationVector;
     getNormVectorIndex(opts?: ProgressOpts): Promise<Record<string, BlockIndexEntry> | undefined>;
     private getParsedNormVectorIndex;
@@ -89,8 +89,21 @@ export declare class HicFile {
     getNormalizationOptions(opts?: ProgressOpts): Promise<string[]>;
     private readNormVectorIndex;
     private readNormExpectedValuesAndNormVectorIndex;
-    private skipExpectedValues;
+    /**
+     * The expected count for a region pair at `binsize`, the denominator of
+     * observed/expected, as straw computes it. `normalization` must be the one
+     * the pair's records carry (`getContactRecords`' `appliedNormalization`),
+     * since each normalization has its own vector. Undefined where the file has
+     * no expected value for it: no vector at this binsize, or a version 5 file
+     * asked for a normalized one.
+     *
+     * Within a chromosome the vector is read for only the distances the pair
+     * spans. Between chromosomes straw divides by the map's average contact, the
+     * raw `sumCounts` over its bins, whatever the normalization.
+     */
+    getExpectedValues(normalization: string, region1: HicRegion, region2: HicRegion, binsize: number, opts?: ProgressOpts): Promise<ExpectedValues | undefined>;
+    private getExpectedSection;
+    private walkExpectedSection;
     private parseNormVectorEntry;
-    getFileChrName(chrAlias: string): string;
 }
 export {};