@gmod/hic 1.2.1 → 1.3.0

package/README.md

@@ -8,8 +8,8 @@ Read `.hic` contact matrix files, in Node.js or the browser.
 A fork of [hic-straw](https://github.com/aidenlab/hic-straw), rewritten in
 TypeScript and retuned for a genome browser's access pattern — many region
 pairs per fetch, arbitrary viewport windows, and contacts handed to a GPU
-renderer. See [docs/optimizations.md](docs/optimizations.md) for what changed
-and why.
+renderer. See [docs/optimizations.md](docs/optimizations.md) and
+[docs/correctness.md](docs/correctness.md) for what changed and why.
 
 ## Install
 
@@ -37,11 +37,11 @@ records.bin2 // Int32Array of bin indices along region2
 records.counts // Float32Array of contact counts
 ```
 
-Both reading methods take an optional trailing `{ onProgress }`, called
-`(current, total)` as the work lands, so a loading indicator can show a bar
-rather than a spinner. The unit is the block for a contact fetch and the
-expected-value chunk for the normalization-index walk — the slow part of opening
-a pre-v9 file. See [docs/api.md](docs/api.md#progress).
+`getNormalizationOptions()` and `getContactRecords()` both take an optional
+trailing `{ onProgress }`, called `(current, total)` as the work lands, so a
+loading indicator can show a bar rather than a spinner. The unit is the block for a
+contact fetch and the expected-value chunk for the normalization-index walk —
+the slow part of opening a pre-v9 file. See [docs/api.md](docs/api.md#progress).
 
 ```js
 await hic.getContactRecords('KR', region, region, 'BP', 2_500_000, {
@@ -49,6 +49,9 @@ await hic.getContactRecords('KR', region, region, 'BP', 2_500_000, {
 })
 ```
 
+Every async method also takes an `AbortSignal` the same way, `{ signal }`, and
+forwards it to each read; see [docs/api.md](docs/api.md#cancellation).
+
 In the browser, or for a file over HTTP, pass a `filehandle` from
 [`generic-filehandle2`](https://www.npmjs.com/package/generic-filehandle2)
 instead of a `path`:
@@ -67,10 +70,10 @@ small blocks scattered through the file — a whole-genome view of the test file
 issues over a thousand of them — and a browser runs about six requests per
 origin at a time, so the request count sets the wall clock.
 [`@gmod/range-cache-filehandle`](https://github.com/GMOD/range-cache-filehandle)
-is a drop-in for `RemoteFile` that coalesces those reads into one request per
-contiguous run. Measured in headless Chrome against a real 69 GB ENCODE file,
-whole chr1 at 5 kb: **24.0 s and 225 requests becomes 1.8 s and 45**, for
-byte-identical output.
+is a drop-in for `RemoteFile` that serves those reads out of 256 KiB chunks, so
+neighboring blocks share a request. Measured in headless Chrome against a real
+69 GB ENCODE file, whole chr1 at 5 kb: **24.0 s and 225 requests becomes 1.8 s
+and 45**, for byte-identical output.
 
 ```js
 import { RemoteFileWithRangeCache } from '@gmod/range-cache-filehandle'
@@ -97,7 +100,7 @@ See [docs/api.md](docs/api.md) for the full API reference.
   requested region.
 - A `.hic` stores only the `bin1 <= bin2` half of the matrix, so a query whose x
   window sits right of its y window gets swapped before it goes out;
-  `transposed` says so, and `bin1` then runs along `region2`.
+  `transposed` marks that swap, and `bin1` then runs along `region2`.
 - `appliedNormalization` names the normalization the file actually applied,
   which is not always the one you asked for — a `.hic` carries normalization
   vectors per (type, chromosome, unit, binsize), so it can offer KR at 5 kb and
@@ -113,8 +116,10 @@ See [docs/api.md](docs/api.md) for the full API reference.
 - [docs/api.md](docs/api.md) — every constructor option, method and return shape
 - [docs/dataflow.md](docs/dataflow.md) — how a fetch flows, and why the path
   forks into two chains
-- [docs/optimizations.md](docs/optimizations.md) — what this fork changed
-  against hic-straw, and what measured it
+- [docs/optimizations.md](docs/optimizations.md) — performance against
+  hic-straw, and what measured it
+- [docs/correctness.md](docs/correctness.md) — parsing differences from
+  hic-straw
 - [CONTRIBUTING.md](CONTRIBUTING.md) — development and release steps
 
 ## Academic use

package/dist/bufferedFile.d.ts

@@ -1,4 +1,4 @@
-import type { Reader } from './types.ts';
+import type { BaseOpts, Reader } from './types.ts';
 export default class BufferedFile {
     private file;
     private size;
@@ -9,5 +9,5 @@ export default class BufferedFile {
         file: Reader;
         size?: number;
     });
-    read(position: number, length: number): Promise<ArrayBuffer>;
+    read(position: number, length: number, opts?: BaseOpts): Promise<ArrayBuffer>;
 }

package/dist/bufferedFile.js

@@ -16,7 +16,7 @@ class BufferedFile {
         this.file = args.file;
         this.size = args.size ?? 64000;
     }
-    async read(position, length) {
+    async read(position, length, opts) {
         const start = position;
         const end = position + length;
         const bufferStart = this.bufferStart;
@@ -28,7 +28,7 @@ class BufferedFile {
             this.buffer = undefined;
             this.bufferStart = 0;
             this.bufferLength = 0;
-            result = await this.file.read(position, length);
+            result = await this.file.read(position, length, opts);
         }
         else if (buf && start >= bufferStart && end <= bufferEnd) {
             // Request within buffer bounds
@@ -38,7 +38,7 @@ class BufferedFile {
         else if (buf && start < bufferStart && end > bufferStart) {
             // Overlap left (unexpected in straw); we don't adjust the buffer
             const l1 = bufferStart - start;
-            const a1 = await this.file.read(position, l1);
+            const a1 = await this.file.read(position, l1, opts);
             const l2 = length - l1;
             if (l2 > 0) {
                 result = concatBuffers(a1, buf.slice(0, l2));
@@ -53,7 +53,7 @@ class BufferedFile {
             const a1 = buf.slice(this.bufferLength - l1, this.bufferLength);
             const l2 = length - l1;
             if (l2 > 0) {
-                const next = await this.file.read(bufferEnd, this.size);
+                const next = await this.file.read(bufferEnd, this.size, opts);
                 this.buffer = next;
                 this.bufferStart = bufferEnd;
                 this.bufferLength = next.byteLength;
@@ -65,7 +65,7 @@ class BufferedFile {
         }
         else {
             // No overlap with buffer
-            const next = await this.file.read(position, this.size);
+            const next = await this.file.read(position, this.size, opts);
             this.buffer = next;
             this.bufferStart = position;
             this.bufferLength = next.byteLength;

package/dist/bufferedFile.js.map

@@ -1 +1 @@
-{"version":3,"file":"bufferedFile.js","sourceRoot":"","sources":["../src/bufferedFile.ts"],"names":[],"mappings":";;AAEA,SAAS,aAAa,CAAC,OAAoB,EAAE,OAAoB;IAC/D,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACnE,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;IACnC,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;IACpD,OAAO,GAAG,CAAC,MAAM,CAAA;AACnB,CAAC;AAED,MAAqB,YAAY;IACvB,IAAI,CAAQ;IACZ,IAAI,CAAQ;IACZ,WAAW,GAAG,CAAC,CAAA;IACf,YAAY,GAAG,CAAC,CAAA;IAChB,MAAM,CAAyB;IAEvC,YAAY,IAAqC;QAC/C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,QAAgB,EAAE,MAAc;QACzC,MAAM,KAAK,GAAG,QAAQ,CAAA;QACtB,MAAM,GAAG,GAAG,QAAQ,GAAG,MAAM,CAAA;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;QAEvB,IAAI,MAAmB,CAAA;QACvB,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YACvB,uEAAuE;YACvE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;YACvB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;YACpB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAA;YACrB,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QACjD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,IAAI,WAAW,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC3D,+BAA+B;YAC/B,MAAM,UAAU,GAAG,KAAK,GAAG,WAAW,CAAA;YACtC,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAAC,CAAA;QACrD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,GAAG,WAAW,IAAI,GAAG,GAAG,WAAW,EAAE,CAAC;YAC3D,iEAAiE;YACjE,MAAM,EAAE,GAAG,WAAW,GAAG,KAAK,CAAA;YAC9B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;YAC7C,MAAM,EAAE,GAAG,MAAM,GAAG,EAAE,CAAA;YACtB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACX,MAAM,GAAG,aAAa,CAAC,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;YAC9C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,GAAG,SAAS,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;YACvD,gBAAgB;YAChB,MAAM,EAAE,GAAG,SAAS,GAAG,KAAK,CAAA;YAC5B,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/D,MAAM,EAAE,GAAG,MAAM,GAAG,EAAE,CAAA;YACtB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACX,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;gBACvD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;gBAClB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAA;gBAC5B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAA;gBACnC,MAAM,GAAG,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;YAC/C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACtD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;YAClB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAA;YAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAA;YACnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;QAChC,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;CACF;AAhED,+BAgEC"}
+{"version":3,"file":"bufferedFile.js","sourceRoot":"","sources":["../src/bufferedFile.ts"],"names":[],"mappings":";;AAEA,SAAS,aAAa,CAAC,OAAoB,EAAE,OAAoB;IAC/D,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACnE,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;IACnC,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;IACpD,OAAO,GAAG,CAAC,MAAM,CAAA;AACnB,CAAC;AAED,MAAqB,YAAY;IACvB,IAAI,CAAQ;IACZ,IAAI,CAAQ;IACZ,WAAW,GAAG,CAAC,CAAA;IACf,YAAY,GAAG,CAAC,CAAA;IAChB,MAAM,CAAyB;IAEvC,YAAY,IAAqC;QAC/C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CACR,QAAgB,EAChB,MAAc,EACd,IAAe;QAEf,MAAM,KAAK,GAAG,QAAQ,CAAA;QACtB,MAAM,GAAG,GAAG,QAAQ,GAAG,MAAM,CAAA;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;QAEvB,IAAI,MAAmB,CAAA;QACvB,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YACvB,uEAAuE;YACvE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;YACvB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;YACpB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAA;YACrB,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAA;QACvD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,IAAI,WAAW,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC3D,+BAA+B;YAC/B,MAAM,UAAU,GAAG,KAAK,GAAG,WAAW,CAAA;YACtC,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAAC,CAAA;QACrD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,GAAG,WAAW,IAAI,GAAG,GAAG,WAAW,EAAE,CAAC;YAC3D,iEAAiE;YACjE,MAAM,EAAE,GAAG,WAAW,GAAG,KAAK,CAAA;YAC9B,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,IAAI,CAAC,CAAA;YACnD,MAAM,EAAE,GAAG,MAAM,GAAG,EAAE,CAAA;YACtB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACX,MAAM,GAAG,aAAa,CAAC,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;YAC9C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,GAAG,SAAS,IAAI,GAAG,GAAG,SAAS,EAAE,CAAC;YACvD,gBAAgB;YAChB,MAAM,EAAE,GAAG,SAAS,GAAG,KAAK,CAAA;YAC5B,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/D,MAAM,EAAE,GAAG,MAAM,GAAG,EAAE,CAAA;YACtB,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACX,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;gBAC7D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;gBAClB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAA;gBAC5B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAA;gBACnC,MAAM,GAAG,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;YAC/C,CAAC;iBAAM,CAAC;gBACN,MAAM,GAAG,EAAE,CAAA;YACb,CAAC;QACH,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YAC5D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;YAClB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAA;YAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAA;YACnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;QAChC,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;CACF;AApED,+BAoEC"}

package/dist/hicFile.d.ts

@@ -2,7 +2,7 @@ import Matrix from './matrix.ts';
 import NormalizationVector from './normalizationVector.ts';
 import type { ContactRecords } from './contactRecords.ts';
 import type MatrixZoomData from './matrixZoomData.ts';
-import type { BlockIndexEntry, HicMetadata, HicRegion, ProgressOpts, Reader } from './types.ts';
+import type { BaseOpts, BlockIndexEntry, HicMetadata, HicRegion, ProgressOpts, Reader } from './types.ts';
 import type { GenericFilehandle } from 'generic-filehandle2';
 /**
  * Prefix of the error thrown when a region pair has no matrix at the requested
@@ -39,19 +39,17 @@ interface Block {
 export declare class HicFile {
     private config;
     private file;
+    private headerCache;
+    private normVectorIndexCache;
     private normVectorCache;
-    private normalizationTypes;
     private matrixCache;
     private blockCache;
     private normVectorIndexPosition;
     private normVectorIndexSize;
-    private initPromise;
-    private normVectorIndexP;
     private version;
     private genomeId;
     private footerPosition;
     normExpectedValueVectorsPosition: number | undefined;
-    private normVectorIndex;
     private chromosomes;
     private chromosomeIndexMap;
     private chrAliasTable;
@@ -59,11 +57,11 @@ export declare class HicFile {
     private masterIndex;
     private meta;
     constructor(config: HicConfig);
-    init(): Promise<void>;
-    getMetaData(): Promise<HicMetadata>;
+    init(opts?: BaseOpts): Promise<void>;
+    getMetaData(opts?: BaseOpts): Promise<HicMetadata>;
     private readHeaderAndFooter;
     private readFooter;
-    getMatrix(chrIdx1: number, chrIdx2: number): Promise<Matrix | undefined>;
+    getMatrix(chrIdx1: number, chrIdx2: number, opts?: BaseOpts): Promise<Matrix | undefined>;
     private readMatrix;
     getContactRecords(normalization: string, region1: HicRegion, region2: HicRegion, units: string, binsize: number, opts?: ProgressOpts): Promise<{
         records: ContactRecords;
@@ -72,7 +70,7 @@ export declare class HicFile {
     }>;
     private getNormVectors;
     getBlocks(region1: HicRegion, region2: HicRegion, binSize: number, opts?: ProgressOpts): Promise<(Block | undefined)[]>;
-    readBlock(blockNumber: number, zd: MatrixZoomData): Promise<Block | undefined>;
+    readBlock(blockNumber: number, zd: MatrixZoomData, opts?: BaseOpts): Promise<Block | undefined>;
     /**
      * Decode one block's records straight into typed arrays.
      *
@@ -83,9 +81,10 @@ export declare class HicFile {
      * of the session, so it is worth one memcpy not to pin an oversized buffer.
      */
     private parseBlockRecords;
-    getNormalizationVector(type: string, chr: string, unit: string, binSize: number): Promise<NormalizationVector | undefined>;
+    getNormalizationVector(type: string, chr: string, unit: string, binSize: number, opts?: BaseOpts): Promise<NormalizationVector | undefined>;
     private readNormalizationVector;
     getNormVectorIndex(opts?: ProgressOpts): Promise<Record<string, BlockIndexEntry> | undefined>;
+    private getParsedNormVectorIndex;
     private loadNormVectorIndex;
     getNormalizationOptions(opts?: ProgressOpts): Promise<string[]>;
     private readNormVectorIndex;