@gmod/hic 1.3.1 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -35
- package/dist/bufferedFile.js +8 -25
- package/dist/bufferedFile.js.map +1 -1
- package/dist/hicFile.d.ts +4 -7
- package/dist/hicFile.js +89 -127
- package/dist/hicFile.js.map +1 -1
- package/dist/normalizationVector.d.ts +3 -1
- package/dist/normalizationVector.js +60 -39
- package/dist/normalizationVector.js.map +1 -1
- package/esm/bufferedFile.js +8 -25
- package/esm/bufferedFile.js.map +1 -1
- package/esm/hicFile.d.ts +4 -7
- package/esm/hicFile.js +89 -127
- package/esm/hicFile.js.map +1 -1
- package/esm/normalizationVector.d.ts +3 -1
- package/esm/normalizationVector.js +60 -39
- package/esm/normalizationVector.js.map +1 -1
- package/package.json +1 -1
- package/src/bufferedFile.ts +8 -22
- package/src/hicFile.ts +92 -161
- package/src/normalizationVector.ts +77 -43
- package/dist/lru.d.ts +0 -29
- package/dist/lru.js +0 -53
- package/dist/lru.js.map +0 -1
- package/esm/lru.d.ts +0 -29
- package/esm/lru.js +0 -50
- package/esm/lru.js.map +0 -1
- package/src/lru.ts +0 -75
package/README.md
CHANGED
|
@@ -5,11 +5,9 @@
|
|
|
5
5
|
|
|
6
6
|
Read `.hic` contact matrix files, in Node.js or the browser.
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
renderer. See [docs/optimizations.md](docs/optimizations.md) and
|
|
12
|
-
[docs/correctness.md](docs/correctness.md) for what changed and why.
|
|
8
|
+
`@gmod/hic` is a fork of [hic-straw](https://github.com/aidenlab/hic-straw),
|
|
9
|
+
rewritten in TypeScript for a genome browser that fetches many region pairs at
|
|
10
|
+
once over arbitrary viewport windows.
|
|
13
11
|
|
|
14
12
|
## Install
|
|
15
13
|
|
|
@@ -29,7 +27,7 @@ await hic.getNormalizationOptions() // ['NONE', 'VC', 'VC_SQRT', 'KR', 'SCALE']
|
|
|
29
27
|
|
|
30
28
|
const region = { chr: '1', start: 0, end: 20_000_000 }
|
|
31
29
|
const { records, appliedNormalization, transposed } =
|
|
32
|
-
await hic.getContactRecords('KR', region, region,
|
|
30
|
+
await hic.getContactRecords('KR', region, region, 2_500_000)
|
|
33
31
|
|
|
34
32
|
// records is struct-of-arrays: three parallel typed arrays
|
|
35
33
|
records.bin1 // Int32Array of bin indices along region1
|
|
@@ -37,14 +35,12 @@ records.bin2 // Int32Array of bin indices along region2
|
|
|
37
35
|
records.counts // Float32Array of contact counts
|
|
38
36
|
```
|
|
39
37
|
|
|
40
|
-
`getNormalizationOptions()` and `getContactRecords()`
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
contact fetch and the expected-value chunk for the normalization-index walk —
|
|
44
|
-
the slow part of opening a pre-v9 file. See [docs/api.md](docs/api.md#progress).
|
|
38
|
+
`getNormalizationOptions()` and `getContactRecords()` take an optional trailing
|
|
39
|
+
`{ onProgress }`, called as `(current, total)`; see
|
|
40
|
+
[docs/api.md](docs/api.md#progress).
|
|
45
41
|
|
|
46
42
|
```js
|
|
47
|
-
await hic.getContactRecords('KR', region, region,
|
|
43
|
+
await hic.getContactRecords('KR', region, region, 2_500_000, {
|
|
48
44
|
onProgress: (current, total) => showBar(current / total),
|
|
49
45
|
})
|
|
50
46
|
```
|
|
@@ -64,16 +60,11 @@ const hic = new HicFile({
|
|
|
64
60
|
})
|
|
65
61
|
```
|
|
66
62
|
|
|
67
|
-
**Over HTTP, put a byte-range cache underneath
|
|
68
|
-
|
|
69
|
-
small blocks scattered through the file — a whole-genome view of the test file
|
|
70
|
-
issues over a thousand of them — and a browser runs about six requests per
|
|
71
|
-
origin at a time, so the request count sets the wall clock.
|
|
63
|
+
**Over HTTP, put a byte-range cache underneath.** A query reads many small
|
|
64
|
+
scattered blocks, so the request count sets the load time.
|
|
72
65
|
[`@gmod/range-cache-filehandle`](https://github.com/GMOD/range-cache-filehandle)
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
69 GB ENCODE file, whole chr1 at 5 kb: **24.0 s and 225 requests becomes 1.8 s
|
|
76
|
-
and 45**, for byte-identical output.
|
|
66
|
+
cut a whole-chr1 fetch from a 69 GB ENCODE file from 24.0 s to 1.8 s
|
|
67
|
+
([docs/optimizations.md](docs/optimizations.md#round-trips-not-cpu-dominate-a-remote-fetch)).
|
|
77
68
|
|
|
78
69
|
```js
|
|
79
70
|
import { RemoteFileWithRangeCache } from '@gmod/range-cache-filehandle'
|
|
@@ -83,8 +74,8 @@ const hic = new HicFile({
|
|
|
83
74
|
})
|
|
84
75
|
```
|
|
85
76
|
|
|
86
|
-
|
|
87
|
-
|
|
77
|
+
A caller with its own IO layer or cache can supply anything that reads `length`
|
|
78
|
+
bytes at `position`:
|
|
88
79
|
|
|
89
80
|
```js
|
|
90
81
|
const hic = new HicFile({
|
|
@@ -99,13 +90,14 @@ See [docs/api.md](docs/api.md) for the full API reference.
|
|
|
99
90
|
- Bin indices are **absolute for their chromosome**, not relative to the
|
|
100
91
|
requested region.
|
|
101
92
|
- A `.hic` stores only the `bin1 <= bin2` half of the matrix, so a query whose x
|
|
102
|
-
window sits right of its y window gets swapped
|
|
103
|
-
|
|
104
|
-
- `appliedNormalization` names the normalization the
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
93
|
+
window sits right of its y window gets swapped. `transposed` marks the swap,
|
|
94
|
+
and `bin1` then runs along `region2`.
|
|
95
|
+
- `appliedNormalization` names the normalization the reader applied, which may
|
|
96
|
+
differ from the one you asked for. The file stores vectors per (type,
|
|
97
|
+
chromosome, unit, binsize), so it can offer KR at 5 kb and nothing at 2.5 Mb.
|
|
98
|
+
- Bins are base pairs only; this fork drops the FRAG (restriction-fragment)
|
|
99
|
+
code paths.
|
|
100
|
+
- Chromosome names match the file's own exactly, with no `chr` aliasing.
|
|
109
101
|
- Each `HicFile` caches up to 128 MB of decompressed contacts. Pass
|
|
110
102
|
`blockCacheMaxBytes` to change that ceiling.
|
|
111
103
|
- Record order is unspecified — the result concatenates whole blocks, cached
|
|
@@ -132,10 +124,6 @@ be linked from [jbrowse.org](http://jbrowse.org).
|
|
|
132
124
|
If you use `.hic` files, please also cite the Juicer/Juicebox papers from the
|
|
133
125
|
[Aiden Lab](https://github.com/aidenlab), whose hic-straw this is derived from.
|
|
134
126
|
|
|
135
|
-
## Contributing
|
|
136
|
-
|
|
137
|
-
See [CONTRIBUTING.md](CONTRIBUTING.md) for development and release workflow.
|
|
138
|
-
|
|
139
127
|
## License
|
|
140
128
|
|
|
141
|
-
MIT, as is upstream hic-straw
|
|
129
|
+
MIT, as is upstream hic-straw; see [LICENSE](LICENSE).
|
package/dist/bufferedFile.js
CHANGED
|
@@ -35,33 +35,16 @@ class BufferedFile {
|
|
|
35
35
|
const sliceStart = start - bufferStart;
|
|
36
36
|
result = buf.slice(sliceStart, sliceStart + length);
|
|
37
37
|
}
|
|
38
|
-
else if (buf && start
|
|
39
|
-
// Overlap
|
|
40
|
-
|
|
41
|
-
const a1 = await this.file.read(position, l1, opts);
|
|
42
|
-
const l2 = length - l1;
|
|
43
|
-
if (l2 > 0) {
|
|
44
|
-
result = concatBuffers(a1, buf.slice(0, l2));
|
|
45
|
-
}
|
|
46
|
-
else {
|
|
47
|
-
result = a1;
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
else if (buf && start < bufferEnd && end > bufferEnd) {
|
|
51
|
-
// Overlap right
|
|
38
|
+
else if (buf && start >= bufferStart && start < bufferEnd) {
|
|
39
|
+
// Overlap right. The expected-values walk only moves forward, so a
|
|
40
|
+
// request starting before the buffer takes the fresh read below.
|
|
52
41
|
const l1 = bufferEnd - start;
|
|
53
42
|
const a1 = buf.slice(this.bufferLength - l1, this.bufferLength);
|
|
54
|
-
const
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
this.bufferLength = next.byteLength;
|
|
60
|
-
result = concatBuffers(a1, next.slice(0, l2));
|
|
61
|
-
}
|
|
62
|
-
else {
|
|
63
|
-
result = a1;
|
|
64
|
-
}
|
|
43
|
+
const next = await this.file.read(bufferEnd, this.size, opts);
|
|
44
|
+
this.buffer = next;
|
|
45
|
+
this.bufferStart = bufferEnd;
|
|
46
|
+
this.bufferLength = next.byteLength;
|
|
47
|
+
result = concatBuffers(a1, next.slice(0, length - l1));
|
|
65
48
|
}
|
|
66
49
|
else {
|
|
67
50
|
// No overlap with buffer
|
package/dist/bufferedFile.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"bufferedFile.js","sourceRoot":"","sources":["../src/bufferedFile.ts"],"names":[],"mappings":";;AAEA,SAAS,aAAa,CAAC,OAAoB,EAAE,OAAoB;IAC/D,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACnE,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;IACnC,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;IACpD,OAAO,GAAG,CAAC,MAAM,CAAA;AACnB,CAAC;AAED,MAAqB,YAAY;IACvB,IAAI,CAAQ;IACZ,IAAI,CAAQ;IACZ,WAAW,GAAG,CAAC,CAAA;IACf,YAAY,GAAG,CAAC,CAAA;IAChB,MAAM,CAAyB;IAEvC,YAAY,IAAqC;QAC/C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CACR,QAAgB,EAChB,MAAc,EACd,IAAe;QAEf,MAAM,KAAK,GAAG,QAAQ,CAAA;QACtB,MAAM,GAAG,GAAG,QAAQ,GAAG,MAAM,CAAA;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;QAEvB,IAAI,MAAmB,CAAA;QACvB,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YACvB,uEAAuE;YACvE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;YACvB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;YACpB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAA;YACrB,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAA;QACvD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,IAAI,WAAW,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC3D,+BAA+B;YAC/B,MAAM,UAAU,GAAG,KAAK,GAAG,WAAW,CAAA;YACtC,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAAC,CAAA;QACrD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,
|
|
1
|
+
{"version":3,"file":"bufferedFile.js","sourceRoot":"","sources":["../src/bufferedFile.ts"],"names":[],"mappings":";;AAEA,SAAS,aAAa,CAAC,OAAoB,EAAE,OAAoB;IAC/D,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACnE,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAA;IACnC,GAAG,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAA;IACpD,OAAO,GAAG,CAAC,MAAM,CAAA;AACnB,CAAC;AAED,MAAqB,YAAY;IACvB,IAAI,CAAQ;IACZ,IAAI,CAAQ;IACZ,WAAW,GAAG,CAAC,CAAA;IACf,YAAY,GAAG,CAAC,CAAA;IAChB,MAAM,CAAyB;IAEvC,YAAY,IAAqC;QAC/C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAA;QACrB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,KAAK,CAAA;IAChC,CAAC;IAED,KAAK,CAAC,IAAI,CACR,QAAgB,EAChB,MAAc,EACd,IAAe;QAEf,MAAM,KAAK,GAAG,QAAQ,CAAA;QACtB,MAAM,GAAG,GAAG,QAAQ,GAAG,MAAM,CAAA;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAA;QACpC,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,CAAA;QACtD,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAA;QAEvB,IAAI,MAAmB,CAAA;QACvB,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YACvB,uEAAuE;YACvE,IAAI,CAAC,MAAM,GAAG,SAAS,CAAA;YACvB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAA;YACpB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAA;YACrB,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAA;QACvD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,IAAI,WAAW,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;YAC3D,+BAA+B;YAC/B,MAAM,UAAU,GAAG,KAAK,GAAG,WAAW,CAAA;YACtC,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,GAAG,MAAM,CAAC,CAAA;QACrD,CAAC;aAAM,IAAI,GAAG,IAAI,KAAK,IAAI,WAAW,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YAC5D,mEAAmE;YACnE,iEAAiE;YACjE,MAAM,EAAE,GAAG,SAAS,GAAG,KAAK,CAAA;YAC5B,MAAM,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,EAAE,EAAE,IAAI,CAAC,YAAY,CAAC,CAAA;YAC/D,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YAC7D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;YAClB,IAAI,CAAC,WAAW,GAAG,SAAS,CAAA;YAC5B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAA;YACnC,MAAM,GAAG,aAAa,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,GAAG,EAAE,CAAC,CAAC,CAAA;QACxD,CAAC;aAAM,CAAC;YACN,yBAAyB;YACzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YAC5D,IAAI,CAAC,MAAM,GAAG,IAAI,CAAA;YAClB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAA;YAC3B,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,CAAA;YACnC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAA;QAChC,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;CACF;AAtDD,+BAsDC"}
|
package/dist/hicFile.d.ts
CHANGED
|
@@ -27,8 +27,7 @@ export interface HicConfig {
|
|
|
27
27
|
/**
|
|
28
28
|
* Ceiling on the decompressed contacts the block cache holds, in bytes.
|
|
29
29
|
* Defaults to 128 MB. Lower it where the budget is tight and re-reading is
|
|
30
|
-
* cheaper than the memory
|
|
31
|
-
* fits, since the entry cap bounds the cache too.
|
|
30
|
+
* cheaper than the memory.
|
|
32
31
|
*/
|
|
33
32
|
blockCacheMaxBytes?: number;
|
|
34
33
|
}
|
|
@@ -37,7 +36,7 @@ interface Block {
|
|
|
37
36
|
records: ContactRecords;
|
|
38
37
|
}
|
|
39
38
|
export declare class HicFile {
|
|
40
|
-
private
|
|
39
|
+
private nvi;
|
|
41
40
|
private file;
|
|
42
41
|
private headerCache;
|
|
43
42
|
private normVectorIndexCache;
|
|
@@ -52,7 +51,6 @@ export declare class HicFile {
|
|
|
52
51
|
normExpectedValueVectorsPosition: number | undefined;
|
|
53
52
|
private chromosomes;
|
|
54
53
|
private chromosomeIndexMap;
|
|
55
|
-
private chrAliasTable;
|
|
56
54
|
private bpResolutions;
|
|
57
55
|
private masterIndex;
|
|
58
56
|
private meta;
|
|
@@ -63,7 +61,7 @@ export declare class HicFile {
|
|
|
63
61
|
private readFooter;
|
|
64
62
|
getMatrix(chrIdx1: number, chrIdx2: number, opts?: BaseOpts): Promise<Matrix | undefined>;
|
|
65
63
|
private readMatrix;
|
|
66
|
-
getContactRecords(normalization: string, region1: HicRegion, region2: HicRegion,
|
|
64
|
+
getContactRecords(normalization: string, region1: HicRegion, region2: HicRegion, binsize: number, opts?: ProgressOpts): Promise<{
|
|
67
65
|
records: ContactRecords;
|
|
68
66
|
appliedNormalization: string;
|
|
69
67
|
transposed: boolean;
|
|
@@ -81,7 +79,7 @@ export declare class HicFile {
|
|
|
81
79
|
* of the session, so it is worth one memcpy not to pin an oversized buffer.
|
|
82
80
|
*/
|
|
83
81
|
private parseBlockRecords;
|
|
84
|
-
getNormalizationVector(type: string, chr: string,
|
|
82
|
+
getNormalizationVector(type: string, chr: string, binSize: number, opts?: BaseOpts): Promise<NormalizationVector | undefined>;
|
|
85
83
|
private readNormalizationVector;
|
|
86
84
|
getNormVectorIndex(opts?: ProgressOpts): Promise<Record<string, BlockIndexEntry> | undefined>;
|
|
87
85
|
private getParsedNormVectorIndex;
|
|
@@ -91,6 +89,5 @@ export declare class HicFile {
|
|
|
91
89
|
private readNormExpectedValuesAndNormVectorIndex;
|
|
92
90
|
private skipExpectedValues;
|
|
93
91
|
private parseNormVectorEntry;
|
|
94
|
-
getFileChrName(chrAlias: string): string;
|
|
95
92
|
}
|
|
96
93
|
export {};
|
package/dist/hicFile.js
CHANGED
|
@@ -10,7 +10,6 @@ const generic_filehandle2_1 = require("generic-filehandle2");
|
|
|
10
10
|
const binWindow_ts_1 = require("./binWindow.js");
|
|
11
11
|
const binary_ts_1 = __importDefault(require("./binary.js"));
|
|
12
12
|
const bufferedFile_ts_1 = __importDefault(require("./bufferedFile.js"));
|
|
13
|
-
const lru_ts_1 = __importDefault(require("./lru.js"));
|
|
14
13
|
const matrix_ts_1 = __importDefault(require("./matrix.js"));
|
|
15
14
|
const normalizationVector_ts_1 = __importDefault(require("./normalizationVector.js"));
|
|
16
15
|
const reader_ts_1 = require("./reader.js");
|
|
@@ -65,15 +64,13 @@ function truncateRecords(records, n) {
|
|
|
65
64
|
// The cliffs are exactly these three capacities. Note the first column: 41% of a
|
|
66
65
|
// cold whole-genome fetch's reads were re-reads of normalization vectors the
|
|
67
66
|
// same fetch had already fetched.
|
|
68
|
-
|
|
69
|
-
//
|
|
70
|
-
//
|
|
71
|
-
//
|
|
72
|
-
//
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
// the entry cap now tracks the working set and this tracks the memory, instead
|
|
76
|
-
// of one number answering both badly. See LRU's WeightOpts.
|
|
67
|
+
//
|
|
68
|
+
// Blocks became struct-of-arrays (see contactRecords.ts), so a cached contact is
|
|
69
|
+
// 12 bytes of untraced ArrayBuffer rather than ~55 on the GC-traced heap — but a
|
|
70
|
+
// block still holds every contact in its bin square, which is the one thing here
|
|
71
|
+
// that can hold real memory, and block sizes vary by more than an order of
|
|
72
|
+
// magnitude with binsize. Measured on the file above: 0.05 MB at 2.5 Mb, 0.23 MB
|
|
73
|
+
// at 100 kb. So the block cache's only bound is its byte budget.
|
|
77
74
|
//
|
|
78
75
|
// This is the one capacity here a caller can override (`blockCacheMaxBytes`),
|
|
79
76
|
// because it is the only one that bounds MEMORY. The two entry caps below are
|
|
@@ -95,6 +92,9 @@ const NORM_VECTOR_CACHE_SIZE = 64;
|
|
|
95
92
|
// hic-straw's extra `resolution` field only made the cache single-resolution
|
|
96
93
|
// (every zoom step threw the previous level away).
|
|
97
94
|
function blockCacheWeight(block) {
|
|
95
|
+
if (!block) {
|
|
96
|
+
return 0;
|
|
97
|
+
}
|
|
98
98
|
const { bin1, bin2, counts } = block.records;
|
|
99
99
|
return bin1.byteLength + bin2.byteLength + counts.byteLength;
|
|
100
100
|
}
|
|
@@ -102,7 +102,7 @@ function getNormalizationVectorKey(type, chrIdx, unit, resolution) {
|
|
|
102
102
|
return `${type}_${chrIdx}_${unit}_${resolution}`;
|
|
103
103
|
}
|
|
104
104
|
class HicFile {
|
|
105
|
-
|
|
105
|
+
nvi;
|
|
106
106
|
file;
|
|
107
107
|
// These caches share the in-flight read, not just its result. A
|
|
108
108
|
// multi-region fetch runs its region pairs concurrently and they share
|
|
@@ -129,17 +129,16 @@ class HicFile {
|
|
|
129
129
|
// make that second shape.
|
|
130
130
|
normExpectedValueVectorsPosition;
|
|
131
131
|
chromosomes = [];
|
|
132
|
-
chromosomeIndexMap =
|
|
133
|
-
chrAliasTable = {};
|
|
132
|
+
chromosomeIndexMap = new Map();
|
|
134
133
|
bpResolutions = [];
|
|
135
134
|
masterIndex = {};
|
|
136
135
|
meta;
|
|
137
136
|
constructor(config) {
|
|
138
|
-
const { filehandle, path, reader, blockCacheMaxBytes } = config;
|
|
139
|
-
this.
|
|
140
|
-
this.blockCache = new
|
|
141
|
-
|
|
142
|
-
|
|
137
|
+
const { filehandle, path, reader, nvi, blockCacheMaxBytes } = config;
|
|
138
|
+
this.nvi = nvi;
|
|
139
|
+
this.blockCache = new shared_read_cache_1.SharedReadCache({
|
|
140
|
+
maxSize: blockCacheMaxBytes ?? BLOCK_CACHE_MAX_BYTES,
|
|
141
|
+
sizeOf: blockCacheWeight,
|
|
143
142
|
});
|
|
144
143
|
if (reader) {
|
|
145
144
|
this.file = reader;
|
|
@@ -194,7 +193,7 @@ class HicFile {
|
|
|
194
193
|
binaryParser.getString(); // attribute value
|
|
195
194
|
}
|
|
196
195
|
const chromosomes = [];
|
|
197
|
-
const chromosomeIndexMap =
|
|
196
|
+
const chromosomeIndexMap = new Map();
|
|
198
197
|
let nChrs = binaryParser.getInt();
|
|
199
198
|
let i = 0;
|
|
200
199
|
while (nChrs-- > 0) {
|
|
@@ -204,7 +203,7 @@ class HicFile {
|
|
|
204
203
|
size: version < 9 ? binaryParser.getInt() : binaryParser.getLong(),
|
|
205
204
|
};
|
|
206
205
|
chromosomes.push(chr);
|
|
207
|
-
chromosomeIndexMap
|
|
206
|
+
chromosomeIndexMap.set(chr.name, chr.index);
|
|
208
207
|
i++;
|
|
209
208
|
}
|
|
210
209
|
const bpResolutions = [];
|
|
@@ -212,18 +211,6 @@ class HicFile {
|
|
|
212
211
|
while (nBpResolutions-- > 0) {
|
|
213
212
|
bpResolutions.push(binaryParser.getInt());
|
|
214
213
|
}
|
|
215
|
-
const chrAliasTable = {};
|
|
216
|
-
for (const chrName of Object.keys(chromosomeIndexMap)) {
|
|
217
|
-
if (chrName.startsWith('chr')) {
|
|
218
|
-
chrAliasTable[chrName.slice(3)] = chrName;
|
|
219
|
-
}
|
|
220
|
-
else if (chrName === 'MT') {
|
|
221
|
-
chrAliasTable.chrM = chrName;
|
|
222
|
-
}
|
|
223
|
-
else {
|
|
224
|
-
chrAliasTable[`chr${chrName}`] = chrName;
|
|
225
|
-
}
|
|
226
|
-
}
|
|
227
214
|
(0, shared_read_cache_1.throwIfAborted)(signal);
|
|
228
215
|
this.version = version;
|
|
229
216
|
this.footerPosition = footerPosition;
|
|
@@ -235,7 +222,6 @@ class HicFile {
|
|
|
235
222
|
this.chromosomes = chromosomes;
|
|
236
223
|
this.chromosomeIndexMap = chromosomeIndexMap;
|
|
237
224
|
this.bpResolutions = bpResolutions;
|
|
238
|
-
this.chrAliasTable = chrAliasTable;
|
|
239
225
|
this.meta = {
|
|
240
226
|
version,
|
|
241
227
|
genome: genomeId,
|
|
@@ -279,10 +265,10 @@ class HicFile {
|
|
|
279
265
|
}
|
|
280
266
|
return matrix;
|
|
281
267
|
}
|
|
282
|
-
async getContactRecords(normalization, region1, region2,
|
|
268
|
+
async getContactRecords(normalization, region1, region2, binsize, opts) {
|
|
283
269
|
await this.init(opts);
|
|
284
|
-
const idx1 = this.chromosomeIndexMap
|
|
285
|
-
const idx2 = this.chromosomeIndexMap
|
|
270
|
+
const idx1 = this.chromosomeIndexMap.get(region1.chr);
|
|
271
|
+
const idx2 = this.chromosomeIndexMap.get(region2.chr);
|
|
286
272
|
// A `.hic` stores only `bin1 <= bin2`, so a query whose x window sits to the
|
|
287
273
|
// right of its y window has to be swapped or it asks for the half of the
|
|
288
274
|
// matrix that does not exist. The same-chromosome test compares **starts**,
|
|
@@ -318,7 +304,7 @@ class HicFile {
|
|
|
318
304
|
// resolved once per pair rather than per block, each paired with the bin
|
|
319
305
|
// offset its values start at.
|
|
320
306
|
const [norm, blocks] = await Promise.all([
|
|
321
|
-
this.getNormVectors(normalization, r1, r2,
|
|
307
|
+
this.getNormVectors(normalization, r1, r2, binsize, opts),
|
|
322
308
|
// The measurable half of the pair: blocks are counted and the two norm
|
|
323
309
|
// vectors are not, because the vector chain is two hops whatever the
|
|
324
310
|
// query and the block chain is the one that grows with the region.
|
|
@@ -393,45 +379,41 @@ class HicFile {
|
|
|
393
379
|
//
|
|
394
380
|
// `transposed` says the query was swapped, so `bin1` runs along `region2`.
|
|
395
381
|
// Reported rather than left for the caller to re-derive: it is decided here
|
|
396
|
-
// from this file's
|
|
397
|
-
//
|
|
398
|
-
// wrong axis.
|
|
382
|
+
// from this file's chromosome indices, which a caller would otherwise have
|
|
383
|
+
// to look up again to know which axis to un-swap.
|
|
399
384
|
return {
|
|
400
385
|
records: contactRecords,
|
|
401
386
|
appliedNormalization: norm ? normalization : 'NONE',
|
|
402
387
|
transposed: transpose,
|
|
403
388
|
};
|
|
404
389
|
}
|
|
405
|
-
async getNormVectors(normalization, r1, r2,
|
|
390
|
+
async getNormVectors(normalization, r1, r2, binsize, opts) {
|
|
391
|
+
if (!normalization || normalization === 'NONE') {
|
|
392
|
+
return undefined;
|
|
393
|
+
}
|
|
406
394
|
const signalOpts = { signal: opts?.signal };
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
if (nv1 && nv2) {
|
|
418
|
-
result = {
|
|
419
|
-
v1: await nv1.getValues(offset1, Math.ceil(r1.end / binsize), signalOpts),
|
|
420
|
-
v2: await nv2.getValues(offset2, Math.ceil(r2.end / binsize), signalOpts),
|
|
421
|
-
offset1,
|
|
422
|
-
offset2,
|
|
423
|
-
};
|
|
424
|
-
}
|
|
395
|
+
const [offset1, end1] = (0, binWindow_ts_1.binWindow)(r1, binsize);
|
|
396
|
+
const [offset2, end2] = (0, binWindow_ts_1.binWindow)(r2, binsize);
|
|
397
|
+
// Awaited in sequence, an inter-chromosomal pair would pay four
|
|
398
|
+
// round-trip waves for what is two hops per vector.
|
|
399
|
+
const [nv1, nv2] = await Promise.all([
|
|
400
|
+
this.getNormalizationVector(normalization, r1.chr, binsize, signalOpts),
|
|
401
|
+
this.getNormalizationVector(normalization, r2.chr, binsize, signalOpts),
|
|
402
|
+
]);
|
|
403
|
+
if (!nv1 || !nv2) {
|
|
404
|
+
return undefined;
|
|
425
405
|
}
|
|
426
|
-
|
|
406
|
+
const [v1, v2] = await Promise.all([
|
|
407
|
+
nv1.getValues(offset1, end1, signalOpts),
|
|
408
|
+
nv2.getValues(offset2, end2, signalOpts),
|
|
409
|
+
]);
|
|
410
|
+
return { v1, v2, offset1, offset2 };
|
|
427
411
|
}
|
|
428
412
|
async getBlocks(region1, region2, binSize, opts) {
|
|
429
413
|
const blockKey = (blockNumber, zd) => `${zd.getKey()}_${blockNumber}`;
|
|
430
414
|
await this.init(opts);
|
|
431
|
-
const
|
|
432
|
-
const
|
|
433
|
-
const idx1 = this.chromosomeIndexMap[chr1];
|
|
434
|
-
const idx2 = this.chromosomeIndexMap[chr2];
|
|
415
|
+
const idx1 = this.chromosomeIndexMap.get(region1.chr);
|
|
416
|
+
const idx2 = this.chromosomeIndexMap.get(region2.chr);
|
|
435
417
|
let blocks = [];
|
|
436
418
|
if (idx1 === undefined) {
|
|
437
419
|
console.warn(`No chromosome named: ${region1.chr}`);
|
|
@@ -452,43 +434,37 @@ class HicFile {
|
|
|
452
434
|
throw new Error(`${exports.NO_DATA_FOR_RESOLUTION}: ${binSize} for map ${region1.chr}-${region2.chr}`);
|
|
453
435
|
}
|
|
454
436
|
const blockNumbers = zd.getBlockNumbers(region1, region2, this.version);
|
|
455
|
-
const
|
|
456
|
-
|
|
457
|
-
const cached = this.blockCache.get(blockKey(num, zd));
|
|
458
|
-
if (cached) {
|
|
459
|
-
blocks.push(cached);
|
|
460
|
-
}
|
|
461
|
-
else {
|
|
462
|
-
blockNumbersToQuery.push(num);
|
|
463
|
-
}
|
|
464
|
-
}
|
|
437
|
+
const keys = blockNumbers.map(num => blockKey(num, zd));
|
|
438
|
+
const cached = keys.map(key => this.blockCache.has(key));
|
|
465
439
|
// Cached blocks are already done, not absent: a pan that reuses most
|
|
466
440
|
// of its blocks reads most of the way along rather than restarting at
|
|
467
441
|
// zero, which is what the bar should say — the work left is the reads.
|
|
442
|
+
// A block another fetch has in flight counts as cached too.
|
|
468
443
|
//
|
|
469
444
|
// A query covering no blocks at all reports nothing rather than `0, 0`:
|
|
470
445
|
// there is no fraction in it, and a caller dividing would get NaN.
|
|
471
446
|
const { onProgress } = opts ?? {};
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
447
|
+
const total = blockNumbers.length;
|
|
448
|
+
let done = cached.filter(Boolean).length;
|
|
449
|
+
if (total > 0) {
|
|
450
|
+
onProgress?.(done, total);
|
|
475
451
|
}
|
|
476
|
-
//
|
|
477
|
-
//
|
|
478
|
-
//
|
|
479
|
-
//
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
452
|
+
// One `Promise.all`, so every block is issued in one wave: what a
|
|
453
|
+
// remote file pays is round-trip DEPTH, and counting the completions
|
|
454
|
+
// must not turn one wave into a queue. `readChainDepth` pins that.
|
|
455
|
+
//
|
|
456
|
+
// A block number missing from the index has no data, so it skips the
|
|
457
|
+
// cache, which would otherwise keep a weightless entry for it forever.
|
|
458
|
+
blocks = await Promise.all(blockNumbers.map(async (num, i) => {
|
|
459
|
+
const block = zd.blockIndex[num]
|
|
460
|
+
? await this.blockCache.get(keys[i], opts?.signal, signal => this.readBlock(num, zd, { signal }))
|
|
461
|
+
: undefined;
|
|
462
|
+
if (!cached[i]) {
|
|
463
|
+
done++;
|
|
464
|
+
onProgress?.(done, total);
|
|
465
|
+
}
|
|
484
466
|
return block;
|
|
485
467
|
}));
|
|
486
|
-
for (const block of newBlocks) {
|
|
487
|
-
if (block) {
|
|
488
|
-
this.blockCache.set(blockKey(block.blockNumber, zd), block);
|
|
489
|
-
}
|
|
490
|
-
}
|
|
491
|
-
blocks = blocks.concat(newBlocks);
|
|
492
468
|
}
|
|
493
469
|
}
|
|
494
470
|
return blocks;
|
|
@@ -595,13 +571,13 @@ class HicFile {
|
|
|
595
571
|
}
|
|
596
572
|
throw new Error(`Unknown block type: ${type}`);
|
|
597
573
|
}
|
|
598
|
-
async getNormalizationVector(type, chr,
|
|
574
|
+
async getNormalizationVector(type, chr, binSize, opts) {
|
|
599
575
|
await this.init(opts);
|
|
600
|
-
const chrIdx = this.chromosomeIndexMap
|
|
576
|
+
const chrIdx = this.chromosomeIndexMap.get(chr);
|
|
601
577
|
if (chrIdx === undefined) {
|
|
602
578
|
return undefined;
|
|
603
579
|
}
|
|
604
|
-
const key = getNormalizationVectorKey(type, chrIdx,
|
|
580
|
+
const key = getNormalizationVectorKey(type, chrIdx, 'BP', binSize);
|
|
605
581
|
// Sharing the in-flight read is what keeps concurrent region pairs sharing
|
|
606
582
|
// one vector: a chromosome appears in every pair it takes part in, so with a
|
|
607
583
|
// result-only cache each of those pairs missed while the first was still in
|
|
@@ -658,8 +634,8 @@ class HicFile {
|
|
|
658
634
|
size: this.normVectorIndexSize,
|
|
659
635
|
}, opts.signal);
|
|
660
636
|
}
|
|
661
|
-
else if (this.
|
|
662
|
-
const nviArray = decodeURIComponent(this.
|
|
637
|
+
else if (this.nvi) {
|
|
638
|
+
const nviArray = decodeURIComponent(this.nvi).split(',');
|
|
663
639
|
return this.readNormVectorIndex({
|
|
664
640
|
start: parseInt(nviArray[0]),
|
|
665
641
|
size: parseInt(nviArray[1]),
|
|
@@ -703,38 +679,27 @@ class HicFile {
|
|
|
703
679
|
let index;
|
|
704
680
|
if (this.normExpectedValueVectorsPosition !== undefined) {
|
|
705
681
|
const nviStart = await this.skipExpectedValues(this.normExpectedValueVectorsPosition, opts);
|
|
706
|
-
|
|
707
|
-
let data = await this.file.read(nviStart, INT, { signal });
|
|
682
|
+
const data = await this.file.read(nviStart, INT, { signal });
|
|
708
683
|
// Possible if there are no norm vectors. A legal v8 file, though uncommon.
|
|
709
684
|
if (data.byteLength !== 0) {
|
|
710
|
-
const
|
|
711
|
-
const nEntries = binaryParser.getInt();
|
|
712
|
-
const sizeEstimate = nEntries * 30;
|
|
713
|
-
data = await this.file.read(nviStart + byteCount, sizeEstimate, {
|
|
714
|
-
signal,
|
|
715
|
-
});
|
|
685
|
+
const nEntries = new binary_ts_1.default(new DataView(data)).getInt();
|
|
716
686
|
const found = { entries: {}, types: ['NONE'] };
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
return;
|
|
730
|
-
}
|
|
731
|
-
this.parseNormVectorEntry(parser, found);
|
|
687
|
+
let position = nviStart + INT;
|
|
688
|
+
let requested = nEntries * 30;
|
|
689
|
+
let buf = await this.file.read(position, requested, { signal });
|
|
690
|
+
let parser = new binary_ts_1.default(new DataView(buf));
|
|
691
|
+
for (let n = nEntries; n > 0; n--) {
|
|
692
|
+
// A short read means the buffer already runs to EOF, and refilling
|
|
693
|
+
// it would only return the same bytes again.
|
|
694
|
+
if (parser.available() < 100 && buf.byteLength === requested) {
|
|
695
|
+
position += parser.position;
|
|
696
|
+
requested = Math.max(1000, n * 30);
|
|
697
|
+
buf = await this.file.read(position, requested, { signal });
|
|
698
|
+
parser = new binary_ts_1.default(new DataView(buf));
|
|
732
699
|
}
|
|
733
|
-
|
|
734
|
-
}
|
|
735
|
-
await processEntries(nEntries, data);
|
|
700
|
+
this.parseNormVectorEntry(parser, found);
|
|
701
|
+
}
|
|
736
702
|
(0, shared_read_cache_1.throwIfAborted)(signal);
|
|
737
|
-
this.config.nvi = `${nviStart},${byteCount}`;
|
|
738
703
|
index = found;
|
|
739
704
|
}
|
|
740
705
|
}
|
|
@@ -806,15 +771,12 @@ class HicFile {
|
|
|
806
771
|
const binSize = binaryParser.getInt(); // 4
|
|
807
772
|
const filePosition = binaryParser.getLong(); // 8
|
|
808
773
|
const sizeInBytes = this.version < 9 ? binaryParser.getInt() : binaryParser.getLong(); // 4:8
|
|
809
|
-
const key =
|
|
774
|
+
const key = getNormalizationVectorKey(type, chrIdx, unit, binSize);
|
|
810
775
|
if (!index.types.includes(type)) {
|
|
811
776
|
index.types.push(type);
|
|
812
777
|
}
|
|
813
778
|
index.entries[key] = { filePosition, size: sizeInBytes };
|
|
814
779
|
}
|
|
815
|
-
getFileChrName(chrAlias) {
|
|
816
|
-
return this.chrAliasTable[chrAlias] ?? chrAlias;
|
|
817
|
-
}
|
|
818
780
|
}
|
|
819
781
|
exports.HicFile = HicFile;
|
|
820
782
|
//# sourceMappingURL=hicFile.js.map
|