@gmod/hic 1.3.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -32
- package/dist/bufferedFile.js +8 -25
- package/dist/bufferedFile.js.map +1 -1
- package/dist/hicFile.d.ts +4 -7
- package/dist/hicFile.js +89 -127
- package/dist/hicFile.js.map +1 -1
- package/dist/normalizationVector.d.ts +3 -1
- package/dist/normalizationVector.js +60 -39
- package/dist/normalizationVector.js.map +1 -1
- package/esm/bufferedFile.js +8 -25
- package/esm/bufferedFile.js.map +1 -1
- package/esm/hicFile.d.ts +4 -7
- package/esm/hicFile.js +89 -127
- package/esm/hicFile.js.map +1 -1
- package/esm/normalizationVector.d.ts +3 -1
- package/esm/normalizationVector.js +60 -39
- package/esm/normalizationVector.js.map +1 -1
- package/package.json +1 -1
- package/src/bufferedFile.ts +8 -22
- package/src/hicFile.ts +92 -161
- package/src/normalizationVector.ts +77 -43
- package/dist/lru.d.ts +0 -29
- package/dist/lru.js +0 -53
- package/dist/lru.js.map +0 -1
- package/esm/lru.d.ts +0 -29
- package/esm/lru.js +0 -50
- package/esm/lru.js.map +0 -1
- package/src/lru.ts +0 -75
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"normalizationVector.js","sourceRoot":"","sources":["../src/normalizationVector.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,aAAa,CAAA;AAItC,MAAM,MAAM,GAAG,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"normalizationVector.js","sourceRoot":"","sources":["../src/normalizationVector.ts"],"names":[],"mappings":"AAAA,OAAO,YAAY,MAAM,aAAa,CAAA;AAItC,MAAM,MAAM,GAAG,CAAC,CAAA;AAQhB,MAAM,CAAC,OAAO,OAAO,mBAAmB;IACtC,yEAAyE;IACzE,+DAA+D;IACvD,KAAK,CAA0B;IAE/B,IAAI,CAAQ;IACZ,YAAY,CAAQ;IACpB,OAAO,CAAQ;IACf,QAAQ,CAAQ;IAExB,YACE,IAAY,EACZ,YAAoB,EACpB,OAAe,EACf,QAAgB;QAEhB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAChB,IAAI,CAAC,YAAY,GAAG,YAAY,CAAA;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;QACtB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC1B,CAAC;IAED,KAAK,CAAC,SAAS,CACb,KAAa,EACb,GAAW,EACX,IAAe;QAEf,sEAAsE;QACtE,0EAA0E;QAC1E,0EAA0E;QAC1E,mBAAmB;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;QAC1C,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAA;QACtB,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,KAAK,IAAI,KAAK,CAAC,KAAK,IAAI,MAAM,IAAI,KAAK,CAAC,GAAG,CAAA;QACrE,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;YACtB,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;YACxC,IAAI,CAAC,KAAK,GAAG,KAAK,CAAA;YAClB,MAAM,OAAO,GAAG,KAAK,CAAA;YACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE;gBACxB,IAAI,IAAI,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;oBAC3B,IAAI,CAAC,KAAK,GAAG,SAAS,CAAA;gBACxB,CAAC;YACH,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,IAAI,MAAoB,CAAA;QACxB,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAA;QAC7B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,qEAAqE;YACrE,gDAAgD;YAChD,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;gBACrC,MAAM,CAAC,CAAA;YACT,CAAC;YACD,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAA;QACzC,CAAC;QAED,4EAA4E;QAC5E,4EAA4E;QAC5E,8BAA8B;QAC9B,MAAM,UAAU,GAAG,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QACtC,OAAO,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,UAAU,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,CAAA;IAChE,CAAC;IAEO,SAAS,CAAC,KAAa,EAAE,GAAW,EAAE,IAAe;QAC3D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,IAAI,CAAC,CAAA;QACtD,0EAA0E;QAC1E,yEAAyE;QACzE,kDAAkD;QAClD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,EAAE,WAAW,CAAC,CAAA;QACtE,MAAM,CAAC,GAAG,WAAW,GAAG,aAAa,CAAA;QACrC,OAAO;YACL,KAAK,EAAE,aAAa;YACpB,GAAG,EAAE,WAAW;YAChB,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,aAAa,EAAE,CAAC,EAAE,IAAI,CAAC;SAChD,CAAA;IACH,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAa,EAAE,CAAS,EAAE,IAAe;QAChE,sEAAsE;QACtE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACX,OAAO,IAAI,YAAY,CAAC,CAAC,CAAC,CAAA;QAC5B,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAC/B,IAAI,CAAC,YAAY,GAAG,KAAK,GAAG,IAAI,CAAC,QAAQ,EACzC,CAAC,GAAG,IAAI,CAAC,QAAQ,EACjB,IAAI,CACL,CAAA;QACD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAA;QACnD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,CAAA;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,CAAC,CAAC,CAAC;gBACP,IAAI,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAA;QACrE,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;CACF"}
|
package/package.json
CHANGED
package/src/bufferedFile.ts
CHANGED
|
@@ -41,30 +41,16 @@ export default class BufferedFile {
|
|
|
41
41
|
// Request within buffer bounds
|
|
42
42
|
const sliceStart = start - bufferStart
|
|
43
43
|
result = buf.slice(sliceStart, sliceStart + length)
|
|
44
|
-
} else if (buf && start
|
|
45
|
-
// Overlap
|
|
46
|
-
|
|
47
|
-
const a1 = await this.file.read(position, l1, opts)
|
|
48
|
-
const l2 = length - l1
|
|
49
|
-
if (l2 > 0) {
|
|
50
|
-
result = concatBuffers(a1, buf.slice(0, l2))
|
|
51
|
-
} else {
|
|
52
|
-
result = a1
|
|
53
|
-
}
|
|
54
|
-
} else if (buf && start < bufferEnd && end > bufferEnd) {
|
|
55
|
-
// Overlap right
|
|
44
|
+
} else if (buf && start >= bufferStart && start < bufferEnd) {
|
|
45
|
+
// Overlap right. The expected-values walk only moves forward, so a
|
|
46
|
+
// request starting before the buffer takes the fresh read below.
|
|
56
47
|
const l1 = bufferEnd - start
|
|
57
48
|
const a1 = buf.slice(this.bufferLength - l1, this.bufferLength)
|
|
58
|
-
const
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
this.bufferLength = next.byteLength
|
|
64
|
-
result = concatBuffers(a1, next.slice(0, l2))
|
|
65
|
-
} else {
|
|
66
|
-
result = a1
|
|
67
|
-
}
|
|
49
|
+
const next = await this.file.read(bufferEnd, this.size, opts)
|
|
50
|
+
this.buffer = next
|
|
51
|
+
this.bufferStart = bufferEnd
|
|
52
|
+
this.bufferLength = next.byteLength
|
|
53
|
+
result = concatBuffers(a1, next.slice(0, length - l1))
|
|
68
54
|
} else {
|
|
69
55
|
// No overlap with buffer
|
|
70
56
|
const next = await this.file.read(position, this.size, opts)
|
package/src/hicFile.ts
CHANGED
|
@@ -5,7 +5,6 @@ import { LocalFile } from 'generic-filehandle2'
|
|
|
5
5
|
import { binWindow } from './binWindow.ts'
|
|
6
6
|
import BinaryParser from './binary.ts'
|
|
7
7
|
import BufferedFile from './bufferedFile.ts'
|
|
8
|
-
import LRU from './lru.ts'
|
|
9
8
|
import Matrix from './matrix.ts'
|
|
10
9
|
import NormalizationVector from './normalizationVector.ts'
|
|
11
10
|
import { readerFromFilehandle } from './reader.ts'
|
|
@@ -66,8 +65,7 @@ export interface HicConfig {
|
|
|
66
65
|
/**
|
|
67
66
|
* Ceiling on the decompressed contacts the block cache holds, in bytes.
|
|
68
67
|
* Defaults to 128 MB. Lower it where the budget is tight and re-reading is
|
|
69
|
-
* cheaper than the memory
|
|
70
|
-
* fits, since the entry cap bounds the cache too.
|
|
68
|
+
* cheaper than the memory.
|
|
71
69
|
*/
|
|
72
70
|
blockCacheMaxBytes?: number
|
|
73
71
|
}
|
|
@@ -75,7 +73,7 @@ export interface HicConfig {
|
|
|
75
73
|
// Cached blocks are the decompressed records and nothing else. hic-straw also
|
|
76
74
|
// hung the MatrixZoomData and the block-index entry off each Block; neither was
|
|
77
75
|
// ever read back, and the zoom data pinned a whole `blockIndex` record per
|
|
78
|
-
// cached block for the lifetime of the
|
|
76
|
+
// cached block for the lifetime of the cache.
|
|
79
77
|
interface Block {
|
|
80
78
|
blockNumber: number
|
|
81
79
|
records: ContactRecords
|
|
@@ -122,15 +120,13 @@ function truncateRecords(records: ContactRecords, n: number): ContactRecords {
|
|
|
122
120
|
// The cliffs are exactly these three capacities. Note the first column: 41% of a
|
|
123
121
|
// cold whole-genome fetch's reads were re-reads of normalization vectors the
|
|
124
122
|
// same fetch had already fetched.
|
|
125
|
-
|
|
126
|
-
//
|
|
127
|
-
//
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
// the entry cap now tracks the working set and this tracks the memory, instead
|
|
133
|
-
// of one number answering both badly. See LRU's WeightOpts.
|
|
123
|
+
//
|
|
124
|
+
// Blocks became struct-of-arrays (see contactRecords.ts), so a cached contact is
|
|
125
|
+
// 12 bytes of untraced ArrayBuffer rather than ~55 on the GC-traced heap — but a
|
|
126
|
+
// block still holds every contact in its bin square, which is the one thing here
|
|
127
|
+
// that can hold real memory, and block sizes vary by more than an order of
|
|
128
|
+
// magnitude with binsize. Measured on the file above: 0.05 MB at 2.5 Mb, 0.23 MB
|
|
129
|
+
// at 100 kb. So the block cache's only bound is its byte budget.
|
|
134
130
|
//
|
|
135
131
|
// This is the one capacity here a caller can override (`blockCacheMaxBytes`),
|
|
136
132
|
// because it is the only one that bounds MEMORY. The two entry caps below are
|
|
@@ -152,7 +148,10 @@ const NORM_VECTOR_CACHE_SIZE = 64
|
|
|
152
148
|
// so no separate resolution generation is needed to keep entries apart —
|
|
153
149
|
// hic-straw's extra `resolution` field only made the cache single-resolution
|
|
154
150
|
// (every zoom step threw the previous level away).
|
|
155
|
-
function blockCacheWeight(block: Block) {
|
|
151
|
+
function blockCacheWeight(block: Block | undefined) {
|
|
152
|
+
if (!block) {
|
|
153
|
+
return 0
|
|
154
|
+
}
|
|
156
155
|
const { bin1, bin2, counts } = block.records
|
|
157
156
|
return bin1.byteLength + bin2.byteLength + counts.byteLength
|
|
158
157
|
}
|
|
@@ -167,7 +166,7 @@ function getNormalizationVectorKey(
|
|
|
167
166
|
}
|
|
168
167
|
|
|
169
168
|
export class HicFile {
|
|
170
|
-
private
|
|
169
|
+
private nvi: string | undefined
|
|
171
170
|
private file: Reader
|
|
172
171
|
|
|
173
172
|
// These caches share the in-flight read, not just its result. A
|
|
@@ -190,7 +189,7 @@ export class HicFile {
|
|
|
190
189
|
})
|
|
191
190
|
// Assigned in the constructor rather than here, since its byte budget is
|
|
192
191
|
// configurable and a field initializer cannot see `config`.
|
|
193
|
-
private blockCache:
|
|
192
|
+
private blockCache: SharedReadCache<string, Block | undefined>
|
|
194
193
|
private normVectorIndexPosition = -1
|
|
195
194
|
private normVectorIndexSize = -1
|
|
196
195
|
|
|
@@ -203,18 +202,17 @@ export class HicFile {
|
|
|
203
202
|
normExpectedValueVectorsPosition: number | undefined
|
|
204
203
|
|
|
205
204
|
private chromosomes: Chromosome[] = []
|
|
206
|
-
private chromosomeIndexMap
|
|
207
|
-
private chrAliasTable: Record<string, string> = {}
|
|
205
|
+
private chromosomeIndexMap = new Map<string, number>()
|
|
208
206
|
private bpResolutions: number[] = []
|
|
209
207
|
private masterIndex: Record<string, MasterIndexEntry> = {}
|
|
210
208
|
private meta: HicMetadata | undefined
|
|
211
209
|
|
|
212
210
|
constructor(config: HicConfig) {
|
|
213
|
-
const { filehandle, path, reader, blockCacheMaxBytes } = config
|
|
214
|
-
this.
|
|
215
|
-
this.blockCache = new
|
|
216
|
-
|
|
217
|
-
|
|
211
|
+
const { filehandle, path, reader, nvi, blockCacheMaxBytes } = config
|
|
212
|
+
this.nvi = nvi
|
|
213
|
+
this.blockCache = new SharedReadCache({
|
|
214
|
+
maxSize: blockCacheMaxBytes ?? BLOCK_CACHE_MAX_BYTES,
|
|
215
|
+
sizeOf: blockCacheWeight,
|
|
218
216
|
})
|
|
219
217
|
if (reader) {
|
|
220
218
|
this.file = reader
|
|
@@ -282,7 +280,7 @@ export class HicFile {
|
|
|
282
280
|
}
|
|
283
281
|
|
|
284
282
|
const chromosomes: Chromosome[] = []
|
|
285
|
-
const chromosomeIndexMap
|
|
283
|
+
const chromosomeIndexMap = new Map<string, number>()
|
|
286
284
|
let nChrs = binaryParser.getInt()
|
|
287
285
|
let i = 0
|
|
288
286
|
while (nChrs-- > 0) {
|
|
@@ -292,7 +290,7 @@ export class HicFile {
|
|
|
292
290
|
size: version < 9 ? binaryParser.getInt() : binaryParser.getLong(),
|
|
293
291
|
}
|
|
294
292
|
chromosomes.push(chr)
|
|
295
|
-
chromosomeIndexMap
|
|
293
|
+
chromosomeIndexMap.set(chr.name, chr.index)
|
|
296
294
|
i++
|
|
297
295
|
}
|
|
298
296
|
|
|
@@ -302,17 +300,6 @@ export class HicFile {
|
|
|
302
300
|
bpResolutions.push(binaryParser.getInt())
|
|
303
301
|
}
|
|
304
302
|
|
|
305
|
-
const chrAliasTable: Record<string, string> = {}
|
|
306
|
-
for (const chrName of Object.keys(chromosomeIndexMap)) {
|
|
307
|
-
if (chrName.startsWith('chr')) {
|
|
308
|
-
chrAliasTable[chrName.slice(3)] = chrName
|
|
309
|
-
} else if (chrName === 'MT') {
|
|
310
|
-
chrAliasTable.chrM = chrName
|
|
311
|
-
} else {
|
|
312
|
-
chrAliasTable[`chr${chrName}`] = chrName
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
|
|
316
303
|
throwIfAborted(signal)
|
|
317
304
|
this.version = version
|
|
318
305
|
this.footerPosition = footerPosition
|
|
@@ -324,7 +311,6 @@ export class HicFile {
|
|
|
324
311
|
this.chromosomes = chromosomes
|
|
325
312
|
this.chromosomeIndexMap = chromosomeIndexMap
|
|
326
313
|
this.bpResolutions = bpResolutions
|
|
327
|
-
this.chrAliasTable = chrAliasTable
|
|
328
314
|
this.meta = {
|
|
329
315
|
version,
|
|
330
316
|
genome: genomeId,
|
|
@@ -397,14 +383,13 @@ export class HicFile {
|
|
|
397
383
|
normalization: string,
|
|
398
384
|
region1: HicRegion,
|
|
399
385
|
region2: HicRegion,
|
|
400
|
-
units: string,
|
|
401
386
|
binsize: number,
|
|
402
387
|
opts?: ProgressOpts,
|
|
403
388
|
) {
|
|
404
389
|
await this.init(opts)
|
|
405
390
|
|
|
406
|
-
const idx1 = this.chromosomeIndexMap
|
|
407
|
-
const idx2 = this.chromosomeIndexMap
|
|
391
|
+
const idx1 = this.chromosomeIndexMap.get(region1.chr)
|
|
392
|
+
const idx2 = this.chromosomeIndexMap.get(region2.chr)
|
|
408
393
|
|
|
409
394
|
// A `.hic` stores only `bin1 <= bin2`, so a query whose x window sits to the
|
|
410
395
|
// right of its y window has to be swapped or it asks for the half of the
|
|
@@ -444,7 +429,7 @@ export class HicFile {
|
|
|
444
429
|
// resolved once per pair rather than per block, each paired with the bin
|
|
445
430
|
// offset its values start at.
|
|
446
431
|
const [norm, blocks] = await Promise.all([
|
|
447
|
-
this.getNormVectors(normalization, r1, r2,
|
|
432
|
+
this.getNormVectors(normalization, r1, r2, binsize, opts),
|
|
448
433
|
// The measurable half of the pair: blocks are counted and the two norm
|
|
449
434
|
// vectors are not, because the vector chain is two hops whatever the
|
|
450
435
|
// query and the block chain is the one that grows with the region.
|
|
@@ -522,9 +507,8 @@ export class HicFile {
|
|
|
522
507
|
//
|
|
523
508
|
// `transposed` says the query was swapped, so `bin1` runs along `region2`.
|
|
524
509
|
// Reported rather than left for the caller to re-derive: it is decided here
|
|
525
|
-
// from this file's
|
|
526
|
-
//
|
|
527
|
-
// wrong axis.
|
|
510
|
+
// from this file's chromosome indices, which a caller would otherwise have
|
|
511
|
+
// to look up again to know which axis to un-swap.
|
|
528
512
|
return {
|
|
529
513
|
records: contactRecords,
|
|
530
514
|
appliedNormalization: norm ? normalization : 'NONE',
|
|
@@ -536,59 +520,29 @@ export class HicFile {
|
|
|
536
520
|
normalization: string,
|
|
537
521
|
r1: HicRegion,
|
|
538
522
|
r2: HicRegion,
|
|
539
|
-
units: string,
|
|
540
523
|
binsize: number,
|
|
541
524
|
opts?: BaseOpts,
|
|
542
525
|
) {
|
|
526
|
+
if (!normalization || normalization === 'NONE') {
|
|
527
|
+
return undefined
|
|
528
|
+
}
|
|
543
529
|
const signalOpts = { signal: opts?.signal }
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
if (
|
|
553
|
-
|
|
554
|
-
const chr2 = this.getFileChrName(r2.chr)
|
|
555
|
-
const offset1 = Math.floor(r1.start / binsize)
|
|
556
|
-
const offset2 = Math.floor(r2.start / binsize)
|
|
557
|
-
const nv1 = await this.getNormalizationVector(
|
|
558
|
-
normalization,
|
|
559
|
-
chr1,
|
|
560
|
-
units,
|
|
561
|
-
binsize,
|
|
562
|
-
signalOpts,
|
|
563
|
-
)
|
|
564
|
-
const nv2 =
|
|
565
|
-
chr1 === chr2
|
|
566
|
-
? nv1
|
|
567
|
-
: await this.getNormalizationVector(
|
|
568
|
-
normalization,
|
|
569
|
-
chr2,
|
|
570
|
-
units,
|
|
571
|
-
binsize,
|
|
572
|
-
signalOpts,
|
|
573
|
-
)
|
|
574
|
-
if (nv1 && nv2) {
|
|
575
|
-
result = {
|
|
576
|
-
v1: await nv1.getValues(
|
|
577
|
-
offset1,
|
|
578
|
-
Math.ceil(r1.end / binsize),
|
|
579
|
-
signalOpts,
|
|
580
|
-
),
|
|
581
|
-
v2: await nv2.getValues(
|
|
582
|
-
offset2,
|
|
583
|
-
Math.ceil(r2.end / binsize),
|
|
584
|
-
signalOpts,
|
|
585
|
-
),
|
|
586
|
-
offset1,
|
|
587
|
-
offset2,
|
|
588
|
-
}
|
|
589
|
-
}
|
|
530
|
+
const [offset1, end1] = binWindow(r1, binsize)
|
|
531
|
+
const [offset2, end2] = binWindow(r2, binsize)
|
|
532
|
+
// Awaited in sequence, an inter-chromosomal pair would pay four
|
|
533
|
+
// round-trip waves for what is two hops per vector.
|
|
534
|
+
const [nv1, nv2] = await Promise.all([
|
|
535
|
+
this.getNormalizationVector(normalization, r1.chr, binsize, signalOpts),
|
|
536
|
+
this.getNormalizationVector(normalization, r2.chr, binsize, signalOpts),
|
|
537
|
+
])
|
|
538
|
+
if (!nv1 || !nv2) {
|
|
539
|
+
return undefined
|
|
590
540
|
}
|
|
591
|
-
|
|
541
|
+
const [v1, v2] = await Promise.all([
|
|
542
|
+
nv1.getValues(offset1, end1, signalOpts),
|
|
543
|
+
nv2.getValues(offset2, end2, signalOpts),
|
|
544
|
+
])
|
|
545
|
+
return { v1, v2, offset1, offset2 }
|
|
592
546
|
}
|
|
593
547
|
|
|
594
548
|
async getBlocks(
|
|
@@ -601,10 +555,8 @@ export class HicFile {
|
|
|
601
555
|
`${zd.getKey()}_${blockNumber}`
|
|
602
556
|
|
|
603
557
|
await this.init(opts)
|
|
604
|
-
const
|
|
605
|
-
const
|
|
606
|
-
const idx1 = this.chromosomeIndexMap[chr1]
|
|
607
|
-
const idx2 = this.chromosomeIndexMap[chr2]
|
|
558
|
+
const idx1 = this.chromosomeIndexMap.get(region1.chr)
|
|
559
|
+
const idx2 = this.chromosomeIndexMap.get(region2.chr)
|
|
608
560
|
|
|
609
561
|
let blocks: (Block | undefined)[] = []
|
|
610
562
|
if (idx1 === undefined) {
|
|
@@ -627,45 +579,42 @@ export class HicFile {
|
|
|
627
579
|
}
|
|
628
580
|
|
|
629
581
|
const blockNumbers = zd.getBlockNumbers(region1, region2, this.version)
|
|
630
|
-
const
|
|
631
|
-
|
|
632
|
-
const cached = this.blockCache.get(blockKey(num, zd))
|
|
633
|
-
if (cached) {
|
|
634
|
-
blocks.push(cached)
|
|
635
|
-
} else {
|
|
636
|
-
blockNumbersToQuery.push(num)
|
|
637
|
-
}
|
|
638
|
-
}
|
|
582
|
+
const keys = blockNumbers.map(num => blockKey(num, zd))
|
|
583
|
+
const cached = keys.map(key => this.blockCache.has(key))
|
|
639
584
|
|
|
640
585
|
// Cached blocks are already done, not absent: a pan that reuses most
|
|
641
586
|
// of its blocks reads most of the way along rather than restarting at
|
|
642
587
|
// zero, which is what the bar should say — the work left is the reads.
|
|
588
|
+
// A block another fetch has in flight counts as cached too.
|
|
643
589
|
//
|
|
644
590
|
// A query covering no blocks at all reports nothing rather than `0, 0`:
|
|
645
591
|
// there is no fraction in it, and a caller dividing would get NaN.
|
|
646
592
|
const { onProgress } = opts ?? {}
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
593
|
+
const total = blockNumbers.length
|
|
594
|
+
let done = cached.filter(Boolean).length
|
|
595
|
+
if (total > 0) {
|
|
596
|
+
onProgress?.(done, total)
|
|
650
597
|
}
|
|
651
|
-
//
|
|
652
|
-
//
|
|
653
|
-
//
|
|
654
|
-
//
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
598
|
+
// One `Promise.all`, so every block is issued in one wave: what a
|
|
599
|
+
// remote file pays is round-trip DEPTH, and counting the completions
|
|
600
|
+
// must not turn one wave into a queue. `readChainDepth` pins that.
|
|
601
|
+
//
|
|
602
|
+
// A block number missing from the index has no data, so it skips the
|
|
603
|
+
// cache, which would otherwise keep a weightless entry for it forever.
|
|
604
|
+
blocks = await Promise.all(
|
|
605
|
+
blockNumbers.map(async (num, i) => {
|
|
606
|
+
const block = zd.blockIndex[num]
|
|
607
|
+
? await this.blockCache.get(keys[i]!, opts?.signal, signal =>
|
|
608
|
+
this.readBlock(num, zd, { signal }),
|
|
609
|
+
)
|
|
610
|
+
: undefined
|
|
611
|
+
if (!cached[i]) {
|
|
612
|
+
done++
|
|
613
|
+
onProgress?.(done, total)
|
|
614
|
+
}
|
|
660
615
|
return block
|
|
661
616
|
}),
|
|
662
617
|
)
|
|
663
|
-
for (const block of newBlocks) {
|
|
664
|
-
if (block) {
|
|
665
|
-
this.blockCache.set(blockKey(block.blockNumber, zd), block)
|
|
666
|
-
}
|
|
667
|
-
}
|
|
668
|
-
blocks = blocks.concat(newBlocks)
|
|
669
618
|
}
|
|
670
619
|
}
|
|
671
620
|
return blocks
|
|
@@ -793,17 +742,16 @@ export class HicFile {
|
|
|
793
742
|
async getNormalizationVector(
|
|
794
743
|
type: string,
|
|
795
744
|
chr: string,
|
|
796
|
-
unit: string,
|
|
797
745
|
binSize: number,
|
|
798
746
|
opts?: BaseOpts,
|
|
799
747
|
) {
|
|
800
748
|
await this.init(opts)
|
|
801
749
|
|
|
802
|
-
const chrIdx = this.chromosomeIndexMap
|
|
750
|
+
const chrIdx = this.chromosomeIndexMap.get(chr)
|
|
803
751
|
if (chrIdx === undefined) {
|
|
804
752
|
return undefined
|
|
805
753
|
}
|
|
806
|
-
const key = getNormalizationVectorKey(type, chrIdx,
|
|
754
|
+
const key = getNormalizationVectorKey(type, chrIdx, 'BP', binSize)
|
|
807
755
|
|
|
808
756
|
// Sharing the in-flight read is what keeps concurrent region pairs sharing
|
|
809
757
|
// one vector: a chromosome appears in every pair it takes part in, so with a
|
|
@@ -872,8 +820,8 @@ export class HicFile {
|
|
|
872
820
|
},
|
|
873
821
|
opts.signal,
|
|
874
822
|
)
|
|
875
|
-
} else if (this.
|
|
876
|
-
const nviArray = decodeURIComponent(this.
|
|
823
|
+
} else if (this.nvi) {
|
|
824
|
+
const nviArray = decodeURIComponent(this.nvi).split(',')
|
|
877
825
|
return this.readNormVectorIndex(
|
|
878
826
|
{
|
|
879
827
|
start: parseInt(nviArray[0]!),
|
|
@@ -927,41 +875,28 @@ export class HicFile {
|
|
|
927
875
|
this.normExpectedValueVectorsPosition,
|
|
928
876
|
opts,
|
|
929
877
|
)
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
let data = await this.file.read(nviStart, INT, { signal })
|
|
878
|
+
const data = await this.file.read(nviStart, INT, { signal })
|
|
933
879
|
// Possible if there are no norm vectors. A legal v8 file, though uncommon.
|
|
934
880
|
if (data.byteLength !== 0) {
|
|
935
|
-
const
|
|
936
|
-
const nEntries = binaryParser.getInt()
|
|
937
|
-
const sizeEstimate = nEntries * 30
|
|
938
|
-
data = await this.file.read(nviStart + byteCount, sizeEstimate, {
|
|
939
|
-
signal,
|
|
940
|
-
})
|
|
881
|
+
const nEntries = new BinaryParser(new DataView(data)).getInt()
|
|
941
882
|
const found: NormVectorIndex = { entries: {}, types: ['NONE'] }
|
|
942
883
|
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
return
|
|
956
|
-
}
|
|
957
|
-
this.parseNormVectorEntry(parser, found)
|
|
884
|
+
let position = nviStart + INT
|
|
885
|
+
let requested = nEntries * 30
|
|
886
|
+
let buf = await this.file.read(position, requested, { signal })
|
|
887
|
+
let parser = new BinaryParser(new DataView(buf))
|
|
888
|
+
for (let n = nEntries; n > 0; n--) {
|
|
889
|
+
// A short read means the buffer already runs to EOF, and refilling
|
|
890
|
+
// it would only return the same bytes again.
|
|
891
|
+
if (parser.available() < 100 && buf.byteLength === requested) {
|
|
892
|
+
position += parser.position
|
|
893
|
+
requested = Math.max(1000, n * 30)
|
|
894
|
+
buf = await this.file.read(position, requested, { signal })
|
|
895
|
+
parser = new BinaryParser(new DataView(buf))
|
|
958
896
|
}
|
|
959
|
-
|
|
897
|
+
this.parseNormVectorEntry(parser, found)
|
|
960
898
|
}
|
|
961
|
-
|
|
962
|
-
await processEntries(nEntries, data)
|
|
963
899
|
throwIfAborted(signal)
|
|
964
|
-
this.config.nvi = `${nviStart},${byteCount}`
|
|
965
900
|
index = found
|
|
966
901
|
}
|
|
967
902
|
}
|
|
@@ -1044,15 +979,11 @@ export class HicFile {
|
|
|
1044
979
|
const filePosition = binaryParser.getLong() // 8
|
|
1045
980
|
const sizeInBytes =
|
|
1046
981
|
this.version < 9 ? binaryParser.getInt() : binaryParser.getLong() // 4:8
|
|
1047
|
-
const key =
|
|
982
|
+
const key = getNormalizationVectorKey(type, chrIdx, unit, binSize)
|
|
1048
983
|
|
|
1049
984
|
if (!index.types.includes(type)) {
|
|
1050
985
|
index.types.push(type)
|
|
1051
986
|
}
|
|
1052
987
|
index.entries[key] = { filePosition, size: sizeInBytes }
|
|
1053
988
|
}
|
|
1054
|
-
|
|
1055
|
-
getFileChrName(chrAlias: string) {
|
|
1056
|
-
return this.chrAliasTable[chrAlias] ?? chrAlias
|
|
1057
|
-
}
|
|
1058
989
|
}
|