@gmod/hic 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -14
- package/dist/bufferedFile.d.ts +2 -2
- package/dist/bufferedFile.js +5 -5
- package/dist/bufferedFile.js.map +1 -1
- package/dist/hicFile.d.ts +9 -10
- package/dist/hicFile.js +167 -174
- package/dist/hicFile.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/lru.js +0 -2
- package/dist/lru.js.map +1 -1
- package/dist/normalizationVector.d.ts +2 -2
- package/dist/normalizationVector.js +2 -2
- package/dist/normalizationVector.js.map +1 -1
- package/dist/reader.js +4 -2
- package/dist/reader.js.map +1 -1
- package/dist/types.d.ts +8 -2
- package/esm/bufferedFile.d.ts +2 -2
- package/esm/bufferedFile.js +5 -5
- package/esm/bufferedFile.js.map +1 -1
- package/esm/hicFile.d.ts +9 -10
- package/esm/hicFile.js +167 -174
- package/esm/hicFile.js.map +1 -1
- package/esm/index.d.ts +1 -1
- package/esm/lru.js +0 -2
- package/esm/lru.js.map +1 -1
- package/esm/normalizationVector.d.ts +2 -2
- package/esm/normalizationVector.js +2 -2
- package/esm/normalizationVector.js.map +1 -1
- package/esm/reader.js +4 -2
- package/esm/reader.js.map +1 -1
- package/esm/types.d.ts +8 -2
- package/package.json +11 -8
- package/src/bufferedFile.ts +10 -6
- package/src/hicFile.ts +227 -188
- package/src/index.ts +1 -0
- package/src/lru.ts +0 -2
- package/src/normalizationVector.ts +3 -3
- package/src/reader.ts +5 -3
- package/src/types.ts +13 -2
package/src/hicFile.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { inflateRawUnknownSize } from '@gmod/inflate'
|
|
2
|
+
import { SharedReadCache, throwIfAborted } from '@gmod/shared-read-cache'
|
|
2
3
|
import { LocalFile } from 'generic-filehandle2'
|
|
3
4
|
|
|
4
5
|
import { binWindow } from './binWindow.ts'
|
|
@@ -12,11 +13,11 @@ import { readerFromFilehandle } from './reader.ts'
|
|
|
12
13
|
import type { ContactRecords } from './contactRecords.ts'
|
|
13
14
|
import type MatrixZoomData from './matrixZoomData.ts'
|
|
14
15
|
import type {
|
|
16
|
+
BaseOpts,
|
|
15
17
|
BlockIndexEntry,
|
|
16
18
|
Chromosome,
|
|
17
19
|
HicMetadata,
|
|
18
20
|
HicRegion,
|
|
19
|
-
ProgressCallback,
|
|
20
21
|
ProgressOpts,
|
|
21
22
|
Reader,
|
|
22
23
|
} from './types.ts'
|
|
@@ -40,6 +41,15 @@ interface MasterIndexEntry {
|
|
|
40
41
|
size: number
|
|
41
42
|
}
|
|
42
43
|
|
|
44
|
+
interface NormVectorIndex {
|
|
45
|
+
entries: Record<string, BlockIndexEntry>
|
|
46
|
+
types: string[]
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
interface SignalOpts extends ProgressOpts {
|
|
50
|
+
signal: AbortSignal
|
|
51
|
+
}
|
|
52
|
+
|
|
43
53
|
export interface HicConfig {
|
|
44
54
|
/** a `generic-filehandle2` handle, e.g. `RemoteFile` for a URL */
|
|
45
55
|
filehandle?: GenericFilehandle
|
|
@@ -160,27 +170,30 @@ export class HicFile {
|
|
|
160
170
|
private config: HicConfig
|
|
161
171
|
private file: Reader
|
|
162
172
|
|
|
163
|
-
//
|
|
173
|
+
// These caches share the in-flight read, not just its result. A
|
|
164
174
|
// multi-region fetch runs its region pairs concurrently and they share
|
|
165
175
|
// chromosomes, so caching only the result had every concurrent pair miss and
|
|
166
176
|
// re-issue the same reads — measured +12 range requests on a 6-pair fetch.
|
|
167
|
-
//
|
|
168
|
-
|
|
177
|
+
// `SharedReadCache` cancels a shared read only once every caller waiting on
|
|
178
|
+
// it has aborted, and drops a rejection so the next caller retries.
|
|
179
|
+
private headerCache = new SharedReadCache<string, void>()
|
|
180
|
+
private normVectorIndexCache = new SharedReadCache<
|
|
181
|
+
string,
|
|
182
|
+
NormVectorIndex | undefined
|
|
183
|
+
>()
|
|
184
|
+
private normVectorCache = new SharedReadCache<
|
|
169
185
|
string,
|
|
170
|
-
|
|
171
|
-
>(NORM_VECTOR_CACHE_SIZE)
|
|
172
|
-
private
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
)
|
|
186
|
+
NormalizationVector | undefined
|
|
187
|
+
>({ maxSize: NORM_VECTOR_CACHE_SIZE })
|
|
188
|
+
private matrixCache = new SharedReadCache<string, Matrix | undefined>({
|
|
189
|
+
maxSize: MATRIX_CACHE_SIZE,
|
|
190
|
+
})
|
|
176
191
|
// Assigned in the constructor rather than here, since its byte budget is
|
|
177
192
|
// configurable and a field initializer cannot see `config`.
|
|
178
193
|
private blockCache: LRU<string, Block>
|
|
179
194
|
private normVectorIndexPosition = -1
|
|
180
195
|
private normVectorIndexSize = -1
|
|
181
196
|
|
|
182
|
-
private initPromise: Promise<void> | undefined
|
|
183
|
-
private normVectorIndexP: Promise<void> | undefined
|
|
184
197
|
private version = 0
|
|
185
198
|
private genomeId = ''
|
|
186
199
|
private footerPosition = 0
|
|
@@ -188,7 +201,6 @@ export class HicFile {
|
|
|
188
201
|
// carrying none of it ends. Public so a test can cut a real file there to
|
|
189
202
|
// make that second shape.
|
|
190
203
|
normExpectedValueVectorsPosition: number | undefined
|
|
191
|
-
private normVectorIndex: Record<string, BlockIndexEntry> | undefined
|
|
192
204
|
|
|
193
205
|
private chromosomes: Chromosome[] = []
|
|
194
206
|
private chromosomeIndexMap: Record<string, number> = {}
|
|
@@ -215,64 +227,52 @@ export class HicFile {
|
|
|
215
227
|
}
|
|
216
228
|
}
|
|
217
229
|
|
|
218
|
-
async init() {
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
// promise forever.
|
|
223
|
-
this.initPromise ??= this.readHeaderAndFooter().catch((e: unknown) => {
|
|
224
|
-
this.initPromise = undefined
|
|
225
|
-
throw e
|
|
226
|
-
})
|
|
227
|
-
return this.initPromise
|
|
230
|
+
async init(opts?: BaseOpts) {
|
|
231
|
+
return this.headerCache.get('header', opts?.signal, signal =>
|
|
232
|
+
this.readHeaderAndFooter(signal),
|
|
233
|
+
)
|
|
228
234
|
}
|
|
229
235
|
|
|
230
|
-
async getMetaData() {
|
|
231
|
-
await this.init()
|
|
236
|
+
async getMetaData(opts?: BaseOpts) {
|
|
237
|
+
await this.init(opts)
|
|
232
238
|
return this.meta!
|
|
233
239
|
}
|
|
234
240
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
this.chromosomes = []
|
|
242
|
-
this.chromosomeIndexMap = {}
|
|
243
|
-
this.chrAliasTable = {}
|
|
244
|
-
this.bpResolutions = []
|
|
245
|
-
this.masterIndex = {}
|
|
246
|
-
|
|
247
|
-
// Read initial fields magic, version, and footer position
|
|
248
|
-
let data = await this.file.read(0, 16)
|
|
241
|
+
// Parsed into locals and committed only at the end: a parse every caller
|
|
242
|
+
// abandoned can still be running beside the one that replaced it, and a
|
|
243
|
+
// transient failure part-way through must leave nothing half-written for the
|
|
244
|
+
// retry to append to.
|
|
245
|
+
private async readHeaderAndFooter(signal: AbortSignal) {
|
|
246
|
+
let data = await this.file.read(0, 16, { signal })
|
|
249
247
|
if (data.byteLength === 0) {
|
|
250
248
|
throw new Error('File content is empty')
|
|
251
249
|
}
|
|
252
250
|
let binaryParser = new BinaryParser(new DataView(data))
|
|
253
251
|
binaryParser.getString() // magic
|
|
254
|
-
|
|
255
|
-
if (
|
|
256
|
-
throw new Error(`Unsupported hic version: ${
|
|
252
|
+
const version = binaryParser.getInt()
|
|
253
|
+
if (version < 5) {
|
|
254
|
+
throw new Error(`Unsupported hic version: ${version}`)
|
|
257
255
|
}
|
|
258
|
-
|
|
256
|
+
const footerPosition = binaryParser.getLong()
|
|
259
257
|
|
|
260
|
-
|
|
261
|
-
|
|
258
|
+
const { masterIndex, normExpectedValueVectorsPosition } =
|
|
259
|
+
await this.readFooter(version, footerPosition, signal)
|
|
262
260
|
|
|
263
|
-
const bodyPosition = Object.values(
|
|
261
|
+
const bodyPosition = Object.values(masterIndex).reduce(
|
|
264
262
|
(min, entry) => Math.min(min, entry.start),
|
|
265
263
|
Number.MAX_VALUE,
|
|
266
264
|
)
|
|
267
265
|
|
|
268
|
-
data = await this.file.read(16, bodyPosition - 16)
|
|
266
|
+
data = await this.file.read(16, bodyPosition - 16, { signal })
|
|
269
267
|
binaryParser = new BinaryParser(new DataView(data))
|
|
270
268
|
|
|
271
|
-
|
|
269
|
+
const genomeId = binaryParser.getString()
|
|
272
270
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
271
|
+
let normVectorIndexPosition = -1
|
|
272
|
+
let normVectorIndexSize = -1
|
|
273
|
+
if (version >= 9) {
|
|
274
|
+
normVectorIndexPosition = binaryParser.getLong()
|
|
275
|
+
normVectorIndexSize = binaryParser.getLong()
|
|
276
276
|
}
|
|
277
277
|
|
|
278
278
|
let nAttributes = binaryParser.getInt()
|
|
@@ -281,96 +281,113 @@ export class HicFile {
|
|
|
281
281
|
binaryParser.getString() // attribute value
|
|
282
282
|
}
|
|
283
283
|
|
|
284
|
+
const chromosomes: Chromosome[] = []
|
|
285
|
+
const chromosomeIndexMap: Record<string, number> = {}
|
|
284
286
|
let nChrs = binaryParser.getInt()
|
|
285
287
|
let i = 0
|
|
286
288
|
while (nChrs-- > 0) {
|
|
287
289
|
const chr = {
|
|
288
290
|
index: i,
|
|
289
291
|
name: binaryParser.getString(),
|
|
290
|
-
size:
|
|
292
|
+
size: version < 9 ? binaryParser.getInt() : binaryParser.getLong(),
|
|
291
293
|
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
+
chromosomes.push(chr)
|
|
295
|
+
chromosomeIndexMap[chr.name] = chr.index
|
|
294
296
|
i++
|
|
295
297
|
}
|
|
296
298
|
|
|
299
|
+
const bpResolutions: number[] = []
|
|
297
300
|
let nBpResolutions = binaryParser.getInt()
|
|
298
301
|
while (nBpResolutions-- > 0) {
|
|
299
|
-
|
|
302
|
+
bpResolutions.push(binaryParser.getInt())
|
|
300
303
|
}
|
|
301
304
|
|
|
302
|
-
|
|
303
|
-
for (const chrName of Object.keys(
|
|
305
|
+
const chrAliasTable: Record<string, string> = {}
|
|
306
|
+
for (const chrName of Object.keys(chromosomeIndexMap)) {
|
|
304
307
|
if (chrName.startsWith('chr')) {
|
|
305
|
-
|
|
308
|
+
chrAliasTable[chrName.slice(3)] = chrName
|
|
306
309
|
} else if (chrName === 'MT') {
|
|
307
|
-
|
|
310
|
+
chrAliasTable.chrM = chrName
|
|
308
311
|
} else {
|
|
309
|
-
|
|
312
|
+
chrAliasTable[`chr${chrName}`] = chrName
|
|
310
313
|
}
|
|
311
314
|
}
|
|
312
315
|
|
|
316
|
+
throwIfAborted(signal)
|
|
317
|
+
this.version = version
|
|
318
|
+
this.footerPosition = footerPosition
|
|
319
|
+
this.masterIndex = masterIndex
|
|
320
|
+
this.normExpectedValueVectorsPosition = normExpectedValueVectorsPosition
|
|
321
|
+
this.genomeId = genomeId
|
|
322
|
+
this.normVectorIndexPosition = normVectorIndexPosition
|
|
323
|
+
this.normVectorIndexSize = normVectorIndexSize
|
|
324
|
+
this.chromosomes = chromosomes
|
|
325
|
+
this.chromosomeIndexMap = chromosomeIndexMap
|
|
326
|
+
this.bpResolutions = bpResolutions
|
|
327
|
+
this.chrAliasTable = chrAliasTable
|
|
313
328
|
this.meta = {
|
|
314
|
-
version
|
|
315
|
-
genome:
|
|
316
|
-
chromosomes
|
|
317
|
-
resolutions:
|
|
329
|
+
version,
|
|
330
|
+
genome: genomeId,
|
|
331
|
+
chromosomes,
|
|
332
|
+
resolutions: bpResolutions,
|
|
318
333
|
}
|
|
319
334
|
}
|
|
320
335
|
|
|
321
|
-
private async readFooter(
|
|
322
|
-
|
|
323
|
-
|
|
336
|
+
private async readFooter(
|
|
337
|
+
version: number,
|
|
338
|
+
footerPosition: number,
|
|
339
|
+
signal: AbortSignal,
|
|
340
|
+
) {
|
|
341
|
+
const skip = version < 9 ? 8 : 12
|
|
342
|
+
let data = await this.file.read(footerPosition, skip, { signal })
|
|
324
343
|
|
|
325
344
|
let binaryParser = new BinaryParser(new DataView(data))
|
|
326
345
|
// Total size, master index + expected values
|
|
327
|
-
const nBytes =
|
|
328
|
-
this.version < 9 ? binaryParser.getInt() : binaryParser.getLong()
|
|
346
|
+
const nBytes = version < 9 ? binaryParser.getInt() : binaryParser.getLong()
|
|
329
347
|
let nEntries = binaryParser.getInt()
|
|
330
348
|
|
|
331
349
|
// Estimate the size of the master index. String key length is unknown, be
|
|
332
350
|
// conservative (100 bytes).
|
|
333
351
|
const miSize = nEntries * (100 + 64 + 32)
|
|
334
352
|
data = await this.file.read(
|
|
335
|
-
|
|
353
|
+
footerPosition + skip,
|
|
336
354
|
Math.min(miSize, nBytes),
|
|
355
|
+
{ signal },
|
|
337
356
|
)
|
|
338
357
|
binaryParser = new BinaryParser(new DataView(data))
|
|
339
358
|
|
|
359
|
+
const masterIndex: Record<string, MasterIndexEntry> = {}
|
|
340
360
|
while (nEntries-- > 0) {
|
|
341
361
|
const key = binaryParser.getString()
|
|
342
362
|
const pos = binaryParser.getLong()
|
|
343
363
|
const size = binaryParser.getInt()
|
|
344
|
-
|
|
364
|
+
masterIndex[key] = { start: pos, size }
|
|
345
365
|
}
|
|
346
366
|
|
|
347
367
|
// Normalized expected values start after the expected values
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
}
|
|
368
|
+
const normExpectedValueVectorsPosition =
|
|
369
|
+
version > 5 ? footerPosition + (version < 9 ? 4 : 8) + nBytes : undefined
|
|
370
|
+
return { masterIndex, normExpectedValueVectorsPosition }
|
|
352
371
|
}
|
|
353
372
|
|
|
354
|
-
async getMatrix(chrIdx1: number, chrIdx2: number) {
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
})
|
|
362
|
-
this.matrixCache.set(key, p)
|
|
363
|
-
}
|
|
364
|
-
return p
|
|
373
|
+
async getMatrix(chrIdx1: number, chrIdx2: number, opts?: BaseOpts) {
|
|
374
|
+
await this.init(opts)
|
|
375
|
+
return this.matrixCache.get(
|
|
376
|
+
Matrix.getKey(chrIdx1, chrIdx2),
|
|
377
|
+
opts?.signal,
|
|
378
|
+
signal => this.readMatrix(chrIdx1, chrIdx2, signal),
|
|
379
|
+
)
|
|
365
380
|
}
|
|
366
381
|
|
|
367
|
-
private async readMatrix(
|
|
368
|
-
|
|
369
|
-
|
|
382
|
+
private async readMatrix(
|
|
383
|
+
chrIdx1: number,
|
|
384
|
+
chrIdx2: number,
|
|
385
|
+
signal: AbortSignal,
|
|
386
|
+
) {
|
|
370
387
|
const idx = this.masterIndex[Matrix.getKey(chrIdx1, chrIdx2)]
|
|
371
388
|
let matrix: Matrix | undefined
|
|
372
389
|
if (idx) {
|
|
373
|
-
const data = await this.file.read(idx.start, idx.size)
|
|
390
|
+
const data = await this.file.read(idx.start, idx.size, { signal })
|
|
374
391
|
matrix = Matrix.parseMatrix(data, this.chromosomes)
|
|
375
392
|
}
|
|
376
393
|
return matrix
|
|
@@ -384,7 +401,7 @@ export class HicFile {
|
|
|
384
401
|
binsize: number,
|
|
385
402
|
opts?: ProgressOpts,
|
|
386
403
|
) {
|
|
387
|
-
await this.init()
|
|
404
|
+
await this.init(opts)
|
|
388
405
|
|
|
389
406
|
const idx1 = this.chromosomeIndexMap[this.getFileChrName(region1.chr)]
|
|
390
407
|
const idx2 = this.chromosomeIndexMap[this.getFileChrName(region2.chr)]
|
|
@@ -427,12 +444,13 @@ export class HicFile {
|
|
|
427
444
|
// resolved once per pair rather than per block, each paired with the bin
|
|
428
445
|
// offset its values start at.
|
|
429
446
|
const [norm, blocks] = await Promise.all([
|
|
430
|
-
this.getNormVectors(normalization, r1, r2, units, binsize),
|
|
447
|
+
this.getNormVectors(normalization, r1, r2, units, binsize, opts),
|
|
431
448
|
// The measurable half of the pair: blocks are counted and the two norm
|
|
432
449
|
// vectors are not, because the vector chain is two hops whatever the
|
|
433
450
|
// query and the block chain is the one that grows with the region.
|
|
434
451
|
this.getBlocks(r1, r2, binsize, opts),
|
|
435
452
|
])
|
|
453
|
+
throwIfAborted(opts?.signal)
|
|
436
454
|
|
|
437
455
|
// Sum of the blocks' record counts bounds the survivors, so the output is
|
|
438
456
|
// allocated once and filled by a write cursor. Blocks overlap the window
|
|
@@ -520,7 +538,9 @@ export class HicFile {
|
|
|
520
538
|
r2: HicRegion,
|
|
521
539
|
units: string,
|
|
522
540
|
binsize: number,
|
|
541
|
+
opts?: BaseOpts,
|
|
523
542
|
) {
|
|
543
|
+
const signalOpts = { signal: opts?.signal }
|
|
524
544
|
let result:
|
|
525
545
|
| {
|
|
526
546
|
v1: Float64Array
|
|
@@ -539,6 +559,7 @@ export class HicFile {
|
|
|
539
559
|
chr1,
|
|
540
560
|
units,
|
|
541
561
|
binsize,
|
|
562
|
+
signalOpts,
|
|
542
563
|
)
|
|
543
564
|
const nv2 =
|
|
544
565
|
chr1 === chr2
|
|
@@ -548,11 +569,20 @@ export class HicFile {
|
|
|
548
569
|
chr2,
|
|
549
570
|
units,
|
|
550
571
|
binsize,
|
|
572
|
+
signalOpts,
|
|
551
573
|
)
|
|
552
574
|
if (nv1 && nv2) {
|
|
553
575
|
result = {
|
|
554
|
-
v1: await nv1.getValues(
|
|
555
|
-
|
|
576
|
+
v1: await nv1.getValues(
|
|
577
|
+
offset1,
|
|
578
|
+
Math.ceil(r1.end / binsize),
|
|
579
|
+
signalOpts,
|
|
580
|
+
),
|
|
581
|
+
v2: await nv2.getValues(
|
|
582
|
+
offset2,
|
|
583
|
+
Math.ceil(r2.end / binsize),
|
|
584
|
+
signalOpts,
|
|
585
|
+
),
|
|
556
586
|
offset1,
|
|
557
587
|
offset2,
|
|
558
588
|
}
|
|
@@ -570,7 +600,7 @@ export class HicFile {
|
|
|
570
600
|
const blockKey = (blockNumber: number, zd: MatrixZoomData) =>
|
|
571
601
|
`${zd.getKey()}_${blockNumber}`
|
|
572
602
|
|
|
573
|
-
await this.init()
|
|
603
|
+
await this.init(opts)
|
|
574
604
|
const chr1 = this.getFileChrName(region1.chr)
|
|
575
605
|
const chr2 = this.getFileChrName(region2.chr)
|
|
576
606
|
const idx1 = this.chromosomeIndexMap[chr1]
|
|
@@ -586,7 +616,8 @@ export class HicFile {
|
|
|
586
616
|
// .hic files store no inter-chromosomal maps, and a multi-region view asks
|
|
587
617
|
// for every pair. Answering with no blocks, rather than warning once per
|
|
588
618
|
// pair per fetch, leaves that for the caller to notice or ignore.
|
|
589
|
-
const
|
|
619
|
+
const signalOpts = { signal: opts?.signal }
|
|
620
|
+
const matrix = await this.getMatrix(idx1, idx2, signalOpts)
|
|
590
621
|
if (matrix) {
|
|
591
622
|
const zd = matrix.getZoomData(binSize)
|
|
592
623
|
if (!zd) {
|
|
@@ -623,7 +654,7 @@ export class HicFile {
|
|
|
623
654
|
// pins that.
|
|
624
655
|
const newBlocks = await Promise.all(
|
|
625
656
|
blockNumbersToQuery.map(async blockNumber => {
|
|
626
|
-
const block = await this.readBlock(blockNumber, zd)
|
|
657
|
+
const block = await this.readBlock(blockNumber, zd, signalOpts)
|
|
627
658
|
done++
|
|
628
659
|
onProgress?.(done, blockNumbers.length)
|
|
629
660
|
return block
|
|
@@ -640,12 +671,15 @@ export class HicFile {
|
|
|
640
671
|
return blocks
|
|
641
672
|
}
|
|
642
673
|
|
|
643
|
-
async readBlock(blockNumber: number, zd: MatrixZoomData) {
|
|
674
|
+
async readBlock(blockNumber: number, zd: MatrixZoomData, opts?: BaseOpts) {
|
|
644
675
|
const idx = zd.blockIndex[blockNumber]
|
|
645
676
|
|
|
646
677
|
let block: Block | undefined
|
|
647
678
|
if (idx) {
|
|
648
|
-
const data = await this.file.read(idx.filePosition, idx.size
|
|
679
|
+
const data = await this.file.read(idx.filePosition, idx.size, {
|
|
680
|
+
signal: opts?.signal,
|
|
681
|
+
})
|
|
682
|
+
throwIfAborted(opts?.signal)
|
|
649
683
|
// `.subarray(2)` drops the zlib header: libdeflate's raw-deflate path is
|
|
650
684
|
// the fast one, and a `.hic` records only a block's *compressed* size, so
|
|
651
685
|
// there is no known output size to hand the exact-size entry point.
|
|
@@ -761,8 +795,9 @@ export class HicFile {
|
|
|
761
795
|
chr: string,
|
|
762
796
|
unit: string,
|
|
763
797
|
binSize: number,
|
|
798
|
+
opts?: BaseOpts,
|
|
764
799
|
) {
|
|
765
|
-
await this.init()
|
|
800
|
+
await this.init(opts)
|
|
766
801
|
|
|
767
802
|
const chrIdx = this.chromosomeIndexMap[this.getFileChrName(chr)]
|
|
768
803
|
if (chrIdx === undefined) {
|
|
@@ -770,35 +805,30 @@ export class HicFile {
|
|
|
770
805
|
}
|
|
771
806
|
const key = getNormalizationVectorKey(type, chrIdx, unit, binSize)
|
|
772
807
|
|
|
773
|
-
//
|
|
774
|
-
// vector: a chromosome appears in every pair it takes part in, so with a
|
|
808
|
+
// Sharing the in-flight read is what keeps concurrent region pairs sharing
|
|
809
|
+
// one vector: a chromosome appears in every pair it takes part in, so with a
|
|
775
810
|
// result-only cache each of those pairs missed while the first was still in
|
|
776
811
|
// flight and read the header again — and then held its own
|
|
777
812
|
// `NormalizationVector`, whose value cache is per instance, so the whole
|
|
778
813
|
// vector was re-read too.
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
throw e
|
|
790
|
-
})
|
|
791
|
-
this.normVectorCache.set(key, p)
|
|
792
|
-
}
|
|
793
|
-
return p
|
|
814
|
+
//
|
|
815
|
+
// A file with no vectors at all, or none for this (type, chr, unit,
|
|
816
|
+
// binsize), simply answers undefined and the caller falls back to raw
|
|
817
|
+
// counts. hic-straw warns to the console here; that fires once per
|
|
818
|
+
// chromosome per region pair per fetch, and the console is the wrong place
|
|
819
|
+
// for it anyway — `getContactRecords` reports the normalization it actually
|
|
820
|
+
// applied instead.
|
|
821
|
+
return this.normVectorCache.get(key, opts?.signal, signal =>
|
|
822
|
+
this.readNormalizationVector(key, signal),
|
|
823
|
+
)
|
|
794
824
|
}
|
|
795
825
|
|
|
796
|
-
private async readNormalizationVector(key: string) {
|
|
797
|
-
const idx = (await this.getNormVectorIndex())?.[key]
|
|
826
|
+
private async readNormalizationVector(key: string, signal: AbortSignal) {
|
|
827
|
+
const idx = (await this.getNormVectorIndex({ signal }))?.[key]
|
|
798
828
|
if (!idx) {
|
|
799
829
|
return undefined
|
|
800
830
|
}
|
|
801
|
-
const data = await this.file.read(idx.filePosition, 8)
|
|
831
|
+
const data = await this.file.read(idx.filePosition, 8, { signal })
|
|
802
832
|
const parser = new BinaryParser(new DataView(data))
|
|
803
833
|
const nValues = this.version < 9 ? parser.getInt() : parser.getLong()
|
|
804
834
|
const dataType = this.version < 9 ? DOUBLE : FLOAT
|
|
@@ -808,52 +838,54 @@ export class HicFile {
|
|
|
808
838
|
}
|
|
809
839
|
|
|
810
840
|
async getNormVectorIndex(opts?: ProgressOpts) {
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
await
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
// walking the whole normalized-expected-values section with a chain of
|
|
822
|
-
// sequential range reads, only to rediscover there is nothing there.
|
|
823
|
-
// Cleared on failure (like `init`) so a transient read error retries
|
|
824
|
-
// rather than caching a rejection forever.
|
|
825
|
-
// The walk runs once, so its progress belongs to the call that performs
|
|
826
|
-
// it: a caller joining an in-flight or finished load is not waiting on
|
|
827
|
-
// reads and is told nothing, which is the truth rather than a silence.
|
|
828
|
-
this.normVectorIndexP ??= this.loadNormVectorIndex(
|
|
829
|
-
opts?.onProgress,
|
|
830
|
-
).catch((e: unknown) => {
|
|
831
|
-
this.normVectorIndexP = undefined
|
|
832
|
-
throw e
|
|
833
|
-
})
|
|
834
|
-
await this.normVectorIndexP
|
|
841
|
+
return (await this.getParsedNormVectorIndex(opts))?.entries
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
private async getParsedNormVectorIndex(opts?: ProgressOpts) {
|
|
845
|
+
// await init() before the version gate, not after: this class defaults
|
|
846
|
+
// `version` to 0, so checking it before the header is parsed would answer
|
|
847
|
+
// "no index" for every file rather than for a v5 one.
|
|
848
|
+
await this.init(opts)
|
|
849
|
+
if (this.version < 6) {
|
|
850
|
+
return undefined
|
|
835
851
|
}
|
|
836
|
-
|
|
852
|
+
// Memoize the *attempt*, not just a populated result. A legal (if uncommon)
|
|
853
|
+
// v8 file with no norm vectors has no index, and re-running the discovery
|
|
854
|
+
// on every call cost two calls per region pair per fetch, each walking the
|
|
855
|
+
// whole normalized-expected-values section with a chain of sequential range
|
|
856
|
+
// reads, only to rediscover there is nothing there.
|
|
857
|
+
//
|
|
858
|
+
// The walk runs once, so its progress belongs to the call that performs
|
|
859
|
+
// it: the fill is per call, so a caller joining an in-flight or finished
|
|
860
|
+
// load is not waiting on reads and is told nothing.
|
|
861
|
+
return this.normVectorIndexCache.get('nvi', opts?.signal, signal =>
|
|
862
|
+
this.loadNormVectorIndex({ signal, onProgress: opts?.onProgress }),
|
|
863
|
+
)
|
|
837
864
|
}
|
|
838
865
|
|
|
839
|
-
private async loadNormVectorIndex(
|
|
840
|
-
// If we know the position of the norm vector index, read it directly.
|
|
841
|
-
// This is the case for hic v9 files.
|
|
866
|
+
private async loadNormVectorIndex(opts: SignalOpts) {
|
|
842
867
|
if (this.normVectorIndexPosition > 0 && this.normVectorIndexSize > 0) {
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
868
|
+
return this.readNormVectorIndex(
|
|
869
|
+
{
|
|
870
|
+
start: this.normVectorIndexPosition,
|
|
871
|
+
size: this.normVectorIndexSize,
|
|
872
|
+
},
|
|
873
|
+
opts.signal,
|
|
874
|
+
)
|
|
847
875
|
} else if (this.config.nvi) {
|
|
848
876
|
const nviArray = decodeURIComponent(this.config.nvi).split(',')
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
877
|
+
return this.readNormVectorIndex(
|
|
878
|
+
{
|
|
879
|
+
start: parseInt(nviArray[0]!),
|
|
880
|
+
size: parseInt(nviArray[1]!),
|
|
881
|
+
},
|
|
882
|
+
opts.signal,
|
|
883
|
+
)
|
|
853
884
|
} else {
|
|
854
885
|
try {
|
|
855
|
-
await this.readNormExpectedValuesAndNormVectorIndex(
|
|
886
|
+
return await this.readNormExpectedValuesAndNormVectorIndex(opts)
|
|
856
887
|
} catch (e) {
|
|
888
|
+
throwIfAborted(opts.signal)
|
|
857
889
|
// Not "expected if the file has no norm vectors" — that case never
|
|
858
890
|
// arrives here. hic-straw's own IO threw a 416 for a read past EOF and
|
|
859
891
|
// this caught it; `generic-filehandle2` deliberately turns a 416 into an
|
|
@@ -862,50 +894,51 @@ export class HicFile {
|
|
|
862
894
|
// the read side owns that case now, and anything reaching here is a real
|
|
863
895
|
// failure worth printing.
|
|
864
896
|
console.error(e)
|
|
897
|
+
return undefined
|
|
865
898
|
}
|
|
866
899
|
}
|
|
867
900
|
}
|
|
868
901
|
|
|
869
902
|
async getNormalizationOptions(opts?: ProgressOpts) {
|
|
870
|
-
|
|
871
|
-
// index. A bit ugly but alternatives are worse.
|
|
872
|
-
await this.getNormVectorIndex(opts)
|
|
873
|
-
return this.normalizationTypes
|
|
903
|
+
return (await this.getParsedNormVectorIndex(opts))?.types ?? ['NONE']
|
|
874
904
|
}
|
|
875
905
|
|
|
876
|
-
private async readNormVectorIndex(
|
|
877
|
-
|
|
878
|
-
|
|
906
|
+
private async readNormVectorIndex(
|
|
907
|
+
range: { start: number; size: number },
|
|
908
|
+
signal: AbortSignal,
|
|
909
|
+
) {
|
|
910
|
+
const data = await this.file.read(range.start, range.size, { signal })
|
|
879
911
|
const binaryParser = new BinaryParser(new DataView(data))
|
|
880
|
-
|
|
912
|
+
const index: NormVectorIndex = { entries: {}, types: ['NONE'] }
|
|
881
913
|
let nEntries = binaryParser.getInt()
|
|
882
914
|
while (nEntries-- > 0) {
|
|
883
|
-
this.parseNormVectorEntry(binaryParser)
|
|
915
|
+
this.parseNormVectorEntry(binaryParser, index)
|
|
884
916
|
}
|
|
885
|
-
return
|
|
917
|
+
return index
|
|
886
918
|
}
|
|
887
919
|
|
|
888
920
|
// Used when the position of the norm vector index is unknown: read through
|
|
889
921
|
// the expected values to find the index.
|
|
890
|
-
private async readNormExpectedValuesAndNormVectorIndex(
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
await this.init()
|
|
922
|
+
private async readNormExpectedValuesAndNormVectorIndex(opts: SignalOpts) {
|
|
923
|
+
const { signal } = opts
|
|
924
|
+
let index: NormVectorIndex | undefined
|
|
894
925
|
if (this.normExpectedValueVectorsPosition !== undefined) {
|
|
895
926
|
const nviStart = await this.skipExpectedValues(
|
|
896
927
|
this.normExpectedValueVectorsPosition,
|
|
897
|
-
|
|
928
|
+
opts,
|
|
898
929
|
)
|
|
899
930
|
let byteCount = INT
|
|
900
931
|
|
|
901
|
-
let data = await this.file.read(nviStart, INT)
|
|
932
|
+
let data = await this.file.read(nviStart, INT, { signal })
|
|
902
933
|
// Possible if there are no norm vectors. A legal v8 file, though uncommon.
|
|
903
934
|
if (data.byteLength !== 0) {
|
|
904
935
|
const binaryParser = new BinaryParser(new DataView(data))
|
|
905
936
|
const nEntries = binaryParser.getInt()
|
|
906
937
|
const sizeEstimate = nEntries * 30
|
|
907
|
-
data = await this.file.read(nviStart + byteCount, sizeEstimate
|
|
908
|
-
|
|
938
|
+
data = await this.file.read(nviStart + byteCount, sizeEstimate, {
|
|
939
|
+
signal,
|
|
940
|
+
})
|
|
941
|
+
const found: NormVectorIndex = { entries: {}, types: ['NONE'] }
|
|
909
942
|
|
|
910
943
|
const processEntries = async (remaining: number, buf: ArrayBuffer) => {
|
|
911
944
|
const parser = new BinaryParser(new DataView(buf))
|
|
@@ -915,30 +948,33 @@ export class HicFile {
|
|
|
915
948
|
n++ // Reset counter as entry is not processed
|
|
916
949
|
byteCount += parser.position
|
|
917
950
|
const est = Math.max(1000, n * 30)
|
|
918
|
-
const more = await this.file.read(nviStart + byteCount, est
|
|
951
|
+
const more = await this.file.read(nviStart + byteCount, est, {
|
|
952
|
+
signal,
|
|
953
|
+
})
|
|
919
954
|
await processEntries(n, more)
|
|
920
955
|
return
|
|
921
956
|
}
|
|
922
|
-
this.parseNormVectorEntry(parser)
|
|
957
|
+
this.parseNormVectorEntry(parser, found)
|
|
923
958
|
}
|
|
924
959
|
byteCount += parser.position
|
|
925
960
|
}
|
|
926
961
|
|
|
927
962
|
await processEntries(nEntries, data)
|
|
963
|
+
throwIfAborted(signal)
|
|
928
964
|
this.config.nvi = `${nviStart},${byteCount}`
|
|
965
|
+
index = found
|
|
929
966
|
}
|
|
930
967
|
}
|
|
968
|
+
return index
|
|
931
969
|
}
|
|
932
970
|
|
|
933
971
|
// Used when the position of the norm vector index is unknown: skip the
|
|
934
972
|
// normalized expected values to find the index.
|
|
935
|
-
private async skipExpectedValues(
|
|
936
|
-
|
|
937
|
-
onProgress?: ProgressCallback,
|
|
938
|
-
) {
|
|
973
|
+
private async skipExpectedValues(start: number, opts: SignalOpts) {
|
|
974
|
+
const { signal, onProgress } = opts
|
|
939
975
|
const version = this.version
|
|
940
976
|
const file = new BufferedFile({ file: this.file, size: 256000 })
|
|
941
|
-
const data = await file.read(start, INT)
|
|
977
|
+
const data = await file.read(start, INT, { signal })
|
|
942
978
|
// A file with no normalization at all ends where this section would start,
|
|
943
979
|
// so the count this is about to read is past EOF. That is the same "no norm
|
|
944
980
|
// vectors" case `readNormExpectedValuesAndNormVectorIndex` guards its own
|
|
@@ -964,7 +1000,7 @@ export class HicFile {
|
|
|
964
1000
|
const skipChunk = async (chunkStart: number) => {
|
|
965
1001
|
let chunkSize = 0
|
|
966
1002
|
|
|
967
|
-
let buf = await file.read(chunkStart, 500)
|
|
1003
|
+
let buf = await file.read(chunkStart, 500, { signal })
|
|
968
1004
|
let parser = new BinaryParser(new DataView(buf))
|
|
969
1005
|
parser.getString() // type
|
|
970
1006
|
parser.getString() // unit
|
|
@@ -972,7 +1008,7 @@ export class HicFile {
|
|
|
972
1008
|
const nValues = version < 9 ? parser.getInt() : parser.getLong()
|
|
973
1009
|
chunkSize += parser.position + nValues * (version < 9 ? DOUBLE : FLOAT)
|
|
974
1010
|
|
|
975
|
-
buf = await file.read(chunkStart + chunkSize, INT)
|
|
1011
|
+
buf = await file.read(chunkStart + chunkSize, INT, { signal })
|
|
976
1012
|
parser = new BinaryParser(new DataView(buf))
|
|
977
1013
|
const nChrScaleFactors = parser.getInt()
|
|
978
1014
|
chunkSize +=
|
|
@@ -997,7 +1033,10 @@ export class HicFile {
|
|
|
997
1033
|
return position
|
|
998
1034
|
}
|
|
999
1035
|
|
|
1000
|
-
private parseNormVectorEntry(
|
|
1036
|
+
private parseNormVectorEntry(
|
|
1037
|
+
binaryParser: BinaryParser,
|
|
1038
|
+
index: NormVectorIndex,
|
|
1039
|
+
) {
|
|
1001
1040
|
const type = binaryParser.getString() // 15
|
|
1002
1041
|
const chrIdx = binaryParser.getInt() // 4
|
|
1003
1042
|
const unit = binaryParser.getString() // 3
|
|
@@ -1007,10 +1046,10 @@ export class HicFile {
|
|
|
1007
1046
|
this.version < 9 ? binaryParser.getInt() : binaryParser.getLong() // 4:8
|
|
1008
1047
|
const key = `${type}_${chrIdx}_${unit}_${binSize}`
|
|
1009
1048
|
|
|
1010
|
-
if (!
|
|
1011
|
-
|
|
1049
|
+
if (!index.types.includes(type)) {
|
|
1050
|
+
index.types.push(type)
|
|
1012
1051
|
}
|
|
1013
|
-
|
|
1052
|
+
index.entries[key] = { filePosition, size: sizeInBytes }
|
|
1014
1053
|
}
|
|
1015
1054
|
|
|
1016
1055
|
getFileChrName(chrAlias: string) {
|