@gmod/hic 1.2.1 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -14
- package/dist/bufferedFile.d.ts +2 -2
- package/dist/bufferedFile.js +5 -5
- package/dist/bufferedFile.js.map +1 -1
- package/dist/hicFile.d.ts +9 -10
- package/dist/hicFile.js +167 -174
- package/dist/hicFile.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/lru.js +0 -2
- package/dist/lru.js.map +1 -1
- package/dist/normalizationVector.d.ts +2 -2
- package/dist/normalizationVector.js +2 -2
- package/dist/normalizationVector.js.map +1 -1
- package/dist/reader.js +4 -2
- package/dist/reader.js.map +1 -1
- package/dist/types.d.ts +8 -2
- package/esm/bufferedFile.d.ts +2 -2
- package/esm/bufferedFile.js +5 -5
- package/esm/bufferedFile.js.map +1 -1
- package/esm/hicFile.d.ts +9 -10
- package/esm/hicFile.js +167 -174
- package/esm/hicFile.js.map +1 -1
- package/esm/index.d.ts +1 -1
- package/esm/lru.js +0 -2
- package/esm/lru.js.map +1 -1
- package/esm/normalizationVector.d.ts +2 -2
- package/esm/normalizationVector.js +2 -2
- package/esm/normalizationVector.js.map +1 -1
- package/esm/reader.js +4 -2
- package/esm/reader.js.map +1 -1
- package/esm/types.d.ts +8 -2
- package/package.json +11 -8
- package/src/bufferedFile.ts +10 -6
- package/src/hicFile.ts +227 -188
- package/src/index.ts +1 -0
- package/src/lru.ts +0 -2
- package/src/normalizationVector.ts +3 -3
- package/src/reader.ts +5 -3
- package/src/types.ts +13 -2
package/esm/hicFile.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { inflateRawUnknownSize } from '@gmod/inflate';
|
|
2
|
+
import { SharedReadCache, throwIfAborted } from '@gmod/shared-read-cache';
|
|
2
3
|
import { LocalFile } from 'generic-filehandle2';
|
|
3
4
|
import { binWindow } from "./binWindow.js";
|
|
4
5
|
import BinaryParser from "./binary.js";
|
|
@@ -97,21 +98,23 @@ function getNormalizationVectorKey(type, chrIdx, unit, resolution) {
|
|
|
97
98
|
export class HicFile {
|
|
98
99
|
config;
|
|
99
100
|
file;
|
|
100
|
-
//
|
|
101
|
+
// These caches share the in-flight read, not just its result. A
|
|
101
102
|
// multi-region fetch runs its region pairs concurrently and they share
|
|
102
103
|
// chromosomes, so caching only the result had every concurrent pair miss and
|
|
103
104
|
// re-issue the same reads — measured +12 range requests on a 6-pair fetch.
|
|
104
|
-
//
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
105
|
+
// `SharedReadCache` cancels a shared read only once every caller waiting on
|
|
106
|
+
// it has aborted, and drops a rejection so the next caller retries.
|
|
107
|
+
headerCache = new SharedReadCache();
|
|
108
|
+
normVectorIndexCache = new SharedReadCache();
|
|
109
|
+
normVectorCache = new SharedReadCache({ maxSize: NORM_VECTOR_CACHE_SIZE });
|
|
110
|
+
matrixCache = new SharedReadCache({
|
|
111
|
+
maxSize: MATRIX_CACHE_SIZE,
|
|
112
|
+
});
|
|
108
113
|
// Assigned in the constructor rather than here, since its byte budget is
|
|
109
114
|
// configurable and a field initializer cannot see `config`.
|
|
110
115
|
blockCache;
|
|
111
116
|
normVectorIndexPosition = -1;
|
|
112
117
|
normVectorIndexSize = -1;
|
|
113
|
-
initPromise;
|
|
114
|
-
normVectorIndexP;
|
|
115
118
|
version = 0;
|
|
116
119
|
genomeId = '';
|
|
117
120
|
footerPosition = 0;
|
|
@@ -119,7 +122,6 @@ export class HicFile {
|
|
|
119
122
|
// carrying none of it ends. Public so a test can cut a real file there to
|
|
120
123
|
// make that second shape.
|
|
121
124
|
normExpectedValueVectorsPosition;
|
|
122
|
-
normVectorIndex;
|
|
123
125
|
chromosomes = [];
|
|
124
126
|
chromosomeIndexMap = {};
|
|
125
127
|
chrAliasTable = {};
|
|
@@ -146,142 +148,133 @@ export class HicFile {
|
|
|
146
148
|
throw new Error('must supply filehandle, path, or reader');
|
|
147
149
|
}
|
|
148
150
|
}
|
|
149
|
-
async init() {
|
|
150
|
-
|
|
151
|
-
// share one header parse rather than both racing readHeaderAndFooter. Clear
|
|
152
|
-
// it on failure so a later call retries instead of caching a rejected
|
|
153
|
-
// promise forever.
|
|
154
|
-
this.initPromise ??= this.readHeaderAndFooter().catch((e) => {
|
|
155
|
-
this.initPromise = undefined;
|
|
156
|
-
throw e;
|
|
157
|
-
});
|
|
158
|
-
return this.initPromise;
|
|
151
|
+
async init(opts) {
|
|
152
|
+
return this.headerCache.get('header', opts?.signal, signal => this.readHeaderAndFooter(signal));
|
|
159
153
|
}
|
|
160
|
-
async getMetaData() {
|
|
161
|
-
await this.init();
|
|
154
|
+
async getMetaData(opts) {
|
|
155
|
+
await this.init(opts);
|
|
162
156
|
return this.meta;
|
|
163
157
|
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
this.chromosomes = [];
|
|
171
|
-
this.chromosomeIndexMap = {};
|
|
172
|
-
this.chrAliasTable = {};
|
|
173
|
-
this.bpResolutions = [];
|
|
174
|
-
this.masterIndex = {};
|
|
175
|
-
// Read initial fields magic, version, and footer position
|
|
176
|
-
let data = await this.file.read(0, 16);
|
|
158
|
+
// Parsed into locals and committed only at the end: a parse every caller
|
|
159
|
+
// abandoned can still be running beside the one that replaced it, and a
|
|
160
|
+
// transient failure part-way through must leave nothing half-written for the
|
|
161
|
+
// retry to append to.
|
|
162
|
+
async readHeaderAndFooter(signal) {
|
|
163
|
+
let data = await this.file.read(0, 16, { signal });
|
|
177
164
|
if (data.byteLength === 0) {
|
|
178
165
|
throw new Error('File content is empty');
|
|
179
166
|
}
|
|
180
167
|
let binaryParser = new BinaryParser(new DataView(data));
|
|
181
168
|
binaryParser.getString(); // magic
|
|
182
|
-
|
|
183
|
-
if (
|
|
184
|
-
throw new Error(`Unsupported hic version: ${
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
data = await this.file.read(16, bodyPosition - 16);
|
|
169
|
+
const version = binaryParser.getInt();
|
|
170
|
+
if (version < 5) {
|
|
171
|
+
throw new Error(`Unsupported hic version: ${version}`);
|
|
172
|
+
}
|
|
173
|
+
const footerPosition = binaryParser.getLong();
|
|
174
|
+
const { masterIndex, normExpectedValueVectorsPosition } = await this.readFooter(version, footerPosition, signal);
|
|
175
|
+
const bodyPosition = Object.values(masterIndex).reduce((min, entry) => Math.min(min, entry.start), Number.MAX_VALUE);
|
|
176
|
+
data = await this.file.read(16, bodyPosition - 16, { signal });
|
|
191
177
|
binaryParser = new BinaryParser(new DataView(data));
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
178
|
+
const genomeId = binaryParser.getString();
|
|
179
|
+
let normVectorIndexPosition = -1;
|
|
180
|
+
let normVectorIndexSize = -1;
|
|
181
|
+
if (version >= 9) {
|
|
182
|
+
normVectorIndexPosition = binaryParser.getLong();
|
|
183
|
+
normVectorIndexSize = binaryParser.getLong();
|
|
196
184
|
}
|
|
197
185
|
let nAttributes = binaryParser.getInt();
|
|
198
186
|
while (nAttributes-- > 0) {
|
|
199
187
|
binaryParser.getString(); // attribute key
|
|
200
188
|
binaryParser.getString(); // attribute value
|
|
201
189
|
}
|
|
190
|
+
const chromosomes = [];
|
|
191
|
+
const chromosomeIndexMap = {};
|
|
202
192
|
let nChrs = binaryParser.getInt();
|
|
203
193
|
let i = 0;
|
|
204
194
|
while (nChrs-- > 0) {
|
|
205
195
|
const chr = {
|
|
206
196
|
index: i,
|
|
207
197
|
name: binaryParser.getString(),
|
|
208
|
-
size:
|
|
198
|
+
size: version < 9 ? binaryParser.getInt() : binaryParser.getLong(),
|
|
209
199
|
};
|
|
210
|
-
|
|
211
|
-
|
|
200
|
+
chromosomes.push(chr);
|
|
201
|
+
chromosomeIndexMap[chr.name] = chr.index;
|
|
212
202
|
i++;
|
|
213
203
|
}
|
|
204
|
+
const bpResolutions = [];
|
|
214
205
|
let nBpResolutions = binaryParser.getInt();
|
|
215
206
|
while (nBpResolutions-- > 0) {
|
|
216
|
-
|
|
207
|
+
bpResolutions.push(binaryParser.getInt());
|
|
217
208
|
}
|
|
218
|
-
|
|
219
|
-
for (const chrName of Object.keys(
|
|
209
|
+
const chrAliasTable = {};
|
|
210
|
+
for (const chrName of Object.keys(chromosomeIndexMap)) {
|
|
220
211
|
if (chrName.startsWith('chr')) {
|
|
221
|
-
|
|
212
|
+
chrAliasTable[chrName.slice(3)] = chrName;
|
|
222
213
|
}
|
|
223
214
|
else if (chrName === 'MT') {
|
|
224
|
-
|
|
215
|
+
chrAliasTable.chrM = chrName;
|
|
225
216
|
}
|
|
226
217
|
else {
|
|
227
|
-
|
|
218
|
+
chrAliasTable[`chr${chrName}`] = chrName;
|
|
228
219
|
}
|
|
229
220
|
}
|
|
221
|
+
throwIfAborted(signal);
|
|
222
|
+
this.version = version;
|
|
223
|
+
this.footerPosition = footerPosition;
|
|
224
|
+
this.masterIndex = masterIndex;
|
|
225
|
+
this.normExpectedValueVectorsPosition = normExpectedValueVectorsPosition;
|
|
226
|
+
this.genomeId = genomeId;
|
|
227
|
+
this.normVectorIndexPosition = normVectorIndexPosition;
|
|
228
|
+
this.normVectorIndexSize = normVectorIndexSize;
|
|
229
|
+
this.chromosomes = chromosomes;
|
|
230
|
+
this.chromosomeIndexMap = chromosomeIndexMap;
|
|
231
|
+
this.bpResolutions = bpResolutions;
|
|
232
|
+
this.chrAliasTable = chrAliasTable;
|
|
230
233
|
this.meta = {
|
|
231
|
-
version
|
|
232
|
-
genome:
|
|
233
|
-
chromosomes
|
|
234
|
-
resolutions:
|
|
234
|
+
version,
|
|
235
|
+
genome: genomeId,
|
|
236
|
+
chromosomes,
|
|
237
|
+
resolutions: bpResolutions,
|
|
235
238
|
};
|
|
236
239
|
}
|
|
237
|
-
async readFooter() {
|
|
238
|
-
const skip =
|
|
239
|
-
let data = await this.file.read(
|
|
240
|
+
async readFooter(version, footerPosition, signal) {
|
|
241
|
+
const skip = version < 9 ? 8 : 12;
|
|
242
|
+
let data = await this.file.read(footerPosition, skip, { signal });
|
|
240
243
|
let binaryParser = new BinaryParser(new DataView(data));
|
|
241
244
|
// Total size, master index + expected values
|
|
242
|
-
const nBytes =
|
|
245
|
+
const nBytes = version < 9 ? binaryParser.getInt() : binaryParser.getLong();
|
|
243
246
|
let nEntries = binaryParser.getInt();
|
|
244
247
|
// Estimate the size of the master index. String key length is unknown, be
|
|
245
248
|
// conservative (100 bytes).
|
|
246
249
|
const miSize = nEntries * (100 + 64 + 32);
|
|
247
|
-
data = await this.file.read(
|
|
250
|
+
data = await this.file.read(footerPosition + skip, Math.min(miSize, nBytes), { signal });
|
|
248
251
|
binaryParser = new BinaryParser(new DataView(data));
|
|
252
|
+
const masterIndex = {};
|
|
249
253
|
while (nEntries-- > 0) {
|
|
250
254
|
const key = binaryParser.getString();
|
|
251
255
|
const pos = binaryParser.getLong();
|
|
252
256
|
const size = binaryParser.getInt();
|
|
253
|
-
|
|
257
|
+
masterIndex[key] = { start: pos, size };
|
|
254
258
|
}
|
|
255
259
|
// Normalized expected values start after the expected values
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
this.normExpectedValueVectorsPosition = this.footerPosition + s + nBytes;
|
|
259
|
-
}
|
|
260
|
+
const normExpectedValueVectorsPosition = version > 5 ? footerPosition + (version < 9 ? 4 : 8) + nBytes : undefined;
|
|
261
|
+
return { masterIndex, normExpectedValueVectorsPosition };
|
|
260
262
|
}
|
|
261
|
-
async getMatrix(chrIdx1, chrIdx2) {
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
if (!p) {
|
|
265
|
-
p = this.readMatrix(chrIdx1, chrIdx2).catch((e) => {
|
|
266
|
-
this.matrixCache.delete(key);
|
|
267
|
-
throw e;
|
|
268
|
-
});
|
|
269
|
-
this.matrixCache.set(key, p);
|
|
270
|
-
}
|
|
271
|
-
return p;
|
|
263
|
+
async getMatrix(chrIdx1, chrIdx2, opts) {
|
|
264
|
+
await this.init(opts);
|
|
265
|
+
return this.matrixCache.get(Matrix.getKey(chrIdx1, chrIdx2), opts?.signal, signal => this.readMatrix(chrIdx1, chrIdx2, signal));
|
|
272
266
|
}
|
|
273
|
-
async readMatrix(chrIdx1, chrIdx2) {
|
|
274
|
-
await this.init();
|
|
267
|
+
async readMatrix(chrIdx1, chrIdx2, signal) {
|
|
275
268
|
const idx = this.masterIndex[Matrix.getKey(chrIdx1, chrIdx2)];
|
|
276
269
|
let matrix;
|
|
277
270
|
if (idx) {
|
|
278
|
-
const data = await this.file.read(idx.start, idx.size);
|
|
271
|
+
const data = await this.file.read(idx.start, idx.size, { signal });
|
|
279
272
|
matrix = Matrix.parseMatrix(data, this.chromosomes);
|
|
280
273
|
}
|
|
281
274
|
return matrix;
|
|
282
275
|
}
|
|
283
276
|
async getContactRecords(normalization, region1, region2, units, binsize, opts) {
|
|
284
|
-
await this.init();
|
|
277
|
+
await this.init(opts);
|
|
285
278
|
const idx1 = this.chromosomeIndexMap[this.getFileChrName(region1.chr)];
|
|
286
279
|
const idx2 = this.chromosomeIndexMap[this.getFileChrName(region2.chr)];
|
|
287
280
|
// A `.hic` stores only `bin1 <= bin2`, so a query whose x window sits to the
|
|
@@ -319,12 +312,13 @@ export class HicFile {
|
|
|
319
312
|
// resolved once per pair rather than per block, each paired with the bin
|
|
320
313
|
// offset its values start at.
|
|
321
314
|
const [norm, blocks] = await Promise.all([
|
|
322
|
-
this.getNormVectors(normalization, r1, r2, units, binsize),
|
|
315
|
+
this.getNormVectors(normalization, r1, r2, units, binsize, opts),
|
|
323
316
|
// The measurable half of the pair: blocks are counted and the two norm
|
|
324
317
|
// vectors are not, because the vector chain is two hops whatever the
|
|
325
318
|
// query and the block chain is the one that grows with the region.
|
|
326
319
|
this.getBlocks(r1, r2, binsize, opts),
|
|
327
320
|
]);
|
|
321
|
+
throwIfAborted(opts?.signal);
|
|
328
322
|
// Sum of the blocks' record counts bounds the survivors, so the output is
|
|
329
323
|
// allocated once and filled by a write cursor. Blocks overlap the window
|
|
330
324
|
// rather than nest in it, so the true count isn't known without either this
|
|
@@ -402,21 +396,22 @@ export class HicFile {
|
|
|
402
396
|
transposed: transpose,
|
|
403
397
|
};
|
|
404
398
|
}
|
|
405
|
-
async getNormVectors(normalization, r1, r2, units, binsize) {
|
|
399
|
+
async getNormVectors(normalization, r1, r2, units, binsize, opts) {
|
|
400
|
+
const signalOpts = { signal: opts?.signal };
|
|
406
401
|
let result;
|
|
407
402
|
if (normalization && normalization !== 'NONE') {
|
|
408
403
|
const chr1 = this.getFileChrName(r1.chr);
|
|
409
404
|
const chr2 = this.getFileChrName(r2.chr);
|
|
410
405
|
const offset1 = Math.floor(r1.start / binsize);
|
|
411
406
|
const offset2 = Math.floor(r2.start / binsize);
|
|
412
|
-
const nv1 = await this.getNormalizationVector(normalization, chr1, units, binsize);
|
|
407
|
+
const nv1 = await this.getNormalizationVector(normalization, chr1, units, binsize, signalOpts);
|
|
413
408
|
const nv2 = chr1 === chr2
|
|
414
409
|
? nv1
|
|
415
|
-
: await this.getNormalizationVector(normalization, chr2, units, binsize);
|
|
410
|
+
: await this.getNormalizationVector(normalization, chr2, units, binsize, signalOpts);
|
|
416
411
|
if (nv1 && nv2) {
|
|
417
412
|
result = {
|
|
418
|
-
v1: await nv1.getValues(offset1, Math.ceil(r1.end / binsize)),
|
|
419
|
-
v2: await nv2.getValues(offset2, Math.ceil(r2.end / binsize)),
|
|
413
|
+
v1: await nv1.getValues(offset1, Math.ceil(r1.end / binsize), signalOpts),
|
|
414
|
+
v2: await nv2.getValues(offset2, Math.ceil(r2.end / binsize), signalOpts),
|
|
420
415
|
offset1,
|
|
421
416
|
offset2,
|
|
422
417
|
};
|
|
@@ -426,7 +421,7 @@ export class HicFile {
|
|
|
426
421
|
}
|
|
427
422
|
async getBlocks(region1, region2, binSize, opts) {
|
|
428
423
|
const blockKey = (blockNumber, zd) => `${zd.getKey()}_${blockNumber}`;
|
|
429
|
-
await this.init();
|
|
424
|
+
await this.init(opts);
|
|
430
425
|
const chr1 = this.getFileChrName(region1.chr);
|
|
431
426
|
const chr2 = this.getFileChrName(region2.chr);
|
|
432
427
|
const idx1 = this.chromosomeIndexMap[chr1];
|
|
@@ -443,7 +438,8 @@ export class HicFile {
|
|
|
443
438
|
// .hic files store no inter-chromosomal maps, and a multi-region view asks
|
|
444
439
|
// for every pair. Answering with no blocks, rather than warning once per
|
|
445
440
|
// pair per fetch, leaves that for the caller to notice or ignore.
|
|
446
|
-
const
|
|
441
|
+
const signalOpts = { signal: opts?.signal };
|
|
442
|
+
const matrix = await this.getMatrix(idx1, idx2, signalOpts);
|
|
447
443
|
if (matrix) {
|
|
448
444
|
const zd = matrix.getZoomData(binSize);
|
|
449
445
|
if (!zd) {
|
|
@@ -476,7 +472,7 @@ export class HicFile {
|
|
|
476
472
|
// completions must not turn one wave into a queue. `readChainDepth`
|
|
477
473
|
// pins that.
|
|
478
474
|
const newBlocks = await Promise.all(blockNumbersToQuery.map(async (blockNumber) => {
|
|
479
|
-
const block = await this.readBlock(blockNumber, zd);
|
|
475
|
+
const block = await this.readBlock(blockNumber, zd, signalOpts);
|
|
480
476
|
done++;
|
|
481
477
|
onProgress?.(done, blockNumbers.length);
|
|
482
478
|
return block;
|
|
@@ -491,11 +487,14 @@ export class HicFile {
|
|
|
491
487
|
}
|
|
492
488
|
return blocks;
|
|
493
489
|
}
|
|
494
|
-
async readBlock(blockNumber, zd) {
|
|
490
|
+
async readBlock(blockNumber, zd, opts) {
|
|
495
491
|
const idx = zd.blockIndex[blockNumber];
|
|
496
492
|
let block;
|
|
497
493
|
if (idx) {
|
|
498
|
-
const data = await this.file.read(idx.filePosition, idx.size
|
|
494
|
+
const data = await this.file.read(idx.filePosition, idx.size, {
|
|
495
|
+
signal: opts?.signal,
|
|
496
|
+
});
|
|
497
|
+
throwIfAborted(opts?.signal);
|
|
499
498
|
// `.subarray(2)` drops the zlib header: libdeflate's raw-deflate path is
|
|
500
499
|
// the fast one, and a `.hic` records only a block's *compressed* size, so
|
|
501
500
|
// there is no known output size to hand the exact-size entry point.
|
|
@@ -590,41 +589,34 @@ export class HicFile {
|
|
|
590
589
|
}
|
|
591
590
|
throw new Error(`Unknown block type: ${type}`);
|
|
592
591
|
}
|
|
593
|
-
async getNormalizationVector(type, chr, unit, binSize) {
|
|
594
|
-
await this.init();
|
|
592
|
+
async getNormalizationVector(type, chr, unit, binSize, opts) {
|
|
593
|
+
await this.init(opts);
|
|
595
594
|
const chrIdx = this.chromosomeIndexMap[this.getFileChrName(chr)];
|
|
596
595
|
if (chrIdx === undefined) {
|
|
597
596
|
return undefined;
|
|
598
597
|
}
|
|
599
598
|
const key = getNormalizationVectorKey(type, chrIdx, unit, binSize);
|
|
600
|
-
//
|
|
601
|
-
// vector: a chromosome appears in every pair it takes part in, so with a
|
|
599
|
+
// Sharing the in-flight read is what keeps concurrent region pairs sharing
|
|
600
|
+
// one vector: a chromosome appears in every pair it takes part in, so with a
|
|
602
601
|
// result-only cache each of those pairs missed while the first was still in
|
|
603
602
|
// flight and read the header again — and then held its own
|
|
604
603
|
// `NormalizationVector`, whose value cache is per instance, so the whole
|
|
605
604
|
// vector was re-read too.
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
p = this.readNormalizationVector(key).catch((e) => {
|
|
615
|
-
this.normVectorCache.delete(key);
|
|
616
|
-
throw e;
|
|
617
|
-
});
|
|
618
|
-
this.normVectorCache.set(key, p);
|
|
619
|
-
}
|
|
620
|
-
return p;
|
|
605
|
+
//
|
|
606
|
+
// A file with no vectors at all, or none for this (type, chr, unit,
|
|
607
|
+
// binsize), simply answers undefined and the caller falls back to raw
|
|
608
|
+
// counts. hic-straw warns to the console here; that fires once per
|
|
609
|
+
// chromosome per region pair per fetch, and the console is the wrong place
|
|
610
|
+
// for it anyway — `getContactRecords` reports the normalization it actually
|
|
611
|
+
// applied instead.
|
|
612
|
+
return this.normVectorCache.get(key, opts?.signal, signal => this.readNormalizationVector(key, signal));
|
|
621
613
|
}
|
|
622
|
-
async readNormalizationVector(key) {
|
|
623
|
-
const idx = (await this.getNormVectorIndex())?.[key];
|
|
614
|
+
async readNormalizationVector(key, signal) {
|
|
615
|
+
const idx = (await this.getNormVectorIndex({ signal }))?.[key];
|
|
624
616
|
if (!idx) {
|
|
625
617
|
return undefined;
|
|
626
618
|
}
|
|
627
|
-
const data = await this.file.read(idx.filePosition, 8);
|
|
619
|
+
const data = await this.file.read(idx.filePosition, 8, { signal });
|
|
628
620
|
const parser = new BinaryParser(new DataView(data));
|
|
629
621
|
const nValues = this.version < 9 ? parser.getInt() : parser.getLong();
|
|
630
622
|
const dataType = this.version < 9 ? DOUBLE : FLOAT;
|
|
@@ -632,52 +624,47 @@ export class HicFile {
|
|
|
632
624
|
return new NormalizationVector(this.file, filePosition, nValues, dataType);
|
|
633
625
|
}
|
|
634
626
|
async getNormVectorIndex(opts) {
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
//
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
// discovery on every call — two calls per region pair per fetch, each
|
|
645
|
-
// walking the whole normalized-expected-values section with a chain of
|
|
646
|
-
// sequential range reads, only to rediscover there is nothing there.
|
|
647
|
-
// Cleared on failure (like `init`) so a transient read error retries
|
|
648
|
-
// rather than caching a rejection forever.
|
|
649
|
-
// The walk runs once, so its progress belongs to the call that performs
|
|
650
|
-
// it: a caller joining an in-flight or finished load is not waiting on
|
|
651
|
-
// reads and is told nothing, which is the truth rather than a silence.
|
|
652
|
-
this.normVectorIndexP ??= this.loadNormVectorIndex(opts?.onProgress).catch((e) => {
|
|
653
|
-
this.normVectorIndexP = undefined;
|
|
654
|
-
throw e;
|
|
655
|
-
});
|
|
656
|
-
await this.normVectorIndexP;
|
|
627
|
+
return (await this.getParsedNormVectorIndex(opts))?.entries;
|
|
628
|
+
}
|
|
629
|
+
async getParsedNormVectorIndex(opts) {
|
|
630
|
+
// await init() before the version gate, not after: this class defaults
|
|
631
|
+
// `version` to 0, so checking it before the header is parsed would answer
|
|
632
|
+
// "no index" for every file rather than for a v5 one.
|
|
633
|
+
await this.init(opts);
|
|
634
|
+
if (this.version < 6) {
|
|
635
|
+
return undefined;
|
|
657
636
|
}
|
|
658
|
-
|
|
637
|
+
// Memoize the *attempt*, not just a populated result. A legal (if uncommon)
|
|
638
|
+
// v8 file with no norm vectors has no index, and re-running the discovery
|
|
639
|
+
// on every call cost two calls per region pair per fetch, each walking the
|
|
640
|
+
// whole normalized-expected-values section with a chain of sequential range
|
|
641
|
+
// reads, only to rediscover there is nothing there.
|
|
642
|
+
//
|
|
643
|
+
// The walk runs once, so its progress belongs to the call that performs
|
|
644
|
+
// it: the fill is per call, so a caller joining an in-flight or finished
|
|
645
|
+
// load is not waiting on reads and is told nothing.
|
|
646
|
+
return this.normVectorIndexCache.get('nvi', opts?.signal, signal => this.loadNormVectorIndex({ signal, onProgress: opts?.onProgress }));
|
|
659
647
|
}
|
|
660
|
-
async loadNormVectorIndex(
|
|
661
|
-
// If we know the position of the norm vector index, read it directly.
|
|
662
|
-
// This is the case for hic v9 files.
|
|
648
|
+
async loadNormVectorIndex(opts) {
|
|
663
649
|
if (this.normVectorIndexPosition > 0 && this.normVectorIndexSize > 0) {
|
|
664
|
-
|
|
650
|
+
return this.readNormVectorIndex({
|
|
665
651
|
start: this.normVectorIndexPosition,
|
|
666
652
|
size: this.normVectorIndexSize,
|
|
667
|
-
});
|
|
653
|
+
}, opts.signal);
|
|
668
654
|
}
|
|
669
655
|
else if (this.config.nvi) {
|
|
670
656
|
const nviArray = decodeURIComponent(this.config.nvi).split(',');
|
|
671
|
-
|
|
657
|
+
return this.readNormVectorIndex({
|
|
672
658
|
start: parseInt(nviArray[0]),
|
|
673
659
|
size: parseInt(nviArray[1]),
|
|
674
|
-
});
|
|
660
|
+
}, opts.signal);
|
|
675
661
|
}
|
|
676
662
|
else {
|
|
677
663
|
try {
|
|
678
|
-
await this.readNormExpectedValuesAndNormVectorIndex(
|
|
664
|
+
return await this.readNormExpectedValuesAndNormVectorIndex(opts);
|
|
679
665
|
}
|
|
680
666
|
catch (e) {
|
|
667
|
+
throwIfAborted(opts.signal);
|
|
681
668
|
// Not "expected if the file has no norm vectors" — that case never
|
|
682
669
|
// arrives here. hic-straw's own IO threw a 416 for a read past EOF and
|
|
683
670
|
// this caught it; `generic-filehandle2` deliberately turns a 416 into an
|
|
@@ -686,41 +673,41 @@ export class HicFile {
|
|
|
686
673
|
// the read side owns that case now, and anything reaching here is a real
|
|
687
674
|
// failure worth printing.
|
|
688
675
|
console.error(e);
|
|
676
|
+
return undefined;
|
|
689
677
|
}
|
|
690
678
|
}
|
|
691
679
|
}
|
|
692
680
|
async getNormalizationOptions(opts) {
|
|
693
|
-
|
|
694
|
-
// index. A bit ugly but alternatives are worse.
|
|
695
|
-
await this.getNormVectorIndex(opts);
|
|
696
|
-
return this.normalizationTypes;
|
|
681
|
+
return (await this.getParsedNormVectorIndex(opts))?.types ?? ['NONE'];
|
|
697
682
|
}
|
|
698
|
-
async readNormVectorIndex(range) {
|
|
699
|
-
await this.
|
|
700
|
-
const data = await this.file.read(range.start, range.size);
|
|
683
|
+
async readNormVectorIndex(range, signal) {
|
|
684
|
+
const data = await this.file.read(range.start, range.size, { signal });
|
|
701
685
|
const binaryParser = new BinaryParser(new DataView(data));
|
|
702
|
-
|
|
686
|
+
const index = { entries: {}, types: ['NONE'] };
|
|
703
687
|
let nEntries = binaryParser.getInt();
|
|
704
688
|
while (nEntries-- > 0) {
|
|
705
|
-
this.parseNormVectorEntry(binaryParser);
|
|
689
|
+
this.parseNormVectorEntry(binaryParser, index);
|
|
706
690
|
}
|
|
707
|
-
return
|
|
691
|
+
return index;
|
|
708
692
|
}
|
|
709
693
|
// Used when the position of the norm vector index is unknown: read through
|
|
710
694
|
// the expected values to find the index.
|
|
711
|
-
async readNormExpectedValuesAndNormVectorIndex(
|
|
712
|
-
|
|
695
|
+
async readNormExpectedValuesAndNormVectorIndex(opts) {
|
|
696
|
+
const { signal } = opts;
|
|
697
|
+
let index;
|
|
713
698
|
if (this.normExpectedValueVectorsPosition !== undefined) {
|
|
714
|
-
const nviStart = await this.skipExpectedValues(this.normExpectedValueVectorsPosition,
|
|
699
|
+
const nviStart = await this.skipExpectedValues(this.normExpectedValueVectorsPosition, opts);
|
|
715
700
|
let byteCount = INT;
|
|
716
|
-
let data = await this.file.read(nviStart, INT);
|
|
701
|
+
let data = await this.file.read(nviStart, INT, { signal });
|
|
717
702
|
// Possible if there are no norm vectors. A legal v8 file, though uncommon.
|
|
718
703
|
if (data.byteLength !== 0) {
|
|
719
704
|
const binaryParser = new BinaryParser(new DataView(data));
|
|
720
705
|
const nEntries = binaryParser.getInt();
|
|
721
706
|
const sizeEstimate = nEntries * 30;
|
|
722
|
-
data = await this.file.read(nviStart + byteCount, sizeEstimate
|
|
723
|
-
|
|
707
|
+
data = await this.file.read(nviStart + byteCount, sizeEstimate, {
|
|
708
|
+
signal,
|
|
709
|
+
});
|
|
710
|
+
const found = { entries: {}, types: ['NONE'] };
|
|
724
711
|
const processEntries = async (remaining, buf) => {
|
|
725
712
|
const parser = new BinaryParser(new DataView(buf));
|
|
726
713
|
let n = remaining;
|
|
@@ -729,25 +716,31 @@ export class HicFile {
|
|
|
729
716
|
n++; // Reset counter as entry is not processed
|
|
730
717
|
byteCount += parser.position;
|
|
731
718
|
const est = Math.max(1000, n * 30);
|
|
732
|
-
const more = await this.file.read(nviStart + byteCount, est
|
|
719
|
+
const more = await this.file.read(nviStart + byteCount, est, {
|
|
720
|
+
signal,
|
|
721
|
+
});
|
|
733
722
|
await processEntries(n, more);
|
|
734
723
|
return;
|
|
735
724
|
}
|
|
736
|
-
this.parseNormVectorEntry(parser);
|
|
725
|
+
this.parseNormVectorEntry(parser, found);
|
|
737
726
|
}
|
|
738
727
|
byteCount += parser.position;
|
|
739
728
|
};
|
|
740
729
|
await processEntries(nEntries, data);
|
|
730
|
+
throwIfAborted(signal);
|
|
741
731
|
this.config.nvi = `${nviStart},${byteCount}`;
|
|
732
|
+
index = found;
|
|
742
733
|
}
|
|
743
734
|
}
|
|
735
|
+
return index;
|
|
744
736
|
}
|
|
745
737
|
// Used when the position of the norm vector index is unknown: skip the
|
|
746
738
|
// normalized expected values to find the index.
|
|
747
|
-
async skipExpectedValues(start,
|
|
739
|
+
async skipExpectedValues(start, opts) {
|
|
740
|
+
const { signal, onProgress } = opts;
|
|
748
741
|
const version = this.version;
|
|
749
742
|
const file = new BufferedFile({ file: this.file, size: 256000 });
|
|
750
|
-
const data = await file.read(start, INT);
|
|
743
|
+
const data = await file.read(start, INT, { signal });
|
|
751
744
|
// A file with no normalization at all ends where this section would start,
|
|
752
745
|
// so the count this is about to read is past EOF. That is the same "no norm
|
|
753
746
|
// vectors" case `readNormExpectedValuesAndNormVectorIndex` guards its own
|
|
@@ -771,14 +764,14 @@ export class HicFile {
|
|
|
771
764
|
// read, each costing about the same.
|
|
772
765
|
const skipChunk = async (chunkStart) => {
|
|
773
766
|
let chunkSize = 0;
|
|
774
|
-
let buf = await file.read(chunkStart, 500);
|
|
767
|
+
let buf = await file.read(chunkStart, 500, { signal });
|
|
775
768
|
let parser = new BinaryParser(new DataView(buf));
|
|
776
769
|
parser.getString(); // type
|
|
777
770
|
parser.getString(); // unit
|
|
778
771
|
parser.getInt(); // binSize
|
|
779
772
|
const nValues = version < 9 ? parser.getInt() : parser.getLong();
|
|
780
773
|
chunkSize += parser.position + nValues * (version < 9 ? DOUBLE : FLOAT);
|
|
781
|
-
buf = await file.read(chunkStart + chunkSize, INT);
|
|
774
|
+
buf = await file.read(chunkStart + chunkSize, INT, { signal });
|
|
782
775
|
parser = new BinaryParser(new DataView(buf));
|
|
783
776
|
const nChrScaleFactors = parser.getInt();
|
|
784
777
|
chunkSize +=
|
|
@@ -800,7 +793,7 @@ export class HicFile {
|
|
|
800
793
|
}
|
|
801
794
|
return position;
|
|
802
795
|
}
|
|
803
|
-
parseNormVectorEntry(binaryParser) {
|
|
796
|
+
parseNormVectorEntry(binaryParser, index) {
|
|
804
797
|
const type = binaryParser.getString(); // 15
|
|
805
798
|
const chrIdx = binaryParser.getInt(); // 4
|
|
806
799
|
const unit = binaryParser.getString(); // 3
|
|
@@ -808,10 +801,10 @@ export class HicFile {
|
|
|
808
801
|
const filePosition = binaryParser.getLong(); // 8
|
|
809
802
|
const sizeInBytes = this.version < 9 ? binaryParser.getInt() : binaryParser.getLong(); // 4:8
|
|
810
803
|
const key = `${type}_${chrIdx}_${unit}_${binSize}`;
|
|
811
|
-
if (!
|
|
812
|
-
|
|
804
|
+
if (!index.types.includes(type)) {
|
|
805
|
+
index.types.push(type);
|
|
813
806
|
}
|
|
814
|
-
|
|
807
|
+
index.entries[key] = { filePosition, size: sizeInBytes };
|
|
815
808
|
}
|
|
816
809
|
getFileChrName(chrAlias) {
|
|
817
810
|
return this.chrAliasTable[chrAlias] ?? chrAlias;
|