@gmod/hic 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -14
- package/dist/bufferedFile.d.ts +2 -2
- package/dist/bufferedFile.js +5 -5
- package/dist/bufferedFile.js.map +1 -1
- package/dist/hicFile.d.ts +9 -10
- package/dist/hicFile.js +167 -174
- package/dist/hicFile.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/lru.js +0 -2
- package/dist/lru.js.map +1 -1
- package/dist/normalizationVector.d.ts +2 -2
- package/dist/normalizationVector.js +18 -3
- package/dist/normalizationVector.js.map +1 -1
- package/dist/reader.js +4 -2
- package/dist/reader.js.map +1 -1
- package/dist/types.d.ts +8 -2
- package/esm/bufferedFile.d.ts +2 -2
- package/esm/bufferedFile.js +5 -5
- package/esm/bufferedFile.js.map +1 -1
- package/esm/hicFile.d.ts +9 -10
- package/esm/hicFile.js +167 -174
- package/esm/hicFile.js.map +1 -1
- package/esm/index.d.ts +1 -1
- package/esm/lru.js +0 -2
- package/esm/lru.js.map +1 -1
- package/esm/normalizationVector.d.ts +2 -2
- package/esm/normalizationVector.js +18 -3
- package/esm/normalizationVector.js.map +1 -1
- package/esm/reader.js +4 -2
- package/esm/reader.js.map +1 -1
- package/esm/types.d.ts +8 -2
- package/package.json +11 -8
- package/src/bufferedFile.ts +10 -6
- package/src/hicFile.ts +227 -188
- package/src/index.ts +1 -0
- package/src/lru.ts +0 -2
- package/src/normalizationVector.ts +20 -4
- package/src/reader.ts +5 -3
- package/src/types.ts +13 -2
package/dist/hicFile.js
CHANGED
|
@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.HicFile = exports.NO_DATA_FOR_RESOLUTION = void 0;
|
|
7
7
|
const inflate_1 = require("@gmod/inflate");
|
|
8
|
+
const shared_read_cache_1 = require("@gmod/shared-read-cache");
|
|
8
9
|
const generic_filehandle2_1 = require("generic-filehandle2");
|
|
9
10
|
const binWindow_ts_1 = require("./binWindow.js");
|
|
10
11
|
const binary_ts_1 = __importDefault(require("./binary.js"));
|
|
@@ -103,21 +104,23 @@ function getNormalizationVectorKey(type, chrIdx, unit, resolution) {
|
|
|
103
104
|
class HicFile {
|
|
104
105
|
config;
|
|
105
106
|
file;
|
|
106
|
-
//
|
|
107
|
+
// These caches share the in-flight read, not just its result. A
|
|
107
108
|
// multi-region fetch runs its region pairs concurrently and they share
|
|
108
109
|
// chromosomes, so caching only the result had every concurrent pair miss and
|
|
109
110
|
// re-issue the same reads — measured +12 range requests on a 6-pair fetch.
|
|
110
|
-
//
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
111
|
+
// `SharedReadCache` cancels a shared read only once every caller waiting on
|
|
112
|
+
// it has aborted, and drops a rejection so the next caller retries.
|
|
113
|
+
headerCache = new shared_read_cache_1.SharedReadCache();
|
|
114
|
+
normVectorIndexCache = new shared_read_cache_1.SharedReadCache();
|
|
115
|
+
normVectorCache = new shared_read_cache_1.SharedReadCache({ maxSize: NORM_VECTOR_CACHE_SIZE });
|
|
116
|
+
matrixCache = new shared_read_cache_1.SharedReadCache({
|
|
117
|
+
maxSize: MATRIX_CACHE_SIZE,
|
|
118
|
+
});
|
|
114
119
|
// Assigned in the constructor rather than here, since its byte budget is
|
|
115
120
|
// configurable and a field initializer cannot see `config`.
|
|
116
121
|
blockCache;
|
|
117
122
|
normVectorIndexPosition = -1;
|
|
118
123
|
normVectorIndexSize = -1;
|
|
119
|
-
initPromise;
|
|
120
|
-
normVectorIndexP;
|
|
121
124
|
version = 0;
|
|
122
125
|
genomeId = '';
|
|
123
126
|
footerPosition = 0;
|
|
@@ -125,7 +128,6 @@ class HicFile {
|
|
|
125
128
|
// carrying none of it ends. Public so a test can cut a real file there to
|
|
126
129
|
// make that second shape.
|
|
127
130
|
normExpectedValueVectorsPosition;
|
|
128
|
-
normVectorIndex;
|
|
129
131
|
chromosomes = [];
|
|
130
132
|
chromosomeIndexMap = {};
|
|
131
133
|
chrAliasTable = {};
|
|
@@ -152,142 +154,133 @@ class HicFile {
|
|
|
152
154
|
throw new Error('must supply filehandle, path, or reader');
|
|
153
155
|
}
|
|
154
156
|
}
|
|
155
|
-
async init() {
|
|
156
|
-
|
|
157
|
-
// share one header parse rather than both racing readHeaderAndFooter. Clear
|
|
158
|
-
// it on failure so a later call retries instead of caching a rejected
|
|
159
|
-
// promise forever.
|
|
160
|
-
this.initPromise ??= this.readHeaderAndFooter().catch((e) => {
|
|
161
|
-
this.initPromise = undefined;
|
|
162
|
-
throw e;
|
|
163
|
-
});
|
|
164
|
-
return this.initPromise;
|
|
157
|
+
async init(opts) {
|
|
158
|
+
return this.headerCache.get('header', opts?.signal, signal => this.readHeaderAndFooter(signal));
|
|
165
159
|
}
|
|
166
|
-
async getMetaData() {
|
|
167
|
-
await this.init();
|
|
160
|
+
async getMetaData(opts) {
|
|
161
|
+
await this.init(opts);
|
|
168
162
|
return this.meta;
|
|
169
163
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
this.chromosomes = [];
|
|
177
|
-
this.chromosomeIndexMap = {};
|
|
178
|
-
this.chrAliasTable = {};
|
|
179
|
-
this.bpResolutions = [];
|
|
180
|
-
this.masterIndex = {};
|
|
181
|
-
// Read initial fields magic, version, and footer position
|
|
182
|
-
let data = await this.file.read(0, 16);
|
|
164
|
+
// Parsed into locals and committed only at the end: a parse every caller
|
|
165
|
+
// abandoned can still be running beside the one that replaced it, and a
|
|
166
|
+
// transient failure part-way through must leave nothing half-written for the
|
|
167
|
+
// retry to append to.
|
|
168
|
+
async readHeaderAndFooter(signal) {
|
|
169
|
+
let data = await this.file.read(0, 16, { signal });
|
|
183
170
|
if (data.byteLength === 0) {
|
|
184
171
|
throw new Error('File content is empty');
|
|
185
172
|
}
|
|
186
173
|
let binaryParser = new binary_ts_1.default(new DataView(data));
|
|
187
174
|
binaryParser.getString(); // magic
|
|
188
|
-
|
|
189
|
-
if (
|
|
190
|
-
throw new Error(`Unsupported hic version: ${
|
|
191
|
-
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
data = await this.file.read(16, bodyPosition - 16);
|
|
175
|
+
const version = binaryParser.getInt();
|
|
176
|
+
if (version < 5) {
|
|
177
|
+
throw new Error(`Unsupported hic version: ${version}`);
|
|
178
|
+
}
|
|
179
|
+
const footerPosition = binaryParser.getLong();
|
|
180
|
+
const { masterIndex, normExpectedValueVectorsPosition } = await this.readFooter(version, footerPosition, signal);
|
|
181
|
+
const bodyPosition = Object.values(masterIndex).reduce((min, entry) => Math.min(min, entry.start), Number.MAX_VALUE);
|
|
182
|
+
data = await this.file.read(16, bodyPosition - 16, { signal });
|
|
197
183
|
binaryParser = new binary_ts_1.default(new DataView(data));
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
184
|
+
const genomeId = binaryParser.getString();
|
|
185
|
+
let normVectorIndexPosition = -1;
|
|
186
|
+
let normVectorIndexSize = -1;
|
|
187
|
+
if (version >= 9) {
|
|
188
|
+
normVectorIndexPosition = binaryParser.getLong();
|
|
189
|
+
normVectorIndexSize = binaryParser.getLong();
|
|
202
190
|
}
|
|
203
191
|
let nAttributes = binaryParser.getInt();
|
|
204
192
|
while (nAttributes-- > 0) {
|
|
205
193
|
binaryParser.getString(); // attribute key
|
|
206
194
|
binaryParser.getString(); // attribute value
|
|
207
195
|
}
|
|
196
|
+
const chromosomes = [];
|
|
197
|
+
const chromosomeIndexMap = {};
|
|
208
198
|
let nChrs = binaryParser.getInt();
|
|
209
199
|
let i = 0;
|
|
210
200
|
while (nChrs-- > 0) {
|
|
211
201
|
const chr = {
|
|
212
202
|
index: i,
|
|
213
203
|
name: binaryParser.getString(),
|
|
214
|
-
size:
|
|
204
|
+
size: version < 9 ? binaryParser.getInt() : binaryParser.getLong(),
|
|
215
205
|
};
|
|
216
|
-
|
|
217
|
-
|
|
206
|
+
chromosomes.push(chr);
|
|
207
|
+
chromosomeIndexMap[chr.name] = chr.index;
|
|
218
208
|
i++;
|
|
219
209
|
}
|
|
210
|
+
const bpResolutions = [];
|
|
220
211
|
let nBpResolutions = binaryParser.getInt();
|
|
221
212
|
while (nBpResolutions-- > 0) {
|
|
222
|
-
|
|
213
|
+
bpResolutions.push(binaryParser.getInt());
|
|
223
214
|
}
|
|
224
|
-
|
|
225
|
-
for (const chrName of Object.keys(
|
|
215
|
+
const chrAliasTable = {};
|
|
216
|
+
for (const chrName of Object.keys(chromosomeIndexMap)) {
|
|
226
217
|
if (chrName.startsWith('chr')) {
|
|
227
|
-
|
|
218
|
+
chrAliasTable[chrName.slice(3)] = chrName;
|
|
228
219
|
}
|
|
229
220
|
else if (chrName === 'MT') {
|
|
230
|
-
|
|
221
|
+
chrAliasTable.chrM = chrName;
|
|
231
222
|
}
|
|
232
223
|
else {
|
|
233
|
-
|
|
224
|
+
chrAliasTable[`chr${chrName}`] = chrName;
|
|
234
225
|
}
|
|
235
226
|
}
|
|
227
|
+
(0, shared_read_cache_1.throwIfAborted)(signal);
|
|
228
|
+
this.version = version;
|
|
229
|
+
this.footerPosition = footerPosition;
|
|
230
|
+
this.masterIndex = masterIndex;
|
|
231
|
+
this.normExpectedValueVectorsPosition = normExpectedValueVectorsPosition;
|
|
232
|
+
this.genomeId = genomeId;
|
|
233
|
+
this.normVectorIndexPosition = normVectorIndexPosition;
|
|
234
|
+
this.normVectorIndexSize = normVectorIndexSize;
|
|
235
|
+
this.chromosomes = chromosomes;
|
|
236
|
+
this.chromosomeIndexMap = chromosomeIndexMap;
|
|
237
|
+
this.bpResolutions = bpResolutions;
|
|
238
|
+
this.chrAliasTable = chrAliasTable;
|
|
236
239
|
this.meta = {
|
|
237
|
-
version
|
|
238
|
-
genome:
|
|
239
|
-
chromosomes
|
|
240
|
-
resolutions:
|
|
240
|
+
version,
|
|
241
|
+
genome: genomeId,
|
|
242
|
+
chromosomes,
|
|
243
|
+
resolutions: bpResolutions,
|
|
241
244
|
};
|
|
242
245
|
}
|
|
243
|
-
async readFooter() {
|
|
244
|
-
const skip =
|
|
245
|
-
let data = await this.file.read(
|
|
246
|
+
async readFooter(version, footerPosition, signal) {
|
|
247
|
+
const skip = version < 9 ? 8 : 12;
|
|
248
|
+
let data = await this.file.read(footerPosition, skip, { signal });
|
|
246
249
|
let binaryParser = new binary_ts_1.default(new DataView(data));
|
|
247
250
|
// Total size, master index + expected values
|
|
248
|
-
const nBytes =
|
|
251
|
+
const nBytes = version < 9 ? binaryParser.getInt() : binaryParser.getLong();
|
|
249
252
|
let nEntries = binaryParser.getInt();
|
|
250
253
|
// Estimate the size of the master index. String key length is unknown, be
|
|
251
254
|
// conservative (100 bytes).
|
|
252
255
|
const miSize = nEntries * (100 + 64 + 32);
|
|
253
|
-
data = await this.file.read(
|
|
256
|
+
data = await this.file.read(footerPosition + skip, Math.min(miSize, nBytes), { signal });
|
|
254
257
|
binaryParser = new binary_ts_1.default(new DataView(data));
|
|
258
|
+
const masterIndex = {};
|
|
255
259
|
while (nEntries-- > 0) {
|
|
256
260
|
const key = binaryParser.getString();
|
|
257
261
|
const pos = binaryParser.getLong();
|
|
258
262
|
const size = binaryParser.getInt();
|
|
259
|
-
|
|
263
|
+
masterIndex[key] = { start: pos, size };
|
|
260
264
|
}
|
|
261
265
|
// Normalized expected values start after the expected values
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
this.normExpectedValueVectorsPosition = this.footerPosition + s + nBytes;
|
|
265
|
-
}
|
|
266
|
+
const normExpectedValueVectorsPosition = version > 5 ? footerPosition + (version < 9 ? 4 : 8) + nBytes : undefined;
|
|
267
|
+
return { masterIndex, normExpectedValueVectorsPosition };
|
|
266
268
|
}
|
|
267
|
-
async getMatrix(chrIdx1, chrIdx2) {
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
if (!p) {
|
|
271
|
-
p = this.readMatrix(chrIdx1, chrIdx2).catch((e) => {
|
|
272
|
-
this.matrixCache.delete(key);
|
|
273
|
-
throw e;
|
|
274
|
-
});
|
|
275
|
-
this.matrixCache.set(key, p);
|
|
276
|
-
}
|
|
277
|
-
return p;
|
|
269
|
+
async getMatrix(chrIdx1, chrIdx2, opts) {
|
|
270
|
+
await this.init(opts);
|
|
271
|
+
return this.matrixCache.get(matrix_ts_1.default.getKey(chrIdx1, chrIdx2), opts?.signal, signal => this.readMatrix(chrIdx1, chrIdx2, signal));
|
|
278
272
|
}
|
|
279
|
-
async readMatrix(chrIdx1, chrIdx2) {
|
|
280
|
-
await this.init();
|
|
273
|
+
async readMatrix(chrIdx1, chrIdx2, signal) {
|
|
281
274
|
const idx = this.masterIndex[matrix_ts_1.default.getKey(chrIdx1, chrIdx2)];
|
|
282
275
|
let matrix;
|
|
283
276
|
if (idx) {
|
|
284
|
-
const data = await this.file.read(idx.start, idx.size);
|
|
277
|
+
const data = await this.file.read(idx.start, idx.size, { signal });
|
|
285
278
|
matrix = matrix_ts_1.default.parseMatrix(data, this.chromosomes);
|
|
286
279
|
}
|
|
287
280
|
return matrix;
|
|
288
281
|
}
|
|
289
282
|
async getContactRecords(normalization, region1, region2, units, binsize, opts) {
|
|
290
|
-
await this.init();
|
|
283
|
+
await this.init(opts);
|
|
291
284
|
const idx1 = this.chromosomeIndexMap[this.getFileChrName(region1.chr)];
|
|
292
285
|
const idx2 = this.chromosomeIndexMap[this.getFileChrName(region2.chr)];
|
|
293
286
|
// A `.hic` stores only `bin1 <= bin2`, so a query whose x window sits to the
|
|
@@ -325,12 +318,13 @@ class HicFile {
|
|
|
325
318
|
// resolved once per pair rather than per block, each paired with the bin
|
|
326
319
|
// offset its values start at.
|
|
327
320
|
const [norm, blocks] = await Promise.all([
|
|
328
|
-
this.getNormVectors(normalization, r1, r2, units, binsize),
|
|
321
|
+
this.getNormVectors(normalization, r1, r2, units, binsize, opts),
|
|
329
322
|
// The measurable half of the pair: blocks are counted and the two norm
|
|
330
323
|
// vectors are not, because the vector chain is two hops whatever the
|
|
331
324
|
// query and the block chain is the one that grows with the region.
|
|
332
325
|
this.getBlocks(r1, r2, binsize, opts),
|
|
333
326
|
]);
|
|
327
|
+
(0, shared_read_cache_1.throwIfAborted)(opts?.signal);
|
|
334
328
|
// Sum of the blocks' record counts bounds the survivors, so the output is
|
|
335
329
|
// allocated once and filled by a write cursor. Blocks overlap the window
|
|
336
330
|
// rather than nest in it, so the true count isn't known without either this
|
|
@@ -408,21 +402,22 @@ class HicFile {
|
|
|
408
402
|
transposed: transpose,
|
|
409
403
|
};
|
|
410
404
|
}
|
|
411
|
-
async getNormVectors(normalization, r1, r2, units, binsize) {
|
|
405
|
+
async getNormVectors(normalization, r1, r2, units, binsize, opts) {
|
|
406
|
+
const signalOpts = { signal: opts?.signal };
|
|
412
407
|
let result;
|
|
413
408
|
if (normalization && normalization !== 'NONE') {
|
|
414
409
|
const chr1 = this.getFileChrName(r1.chr);
|
|
415
410
|
const chr2 = this.getFileChrName(r2.chr);
|
|
416
411
|
const offset1 = Math.floor(r1.start / binsize);
|
|
417
412
|
const offset2 = Math.floor(r2.start / binsize);
|
|
418
|
-
const nv1 = await this.getNormalizationVector(normalization, chr1, units, binsize);
|
|
413
|
+
const nv1 = await this.getNormalizationVector(normalization, chr1, units, binsize, signalOpts);
|
|
419
414
|
const nv2 = chr1 === chr2
|
|
420
415
|
? nv1
|
|
421
|
-
: await this.getNormalizationVector(normalization, chr2, units, binsize);
|
|
416
|
+
: await this.getNormalizationVector(normalization, chr2, units, binsize, signalOpts);
|
|
422
417
|
if (nv1 && nv2) {
|
|
423
418
|
result = {
|
|
424
|
-
v1: await nv1.getValues(offset1, Math.ceil(r1.end / binsize)),
|
|
425
|
-
v2: await nv2.getValues(offset2, Math.ceil(r2.end / binsize)),
|
|
419
|
+
v1: await nv1.getValues(offset1, Math.ceil(r1.end / binsize), signalOpts),
|
|
420
|
+
v2: await nv2.getValues(offset2, Math.ceil(r2.end / binsize), signalOpts),
|
|
426
421
|
offset1,
|
|
427
422
|
offset2,
|
|
428
423
|
};
|
|
@@ -432,7 +427,7 @@ class HicFile {
|
|
|
432
427
|
}
|
|
433
428
|
async getBlocks(region1, region2, binSize, opts) {
|
|
434
429
|
const blockKey = (blockNumber, zd) => `${zd.getKey()}_${blockNumber}`;
|
|
435
|
-
await this.init();
|
|
430
|
+
await this.init(opts);
|
|
436
431
|
const chr1 = this.getFileChrName(region1.chr);
|
|
437
432
|
const chr2 = this.getFileChrName(region2.chr);
|
|
438
433
|
const idx1 = this.chromosomeIndexMap[chr1];
|
|
@@ -449,7 +444,8 @@ class HicFile {
|
|
|
449
444
|
// .hic files store no inter-chromosomal maps, and a multi-region view asks
|
|
450
445
|
// for every pair. Answering with no blocks, rather than warning once per
|
|
451
446
|
// pair per fetch, leaves that for the caller to notice or ignore.
|
|
452
|
-
const
|
|
447
|
+
const signalOpts = { signal: opts?.signal };
|
|
448
|
+
const matrix = await this.getMatrix(idx1, idx2, signalOpts);
|
|
453
449
|
if (matrix) {
|
|
454
450
|
const zd = matrix.getZoomData(binSize);
|
|
455
451
|
if (!zd) {
|
|
@@ -482,7 +478,7 @@ class HicFile {
|
|
|
482
478
|
// completions must not turn one wave into a queue. `readChainDepth`
|
|
483
479
|
// pins that.
|
|
484
480
|
const newBlocks = await Promise.all(blockNumbersToQuery.map(async (blockNumber) => {
|
|
485
|
-
const block = await this.readBlock(blockNumber, zd);
|
|
481
|
+
const block = await this.readBlock(blockNumber, zd, signalOpts);
|
|
486
482
|
done++;
|
|
487
483
|
onProgress?.(done, blockNumbers.length);
|
|
488
484
|
return block;
|
|
@@ -497,11 +493,14 @@ class HicFile {
|
|
|
497
493
|
}
|
|
498
494
|
return blocks;
|
|
499
495
|
}
|
|
500
|
-
async readBlock(blockNumber, zd) {
|
|
496
|
+
async readBlock(blockNumber, zd, opts) {
|
|
501
497
|
const idx = zd.blockIndex[blockNumber];
|
|
502
498
|
let block;
|
|
503
499
|
if (idx) {
|
|
504
|
-
const data = await this.file.read(idx.filePosition, idx.size
|
|
500
|
+
const data = await this.file.read(idx.filePosition, idx.size, {
|
|
501
|
+
signal: opts?.signal,
|
|
502
|
+
});
|
|
503
|
+
(0, shared_read_cache_1.throwIfAborted)(opts?.signal);
|
|
505
504
|
// `.subarray(2)` drops the zlib header: libdeflate's raw-deflate path is
|
|
506
505
|
// the fast one, and a `.hic` records only a block's *compressed* size, so
|
|
507
506
|
// there is no known output size to hand the exact-size entry point.
|
|
@@ -596,41 +595,34 @@ class HicFile {
|
|
|
596
595
|
}
|
|
597
596
|
throw new Error(`Unknown block type: ${type}`);
|
|
598
597
|
}
|
|
599
|
-
async getNormalizationVector(type, chr, unit, binSize) {
|
|
600
|
-
await this.init();
|
|
598
|
+
async getNormalizationVector(type, chr, unit, binSize, opts) {
|
|
599
|
+
await this.init(opts);
|
|
601
600
|
const chrIdx = this.chromosomeIndexMap[this.getFileChrName(chr)];
|
|
602
601
|
if (chrIdx === undefined) {
|
|
603
602
|
return undefined;
|
|
604
603
|
}
|
|
605
604
|
const key = getNormalizationVectorKey(type, chrIdx, unit, binSize);
|
|
606
|
-
//
|
|
607
|
-
// vector: a chromosome appears in every pair it takes part in, so with a
|
|
605
|
+
// Sharing the in-flight read is what keeps concurrent region pairs sharing
|
|
606
|
+
// one vector: a chromosome appears in every pair it takes part in, so with a
|
|
608
607
|
// result-only cache each of those pairs missed while the first was still in
|
|
609
608
|
// flight and read the header again — and then held its own
|
|
610
609
|
// `NormalizationVector`, whose value cache is per instance, so the whole
|
|
611
610
|
// vector was re-read too.
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
p = this.readNormalizationVector(key).catch((e) => {
|
|
621
|
-
this.normVectorCache.delete(key);
|
|
622
|
-
throw e;
|
|
623
|
-
});
|
|
624
|
-
this.normVectorCache.set(key, p);
|
|
625
|
-
}
|
|
626
|
-
return p;
|
|
611
|
+
//
|
|
612
|
+
// A file with no vectors at all, or none for this (type, chr, unit,
|
|
613
|
+
// binsize), simply answers undefined and the caller falls back to raw
|
|
614
|
+
// counts. hic-straw warns to the console here; that fires once per
|
|
615
|
+
// chromosome per region pair per fetch, and the console is the wrong place
|
|
616
|
+
// for it anyway — `getContactRecords` reports the normalization it actually
|
|
617
|
+
// applied instead.
|
|
618
|
+
return this.normVectorCache.get(key, opts?.signal, signal => this.readNormalizationVector(key, signal));
|
|
627
619
|
}
|
|
628
|
-
async readNormalizationVector(key) {
|
|
629
|
-
const idx = (await this.getNormVectorIndex())?.[key];
|
|
620
|
+
async readNormalizationVector(key, signal) {
|
|
621
|
+
const idx = (await this.getNormVectorIndex({ signal }))?.[key];
|
|
630
622
|
if (!idx) {
|
|
631
623
|
return undefined;
|
|
632
624
|
}
|
|
633
|
-
const data = await this.file.read(idx.filePosition, 8);
|
|
625
|
+
const data = await this.file.read(idx.filePosition, 8, { signal });
|
|
634
626
|
const parser = new binary_ts_1.default(new DataView(data));
|
|
635
627
|
const nValues = this.version < 9 ? parser.getInt() : parser.getLong();
|
|
636
628
|
const dataType = this.version < 9 ? DOUBLE : FLOAT;
|
|
@@ -638,52 +630,47 @@ class HicFile {
|
|
|
638
630
|
return new normalizationVector_ts_1.default(this.file, filePosition, nValues, dataType);
|
|
639
631
|
}
|
|
640
632
|
async getNormVectorIndex(opts) {
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
//
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
// discovery on every call — two calls per region pair per fetch, each
|
|
651
|
-
// walking the whole normalized-expected-values section with a chain of
|
|
652
|
-
// sequential range reads, only to rediscover there is nothing there.
|
|
653
|
-
// Cleared on failure (like `init`) so a transient read error retries
|
|
654
|
-
// rather than caching a rejection forever.
|
|
655
|
-
// The walk runs once, so its progress belongs to the call that performs
|
|
656
|
-
// it: a caller joining an in-flight or finished load is not waiting on
|
|
657
|
-
// reads and is told nothing, which is the truth rather than a silence.
|
|
658
|
-
this.normVectorIndexP ??= this.loadNormVectorIndex(opts?.onProgress).catch((e) => {
|
|
659
|
-
this.normVectorIndexP = undefined;
|
|
660
|
-
throw e;
|
|
661
|
-
});
|
|
662
|
-
await this.normVectorIndexP;
|
|
633
|
+
return (await this.getParsedNormVectorIndex(opts))?.entries;
|
|
634
|
+
}
|
|
635
|
+
async getParsedNormVectorIndex(opts) {
|
|
636
|
+
// await init() before the version gate, not after: this class defaults
|
|
637
|
+
// `version` to 0, so checking it before the header is parsed would answer
|
|
638
|
+
// "no index" for every file rather than for a v5 one.
|
|
639
|
+
await this.init(opts);
|
|
640
|
+
if (this.version < 6) {
|
|
641
|
+
return undefined;
|
|
663
642
|
}
|
|
664
|
-
|
|
643
|
+
// Memoize the *attempt*, not just a populated result. A legal (if uncommon)
|
|
644
|
+
// v8 file with no norm vectors has no index, and re-running the discovery
|
|
645
|
+
// on every call cost two calls per region pair per fetch, each walking the
|
|
646
|
+
// whole normalized-expected-values section with a chain of sequential range
|
|
647
|
+
// reads, only to rediscover there is nothing there.
|
|
648
|
+
//
|
|
649
|
+
// The walk runs once, so its progress belongs to the call that performs
|
|
650
|
+
// it: the fill is per call, so a caller joining an in-flight or finished
|
|
651
|
+
// load is not waiting on reads and is told nothing.
|
|
652
|
+
return this.normVectorIndexCache.get('nvi', opts?.signal, signal => this.loadNormVectorIndex({ signal, onProgress: opts?.onProgress }));
|
|
665
653
|
}
|
|
666
|
-
async loadNormVectorIndex(
|
|
667
|
-
// If we know the position of the norm vector index, read it directly.
|
|
668
|
-
// This is the case for hic v9 files.
|
|
654
|
+
async loadNormVectorIndex(opts) {
|
|
669
655
|
if (this.normVectorIndexPosition > 0 && this.normVectorIndexSize > 0) {
|
|
670
|
-
|
|
656
|
+
return this.readNormVectorIndex({
|
|
671
657
|
start: this.normVectorIndexPosition,
|
|
672
658
|
size: this.normVectorIndexSize,
|
|
673
|
-
});
|
|
659
|
+
}, opts.signal);
|
|
674
660
|
}
|
|
675
661
|
else if (this.config.nvi) {
|
|
676
662
|
const nviArray = decodeURIComponent(this.config.nvi).split(',');
|
|
677
|
-
|
|
663
|
+
return this.readNormVectorIndex({
|
|
678
664
|
start: parseInt(nviArray[0]),
|
|
679
665
|
size: parseInt(nviArray[1]),
|
|
680
|
-
});
|
|
666
|
+
}, opts.signal);
|
|
681
667
|
}
|
|
682
668
|
else {
|
|
683
669
|
try {
|
|
684
|
-
await this.readNormExpectedValuesAndNormVectorIndex(
|
|
670
|
+
return await this.readNormExpectedValuesAndNormVectorIndex(opts);
|
|
685
671
|
}
|
|
686
672
|
catch (e) {
|
|
673
|
+
(0, shared_read_cache_1.throwIfAborted)(opts.signal);
|
|
687
674
|
// Not "expected if the file has no norm vectors" — that case never
|
|
688
675
|
// arrives here. hic-straw's own IO threw a 416 for a read past EOF and
|
|
689
676
|
// this caught it; `generic-filehandle2` deliberately turns a 416 into an
|
|
@@ -692,41 +679,41 @@ class HicFile {
|
|
|
692
679
|
// the read side owns that case now, and anything reaching here is a real
|
|
693
680
|
// failure worth printing.
|
|
694
681
|
console.error(e);
|
|
682
|
+
return undefined;
|
|
695
683
|
}
|
|
696
684
|
}
|
|
697
685
|
}
|
|
698
686
|
async getNormalizationOptions(opts) {
|
|
699
|
-
|
|
700
|
-
// index. A bit ugly but alternatives are worse.
|
|
701
|
-
await this.getNormVectorIndex(opts);
|
|
702
|
-
return this.normalizationTypes;
|
|
687
|
+
return (await this.getParsedNormVectorIndex(opts))?.types ?? ['NONE'];
|
|
703
688
|
}
|
|
704
|
-
async readNormVectorIndex(range) {
|
|
705
|
-
await this.
|
|
706
|
-
const data = await this.file.read(range.start, range.size);
|
|
689
|
+
async readNormVectorIndex(range, signal) {
|
|
690
|
+
const data = await this.file.read(range.start, range.size, { signal });
|
|
707
691
|
const binaryParser = new binary_ts_1.default(new DataView(data));
|
|
708
|
-
|
|
692
|
+
const index = { entries: {}, types: ['NONE'] };
|
|
709
693
|
let nEntries = binaryParser.getInt();
|
|
710
694
|
while (nEntries-- > 0) {
|
|
711
|
-
this.parseNormVectorEntry(binaryParser);
|
|
695
|
+
this.parseNormVectorEntry(binaryParser, index);
|
|
712
696
|
}
|
|
713
|
-
return
|
|
697
|
+
return index;
|
|
714
698
|
}
|
|
715
699
|
// Used when the position of the norm vector index is unknown: read through
|
|
716
700
|
// the expected values to find the index.
|
|
717
|
-
async readNormExpectedValuesAndNormVectorIndex(
|
|
718
|
-
|
|
701
|
+
async readNormExpectedValuesAndNormVectorIndex(opts) {
|
|
702
|
+
const { signal } = opts;
|
|
703
|
+
let index;
|
|
719
704
|
if (this.normExpectedValueVectorsPosition !== undefined) {
|
|
720
|
-
const nviStart = await this.skipExpectedValues(this.normExpectedValueVectorsPosition,
|
|
705
|
+
const nviStart = await this.skipExpectedValues(this.normExpectedValueVectorsPosition, opts);
|
|
721
706
|
let byteCount = INT;
|
|
722
|
-
let data = await this.file.read(nviStart, INT);
|
|
707
|
+
let data = await this.file.read(nviStart, INT, { signal });
|
|
723
708
|
// Possible if there are no norm vectors. A legal v8 file, though uncommon.
|
|
724
709
|
if (data.byteLength !== 0) {
|
|
725
710
|
const binaryParser = new binary_ts_1.default(new DataView(data));
|
|
726
711
|
const nEntries = binaryParser.getInt();
|
|
727
712
|
const sizeEstimate = nEntries * 30;
|
|
728
|
-
data = await this.file.read(nviStart + byteCount, sizeEstimate
|
|
729
|
-
|
|
713
|
+
data = await this.file.read(nviStart + byteCount, sizeEstimate, {
|
|
714
|
+
signal,
|
|
715
|
+
});
|
|
716
|
+
const found = { entries: {}, types: ['NONE'] };
|
|
730
717
|
const processEntries = async (remaining, buf) => {
|
|
731
718
|
const parser = new binary_ts_1.default(new DataView(buf));
|
|
732
719
|
let n = remaining;
|
|
@@ -735,25 +722,31 @@ class HicFile {
|
|
|
735
722
|
n++; // Reset counter as entry is not processed
|
|
736
723
|
byteCount += parser.position;
|
|
737
724
|
const est = Math.max(1000, n * 30);
|
|
738
|
-
const more = await this.file.read(nviStart + byteCount, est
|
|
725
|
+
const more = await this.file.read(nviStart + byteCount, est, {
|
|
726
|
+
signal,
|
|
727
|
+
});
|
|
739
728
|
await processEntries(n, more);
|
|
740
729
|
return;
|
|
741
730
|
}
|
|
742
|
-
this.parseNormVectorEntry(parser);
|
|
731
|
+
this.parseNormVectorEntry(parser, found);
|
|
743
732
|
}
|
|
744
733
|
byteCount += parser.position;
|
|
745
734
|
};
|
|
746
735
|
await processEntries(nEntries, data);
|
|
736
|
+
(0, shared_read_cache_1.throwIfAborted)(signal);
|
|
747
737
|
this.config.nvi = `${nviStart},${byteCount}`;
|
|
738
|
+
index = found;
|
|
748
739
|
}
|
|
749
740
|
}
|
|
741
|
+
return index;
|
|
750
742
|
}
|
|
751
743
|
// Used when the position of the norm vector index is unknown: skip the
|
|
752
744
|
// normalized expected values to find the index.
|
|
753
|
-
async skipExpectedValues(start,
|
|
745
|
+
async skipExpectedValues(start, opts) {
|
|
746
|
+
const { signal, onProgress } = opts;
|
|
754
747
|
const version = this.version;
|
|
755
748
|
const file = new bufferedFile_ts_1.default({ file: this.file, size: 256000 });
|
|
756
|
-
const data = await file.read(start, INT);
|
|
749
|
+
const data = await file.read(start, INT, { signal });
|
|
757
750
|
// A file with no normalization at all ends where this section would start,
|
|
758
751
|
// so the count this is about to read is past EOF. That is the same "no norm
|
|
759
752
|
// vectors" case `readNormExpectedValuesAndNormVectorIndex` guards its own
|
|
@@ -777,14 +770,14 @@ class HicFile {
|
|
|
777
770
|
// read, each costing about the same.
|
|
778
771
|
const skipChunk = async (chunkStart) => {
|
|
779
772
|
let chunkSize = 0;
|
|
780
|
-
let buf = await file.read(chunkStart, 500);
|
|
773
|
+
let buf = await file.read(chunkStart, 500, { signal });
|
|
781
774
|
let parser = new binary_ts_1.default(new DataView(buf));
|
|
782
775
|
parser.getString(); // type
|
|
783
776
|
parser.getString(); // unit
|
|
784
777
|
parser.getInt(); // binSize
|
|
785
778
|
const nValues = version < 9 ? parser.getInt() : parser.getLong();
|
|
786
779
|
chunkSize += parser.position + nValues * (version < 9 ? DOUBLE : FLOAT);
|
|
787
|
-
buf = await file.read(chunkStart + chunkSize, INT);
|
|
780
|
+
buf = await file.read(chunkStart + chunkSize, INT, { signal });
|
|
788
781
|
parser = new binary_ts_1.default(new DataView(buf));
|
|
789
782
|
const nChrScaleFactors = parser.getInt();
|
|
790
783
|
chunkSize +=
|
|
@@ -806,7 +799,7 @@ class HicFile {
|
|
|
806
799
|
}
|
|
807
800
|
return position;
|
|
808
801
|
}
|
|
809
|
-
parseNormVectorEntry(binaryParser) {
|
|
802
|
+
parseNormVectorEntry(binaryParser, index) {
|
|
810
803
|
const type = binaryParser.getString(); // 15
|
|
811
804
|
const chrIdx = binaryParser.getInt(); // 4
|
|
812
805
|
const unit = binaryParser.getString(); // 3
|
|
@@ -814,10 +807,10 @@ class HicFile {
|
|
|
814
807
|
const filePosition = binaryParser.getLong(); // 8
|
|
815
808
|
const sizeInBytes = this.version < 9 ? binaryParser.getInt() : binaryParser.getLong(); // 4:8
|
|
816
809
|
const key = `${type}_${chrIdx}_${unit}_${binSize}`;
|
|
817
|
-
if (!
|
|
818
|
-
|
|
810
|
+
if (!index.types.includes(type)) {
|
|
811
|
+
index.types.push(type);
|
|
819
812
|
}
|
|
820
|
-
|
|
813
|
+
index.entries[key] = { filePosition, size: sizeInBytes };
|
|
821
814
|
}
|
|
822
815
|
getFileChrName(chrAlias) {
|
|
823
816
|
return this.chrAliasTable[chrAlias] ?? chrAlias;
|