@gmod/hic 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/hicFile.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { inflateRawUnknownSize } from '@gmod/inflate'
2
+ import { SharedReadCache, throwIfAborted } from '@gmod/shared-read-cache'
2
3
  import { LocalFile } from 'generic-filehandle2'
3
4
 
4
5
  import { binWindow } from './binWindow.ts'
@@ -12,11 +13,11 @@ import { readerFromFilehandle } from './reader.ts'
12
13
  import type { ContactRecords } from './contactRecords.ts'
13
14
  import type MatrixZoomData from './matrixZoomData.ts'
14
15
  import type {
16
+ BaseOpts,
15
17
  BlockIndexEntry,
16
18
  Chromosome,
17
19
  HicMetadata,
18
20
  HicRegion,
19
- ProgressCallback,
20
21
  ProgressOpts,
21
22
  Reader,
22
23
  } from './types.ts'
@@ -40,6 +41,15 @@ interface MasterIndexEntry {
40
41
  size: number
41
42
  }
42
43
 
44
+ interface NormVectorIndex {
45
+ entries: Record<string, BlockIndexEntry>
46
+ types: string[]
47
+ }
48
+
49
+ interface SignalOpts extends ProgressOpts {
50
+ signal: AbortSignal
51
+ }
52
+
43
53
  export interface HicConfig {
44
54
  /** a `generic-filehandle2` handle, e.g. `RemoteFile` for a URL */
45
55
  filehandle?: GenericFilehandle
@@ -160,27 +170,30 @@ export class HicFile {
160
170
  private config: HicConfig
161
171
  private file: Reader
162
172
 
163
- // Both caches hold the in-flight PROMISE, not the resolved value. A
173
+ // These caches share the in-flight read, not just its result. A
164
174
  // multi-region fetch runs its region pairs concurrently and they share
165
175
  // chromosomes, so caching only the result had every concurrent pair miss and
166
176
  // re-issue the same reads — measured +12 range requests on a 6-pair fetch.
167
- // Same shape as `initPromise`/`normVectorIndexP`, rejection eviction included.
168
- private normVectorCache = new LRU<
177
+ // `SharedReadCache` cancels a shared read only once every caller waiting on
178
+ // it has aborted, and drops a rejection so the next caller retries.
179
+ private headerCache = new SharedReadCache<string, void>()
180
+ private normVectorIndexCache = new SharedReadCache<
181
+ string,
182
+ NormVectorIndex | undefined
183
+ >()
184
+ private normVectorCache = new SharedReadCache<
169
185
  string,
170
- Promise<NormalizationVector | undefined>
171
- >(NORM_VECTOR_CACHE_SIZE)
172
- private normalizationTypes = ['NONE']
173
- private matrixCache = new LRU<string, Promise<Matrix | undefined>>(
174
- MATRIX_CACHE_SIZE,
175
- )
186
+ NormalizationVector | undefined
187
+ >({ maxSize: NORM_VECTOR_CACHE_SIZE })
188
+ private matrixCache = new SharedReadCache<string, Matrix | undefined>({
189
+ maxSize: MATRIX_CACHE_SIZE,
190
+ })
176
191
  // Assigned in the constructor rather than here, since its byte budget is
177
192
  // configurable and a field initializer cannot see `config`.
178
193
  private blockCache: LRU<string, Block>
179
194
  private normVectorIndexPosition = -1
180
195
  private normVectorIndexSize = -1
181
196
 
182
- private initPromise: Promise<void> | undefined
183
- private normVectorIndexP: Promise<void> | undefined
184
197
  private version = 0
185
198
  private genomeId = ''
186
199
  private footerPosition = 0
@@ -188,7 +201,6 @@ export class HicFile {
188
201
  // carrying none of it ends. Public so a test can cut a real file there to
189
202
  // make that second shape.
190
203
  normExpectedValueVectorsPosition: number | undefined
191
- private normVectorIndex: Record<string, BlockIndexEntry> | undefined
192
204
 
193
205
  private chromosomes: Chromosome[] = []
194
206
  private chromosomeIndexMap: Record<string, number> = {}
@@ -215,64 +227,52 @@ export class HicFile {
215
227
  }
216
228
  }
217
229
 
218
- async init() {
219
- // Memoize the promise, not a boolean flag: two concurrent callers must
220
- // share one header parse rather than both racing readHeaderAndFooter. Clear
221
- // it on failure so a later call retries instead of caching a rejected
222
- // promise forever.
223
- this.initPromise ??= this.readHeaderAndFooter().catch((e: unknown) => {
224
- this.initPromise = undefined
225
- throw e
226
- })
227
- return this.initPromise
230
+ async init(opts?: BaseOpts) {
231
+ return this.headerCache.get('header', opts?.signal, signal =>
232
+ this.readHeaderAndFooter(signal),
233
+ )
228
234
  }
229
235
 
230
- async getMetaData() {
231
- await this.init()
236
+ async getMetaData(opts?: BaseOpts) {
237
+ await this.init(opts)
232
238
  return this.meta!
233
239
  }
234
240
 
235
- private async readHeaderAndFooter() {
236
- // `init` clears its memoized promise on failure so a transient read error
237
- // retries, and this parse appends as it goes — so a retry has to start from
238
- // empty or a failure part-way through the chromosome loop leaves the second
239
- // run appending to the first's partial output (duplicate refNames out of
240
- // `getRefNames`, a duplicated binsize list out of `getMetaData`).
241
- this.chromosomes = []
242
- this.chromosomeIndexMap = {}
243
- this.chrAliasTable = {}
244
- this.bpResolutions = []
245
- this.masterIndex = {}
246
-
247
- // Read initial fields magic, version, and footer position
248
- let data = await this.file.read(0, 16)
241
+ // Parsed into locals and committed only at the end: a parse every caller
242
+ // abandoned can still be running beside the one that replaced it, and a
243
+ // transient failure part-way through must leave nothing half-written for the
244
+ // retry to append to.
245
+ private async readHeaderAndFooter(signal: AbortSignal) {
246
+ let data = await this.file.read(0, 16, { signal })
249
247
  if (data.byteLength === 0) {
250
248
  throw new Error('File content is empty')
251
249
  }
252
250
  let binaryParser = new BinaryParser(new DataView(data))
253
251
  binaryParser.getString() // magic
254
- this.version = binaryParser.getInt()
255
- if (this.version < 5) {
256
- throw new Error(`Unsupported hic version: ${this.version}`)
252
+ const version = binaryParser.getInt()
253
+ if (version < 5) {
254
+ throw new Error(`Unsupported hic version: ${version}`)
257
255
  }
258
- this.footerPosition = binaryParser.getLong()
256
+ const footerPosition = binaryParser.getLong()
259
257
 
260
- // Read footer to determine file position of body section (end of header)
261
- await this.readFooter()
258
+ const { masterIndex, normExpectedValueVectorsPosition } =
259
+ await this.readFooter(version, footerPosition, signal)
262
260
 
263
- const bodyPosition = Object.values(this.masterIndex).reduce(
261
+ const bodyPosition = Object.values(masterIndex).reduce(
264
262
  (min, entry) => Math.min(min, entry.start),
265
263
  Number.MAX_VALUE,
266
264
  )
267
265
 
268
- data = await this.file.read(16, bodyPosition - 16)
266
+ data = await this.file.read(16, bodyPosition - 16, { signal })
269
267
  binaryParser = new BinaryParser(new DataView(data))
270
268
 
271
- this.genomeId = binaryParser.getString()
269
+ const genomeId = binaryParser.getString()
272
270
 
273
- if (this.version >= 9) {
274
- this.normVectorIndexPosition = binaryParser.getLong()
275
- this.normVectorIndexSize = binaryParser.getLong()
271
+ let normVectorIndexPosition = -1
272
+ let normVectorIndexSize = -1
273
+ if (version >= 9) {
274
+ normVectorIndexPosition = binaryParser.getLong()
275
+ normVectorIndexSize = binaryParser.getLong()
276
276
  }
277
277
 
278
278
  let nAttributes = binaryParser.getInt()
@@ -281,96 +281,113 @@ export class HicFile {
281
281
  binaryParser.getString() // attribute value
282
282
  }
283
283
 
284
+ const chromosomes: Chromosome[] = []
285
+ const chromosomeIndexMap: Record<string, number> = {}
284
286
  let nChrs = binaryParser.getInt()
285
287
  let i = 0
286
288
  while (nChrs-- > 0) {
287
289
  const chr = {
288
290
  index: i,
289
291
  name: binaryParser.getString(),
290
- size: this.version < 9 ? binaryParser.getInt() : binaryParser.getLong(),
292
+ size: version < 9 ? binaryParser.getInt() : binaryParser.getLong(),
291
293
  }
292
- this.chromosomes.push(chr)
293
- this.chromosomeIndexMap[chr.name] = chr.index
294
+ chromosomes.push(chr)
295
+ chromosomeIndexMap[chr.name] = chr.index
294
296
  i++
295
297
  }
296
298
 
299
+ const bpResolutions: number[] = []
297
300
  let nBpResolutions = binaryParser.getInt()
298
301
  while (nBpResolutions-- > 0) {
299
- this.bpResolutions.push(binaryParser.getInt())
302
+ bpResolutions.push(binaryParser.getInt())
300
303
  }
301
304
 
302
- // Build lookup table for well-known chr aliases
303
- for (const chrName of Object.keys(this.chromosomeIndexMap)) {
305
+ const chrAliasTable: Record<string, string> = {}
306
+ for (const chrName of Object.keys(chromosomeIndexMap)) {
304
307
  if (chrName.startsWith('chr')) {
305
- this.chrAliasTable[chrName.slice(3)] = chrName
308
+ chrAliasTable[chrName.slice(3)] = chrName
306
309
  } else if (chrName === 'MT') {
307
- this.chrAliasTable.chrM = chrName
310
+ chrAliasTable.chrM = chrName
308
311
  } else {
309
- this.chrAliasTable[`chr${chrName}`] = chrName
312
+ chrAliasTable[`chr${chrName}`] = chrName
310
313
  }
311
314
  }
312
315
 
316
+ throwIfAborted(signal)
317
+ this.version = version
318
+ this.footerPosition = footerPosition
319
+ this.masterIndex = masterIndex
320
+ this.normExpectedValueVectorsPosition = normExpectedValueVectorsPosition
321
+ this.genomeId = genomeId
322
+ this.normVectorIndexPosition = normVectorIndexPosition
323
+ this.normVectorIndexSize = normVectorIndexSize
324
+ this.chromosomes = chromosomes
325
+ this.chromosomeIndexMap = chromosomeIndexMap
326
+ this.bpResolutions = bpResolutions
327
+ this.chrAliasTable = chrAliasTable
313
328
  this.meta = {
314
- version: this.version,
315
- genome: this.genomeId,
316
- chromosomes: this.chromosomes,
317
- resolutions: this.bpResolutions,
329
+ version,
330
+ genome: genomeId,
331
+ chromosomes,
332
+ resolutions: bpResolutions,
318
333
  }
319
334
  }
320
335
 
321
- private async readFooter() {
322
- const skip = this.version < 9 ? 8 : 12
323
- let data = await this.file.read(this.footerPosition, skip)
336
+ private async readFooter(
337
+ version: number,
338
+ footerPosition: number,
339
+ signal: AbortSignal,
340
+ ) {
341
+ const skip = version < 9 ? 8 : 12
342
+ let data = await this.file.read(footerPosition, skip, { signal })
324
343
 
325
344
  let binaryParser = new BinaryParser(new DataView(data))
326
345
  // Total size, master index + expected values
327
- const nBytes =
328
- this.version < 9 ? binaryParser.getInt() : binaryParser.getLong()
346
+ const nBytes = version < 9 ? binaryParser.getInt() : binaryParser.getLong()
329
347
  let nEntries = binaryParser.getInt()
330
348
 
331
349
  // Estimate the size of the master index. String key length is unknown, be
332
350
  // conservative (100 bytes).
333
351
  const miSize = nEntries * (100 + 64 + 32)
334
352
  data = await this.file.read(
335
- this.footerPosition + skip,
353
+ footerPosition + skip,
336
354
  Math.min(miSize, nBytes),
355
+ { signal },
337
356
  )
338
357
  binaryParser = new BinaryParser(new DataView(data))
339
358
 
359
+ const masterIndex: Record<string, MasterIndexEntry> = {}
340
360
  while (nEntries-- > 0) {
341
361
  const key = binaryParser.getString()
342
362
  const pos = binaryParser.getLong()
343
363
  const size = binaryParser.getInt()
344
- this.masterIndex[key] = { start: pos, size }
364
+ masterIndex[key] = { start: pos, size }
345
365
  }
346
366
 
347
367
  // Normalized expected values start after the expected values
348
- if (this.version > 5) {
349
- const s = this.version < 9 ? 4 : 8
350
- this.normExpectedValueVectorsPosition = this.footerPosition + s + nBytes
351
- }
368
+ const normExpectedValueVectorsPosition =
369
+ version > 5 ? footerPosition + (version < 9 ? 4 : 8) + nBytes : undefined
370
+ return { masterIndex, normExpectedValueVectorsPosition }
352
371
  }
353
372
 
354
- async getMatrix(chrIdx1: number, chrIdx2: number) {
355
- const key = Matrix.getKey(chrIdx1, chrIdx2)
356
- let p = this.matrixCache.get(key)
357
- if (!p) {
358
- p = this.readMatrix(chrIdx1, chrIdx2).catch((e: unknown) => {
359
- this.matrixCache.delete(key)
360
- throw e
361
- })
362
- this.matrixCache.set(key, p)
363
- }
364
- return p
373
+ async getMatrix(chrIdx1: number, chrIdx2: number, opts?: BaseOpts) {
374
+ await this.init(opts)
375
+ return this.matrixCache.get(
376
+ Matrix.getKey(chrIdx1, chrIdx2),
377
+ opts?.signal,
378
+ signal => this.readMatrix(chrIdx1, chrIdx2, signal),
379
+ )
365
380
  }
366
381
 
367
- private async readMatrix(chrIdx1: number, chrIdx2: number) {
368
- await this.init()
369
-
382
+ private async readMatrix(
383
+ chrIdx1: number,
384
+ chrIdx2: number,
385
+ signal: AbortSignal,
386
+ ) {
370
387
  const idx = this.masterIndex[Matrix.getKey(chrIdx1, chrIdx2)]
371
388
  let matrix: Matrix | undefined
372
389
  if (idx) {
373
- const data = await this.file.read(idx.start, idx.size)
390
+ const data = await this.file.read(idx.start, idx.size, { signal })
374
391
  matrix = Matrix.parseMatrix(data, this.chromosomes)
375
392
  }
376
393
  return matrix
@@ -384,7 +401,7 @@ export class HicFile {
384
401
  binsize: number,
385
402
  opts?: ProgressOpts,
386
403
  ) {
387
- await this.init()
404
+ await this.init(opts)
388
405
 
389
406
  const idx1 = this.chromosomeIndexMap[this.getFileChrName(region1.chr)]
390
407
  const idx2 = this.chromosomeIndexMap[this.getFileChrName(region2.chr)]
@@ -427,12 +444,13 @@ export class HicFile {
427
444
  // resolved once per pair rather than per block, each paired with the bin
428
445
  // offset its values start at.
429
446
  const [norm, blocks] = await Promise.all([
430
- this.getNormVectors(normalization, r1, r2, units, binsize),
447
+ this.getNormVectors(normalization, r1, r2, units, binsize, opts),
431
448
  // The measurable half of the pair: blocks are counted and the two norm
432
449
  // vectors are not, because the vector chain is two hops whatever the
433
450
  // query and the block chain is the one that grows with the region.
434
451
  this.getBlocks(r1, r2, binsize, opts),
435
452
  ])
453
+ throwIfAborted(opts?.signal)
436
454
 
437
455
  // Sum of the blocks' record counts bounds the survivors, so the output is
438
456
  // allocated once and filled by a write cursor. Blocks overlap the window
@@ -520,7 +538,9 @@ export class HicFile {
520
538
  r2: HicRegion,
521
539
  units: string,
522
540
  binsize: number,
541
+ opts?: BaseOpts,
523
542
  ) {
543
+ const signalOpts = { signal: opts?.signal }
524
544
  let result:
525
545
  | {
526
546
  v1: Float64Array
@@ -539,6 +559,7 @@ export class HicFile {
539
559
  chr1,
540
560
  units,
541
561
  binsize,
562
+ signalOpts,
542
563
  )
543
564
  const nv2 =
544
565
  chr1 === chr2
@@ -548,11 +569,20 @@ export class HicFile {
548
569
  chr2,
549
570
  units,
550
571
  binsize,
572
+ signalOpts,
551
573
  )
552
574
  if (nv1 && nv2) {
553
575
  result = {
554
- v1: await nv1.getValues(offset1, Math.ceil(r1.end / binsize)),
555
- v2: await nv2.getValues(offset2, Math.ceil(r2.end / binsize)),
576
+ v1: await nv1.getValues(
577
+ offset1,
578
+ Math.ceil(r1.end / binsize),
579
+ signalOpts,
580
+ ),
581
+ v2: await nv2.getValues(
582
+ offset2,
583
+ Math.ceil(r2.end / binsize),
584
+ signalOpts,
585
+ ),
556
586
  offset1,
557
587
  offset2,
558
588
  }
@@ -570,7 +600,7 @@ export class HicFile {
570
600
  const blockKey = (blockNumber: number, zd: MatrixZoomData) =>
571
601
  `${zd.getKey()}_${blockNumber}`
572
602
 
573
- await this.init()
603
+ await this.init(opts)
574
604
  const chr1 = this.getFileChrName(region1.chr)
575
605
  const chr2 = this.getFileChrName(region2.chr)
576
606
  const idx1 = this.chromosomeIndexMap[chr1]
@@ -586,7 +616,8 @@ export class HicFile {
586
616
  // .hic files store no inter-chromosomal maps, and a multi-region view asks
587
617
  // for every pair. Answering with no blocks, rather than warning once per
588
618
  // pair per fetch, leaves that for the caller to notice or ignore.
589
- const matrix = await this.getMatrix(idx1, idx2)
619
+ const signalOpts = { signal: opts?.signal }
620
+ const matrix = await this.getMatrix(idx1, idx2, signalOpts)
590
621
  if (matrix) {
591
622
  const zd = matrix.getZoomData(binSize)
592
623
  if (!zd) {
@@ -623,7 +654,7 @@ export class HicFile {
623
654
  // pins that.
624
655
  const newBlocks = await Promise.all(
625
656
  blockNumbersToQuery.map(async blockNumber => {
626
- const block = await this.readBlock(blockNumber, zd)
657
+ const block = await this.readBlock(blockNumber, zd, signalOpts)
627
658
  done++
628
659
  onProgress?.(done, blockNumbers.length)
629
660
  return block
@@ -640,12 +671,15 @@ export class HicFile {
640
671
  return blocks
641
672
  }
642
673
 
643
- async readBlock(blockNumber: number, zd: MatrixZoomData) {
674
+ async readBlock(blockNumber: number, zd: MatrixZoomData, opts?: BaseOpts) {
644
675
  const idx = zd.blockIndex[blockNumber]
645
676
 
646
677
  let block: Block | undefined
647
678
  if (idx) {
648
- const data = await this.file.read(idx.filePosition, idx.size)
679
+ const data = await this.file.read(idx.filePosition, idx.size, {
680
+ signal: opts?.signal,
681
+ })
682
+ throwIfAborted(opts?.signal)
649
683
  // `.subarray(2)` drops the zlib header: libdeflate's raw-deflate path is
650
684
  // the fast one, and a `.hic` records only a block's *compressed* size, so
651
685
  // there is no known output size to hand the exact-size entry point.
@@ -761,8 +795,9 @@ export class HicFile {
761
795
  chr: string,
762
796
  unit: string,
763
797
  binSize: number,
798
+ opts?: BaseOpts,
764
799
  ) {
765
- await this.init()
800
+ await this.init(opts)
766
801
 
767
802
  const chrIdx = this.chromosomeIndexMap[this.getFileChrName(chr)]
768
803
  if (chrIdx === undefined) {
@@ -770,35 +805,30 @@ export class HicFile {
770
805
  }
771
806
  const key = getNormalizationVectorKey(type, chrIdx, unit, binSize)
772
807
 
773
- // Caching the promise is what keeps concurrent region pairs sharing one
774
- // vector: a chromosome appears in every pair it takes part in, so with a
808
+ // Sharing the in-flight read is what keeps concurrent region pairs sharing
809
+ // one vector: a chromosome appears in every pair it takes part in, so with a
775
810
  // result-only cache each of those pairs missed while the first was still in
776
811
  // flight and read the header again — and then held its own
777
812
  // `NormalizationVector`, whose value cache is per instance, so the whole
778
813
  // vector was re-read too.
779
- let p = this.normVectorCache.get(key)
780
- if (!p) {
781
- // A file with no vectors at all, or none for this (type, chr, unit,
782
- // binsize), simply answers undefined and the caller falls back to raw
783
- // counts. hic-straw warns to the console here; that fires once per
784
- // chromosome per region pair per fetch, and the console is the wrong place
785
- // for it anyway — `getContactRecords` reports the normalization it
786
- // actually applied instead.
787
- p = this.readNormalizationVector(key).catch((e: unknown) => {
788
- this.normVectorCache.delete(key)
789
- throw e
790
- })
791
- this.normVectorCache.set(key, p)
792
- }
793
- return p
814
+ //
815
+ // A file with no vectors at all, or none for this (type, chr, unit,
816
+ // binsize), simply answers undefined and the caller falls back to raw
817
+ // counts. hic-straw warns to the console here; that fires once per
818
+ // chromosome per region pair per fetch, and the console is the wrong place
819
+ // for it anyway — `getContactRecords` reports the normalization it actually
820
+ // applied instead.
821
+ return this.normVectorCache.get(key, opts?.signal, signal =>
822
+ this.readNormalizationVector(key, signal),
823
+ )
794
824
  }
795
825
 
796
- private async readNormalizationVector(key: string) {
797
- const idx = (await this.getNormVectorIndex())?.[key]
826
+ private async readNormalizationVector(key: string, signal: AbortSignal) {
827
+ const idx = (await this.getNormVectorIndex({ signal }))?.[key]
798
828
  if (!idx) {
799
829
  return undefined
800
830
  }
801
- const data = await this.file.read(idx.filePosition, 8)
831
+ const data = await this.file.read(idx.filePosition, 8, { signal })
802
832
  const parser = new BinaryParser(new DataView(data))
803
833
  const nValues = this.version < 9 ? parser.getInt() : parser.getLong()
804
834
  const dataType = this.version < 9 ? DOUBLE : FLOAT
@@ -808,52 +838,54 @@ export class HicFile {
808
838
  }
809
839
 
810
840
  async getNormVectorIndex(opts?: ProgressOpts) {
811
- // Before the version gate, not after it: `version` is 0 until the header is
812
- // parsed, so calling this first — which `getNormalizationOptions` on its own
813
- // does — answered "no index" for every file rather than for a v5 one. A
814
- // caller that reads metadata first never sees it.
815
- await this.init()
816
- if (this.version >= 6) {
817
- // Memoize the *attempt*, not just a populated result. A legal (if
818
- // uncommon) v8 file with no norm vectors leaves `normVectorIndex`
819
- // undefined, and the old `!this.normVectorIndex` guard then re-ran the
820
- // discovery on every call — two calls per region pair per fetch, each
821
- // walking the whole normalized-expected-values section with a chain of
822
- // sequential range reads, only to rediscover there is nothing there.
823
- // Cleared on failure (like `init`) so a transient read error retries
824
- // rather than caching a rejection forever.
825
- // The walk runs once, so its progress belongs to the call that performs
826
- // it: a caller joining an in-flight or finished load is not waiting on
827
- // reads and is told nothing, which is the truth rather than a silence.
828
- this.normVectorIndexP ??= this.loadNormVectorIndex(
829
- opts?.onProgress,
830
- ).catch((e: unknown) => {
831
- this.normVectorIndexP = undefined
832
- throw e
833
- })
834
- await this.normVectorIndexP
841
+ return (await this.getParsedNormVectorIndex(opts))?.entries
842
+ }
843
+
844
+ private async getParsedNormVectorIndex(opts?: ProgressOpts) {
845
+ // await init() before the version gate, not after: this class defaults
846
+ // `version` to 0, so checking it before the header is parsed would answer
847
+ // "no index" for every file rather than for a v5 one.
848
+ await this.init(opts)
849
+ if (this.version < 6) {
850
+ return undefined
835
851
  }
836
- return this.normVectorIndex
852
+ // Memoize the *attempt*, not just a populated result. A legal (if uncommon)
853
+ // v8 file with no norm vectors has no index, and re-running the discovery
854
+ // on every call cost two calls per region pair per fetch, each walking the
855
+ // whole normalized-expected-values section with a chain of sequential range
856
+ // reads, only to rediscover there is nothing there.
857
+ //
858
+ // The walk runs once, so its progress belongs to the call that performs
859
+ // it: the fill is per call, so a caller joining an in-flight or finished
860
+ // load is not waiting on reads and is told nothing.
861
+ return this.normVectorIndexCache.get('nvi', opts?.signal, signal =>
862
+ this.loadNormVectorIndex({ signal, onProgress: opts?.onProgress }),
863
+ )
837
864
  }
838
865
 
839
- private async loadNormVectorIndex(onProgress?: ProgressCallback) {
840
- // If we know the position of the norm vector index, read it directly.
841
- // This is the case for hic v9 files.
866
+ private async loadNormVectorIndex(opts: SignalOpts) {
842
867
  if (this.normVectorIndexPosition > 0 && this.normVectorIndexSize > 0) {
843
- await this.readNormVectorIndex({
844
- start: this.normVectorIndexPosition,
845
- size: this.normVectorIndexSize,
846
- })
868
+ return this.readNormVectorIndex(
869
+ {
870
+ start: this.normVectorIndexPosition,
871
+ size: this.normVectorIndexSize,
872
+ },
873
+ opts.signal,
874
+ )
847
875
  } else if (this.config.nvi) {
848
876
  const nviArray = decodeURIComponent(this.config.nvi).split(',')
849
- await this.readNormVectorIndex({
850
- start: parseInt(nviArray[0]!),
851
- size: parseInt(nviArray[1]!),
852
- })
877
+ return this.readNormVectorIndex(
878
+ {
879
+ start: parseInt(nviArray[0]!),
880
+ size: parseInt(nviArray[1]!),
881
+ },
882
+ opts.signal,
883
+ )
853
884
  } else {
854
885
  try {
855
- await this.readNormExpectedValuesAndNormVectorIndex(onProgress)
886
+ return await this.readNormExpectedValuesAndNormVectorIndex(opts)
856
887
  } catch (e) {
888
+ throwIfAborted(opts.signal)
857
889
  // Not "expected if the file has no norm vectors" — that case never
858
890
  // arrives here. hic-straw's own IO threw a 416 for a read past EOF and
859
891
  // this caught it; `generic-filehandle2` deliberately turns a 416 into an
@@ -862,50 +894,51 @@ export class HicFile {
862
894
  // the read side owns that case now, and anything reaching here is a real
863
895
  // failure worth printing.
864
896
  console.error(e)
897
+ return undefined
865
898
  }
866
899
  }
867
900
  }
868
901
 
869
902
  async getNormalizationOptions(opts?: ProgressOpts) {
870
- // Normalization options are computed as a side effect of loading the
871
- // index. A bit ugly but alternatives are worse.
872
- await this.getNormVectorIndex(opts)
873
- return this.normalizationTypes
903
+ return (await this.getParsedNormVectorIndex(opts))?.types ?? ['NONE']
874
904
  }
875
905
 
876
- private async readNormVectorIndex(range: { start: number; size: number }) {
877
- await this.init()
878
- const data = await this.file.read(range.start, range.size)
906
+ private async readNormVectorIndex(
907
+ range: { start: number; size: number },
908
+ signal: AbortSignal,
909
+ ) {
910
+ const data = await this.file.read(range.start, range.size, { signal })
879
911
  const binaryParser = new BinaryParser(new DataView(data))
880
- this.normVectorIndex = {}
912
+ const index: NormVectorIndex = { entries: {}, types: ['NONE'] }
881
913
  let nEntries = binaryParser.getInt()
882
914
  while (nEntries-- > 0) {
883
- this.parseNormVectorEntry(binaryParser)
915
+ this.parseNormVectorEntry(binaryParser, index)
884
916
  }
885
- return this.normVectorIndex
917
+ return index
886
918
  }
887
919
 
888
920
  // Used when the position of the norm vector index is unknown: read through
889
921
  // the expected values to find the index.
890
- private async readNormExpectedValuesAndNormVectorIndex(
891
- onProgress?: ProgressCallback,
892
- ) {
893
- await this.init()
922
+ private async readNormExpectedValuesAndNormVectorIndex(opts: SignalOpts) {
923
+ const { signal } = opts
924
+ let index: NormVectorIndex | undefined
894
925
  if (this.normExpectedValueVectorsPosition !== undefined) {
895
926
  const nviStart = await this.skipExpectedValues(
896
927
  this.normExpectedValueVectorsPosition,
897
- onProgress,
928
+ opts,
898
929
  )
899
930
  let byteCount = INT
900
931
 
901
- let data = await this.file.read(nviStart, INT)
932
+ let data = await this.file.read(nviStart, INT, { signal })
902
933
  // Possible if there are no norm vectors. A legal v8 file, though uncommon.
903
934
  if (data.byteLength !== 0) {
904
935
  const binaryParser = new BinaryParser(new DataView(data))
905
936
  const nEntries = binaryParser.getInt()
906
937
  const sizeEstimate = nEntries * 30
907
- data = await this.file.read(nviStart + byteCount, sizeEstimate)
908
- this.normVectorIndex = {}
938
+ data = await this.file.read(nviStart + byteCount, sizeEstimate, {
939
+ signal,
940
+ })
941
+ const found: NormVectorIndex = { entries: {}, types: ['NONE'] }
909
942
 
910
943
  const processEntries = async (remaining: number, buf: ArrayBuffer) => {
911
944
  const parser = new BinaryParser(new DataView(buf))
@@ -915,30 +948,33 @@ export class HicFile {
915
948
  n++ // Reset counter as entry is not processed
916
949
  byteCount += parser.position
917
950
  const est = Math.max(1000, n * 30)
918
- const more = await this.file.read(nviStart + byteCount, est)
951
+ const more = await this.file.read(nviStart + byteCount, est, {
952
+ signal,
953
+ })
919
954
  await processEntries(n, more)
920
955
  return
921
956
  }
922
- this.parseNormVectorEntry(parser)
957
+ this.parseNormVectorEntry(parser, found)
923
958
  }
924
959
  byteCount += parser.position
925
960
  }
926
961
 
927
962
  await processEntries(nEntries, data)
963
+ throwIfAborted(signal)
928
964
  this.config.nvi = `${nviStart},${byteCount}`
965
+ index = found
929
966
  }
930
967
  }
968
+ return index
931
969
  }
932
970
 
933
971
  // Used when the position of the norm vector index is unknown: skip the
934
972
  // normalized expected values to find the index.
935
- private async skipExpectedValues(
936
- start: number,
937
- onProgress?: ProgressCallback,
938
- ) {
973
+ private async skipExpectedValues(start: number, opts: SignalOpts) {
974
+ const { signal, onProgress } = opts
939
975
  const version = this.version
940
976
  const file = new BufferedFile({ file: this.file, size: 256000 })
941
- const data = await file.read(start, INT)
977
+ const data = await file.read(start, INT, { signal })
942
978
  // A file with no normalization at all ends where this section would start,
943
979
  // so the count this is about to read is past EOF. That is the same "no norm
944
980
  // vectors" case `readNormExpectedValuesAndNormVectorIndex` guards its own
@@ -964,7 +1000,7 @@ export class HicFile {
964
1000
  const skipChunk = async (chunkStart: number) => {
965
1001
  let chunkSize = 0
966
1002
 
967
- let buf = await file.read(chunkStart, 500)
1003
+ let buf = await file.read(chunkStart, 500, { signal })
968
1004
  let parser = new BinaryParser(new DataView(buf))
969
1005
  parser.getString() // type
970
1006
  parser.getString() // unit
@@ -972,7 +1008,7 @@ export class HicFile {
972
1008
  const nValues = version < 9 ? parser.getInt() : parser.getLong()
973
1009
  chunkSize += parser.position + nValues * (version < 9 ? DOUBLE : FLOAT)
974
1010
 
975
- buf = await file.read(chunkStart + chunkSize, INT)
1011
+ buf = await file.read(chunkStart + chunkSize, INT, { signal })
976
1012
  parser = new BinaryParser(new DataView(buf))
977
1013
  const nChrScaleFactors = parser.getInt()
978
1014
  chunkSize +=
@@ -997,7 +1033,10 @@ export class HicFile {
997
1033
  return position
998
1034
  }
999
1035
 
1000
- private parseNormVectorEntry(binaryParser: BinaryParser) {
1036
+ private parseNormVectorEntry(
1037
+ binaryParser: BinaryParser,
1038
+ index: NormVectorIndex,
1039
+ ) {
1001
1040
  const type = binaryParser.getString() // 15
1002
1041
  const chrIdx = binaryParser.getInt() // 4
1003
1042
  const unit = binaryParser.getString() // 3
@@ -1007,10 +1046,10 @@ export class HicFile {
1007
1046
  this.version < 9 ? binaryParser.getInt() : binaryParser.getLong() // 4:8
1008
1047
  const key = `${type}_${chrIdx}_${unit}_${binSize}`
1009
1048
 
1010
- if (!this.normalizationTypes.includes(type)) {
1011
- this.normalizationTypes.push(type)
1049
+ if (!index.types.includes(type)) {
1050
+ index.types.push(type)
1012
1051
  }
1013
- this.normVectorIndex![key] = { filePosition, size: sizeInBytes }
1052
+ index.entries[key] = { filePosition, size: sizeInBytes }
1014
1053
  }
1015
1054
 
1016
1055
  getFileChrName(chrAlias: string) {