@gmod/cram 4.0.3 → 4.0.5

package/src/cramFile/file.ts

@@ -4,11 +4,12 @@ import QuickLRU from 'quick-lru'
 import { XzReadableStream } from 'xz-decompress'
 
 import { CramMalformedError, CramUnimplementedError } from '../errors'
-import htscodecs from '../htscodecs'
+import * as htscodecs from '../htscodecs'
 import { open } from '../io'
 import ransuncompress from '../rans'
 import { parseHeaderText } from '../sam'
 import { unzip } from '../unzip'
+import { concatUint8Array } from '../util'
 import CramContainer from './container'
 import CramRecord from './record'
 import {
@@ -17,7 +18,7 @@ import {
   cramFileDefinition,
   getSectionParsers,
 } from './sectionParsers'
-import { concatUint8Array, parseItem, tinyMemoize } from './util'
+import { parseItem, tinyMemoize } from './util'
 
 import type { GenericFilehandle } from 'generic-filehandle2'
 
@@ -101,12 +102,6 @@ export default class CramFile {
     }
   }
 
-  // can just stat this object like a filehandle
-  stat() {
-    return this.file.stat()
-  }
-
-  // can just stat this object like a filehandle
   read(length: number, position: number) {
     return this.file.read(length, position)
   }
@@ -132,20 +127,17 @@ export default class CramFile {
     }
 
     const firstBlock = await firstContainer.getFirstBlock()
-    if (firstBlock === undefined) {
-      return parseHeaderText('')
-    } else {
-      const content = firstBlock.content
-      const dataView = new DataView(content.buffer)
-      const headerLength = dataView.getInt32(0, true)
-      const textStart = 4
-      const decoder = new TextDecoder('utf8')
-      const text = decoder.decode(
-        content.subarray(textStart, textStart + headerLength),
-      )
-      this.header = text
-      return parseHeaderText(text)
-    }
+
+    const content = firstBlock.content
+    const dataView = new DataView(content.buffer)
+    const headerLength = dataView.getInt32(0, true)
+    const textStart = 4
+    const decoder = new TextDecoder('utf8')
+    const text = decoder.decode(
+      content.subarray(textStart, textStart + headerLength),
+    )
+    this.header = text
+    return parseHeaderText(text)
   }
 
   async getHeaderText() {
@@ -157,25 +149,19 @@ export default class CramFile {
     const { majorVersion } = await this.getDefinition()
     const sectionParsers = getSectionParsers(majorVersion)
     let position = sectionParsers.cramFileDefinition.maxLength
-    const { size: fileSize } = await this.file.stat()
-    const { cramContainerHeader1 } = sectionParsers
 
     // skip with a series of reads to the proper container
     let currentContainer: CramContainer | undefined
     for (let i = 0; i <= containerNumber; i++) {
       // if we are about to go off the end of the file
       // and have not found that container, it does not exist
-      if (position + cramContainerHeader1.maxLength + 8 >= fileSize) {
-        return undefined
-      }
+      // if (position + cramContainerHeader1.maxLength + 8 >= fileSize) {
+      //   return undefined
+      // }
 
       currentContainer = this.getContainerAtPosition(position)
       const currentHeader = await currentContainer.getHeader()
-      if (!currentHeader) {
-        throw new CramMalformedError(
-          `container ${containerNumber} not found in file`,
-        )
-      }
+
       // if this is the first container, read all the blocks in the container
       // to determine its length, because we cannot trust the container
       // header's given length due to a bug somewhere in htslib
@@ -183,9 +169,6 @@ export default class CramFile {
         position = currentHeader._endPosition
         for (let j = 0; j < currentHeader.numBlocks; j++) {
           const block = await this.readBlock(position)
-          if (block === undefined) {
-            return undefined
-          }
           position = block._endPosition
         }
       } else {
@@ -218,39 +201,41 @@ export default class CramFile {
 
   /**
    * @returns {Promise[number]} the number of containers in the file
+   *
+   * note: this is currently used only in unit tests, and after removing file
+   * length check, relies on a try catch to read return an error to break
    */
   async containerCount(): Promise<number | undefined> {
     const { majorVersion } = await this.getDefinition()
     const sectionParsers = getSectionParsers(majorVersion)
-    const { size: fileSize } = await this.file.stat()
-    const { cramContainerHeader1 } = sectionParsers
 
     let containerCount = 0
     let position = sectionParsers.cramFileDefinition.maxLength
-    while (position + cramContainerHeader1.maxLength + 8 < fileSize) {
-      const currentHeader =
-        await this.getContainerAtPosition(position).getHeader()
-      if (!currentHeader) {
-        break
-      }
-      // if this is the first container, read all the blocks in the container,
-      // because we cannot trust the container header's given length due to a
-      // bug somewhere in htslib
-      if (containerCount === 0) {
-        position = currentHeader._endPosition
-        for (let j = 0; j < currentHeader.numBlocks; j++) {
-          const block = await this.readBlock(position)
-          if (block === undefined) {
-            return undefined
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+      while (true) {
+        const currentHeader =
+          await this.getContainerAtPosition(position).getHeader()
+
+        // if this is the first container, read all the blocks in the container,
+        // because we cannot trust the container header's given length due to a
+        // bug somewhere in htslib
+        if (containerCount === 0) {
+          position = currentHeader._endPosition
+          for (let j = 0; j < currentHeader.numBlocks; j++) {
+            const block = await this.readBlock(position)
+            position = block._endPosition
           }
-          position = block._endPosition
+        } else {
+          // otherwise, just traverse to the next container using the container's
+          // length
+          position += currentHeader._size + currentHeader.length
         }
-      } else {
-        // otherwise, just traverse to the next container using the container's
-        // length
-        position += currentHeader._size + currentHeader.length
+        containerCount += 1
       }
-      containerCount += 1
+    } catch (e) {
+      containerCount--
+      /* do nothing */
     }
 
     return containerCount
@@ -264,11 +249,6 @@ export default class CramFile {
     const { majorVersion } = await this.getDefinition()
     const sectionParsers = getSectionParsers(majorVersion)
     const { cramBlockHeader } = sectionParsers
-    const { size: fileSize } = await this.file.stat()
-
-    if (position + cramBlockHeader.maxLength >= fileSize) {
-      return undefined
-    }
 
     const buffer = await this.file.read(cramBlockHeader.maxLength, position)
     return parseItem(buffer, cramBlockHeader.parser, 0, position)
@@ -286,16 +266,7 @@ export default class CramFile {
     size = section.maxLength,
     preReadBuffer?: Uint8Array,
   ) {
-    let buffer: Uint8Array
-    if (preReadBuffer) {
-      buffer = preReadBuffer
-    } else {
-      const { size: fileSize } = await this.file.stat()
-      if (position + size >= fileSize) {
-        return undefined
-      }
-      buffer = await this.file.read(size, position)
-    }
+    const buffer = preReadBuffer ?? (await this.file.read(size, position))
     const data = parseItem(buffer, section.parser, 0, position)
     if (data._size !== size) {
       throw new CramMalformedError(
@@ -355,9 +326,6 @@ export default class CramFile {
     const { majorVersion } = await this.getDefinition()
     const sectionParsers = getSectionParsers(majorVersion)
     const blockHeader = await this.readBlockHeader(position)
-    if (blockHeader === undefined) {
-      return undefined
-    }
     const blockContentPosition = blockHeader._endPosition
 
     const d = await this.file.read(
@@ -385,9 +353,6 @@ export default class CramFile {
         sectionParsers.cramBlockCrc32,
         blockContentPosition + blockHeader.compressedSize,
       )
-      if (crc === undefined) {
-        return undefined
-      }
       block.crc32 = crc.crc32
 
       // check the block data crc32

package/src/cramFile/record.ts

@@ -271,7 +271,7 @@ export default class CramRecord {
 
     this.readGroupId = readGroupId
     this.readName = readName
-    this.sequenceId = sequenceId
+    this.sequenceId = sequenceId!
     this.uniqueId = uniqueId
     this.templateSize = templateSize
     this.alignmentStart = alignmentStart

package/src/cramFile/sectionParsers.ts

@@ -652,11 +652,14 @@ function cramContainerHeader1(majorVersion: number) {
     parser: (buffer: Uint8Array, offset: number) => {
       const b = buffer
       const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+
       // byte size of the container data (blocks)
       const length = dataView.getInt32(offset, true)
       offset += 4
-      // reference sequence identifier, -1 for unmapped reads, -2 for multiple
-      // reference sequences
+
+      // reference sequence identifier:
+      // -1 for unmapped reads,
+      // -2 for multiple reference sequences
       const [refSeqId, newOffset1] = parseItf8(buffer, offset)
       offset += newOffset1
       const [refSeqStart, newOffset2] = parseItf8(buffer, offset)

package/src/cramFile/slice/decodeRecord.ts

@@ -141,16 +141,14 @@ function decodeReadFeatures(
     const data = decodeDataSeries(dataSeriesName)
     if (type === 'character') {
       return String.fromCharCode(data)
-    }
-    if (type === 'string') {
+    } else if (type === 'string') {
       let r = ''
       for (let i = 0; i < data.byteLength; i++) {
         r += String.fromCharCode(data[i])
       }
       return r
-    }
-    if (type === 'numArray') {
-      return data.toArray()
+    } else if (type === 'numArray') {
+      return Array.from(data)
     }
     // else if (type === 'number') {
     //   return data[0]
@@ -213,7 +211,7 @@ function decodeReadFeatures(
 
 export type DataSeriesDecoder = <T extends DataSeriesEncodingKey>(
   dataSeriesName: T,
-) => DataTypeMapping[DataSeriesTypes[T]]
+) => DataTypeMapping[DataSeriesTypes[T]] | undefined
 
 export default function decodeRecord(
   slice: CramSlice,
@@ -226,12 +224,11 @@ export default function decodeRecord(
   majorVersion: number,
   recordNumber: number,
 ) {
-  let flags = decodeDataSeries('BF')
+  let flags = decodeDataSeries('BF')!
 
-  // note: the C data type of compressionFlags is byte in cram v1
-  // and int32 in cram v2+, but that does not matter for us here
-  // in javascript land.
-  const cramFlags = decodeDataSeries('CF')
+  // note: the C data type of compressionFlags is byte in cram v1 and int32 in
+  // cram v2+, but that does not matter for us here in javascript land.
+  const cramFlags = decodeDataSeries('CF')!
 
   if (!isMappedSliceHeader(sliceHeader.parsedContent)) {
     throw new Error('slice header not mapped')
@@ -242,18 +239,18 @@ export default function decodeRecord(
       ? decodeDataSeries('RI')
       : sliceHeader.parsedContent.refSeqId
 
-  const readLength = decodeDataSeries('RL')
+  const readLength = decodeDataSeries('RL')!
   // if APDelta, will calculate the true start in a second pass
-  let alignmentStart = decodeDataSeries('AP')
+  let alignmentStart = decodeDataSeries('AP')!
   if (compressionScheme.APdelta) {
     alignmentStart = alignmentStart + cursors.lastAlignmentStart
   }
   cursors.lastAlignmentStart = alignmentStart
-  const readGroupId = decodeDataSeries('RG')
+  const readGroupId = decodeDataSeries('RG')!
 
   let readName: string | undefined
   if (compressionScheme.readNamesIncluded) {
-    readName = readNullTerminatedString(decodeDataSeries('RN'))
+    readName = readNullTerminatedString(decodeDataSeries('RN')!)
   }
 
   let mateToUse:
@@ -270,14 +267,14 @@ export default function decodeRecord(
   if (CramFlagsDecoder.isDetached(cramFlags)) {
     // note: the MF is a byte in 1.0, int32 in 2+, but once again this doesn't
     // matter for javascript
-    const mateFlags = decodeDataSeries('MF')
+    const mateFlags = decodeDataSeries('MF')!
     let mateReadName: string | undefined
     if (!compressionScheme.readNamesIncluded) {
-      mateReadName = readNullTerminatedString(decodeDataSeries('RN'))
+      mateReadName = readNullTerminatedString(decodeDataSeries('RN')!)
       readName = mateReadName
     }
-    const mateSequenceId = decodeDataSeries('NS')
-    const mateAlignmentStart = decodeDataSeries('NP')
+    const mateSequenceId = decodeDataSeries('NS')!
+    const mateAlignmentStart = decodeDataSeries('NP')!
     if (mateFlags || mateSequenceId > -1) {
       mateToUse = {
         mateFlags,
@@ -287,7 +284,7 @@ export default function decodeRecord(
       }
     }
 
-    templateSize = decodeDataSeries('TS')
+    templateSize = decodeDataSeries('TS')!
 
     // set mate unmapped if needed
     if (MateFlagsDecoder.isUnmapped(mateFlags)) {
@@ -300,12 +297,12 @@ export default function decodeRecord(
 
     // detachedCount++
   } else if (CramFlagsDecoder.isWithMateDownstream(cramFlags)) {
-    mateRecordNumber = decodeDataSeries('NF') + recordNumber + 1
+    mateRecordNumber = decodeDataSeries('NF')! + recordNumber + 1
   }
 
   // TODO: the aux tag parsing will have to be refactored if we want to support
   // cram v1
-  const TLindex = decodeDataSeries('TL')
+  const TLindex = decodeDataSeries('TL')!
   if (TLindex < 0) {
     /* TODO: check nTL: TLindex >= compressionHeader.tagEncoding.size */
     throw new CramMalformedError('invalid TL index')
@@ -324,7 +321,11 @@ export default function decodeRecord(
       .getCodecForTag(tagId)
       .decode(slice, coreDataBlock, blocksByContentId, cursors)
     tags[tagName] =
-      typeof tagData === 'number' ? tagData : parseTagData(tagType, tagData)
+      tagData === undefined
+        ? undefined
+        : typeof tagData === 'number'
+          ? tagData
+          : parseTagData(tagType, tagData)
   }
 
   let readFeatures: ReadFeature[] | undefined
@@ -334,7 +335,7 @@ export default function decodeRecord(
   let readBases = undefined
   if (!BamFlagsDecoder.isSegmentUnmapped(flags)) {
     // reading read features
-    const readFeatureCount = decodeDataSeries('FN')
+    const readFeatureCount = decodeDataSeries('FN')!
     if (readFeatureCount) {
       readFeatures = decodeReadFeatures(
         alignmentStart,
@@ -369,11 +370,11 @@ export default function decodeRecord(
     }
 
     // mapping quality
-    mappingQuality = decodeDataSeries('MQ')
+    mappingQuality = decodeDataSeries('MQ')!
     if (CramFlagsDecoder.isPreservingQualityScores(cramFlags)) {
       qualityScores = new Array(readLength)
       for (let i = 0; i < qualityScores.length; i++) {
-        qualityScores[i] = decodeDataSeries('QS')
+        qualityScores[i] = decodeDataSeries('QS')!
       }
     }
   } else if (CramFlagsDecoder.isDecodeSequenceAsStar(cramFlags)) {
@@ -382,14 +383,14 @@ export default function decodeRecord(
   } else {
     const bases = new Array(readLength) as number[]
     for (let i = 0; i < bases.length; i++) {
-      bases[i] = decodeDataSeries('BA')
+      bases[i] = decodeDataSeries('BA')!
     }
     readBases = String.fromCharCode(...bases)
 
     if (CramFlagsDecoder.isPreservingQualityScores(cramFlags)) {
       qualityScores = new Array(readLength)
       for (let i = 0; i < bases.length; i++) {
-        qualityScores[i] = decodeDataSeries('QS')
+        qualityScores[i] = decodeDataSeries('QS')!
       }
     }
   }

package/src/cramFile/slice/index.ts

@@ -20,6 +20,13 @@ export type SliceHeader = CramFileBlock & {
   parsedContent: MappedSliceHeader | UnmappedSliceHeader
 }
 
+interface RefRegion {
+  id: number
+  start: number
+  end: number
+  seq: string | null
+}
+
 /**
  * @private
  * Try to estimate the template length from a bunch of interrelated multi-segment reads.
@@ -191,16 +198,10 @@ export default class CramSlice {
     const { majorVersion } = await this.file.getDefinition()
     const sectionParsers = getSectionParsers(majorVersion)
     const containerHeader = await this.container.getHeader()
-    if (!containerHeader) {
-      throw new Error('no container header detected')
-    }
 
     const header = await this.file.readBlock(
       containerHeader._endPosition + this.containerPosition,
     )
-    if (header === undefined) {
-      throw new Error('block header undefined')
-    }
     if (header.contentType === 'MAPPED_SLICE_HEADER') {
       const content = parseItem(
         header.content,
@@ -232,9 +233,6 @@ export default class CramSlice {
     const blocks: CramFileBlock[] = new Array(header.parsedContent.numBlocks)
     for (let i = 0; i < blocks.length; i++) {
       const block = await this.file.readBlock(blockPosition)
-      if (block === undefined) {
-        throw new Error('block undefined')
-      }
       blocks[i] = block
       blockPosition = blocks[i]!._endPosition
     }
@@ -404,21 +402,14 @@ export default class CramSlice {
       T extends DataSeriesEncodingKey,
     >(
       dataSeriesName: T,
-    ): DataTypeMapping[DataSeriesTypes[T]] => {
+    ): DataTypeMapping[DataSeriesTypes[T]] | undefined => {
       const codec = compressionScheme.getCodecForDataSeries(dataSeriesName)
       if (!codec) {
         throw new CramMalformedError(
           `no codec defined for ${dataSeriesName} data series`,
         )
       }
-      // console.log(dataSeriesName, Object.getPrototypeOf(codec))
-      const decoded = codec.decode(
-        this,
-        coreDataBlock,
-        blocksByContentId,
-        cursors,
-      )
-      return decoded
+      return codec.decode(this, coreDataBlock, blocksByContentId, cursors)
     }
     const records: CramRecord[] = new Array(
       sliceHeader.parsedContent.numRecords,
@@ -457,16 +448,22 @@ export default class CramSlice {
     }
 
     // interpret `recordsToNextFragment` attributes to make standard `mate`
-    // objects Resolve mate pair cross-references between records in this slice
+    // objects
+    //
+    // Resolve mate pair cross-references between records in this slice
     for (let i = 0; i < records.length; i += 1) {
-      const { mateRecordNumber } = records[i]!
-      if (mateRecordNumber !== undefined && mateRecordNumber >= 0) {
-        associateIntraSliceMate(
-          records,
-          i,
-          records[i]!,
-          records[mateRecordNumber]!,
-        )
+      const r = records[i]
+      // check for !!r added after removal  of "stat" file size check: found
+      // some undefined entries
+      if (r) {
+        const { mateRecordNumber } = r
+        if (
+          mateRecordNumber !== undefined &&
+          mateRecordNumber >= 0 &&
+          records[mateRecordNumber]
+        ) {
+          associateIntraSliceMate(records, i, r, records[mateRecordNumber])
+        }
       }
     }
 
@@ -501,10 +498,7 @@ export default class CramSlice {
         if (compressionScheme === undefined) {
           throw new Error('compression scheme undefined')
         }
-        const refRegions: Record<
-          string,
-          { id: number; start: number; end: number; seq: string | null }
-        > = {}
+        const refRegions: Record<string, RefRegion> = {}
 
         // iterate over the records to find the spans of the reference
         // sequences we need to fetch