@gmod/cram 1.6.3 → 1.7.0

package/src/cramFile/{file.js → file.ts}

@@ -1,10 +1,12 @@
 import { unzip } from '../unzip'
 import crc32 from 'buffer-crc32'
-import LRU from 'quick-lru'
+import QuickLRU from 'quick-lru'
 
-import { CramUnimplementedError, CramMalformedError } from '../errors'
+import { CramMalformedError, CramUnimplementedError } from '../errors'
 import ransuncompress from '../rans'
 import {
+  BlockHeader,
+  CompressionMethod,
   cramFileDefinition as cramFileDefinitionParser,
   getSectionParsers,
 } from './sectionParsers'
@@ -14,10 +16,14 @@ import CramContainer from './container'
 import { open } from '../io'
 import { parseItem, tinyMemoize } from './util'
 import { parseHeaderText } from '../sam'
+import { Parser } from '@gmod/binary-parser'
+import CramRecord from './record'
+import { Filehandle } from './filehandle'
+
 //source:https://abdulapopoola.com/2019/01/20/check-endianness-with-javascript/
 function getEndianness() {
-  let uInt32 = new Uint32Array([0x11223344])
-  let uInt8 = new Uint8Array(uInt32.buffer)
+  const uInt32 = new Uint32Array([0x11223344])
+  const uInt8 = new Uint8Array(uInt32.buffer)
 
   if (uInt8[0] === 0x44) {
     return 0 //little-endian
@@ -28,33 +34,62 @@ function getEndianness() {
   }
 }
 
+// export type CramFileSource =
+//   | { url: string; path?: undefined; filehandle?: undefined }
+//   | { path: string; url?: undefined; filehandle?: undefined }
+//   | { filehandle: Filehandle; url?: undefined; path?: undefined }
+
+export type CramFileSource = {
+  filehandle?: Filehandle
+  url?: string
+  path?: string
+}
+
+export type SeqFetch = (
+  seqId: number,
+  start: number,
+  end: number,
+) => Promise<string>
+
+export type CramFileArgs = CramFileSource & {
+  checkSequenceMD5: boolean
+  cacheSize?: number
+  seqFetch: SeqFetch
+}
+
+export type CramFileBlock = BlockHeader & {
+  _endPosition: number
+  contentPosition: number
+  _size: number
+  content: Buffer
+  crc32?: number
+}
+
 export default class CramFile {
-  /**
-   * @param {object} args
-   * @param {object} [args.filehandle] - a filehandle that implements the stat() and
-   * read() methods of the Node filehandle API https://nodejs.org/api/fs.html#fs_class_filehandle
-   * @param {object} [args.path] - path to the cram file
-   * @param {object} [args.url] - url for the cram file.  also supports file:// urls for local files
-   * @param {function} [args.seqFetch] - a function with signature
-   * `(seqId, startCoordinate, endCoordinate)` that returns a promise for a string of sequence bases
-   * @param {number} [args.cacheSize] optional maximum number of CRAM records to cache.  default 20,000
-   * @param {boolean} [args.checkSequenceMD5] - default true. if false, disables verifying the MD5
-   * checksum of the reference sequence underlying a slice. In some applications, this check can cause an inconvenient amount (many megabases) of sequences to be fetched.
-   */
-  constructor(args) {
+  private file: Filehandle
+  public validateChecksums: boolean
+  public fetchReferenceSequenceCallback: SeqFetch
+  public options: {
+    checkSequenceMD5: boolean
+    cacheSize: number
+  }
+  public featureCache: QuickLRU<string, Promise<CramRecord[]>>
+  private header: string | undefined
+
+  constructor(args: CramFileArgs) {
     this.file = open(args.url, args.path, args.filehandle)
     this.validateChecksums = true
     this.fetchReferenceSequenceCallback = args.seqFetch
     this.options = {
-      checkSequenceMD5: args.checkSequenceMD5 !== false,
-      cacheSize: args.cacheSize !== undefined ? args.cacheSize : 20000,
+      checkSequenceMD5: args.checkSequenceMD5,
+      cacheSize: args.cacheSize ?? 20000,
     }
 
     // cache of features in a slice, keyed by the
     // slice offset. caches all of the features in a slice, or none.
     // the cache is actually used by the slice object, it's just
     // kept here at the level of the file
-    this.featureCache = new LRU({
+    this.featureCache = new QuickLRU({
       maxSize: this.options.cacheSize,
     })
     if (getEndianness() > 0) {
@@ -62,19 +97,27 @@ export default class CramFile {
     }
   }
 
-  toString() {
-    if (this.file.filename) {
-      return this.file.filename
-    }
-    if (this.file.url) {
-      return this.file.url
-    }
-
-    return '(cram file)'
-  }
+  // toString() {
+  //   if (this.file.filename) {
+  //     return this.file.filename
+  //   }
+  //   if (this.file.url) {
+  //     return this.file.url
+  //   }
+  //
+  //   return '(cram file)'
+  // }
 
   // can just read this object like a filehandle
-  read(buffer, offset, length, position) {
+  read(
+    buffer: Buffer,
+    offset: number,
+    length: number,
+    position: number,
+  ): Promise<{
+    bytesRead: number
+    buffer: Buffer
+  }> {
     return this.file.read(buffer, offset, length, position)
   }
 
@@ -87,7 +130,8 @@ export default class CramFile {
   async getDefinition() {
     const headbytes = Buffer.allocUnsafe(cramFileDefinitionParser.maxLength)
     await this.file.read(headbytes, 0, cramFileDefinitionParser.maxLength, 0)
-    const definition = cramFileDefinitionParser.parser.parse(headbytes).result
+    const definition = cramFileDefinitionParser.parser.parse(headbytes)
+      .result as any
     if (definition.majorVersion !== 2 && definition.majorVersion !== 3) {
       throw new CramUnimplementedError(
         `CRAM version ${definition.majorVersion} not supported`,
@@ -103,7 +147,11 @@ export default class CramFile {
       throw new CramMalformedError('file contains no containers')
     }
 
-    const { content } = await firstContainer.getFirstBlock()
+    const firstBlock = await firstContainer.getFirstBlock()
+    if (firstBlock === undefined) {
+      return parseHeaderText('')
+    }
+    const content = firstBlock.content
     // find the end of the trailing zeros in the header text
     const headerLength = content.readInt32LE(0)
     const textStart = 4
@@ -126,7 +174,7 @@ export default class CramFile {
     return getSectionParsers(majorVersion)
   }
 
-  async getContainerById(containerNumber) {
+  async getContainerById(containerNumber: number) {
     const sectionParsers = await this.getSectionParsers()
     let position = sectionParsers.cramFileDefinition.maxLength
     const { size: fileSize } = await this.file.stat()
@@ -156,6 +204,9 @@ export default class CramFile {
         position = currentHeader._endPosition
         for (let j = 0; j < currentHeader.numBlocks; j += 1) {
           const block = await this.readBlock(position)
+          if (block === undefined) {
+            return undefined
+          }
           position = block._endPosition
         }
       } else {
@@ -167,7 +218,12 @@ export default class CramFile {
     return currentContainer
   }
 
-  async checkCrc32(position, length, recordedCrc32, description) {
+  async checkCrc32(
+    position: number,
+    length: number,
+    recordedCrc32: number,
+    description: string,
+  ) {
     const b = Buffer.allocUnsafe(length)
     await this.file.read(b, 0, length, position)
     const calculatedCrc32 = crc32.unsigned(b)
@@ -181,7 +237,7 @@ export default class CramFile {
   /**
    * @returns {Promise[number]} the number of containers in the file
    */
-  async containerCount() {
+  async containerCount(): Promise<number | undefined> {
     const sectionParsers = await this.getSectionParsers()
     const { size: fileSize } = await this.file.stat()
     const { cramContainerHeader1 } = sectionParsers
@@ -202,6 +258,9 @@ export default class CramFile {
         position = currentHeader._endPosition
         for (let j = 0; j < currentHeader.numBlocks; j += 1) {
           const block = await this.readBlock(position)
+          if (block === undefined) {
+            return undefined
+          }
           position = block._endPosition
         }
       } else {
@@ -214,11 +273,11 @@ export default class CramFile {
     return containerCount
   }
 
-  getContainerAtPosition(position) {
+  getContainerAtPosition(position: number) {
     return new CramContainer(this, position)
   }
 
-  async readBlockHeader(position) {
+  async readBlockHeader(position: number) {
     const sectionParsers = await this.getSectionParsers()
     const { cramBlockHeader } = sectionParsers
     const { size: fileSize } = await this.file.stat()
@@ -232,11 +291,11 @@ export default class CramFile {
     return parseItem(buffer, cramBlockHeader.parser, 0, position)
   }
 
-  async _parseSection(
-    section,
-    position,
+  async _parseSection<T>(
+    section: { parser: Parser<T>; maxLength: number },
+    position: number,
     size = section.maxLength,
-    preReadBuffer,
+    preReadBuffer = undefined,
   ) {
     let buffer
     if (preReadBuffer) {
@@ -258,16 +317,21 @@ export default class CramFile {
     return data
   }
 
-  _uncompress(compressionMethod, inputBuffer, outputBuffer) {
+  _uncompress(
+    compressionMethod: CompressionMethod,
+    inputBuffer: Buffer,
+    outputBuffer: Buffer,
+  ) {
     if (compressionMethod === 'gzip') {
       const result = unzip(inputBuffer)
       result.copy(outputBuffer)
     } else if (compressionMethod === 'bzip2') {
-      var bits = bzip2.array(inputBuffer)
-      var size = bzip2.header(bits)
-      var j = 0
+      const bits = bzip2.array(inputBuffer)
+      let size = bzip2.header(bits)
+      let j = 0
+      let chunk
       do {
-        var chunk = bzip2.decompress(bits, size)
+        chunk = bzip2.decompress(bits, size)
         if (chunk != -1) {
           Buffer.from(chunk).copy(outputBuffer, j)
           j += chunk.length
@@ -293,26 +357,35 @@ export default class CramFile {
     }
   }
 
-  async readBlock(position) {
+  async readBlock(position: number): Promise<CramFileBlock | undefined> {
     const { majorVersion } = await this.getDefinition()
     const sectionParsers = await this.getSectionParsers()
-    const block = await this.readBlockHeader(position)
-    const blockContentPosition = block._endPosition
-    block.contentPosition = block._endPosition
+    const blockHeader = await this.readBlockHeader(position)
+    if (blockHeader === undefined) {
+      return undefined
+    }
+    const blockContentPosition = blockHeader._endPosition
+
+    const uncompressedData = Buffer.allocUnsafe(blockHeader.uncompressedSize)
 
-    const uncompressedData = Buffer.allocUnsafe(block.uncompressedSize)
+    const block: CramFileBlock = {
+      ...blockHeader,
+      _endPosition: blockContentPosition,
+      contentPosition: blockContentPosition,
+      content: uncompressedData,
+    }
 
-    if (block.compressionMethod !== 'raw') {
-      const compressedData = Buffer.allocUnsafe(block.compressedSize)
+    if (blockHeader.compressionMethod !== 'raw') {
+      const compressedData = Buffer.allocUnsafe(blockHeader.compressedSize)
       await this.read(
         compressedData,
         0,
-        block.compressedSize,
+        blockHeader.compressedSize,
         blockContentPosition,
       )
 
       this._uncompress(
-        block.compressionMethod,
+        blockHeader.compressionMethod,
         compressedData,
         uncompressedData,
       )
@@ -320,27 +393,28 @@ export default class CramFile {
       await this.read(
         uncompressedData,
         0,
-        block.uncompressedSize,
+        blockHeader.uncompressedSize,
         blockContentPosition,
       )
     }
 
-    block.content = uncompressedData
-
     if (majorVersion >= 3) {
       // parse the crc32
       const crc = await this._parseSection(
         sectionParsers.cramBlockCrc32,
-        blockContentPosition + block.compressedSize,
+        blockContentPosition + blockHeader.compressedSize,
       )
+      if (crc === undefined) {
+        return undefined
+      }
       block.crc32 = crc.crc32
 
       // check the block data crc32
       if (this.validateChecksums) {
         await this.checkCrc32(
           position,
-          block._size + block.compressedSize,
-          block.crc32,
+          blockHeader._size + blockHeader.compressedSize,
+          crc.crc32,
           'block data',
         )
       }

package/src/cramFile/filehandle.ts

@@ -0,0 +1,3 @@
+import { GenericFilehandle } from 'generic-filehandle'
+
+export { GenericFilehandle as Filehandle }

package/src/cramFile/{record.js → record.ts}

@@ -1,13 +1,34 @@
 import Constants from './constants'
+import CramContainerCompressionScheme from './container/compressionScheme'
+import decodeRecord from './slice/decodeRecord'
 
-function decodeReadSequence(cramRecord, refRegion) {
+export type RefRegion = {
+  start: number
+  end: number
+  seq: string
+}
+
+export type ReadFeature = {
+  code: string
+  pos: number
+  refPos: number
+  data: any
+
+  ref?: string
+  sub?: string
+}
+
+function decodeReadSequence(
+  cramRecord: CramRecord,
+  refRegion: RefRegion,
+): string | null {
   // if it has no length, it has no sequence
   if (!cramRecord.lengthOnRef && !cramRecord.readLength) {
-    return undefined
+    return null
   }
 
   if (cramRecord.isUnknownBases()) {
-    return undefined
+    return null
   }
 
   // remember: all coordinates are 1-based closed
@@ -106,10 +127,10 @@ const baseNumbers = {
 }
 
 function decodeBaseSubstitution(
-  cramRecord,
-  refRegion,
-  compressionScheme,
-  readFeature,
+  cramRecord: CramRecord,
+  refRegion: RefRegion,
+  compressionScheme: CramContainerCompressionScheme,
+  readFeature: ReadFeature,
 ) {
   if (!refRegion) {
     return
@@ -121,7 +142,7 @@ function decodeBaseSubstitution(
   if (refBase) {
     readFeature.ref = refBase
   }
-  let baseNumber = baseNumbers[refBase]
+  let baseNumber = (baseNumbers as any)[refBase]
   if (baseNumber === undefined) {
     baseNumber = 4
   }
@@ -132,12 +153,153 @@ function decodeBaseSubstitution(
   }
 }
 
+export type MateRecord = {
+  readName?: string
+  sequenceId: number
+  alignmentStart: number
+  flags?: number
+
+  uniqueId?: number
+}
+
+export const BamFlags = [
+  [0x1, 'Paired'],
+  [0x2, 'ProperlyPaired'],
+  [0x4, 'SegmentUnmapped'],
+  [0x8, 'MateUnmapped'],
+  [0x10, 'ReverseComplemented'],
+  //  the mate is mapped to the reverse strand
+  [0x20, 'MateReverseComplemented'],
+  //  this is read1
+  [0x40, 'Read1'],
+  //  this is read2
+  [0x80, 'Read2'],
+  //  not primary alignment
+  [0x100, 'Secondary'],
+  //  QC failure
+  [0x200, 'FailedQc'],
+  //  optical or PCR duplicate
+  [0x400, 'Duplicate'],
+  //  supplementary alignment
+  [0x800, 'Supplementary'],
+] as const
+
+export const CramFlags = [
+  [0x1, 'PreservingQualityScores'],
+  [0x2, 'Detached'],
+  [0x4, 'WithMateDownstream'],
+  [0x8, 'DecodeSequenceAsStar'],
+] as const
+
+export const MateFlags = [
+  [0x1, 'OnNegativeStrand'],
+  [0x2, 'Unmapped'],
+] as const
+
+type FlagsDecoder<Type> = {
+  [Property in Type as `is${Capitalize<string & Property>}`]: (
+    flags: number,
+  ) => boolean
+}
+
+type FlagsEncoder<Type> = {
+  [Property in Type as `set${Capitalize<string & Property>}`]: (
+    flags: number,
+  ) => number
+}
+
+function makeFlagsHelper<T>(
+  x: ReadonlyArray<readonly [number, T]>,
+): FlagsDecoder<T> & FlagsEncoder<T> {
+  const r: any = {}
+  for (const [code, name] of x) {
+    r['is' + name] = (flags: number) => !!(flags & code)
+    r['set' + name] = (flags: number) => flags | code
+  }
+
+  return r
+}
+
+export const BamFlagsDecoder = makeFlagsHelper(BamFlags)
+export const CramFlagsDecoder = makeFlagsHelper(CramFlags)
+export const MateFlagsDecoder = makeFlagsHelper(MateFlags)
+
 /**
  * Class of each CRAM record returned by this API.
  */
 export default class CramRecord {
-  constructor() {
-    this.tags = {}
+  public tags: Record<string, string>
+  public flags: number
+  public cramFlags: number
+  public readBases?: string | null
+  public _refRegion?: RefRegion
+  public readFeatures?: ReadFeature[]
+  public alignmentStart: number
+  public lengthOnRef: number | undefined
+  public readLength: number
+  public templateLength?: number
+  public templateSize?: number
+  public readName?: string
+  public mateRecordNumber?: number
+  public mate?: MateRecord
+  public uniqueId: number
+  public sequenceId: number
+  public readGroupId: number
+  public mappingQuality: number | undefined
+  public qualityScores: number[] | null | undefined
+
+  constructor({
+    flags,
+    cramFlags,
+    readLength,
+    mappingQuality,
+    lengthOnRef,
+    qualityScores,
+    mateRecordNumber,
+    readBases,
+    readFeatures,
+    mateToUse,
+    readGroupId,
+    readName,
+    sequenceId,
+    uniqueId,
+    templateSize,
+    alignmentStart,
+    tags,
+  }: ReturnType<typeof decodeRecord> & { uniqueId: number }) {
+    this.flags = flags
+    this.cramFlags = cramFlags
+    this.readLength = readLength
+    this.mappingQuality = mappingQuality
+    this.lengthOnRef = lengthOnRef
+    this.qualityScores = qualityScores
+    if (readBases) {
+      this.readBases = readBases
+    }
+
+    this.readGroupId = readGroupId
+    this.readName = readName
+    this.sequenceId = sequenceId
+    this.uniqueId = uniqueId
+    this.templateSize = templateSize
+    this.alignmentStart = alignmentStart
+    this.tags = tags
+
+    // backwards compatibility
+    if (readFeatures) {
+      this.readFeatures = readFeatures
+    }
+    if (mateToUse) {
+      this.mate = {
+        flags: mateToUse.mateFlags,
+        readName: mateToUse.mateReadName,
+        sequenceId: mateToUse.mateSequenceId,
+        alignmentStart: mateToUse.mateAlignmentStart,
+      }
+    }
+    if (mateRecordNumber) {
+      this.mateRecordNumber = mateRecordNumber
+    }
   }
 
   /**
@@ -230,7 +392,10 @@ export default class CramRecord {
    */
   getReadBases() {
     if (!this.readBases && this._refRegion) {
-      this.readBases = decodeReadSequence(this, this._refRegion)
+      const decoded = decodeReadSequence(this, this._refRegion)
+      if (decoded) {
+        this.readBases = decoded
+      }
     }
     return this.readBases
   }
@@ -261,6 +426,9 @@ export default class CramRecord {
 
       const tmp = []
       let isize = this.templateLength || this.templateSize
+      if (isize === undefined) {
+        throw new Error('One of templateSize and templateLength must be set')
+      }
       if (this.alignmentStart > this.mate.alignmentStart && isize > 0) {
         isize = -isize
       }
@@ -293,7 +461,10 @@ export default class CramRecord {
    * @param {CramContainerCompressionScheme} compressionScheme
    * @returns {undefined} nothing
    */
-  addReferenceSequence(refRegion, compressionScheme) {
+  addReferenceSequence(
+    refRegion: RefRegion,
+    compressionScheme: CramContainerCompressionScheme,
+  ) {
     if (this.readFeatures) {
       // use the reference bases to decode the bases
       // substituted in each base substitution
@@ -322,12 +493,12 @@ export default class CramRecord {
   }
 
   toJSON() {
-    const data = {}
+    const data: any = {}
     Object.keys(this).forEach(k => {
       if (k.charAt(0) === '_') {
         return
       }
-      data[k] = this[k]
+      data[k] = (this as any)[k]
     })
 
     data.readBases = this.getReadBases()