@gmod/cram 1.6.1 → 1.6.4

package/src/cramFile/constants.js

@@ -0,0 +1,55 @@
+const Constants = {
+  CRAM_FLAG_PRESERVE_QUAL_SCORES: 1 << 0,
+  CRAM_FLAG_DETACHED: 1 << 1,
+  CRAM_FLAG_MATE_DOWNSTREAM: 1 << 2,
+  CRAM_FLAG_NO_SEQ: 1 << 3,
+  CRAM_FLAG_MASK: (1 << 4) - 1,
+
+  // mate read is reversed
+  CRAM_M_REVERSE: 1,
+  // mated read is unmapped
+  CRAM_M_UNMAP: 2,
+
+  //  the read is paired in sequencing, no matter whether it is mapped in a pair
+  BAM_FPAIRED: 1,
+  //  the read is mapped in a proper pair
+  BAM_FPROPER_PAIR: 2,
+  //  the read itself is unmapped; conflictive with BAM_FPROPER_PAIR
+  BAM_FUNMAP: 4,
+  //  the mate is unmapped
+  BAM_FMUNMAP: 8,
+  //  the read is mapped to the reverse strand
+  BAM_FREVERSE: 16,
+  //  the mate is mapped to the reverse strand
+  BAM_FMREVERSE: 32,
+  //  this is read1
+  BAM_FREAD1: 64,
+  //  this is read2
+  BAM_FREAD2: 128,
+  //  not primary alignment
+  BAM_FSECONDARY: 256,
+  //  QC failure
+  BAM_FQCFAIL: 512,
+  //  optical or PCR duplicate
+  BAM_FDUP: 1024,
+  //  supplementary alignment
+  BAM_FSUPPLEMENTARY: 2048,
+
+  BAM_CMATCH: 0,
+  BAM_CINS: 1,
+  BAM_CDEL: 2,
+  BAM_CREF_SKIP: 3,
+  BAM_CSOFT_CLIP: 4,
+  BAM_CHARD_CLIP: 5,
+  BAM_CPAD: 6,
+  BAM_CEQUAL: 7,
+  BAM_CDIFF: 8,
+  BAM_CBACK: 9,
+
+  BAM_CIGAR_STR: 'MIDNSHP:XB',
+  BAM_CIGAR_SHIFT: 4,
+  BAM_CIGAR_MASK: 0xf,
+  BAM_CIGAR_TYPE: 0x3c1a7,
+}
+
+export default Constants

package/src/cramFile/container/compressionScheme.js

@@ -0,0 +1,143 @@
+import { CramMalformedError } from '../../errors'
+import { instantiateCodec } from '../codecs'
+
+// the hardcoded data type to be decoded for each core
+// data field
+const dataSeriesTypes = {
+  BF: 'int',
+  CF: 'int',
+  RI: 'int',
+  RL: 'int',
+  AP: 'int',
+  RG: 'int',
+  MF: 'int',
+  NS: 'int',
+  NP: 'int',
+  TS: 'int',
+  NF: 'int',
+  TC: 'byte',
+  TN: 'int',
+  FN: 'int',
+  FC: 'byte',
+  FP: 'int',
+  BS: 'byte',
+  IN: 'byteArray',
+  SC: 'byteArray',
+  DL: 'int',
+  BA: 'byte',
+  BB: 'byteArray',
+  RS: 'int',
+  PD: 'int',
+  HC: 'int',
+  MQ: 'int',
+  RN: 'byteArray',
+  QS: 'byte',
+  QQ: 'byteArray',
+  TL: 'int',
+  TM: 'ignore',
+  TV: 'ignore',
+}
+
+function parseSubstitutionMatrix(byteArray) {
+  const matrix = new Array(5)
+  for (let i = 0; i < 5; i += 1) {
+    matrix[i] = new Array(4)
+  }
+
+  matrix[0][(byteArray[0] >> 6) & 3] = 'C'
+  matrix[0][(byteArray[0] >> 4) & 3] = 'G'
+  matrix[0][(byteArray[0] >> 2) & 3] = 'T'
+  matrix[0][(byteArray[0] >> 0) & 3] = 'N'
+
+  matrix[1][(byteArray[1] >> 6) & 3] = 'A'
+  matrix[1][(byteArray[1] >> 4) & 3] = 'G'
+  matrix[1][(byteArray[1] >> 2) & 3] = 'T'
+  matrix[1][(byteArray[1] >> 0) & 3] = 'N'
+
+  matrix[2][(byteArray[2] >> 6) & 3] = 'A'
+  matrix[2][(byteArray[2] >> 4) & 3] = 'C'
+  matrix[2][(byteArray[2] >> 2) & 3] = 'T'
+  matrix[2][(byteArray[2] >> 0) & 3] = 'N'
+
+  matrix[3][(byteArray[3] >> 6) & 3] = 'A'
+  matrix[3][(byteArray[3] >> 4) & 3] = 'C'
+  matrix[3][(byteArray[3] >> 2) & 3] = 'G'
+  matrix[3][(byteArray[3] >> 0) & 3] = 'N'
+
+  matrix[4][(byteArray[4] >> 6) & 3] = 'A'
+  matrix[4][(byteArray[4] >> 4) & 3] = 'C'
+  matrix[4][(byteArray[4] >> 2) & 3] = 'G'
+  matrix[4][(byteArray[4] >> 0) & 3] = 'T'
+
+  return matrix
+}
+
+export default class CramContainerCompressionScheme {
+  constructor(content) {
+    Object.assign(this, content)
+    // interpret some of the preservation map tags for convenient use
+    this.readNamesIncluded = content.preservation.RN
+    this.APdelta = content.preservation.AP
+    this.referenceRequired = !!content.preservation.RR
+    this.tagIdsDictionary = content.preservation.TD
+    this.substitutionMatrix = parseSubstitutionMatrix(content.preservation.SM)
+
+    this.dataSeriesCodecCache = new Map()
+    this.tagCodecCache = {}
+  }
+
+  /**
+   * @param {string} tagName three-character tag name
+   * @private
+   */
+  getCodecForTag(tagName) {
+    if (!this.tagCodecCache[tagName]) {
+      const encodingData = this.tagEncoding[tagName]
+      if (encodingData) {
+        this.tagCodecCache[tagName] = instantiateCodec(
+          encodingData,
+          'byteArray', // all tags are byte array data
+        )
+      }
+    }
+    return this.tagCodecCache[tagName]
+  }
+
+  /**
+   *
+   * @param {number} tagListId ID of the tag list to fetch from the tag dictionary
+   * @private
+   */
+  getTagNames(tagListId) {
+    return this.tagIdsDictionary[tagListId]
+  }
+
+  getCodecForDataSeries(dataSeriesName) {
+    let r = this.dataSeriesCodecCache.get(dataSeriesName)
+    if (r === undefined) {
+      const encodingData = this.dataSeriesEncoding[dataSeriesName]
+      if (encodingData) {
+        const dataType = dataSeriesTypes[dataSeriesName]
+        if (!dataType) {
+          throw new CramMalformedError(
+            `data series name ${dataSeriesName} not defined in file compression header`,
+          )
+        }
+        r = instantiateCodec(encodingData, dataType)
+        this.dataSeriesCodecCache.set(dataSeriesName, r)
+      }
+    }
+    return r
+  }
+
+  toJSON() {
+    const data = {}
+    Object.keys(this).forEach(k => {
+      if (/Cache$/.test(k)) {
+        return
+      }
+      data[k] = this[k]
+    })
+    return data
+  }
+}

package/src/cramFile/container/index.js

@@ -0,0 +1,119 @@
+import { CramMalformedError } from '../../errors'
+
+import { itf8Size, parseItem, tinyMemoize } from '../util'
+import CramSlice from '../slice'
+import CramContainerCompressionScheme from './compressionScheme'
+
+export default class CramContainer {
+  constructor(cramFile, position) {
+    // cram file this container comes from
+    this.file = cramFile
+    // position of this container in the file
+    this.filePosition = position
+    // console.log(`container: ${this.filePosition}`)
+  }
+
+  // memoize
+  getHeader() {
+    return this._readContainerHeader(this.filePosition)
+  }
+
+  // memoize
+  async getCompressionHeaderBlock() {
+    const containerHeader = await this.getHeader()
+
+    // if there are no records in the container, there will be no compression header
+    if (!containerHeader.numRecords) {
+      return null
+    }
+    const sectionParsers = await this.file.getSectionParsers()
+    const block = await this.getFirstBlock()
+    if (block.contentType !== 'COMPRESSION_HEADER') {
+      throw new CramMalformedError(
+        `invalid content type ${block.contentType} in what is supposed to be the compression header block`,
+      )
+    }
+    const content = parseItem(
+      block.content,
+      sectionParsers.cramCompressionHeader.parser,
+      0,
+      block.contentPosition,
+    )
+    block.content = content
+    return block
+  }
+
+  async getFirstBlock() {
+    const containerHeader = await this.getHeader()
+    return this.file.readBlock(containerHeader._endPosition)
+  }
+
+  // parses the compression header data into a CramContainerCompressionScheme object
+  // memoize
+  async getCompressionScheme() {
+    const header = await this.getCompressionHeaderBlock()
+    if (!header) {
+      return undefined
+    }
+    return new CramContainerCompressionScheme(header.content)
+  }
+
+  getSlice(slicePosition, sliceSize) {
+    // note: slicePosition is relative to the end of the container header
+    // TODO: perhaps we should cache slices?
+    return new CramSlice(this, slicePosition, sliceSize)
+  }
+
+  async _readContainerHeader(position) {
+    const sectionParsers = await this.file.getSectionParsers()
+    const { cramContainerHeader1, cramContainerHeader2 } = sectionParsers
+    const { size: fileSize } = await this.file.stat()
+
+    if (position >= fileSize) {
+      return undefined
+    }
+
+    // parse the container header. do it in 2 pieces because you cannot tell
+    // how much to buffer until you read numLandmarks
+    const bytes1 = Buffer.allocUnsafe(cramContainerHeader1.maxLength)
+    await this.file.read(bytes1, 0, cramContainerHeader1.maxLength, position)
+    const header1 = parseItem(bytes1, cramContainerHeader1.parser)
+    const numLandmarksSize = itf8Size(header1.numLandmarks)
+    if (position + header1.length >= fileSize) {
+      console.warn(
+        `${this.file}: container header at ${position} indicates that the container has length ${header1.length}, which extends beyond the length of the file. Skipping this container.`,
+      )
+      return undefined
+    }
+    const bytes2 = Buffer.allocUnsafe(
+      cramContainerHeader2.maxLength(header1.numLandmarks),
+    )
+    await this.file.read(
+      bytes2,
+      0,
+      cramContainerHeader2.maxLength(header1.numLandmarks),
+      position + header1._size - numLandmarksSize,
+    )
+    const header2 = parseItem(bytes2, cramContainerHeader2.parser)
+
+    if (this.file.validateChecksums && header2.crc32 !== undefined) {
+      await this.file.checkCrc32(
+        position,
+        header1._size + header2._size - numLandmarksSize - 4,
+        header2.crc32,
+        `container header beginning at position ${position}`,
+      )
+    }
+
+    const completeHeader = Object.assign(header1, header2, {
+      _size: header1._size + header2._size - numLandmarksSize,
+      _endPosition: header1._size + header2._size - numLandmarksSize + position,
+    })
+
+    return completeHeader
+  }
+}
+
+'getHeader getCompressionHeaderBlock getCompressionScheme'
+  .split(' ')
+  .forEach(method => tinyMemoize(CramContainer, method))

package/src/cramFile/file.js

@@ -0,0 +1,363 @@
+import { unzip } from '../unzip'
+import crc32 from 'buffer-crc32'
+import LRU from 'quick-lru'
+
+import { CramUnimplementedError, CramMalformedError } from '../errors'
+import ransuncompress from '../rans'
+import {
+  cramFileDefinition as cramFileDefinitionParser,
+  getSectionParsers,
+} from './sectionParsers'
+import htscodecs from '@jkbonfield/htscodecs'
+import CramContainer from './container'
+
+import { open } from '../io'
+import { parseItem, tinyMemoize } from './util'
+import { parseHeaderText } from '../sam'
+//source:https://abdulapopoola.com/2019/01/20/check-endianness-with-javascript/
+function getEndianness() {
+  let uInt32 = new Uint32Array([0x11223344])
+  let uInt8 = new Uint8Array(uInt32.buffer)
+
+  if (uInt8[0] === 0x44) {
+    return 0 //little-endian
+  } else if (uInt8[0] === 0x11) {
+    return 1 //big-endian
+  } else {
+    return 2 //mixed-endian?
+  }
+}
+
+export default class CramFile {
+  /**
+   * @param {object} args
+   * @param {object} [args.filehandle] - a filehandle that implements the stat() and
+   * read() methods of the Node filehandle API https://nodejs.org/api/fs.html#fs_class_filehandle
+   * @param {object} [args.path] - path to the cram file
+   * @param {object} [args.url] - url for the cram file.  also supports file:// urls for local files
+   * @param {function} [args.seqFetch] - a function with signature
+   * `(seqId, startCoordinate, endCoordinate)` that returns a promise for a string of sequence bases
+   * @param {number} [args.cacheSize] optional maximum number of CRAM records to cache.  default 20,000
+   * @param {boolean} [args.checkSequenceMD5] - default true. if false, disables verifying the MD5
+   * checksum of the reference sequence underlying a slice. In some applications, this check can cause an inconvenient amount (many megabases) of sequences to be fetched.
+   */
+  constructor(args) {
+    this.file = open(args.url, args.path, args.filehandle)
+    this.validateChecksums = true
+    this.fetchReferenceSequenceCallback = args.seqFetch
+    this.options = {
+      checkSequenceMD5: args.checkSequenceMD5 !== false,
+      cacheSize: args.cacheSize !== undefined ? args.cacheSize : 20000,
+    }
+
+    // cache of features in a slice, keyed by the
+    // slice offset. caches all of the features in a slice, or none.
+    // the cache is actually used by the slice object, it's just
+    // kept here at the level of the file
+    this.featureCache = new LRU({
+      maxSize: this.options.cacheSize,
+    })
+    if (getEndianness() > 0) {
+      throw new Error('Detected big-endian machine, may be unable to run')
+    }
+  }
+
+  toString() {
+    if (this.file.filename) {
+      return this.file.filename
+    }
+    if (this.file.url) {
+      return this.file.url
+    }
+
+    return '(cram file)'
+  }
+
+  // can just read this object like a filehandle
+  read(buffer, offset, length, position) {
+    return this.file.read(buffer, offset, length, position)
+  }
+
+  // can just stat this object like a filehandle
+  stat() {
+    return this.file.stat()
+  }
+
+  // memoized
+  async getDefinition() {
+    const headbytes = Buffer.allocUnsafe(cramFileDefinitionParser.maxLength)
+    await this.file.read(headbytes, 0, cramFileDefinitionParser.maxLength, 0)
+    const definition = cramFileDefinitionParser.parser.parse(headbytes).result
+    if (definition.majorVersion !== 2 && definition.majorVersion !== 3) {
+      throw new CramUnimplementedError(
+        `CRAM version ${definition.majorVersion} not supported`,
+      )
+    }
+    return definition
+  }
+
+  // memoize
+  async getSamHeader() {
+    const firstContainer = await this.getContainerById(0)
+    if (!firstContainer) {
+      throw new CramMalformedError('file contains no containers')
+    }
+
+    const { content } = await firstContainer.getFirstBlock()
+    // find the end of the trailing zeros in the header text
+    const headerLength = content.readInt32LE(0)
+    const textStart = 4
+    // let textEnd = content.length - 1
+    // while (textEnd >= textStart && !content[textEnd]) textEnd -= 1
+    // trim off the trailing zeros
+    const text = content.toString('utf8', textStart, textStart + headerLength)
+    this.header = text
+    return parseHeaderText(text)
+  }
+
+  async getHeaderText() {
+    await this.getSamHeader()
+    return this.header
+  }
+
+  // memoize
+  async getSectionParsers() {
+    const { majorVersion } = await this.getDefinition()
+    return getSectionParsers(majorVersion)
+  }
+
+  async getContainerById(containerNumber) {
+    const sectionParsers = await this.getSectionParsers()
+    let position = sectionParsers.cramFileDefinition.maxLength
+    const { size: fileSize } = await this.file.stat()
+    const { cramContainerHeader1 } = sectionParsers
+
+    // skip with a series of reads to the proper container
+    let currentContainer
+    for (let i = 0; i <= containerNumber; i += 1) {
+      // if we are about to go off the end of the file
+      // and have not found that container, it does not exist
+      if (position + cramContainerHeader1.maxLength + 8 >= fileSize) {
+        return undefined
+      }
+
+      currentContainer = this.getContainerAtPosition(position)
+      const currentHeader = await currentContainer.getHeader()
+      if (!currentHeader) {
+        throw new CramMalformedError(
+          `container ${containerNumber} not found in file`,
+        )
+      }
+      // if this is the first container, read all the blocks in the
+      // container to determine its length, because we cannot trust
+      // the container header's given length due to a bug somewhere
+      // in htslib
+      if (i === 0) {
+        position = currentHeader._endPosition
+        for (let j = 0; j < currentHeader.numBlocks; j += 1) {
+          const block = await this.readBlock(position)
+          position = block._endPosition
+        }
+      } else {
+        // otherwise, just traverse to the next container using the container's length
+        position += currentHeader._size + currentHeader.length
+      }
+    }
+
+    return currentContainer
+  }
+
+  async checkCrc32(position, length, recordedCrc32, description) {
+    const b = Buffer.allocUnsafe(length)
+    await this.file.read(b, 0, length, position)
+    const calculatedCrc32 = crc32.unsigned(b)
+    if (calculatedCrc32 !== recordedCrc32) {
+      throw new CramMalformedError(
+        `crc mismatch in ${description}: recorded CRC32 = ${recordedCrc32}, but calculated CRC32 = ${calculatedCrc32}`,
+      )
+    }
+  }
+
+  /**
+   * @returns {Promise[number]} the number of containers in the file
+   */
+  async containerCount() {
+    const sectionParsers = await this.getSectionParsers()
+    const { size: fileSize } = await this.file.stat()
+    const { cramContainerHeader1 } = sectionParsers
+
+    let containerCount = 0
+    let position = sectionParsers.cramFileDefinition.maxLength
+    while (position + cramContainerHeader1.maxLength + 8 < fileSize) {
+      const currentHeader = await this.getContainerAtPosition(
+        position,
+      ).getHeader()
+      if (!currentHeader) {
+        break
+      }
+      // if this is the first container, read all the blocks in the
+      // container, because we cannot trust the container
+      // header's given length due to a bug somewhere in htslib
+      if (containerCount === 0) {
+        position = currentHeader._endPosition
+        for (let j = 0; j < currentHeader.numBlocks; j += 1) {
+          const block = await this.readBlock(position)
+          position = block._endPosition
+        }
+      } else {
+        // otherwise, just traverse to the next container using the container's length
+        position += currentHeader._size + currentHeader.length
+      }
+      containerCount += 1
+    }
+
+    return containerCount
+  }
+
+  getContainerAtPosition(position) {
+    return new CramContainer(this, position)
+  }
+
+  async readBlockHeader(position) {
+    const sectionParsers = await this.getSectionParsers()
+    const { cramBlockHeader } = sectionParsers
+    const { size: fileSize } = await this.file.stat()
+
+    if (position + cramBlockHeader.maxLength >= fileSize) {
+      return undefined
+    }
+
+    const buffer = Buffer.allocUnsafe(cramBlockHeader.maxLength)
+    await this.file.read(buffer, 0, cramBlockHeader.maxLength, position)
+    return parseItem(buffer, cramBlockHeader.parser, 0, position)
+  }
+
+  async _parseSection(
+    section,
+    position,
+    size = section.maxLength,
+    preReadBuffer,
+  ) {
+    let buffer
+    if (preReadBuffer) {
+      buffer = preReadBuffer
+    } else {
+      const { size: fileSize } = await this.file.stat()
+      if (position + size >= fileSize) {
+        return undefined
+      }
+      buffer = Buffer.allocUnsafe(size)
+      await this.file.read(buffer, 0, size, position)
+    }
+    const data = parseItem(buffer, section.parser, 0, position)
+    if (data._size !== size) {
+      throw new CramMalformedError(
+        `section read error: requested size ${size} does not equal parsed size ${data._size}`,
+      )
+    }
+    return data
+  }
+
+  _uncompress(compressionMethod, inputBuffer, outputBuffer) {
+    if (compressionMethod === 'gzip') {
+      const result = unzip(inputBuffer)
+      result.copy(outputBuffer)
+    } else if (compressionMethod === 'bzip2') {
+      var bits = bzip2.array(inputBuffer)
+      var size = bzip2.header(bits)
+      var j = 0
+      do {
+        var chunk = bzip2.decompress(bits, size)
+        if (chunk != -1) {
+          Buffer.from(chunk).copy(outputBuffer, j)
+          j += chunk.length
+          size -= chunk.length
+        }
+      } while (chunk != -1)
+    } else if (compressionMethod === 'rans') {
+      ransuncompress(inputBuffer, outputBuffer)
+      //htscodecs r4x8 is slower, but compatible.
+      //htscodecs.r4x8_uncompress(inputBuffer, outputBuffer);
+    } else if (compressionMethod === 'rans4x16') {
+      htscodecs.r4x16_uncompress(inputBuffer, outputBuffer)
+    } else if (compressionMethod === 'arith') {
+      htscodecs.arith_uncompress(inputBuffer, outputBuffer)
+    } else if (compressionMethod === 'fqzcomp') {
+      htscodecs.fqzcomp_uncompress(inputBuffer, outputBuffer)
+    } else if (compressionMethod === 'tok3') {
+      htscodecs.tok3_uncompress(inputBuffer, outputBuffer)
+    } else {
+      throw new CramUnimplementedError(
+        `${compressionMethod} decompression not yet implemented`,
+      )
+    }
+  }
+
+  async readBlock(position) {
+    const { majorVersion } = await this.getDefinition()
+    const sectionParsers = await this.getSectionParsers()
+    const block = await this.readBlockHeader(position)
+    const blockContentPosition = block._endPosition
+    block.contentPosition = block._endPosition
+
+    const uncompressedData = Buffer.allocUnsafe(block.uncompressedSize)
+
+    if (block.compressionMethod !== 'raw') {
+      const compressedData = Buffer.allocUnsafe(block.compressedSize)
+      await this.read(
+        compressedData,
+        0,
+        block.compressedSize,
+        blockContentPosition,
+      )
+
+      this._uncompress(
+        block.compressionMethod,
+        compressedData,
+        uncompressedData,
+      )
+    } else {
+      await this.read(
+        uncompressedData,
+        0,
+        block.uncompressedSize,
+        blockContentPosition,
+      )
+    }
+
+    block.content = uncompressedData
+
+    if (majorVersion >= 3) {
+      // parse the crc32
+      const crc = await this._parseSection(
+        sectionParsers.cramBlockCrc32,
+        blockContentPosition + block.compressedSize,
+      )
+      block.crc32 = crc.crc32
+
+      // check the block data crc32
+      if (this.validateChecksums) {
+        await this.checkCrc32(
+          position,
+          block._size + block.compressedSize,
+          block.crc32,
+          'block data',
+        )
+      }
+
+      // make the endposition and size reflect the whole block
+      block._endPosition = crc._endPosition
+      block._size =
+        block.compressedSize + sectionParsers.cramBlockCrc32.maxLength
+    } else {
+      block._endPosition = blockContentPosition + block.compressedSize
+      block._size = block.compressedSize
+    }
+
+    return block
+  }
+}
+
+'getDefinition getSectionParsers getSamHeader'
+  .split(' ')
+  .forEach(method => tinyMemoize(CramFile, method))

package/src/cramFile/index.js

@@ -0,0 +1,3 @@
+import CramFile from './file'
+
+export default CramFile