@gmod/cram 1.5.9 → 1.6.2

package/src/cramFile/record.js

@@ -0,0 +1,337 @@
+import Constants from './constants'
+
+function decodeReadSequence(cramRecord, refRegion) {
+  // if it has no length, it has no sequence
+  if (!cramRecord.lengthOnRef && !cramRecord.readLength) {
+    return undefined
+  }
+
+  if (cramRecord.isUnknownBases()) {
+    return undefined
+  }
+
+  // remember: all coordinates are 1-based closed
+  const regionSeqOffset = cramRecord.alignmentStart - refRegion.start
+
+  if (!cramRecord.readFeatures) {
+    return refRegion.seq
+      .substr(regionSeqOffset, cramRecord.lengthOnRef)
+      .toUpperCase()
+  }
+
+  let bases = ''
+  let regionPos = regionSeqOffset
+  let currentReadFeature = 0
+  while (bases.length < cramRecord.readLength) {
+    if (currentReadFeature < cramRecord.readFeatures.length) {
+      const feature = cramRecord.readFeatures[currentReadFeature]
+      if (feature.code === 'Q' || feature.code === 'q') {
+        currentReadFeature += 1
+      } else if (feature.pos === bases.length + 1) {
+        // process the read feature
+        currentReadFeature += 1
+
+        if (feature.code === 'b') {
+          // specify a base pair for some reason
+          const ret = feature.data.split(',')
+          const added = String.fromCharCode(...ret)
+          bases += added
+          regionPos += added.length
+        } else if (feature.code === 'B') {
+          // base pair and associated quality
+          // TODO: do we need to set the quality in the qual scores?
+          bases += feature.data[0]
+          regionPos += 1
+        } else if (feature.code === 'X') {
+          // base substitution
+          bases += feature.sub
+          regionPos += 1
+        } else if (feature.code === 'I') {
+          // insertion
+          bases += feature.data
+        } else if (feature.code === 'D') {
+          // deletion
+          regionPos += feature.data
+        } else if (feature.code === 'i') {
+          // insert single base
+          bases += feature.data
+        } else if (feature.code === 'N') {
+          // reference skip. delete some bases
+          // do nothing
+          // seqBases.splice(feature.pos - 1, feature.data)
+          regionPos += feature.data
+        } else if (feature.code === 'S') {
+          // soft clipped bases that should be present in the read seq
+          // seqBases.splice(feature.pos - 1, 0, ...feature.data.split(''))
+          bases += feature.data
+        } else if (feature.code === 'P') {
+          // padding, do nothing
+        } else if (feature.code === 'H') {
+          // hard clip, do nothing
+        }
+      } else if (currentReadFeature < cramRecord.readFeatures.length) {
+        // put down a chunk of sequence up to the next read feature
+        const chunk = refRegion.seq.substr(
+          regionPos,
+          cramRecord.readFeatures[currentReadFeature].pos - bases.length - 1,
+        )
+        bases += chunk
+        regionPos += chunk.length
+      }
+    } else {
+      // put down a chunk of reference up to the full read length
+      const chunk = refRegion.seq.substr(
+        regionPos,
+        cramRecord.readLength - bases.length,
+      )
+      bases += chunk
+      regionPos += chunk.length
+    }
+  }
+
+  return bases.toUpperCase()
+}
+
+const baseNumbers = {
+  a: 0,
+  A: 0,
+  c: 1,
+  C: 1,
+  g: 2,
+  G: 2,
+  t: 3,
+  T: 3,
+  n: 4,
+  N: 4,
+}
+
+function decodeBaseSubstitution(
+  cramRecord,
+  refRegion,
+  compressionScheme,
+  readFeature,
+) {
+  if (!refRegion) {
+    return
+  }
+
+  // decode base substitution code using the substitution matrix
+  const refCoord = readFeature.refPos - refRegion.start
+  const refBase = refRegion.seq.charAt(refCoord)
+  if (refBase) {
+    readFeature.ref = refBase
+  }
+  let baseNumber = baseNumbers[refBase]
+  if (baseNumber === undefined) {
+    baseNumber = 4
+  }
+  const substitutionScheme = compressionScheme.substitutionMatrix[baseNumber]
+  const base = substitutionScheme[readFeature.data]
+  if (base) {
+    readFeature.sub = base
+  }
+}
+
+/**
+ * Class of each CRAM record returned by this API.
+ */
+export default class CramRecord {
+  constructor() {
+    this.tags = {}
+  }
+
+  /**
+   * @returns {boolean} true if the read is paired, regardless of whether both segments are mapped
+   */
+  isPaired() {
+    return !!(this.flags & Constants.BAM_FPAIRED)
+  }
+
+  /** @returns {boolean} true if the read is paired, and both segments are mapped */
+  isProperlyPaired() {
+    return !!(this.flags & Constants.BAM_FPROPER_PAIR)
+  }
+
+  /** @returns {boolean} true if the read itself is unmapped; conflictive with isProperlyPaired */
+  isSegmentUnmapped() {
+    return !!(this.flags & Constants.BAM_FUNMAP)
+  }
+
+  /** @returns {boolean} true if the read itself is unmapped; conflictive with isProperlyPaired */
+  isMateUnmapped() {
+    return !!(this.flags & Constants.BAM_FMUNMAP)
+  }
+
+  /** @returns {boolean} true if the read is mapped to the reverse strand */
+  isReverseComplemented() {
+    return !!(this.flags & Constants.BAM_FREVERSE)
+  }
+
+  /** @returns {boolean} true if the mate is mapped to the reverse strand */
+  isMateReverseComplemented() {
+    return !!(this.flags & Constants.BAM_FMREVERSE)
+  }
+
+  /** @returns {boolean} true if this is read number 1 in a pair */
+  isRead1() {
+    return !!(this.flags & Constants.BAM_FREAD1)
+  }
+
+  /** @returns {boolean} true if this is read number 2 in a pair */
+  isRead2() {
+    return !!(this.flags & Constants.BAM_FREAD2)
+  }
+
+  /** @returns {boolean} true if this is a secondary alignment */
+  isSecondary() {
+    return !!(this.flags & Constants.BAM_FSECONDARY)
+  }
+
+  /** @returns {boolean} true if this read has failed QC checks */
+  isFailedQc() {
+    return !!(this.flags & Constants.BAM_FQCFAIL)
+  }
+
+  /** @returns {boolean} true if the read is an optical or PCR duplicate */
+  isDuplicate() {
+    return !!(this.flags & Constants.BAM_FDUP)
+  }
+
+  /** @returns {boolean} true if this is a supplementary alignment */
+  isSupplementary() {
+    return !!(this.flags & Constants.BAM_FSUPPLEMENTARY)
+  }
+
+  /**
+   * @returns {boolean} true if the read is detached
+   */
+  isDetached() {
+    return !!(this.cramFlags & Constants.CRAM_FLAG_DETACHED)
+  }
+
+  /** @returns {boolean} true if the read has a mate in this same CRAM segment */
+  hasMateDownStream() {
+    return !!(this.cramFlags & Constants.CRAM_FLAG_MATE_DOWNSTREAM)
+  }
+
+  /** @returns {boolean} true if the read contains qual scores */
+  isPreservingQualityScores() {
+    return !!(this.cramFlags & Constants.CRAM_FLAG_PRESERVE_QUAL_SCORES)
+  }
+
+  /** @returns {boolean} true if the read has no sequence bases */
+  isUnknownBases() {
+    return !!(this.cramFlags & Constants.CRAM_FLAG_NO_SEQ)
+  }
+
+  /**
+   * Get the original sequence of this read.
+   * @returns {String} sequence basepairs
+   */
+  getReadBases() {
+    if (!this.readBases && this._refRegion) {
+      this.readBases = decodeReadSequence(this, this._refRegion)
+    }
+    return this.readBases
+  }
+
+  /**
+   * Get the pair orientation of a paired read. Adapted from igv.js
+   * @returns {String} of paired orientatin
+   */
+  getPairOrientation() {
+    if (
+      !this.isSegmentUnmapped() &&
+      this.isPaired() &&
+      !this.isMateUnmapped() &&
+      this.mate &&
+      this.sequenceId === this.mate.sequenceId
+    ) {
+      const s1 = this.isReverseComplemented() ? 'R' : 'F'
+      const s2 = this.isMateReverseComplemented() ? 'R' : 'F'
+      let o1 = ' '
+      let o2 = ' '
+      if (this.isRead1()) {
+        o1 = '1'
+        o2 = '2'
+      } else if (this.isRead2()) {
+        o1 = '2'
+        o2 = '1'
+      }
+
+      const tmp = []
+      let isize = this.templateLength || this.templateSize
+      if (this.alignmentStart > this.mate.alignmentStart && isize > 0) {
+        isize = -isize
+      }
+      if (isize > 0) {
+        tmp[0] = s1
+        tmp[1] = o1
+        tmp[2] = s2
+        tmp[3] = o2
+      } else {
+        tmp[2] = s1
+        tmp[3] = o1
+        tmp[0] = s2
+        tmp[1] = o2
+      }
+      return tmp.join('')
+    }
+    return null
+  }
+
+  /**
+   * Annotates this feature with the given reference sequence basepair
+   * information. This will add a `sub` and a `ref` item to base
+   * subsitution read features given the actual substituted and reference
+   * base pairs, and will make the `getReadSequence()` method work.
+   *
+   * @param {object} refRegion
+   * @param {number} refRegion.start
+   * @param {number} refRegion.end
+   * @param {string} refRegion.seq
+   * @param {CramContainerCompressionScheme} compressionScheme
+   * @returns {undefined} nothing
+   */
+  addReferenceSequence(refRegion, compressionScheme) {
+    if (this.readFeatures) {
+      // use the reference bases to decode the bases
+      // substituted in each base substitution
+      this.readFeatures.forEach(readFeature => {
+        if (readFeature.code === 'X') {
+          decodeBaseSubstitution(
+            this,
+            refRegion,
+            compressionScheme,
+            readFeature,
+          )
+        }
+      })
+    }
+
+    // if this region completely covers this read,
+    // keep a reference to it
+    if (
+      !this.readBases &&
+      refRegion.start <= this.alignmentStart &&
+      refRegion.end >=
+        this.alignmentStart + (this.lengthOnRef || this.readLength) - 1
+    ) {
+      this._refRegion = refRegion
+    }
+  }
+
+  toJSON() {
+    const data = {}
+    Object.keys(this).forEach(k => {
+      if (k.charAt(0) === '_') {
+        return
+      }
+      data[k] = this[k]
+    })
+
+    data.readBases = this.getReadBases()
+
+    return data
+  }
+}

package/src/cramFile/sectionParsers.js

@@ -0,0 +1,379 @@
+import { Parser } from '@gmod/binary-parser'
+
+const singleItf8 = new Parser().itf8()
+
+const cramFileDefinition = {
+  parser: new Parser()
+    .string('magic', { length: 4 })
+    .uint8('majorVersion')
+    .uint8('minorVersion')
+    .string('fileId', { length: 20, stripNull: true }),
+  maxLength: 26,
+}
+
+const cramBlockHeader = {
+  parser: new Parser()
+    .uint8('compressionMethod', {
+      formatter: /* istanbul ignore next */ b => {
+        const method = [
+          'raw',
+          'gzip',
+          'bzip2',
+          'lzma',
+          'rans',
+          'rans4x16',
+          'arith',
+          'fqzcomp',
+          'tok3',
+        ][b]
+        if (!method) {
+          throw new Error(`compression method number ${b} not implemented`)
+        }
+        return method
+      },
+    })
+    .uint8('contentType', {
+      formatter: /* istanbul ignore next */ b => {
+        const type = [
+          'FILE_HEADER',
+          'COMPRESSION_HEADER',
+          'MAPPED_SLICE_HEADER',
+          'UNMAPPED_SLICE_HEADER', // < only used in cram v1
+          'EXTERNAL_DATA',
+          'CORE_DATA',
+        ][b]
+        if (!type) {
+          throw new Error(`invalid block content type id ${b}`)
+        }
+        return type
+      },
+    })
+    .itf8('contentId')
+    .itf8('compressedSize')
+    .itf8('uncompressedSize'),
+  maxLength: 17,
+}
+
+const cramBlockCrc32 = {
+  parser: new Parser().uint32('crc32'),
+  maxLength: 4,
+}
+
+// const ENCODING_NAMES = [
+//   'NULL', // 0
+//   'EXTERNAL', // 1
+//   'GOLOMB', // 2
+//   'HUFFMAN_INT', // 3
+//   'BYTE_ARRAY_LEN', // 4
+//   'BYTE_ARRAY_STOP', // 5
+//   'BETA', // 6
+//   'SUBEXP', // 7
+//   'GOLOMB_RICE', // 8
+//   'GAMMA', // 9
+// ]
+
+const cramTagDictionary = new Parser().itf8('size').buffer('ents', {
+  length: 'size',
+  formatter: /* istanbul ignore next */ buffer => {
+    function makeTagSet(stringStart, stringEnd) {
+      const str = buffer.toString('utf8', stringStart, stringEnd)
+      const tags = []
+      for (let i = 0; i < str.length; i += 3) {
+        tags.push(str.substr(i, 3))
+      }
+      return tags
+    }
+    /* eslint-disable */
+    var tagSets = []
+    var stringStart = 0
+    var i
+    /* eslint-enable */
+    for (i = 0; i < buffer.length; i += 1) {
+      if (!buffer[i]) {
+        tagSets.push(makeTagSet(stringStart, i))
+        stringStart = i + 1
+      }
+    }
+    if (i > stringStart) {
+      tagSets.push(makeTagSet(stringStart, i))
+    }
+    return tagSets
+  },
+})
+
+// const cramPreservationMapKeys = 'XX RN AP RR SM TD'.split(' ')
+const parseByteAsBool = new Parser().uint8(null, {
+  formatter: /* istanbul ignore next */ val => !!val,
+})
+
+const cramPreservationMap = new Parser()
+  .itf8('mapSize')
+  .itf8('mapCount')
+  .array('ents', {
+    length: 'mapCount',
+    type: new Parser()
+      .string('key', {
+        length: 2,
+        stripNull: false,
+        // formatter: val => cramPreservationMapKeys[val] || 0,
+      })
+      .choice('value', {
+        tag: 'key',
+        choices: {
+          MI: parseByteAsBool,
+          UI: parseByteAsBool,
+          PI: parseByteAsBool,
+          RN: parseByteAsBool,
+          AP: parseByteAsBool,
+          RR: parseByteAsBool,
+          SM: new Parser().array(null, { type: 'uint8', length: 5 }),
+          TD: new Parser().nest(null, {
+            type: cramTagDictionary,
+            formatter: /* istanbul ignore next */ data => data.ents,
+          }),
+        },
+      }),
+  })
+
+/* istanbul ignore next */
+function formatMap(data) {
+  const map = {}
+  for (let i = 0; i < data.ents.length; i += 1) {
+    const { key, value } = data.ents[i]
+    if (map[key]) {
+      console.warn(`duplicate key ${key} in map`)
+    }
+    map[key] = value
+  }
+  return map
+}
+
+const unversionedParsers = {
+  cramFileDefinition,
+  cramBlockHeader,
+  cramBlockCrc32,
+}
+
+// each of these is a function of the major and minor version
+const versionedParsers = {
+  // assemble a section parser for the unmapped slice header, with slight
+  // variations depending on the major version of the cram file
+  cramUnmappedSliceHeader(majorVersion) {
+    let maxLength = 0
+    let parser = new Parser().itf8('numRecords')
+    maxLength += 5
+
+    // recordCounter is itf8 in a CRAM v2 file, absent in CRAM v1
+    if (majorVersion >= 3) {
+      parser = parser.ltf8('recordCounter')
+      maxLength += 9
+    } else if (majorVersion === 2) {
+      parser = parser.itf8('recordCounter')
+      maxLength += 5
+    }
+
+    parser = parser
+      .itf8('numBlocks')
+      .itf8('numContentIds')
+      .array('contentIds', {
+        type: singleItf8,
+        length: 'numContentIds',
+      })
+    maxLength += 5 * 2 // + numContentIds*5
+
+    // the md5 sum is missing in cram v1
+    if (majorVersion >= 2) {
+      parser = parser.array('md5', { type: 'uint8', length: 16 })
+      maxLength += 16
+    }
+
+    const maxLengthFunc = numContentIds => maxLength + numContentIds * 5
+
+    return { parser, maxLength: maxLengthFunc } // : p, maxLength: numContentIds => 5 + 9 + 5 * 2 + 5 * numContentIds + 16 }
+  },
+
+  // assembles a section parser for the unmapped slice header, with slight
+  // variations depending on the major version of the cram file
+  cramMappedSliceHeader(majorVersion) {
+    let parser = new Parser()
+      .itf8('refSeqId')
+      .itf8('refSeqStart')
+      .itf8('refSeqSpan')
+      .itf8('numRecords')
+    let maxLength = 5 * 4
+
+    if (majorVersion >= 3) {
+      parser = parser.ltf8('recordCounter')
+      maxLength += 9
+    } else if (majorVersion === 2) {
+      parser = parser.itf8('recordCounter')
+      maxLength += 5
+    }
+
+    parser = parser
+      .itf8('numBlocks')
+      .itf8('numContentIds')
+      .array('contentIds', {
+        type: singleItf8,
+        length: 'numContentIds',
+      })
+      .itf8('refBaseBlockId')
+    maxLength += 5 * 3
+
+    // the md5 sum is missing in cram v1
+    if (majorVersion >= 2) {
+      parser = parser.array('md5', { type: 'uint8', length: 16 })
+      maxLength += 16
+    }
+
+    const maxLengthFunc = numContentIds => maxLength + numContentIds * 5
+
+    return { parser, maxLength: maxLengthFunc }
+  },
+
+  cramEncoding(majorVersion) {
+    const parser = new Parser()
+      .namely('cramEncoding')
+      .itf8('codecId')
+      .itf8('parametersBytes')
+      .choice('parameters', {
+        tag: 'codecId',
+        choices: {
+          0: new Parser(), // NULL
+          1: new Parser().itf8('blockContentId'), // EXTERNAL
+          2: new Parser().itf8('offset').itf8('M'), // GOLOMB,
+          // HUFFMAN_INT
+          3: Parser.start()
+            .itf8('numCodes')
+            .array('symbols', { length: 'numCodes', type: singleItf8 })
+            .itf8('numLengths')
+            .array('bitLengths', { length: 'numLengths', type: singleItf8 }),
+          4: Parser.start() // BYTE_ARRAY_LEN
+            .nest('lengthsEncoding', { type: 'cramEncoding' })
+            .nest('valuesEncoding', { type: 'cramEncoding' }),
+          // BYTE_ARRAY_STOP is a little different for CRAM v1
+          5: new Parser()
+            .uint8('stopByte')
+            [majorVersion > 1 ? 'itf8' : 'int']('blockContentId'),
+          6: new Parser().itf8('offset').itf8('length'), // BETA
+          7: new Parser().itf8('offset').itf8('K'), // SUBEXP
+          8: new Parser().itf8('offset').itf8('log2m'), // GOLOMB_RICE
+          9: new Parser().itf8('offset'), // GAMMA
+        },
+      })
+
+    return { parser }
+  },
+
+  cramDataSeriesEncodingMap(majorVersion) {
+    return new Parser()
+      .itf8('mapSize')
+      .itf8('mapCount')
+      .array('ents', {
+        length: 'mapCount',
+        type: new Parser()
+          .string('key', { length: 2, stripNull: false })
+          .nest('value', { type: this.cramEncoding(majorVersion).parser }),
+      })
+  },
+
+  cramTagEncodingMap(majorVersion) {
+    return new Parser()
+      .itf8('mapSize')
+      .itf8('mapCount')
+      .array('ents', {
+        length: 'mapCount',
+        type: new Parser()
+          .itf8('key', {
+            formatter: /* istanbul ignore next */ integerRepresentation =>
+              /* istanbul ignore next */
+              String.fromCharCode((integerRepresentation >> 16) & 0xff) +
+              String.fromCharCode((integerRepresentation >> 8) & 0xff) +
+              String.fromCharCode(integerRepresentation & 0xff),
+          })
+          .nest('value', { type: this.cramEncoding(majorVersion).parser }),
+      })
+  },
+
+  cramCompressionHeader(majorVersion) {
+    let parser = new Parser()
+    // TODO: if we want to support CRAM v1, we will need to refactor
+    // compression header into 2 parts to parse the landmarks,
+    // like the container header
+    parser = parser
+      .nest('preservation', {
+        type: cramPreservationMap,
+        formatter: formatMap,
+      })
+      .nest('dataSeriesEncoding', {
+        type: this.cramDataSeriesEncodingMap(majorVersion),
+        formatter: formatMap,
+      })
+      .nest('tagEncoding', {
+        type: this.cramTagEncodingMap(majorVersion),
+        formatter: formatMap,
+      })
+    return { parser }
+  },
+
+  cramContainerHeader1(majorVersion) {
+    let parser = new Parser()
+      .int32('length') // byte size of the container data (blocks)
+      .itf8('refSeqId') // reference sequence identifier, -1 for unmapped reads, -2 for multiple reference sequences
+      .itf8('refSeqStart') // the alignment start position or 0 for unmapped reads
+      .itf8('alignmentSpan') // the length of the alignment or 0 for unmapped reads
+      .itf8('numRecords') // number of records in the container
+    let maxLength = 4 + 5 * 4
+
+    if (majorVersion >= 3) {
+      parser = parser.ltf8('recordCounter') // 1-based sequential index of records in the file/stream.
+      maxLength += 9
+    } else if (majorVersion === 2) {
+      parser = parser.itf8('recordCounter')
+      maxLength += 5
+    }
+
+    if (majorVersion > 1) {
+      parser = parser.ltf8('numBases') // number of read bases
+      maxLength += 9
+    }
+    parser = parser
+      .itf8('numBlocks') // the number of blocks
+      .itf8('numLandmarks') // the number of landmarks
+    maxLength += 5 + 5
+
+    return { parser, maxLength }
+  },
+
+  cramContainerHeader2(majorVersion) {
+    let parser = new Parser()
+      .itf8('numLandmarks') // the number of blocks
+      // Each integer value of this array is a byte offset
+      // into the blocks byte array. Landmarks are used for
+      // random access indexing.
+      .array('landmarks', {
+        type: new Parser().itf8(),
+        length: 'numLandmarks',
+      })
+
+    let crcLength = 0
+    if (majorVersion >= 3) {
+      parser = parser.uint32('crc32')
+      crcLength = 4
+    }
+    return {
+      parser,
+      maxLength: numLandmarks => 5 + numLandmarks * 5 + crcLength,
+    }
+  },
+}
+
+function getSectionParsers(majorVersion) {
+  const parsers = Object.assign({}, unversionedParsers)
+  Object.keys(versionedParsers).forEach(parserName => {
+    parsers[parserName] = versionedParsers[parserName](majorVersion)
+  })
+  return parsers
+}
+
+export { cramFileDefinition, getSectionParsers }