@gmod/cram 1.6.1 → 1.6.2

package/CHANGELOG.md

@@ -1,3 +1,7 @@
+# v1.6.2
+
+- Publish src directory for better source maps
+
 # v1.6.1
 
 - Explicitly use pako for browser bundle to help avoid buggy zlib polyfills

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gmod/cram",
-  "version": "1.6.1",
+  "version": "1.6.2",
   "description": "read CRAM files with pure Javascript",
   "license": "MIT",
   "repository": "GMOD/cram-js",
@@ -16,6 +16,7 @@
   },
   "files": [
     "errors.js",
+    "src",
     "dist",
     "esm"
   ],

package/src/craiIndex.js

@@ -0,0 +1,180 @@
+import AbortablePromiseCache from 'abortable-promise-cache'
+import QuickLRU from 'quick-lru'
+import { unzip } from './unzip'
+import { open } from './io'
+import { CramMalformedError } from './errors'
+
+const BAI_MAGIC = 21578050 // BAI\1
+
+class Slice {
+  constructor(args) {
+    Object.assign(this, args)
+  }
+
+  toString() {
+    return `${this.start}:${this.span}:${this.containerStart}:${this.sliceStart}:${this.sliceBytes}`
+  }
+}
+
+function addRecordToIndex(index, record) {
+  if (record.some(el => el === undefined)) {
+    throw new CramMalformedError('invalid .crai index file')
+  }
+
+  const [seqId, start, span, containerStart, sliceStart, sliceBytes] = record
+
+  if (!index[seqId]) {
+    index[seqId] = []
+  }
+
+  index[seqId].push(
+    new Slice({
+      start,
+      span,
+      containerStart,
+      sliceStart,
+      sliceBytes,
+    }),
+  )
+}
+
+export default class CraiIndex {
+  // A CRAM index (.crai) is a gzipped tab delimited file containing the following columns:
+  // 1. Sequence id
+  // 2. Alignment start
+  // 3. Alignment span
+  // 4. Container start byte position in the file
+  // 5. Slice start byte position in the container data (‘blocks’)
+  // 6. Slice size in bytes
+  // Each line represents a slice in the CRAM file. Please note that all slices must be listed in index file.
+
+  /**
+   *
+   * @param {object} args
+   * @param {string} [args.path]
+   * @param {string} [args.url]
+   * @param {FileHandle} [args.filehandle]
+   */
+  constructor(args) {
+    const filehandle = open(args.url, args.path, args.filehandle)
+    this._parseCache = new AbortablePromiseCache({
+      cache: new QuickLRU({ maxSize: 1 }),
+      fill: (data, signal) => this.parseIndex({ signal }),
+    })
+    this.readFile = filehandle.readFile.bind(filehandle)
+  }
+
+  parseIndex() {
+    const index = {}
+    return this.readFile()
+      .then(data => {
+        if (data[0] === 31 && data[1] === 139) {
+          return unzip(data)
+        }
+        return data
+      })
+      .then(uncompressedBuffer => {
+        if (
+          uncompressedBuffer.length > 4 &&
+          uncompressedBuffer.readUInt32LE(0) === BAI_MAGIC
+        ) {
+          throw new CramMalformedError(
+            'invalid .crai index file. note: file appears to be a .bai index. this is technically legal but please open a github issue if you need support',
+          )
+        }
+        // interpret the text as regular ascii, since it is
+        // supposed to be only digits and whitespace characters
+        // this is written in a deliberately low-level fashion for performance,
+        // because some .crai files can be pretty large.
+        let currentRecord = []
+        let currentString = ''
+        for (let i = 0; i < uncompressedBuffer.length; i += 1) {
+          const charCode = uncompressedBuffer[i]
+          if (
+            (charCode >= 48 && charCode <= 57) /* 0-9 */ ||
+            (!currentString && charCode === 45) /* leading - */
+          ) {
+            currentString += String.fromCharCode(charCode)
+          } else if (charCode === 9 /* \t */) {
+            currentRecord.push(Number.parseInt(currentString, 10))
+            currentString = ''
+          } else if (charCode === 10 /* \n */) {
+            currentRecord.push(Number.parseInt(currentString, 10))
+            currentString = ''
+            addRecordToIndex(index, currentRecord)
+            currentRecord = []
+          } else if (charCode !== 13 /* \r */ && charCode !== 32 /* space */) {
+            // if there are other characters in the file besides
+            // space and \r, something is wrong.
+            throw new CramMalformedError('invalid .crai index file')
+          }
+        }
+
+        // if the file ends without a \n, we need to flush our buffers
+        if (currentString) {
+          currentRecord.push(Number.parseInt(currentString, 10))
+        }
+        if (currentRecord.length === 6) {
+          addRecordToIndex(index, currentRecord)
+        }
+
+        // sort each of them by start
+        Object.entries(index).forEach(([seqId, ent]) => {
+          index[seqId] = ent.sort(
+            (a, b) => a.start - b.start || a.span - b.span,
+          )
+        })
+        return index
+      })
+  }
+
+  getIndex(opts = {}) {
+    return this._parseCache.get('index', null, opts.signal)
+  }
+
+  /**
+   * @param {number} seqId
+   * @returns {Promise} true if the index contains entries for
+   * the given reference sequence ID, false otherwise
+   */
+  async hasDataForReferenceSequence(seqId) {
+    return !!(await this.getIndex())[seqId]
+  }
+
+  /**
+   * fetch index entries for the given range
+   *
+   * @param {number} seqId
+   * @param {number} queryStart
+   * @param {number} queryEnd
+   *
+   * @returns {Promise} promise for
+   * an array of objects of the form
+   * `{start, span, containerStart, sliceStart, sliceBytes }`
+   */
+  async getEntriesForRange(seqId, queryStart, queryEnd) {
+    const seqEntries = (await this.getIndex())[seqId]
+    if (!seqEntries) {
+      return []
+    }
+
+    const compare = entry => {
+      const entryStart = entry.start
+      const entryEnd = entry.start + entry.span
+      if (entryStart >= queryEnd) {
+        return -1
+      } // entry is ahead of query
+      if (entryEnd <= queryStart) {
+        return 1
+      } // entry is behind query
+      return 0 // entry overlaps query
+    }
+    const bins = []
+    for (let i = 0; i < seqEntries.length; i += 1) {
+      if (compare(seqEntries[i]) === 0) {
+        bins.push(seqEntries[i])
+      }
+    }
+    return bins
+  }
+}

package/src/cramFile/codecs/_base.js

@@ -0,0 +1,49 @@
+import { CramBufferOverrunError } from '../../errors'
+
+const validDataTypes = {
+  int: true,
+  byte: true,
+  long: true,
+  byteArray: true,
+  byteArrayBlock: true,
+}
+
+// codec base class
+export default class CramCodec {
+  constructor(parameters = {}, dataType) {
+    this.parameters = parameters
+    this.dataType = dataType
+    if (!dataType) {
+      throw new TypeError('must provide a data type to codec constructor')
+    }
+    if (!validDataTypes[dataType]) {
+      throw new TypeError(`invalid data type ${dataType}`)
+    }
+  }
+
+  // decode(slice, coreDataBlock, blocksByContentId, cursors) {
+  // }
+
+  _getBits(data, cursor, numBits) {
+    let val = 0
+    if (
+      cursor.bytePosition + (7 - cursor.bitPosition + numBits) / 8 >
+      data.length
+    ) {
+      throw new CramBufferOverrunError(
+        'read error during decoding. the file seems to be truncated.',
+      )
+    }
+    for (let dlen = numBits; dlen; dlen -= 1) {
+      // get the next `dlen` bits in the input, put them in val
+      val <<= 1
+      val |= (data[cursor.bytePosition] >> cursor.bitPosition) & 1
+      cursor.bitPosition -= 1
+      if (cursor.bitPosition < 0) {
+        cursor.bytePosition += 1
+      }
+      cursor.bitPosition &= 7
+    }
+    return val
+  }
+}

package/src/cramFile/codecs/beta.js

@@ -0,0 +1,23 @@
+import { CramUnimplementedError } from '../../errors'
+import CramCodec from './_base'
+
+export default class BetaCodec extends CramCodec {
+  constructor(parameters = {}, dataType) {
+    super(parameters, dataType)
+    if (this.dataType !== 'int') {
+      throw new CramUnimplementedError(
+        `${this.dataType} decoding not yet implemented by BETA codec`,
+      )
+    }
+  }
+
+  decode(slice, coreDataBlock, blocksByContentId, cursors) {
+    const data =
+      this._getBits(
+        coreDataBlock.content,
+        cursors.coreBlock,
+        this.parameters.length,
+      ) - this.parameters.offset
+    return data
+  }
+}

package/src/cramFile/codecs/byteArrayLength.js

@@ -0,0 +1,55 @@
+import { tinyMemoize } from '../util'
+
+import CramCodec from './_base'
+
+export default class ByteArrayStopCodec extends CramCodec {
+  constructor(parameters = {}, dataType, instantiateCodec) {
+    super(parameters, dataType)
+    this.instantiateCodec = instantiateCodec
+    if (dataType !== 'byteArray') {
+      throw new TypeError(
+        `byteArrayLength does not support data type ${dataType}`,
+      )
+    }
+  }
+
+  decode(slice, coreDataBlock, blocksByContentId, cursors) {
+    const lengthCodec = this._getLengthCodec()
+    const arrayLength = lengthCodec.decode(
+      slice,
+      coreDataBlock,
+      blocksByContentId,
+      cursors,
+    )
+
+    const dataCodec = this._getDataCodec()
+    const data = new Array(arrayLength)
+    for (let i = 0; i < arrayLength; i += 1) {
+      data[i] = dataCodec.decode(
+        slice,
+        coreDataBlock,
+        blocksByContentId,
+        cursors,
+      )
+    }
+
+    return data
+  }
+
+  // memoize
+  _getLengthCodec() {
+    const encodingParams = this.parameters.lengthsEncoding
+    return this.instantiateCodec(encodingParams, 'int')
+  }
+
+  // memoize
+  _getDataCodec() {
+    const encodingParams = this.parameters.valuesEncoding
+
+    return this.instantiateCodec(encodingParams, 'byte')
+  }
+}
+
+'_getLengthCodec _getDataCodec'
+  .split(' ')
+  .forEach(method => tinyMemoize(ByteArrayStopCodec, method))

package/src/cramFile/codecs/byteArrayStop.js

@@ -0,0 +1,50 @@
+import { CramBufferOverrunError, CramMalformedError } from '../../errors'
+
+import CramCodec from './_base'
+
+export default class ByteArrayStopCodec extends CramCodec {
+  constructor(parameters = {}, dataType) {
+    super(parameters, dataType)
+    if (dataType === 'byteArray') {
+      this._decode = this._decodeByteArray
+    } else {
+      throw new TypeError(
+        `byteArrayStop codec does not support data type ${dataType}`,
+      )
+    }
+  }
+
+  decode(slice, coreDataBlock, blocksByContentId, cursors) {
+    const { blockContentId } = this.parameters
+    const contentBlock = blocksByContentId[blockContentId]
+    if (!contentBlock) {
+      throw new CramMalformedError(
+        `no block found with content ID ${blockContentId}`,
+      )
+    }
+    const cursor = cursors.externalBlocks.getCursor(blockContentId)
+    return this._decode(contentBlock, cursor)
+  }
+
+  _decodeByteArray(contentBlock, cursor) {
+    const dataBuffer = contentBlock.content
+    const { stopByte } = this.parameters
+    // scan to the next stop byte
+    const startPosition = cursor.bytePosition
+    let stopPosition = cursor.bytePosition
+    while (
+      dataBuffer[stopPosition] !== stopByte &&
+      stopPosition < dataBuffer.length
+    ) {
+      if (stopPosition === dataBuffer.length) {
+        throw new CramBufferOverrunError(
+          `byteArrayStop reading beyond length of data buffer?`,
+        )
+      }
+      stopPosition += 1
+    }
+    cursor.bytePosition = stopPosition + 1
+    const data = dataBuffer.slice(startPosition, stopPosition)
+    return data
+  }
+}

package/src/cramFile/codecs/external.js

@@ -0,0 +1,54 @@
+import {
+  CramUnimplementedError,
+  CramMalformedError,
+  CramBufferOverrunError,
+} from '../../errors'
+import CramCodec from './_base'
+import { parseItf8 } from '../util'
+
+export default class ExternalCodec extends CramCodec {
+  constructor(parameters = {}, dataType) {
+    super(parameters, dataType)
+    if (this.dataType === 'int') {
+      this._decodeData = this._decodeInt
+    } else if (this.dataType === 'byte') {
+      this._decodeData = this._decodeByte
+    } else {
+      throw new CramUnimplementedError(
+        `${this.dataType} decoding not yet implemented by EXTERNAL codec`,
+      )
+    }
+  }
+
+  decode(slice, coreDataBlock, blocksByContentId, cursors) {
+    const { blockContentId } = this.parameters
+    const contentBlock = blocksByContentId[blockContentId]
+    if (!contentBlock) {
+      throw new CramMalformedError(
+        `no block found with content ID ${blockContentId}`,
+      )
+    }
+    const cursor = cursors.externalBlocks.getCursor(blockContentId)
+    return this._decodeData(contentBlock, cursor)
+  }
+
+  _decodeInt(contentBlock, cursor) {
+    const [result, bytesRead] = parseItf8(
+      contentBlock.content,
+      cursor.bytePosition,
+    )
+    cursor.bytePosition += bytesRead
+    return result
+  }
+
+  _decodeByte(contentBlock, cursor) {
+    if (cursor.bytePosition >= contentBlock.content.length) {
+      throw new CramBufferOverrunError(
+        'attempted to read beyond end of block. this file seems truncated.',
+      )
+    }
+    const result = contentBlock.content[cursor.bytePosition]
+    cursor.bytePosition += 1
+    return result
+  }
+}

package/src/cramFile/codecs/gamma.js

@@ -0,0 +1,30 @@
+import { CramUnimplementedError } from '../../errors'
+import CramCodec from './_base'
+
+export default class GammaCodec extends CramCodec {
+  constructor(parameters = {}, dataType) {
+    super(parameters, dataType)
+    if (this.dataType !== 'int') {
+      throw new CramUnimplementedError(
+        `${this.dataType} decoding not yet implemented by GAMMA codec`,
+      )
+    }
+  }
+
+  decode(slice, coreDataBlock, blocksByContentId, cursors) {
+    let length = 1
+
+    while (this._getBits(coreDataBlock.content, cursors.coreBlock, 1) === 0) {
+      length += 1
+    }
+
+    const readBits = this._getBits(
+      coreDataBlock.content,
+      cursors.coreBlock,
+      length - 1,
+    )
+
+    const value = readBits | (1 << (length - 1))
+    return value - this.parameters.offset
+  }
+}

package/src/cramFile/codecs/huffman.js

@@ -0,0 +1,137 @@
+import { CramMalformedError } from '../../errors'
+import CramCodec from './_base'
+
+function numberOfSetBits(ii) {
+  let i = (ii - (ii >> 1)) & 0x55555555
+  i = (i & 0x33333333) + ((i >> 2) & 0x33333333)
+  return (((i + (i >> 4)) & 0x0f0f0f0f) * 0x01010101) >> 24
+}
+
+export default class HuffmanIntCodec extends CramCodec {
+  constructor(parameters = {}, dataType) {
+    super(parameters, dataType)
+    if (!['byte', 'int'].includes(this.dataType)) {
+      throw new TypeError(
+        `${this.dataType} decoding not yet implemented by HUFFMAN_INT codec`,
+      )
+    }
+
+    this.buildCodeBook()
+    this.buildCodes()
+    this.buildCaches()
+
+    // if this is a degenerate zero-length huffman code, special-case the decoding
+    if (this.sortedCodes[0].bitLength === 0) {
+      this._decode = this._decodeZeroLengthCode
+    }
+  }
+
+  buildCodeBook() {
+    // parse the parameters together into a `codes` data structure
+    let codes = new Array(this.parameters.numCodes)
+    for (let i = 0; i < this.parameters.numCodes; i += 1) {
+      codes[i] = {
+        symbol: this.parameters.symbols[i],
+        bitLength: this.parameters.bitLengths[i],
+      }
+    }
+    // sort the codes by bit length and symbol value
+    codes = codes.sort(
+      (a, b) => a.bitLength - b.bitLength || a.symbol - b.symbol,
+    )
+
+    this.codeBook = {}
+    codes.forEach(code => {
+      if (!this.codeBook[code.bitLength]) {
+        this.codeBook[code.bitLength] = []
+      }
+      this.codeBook[code.bitLength].push(code.symbol)
+    })
+  }
+
+  buildCodes() {
+    this.codes = {} /*  new TreeMap<Integer, HuffmanBitCode>(); */
+    let codeLength = 0
+    let codeValue = -1
+    Object.entries(this.codeBook).forEach(([bitLength, symbols]) => {
+      bitLength = parseInt(bitLength, 10)
+      symbols.forEach(symbol => {
+        const code = { bitLength, value: symbol }
+        codeValue += 1
+        const delta = bitLength - codeLength // new length?
+        codeValue <<= delta // pad with 0's
+        code.bitCode = codeValue // calculated: huffman code
+        codeLength += delta // adjust current code length
+
+        if (numberOfSetBits(codeValue) > bitLength) {
+          throw new CramMalformedError('Symbol out of range')
+        }
+
+        this.codes[symbol] = code
+      })
+    })
+  }
+
+  buildCaches() {
+    this.sortedCodes = Object.values(this.codes).sort(
+      (a, b) => a.bitLength - b.bitLength || a.bitCode - b.bitCode,
+    )
+
+    // this.sortedValues = this.parameters.values.sort((a,b) => a-b)
+    this.sortedByValue = Object.values(this.codes).sort(
+      (a, b) => a.value - b.value,
+    )
+
+    this.sortedValuesByBitCode = this.sortedCodes.map(c => c.value)
+    this.sortedBitCodes = this.sortedCodes.map(c => c.bitCode)
+    this.sortedBitLengthsByBitCode = this.sortedCodes.map(c => c.bitLength)
+    const maxBitCode = Math.max(...this.sortedBitCodes)
+
+    this.bitCodeToValue = new Array(maxBitCode + 1).fill(-1)
+    for (let i = 0; i < this.sortedBitCodes.length; i += 1) {
+      this.bitCodeToValue[this.sortedCodes[i].bitCode] = i
+    }
+  }
+
+  decode(slice, coreDataBlock, blocksByContentId, cursors) {
+    return this._decode(slice, coreDataBlock, cursors.coreBlock)
+  }
+
+  // _decodeNull() {
+  //   return -1
+  // }
+
+  // the special case for zero-length codes
+  _decodeZeroLengthCode() {
+    return this.sortedCodes[0].value
+  }
+
+  _decode(slice, coreDataBlock, coreCursor) {
+    const input = coreDataBlock.content
+
+    let prevLen = 0
+    let bits = 0
+    for (let i = 0; i < this.sortedCodes.length; i += 1) {
+      const length = this.sortedCodes[i].bitLength
+      bits <<= length - prevLen
+      bits |= this._getBits(input, coreCursor, length - prevLen)
+      prevLen = length
+      {
+        const index = this.bitCodeToValue[bits]
+        if (index > -1 && this.sortedBitLengthsByBitCode[index] === length) {
+          return this.sortedValuesByBitCode[index]
+        }
+
+        for (
+          let j = i;
+          this.sortedCodes[j + 1].bitLength === length &&
+          j < this.sortedCodes.length;
+          j += 1
+        ) {
+          i += 1
+        }
+      }
+    }
+    throw new CramMalformedError('Huffman symbol not found.')
+  }
+}

package/src/cramFile/codecs/index.js

@@ -0,0 +1,38 @@
+import { CramUnimplementedError } from '../../errors'
+
+import HuffmanIntCodec from './huffman'
+import ExternalCodec from './external'
+import ByteArrayStopCodec from './byteArrayStop'
+import ByteArrayLengthCodec from './byteArrayLength'
+import BetaCodec from './beta'
+import GammaCodec from './gamma'
+import SubexpCodec from './subexp'
+
+const codecClasses = {
+  1: ExternalCodec,
+  // 2: GolombCodec,
+  3: HuffmanIntCodec,
+  4: ByteArrayLengthCodec,
+  5: ByteArrayStopCodec,
+  6: BetaCodec,
+  7: SubexpCodec,
+  // 8: GolombRiceCodec,
+  9: GammaCodec,
+}
+
+export function getCodecClassWithId(id) {
+  return codecClasses[id]
+}
+
+export function instantiateCodec(encodingData, dataType) {
+  const CodecClass = getCodecClassWithId(
+    dataType === 'ignore' ? 0 : encodingData.codecId,
+  )
+  if (!CodecClass) {
+    throw new CramUnimplementedError(
+      `no codec implemented for codec ID ${encodingData.codecId}`,
+    )
+  }
+
+  return new CodecClass(encodingData.parameters, dataType, instantiateCodec)
+}

package/src/cramFile/codecs/subexp.js

@@ -0,0 +1,32 @@
+import { CramUnimplementedError } from '../../errors'
+import CramCodec from './_base'
+
+export default class SubexpCodec extends CramCodec {
+  constructor(parameters = {}, dataType) {
+    super(parameters, dataType)
+    if (this.dataType !== 'int') {
+      throw new CramUnimplementedError(
+        `${this.dataType} decoding not yet implemented by SUBEXP codec`,
+      )
+    }
+  }
+
+  decode(slice, coreDataBlock, blocksByContentId, cursors) {
+    let numLeadingOnes = 0
+    while (this._getBits(coreDataBlock.content, cursors.coreBlock, 1)) {
+      numLeadingOnes += 1
+    }
+
+    let b
+    let n
+    if (numLeadingOnes === 0) {
+      b = this.parameters.K
+      n = this._getBits(coreDataBlock.content, cursors.coreBlock, b)
+    } else {
+      b = numLeadingOnes + this.parameters.K - 1
+      n = (1 << b) | this._getBits(coreDataBlock.content, cursors.coreBlock, b)
+    }
+
+    return n - this.parameters.offset
+  }
+}