@gmod/cram 1.5.9 → 1.6.2

package/src/indexedCramFile.js

@@ -0,0 +1,191 @@
+import { CramUnimplementedError, CramSizeLimitError } from './errors'
+
+import CramFile from './cramFile'
+
+export default class IndexedCramFile {
+  /**
+   *
+   * @param {object} args
+   * @param {CramFile} args.cram
+   * @param {Index-like} args.index object that supports getEntriesForRange(seqId,start,end) -> Promise[Array[index entries]]
+   * @param {number} [args.cacheSize] optional maximum number of CRAM records to cache.  default 20,000
+   * @param {number} [args.fetchSizeLimit] optional maximum number of bytes to fetch in a single getRecordsForRange call.  Default 3 MiB.
+   * @param {boolean} [args.checkSequenceMD5] - default true. if false, disables verifying the MD5
+   * checksum of the reference sequence underlying a slice. In some applications, this check can cause an inconvenient amount (many megabases) of sequences to be fetched.
+   */
+  constructor(args) {
+    // { cram, index, seqFetch /* fasta, fastaIndex */ }) {
+    if (args.cram) {
+      this.cram = args.cram
+    } else {
+      this.cram = new CramFile({
+        url: args.cramUrl,
+        path: args.cramPath,
+        filehandle: args.cramFilehandle,
+        seqFetch: args.seqFetch,
+        checkSequenceMD5: args.checkSequenceMD5,
+        cacheSize: args.cacheSize,
+      })
+    }
+
+    if (!(this.cram instanceof CramFile)) {
+      throw new Error('invalid arguments: no cramfile')
+    }
+
+    this.index = args.index
+    if (!this.index.getEntriesForRange) {
+      throw new Error('invalid arguments: not an index')
+    }
+
+    this.fetchSizeLimit = args.fetchSizeLimit || 3000000
+  }
+
+  /**
+   *
+   * @param {number} seq numeric ID of the reference sequence
+   * @param {number} start start of the range of interest. 1-based closed coordinates.
+   * @param {number} end end of the range of interest. 1-based closed coordinates.
+   * @returns {Promise[Array[CramRecord]]}
+   */
+  async getRecordsForRange(seq, start, end, opts = {}) {
+    opts.viewAsPairs = opts.viewAsPairs || false
+    opts.pairAcrossChr = opts.pairAcrossChr || false
+    opts.maxInsertSize = opts.maxInsertSize || 200000
+
+    if (typeof seq === 'string') {
+      // TODO: support string reference sequence names somehow
+      throw new CramUnimplementedError(
+        'string sequence names not yet supported',
+      )
+    }
+    const seqId = seq
+    const slices = await this.index.getEntriesForRange(seqId, start, end)
+    const totalSize = slices.map(s => s.sliceBytes).reduce((a, b) => a + b, 0)
+    if (totalSize > this.fetchSizeLimit) {
+      throw new CramSizeLimitError(
+        `data size of ${totalSize.toLocaleString()} bytes exceeded fetch size limit of ${this.fetchSizeLimit.toLocaleString()} bytes`,
+      )
+    }
+
+    // TODO: do we need to merge or de-duplicate the blocks?
+
+    // fetch all the slices and parse the feature data
+    const filter = feature =>
+      feature.sequenceId === seq &&
+      feature.alignmentStart <= end &&
+      feature.alignmentStart + feature.lengthOnRef - 1 >= start
+    const sliceResults = await Promise.all(
+      slices.map(slice => this.getRecordsInSlice(slice, filter)),
+    )
+
+    let ret = Array.prototype.concat(...sliceResults)
+    if (opts.viewAsPairs) {
+      const readNames = {}
+      const readIds = {}
+      for (let i = 0; i < ret.length; i += 1) {
+        const name = ret[i].readName
+        const id = ret[i].uniqueId
+        if (!readNames[name]) {
+          readNames[name] = 0
+        }
+        readNames[name] += 1
+        readIds[id] = 1
+      }
+      const unmatedPairs = {}
+      Object.entries(readNames).forEach(([k, v]) => {
+        if (v === 1) {
+          unmatedPairs[k] = true
+        }
+      })
+      const matePromises = []
+      for (let i = 0; i < ret.length; i += 1) {
+        const name = ret[i].readName
+        if (
+          unmatedPairs[name] &&
+          ret[i].mate &&
+          (ret[i].mate.sequenceId === seqId || opts.pairAcrossChr) &&
+          Math.abs(ret[i].alignmentStart - ret[i].mate.alignmentStart) <
+            opts.maxInsertSize
+        ) {
+          const mateSlices = this.index.getEntriesForRange(
+            ret[i].mate.sequenceId,
+            ret[i].mate.alignmentStart,
+            ret[i].mate.alignmentStart + 1,
+          )
+          matePromises.push(mateSlices)
+        }
+      }
+      const mateBlocks = await Promise.all(matePromises)
+      let mateChunks = []
+      for (let i = 0; i < mateBlocks.length; i += 1) {
+        mateChunks.push(...mateBlocks[i])
+      }
+      // filter out duplicates
+      mateChunks = mateChunks
+        .sort((a, b) => a.toString().localeCompare(b.toString()))
+        .filter(
+          (item, pos, ary) =>
+            !pos || item.toString() !== ary[pos - 1].toString(),
+        )
+
+      const mateRecordPromises = []
+      const mateFeatPromises = []
+
+      const mateTotalSize = mateChunks
+        .map(s => s.sliceBytes)
+        .reduce((a, b) => a + b, 0)
+      if (mateTotalSize > this.fetchSizeLimit) {
+        throw new Error(
+          `mate data size of ${mateTotalSize.toLocaleString()} bytes exceeded fetch size limit of ${this.fetchSizeLimit.toLocaleString()} bytes`,
+        )
+      }
+
+      mateChunks.forEach(c => {
+        let recordPromise = this.cram.featureCache.get(c.toString())
+        if (!recordPromise) {
+          recordPromise = this.getRecordsInSlice(c, () => true)
+          this.cram.featureCache.set(c.toString(), recordPromise)
+        }
+        mateRecordPromises.push(recordPromise)
+        const featPromise = recordPromise.then(feats => {
+          const mateRecs = []
+          for (let i = 0; i < feats.length; i += 1) {
+            const feature = feats[i]
+            if (unmatedPairs[feature.readName] && !readIds[feature.uniqueId]) {
+              mateRecs.push(feature)
+            }
+          }
+          return mateRecs
+        })
+        mateFeatPromises.push(featPromise)
+      })
+      const newMateFeats = await Promise.all(mateFeatPromises)
+      if (newMateFeats.length) {
+        const newMates = newMateFeats.reduce((result, current) =>
+          result.concat(current),
+        )
+        ret = ret.concat(newMates)
+      }
+    }
+    return ret
+  }
+
+  getRecordsInSlice(
+    { containerStart, sliceStart, sliceBytes },
+    filterFunction,
+  ) {
+    const container = this.cram.getContainerAtPosition(containerStart)
+    const slice = container.getSlice(sliceStart, sliceBytes)
+    return slice.getRecords(filterFunction)
+  }
+
+  /**
+   *
+   * @param {number} seqId
+   * @returns {Promise} true if the CRAM file contains data for the given
+   * reference sequence numerical ID
+   */
+  hasDataForReferenceSequence(seqId) {
+    return this.index.hasDataForReferenceSequence(seqId)
+  }
+}

package/src/io/bufferCache.js

@@ -0,0 +1,66 @@
+import LRU from 'quick-lru'
+
+export default class BufferCache {
+  constructor({ fetch, size = 10000000, chunkSize = 32768 }) {
+    if (!fetch) {
+      throw new Error('fetch function required')
+    }
+    this.fetch = fetch
+    this.chunkSize = chunkSize
+    this.lruCache = new LRU({ maxSize: Math.floor(size / chunkSize) })
+  }
+
+  async get(outputBuffer, offset, length, position) {
+    if (outputBuffer.length < offset + length) {
+      throw new Error('output buffer not big enough for request')
+    }
+
+    // calculate the list of chunks involved in this fetch
+    const firstChunk = Math.floor(position / this.chunkSize)
+    const lastChunk = Math.floor((position + length) / this.chunkSize)
+
+    // fetch them all as necessary
+    const fetches = new Array(lastChunk - firstChunk + 1)
+    for (let chunk = firstChunk; chunk <= lastChunk; chunk += 1) {
+      fetches[chunk - firstChunk] = this._getChunk(chunk).then(data => ({
+        data,
+        chunkNumber: chunk,
+      }))
+    }
+
+    // stitch together the response buffer using them
+    const chunks = await Promise.all(fetches)
+    const chunksOffset = position - chunks[0].chunkNumber * this.chunkSize
+    chunks.forEach(({ data, chunkNumber }) => {
+      const chunkPositionStart = chunkNumber * this.chunkSize
+      let copyStart = 0
+      let copyEnd = this.chunkSize
+      let copyOffset =
+        offset + (chunkNumber - firstChunk) * this.chunkSize - chunksOffset
+
+      if (chunkNumber === firstChunk) {
+        copyOffset = offset
+        copyStart = chunksOffset
+      }
+      if (chunkNumber === lastChunk) {
+        copyEnd = position + length - chunkPositionStart
+      }
+
+      data.copy(outputBuffer, copyOffset, copyStart, copyEnd)
+    })
+  }
+
+  _getChunk(chunkNumber) {
+    const cachedPromise = this.lruCache.get(chunkNumber)
+    if (cachedPromise) {
+      return cachedPromise
+    }
+
+    const freshPromise = this.fetch(
+      chunkNumber * this.chunkSize,
+      this.chunkSize,
+    )
+    this.lruCache.set(chunkNumber, freshPromise)
+    return freshPromise
+  }
+}

package/src/io/index.js

@@ -0,0 +1,26 @@
+import url from 'url'
+import RemoteFile from './remoteFile'
+import LocalFile from './localFile'
+
+function fromUrl(source) {
+  const { protocol, pathname } = url.parse(source)
+  if (protocol === 'file:') {
+    return new LocalFile(unescape(pathname))
+  }
+  return new RemoteFile(source)
+}
+
+function open(maybeUrl, maybePath, maybeFilehandle) {
+  if (maybeFilehandle) {
+    return maybeFilehandle
+  }
+  if (maybeUrl) {
+    return fromUrl(maybeUrl)
+  }
+  if (maybePath) {
+    return new LocalFile(maybePath)
+  }
+  throw new Error('no url, path, or filehandle provided, cannot open')
+}
+
+export { LocalFile, RemoteFile, fromUrl, open }

package/src/io/localFile.js

@@ -0,0 +1,35 @@
+import { promisify } from 'es6-promisify'
+import fs from 'fs'
+
+const fsOpen = fs && promisify(fs.open)
+const fsRead = fs && promisify(fs.read)
+const fsFStat = fs && promisify(fs.fstat)
+const fsReadFile = fs && promisify(fs.readFile)
+
+export default class LocalFile {
+  constructor(source) {
+    this.position = 0
+    this.filename = source
+    this.fd = fsOpen(this.filename, 'r')
+  }
+
+  async read(buffer, offset = 0, length, position) {
+    let readPosition = position
+    if (readPosition === null) {
+      readPosition = this.position
+      this.position += length
+    }
+    return fsRead(await this.fd, buffer, offset, length, position)
+  }
+
+  async readFile() {
+    return fsReadFile(await this.fd)
+  }
+
+  async stat() {
+    if (!this._stat) {
+      this._stat = await fsFStat(await this.fd)
+    }
+    return this._stat
+  }
+}

package/src/io/remoteFile.js

@@ -0,0 +1,71 @@
+import fetch from 'cross-fetch'
+import BufferCache from './bufferCache'
+
+export default class RemoteFile {
+  constructor(source) {
+    this.position = 0
+    this.url = source
+    this.cache = new BufferCache({
+      fetch: (start, length) => this._fetch(start, length),
+    })
+  }
+
+  async _fetch(position, length) {
+    const headers = {}
+    if (length < Infinity) {
+      headers.range = `bytes=${position}-${position + length}`
+    } else if (length === Infinity && position !== 0) {
+      headers.range = `bytes=${position}-`
+    }
+    const response = await fetch(this.url, {
+      method: 'GET',
+      headers,
+      redirect: 'follow',
+      mode: 'cors',
+    })
+    if (
+      (response.status === 200 && position === 0) ||
+      response.status === 206
+    ) {
+      const nodeBuffer = Buffer.from(await response.arrayBuffer())
+
+      // try to parse out the size of the remote file
+      const sizeMatch = /\/(\d+)$/.exec(response.headers.get('content-range'))
+      if (sizeMatch[1]) {
+        this._stat = { size: parseInt(sizeMatch[1], 10) }
+      }
+
+      return nodeBuffer
+    }
+    throw new Error(`HTTP ${response.status} fetching ${this.url}`)
+  }
+
+  read(buffer, offset = 0, length = Infinity, position = 0) {
+    let readPosition = position
+    if (readPosition === null) {
+      readPosition = this.position
+      this.position += length
+    }
+    return this.cache.get(buffer, offset, length, position)
+  }
+
+  async readFile() {
+    const response = await fetch(this.url, {
+      method: 'GET',
+      redirect: 'follow',
+      mode: 'cors',
+    })
+    return Buffer.from(await response.arrayBuffer())
+  }
+
+  async stat() {
+    if (!this._stat) {
+      const buf = Buffer.allocUnsafe(10)
+      await this.read(buf, 0, 10, 0)
+      if (!this._stat) {
+        throw new Error(`unable to determine size of file at ${this.url}`)
+      }
+    }
+    return this._stat
+  }
+}

package/src/rans/README.md

@@ -0,0 +1 @@
+Ported from htsjdk's rANS implementation

package/src/rans/constants.js

@@ -0,0 +1,5 @@
+const TF_SHIFT = 12
+const TOTFREQ = 1 << TF_SHIFT
+const RANS_BYTE_L = 1 << 23
+
+export { TF_SHIFT, TOTFREQ, RANS_BYTE_L }

package/src/rans/d04.js

@@ -0,0 +1,83 @@
+import { CramMalformedError } from '../errors'
+
+import { TF_SHIFT } from './constants'
+import Decoding from './decoding'
+
+export default function uncompress(
+  /* ByteBuffer */ input,
+  /* Decoding.AriDecoder */ D,
+  /* Decoding.Symbol[] */ syms,
+  /* ByteBuffer */ out,
+) {
+  let rans0 = input.getInt()
+  let rans1 = input.getInt()
+  let rans2 = input.getInt()
+  let rans3 = input.getInt()
+
+  const /* int */ outputSize = out.remaining()
+  const /* int */ outputEnd = outputSize & ~3
+  for (let i = 0; i < outputEnd; i += 4) {
+    const /* byte */ c0 = D.R[Decoding.get(rans0, TF_SHIFT)]
+    const /* byte */ c1 = D.R[Decoding.get(rans1, TF_SHIFT)]
+    const /* byte */ c2 = D.R[Decoding.get(rans2, TF_SHIFT)]
+    const /* byte */ c3 = D.R[Decoding.get(rans3, TF_SHIFT)]
+
+    out.putAt(i, c0)
+    out.putAt(i + 1, c1)
+    out.putAt(i + 2, c2)
+    out.putAt(i + 3, c3)
+
+    rans0 = Decoding.advanceSymbolStep(rans0, syms[0xff & c0], TF_SHIFT)
+    rans1 = Decoding.advanceSymbolStep(rans1, syms[0xff & c1], TF_SHIFT)
+    rans2 = Decoding.advanceSymbolStep(rans2, syms[0xff & c2], TF_SHIFT)
+    rans3 = Decoding.advanceSymbolStep(rans3, syms[0xff & c3], TF_SHIFT)
+
+    rans0 = Decoding.renormalize(rans0, input)
+    rans1 = Decoding.renormalize(rans1, input)
+    rans2 = Decoding.renormalize(rans2, input)
+    rans3 = Decoding.renormalize(rans3, input)
+  }
+
+  out.setPosition(outputEnd)
+  let /* byte */ c
+  switch (outputSize & 3) {
+    case 0:
+      break
+    case 1:
+      c = D.R[Decoding.get(rans0, TF_SHIFT)]
+      Decoding.advanceSymbol(rans0, input, syms[0xff & c], TF_SHIFT)
+      out.put(c)
+      break
+
+    case 2:
+      c = D.R[Decoding.get(rans0, TF_SHIFT)]
+      Decoding.advanceSymbol(rans0, input, syms[0xff & c], TF_SHIFT)
+      out.put(c)
+
+      c = D.R[Decoding.get(rans1, TF_SHIFT)]
+      Decoding.advanceSymbol(rans1, input, syms[0xff & c], TF_SHIFT)
+      out.put(c)
+      break
+
+    case 3:
+      c = D.R[Decoding.get(rans0, TF_SHIFT)]
+      Decoding.advanceSymbol(rans0, input, syms[0xff & c], TF_SHIFT)
+      out.put(c)
+
+      c = D.R[Decoding.get(rans1, TF_SHIFT)]
+      Decoding.advanceSymbol(rans1, input, syms[0xff & c], TF_SHIFT)
+      out.put(c)
+
+      c = D.R[Decoding.get(rans2, TF_SHIFT)]
+      Decoding.advanceSymbol(rans2, input, syms[0xff & c], TF_SHIFT)
+      out.put(c)
+      break
+
+    default:
+      throw new CramMalformedError(
+        'invalid output size encountered during rANS decoding',
+      )
+  }
+
+  out.setPosition(0)
+}

package/src/rans/d14.js

@@ -0,0 +1,59 @@
+import { TF_SHIFT } from './constants'
+import Decoding from './decoding'
+
+export default function uncompress(
+  /* ByteBuffer */ input,
+  /* ByteBuffer */ output,
+  /* Decoding.AriDecoder[] */ D,
+  /* Decoding.Symbol[][] */ syms,
+) {
+  const /* int */ outputSize = output.remaining()
+  let rans0 = input.getInt()
+  let rans1 = input.getInt()
+  let rans2 = input.getInt()
+  let rans7 = input.getInt()
+
+  const /* int */ isz4 = outputSize >> 2
+  let /* int */ i0 = 0
+  let /* int */ i1 = isz4
+  let /* int */ i2 = 2 * isz4
+  let /* int */ i7 = 3 * isz4
+  let /* int */ l0 = 0
+  let /* int */ l1 = 0
+  let /* int */ l2 = 0
+  let /* int */ l7 = 0
+  for (; i0 < isz4; i0 += 1, i1 += 1, i2 += 1, i7 += 1) {
+    const /* int */ c0 = 0xff & D[l0].R[Decoding.get(rans0, TF_SHIFT)]
+    const /* int */ c1 = 0xff & D[l1].R[Decoding.get(rans1, TF_SHIFT)]
+    const /* int */ c2 = 0xff & D[l2].R[Decoding.get(rans2, TF_SHIFT)]
+    const /* int */ c7 = 0xff & D[l7].R[Decoding.get(rans7, TF_SHIFT)]
+
+    output.putAt(i0, c0)
+    output.putAt(i1, c1)
+    output.putAt(i2, c2)
+    output.putAt(i7, c7)
+
+    rans0 = Decoding.advanceSymbolStep(rans0, syms[l0][c0], TF_SHIFT)
+    rans1 = Decoding.advanceSymbolStep(rans1, syms[l1][c1], TF_SHIFT)
+    rans2 = Decoding.advanceSymbolStep(rans2, syms[l2][c2], TF_SHIFT)
+    rans7 = Decoding.advanceSymbolStep(rans7, syms[l7][c7], TF_SHIFT)
+
+    rans0 = Decoding.renormalize(rans0, input)
+    rans1 = Decoding.renormalize(rans1, input)
+    rans2 = Decoding.renormalize(rans2, input)
+    rans7 = Decoding.renormalize(rans7, input)
+
+    l0 = c0
+    l1 = c1
+    l2 = c2
+    l7 = c7
+  }
+
+  // Remainder
+  for (; i7 < outputSize; i7 += 1) {
+    const /* int */ c7 = 0xff & D[l7].R[Decoding.get(rans7, TF_SHIFT)]
+    output.putAt(i7, c7)
+    rans7 = Decoding.advanceSymbol(rans7, input, syms[l7][c7], TF_SHIFT)
+    l7 = c7
+  }
+}

package/src/rans/decoding.js

@@ -0,0 +1,141 @@
+import { CramMalformedError } from '../errors'
+
+import { RANS_BYTE_L } from './constants'
+
+class FC {
+  // int F, C;
+  constructor() {
+    this.F = undefined
+    this.C = undefined
+  }
+}
+
+class AriDecoder {
+  // final FC[] fc = new FC[256];
+  // byte[] R;
+
+  constructor() {
+    this.fc = new Array(256)
+    for (let i = 0; i < this.fc.length; i += 1) {
+      this.fc[i] = new FC()
+    }
+    this.R = null
+  }
+}
+
+class Symbol {
+  // int start; // Start of range.
+  // int freq; // Symbol frequency.
+  constructor() {
+    this.start = undefined
+    this.freq = undefined
+  }
+}
+
+// Initialize a decoder symbol to start "start" and frequency "freq"
+function symbolInit(sym, start, freq) {
+  if (!(start <= 1 << 16)) {
+    throw new CramMalformedError(`assertion failed: start <= 1<<16`)
+  }
+  if (!(freq <= (1 << 16) - start)) {
+    throw new CramMalformedError(`assertion failed: freq <= 1<<16`)
+  }
+  sym.start = start
+  sym.freq = freq
+}
+
+// Advances in the bit stream by "popping" a single symbol with range start
+// "start" and frequency "freq". All frequencies are assumed to sum to
+// "1 << scaleBits".
+// No renormalization or output happens.
+/* private static int */ function advanceStep(
+  /* final int */ r,
+  /* final int */ start,
+  /* final int */ freq,
+  /* final int */ scaleBits,
+) {
+  /* final int */ const mask = (1 << scaleBits) - 1
+
+  // s, x = D(x)
+  return freq * (r >> scaleBits) + (r & mask) - start
+}
+
+// Equivalent to RansDecAdvanceStep that takes a symbol.
+/* static int  */ function advanceSymbolStep(
+  /* final int */ r,
+  /* final RansDecSymbol */ sym,
+  /* final int */ scaleBits,
+) {
+  return advanceStep(r, sym.start, sym.freq, scaleBits)
+}
+
+// Returns the current cumulative frequency (map it to a symbol yourself!)
+/* static int */ function get(/* final int */ r, /* final int */ scaleBits) {
+  return r & ((1 << scaleBits) - 1)
+}
+
+// Advances in the bit stream by "popping" a single symbol with range start
+// "start" and frequency "freq". All frequencies are assumed to sum to
+// "1 << scaleBits",
+// and the resulting bytes get written to ptr (which is updated).
+/* private static int */ function advance(
+  /* int */ r,
+  /* final ByteBuffer */ pptr,
+  /* final int */ start,
+  /* final int */ freq,
+  /* final int */ scaleBits,
+) {
+  /* final int */ const mask = (1 << scaleBits) - 1
+
+  // s, x = D(x)
+  r = freq * (r >> scaleBits) + (r & mask) - start
+
+  // re-normalize
+  if (r < RANS_BYTE_L) {
+    do {
+      /* final int */ const b = 0xff & pptr.get()
+      r = (r << 8) | b
+    } while (r < RANS_BYTE_L)
+  }
+
+  return r
+}
+
+// Equivalent to RansDecAdvance that takes a symbol.
+/*  static int */ function advanceSymbol(
+  /* final int */ r,
+  /* final ByteBuffer */ pptr,
+  /* final RansDecSymbol */ sym,
+  /* final int */ scaleBits,
+) {
+  return advance(r, pptr, sym.start, sym.freq, scaleBits)
+}
+
+// Re-normalize.
+/*  static int */ function renormalize(
+  /* int */ r,
+  /* final ByteBuffer */ pptr,
+) {
+  // re-normalize
+  if (r < RANS_BYTE_L) {
+    do {
+      r = (r << 8) | (0xff & pptr.get())
+    } while (r < RANS_BYTE_L)
+  }
+
+  return r
+}
+
+const Decode = {
+  FC,
+  AriDecoder,
+  Symbol,
+  symbolInit,
+  advanceStep,
+  advanceSymbolStep,
+  get,
+  advanceSymbol,
+  renormalize,
+}
+
+export default Decode