@gmod/bbi 1.0.32 → 1.0.35

package/src/blockView.ts

@@ -0,0 +1,462 @@
+/* eslint no-bitwise: ["error", { "allow": ["|"] }] */
+import { Observer } from 'rxjs'
+import { Parser } from '@gmod/binary-parser'
+import AbortablePromiseCache from 'abortable-promise-cache'
+import { GenericFilehandle } from 'generic-filehandle'
+import { unzip } from './unzip'
+import QuickLRU from 'quick-lru'
+import { Feature } from './bbi'
+import Range from './range'
+import { groupBlocks, checkAbortSignal } from './util'
+
+interface CoordRequest {
+  chrId: number
+  start: number
+  end: number
+}
+interface DataBlock {
+  startChrom: number
+  endChrom: number
+  startBase: number
+  endBase: number
+  validCnt: number
+  minVal: number
+  maxVal: number
+  sumData: number
+  sumSqData: number
+}
+interface ReadData {
+  offset: number
+  length: number
+}
+
+interface SummaryBlock {
+  chromId: number
+  start: number
+  end: number
+  validCnt: number
+  minScore: number
+  maxScore: number
+  sumData: number
+  sumSqData: number
+}
+interface Options {
+  signal?: AbortSignal
+  request?: CoordRequest
+}
+
+const BIG_WIG_TYPE_GRAPH = 1
+const BIG_WIG_TYPE_VSTEP = 2
+const BIG_WIG_TYPE_FSTEP = 3
+
+function getParsers(isBigEndian: boolean): any {
+  const le = isBigEndian ? 'big' : 'little'
+  const summaryParser = new Parser()
+    .endianess(le)
+    .uint32('chromId')
+    .uint32('start')
+    .uint32('end')
+    .uint32('validCnt')
+    .float('minScore')
+    .float('maxScore')
+    .float('sumData')
+    .float('sumSqData')
+
+  const leafParser = new Parser()
+    .endianess(le)
+    .uint8('isLeaf')
+    .skip(1)
+    .uint16('cnt')
+    .choice({
+      tag: 'isLeaf',
+      choices: {
+        1: new Parser().array('blocksToFetch', {
+          length: 'cnt',
+          type: new Parser()
+            .uint32('startChrom')
+            .uint32('startBase')
+            .uint32('endChrom')
+            .uint32('endBase')
+            .uint64('blockOffset')
+            .uint64('blockSize'),
+        }),
+        0: new Parser().array('recurOffsets', {
+          length: 'cnt',
+          type: new Parser()
+            .uint32('startChrom')
+            .uint32('startBase')
+            .uint32('endChrom')
+            .uint32('endBase')
+            .uint64('blockOffset'),
+        }),
+      },
+    })
+  const bigBedParser = new Parser()
+    .endianess(le)
+    .uint32('chromId')
+    .int32('start')
+    .int32('end')
+    .string('rest', {
+      zeroTerminated: true,
+    })
+
+  const bigWigParser = new Parser()
+    .endianess(le)
+    .skip(4)
+    .int32('blockStart')
+    .skip(4)
+    .uint32('itemStep')
+    .uint32('itemSpan')
+    .uint8('blockType')
+    .skip(1)
+    .uint16('itemCount')
+    .choice({
+      tag: 'blockType',
+      choices: {
+        [BIG_WIG_TYPE_FSTEP]: new Parser().array('items', {
+          length: 'itemCount',
+          type: new Parser().float('score'),
+        }),
+        [BIG_WIG_TYPE_VSTEP]: new Parser().array('items', {
+          length: 'itemCount',
+          type: new Parser().int32('start').float('score'),
+        }),
+        [BIG_WIG_TYPE_GRAPH]: new Parser().array('items', {
+          length: 'itemCount',
+          type: new Parser().int32('start').int32('end').float('score'),
+        }),
+      },
+    })
+  return {
+    bigWigParser,
+    bigBedParser,
+    summaryParser,
+    leafParser,
+  }
+}
+
+/**
+ * View into a subset of the data in a BigWig file.
+ *
+ * Adapted by Robert Buels and Colin Diesh from bigwig.js in the Dalliance Genome
+ * Explorer by Thomas Down.
+ * @constructs
+ */
+
+export class BlockView {
+  private cirTreeOffset: number
+
+  private cirTreeLength: number
+
+  private bbi: GenericFilehandle
+
+  private isCompressed: boolean
+
+  private isBigEndian: boolean
+
+  private refsByName: any
+
+  private blockType: string
+
+  private cirTreePromise?: Promise<{ bytesRead: number; buffer: Buffer }>
+
+  private featureCache = new AbortablePromiseCache({
+    cache: new QuickLRU({ maxSize: 1000 }),
+
+    fill: async (requestData: ReadData, signal: AbortSignal) => {
+      const { length, offset } = requestData
+      const { buffer } = await this.bbi.read(
+        Buffer.alloc(length),
+        0,
+        length,
+        offset,
+        { signal },
+      )
+      return buffer
+    },
+  })
+
+  private leafParser: any
+
+  private bigWigParser: any
+
+  private bigBedParser: any
+
+  private summaryParser: any
+
+  public constructor(
+    bbi: GenericFilehandle,
+    refsByName: any,
+    cirTreeOffset: number,
+    cirTreeLength: number,
+    isBigEndian: boolean,
+    isCompressed: boolean,
+    blockType: string,
+  ) {
+    if (!(cirTreeOffset >= 0)) {
+      throw new Error('invalid cirTreeOffset!')
+    }
+    if (!(cirTreeLength > 0)) {
+      throw new Error('invalid cirTreeLength!')
+    }
+
+    this.cirTreeOffset = cirTreeOffset
+    this.cirTreeLength = cirTreeLength
+    this.isCompressed = isCompressed
+    this.refsByName = refsByName
+    this.isBigEndian = isBigEndian
+    this.bbi = bbi
+    this.blockType = blockType
+    Object.assign(this, getParsers(isBigEndian))
+  }
+
+  public async readWigData(
+    chrName: string,
+    start: number,
+    end: number,
+    observer: Observer<Feature[]>,
+    opts: Options,
+  ) {
+    try {
+      const { refsByName, bbi, cirTreeOffset, isBigEndian } = this
+      const { signal } = opts
+      const chrId = refsByName[chrName]
+      if (chrId === undefined) {
+        observer.complete()
+      }
+      const request = { chrId, start, end }
+      if (!this.cirTreePromise) {
+        this.cirTreePromise = bbi.read(Buffer.alloc(48), 0, 48, cirTreeOffset, {
+          signal,
+        })
+      }
+      const { buffer } = await this.cirTreePromise
+      const cirBlockSize = isBigEndian
+        ? buffer.readUInt32BE(4)
+        : buffer.readUInt32LE(4)
+      let blocksToFetch: any[] = []
+      let outstanding = 0
+
+      const cirFobRecur2 = (
+        cirBlockData: Buffer,
+        offset: number,
+        level: number,
+      ) => {
+        try {
+          const data = cirBlockData.slice(offset)
+
+          const p = this.leafParser.parse(data).result
+          if (p.blocksToFetch) {
+            blocksToFetch = blocksToFetch.concat(
+              p.blocksToFetch.filter(filterFeats).map((l: any): any => ({
+                offset: l.blockOffset,
+                length: l.blockSize,
+              })),
+            )
+          }
+          if (p.recurOffsets) {
+            const recurOffsets = p.recurOffsets
+              .filter(filterFeats)
+              .map((l: any): any => l.blockOffset)
+            if (recurOffsets.length > 0) {
+              cirFobRecur(recurOffsets, level + 1)
+            }
+          }
+        } catch (e) {
+          observer.error(e)
+        }
+      }
+
+      const filterFeats = (b: DataBlock) => {
+        const { startChrom, startBase, endChrom, endBase } = b
+        return (
+          (startChrom < chrId || (startChrom === chrId && startBase <= end)) &&
+          (endChrom > chrId || (endChrom === chrId && endBase >= start))
+        )
+      }
+
+      const cirFobStartFetch = async (off: any, fr: any, level: number) => {
+        try {
+          const length = fr.max() - fr.min()
+          const offset = fr.min()
+          const resultBuffer = await this.featureCache.get(
+            `${length}_${offset}`,
+            { length, offset },
+            signal,
+          )
+          for (let i = 0; i < off.length; i += 1) {
+            if (fr.contains(off[i])) {
+              cirFobRecur2(resultBuffer, off[i] - offset, level)
+              outstanding -= 1
+              if (outstanding === 0) {
+                this.readFeatures(observer, blocksToFetch, { ...opts, request })
+              }
+            }
+          }
+        } catch (e) {
+          observer.error(e)
+        }
+      }
+      const cirFobRecur = (offset: any, level: number) => {
+        try {
+          outstanding += offset.length
+
+          const maxCirBlockSpan = 4 + cirBlockSize * 32 // Upper bound on size, based on a completely full leaf node.
+          let spans = new Range(offset[0], offset[0] + maxCirBlockSpan)
+          for (let i = 1; i < offset.length; i += 1) {
+            const blockSpan = new Range(offset[i], offset[i] + maxCirBlockSpan)
+            spans = spans.union(blockSpan)
+          }
+          spans.getRanges().map(fr => cirFobStartFetch(offset, fr, level))
+        } catch (e) {
+          observer.error(e)
+        }
+      }
+
+      return cirFobRecur([cirTreeOffset + 48], 1)
+    } catch (e) {
+      observer.error(e)
+    }
+  }
+
+  private parseSummaryBlock(
+    data: Buffer,
+    startOffset: number,
+    request?: CoordRequest,
+  ) {
+    const features = [] as SummaryBlock[]
+    let currOffset = startOffset
+    while (currOffset < data.byteLength) {
+      const res = this.summaryParser.parse(data.slice(currOffset))
+      features.push(res.result)
+      currOffset += res.offset
+    }
+    let items = features
+    if (request) {
+      items = items.filter(elt => elt.chromId === request.chrId)
+    }
+    const feats = items.map(
+      (elt: SummaryBlock): Feature => ({
+        start: elt.start,
+        end: elt.end,
+        maxScore: elt.maxScore,
+        minScore: elt.minScore,
+        score: elt.sumData / (elt.validCnt || 1),
+        summary: true,
+      }),
+    )
+    return request
+      ? feats.filter(f => BlockView.coordFilter(f, request))
+      : feats
+  }
+
+  private parseBigBedBlock(
+    data: Buffer,
+    startOffset: number,
+    offset: number,
+    request?: CoordRequest,
+  ) {
+    const items = [] as Feature[]
+    let currOffset = startOffset
+    while (currOffset < data.byteLength) {
+      const res = this.bigBedParser.parse(data.slice(currOffset))
+      res.result.uniqueId = `bb-${offset + currOffset}`
+      items.push(res.result)
+      currOffset += res.offset
+    }
+
+    return request
+      ? items.filter((f: any) => BlockView.coordFilter(f, request))
+      : items
+  }
+
+  private parseBigWigBlock(
+    bytes: Buffer,
+    startOffset: number,
+    request?: CoordRequest,
+  ): Feature[] {
+    const data = bytes.slice(startOffset)
+    const results = this.bigWigParser.parse(data).result
+    const { items, itemSpan, itemStep, blockStart, blockType } = results
+    if (blockType === BIG_WIG_TYPE_FSTEP) {
+      for (let i = 0; i < items.length; i++) {
+        items[i].start = blockStart + i * itemStep
+        items[i].end = blockStart + i * itemStep + itemSpan
+      }
+    } else if (blockType === BIG_WIG_TYPE_VSTEP) {
+      for (let i = 0; i < items.length; i++) {
+        items[i].end = items[i].start + itemSpan
+      }
+    }
+    return request
+      ? items.filter((f: any) => BlockView.coordFilter(f, request))
+      : items
+  }
+
+  private static coordFilter(f: Feature, range: CoordRequest): boolean {
+    return f.start < range.end && f.end >= range.start
+  }
+
+  public async readFeatures(
+    observer: Observer<Feature[]>,
+    blocks: any,
+    opts: Options = {},
+  ): Promise<void> {
+    try {
+      const { blockType, isCompressed } = this
+      const { signal, request } = opts
+      const blockGroupsToFetch = groupBlocks(blocks)
+      checkAbortSignal(signal)
+      await Promise.all(
+        blockGroupsToFetch.map(async (blockGroup: any) => {
+          checkAbortSignal(signal)
+          const { length, offset } = blockGroup
+          const data = await this.featureCache.get(
+            `${length}_${offset}`,
+            blockGroup,
+            signal,
+          )
+          blockGroup.blocks.forEach((block: any) => {
+            checkAbortSignal(signal)
+            let blockOffset = block.offset - blockGroup.offset
+            let resultData = data
+            if (isCompressed) {
+              resultData = unzip(data.slice(blockOffset))
+              blockOffset = 0
+            }
+            checkAbortSignal(signal)
+
+            switch (blockType) {
+              case 'summary':
+                observer.next(
+                  this.parseSummaryBlock(resultData, blockOffset, request),
+                )
+                break
+              case 'bigwig':
+                observer.next(
+                  this.parseBigWigBlock(resultData, blockOffset, request),
+                )
+                break
+              case 'bigbed':
+                observer.next(
+                  this.parseBigBedBlock(
+                    resultData,
+                    blockOffset,
+                    // eslint-disable-next-line no-bitwise
+                    block.offset * (1 << 8),
+                    request,
+                  ),
+                )
+                break
+              default:
+                console.warn(`Don't know what to do with ${blockType}`)
+            }
+          })
+        }),
+      )
+      observer.complete()
+    } catch (e) {
+      observer.error(e)
+    }
+  }
+}

package/src/declare.d.ts

@@ -0,0 +1,4 @@
+declare module 'es6-promisify'
+declare module 'abortable-promise-cache'
+declare module '@gmod/binary-parser'
+declare module '@gmod/bed'

package/src/index.ts

@@ -0,0 +1,3 @@
+export { BigWig } from './bigwig'
+export { BigBed } from './bigbed'
+export { Feature, Header, RequestOptions } from './bbi'

package/src/range.ts

@@ -0,0 +1,142 @@
+/* eslint prefer-rest-params:0, no-nested-ternary:0 */
+
+/**
+ * Adapted from a combination of Range and _Compound in the
+ * Dalliance Genome Explorer, (c) Thomas Down 2006-2010.
+ */
+export default class Range {
+  public ranges: any
+
+  public constructor(arg1: any, arg2?: any) {
+    this.ranges =
+      arguments.length === 2
+        ? [{ min: arg1, max: arg2 }]
+        : 0 in arg1
+        ? Object.assign({}, arg1)
+        : [arg1]
+  }
+
+  public min(): number {
+    return this.ranges[0].min
+  }
+
+  public max(): number {
+    return this.ranges[this.ranges.length - 1].max
+  }
+
+  public contains(pos: number): boolean {
+    for (let s = 0; s < this.ranges.length; s += 1) {
+      const r = this.ranges[s]
+      if (r.min <= pos && r.max >= pos) {
+        return true
+      }
+    }
+    return false
+  }
+
+  public isContiguous(): boolean {
+    return this.ranges.length > 1
+  }
+
+  public getRanges(): Range[] {
+    return this.ranges.map((r: Range) => new Range(r.min, r.max))
+  }
+
+  public toString(): string {
+    return this.ranges.map((r: Range) => `[${r.min}-${r.max}]`).join(',')
+  }
+
+  public union(s1: Range): Range {
+    const ranges = this.getRanges().concat(s1.getRanges()).sort(this.rangeOrder)
+    const oranges = []
+    let current = ranges[0]
+
+    for (let i = 1; i < ranges.length; i += 1) {
+      const nxt = ranges[i]
+      if (nxt.min() > current.max() + 1) {
+        oranges.push(current)
+        current = nxt
+      } else if (nxt.max() > current.max()) {
+        current = new Range(current.min(), nxt.max())
+      }
+    }
+    oranges.push(current)
+
+    if (oranges.length === 1) {
+      return oranges[0]
+    }
+    return new Range(oranges)
+  }
+
+  public intersection(arg: Range): Range {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    let s0 = this
+    let s1 = arg
+    const r0 = this.ranges()
+    const r1 = s1.ranges()
+    const l0 = r0.length
+
+    const l1 = r1.length
+    let i0 = 0
+
+    let i1 = 0
+    const or = []
+
+    while (i0 < l0 && i1 < l1) {
+      s0 = r0[i0]
+      s1 = r1[i1]
+      const lapMin = Math.max(s0.min(), s1.min())
+      const lapMax = Math.min(s0.max(), s1.max())
+      if (lapMax >= lapMin) {
+        or.push(new Range(lapMin, lapMax))
+      }
+      if (s0.max() > s1.max()) {
+        i1 += 1
+      } else {
+        i0 += 1
+      }
+    }
+
+    if (or.length === 0) {
+      throw new Error('found range of length 0')
+    }
+    if (or.length === 1) {
+      return or[0]
+    }
+    return new Range(or)
+  }
+
+  public coverage(): number {
+    let tot = 0
+    const rl = this.ranges()
+    for (let ri = 0; ri < rl.length; ri += 1) {
+      const r = rl[ri]
+      tot += r.max() - r.min() + 1
+    }
+    return tot
+  }
+
+  public rangeOrder(tmpa: Range, tmpb: Range): number {
+    let a = tmpa
+    let b = tmpb
+    if (arguments.length < 2) {
+      b = a
+      // eslint-disable-next-line @typescript-eslint/no-this-alias
+      a = this
+    }
+
+    if (a.min() < b.min()) {
+      return -1
+    }
+    if (a.min() > b.min()) {
+      return 1
+    }
+    if (a.max() < b.max()) {
+      return -1
+    }
+    if (b.max() > a.max()) {
+      return 1
+    }
+    return 0
+  }
+}

package/src/unzip-pako.ts

@@ -0,0 +1,5 @@
+import { inflate } from 'pako'
+
+export function unzip(input: Buffer) {
+  return Buffer.from(inflate(input))
+}

package/src/unzip.ts

@@ -0,0 +1,2 @@
+import { inflateSync } from 'zlib'
+export { inflateSync as unzip }

package/src/util.ts

@@ -0,0 +1,74 @@
+/* eslint no-bitwise: ["error", { "allow": ["|"] }] */
+export class AbortError extends Error {
+  public code: string
+
+  public constructor(message: string) {
+    super(message)
+    this.code = 'ERR_ABORTED'
+  }
+}
+// sort blocks by file offset and
+// group blocks that are within 2KB of eachother
+export function groupBlocks(blocks: any[]): any[] {
+  blocks.sort((b0, b1) => (b0.offset | 0) - (b1.offset | 0))
+
+  const blockGroups = []
+  let lastBlock
+  let lastBlockEnd
+  for (let i = 0; i < blocks.length; i += 1) {
+    if (lastBlock && blocks[i].offset - lastBlockEnd <= 2000) {
+      lastBlock.length += blocks[i].length - lastBlockEnd + blocks[i].offset
+      lastBlock.blocks.push(blocks[i])
+    } else {
+      blockGroups.push(
+        (lastBlock = {
+          blocks: [blocks[i]],
+          length: blocks[i].length,
+          offset: blocks[i].offset,
+        }),
+      )
+    }
+    lastBlockEnd = lastBlock.offset + lastBlock.length
+  }
+
+  return blockGroups
+}
+
+/**
+ * Properly check if the given AbortSignal is aborted.
+ * Per the standard, if the signal reads as aborted,
+ * this function throws either a DOMException AbortError, or a regular error
+ * with a `code` attribute set to `ERR_ABORTED`.
+ *
+ * For convenience, passing `undefined` is a no-op
+ *
+ * @param {AbortSignal} [signal] an AbortSignal, or anything with an `aborted` attribute
+ * @returns nothing
+ */
+export function checkAbortSignal(signal?: AbortSignal): void {
+  if (!signal) {
+    return
+  }
+
+  if (signal.aborted) {
+    // console.log('bam aborted!')
+    if (typeof DOMException !== 'undefined') {
+      throw new DOMException('aborted', 'AbortError')
+    } else {
+      const e = new AbortError('aborted')
+      e.code = 'ERR_ABORTED'
+      throw e
+    }
+  }
+}
+
+/**
+ * Skips to the next tick, then runs `checkAbortSignal`.
+ * Await this to inside an otherwise synchronous loop to
+ * provide a place to break when an abort signal is received.
+ * @param {AbortSignal} signal
+ */
+export async function abortBreakPoint(signal?: AbortSignal): Promise<void> {
+  await Promise.resolve()
+  checkAbortSignal(signal)
+}