@gmod/bbi 1.0.33 → 2.0.0

package/src/blockView.ts

@@ -0,0 +1,496 @@
+import { Observer } from 'rxjs'
+import { Parser } from 'binary-parser'
+import AbortablePromiseCache from 'abortable-promise-cache'
+import { GenericFilehandle } from 'generic-filehandle'
+import QuickLRU from 'quick-lru'
+
+// locals
+import Range from './range'
+import { unzip } from './unzip'
+import { Feature } from './bbi'
+import { groupBlocks, checkAbortSignal } from './util'
+
+interface CoordRequest {
+  chrId: number
+  start: number
+  end: number
+}
+interface DataBlock {
+  startChrom: number
+  endChrom: number
+  startBase: number
+  endBase: number
+  validCnt: number
+  minVal: number
+  maxVal: number
+  sumData: number
+  sumSqData: number
+}
+interface ReadData {
+  offset: number
+  length: number
+}
+
+interface Options {
+  signal?: AbortSignal
+  request?: CoordRequest
+}
+
+const BIG_WIG_TYPE_GRAPH = 1
+const BIG_WIG_TYPE_VSTEP = 2
+const BIG_WIG_TYPE_FSTEP = 3
+
+function coordFilter(s1: number, e1: number, s2: number, e2: number): boolean {
+  return s1 < e2 && e1 >= s2
+}
+
+function getParsers(isBigEndian: boolean) {
+  const le = isBigEndian ? 'big' : 'little'
+  const summaryParser = new Parser()
+    .endianess(le)
+    .uint32('chromId')
+    .uint32('start')
+    .uint32('end')
+    .uint32('validCnt')
+    .floatle('minScore')
+    .floatle('maxScore')
+    .floatle('sumData')
+    .floatle('sumSqData')
+    .saveOffset('offset')
+
+  const leafParser = new Parser()
+    .endianess(le)
+    .uint8('isLeaf')
+    .skip(1)
+    .uint16('cnt')
+    .choice({
+      tag: 'isLeaf',
+      choices: {
+        1: new Parser().endianess(le).array('blocksToFetch', {
+          length: 'cnt',
+          type: new Parser()
+            .endianess(le)
+            .uint32('startChrom')
+            .uint32('startBase')
+            .uint32('endChrom')
+            .uint32('endBase')
+            .uint64('blockOffset')
+            .uint64('blockSize')
+            .saveOffset('offset'),
+        }),
+        0: new Parser().array('recurOffsets', {
+          length: 'cnt',
+          type: new Parser()
+            .endianess(le)
+            .uint32('startChrom')
+            .uint32('startBase')
+            .uint32('endChrom')
+            .uint32('endBase')
+            .uint64('blockOffset')
+            .saveOffset('offset'),
+        }),
+      },
+    })
+  const bigBedParser = new Parser()
+    .endianess(le)
+    .uint32('chromId')
+    .int32('start')
+    .int32('end')
+    .string('rest', {
+      zeroTerminated: true,
+    })
+    .saveOffset('offset')
+
+  const bigWigParser = new Parser()
+    .endianess(le)
+    .skip(4)
+    .int32('blockStart')
+    .skip(4)
+    .uint32('itemStep')
+    .uint32('itemSpan')
+    .uint8('blockType')
+    .skip(1)
+    .uint16('itemCount')
+    .choice({
+      tag: 'blockType',
+      choices: {
+        [BIG_WIG_TYPE_FSTEP]: new Parser().array('items', {
+          length: 'itemCount',
+          type: new Parser().floatle('score'),
+        }),
+        [BIG_WIG_TYPE_VSTEP]: new Parser().array('items', {
+          length: 'itemCount',
+          type: new Parser().endianess(le).int32('start').floatle('score'),
+        }),
+        [BIG_WIG_TYPE_GRAPH]: new Parser().array('items', {
+          length: 'itemCount',
+          type: new Parser()
+            .endianess(le)
+            .int32('start')
+            .int32('end')
+            .floatle('score'),
+        }),
+      },
+    })
+  return {
+    bigWigParser,
+    bigBedParser,
+    summaryParser,
+    leafParser,
+  }
+}
+
+/**
+ * View into a subset of the data in a BigWig file.
+ *
+ * Adapted by Robert Buels and Colin Diesh from bigwig.js in the Dalliance Genome
+ * Explorer by Thomas Down.
+ * @constructs
+ */
+
+export class BlockView {
+  private cirTreePromise?: Promise<{ bytesRead: number; buffer: Buffer }>
+
+  private featureCache = new AbortablePromiseCache({
+    cache: new QuickLRU({ maxSize: 1000 }),
+
+    fill: async (requestData: ReadData, signal: AbortSignal) => {
+      const len = Number(requestData.length)
+      const off = Number(requestData.offset)
+      const { buffer } = await this.bbi.read(Buffer.alloc(len), 0, len, off, {
+        signal,
+      })
+      return buffer
+    },
+  })
+
+  private leafParser: ReturnType<typeof getParsers>['leafParser']
+
+  private bigBedParser: ReturnType<typeof getParsers>['bigBedParser']
+
+  public constructor(
+    private bbi: GenericFilehandle,
+    private refsByName: any,
+    private cirTreeOffset: number,
+    private isBigEndian: boolean,
+    private isCompressed: boolean,
+    private blockType: string,
+  ) {
+    if (!(cirTreeOffset >= 0)) {
+      throw new Error('invalid cirTreeOffset!')
+    }
+
+    const parsers = getParsers(isBigEndian)
+    this.leafParser = parsers.leafParser
+    this.bigBedParser = parsers.bigBedParser
+  }
+
+  public async readWigData(
+    chrName: string,
+    start: number,
+    end: number,
+    observer: Observer<Feature[]>,
+    opts: Options,
+  ) {
+    try {
+      const { refsByName, bbi, cirTreeOffset, isBigEndian } = this
+      const chrId = refsByName[chrName]
+      if (chrId === undefined) {
+        observer.complete()
+      }
+      const request = { chrId, start, end }
+      if (!this.cirTreePromise) {
+        const off = Number(cirTreeOffset)
+        this.cirTreePromise = bbi.read(Buffer.alloc(48), 0, 48, off, opts)
+      }
+      const { buffer } = await this.cirTreePromise
+      const cirBlockSize = isBigEndian
+        ? buffer.readUInt32BE(4)
+        : buffer.readUInt32LE(4)
+      let blocksToFetch: any[] = []
+      let outstanding = 0
+
+      const cirFobRecur2 = (
+        cirBlockData: Buffer,
+        offset: number,
+        level: number,
+      ) => {
+        try {
+          const data = cirBlockData.subarray(offset)
+
+          const p = this.leafParser.parse(data)
+          if (p.blocksToFetch) {
+            blocksToFetch = blocksToFetch.concat(
+              p.blocksToFetch
+                .filter(filterFeats)
+                .map((l: { blockOffset: bigint; blockSize: bigint }) => ({
+                  offset: l.blockOffset,
+                  length: l.blockSize,
+                })),
+            )
+          }
+          if (p.recurOffsets) {
+            const recurOffsets = p.recurOffsets
+              .filter(filterFeats)
+              .map((l: { blockOffset: bigint }) => Number(l.blockOffset))
+            if (recurOffsets.length > 0) {
+              cirFobRecur(recurOffsets, level + 1)
+            }
+          }
+        } catch (e) {
+          observer.error(e)
+        }
+      }
+
+      const filterFeats = (b: DataBlock) => {
+        const { startChrom, startBase, endChrom, endBase } = b
+        return (
+          (startChrom < chrId || (startChrom === chrId && startBase <= end)) &&
+          (endChrom > chrId || (endChrom === chrId && endBase >= start))
+        )
+      }
+
+      const cirFobStartFetch = async (
+        off: number[],
+        fr: Range,
+        level: number,
+      ) => {
+        try {
+          const length = fr.max() - fr.min()
+          const offset = fr.min()
+          const resultBuffer: Buffer = await this.featureCache.get(
+            `${length}_${offset}`,
+            { length, offset },
+            opts.signal,
+          )
+          for (let i = 0; i < off.length; i += 1) {
+            if (fr.contains(off[i])) {
+              cirFobRecur2(resultBuffer, off[i] - offset, level)
+              outstanding -= 1
+              if (outstanding === 0) {
+                this.readFeatures(observer, blocksToFetch, { ...opts, request })
+              }
+            }
+          }
+        } catch (e) {
+          observer.error(e)
+        }
+      }
+      const cirFobRecur = (offset: number[], level: number) => {
+        try {
+          outstanding += offset.length
+
+          const maxCirBlockSpan = 4 + Number(cirBlockSize) * 32 // Upper bound on size, based on a completely full leaf node.
+          let spans = new Range(offset[0], offset[0] + maxCirBlockSpan)
+          for (let i = 1; i < offset.length; i += 1) {
+            const blockSpan = new Range(offset[i], offset[i] + maxCirBlockSpan)
+            spans = spans.union(blockSpan)
+          }
+          spans.getRanges().map(fr => cirFobStartFetch(offset, fr, level))
+        } catch (e) {
+          observer.error(e)
+        }
+      }
+
+      return cirFobRecur([Number(cirTreeOffset) + 48], 1)
+    } catch (e) {
+      observer.error(e)
+    }
+  }
+
+  private parseSummaryBlock(
+    buffer: Buffer,
+    startOffset: number,
+    request?: CoordRequest,
+  ) {
+    const features = [] as any[]
+    let offset = startOffset
+
+    const dataView = new DataView(
+      buffer.buffer,
+      buffer.byteOffset,
+      buffer.length,
+    )
+    while (offset < buffer.byteLength) {
+      // this was extracted from looking at the runtime code generated by
+      // binary-parser
+      const chromId = dataView.getUint32(offset, true)
+      offset += 4
+      const start = dataView.getUint32(offset, true)
+      offset += 4
+      const end = dataView.getUint32(offset, true)
+      offset += 4
+      const validCnt = dataView.getUint32(offset, true)
+      offset += 4
+      const minScore = dataView.getFloat32(offset, true)
+      offset += 4
+      const maxScore = dataView.getFloat32(offset, true)
+      offset += 4
+      const sumData = dataView.getFloat32(offset, true)
+      offset += 4
+      // unused
+      // const sumSqData = dataView.getFloat32(offset, true)
+      offset += 4
+
+      if (
+        request
+          ? chromId === request.chrId &&
+            coordFilter(start, end, request.start, request.end)
+          : true
+      ) {
+        features.push({
+          start,
+          end,
+          maxScore,
+          minScore,
+          summary: true,
+          score: sumData / (validCnt || 1),
+        })
+      }
+    }
+
+    return features
+  }
+
+  private parseBigBedBlock(
+    data: Buffer,
+    startOffset: number,
+    offset: number,
+    request?: CoordRequest,
+  ) {
+    const items = [] as Feature[]
+    let currOffset = startOffset
+    while (currOffset < data.byteLength) {
+      const res = this.bigBedParser.parse(data.subarray(currOffset))
+      items.push({ ...res, uniqueId: `bb-${offset + currOffset}` })
+      currOffset += res.offset
+    }
+
+    return request
+      ? items.filter((f: any) =>
+          coordFilter(f.start, f.end, request.start, request.end),
+        )
+      : items
+  }
+
+  private parseBigWigBlock(
+    buffer: Buffer,
+    startOffset: number,
+    request?: CoordRequest,
+  ) {
+    const b = buffer.subarray(startOffset)
+
+    const dataView = new DataView(b.buffer, b.byteOffset, b.length)
+    let offset = 0
+    offset += 4
+    const blockStart = dataView.getInt32(offset, true)
+    offset += 8
+    const itemStep = dataView.getUint32(offset, true)
+    offset += 4
+    const itemSpan = dataView.getUint32(offset, true)
+    offset += 4
+    const blockType = dataView.getUint8(offset)
+    offset += 2
+    const itemCount = dataView.getUint16(offset, true)
+    offset += 2
+    const items = new Array(itemCount)
+    switch (blockType) {
+      case 1:
+        for (let i = 0; i < itemCount; i++) {
+          const start = dataView.getInt32(offset, true)
+          offset += 4
+          const end = dataView.getInt32(offset, true)
+          offset += 4
+          const score = dataView.getFloat32(offset, true)
+          offset += 4
+          items[i] = { start, end, score }
+        }
+        break
+      case 2:
+        for (let i = 0; i < itemCount; i++) {
+          const start = dataView.getInt32(offset, true)
+          offset += 4
+          const score = dataView.getFloat32(offset, true)
+          offset += 4
+          items[i] = { score, start, end: start + itemSpan }
+        }
+        break
+      case 3:
+        for (let i = 0; i < itemCount; i++) {
+          const score = dataView.getFloat32(offset, true)
+          offset += 4
+          const start = blockStart + i * itemStep
+          items[i] = { score, start, end: start + itemSpan }
+        }
+        break
+    }
+
+    return request
+      ? items.filter((f: any) =>
+          coordFilter(f.start, f.end, request.start, request.end),
+        )
+      : items
+  }
+
+  public async readFeatures(
+    observer: Observer<Feature[]>,
+    blocks: { offset: bigint; length: bigint }[],
+    opts: Options = {},
+  ) {
+    try {
+      const { blockType, isCompressed } = this
+      const { signal, request } = opts
+      const blockGroupsToFetch = groupBlocks(blocks)
+      checkAbortSignal(signal)
+      await Promise.all(
+        blockGroupsToFetch.map(async blockGroup => {
+          checkAbortSignal(signal)
+          const { length, offset } = blockGroup
+          const data = await this.featureCache.get(
+            `${length}_${offset}`,
+            blockGroup,
+            signal,
+          )
+          blockGroup.blocks.forEach(block => {
+            checkAbortSignal(signal)
+            let blockOffset = Number(block.offset) - Number(blockGroup.offset)
+            let resultData = data
+            if (isCompressed) {
+              resultData = unzip(data.subarray(blockOffset))
+              blockOffset = 0
+            }
+            checkAbortSignal(signal)
+
+            switch (blockType) {
+              case 'summary':
+                observer.next(
+                  this.parseSummaryBlock(resultData, blockOffset, request),
+                )
+                break
+              case 'bigwig':
+                observer.next(
+                  this.parseBigWigBlock(resultData, blockOffset, request),
+                )
+                break
+              case 'bigbed':
+                observer.next(
+                  this.parseBigBedBlock(
+                    resultData,
+                    blockOffset,
+                    Number(block.offset) * (1 << 8),
+                    request,
+                  ),
+                )
+                break
+              default:
+                console.warn(`Don't know what to do with ${blockType}`)
+            }
+          })
+        }),
+      )
+      observer.complete()
+    } catch (e) {
+      observer.error(e)
+    }
+  }
+}

package/src/declare.d.ts

@@ -0,0 +1,2 @@
+declare module 'abortable-promise-cache'
+declare module '@gmod/bed'

package/src/index.ts

@@ -0,0 +1,3 @@
+export { BigWig } from './bigwig'
+export { BigBed } from './bigbed'
+export { Feature, Header, RequestOptions } from './bbi'

package/src/range.ts

@@ -0,0 +1,142 @@
+/* eslint prefer-rest-params:0, no-nested-ternary:0 */
+
+/**
+ * Adapted from a combination of Range and _Compound in the
+ * Dalliance Genome Explorer, (c) Thomas Down 2006-2010.
+ */
+export default class Range {
+  public ranges: any
+
+  public constructor(arg1: any, arg2?: any) {
+    this.ranges =
+      arguments.length === 2
+        ? [{ min: arg1, max: arg2 }]
+        : 0 in arg1
+        ? Object.assign({}, arg1)
+        : [arg1]
+  }
+
+  public min(): number {
+    return this.ranges[0].min
+  }
+
+  public max(): number {
+    return this.ranges[this.ranges.length - 1].max
+  }
+
+  public contains(pos: number): boolean {
+    for (let s = 0; s < this.ranges.length; s += 1) {
+      const r = this.ranges[s]
+      if (r.min <= pos && r.max >= pos) {
+        return true
+      }
+    }
+    return false
+  }
+
+  public isContiguous(): boolean {
+    return this.ranges.length > 1
+  }
+
+  public getRanges(): Range[] {
+    return this.ranges.map((r: Range) => new Range(r.min, r.max))
+  }
+
+  public toString(): string {
+    return this.ranges.map((r: Range) => `[${r.min}-${r.max}]`).join(',')
+  }
+
+  public union(s1: Range): Range {
+    const ranges = this.getRanges().concat(s1.getRanges()).sort(this.rangeOrder)
+    const oranges = []
+    let current = ranges[0]
+
+    for (let i = 1; i < ranges.length; i += 1) {
+      const nxt = ranges[i]
+      if (nxt.min() > current.max() + 1) {
+        oranges.push(current)
+        current = nxt
+      } else if (nxt.max() > current.max()) {
+        current = new Range(current.min(), nxt.max())
+      }
+    }
+    oranges.push(current)
+
+    if (oranges.length === 1) {
+      return oranges[0]
+    }
+    return new Range(oranges)
+  }
+
+  public intersection(arg: Range): Range {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    let s0 = this
+    let s1 = arg
+    const r0 = this.ranges()
+    const r1 = s1.ranges()
+    const l0 = r0.length
+
+    const l1 = r1.length
+    let i0 = 0
+
+    let i1 = 0
+    const or = []
+
+    while (i0 < l0 && i1 < l1) {
+      s0 = r0[i0]
+      s1 = r1[i1]
+      const lapMin = Math.max(s0.min(), s1.min())
+      const lapMax = Math.min(s0.max(), s1.max())
+      if (lapMax >= lapMin) {
+        or.push(new Range(lapMin, lapMax))
+      }
+      if (s0.max() > s1.max()) {
+        i1 += 1
+      } else {
+        i0 += 1
+      }
+    }
+
+    if (or.length === 0) {
+      throw new Error('found range of length 0')
+    }
+    if (or.length === 1) {
+      return or[0]
+    }
+    return new Range(or)
+  }
+
+  public coverage(): number {
+    let tot = 0
+    const rl = this.ranges()
+    for (let ri = 0; ri < rl.length; ri += 1) {
+      const r = rl[ri]
+      tot += r.max() - r.min() + 1
+    }
+    return tot
+  }
+
+  public rangeOrder(tmpa: Range, tmpb: Range): number {
+    let a = tmpa
+    let b = tmpb
+    if (arguments.length < 2) {
+      b = a
+      // eslint-disable-next-line @typescript-eslint/no-this-alias
+      a = this
+    }
+
+    if (a.min() < b.min()) {
+      return -1
+    }
+    if (a.min() > b.min()) {
+      return 1
+    }
+    if (a.max() < b.max()) {
+      return -1
+    }
+    if (b.max() > a.max()) {
+      return 1
+    }
+    return 0
+  }
+}

package/src/unzip-pako.ts

@@ -0,0 +1,5 @@
+import { inflateRaw } from 'pako'
+
+export function unzip(input: Buffer) {
+  return inflateRaw(input.subarray(2))
+}

package/src/unzip.ts

@@ -0,0 +1,2 @@
+import { inflateSync } from 'zlib'
+export { inflateSync as unzip }