@gmod/bbi 1.0.33 → 2.0.0

package/esm/util.js

@@ -1,24 +1,25 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.abortBreakPoint = exports.checkAbortSignal = exports.groupBlocks = exports.AbortError = void 0;
 /* eslint no-bitwise: ["error", { "allow": ["|"] }] */
-class AbortError extends Error {
+export class AbortError extends Error {
     constructor(message) {
         super(message);
         this.code = 'ERR_ABORTED';
     }
 }
-exports.AbortError = AbortError;
 // sort blocks by file offset and
 // group blocks that are within 2KB of eachother
-function groupBlocks(blocks) {
-    blocks.sort((b0, b1) => (b0.offset | 0) - (b1.offset | 0));
+export function groupBlocks(blocks) {
+    blocks.sort((b0, b1) => Number(b0.offset) - Number(b1.offset));
     const blockGroups = [];
     let lastBlock;
     let lastBlockEnd;
     for (let i = 0; i < blocks.length; i += 1) {
-        if (lastBlock && blocks[i].offset - lastBlockEnd <= 2000) {
-            lastBlock.length += blocks[i].length - lastBlockEnd + blocks[i].offset;
+        if (lastBlock &&
+            lastBlockEnd &&
+            Number(blocks[i].offset) - lastBlockEnd <= 2000) {
+            lastBlock.length = BigInt(Number(lastBlock.length) +
+                Number(blocks[i].length) -
+                lastBlockEnd +
+                Number(blocks[i].offset));
             lastBlock.blocks.push(blocks[i]);
         }
         else {
@@ -28,11 +29,10 @@ function groupBlocks(blocks) {
                 offset: blocks[i].offset,
             }));
         }
-        lastBlockEnd = lastBlock.offset + lastBlock.length;
+        lastBlockEnd = Number(lastBlock.offset) + Number(lastBlock.length);
     }
     return blockGroups;
 }
-exports.groupBlocks = groupBlocks;
 /**
  * Properly check if the given AbortSignal is aborted.
  * Per the standard, if the signal reads as aborted,
@@ -44,7 +44,7 @@ exports.groupBlocks = groupBlocks;
  * @param {AbortSignal} [signal] an AbortSignal, or anything with an `aborted` attribute
  * @returns nothing
  */
-function checkAbortSignal(signal) {
+export function checkAbortSignal(signal) {
     if (!signal) {
         return;
     }
@@ -60,15 +60,14 @@ function checkAbortSignal(signal) {
         }
     }
 }
-exports.checkAbortSignal = checkAbortSignal;
 /**
  * Skips to the next tick, then runs `checkAbortSignal`.
  * Await this to inside an otherwise synchronous loop to
  * provide a place to break when an abort signal is received.
  * @param {AbortSignal} signal
  */
-async function abortBreakPoint(signal) {
+export async function abortBreakPoint(signal) {
     await Promise.resolve();
     checkAbortSignal(signal);
 }
-exports.abortBreakPoint = abortBreakPoint;
+//# sourceMappingURL=util.js.map

package/esm/util.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,sDAAsD;AACtD,MAAM,OAAO,UAAW,SAAQ,KAAK;IAGnC,YAAmB,OAAe;QAChC,KAAK,CAAC,OAAO,CAAC,CAAA;QACd,IAAI,CAAC,IAAI,GAAG,aAAa,CAAA;IAC3B,CAAC;CACF;AACD,iCAAiC;AACjC,gDAAgD;AAChD,MAAM,UAAU,WAAW,CAAC,MAA4C;IACtE,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAA;IAE9D,MAAM,WAAW,GAAG,EAAE,CAAA;IACtB,IAAI,SAAS,CAAA;IACb,IAAI,YAAY,CAAA;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE;QACzC,IACE,SAAS;YACT,YAAY;YACZ,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,YAAY,IAAI,IAAI,EAC/C;YACA,SAAS,CAAC,MAAM,GAAG,MAAM,CACvB,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;gBACtB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;gBACxB,YAAY;gBACZ,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAC3B,CAAA;YACD,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;SACjC;aAAM;YACL,WAAW,CAAC,IAAI,CACd,CAAC,SAAS,GAAG;gBACX,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBACnB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM;gBACxB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM;aACzB,CAAC,CACH,CAAA;SACF;QACD,YAAY,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;KACnE;IAED,OAAO,WAAW,CAAA;AACpB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAoB;IACnD,IAAI,CAAC,MAAM,EAAE;QACX,OAAM;KACP;IAED,IAAI,MAAM,CAAC,OAAO,EAAE;QAClB,8BAA8B;QAC9B,IAAI,OAAO,YAAY,KAAK,WAAW,EAAE;YACvC,MAAM,IAAI,YAAY,CAAC,SAAS,EAAE,YAAY,CAAC,CAAA;SAChD;aAAM;YACL,MAAM,CAAC,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAA;YACnC,CAAC,CAAC,IAAI,GAAG,aAAa,CAAA;YACtB,MAAM,CAAC,CAAA;SACR;KACF;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAoB;IACxD,MAAM,OAAO,CAAC,OAAO,EAAE,CAAA;IACvB,gBAAgB,CAAC,MAAM,CAAC,CAAA;AAC1B,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@gmod/bbi",
-  "version": "1.0.33",
+  "version": "2.0.0",
   "description": "Parser for BigWig/BigBed files",
   "license": "MIT",
   "repository": "GMOD/bbi-js",
@@ -16,7 +16,8 @@
   },
   "files": [
     "dist",
-    "esm"
+    "esm",
+    "src"
   ],
   "scripts": {
     "test": "jest",
@@ -24,8 +25,8 @@
     "lint": "eslint --report-unused-disable-directives --max-warnings 0 --ext .js,.ts src",
     "clean": "rimraf dist esm",
     "prebuild": "npm run clean",
-    "build:esm": "tsc --target es2018 --outDir esm",
-    "build:es5": "tsc --target es5 --outDir dist",
+    "build:esm": "tsc --target es2018 --outDir esm --module es2020",
+    "build:es5": "tsc --target es5 --outDir dist --module commonjs",
     "build": "npm run build:esm && npm run build:es5",
     "preversion": "npm run lint && npm test && npm run build",
     "version": "standard-changelog && git add CHANGELOG.md",
@@ -40,21 +41,19 @@
     "genomics"
   ],
   "dependencies": {
-    "@gmod/binary-parser": "^1.4.0",
     "abortable-promise-cache": "^1.4.1",
-    "buffer-crc32": "^0.2.13",
-    "es6-promisify": "^6.0.1",
+    "binary-parser": "^2.1.0",
     "generic-filehandle": "^2.0.0",
-    "pako": "^1.0.0",
+    "pako": "^2.0.0",
     "quick-lru": "^4.0.0",
     "rxjs": "^6.5.2"
   },
   "devDependencies": {
     "@gmod/bed": "^2.0.0",
-    "@types/jest": "^27.0.3",
+    "@types/jest": "^28.1.4",
     "@types/long": "^4.0.0",
-    "@types/node": "^12.0.2",
-    "@types/pako": "^1.0.3",
+    "@types/node": "^18.0.0",
+    "@types/pako": "^2.0.0",
     "@typescript-eslint/eslint-plugin": "^5.10.0",
     "@typescript-eslint/parser": "^5.10.0",
     "cross-fetch": "^3.0.2",
@@ -62,11 +61,12 @@
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-import": "^2.25.3",
     "eslint-plugin-prettier": "^4.0.0",
-    "jest": "^27.4.3",
+    "jest": "^28.1.2",
+    "jest-environment-jsdom": "^28.1.2",
     "prettier": "^2.5.1",
     "rimraf": "^3.0.2",
     "standard-changelog": "^2.0.11",
-    "ts-jest": "^27.0.7",
+    "ts-jest": "^28.0.5",
     "typescript": "^4.5.2"
   },
   "publishConfig": {

package/src/bbi.ts

@@ -0,0 +1,375 @@
+import { Parser } from 'binary-parser'
+import { LocalFile, RemoteFile, GenericFilehandle } from 'generic-filehandle'
+import { Observable, Observer } from 'rxjs'
+import { reduce } from 'rxjs/operators'
+import { BlockView } from './blockView'
+
+const BIG_WIG_MAGIC = -2003829722
+const BIG_BED_MAGIC = -2021002517
+
+export interface Feature {
+  start: number
+  end: number
+  score: number
+  rest?: string // for bigbed line
+  minScore?: number // for summary line
+  maxScore?: number // for summary line
+  summary?: boolean // is summary line
+  uniqueId?: string // for bigbed contains uniqueId calculated from file offset
+  field?: number // used in bigbed searching
+}
+interface Statistics {
+  scoreSum: number
+  basesCovered: number
+  scoreSumSquares: number
+}
+
+interface RefInfo {
+  name: string
+  id: number
+  length: number
+}
+export interface Header {
+  autoSql: string
+  totalSummary: Statistics
+  zoomLevels: any
+  unzoomedIndexOffset: number
+  unzoomedDataOffset: number
+  definedFieldCount: number
+  uncompressBufSize: number
+  chromTreeOffset: number
+  fileSize: number
+  extHeaderOffset: number
+  isBigEndian: boolean
+  fileType: string
+  refsByName: { [key: string]: number }
+  refsByNumber: { [key: number]: RefInfo }
+}
+
+/* get the compiled parsers for different sections of the bigwig file
+ *
+ * @param isBE - is big endian, typically false
+ * @return an object with compiled parsers
+ */
+function getParsers(isBE: boolean) {
+  const le = isBE ? 'big' : 'little'
+  const headerParser = new Parser()
+    .endianess(le)
+    .int32('magic')
+    .uint16('version')
+    .uint16('numZoomLevels')
+    .uint64('chromTreeOffset')
+    .uint64('unzoomedDataOffset')
+    .uint64('unzoomedIndexOffset')
+    .uint16('fieldCount')
+    .uint16('definedFieldCount')
+    .uint64('asOffset') // autoSql offset, used in bigbed
+    .uint64('totalSummaryOffset')
+    .uint32('uncompressBufSize')
+    .uint64('extHeaderOffset') // name index offset, used in bigbed
+    .array('zoomLevels', {
+      length: 'numZoomLevels',
+      type: new Parser()
+        .endianess(le)
+        .uint32('reductionLevel')
+        .uint32('reserved')
+        .uint64('dataOffset')
+        .uint64('indexOffset'),
+    })
+
+  const totalSummaryParser = new Parser()
+    .endianess(le)
+    .uint64('basesCovered')
+    .doublele('scoreMin')
+    .doublele('scoreMax')
+    .doublele('scoreSum')
+    .doublele('scoreSumSquares')
+
+  const chromTreeParser = new Parser()
+    .endianess(le)
+    .uint32('magic')
+    .uint32('blockSize')
+    .uint32('keySize')
+    .uint32('valSize')
+    .uint64('itemCount')
+
+  const isLeafNode = new Parser()
+    .endianess(le)
+    .uint8('isLeafNode')
+    .skip(1)
+    .uint16('cnt')
+    .saveOffset('offset')
+
+  return {
+    chromTreeParser,
+    totalSummaryParser,
+    headerParser,
+    isLeafNode,
+  }
+}
+
+export interface RequestOptions {
+  signal?: AbortSignal
+  headers?: Record<string, string>
+  [key: string]: unknown
+}
+
+export abstract class BBI {
+  protected bbi: GenericFilehandle
+
+  private headerP?: Promise<Header>
+
+  protected renameRefSeqs: (a: string) => string
+
+  /* fetch and parse header information from a bigwig or bigbed file
+   * @param abortSignal - abort the operation, can be null
+   * @return a Header object
+   */
+  public getHeader(opts: RequestOptions | AbortSignal = {}) {
+    const options = 'aborted' in opts ? { signal: opts as AbortSignal } : opts
+    if (!this.headerP) {
+      this.headerP = this._getHeader(options).catch(e => {
+        this.headerP = undefined
+        throw e
+      })
+    }
+    return this.headerP
+  }
+
+  /*
+   * @param filehandle - a filehandle from generic-filehandle or implementing something similar to the node10 fs.promises API
+   * @param path - a Local file path as a string
+   * @param url - a URL string
+   * @param renameRefSeqs - an optional method to rename the internal reference sequences using a mapping function
+   */
+  public constructor(
+    options: {
+      filehandle?: GenericFilehandle
+      path?: string
+      url?: string
+      renameRefSeqs?: (a: string) => string
+    } = {},
+  ) {
+    const { filehandle, renameRefSeqs = s => s, path, url } = options
+    this.renameRefSeqs = renameRefSeqs
+    if (filehandle) {
+      this.bbi = filehandle
+    } else if (url) {
+      this.bbi = new RemoteFile(url)
+    } else if (path) {
+      this.bbi = new LocalFile(path)
+    } else {
+      throw new Error('no file given')
+    }
+  }
+
+  private async _getHeader(opts: RequestOptions) {
+    const header = await this._getMainHeader(opts)
+    const chroms = await this._readChromTree(header, opts)
+    return { ...header, ...chroms }
+  }
+
+  private async _getMainHeader(
+    opts: RequestOptions,
+    requestSize = 2000,
+  ): Promise<Header> {
+    const { buffer } = await this.bbi.read(
+      Buffer.alloc(requestSize),
+      0,
+      requestSize,
+      0,
+      opts,
+    )
+    const isBigEndian = this._isBigEndian(buffer)
+    const ret = getParsers(isBigEndian)
+    const header = ret.headerParser.parse(buffer)
+    const { magic, asOffset, totalSummaryOffset } = header
+    header.fileType = magic === BIG_BED_MAGIC ? 'bigbed' : 'bigwig'
+    if (asOffset > requestSize || totalSummaryOffset > requestSize) {
+      return this._getMainHeader(opts, requestSize * 2)
+    }
+    if (asOffset) {
+      const off = Number(header.asOffset)
+      header.autoSql = buffer
+        .subarray(off, buffer.indexOf(0, off))
+        .toString('utf8')
+    }
+    if (header.totalSummaryOffset > requestSize) {
+      return this._getMainHeader(opts, requestSize * 2)
+    }
+    if (header.totalSummaryOffset) {
+      const tail = buffer.subarray(Number(header.totalSummaryOffset))
+      header.totalSummary = ret.totalSummaryParser.parse(tail)
+    }
+    return { ...header, isBigEndian }
+  }
+
+  private _isBigEndian(buffer: Buffer): boolean {
+    let ret = buffer.readInt32LE(0)
+    if (ret === BIG_WIG_MAGIC || ret === BIG_BED_MAGIC) {
+      return false
+    }
+    ret = buffer.readInt32BE(0)
+    if (ret === BIG_WIG_MAGIC || ret === BIG_BED_MAGIC) {
+      return true
+    }
+    throw new Error('not a BigWig/BigBed file')
+  }
+
+  // todo: add progress if long running
+  private async _readChromTree(header: Header, opts: { signal?: AbortSignal }) {
+    const isBE = header.isBigEndian
+    const le = isBE ? 'big' : 'little'
+    const refsByNumber: {
+      [key: number]: { name: string; id: number; length: number }
+    } = []
+    const refsByName: { [key: string]: number } = {}
+
+    let unzoomedDataOffset = Number(header.unzoomedDataOffset)
+    const chromTreeOffset = Number(header.chromTreeOffset)
+    while (unzoomedDataOffset % 4 !== 0) {
+      unzoomedDataOffset += 1
+    }
+    const off = unzoomedDataOffset - chromTreeOffset
+    const { buffer } = await this.bbi.read(
+      Buffer.alloc(off),
+      0,
+      off,
+      Number(chromTreeOffset),
+      opts,
+    )
+
+    const p = getParsers(isBE)
+    const { keySize } = p.chromTreeParser.parse(buffer)
+    const leafNodeParser = new Parser()
+      .endianess(le)
+      .string('key', { stripNull: true, length: keySize })
+      .uint32('refId')
+      .uint32('refSize')
+      .saveOffset('offset')
+    const nonleafNodeParser = new Parser()
+      .endianess(le)
+      .skip(keySize)
+      .uint64('childOffset')
+      .saveOffset('offset')
+    const rootNodeOffset = 32
+    const bptReadNode = async (currentOffset: number) => {
+      let offset = currentOffset
+      if (offset >= buffer.length) {
+        throw new Error('reading beyond end of buffer')
+      }
+      const ret = p.isLeafNode.parse(buffer.subarray(offset))
+      const { isLeafNode, cnt } = ret
+      offset += ret.offset
+      if (isLeafNode) {
+        for (let n = 0; n < cnt; n += 1) {
+          const leafRet = leafNodeParser.parse(buffer.subarray(offset))
+          offset += leafRet.offset
+          const { key, refId, refSize } = leafRet
+          const refRec = { name: key, id: refId, length: refSize }
+          refsByName[this.renameRefSeqs(key)] = refId
+          refsByNumber[refId] = refRec
+        }
+      } else {
+        // parse index node
+        const nextNodes = []
+        for (let n = 0; n < cnt; n += 1) {
+          const nonleafRet = nonleafNodeParser.parse(buffer.subarray(offset))
+          const { childOffset } = nonleafRet
+          offset += nonleafRet.offset
+          nextNodes.push(
+            bptReadNode(Number(childOffset) - Number(chromTreeOffset)),
+          )
+        }
+        await Promise.all(nextNodes)
+      }
+    }
+    await bptReadNode(rootNodeOffset)
+    return {
+      refsByName,
+      refsByNumber,
+    }
+  }
+
+  /*
+   * fetches the "unzoomed" view of the bigwig data. this is the default for bigbed
+   * @param abortSignal - a signal to optionally abort this operation
+   */
+  protected async getUnzoomedView(opts: RequestOptions): Promise<BlockView> {
+    const {
+      unzoomedIndexOffset,
+      refsByName,
+      uncompressBufSize,
+      isBigEndian,
+      fileType,
+    } = await this.getHeader(opts)
+    return new BlockView(
+      this.bbi,
+      refsByName,
+      unzoomedIndexOffset,
+      isBigEndian,
+      uncompressBufSize > 0,
+      fileType,
+    )
+  }
+
+  /*
+   * abstract method - get the view for a given scale
+   */
+  protected abstract getView(
+    scale: number,
+    opts: RequestOptions,
+  ): Promise<BlockView>
+
+  /**
+   * Gets features from a BigWig file
+   *
+   * @param refName - The chromosome name
+   * @param start - The start of a region
+   * @param end - The end of a region
+   * @param opts - An object containing basesPerSpan (e.g. pixels per basepair) or scale used to infer the zoomLevel to use
+   */
+  public async getFeatureStream(
+    refName: string,
+    start: number,
+    end: number,
+    opts: RequestOptions & { scale?: number; basesPerSpan?: number } = {
+      scale: 1,
+    },
+  ): Promise<Observable<Feature[]>> {
+    await this.getHeader(opts)
+    const chrName = this.renameRefSeqs(refName)
+    let view: BlockView
+
+    if (opts.basesPerSpan) {
+      view = await this.getView(1 / opts.basesPerSpan, opts)
+    } else if (opts.scale) {
+      view = await this.getView(opts.scale, opts)
+    } else {
+      view = await this.getView(1, opts)
+    }
+
+    if (!view) {
+      throw new Error('unable to get block view for data')
+    }
+    return new Observable((observer: Observer<Feature[]>): void => {
+      view.readWigData(chrName, start, end, observer, opts)
+    })
+  }
+
+  public async getFeatures(
+    refName: string,
+    start: number,
+    end: number,
+    opts: RequestOptions & { scale?: number; basesPerSpan?: number } = {
+      scale: 1,
+    },
+  ): Promise<Feature[]> {
+    const ob = await this.getFeatureStream(refName, start, end, opts)
+
+    const ret = await ob
+      .pipe(reduce((acc, curr) => acc.concat(curr)))
+      .toPromise()
+    return ret || []
+  }
+}