@gmod/bbi 1.0.30 → 1.0.31

package/esm/bbi.d.ts

@@ -0,0 +1,84 @@
+import { GenericFilehandle } from 'generic-filehandle';
+import { Observable } from 'rxjs';
+import { BlockView } from './blockView';
+export interface Feature {
+    start: number;
+    end: number;
+    score: number;
+    rest?: string;
+    minScore?: number;
+    maxScore?: number;
+    summary?: boolean;
+    uniqueId?: string;
+    field?: number;
+}
+interface Statistics {
+    scoreSum: number;
+    basesCovered: number;
+    scoreSumSquares: number;
+}
+interface RefInfo {
+    name: string;
+    id: number;
+    length: number;
+}
+export interface Header {
+    autoSql: string;
+    totalSummary: Statistics;
+    zoomLevels: any;
+    unzoomedIndexOffset: number;
+    unzoomedDataOffset: number;
+    definedFieldCount: number;
+    uncompressBufSize: number;
+    chromTreeOffset: number;
+    fileSize: number;
+    extHeaderOffset: number;
+    isBigEndian: boolean;
+    fileType: string;
+    refsByName: {
+        [key: string]: number;
+    };
+    refsByNumber: {
+        [key: number]: RefInfo;
+    };
+}
+export interface RequestOptions {
+    signal?: AbortSignal;
+    headers?: Record<string, string>;
+    [key: string]: unknown;
+}
+export declare abstract class BBI {
+    protected bbi: GenericFilehandle;
+    protected headerCache: any;
+    protected renameRefSeqs: (a: string) => string;
+    getHeader(opts?: RequestOptions | AbortSignal): any;
+    constructor(options?: {
+        filehandle?: GenericFilehandle;
+        path?: string;
+        url?: string;
+        renameRefSeqs?: (a: string) => string;
+    });
+    private _getHeader;
+    private _getMainHeader;
+    private _isBigEndian;
+    private _readChromTree;
+    protected getUnzoomedView(opts: RequestOptions): Promise<BlockView>;
+    protected abstract getView(scale: number, opts: RequestOptions): Promise<BlockView>;
+    /**
+     * Gets features from a BigWig file
+     *
+     * @param refName - The chromosome name
+     * @param start - The start of a region
+     * @param end - The end of a region
+     * @param opts - An object containing basesPerSpan (e.g. pixels per basepair) or scale used to infer the zoomLevel to use
+     */
+    getFeatureStream(refName: string, start: number, end: number, opts?: RequestOptions & {
+        scale?: number;
+        basesPerSpan?: number;
+    }): Promise<Observable<Feature[]>>;
+    getFeatures(refName: string, start: number, end: number, opts?: RequestOptions & {
+        scale?: number;
+        basesPerSpan?: number;
+    }): Promise<Feature[]>;
+}
+export {};

package/esm/bbi.js

@@ -0,0 +1,259 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BBI = void 0;
+const binary_parser_1 = require("@gmod/binary-parser");
+const generic_filehandle_1 = require("generic-filehandle");
+const rxjs_1 = require("rxjs");
+const operators_1 = require("rxjs/operators");
+const abortable_promise_cache_1 = __importDefault(require("abortable-promise-cache"));
+const quick_lru_1 = __importDefault(require("quick-lru"));
+const blockView_1 = require("./blockView");
+const BIG_WIG_MAGIC = -2003829722;
+const BIG_BED_MAGIC = -2021002517;
+/* get the compiled parsers for different sections of the bigwig file
+ *
+ * @param isBE - is big endian, typically false
+ * @return an object with compiled parsers
+ */
+function getParsers(isBE) {
+    const le = isBE ? 'big' : 'little';
+    const headerParser = new binary_parser_1.Parser()
+        .endianess(le)
+        .int32('magic')
+        .uint16('version')
+        .uint16('numZoomLevels')
+        .uint64('chromTreeOffset')
+        .uint64('unzoomedDataOffset')
+        .uint64('unzoomedIndexOffset')
+        .uint16('fieldCount')
+        .uint16('definedFieldCount')
+        .uint64('asOffset') // autoSql offset, used in bigbed
+        .uint64('totalSummaryOffset')
+        .uint32('uncompressBufSize')
+        .uint64('extHeaderOffset') // name index offset, used in bigbed
+        .array('zoomLevels', {
+        length: 'numZoomLevels',
+        type: new binary_parser_1.Parser()
+            .uint32('reductionLevel')
+            .uint32('reserved')
+            .uint64('dataOffset')
+            .uint64('indexOffset'),
+    });
+    const totalSummaryParser = new binary_parser_1.Parser()
+        .endianess(le)
+        .uint64('basesCovered')
+        .double('scoreMin')
+        .double('scoreMax')
+        .double('scoreSum')
+        .double('scoreSumSquares');
+    const chromTreeParser = new binary_parser_1.Parser()
+        .endianess(le)
+        .uint32('magic')
+        .uint32('blockSize')
+        .uint32('keySize')
+        .uint32('valSize')
+        .uint64('itemCount');
+    const isLeafNode = new binary_parser_1.Parser()
+        .endianess(le)
+        .uint8('isLeafNode')
+        .skip(1)
+        .uint16('cnt');
+    return {
+        chromTreeParser,
+        totalSummaryParser,
+        headerParser,
+        isLeafNode,
+    };
+}
+class BBI {
+    /*
+     * @param filehandle - a filehandle from generic-filehandle or implementing something similar to the node10 fs.promises API
+     * @param path - a Local file path as a string
+     * @param url - a URL string
+     * @param renameRefSeqs - an optional method to rename the internal reference sequences using a mapping function
+     */
+    constructor(options = {}) {
+        this.headerCache = new abortable_promise_cache_1.default({
+            cache: new quick_lru_1.default({ maxSize: 1 }),
+            fill: async (params, signal) => {
+                return this._getHeader({ ...params, signal });
+            },
+        });
+        const { filehandle, renameRefSeqs, path, url } = options;
+        this.renameRefSeqs = renameRefSeqs || ((s) => s);
+        if (filehandle) {
+            this.bbi = filehandle;
+        }
+        else if (url) {
+            this.bbi = new generic_filehandle_1.RemoteFile(url);
+        }
+        else if (path) {
+            this.bbi = new generic_filehandle_1.LocalFile(path);
+        }
+        else {
+            throw new Error('no file given');
+        }
+    }
+    /* fetch and parse header information from a bigwig or bigbed file
+     * @param abortSignal - abort the operation, can be null
+     * @return a Header object
+     */
+    getHeader(opts = {}) {
+        const options = 'aborted' in opts ? { signal: opts } : opts;
+        return this.headerCache.get(JSON.stringify(options), options, options.signal);
+    }
+    async _getHeader(opts) {
+        const header = await this._getMainHeader(opts);
+        const chroms = await this._readChromTree(header, opts);
+        return { ...header, ...chroms };
+    }
+    async _getMainHeader(opts, requestSize = 2000) {
+        const { buffer } = await this.bbi.read(Buffer.alloc(requestSize), 0, requestSize, 0, opts);
+        const isBigEndian = this._isBigEndian(buffer);
+        const ret = getParsers(isBigEndian);
+        const header = ret.headerParser.parse(buffer).result;
+        header.fileType = header.magic === BIG_BED_MAGIC ? 'bigbed' : 'bigwig';
+        if (header.asOffset > requestSize ||
+            header.totalSummaryOffset > requestSize) {
+            return this._getMainHeader(opts, requestSize * 2);
+        }
+        if (header.asOffset) {
+            header.autoSql = buffer
+                .slice(header.asOffset, buffer.indexOf(0, header.asOffset))
+                .toString('utf8');
+        }
+        if (header.totalSummaryOffset > requestSize) {
+            return this._getMainHeader(opts, requestSize * 2);
+        }
+        if (header.totalSummaryOffset) {
+            const tail = buffer.slice(header.totalSummaryOffset);
+            header.totalSummary = ret.totalSummaryParser.parse(tail).result;
+        }
+        return { ...header, isBigEndian };
+    }
+    _isBigEndian(buffer) {
+        let ret = buffer.readInt32LE(0);
+        if (ret === BIG_WIG_MAGIC || ret === BIG_BED_MAGIC) {
+            return false;
+        }
+        ret = buffer.readInt32BE(0);
+        if (ret === BIG_WIG_MAGIC || ret === BIG_BED_MAGIC) {
+            return true;
+        }
+        throw new Error('not a BigWig/BigBed file');
+    }
+    // todo: add progress if long running
+    async _readChromTree(header, opts) {
+        const isBE = header.isBigEndian;
+        const le = isBE ? 'big' : 'little';
+        const refsByNumber = [];
+        const refsByName = {};
+        const { chromTreeOffset } = header;
+        let { unzoomedDataOffset } = header;
+        while (unzoomedDataOffset % 4 !== 0) {
+            unzoomedDataOffset += 1;
+        }
+        const { buffer: data } = await this.bbi.read(Buffer.alloc(unzoomedDataOffset - chromTreeOffset), 0, unzoomedDataOffset - chromTreeOffset, chromTreeOffset, opts);
+        const p = getParsers(isBE);
+        const { keySize } = p.chromTreeParser.parse(data).result;
+        const leafNodeParser = new binary_parser_1.Parser()
+            .endianess(le)
+            .string('key', { stripNull: true, length: keySize })
+            .uint32('refId')
+            .uint32('refSize');
+        const nonleafNodeParser = new binary_parser_1.Parser()
+            .endianess(le)
+            .skip(keySize)
+            .uint64('childOffset');
+        const rootNodeOffset = 32;
+        const bptReadNode = async (currentOffset) => {
+            let offset = currentOffset;
+            if (offset >= data.length) {
+                throw new Error('reading beyond end of buffer');
+            }
+            const ret = p.isLeafNode.parse(data.slice(offset));
+            const { isLeafNode, cnt } = ret.result;
+            offset += ret.offset;
+            if (isLeafNode) {
+                for (let n = 0; n < cnt; n += 1) {
+                    const leafRet = leafNodeParser.parse(data.slice(offset));
+                    offset += leafRet.offset;
+                    const { key, refId, refSize } = leafRet.result;
+                    const refRec = { name: key, id: refId, length: refSize };
+                    refsByName[this.renameRefSeqs(key)] = refId;
+                    refsByNumber[refId] = refRec;
+                }
+            }
+            else {
+                // parse index node
+                const nextNodes = [];
+                for (let n = 0; n < cnt; n += 1) {
+                    const nonleafRet = nonleafNodeParser.parse(data.slice(offset));
+                    let { childOffset } = nonleafRet.result;
+                    offset += nonleafRet.offset;
+                    childOffset -= chromTreeOffset;
+                    nextNodes.push(bptReadNode(childOffset));
+                }
+                await Promise.all(nextNodes);
+            }
+        };
+        await bptReadNode(rootNodeOffset);
+        return {
+            refsByName,
+            refsByNumber,
+        };
+    }
+    /*
+     * fetches the "unzoomed" view of the bigwig data. this is the default for bigbed
+     * @param abortSignal - a signal to optionally abort this operation
+     */
+    async getUnzoomedView(opts) {
+        const { unzoomedIndexOffset, zoomLevels, refsByName, uncompressBufSize, isBigEndian, fileType, } = await this.getHeader(opts);
+        const nzl = zoomLevels[0];
+        const cirLen = nzl ? nzl.dataOffset - unzoomedIndexOffset : 4000;
+        return new blockView_1.BlockView(this.bbi, refsByName, unzoomedIndexOffset, cirLen, isBigEndian, uncompressBufSize > 0, fileType);
+    }
+    /**
+     * Gets features from a BigWig file
+     *
+     * @param refName - The chromosome name
+     * @param start - The start of a region
+     * @param end - The end of a region
+     * @param opts - An object containing basesPerSpan (e.g. pixels per basepair) or scale used to infer the zoomLevel to use
+     */
+    async getFeatureStream(refName, start, end, opts = {
+        scale: 1,
+    }) {
+        await this.getHeader(opts);
+        const chrName = this.renameRefSeqs(refName);
+        let view;
+        if (opts.basesPerSpan) {
+            view = await this.getView(1 / opts.basesPerSpan, opts);
+        }
+        else if (opts.scale) {
+            view = await this.getView(opts.scale, opts);
+        }
+        else {
+            view = await this.getView(1, opts);
+        }
+        if (!view) {
+            throw new Error('unable to get block view for data');
+        }
+        return new rxjs_1.Observable((observer) => {
+            view.readWigData(chrName, start, end, observer, opts);
+        });
+    }
+    async getFeatures(refName, start, end, opts = {
+        scale: 1,
+    }) {
+        const ob = await this.getFeatureStream(refName, start, end, opts);
+        const ret = await ob
+            .pipe((0, operators_1.reduce)((acc, curr) => acc.concat(curr)))
+            .toPromise();
+        return ret || [];
+    }
+}
+exports.BBI = BBI;

package/esm/bigbed.d.ts

@@ -0,0 +1,12 @@
+import { BBI, Feature, RequestOptions } from './bbi';
+import { BlockView } from './blockView';
+export declare function filterUndef<T>(ts: (T | undefined)[]): T[];
+export declare class BigBed extends BBI {
+    readIndicesCache: any;
+    constructor(opts?: any);
+    readIndices(opts?: AbortSignal | RequestOptions): any;
+    protected getView(scale: number, opts: RequestOptions): Promise<BlockView>;
+    private _readIndices;
+    private searchExtraIndexBlocks;
+    searchExtraIndex(name: string, opts?: RequestOptions): Promise<Feature[]>;
+}

package/esm/bigbed.js

@@ -0,0 +1,182 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigBed = exports.filterUndef = void 0;
+const binary_parser_1 = require("@gmod/binary-parser");
+const rxjs_1 = require("rxjs");
+const operators_1 = require("rxjs/operators");
+const abortable_promise_cache_1 = __importDefault(require("abortable-promise-cache"));
+const quick_lru_1 = __importDefault(require("quick-lru"));
+const bbi_1 = require("./bbi");
+function filterUndef(ts) {
+    return ts.filter((t) => !!t);
+}
+exports.filterUndef = filterUndef;
+class BigBed extends bbi_1.BBI {
+    constructor(opts) {
+        super(opts);
+        this.readIndicesCache = new abortable_promise_cache_1.default({
+            cache: new quick_lru_1.default({ maxSize: 1 }),
+            fill: async (args, signal) => {
+                return this._readIndices({ ...args, signal });
+            },
+        });
+    }
+    readIndices(opts = {}) {
+        const options = 'aborted' in opts ? { signal: opts } : opts;
+        return this.readIndicesCache.get(JSON.stringify(options), options, options.signal);
+    }
+    /*
+     * retrieve unzoomed view for any scale
+     * @param scale - unused
+     * @param abortSignal - an optional AbortSignal to kill operation
+     * @return promise for a BlockView
+     */
+    async getView(scale, opts) {
+        return this.getUnzoomedView(opts);
+    }
+    /*
+     * parse the bigbed extraIndex fields
+     * @param abortSignal to abort operation
+     * @return a Promise for an array of Index data structure since there can be multiple extraIndexes in a bigbed, see bedToBigBed documentation
+     */
+    async _readIndices(opts) {
+        const { extHeaderOffset, isBigEndian } = await this.getHeader(opts);
+        const { buffer: data } = await this.bbi.read(Buffer.alloc(64), 0, 64, extHeaderOffset);
+        const le = isBigEndian ? 'big' : 'little';
+        const ret = new binary_parser_1.Parser()
+            .endianess(le)
+            .uint16('size')
+            .uint16('count')
+            .uint64('offset')
+            .parse(data).result;
+        const { count, offset } = ret;
+        // no extra index is defined if count==0
+        if (count === 0) {
+            return [];
+        }
+        const blocklen = 20;
+        const len = blocklen * count;
+        const { buffer } = await this.bbi.read(Buffer.alloc(len), 0, len, offset);
+        const extParser = new binary_parser_1.Parser()
+            .endianess(le)
+            .int16('type')
+            .int16('fieldcount')
+            .uint64('offset')
+            .skip(4)
+            .int16('field');
+        const indices = [];
+        for (let i = 0; i < count; i += 1) {
+            indices.push(extParser.parse(buffer.slice(i * blocklen)).result);
+        }
+        return indices;
+    }
+    /*
+     * perform a search in the bigbed extraIndex to find which blocks in the bigbed data to look for the
+     * actual feature data
+     *
+     * @param name - the name to search for
+     * @param opts - a SearchOptions argument with optional signal
+     * @return a Promise for an array of bigbed block Loc entries
+     */
+    async searchExtraIndexBlocks(name, opts = {}) {
+        const { isBigEndian } = await this.getHeader(opts);
+        const indices = await this.readIndices(opts);
+        if (!indices.length) {
+            return [];
+        }
+        const locs = indices.map(async (index) => {
+            const { offset, field } = index;
+            const { buffer: data } = await this.bbi.read(Buffer.alloc(32), 0, 32, offset, opts);
+            const p = new binary_parser_1.Parser()
+                .endianess(isBigEndian ? 'big' : 'little')
+                .int32('magic')
+                .int32('blockSize')
+                .int32('keySize')
+                .int32('valSize')
+                .uint64('itemCount');
+            const { blockSize, keySize, valSize } = p.parse(data).result;
+            const bpt = new binary_parser_1.Parser()
+                .endianess(isBigEndian ? 'big' : 'little')
+                .int8('nodeType')
+                .skip(1)
+                .int16('cnt')
+                .choice({
+                tag: 'nodeType',
+                choices: {
+                    0: new binary_parser_1.Parser().array('leafkeys', {
+                        length: 'cnt',
+                        type: new binary_parser_1.Parser()
+                            .string('key', { length: keySize, stripNull: true })
+                            .uint64('offset'),
+                    }),
+                    1: new binary_parser_1.Parser().array('keys', {
+                        length: 'cnt',
+                        type: new binary_parser_1.Parser()
+                            .string('key', { length: keySize, stripNull: true })
+                            .uint64('offset')
+                            .uint32('length')
+                            .uint32('reserved'),
+                    }),
+                },
+            });
+            const bptReadNode = async (nodeOffset) => {
+                const len = 4 + blockSize * (keySize + valSize);
+                const { buffer } = await this.bbi.read(Buffer.alloc(len), 0, len, nodeOffset, opts);
+                const node = bpt.parse(buffer).result;
+                if (node.leafkeys) {
+                    let lastOffset;
+                    for (let i = 0; i < node.leafkeys.length; i += 1) {
+                        const { key } = node.leafkeys[i];
+                        if (name.localeCompare(key) < 0 && lastOffset) {
+                            return bptReadNode(lastOffset);
+                        }
+                        lastOffset = node.leafkeys[i].offset;
+                    }
+                    return bptReadNode(lastOffset);
+                }
+                for (let i = 0; i < node.keys.length; i += 1) {
+                    if (node.keys[i].key === name) {
+                        return { ...node.keys[i], field };
+                    }
+                }
+                return undefined;
+            };
+            const rootNodeOffset = 32;
+            return bptReadNode(offset + rootNodeOffset);
+        });
+        return filterUndef(await Promise.all(locs));
+    }
+    /*
+     * retrieve the features from the bigbed data that were found through the lookup of the extraIndex
+     * note that there can be multiple extraIndex, see the BigBed specification and the -extraIndex argument to bedToBigBed
+     *
+     * @param name - the name to search for
+     * @param opts - a SearchOptions argument with optional signal
+     * @return a Promise for an array of Feature
+     */
+    async searchExtraIndex(name, opts = {}) {
+        const blocks = await this.searchExtraIndexBlocks(name, opts);
+        if (!blocks.length) {
+            return [];
+        }
+        const view = await this.getUnzoomedView(opts);
+        const res = blocks.map(block => {
+            return new rxjs_1.Observable((observer) => {
+                view.readFeatures(observer, [block], opts);
+            }).pipe((0, operators_1.reduce)((acc, curr) => acc.concat(curr)), (0, operators_1.map)(x => {
+                for (let i = 0; i < x.length; i += 1) {
+                    x[i].field = block.field;
+                }
+                return x;
+            }));
+        });
+        const ret = await (0, rxjs_1.merge)(...res).toPromise();
+        return ret.filter((f) => {
+            return f.rest.split('\t')[f.field - 3] === name;
+        });
+    }
+}
+exports.BigBed = BigBed;

package/esm/bigwig.d.ts

@@ -0,0 +1,13 @@
+import { BlockView } from './blockView';
+import { BBI, RequestOptions } from './bbi';
+export declare class BigWig extends BBI {
+    /**
+     * Retrieves a BlockView of a specific zoomLevel
+     *
+     * @param refName - The chromosome name
+     * @param start - The start of a region
+     * @param end - The end of a region
+     * @param opts - An object containing basesPerSpan (e.g. pixels per basepair) or scale used to infer the zoomLevel to use
+     */
+    protected getView(scale: number, opts: RequestOptions): Promise<BlockView>;
+}

package/esm/bigwig.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigWig = void 0;
+const blockView_1 = require("./blockView");
+const bbi_1 = require("./bbi");
+class BigWig extends bbi_1.BBI {
+    /**
+     * Retrieves a BlockView of a specific zoomLevel
+     *
+     * @param refName - The chromosome name
+     * @param start - The start of a region
+     * @param end - The end of a region
+     * @param opts - An object containing basesPerSpan (e.g. pixels per basepair) or scale used to infer the zoomLevel to use
+     */
+    async getView(scale, opts) {
+        const { zoomLevels, refsByName, fileSize, isBigEndian, uncompressBufSize } = await this.getHeader(opts);
+        const basesPerPx = 1 / scale;
+        let maxLevel = zoomLevels.length;
+        if (!fileSize) {
+            // if we don't know the file size, we can't fetch the highest zoom level :-(
+            maxLevel -= 1;
+        }
+        for (let i = maxLevel; i >= 0; i -= 1) {
+            const zh = zoomLevels[i];
+            if (zh && zh.reductionLevel <= 2 * basesPerPx) {
+                const indexLength = i < zoomLevels.length - 1
+                    ? zoomLevels[i + 1].dataOffset - zh.indexOffset
+                    : fileSize - 4 - zh.indexOffset;
+                return new blockView_1.BlockView(this.bbi, refsByName, zh.indexOffset, indexLength, isBigEndian, uncompressBufSize > 0, 'summary');
+            }
+        }
+        return this.getUnzoomedView(opts);
+    }
+}
+exports.BigWig = BigWig;

package/esm/blockView.d.ts

@@ -0,0 +1,42 @@
+import { Observer } from 'rxjs';
+import { GenericFilehandle } from 'generic-filehandle';
+import { Feature } from './bbi';
+interface CoordRequest {
+    chrId: number;
+    start: number;
+    end: number;
+}
+interface Options {
+    signal?: AbortSignal;
+    request?: CoordRequest;
+}
+/**
+ * View into a subset of the data in a BigWig file.
+ *
+ * Adapted by Robert Buels and Colin Diesh from bigwig.js in the Dalliance Genome
+ * Explorer by Thomas Down.
+ * @constructs
+ */
+export declare class BlockView {
+    private cirTreeOffset;
+    private cirTreeLength;
+    private bbi;
+    private isCompressed;
+    private isBigEndian;
+    private refsByName;
+    private blockType;
+    private cirTreePromise?;
+    private featureCache;
+    private leafParser;
+    private bigWigParser;
+    private bigBedParser;
+    private summaryParser;
+    constructor(bbi: GenericFilehandle, refsByName: any, cirTreeOffset: number, cirTreeLength: number, isBigEndian: boolean, isCompressed: boolean, blockType: string);
+    readWigData(chrName: string, start: number, end: number, observer: Observer<Feature[]>, opts: Options): Promise<void>;
+    private parseSummaryBlock;
+    private parseBigBedBlock;
+    private parseBigWigBlock;
+    private static coordFilter;
+    readFeatures(observer: Observer<Feature[]>, blocks: any, opts?: Options): Promise<void>;
+}
+export {};