@gmod/bbi 1.0.30 → 1.0.33

package/esm/bigbed.js

@@ -0,0 +1,182 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigBed = exports.filterUndef = void 0;
+const binary_parser_1 = require("@gmod/binary-parser");
+const rxjs_1 = require("rxjs");
+const operators_1 = require("rxjs/operators");
+const abortable_promise_cache_1 = __importDefault(require("abortable-promise-cache"));
+const quick_lru_1 = __importDefault(require("quick-lru"));
+const bbi_1 = require("./bbi");
+function filterUndef(ts) {
+    return ts.filter((t) => !!t);
+}
+exports.filterUndef = filterUndef;
+class BigBed extends bbi_1.BBI {
+    constructor(opts) {
+        super(opts);
+        this.readIndicesCache = new abortable_promise_cache_1.default({
+            cache: new quick_lru_1.default({ maxSize: 1 }),
+            fill: async (args, signal) => {
+                return this._readIndices({ ...args, signal });
+            },
+        });
+    }
+    readIndices(opts = {}) {
+        const options = 'aborted' in opts ? { signal: opts } : opts;
+        return this.readIndicesCache.get(JSON.stringify(options), options, options.signal);
+    }
+    /*
+     * retrieve unzoomed view for any scale
+     * @param scale - unused
+     * @param abortSignal - an optional AbortSignal to kill operation
+     * @return promise for a BlockView
+     */
+    async getView(scale, opts) {
+        return this.getUnzoomedView(opts);
+    }
+    /*
+     * parse the bigbed extraIndex fields
+     * @param abortSignal to abort operation
+     * @return a Promise for an array of Index data structure since there can be multiple extraIndexes in a bigbed, see bedToBigBed documentation
+     */
+    async _readIndices(opts) {
+        const { extHeaderOffset, isBigEndian } = await this.getHeader(opts);
+        const { buffer: data } = await this.bbi.read(Buffer.alloc(64), 0, 64, extHeaderOffset);
+        const le = isBigEndian ? 'big' : 'little';
+        const ret = new binary_parser_1.Parser()
+            .endianess(le)
+            .uint16('size')
+            .uint16('count')
+            .uint64('offset')
+            .parse(data).result;
+        const { count, offset } = ret;
+        // no extra index is defined if count==0
+        if (count === 0) {
+            return [];
+        }
+        const blocklen = 20;
+        const len = blocklen * count;
+        const { buffer } = await this.bbi.read(Buffer.alloc(len), 0, len, offset);
+        const extParser = new binary_parser_1.Parser()
+            .endianess(le)
+            .int16('type')
+            .int16('fieldcount')
+            .uint64('offset')
+            .skip(4)
+            .int16('field');
+        const indices = [];
+        for (let i = 0; i < count; i += 1) {
+            indices.push(extParser.parse(buffer.subarray(i * blocklen)).result);
+        }
+        return indices;
+    }
+    /*
+     * perform a search in the bigbed extraIndex to find which blocks in the bigbed data to look for the
+     * actual feature data
+     *
+     * @param name - the name to search for
+     * @param opts - a SearchOptions argument with optional signal
+     * @return a Promise for an array of bigbed block Loc entries
+     */
+    async searchExtraIndexBlocks(name, opts = {}) {
+        const { isBigEndian } = await this.getHeader(opts);
+        const indices = await this.readIndices(opts);
+        if (!indices.length) {
+            return [];
+        }
+        const locs = indices.map(async (index) => {
+            const { offset, field } = index;
+            const { buffer: data } = await this.bbi.read(Buffer.alloc(32), 0, 32, offset, opts);
+            const p = new binary_parser_1.Parser()
+                .endianess(isBigEndian ? 'big' : 'little')
+                .int32('magic')
+                .int32('blockSize')
+                .int32('keySize')
+                .int32('valSize')
+                .uint64('itemCount');
+            const { blockSize, keySize, valSize } = p.parse(data).result;
+            const bpt = new binary_parser_1.Parser()
+                .endianess(isBigEndian ? 'big' : 'little')
+                .int8('nodeType')
+                .skip(1)
+                .int16('cnt')
+                .choice({
+                tag: 'nodeType',
+                choices: {
+                    0: new binary_parser_1.Parser().array('leafkeys', {
+                        length: 'cnt',
+                        type: new binary_parser_1.Parser()
+                            .string('key', { length: keySize, stripNull: true })
+                            .uint64('offset'),
+                    }),
+                    1: new binary_parser_1.Parser().array('keys', {
+                        length: 'cnt',
+                        type: new binary_parser_1.Parser()
+                            .string('key', { length: keySize, stripNull: true })
+                            .uint64('offset')
+                            .uint32('length')
+                            .uint32('reserved'),
+                    }),
+                },
+            });
+            const bptReadNode = async (nodeOffset) => {
+                const len = 4 + blockSize * (keySize + valSize);
+                const { buffer } = await this.bbi.read(Buffer.alloc(len), 0, len, nodeOffset, opts);
+                const node = bpt.parse(buffer).result;
+                if (node.leafkeys) {
+                    let lastOffset;
+                    for (let i = 0; i < node.leafkeys.length; i += 1) {
+                        const { key } = node.leafkeys[i];
+                        if (name.localeCompare(key) < 0 && lastOffset) {
+                            return bptReadNode(lastOffset);
+                        }
+                        lastOffset = node.leafkeys[i].offset;
+                    }
+                    return bptReadNode(lastOffset);
+                }
+                for (let i = 0; i < node.keys.length; i += 1) {
+                    if (node.keys[i].key === name) {
+                        return { ...node.keys[i], field };
+                    }
+                }
+                return undefined;
+            };
+            const rootNodeOffset = 32;
+            return bptReadNode(offset + rootNodeOffset);
+        });
+        return filterUndef(await Promise.all(locs));
+    }
+    /*
+     * retrieve the features from the bigbed data that were found through the lookup of the extraIndex
+     * note that there can be multiple extraIndex, see the BigBed specification and the -extraIndex argument to bedToBigBed
+     *
+     * @param name - the name to search for
+     * @param opts - a SearchOptions argument with optional signal
+     * @return a Promise for an array of Feature
+     */
+    async searchExtraIndex(name, opts = {}) {
+        const blocks = await this.searchExtraIndexBlocks(name, opts);
+        if (!blocks.length) {
+            return [];
+        }
+        const view = await this.getUnzoomedView(opts);
+        const res = blocks.map(block => {
+            return new rxjs_1.Observable((observer) => {
+                view.readFeatures(observer, [block], opts);
+            }).pipe((0, operators_1.reduce)((acc, curr) => acc.concat(curr)), (0, operators_1.map)(x => {
+                for (let i = 0; i < x.length; i += 1) {
+                    x[i].field = block.field;
+                }
+                return x;
+            }));
+        });
+        const ret = await (0, rxjs_1.merge)(...res).toPromise();
+        return ret.filter((f) => {
+            return f.rest.split('\t')[f.field - 3] === name;
+        });
+    }
+}
+exports.BigBed = BigBed;

package/esm/bigwig.d.ts

@@ -0,0 +1,13 @@
+import { BlockView } from './blockView';
+import { BBI, RequestOptions } from './bbi';
+export declare class BigWig extends BBI {
+    /**
+     * Retrieves a BlockView of a specific zoomLevel
+     *
+     * @param refName - The chromosome name
+     * @param start - The start of a region
+     * @param end - The end of a region
+     * @param opts - An object containing basesPerSpan (e.g. pixels per basepair) or scale used to infer the zoomLevel to use
+     */
+    protected getView(scale: number, opts: RequestOptions): Promise<BlockView>;
+}

package/esm/bigwig.js

@@ -0,0 +1,35 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigWig = void 0;
+const blockView_1 = require("./blockView");
+const bbi_1 = require("./bbi");
+class BigWig extends bbi_1.BBI {
+    /**
+     * Retrieves a BlockView of a specific zoomLevel
+     *
+     * @param refName - The chromosome name
+     * @param start - The start of a region
+     * @param end - The end of a region
+     * @param opts - An object containing basesPerSpan (e.g. pixels per basepair) or scale used to infer the zoomLevel to use
+     */
+    async getView(scale, opts) {
+        const { zoomLevels, refsByName, fileSize, isBigEndian, uncompressBufSize } = await this.getHeader(opts);
+        const basesPerPx = 1 / scale;
+        let maxLevel = zoomLevels.length;
+        if (!fileSize) {
+            // if we don't know the file size, we can't fetch the highest zoom level :-(
+            maxLevel -= 1;
+        }
+        for (let i = maxLevel; i >= 0; i -= 1) {
+            const zh = zoomLevels[i];
+            if (zh && zh.reductionLevel <= 2 * basesPerPx) {
+                const indexLength = i < zoomLevels.length - 1
+                    ? zoomLevels[i + 1].dataOffset - zh.indexOffset
+                    : fileSize - 4 - zh.indexOffset;
+                return new blockView_1.BlockView(this.bbi, refsByName, zh.indexOffset, indexLength, isBigEndian, uncompressBufSize > 0, 'summary');
+            }
+        }
+        return this.getUnzoomedView(opts);
+    }
+}
+exports.BigWig = BigWig;

package/esm/blockView.d.ts

@@ -0,0 +1,42 @@
+import { Observer } from 'rxjs';
+import { GenericFilehandle } from 'generic-filehandle';
+import { Feature } from './bbi';
+interface CoordRequest {
+    chrId: number;
+    start: number;
+    end: number;
+}
+interface Options {
+    signal?: AbortSignal;
+    request?: CoordRequest;
+}
+/**
+ * View into a subset of the data in a BigWig file.
+ *
+ * Adapted by Robert Buels and Colin Diesh from bigwig.js in the Dalliance Genome
+ * Explorer by Thomas Down.
+ * @constructs
+ */
+export declare class BlockView {
+    private cirTreeOffset;
+    private cirTreeLength;
+    private bbi;
+    private isCompressed;
+    private isBigEndian;
+    private refsByName;
+    private blockType;
+    private cirTreePromise?;
+    private featureCache;
+    private leafParser;
+    private bigWigParser;
+    private bigBedParser;
+    private summaryParser;
+    constructor(bbi: GenericFilehandle, refsByName: any, cirTreeOffset: number, cirTreeLength: number, isBigEndian: boolean, isCompressed: boolean, blockType: string);
+    readWigData(chrName: string, start: number, end: number, observer: Observer<Feature[]>, opts: Options): Promise<void>;
+    private parseSummaryBlock;
+    private parseBigBedBlock;
+    private parseBigWigBlock;
+    private static coordFilter;
+    readFeatures(observer: Observer<Feature[]>, blocks: any, opts?: Options): Promise<void>;
+}
+export {};

package/esm/blockView.js

@@ -0,0 +1,321 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BlockView = void 0;
+const binary_parser_1 = require("@gmod/binary-parser");
+const abortable_promise_cache_1 = __importDefault(require("abortable-promise-cache"));
+const unzip_1 = require("./unzip");
+const quick_lru_1 = __importDefault(require("quick-lru"));
+const range_1 = __importDefault(require("./range"));
+const util_1 = require("./util");
+const BIG_WIG_TYPE_GRAPH = 1;
+const BIG_WIG_TYPE_VSTEP = 2;
+const BIG_WIG_TYPE_FSTEP = 3;
+function getParsers(isBigEndian) {
+    const le = isBigEndian ? 'big' : 'little';
+    const summaryParser = new binary_parser_1.Parser()
+        .endianess(le)
+        .uint32('chromId')
+        .uint32('start')
+        .uint32('end')
+        .uint32('validCnt')
+        .float('minScore')
+        .float('maxScore')
+        .float('sumData')
+        .float('sumSqData');
+    const leafParser = new binary_parser_1.Parser()
+        .endianess(le)
+        .uint8('isLeaf')
+        .skip(1)
+        .uint16('cnt')
+        .choice({
+        tag: 'isLeaf',
+        choices: {
+            1: new binary_parser_1.Parser().array('blocksToFetch', {
+                length: 'cnt',
+                type: new binary_parser_1.Parser()
+                    .uint32('startChrom')
+                    .uint32('startBase')
+                    .uint32('endChrom')
+                    .uint32('endBase')
+                    .uint64('blockOffset')
+                    .uint64('blockSize'),
+            }),
+            0: new binary_parser_1.Parser().array('recurOffsets', {
+                length: 'cnt',
+                type: new binary_parser_1.Parser()
+                    .uint32('startChrom')
+                    .uint32('startBase')
+                    .uint32('endChrom')
+                    .uint32('endBase')
+                    .uint64('blockOffset'),
+            }),
+        },
+    });
+    const bigBedParser = new binary_parser_1.Parser()
+        .endianess(le)
+        .uint32('chromId')
+        .int32('start')
+        .int32('end')
+        .string('rest', {
+        zeroTerminated: true,
+    });
+    const bigWigParser = new binary_parser_1.Parser()
+        .endianess(le)
+        .skip(4)
+        .int32('blockStart')
+        .skip(4)
+        .uint32('itemStep')
+        .uint32('itemSpan')
+        .uint8('blockType')
+        .skip(1)
+        .uint16('itemCount')
+        .choice({
+        tag: 'blockType',
+        choices: {
+            [BIG_WIG_TYPE_FSTEP]: new binary_parser_1.Parser().array('items', {
+                length: 'itemCount',
+                type: new binary_parser_1.Parser().float('score'),
+            }),
+            [BIG_WIG_TYPE_VSTEP]: new binary_parser_1.Parser().array('items', {
+                length: 'itemCount',
+                type: new binary_parser_1.Parser().int32('start').float('score'),
+            }),
+            [BIG_WIG_TYPE_GRAPH]: new binary_parser_1.Parser().array('items', {
+                length: 'itemCount',
+                type: new binary_parser_1.Parser().int32('start').int32('end').float('score'),
+            }),
+        },
+    });
+    return {
+        bigWigParser,
+        bigBedParser,
+        summaryParser,
+        leafParser,
+    };
+}
+/**
+ * View into a subset of the data in a BigWig file.
+ *
+ * Adapted by Robert Buels and Colin Diesh from bigwig.js in the Dalliance Genome
+ * Explorer by Thomas Down.
+ * @constructs
+ */
+class BlockView {
+    constructor(bbi, refsByName, cirTreeOffset, cirTreeLength, isBigEndian, isCompressed, blockType) {
+        this.featureCache = new abortable_promise_cache_1.default({
+            cache: new quick_lru_1.default({ maxSize: 1000 }),
+            fill: async (requestData, signal) => {
+                const { length, offset } = requestData;
+                const { buffer } = await this.bbi.read(Buffer.alloc(length), 0, length, offset, { signal });
+                return buffer;
+            },
+        });
+        if (!(cirTreeOffset >= 0)) {
+            throw new Error('invalid cirTreeOffset!');
+        }
+        if (!(cirTreeLength > 0)) {
+            throw new Error('invalid cirTreeLength!');
+        }
+        this.cirTreeOffset = cirTreeOffset;
+        this.cirTreeLength = cirTreeLength;
+        this.isCompressed = isCompressed;
+        this.refsByName = refsByName;
+        this.isBigEndian = isBigEndian;
+        this.bbi = bbi;
+        this.blockType = blockType;
+        Object.assign(this, getParsers(isBigEndian));
+    }
+    async readWigData(chrName, start, end, observer, opts) {
+        try {
+            const { refsByName, bbi, cirTreeOffset, isBigEndian } = this;
+            const { signal } = opts;
+            const chrId = refsByName[chrName];
+            if (chrId === undefined) {
+                observer.complete();
+            }
+            const request = { chrId, start, end };
+            if (!this.cirTreePromise) {
+                this.cirTreePromise = bbi.read(Buffer.alloc(48), 0, 48, cirTreeOffset, {
+                    signal,
+                });
+            }
+            const { buffer } = await this.cirTreePromise;
+            const cirBlockSize = isBigEndian
+                ? buffer.readUInt32BE(4)
+                : buffer.readUInt32LE(4);
+            let blocksToFetch = [];
+            let outstanding = 0;
+            const cirFobRecur2 = (cirBlockData, offset, level) => {
+                try {
+                    const data = cirBlockData.subarray(offset);
+                    const p = this.leafParser.parse(data).result;
+                    if (p.blocksToFetch) {
+                        blocksToFetch = blocksToFetch.concat(p.blocksToFetch.filter(filterFeats).map((l) => ({
+                            offset: l.blockOffset,
+                            length: l.blockSize,
+                        })));
+                    }
+                    if (p.recurOffsets) {
+                        const recurOffsets = p.recurOffsets
+                            .filter(filterFeats)
+                            .map((l) => l.blockOffset);
+                        if (recurOffsets.length > 0) {
+                            cirFobRecur(recurOffsets, level + 1);
+                        }
+                    }
+                }
+                catch (e) {
+                    observer.error(e);
+                }
+            };
+            const filterFeats = (b) => {
+                const { startChrom, startBase, endChrom, endBase } = b;
+                return ((startChrom < chrId || (startChrom === chrId && startBase <= end)) &&
+                    (endChrom > chrId || (endChrom === chrId && endBase >= start)));
+            };
+            const cirFobStartFetch = async (off, fr, level) => {
+                try {
+                    const length = fr.max() - fr.min();
+                    const offset = fr.min();
+                    const resultBuffer = await this.featureCache.get(`${length}_${offset}`, { length, offset }, signal);
+                    for (let i = 0; i < off.length; i += 1) {
+                        if (fr.contains(off[i])) {
+                            cirFobRecur2(resultBuffer, off[i] - offset, level);
+                            outstanding -= 1;
+                            if (outstanding === 0) {
+                                this.readFeatures(observer, blocksToFetch, { ...opts, request });
+                            }
+                        }
+                    }
+                }
+                catch (e) {
+                    observer.error(e);
+                }
+            };
+            const cirFobRecur = (offset, level) => {
+                try {
+                    outstanding += offset.length;
+                    const maxCirBlockSpan = 4 + cirBlockSize * 32; // Upper bound on size, based on a completely full leaf node.
+                    let spans = new range_1.default(offset[0], offset[0] + maxCirBlockSpan);
+                    for (let i = 1; i < offset.length; i += 1) {
+                        const blockSpan = new range_1.default(offset[i], offset[i] + maxCirBlockSpan);
+                        spans = spans.union(blockSpan);
+                    }
+                    spans.getRanges().map(fr => cirFobStartFetch(offset, fr, level));
+                }
+                catch (e) {
+                    observer.error(e);
+                }
+            };
+            return cirFobRecur([cirTreeOffset + 48], 1);
+        }
+        catch (e) {
+            observer.error(e);
+        }
+    }
+    parseSummaryBlock(data, startOffset, request) {
+        const features = [];
+        let currOffset = startOffset;
+        while (currOffset < data.byteLength) {
+            const res = this.summaryParser.parse(data.subarray(currOffset));
+            features.push(res.result);
+            currOffset += res.offset;
+        }
+        let items = features;
+        if (request) {
+            items = items.filter(elt => elt.chromId === request.chrId);
+        }
+        const feats = items.map((elt) => ({
+            start: elt.start,
+            end: elt.end,
+            maxScore: elt.maxScore,
+            minScore: elt.minScore,
+            score: elt.sumData / (elt.validCnt || 1),
+            summary: true,
+        }));
+        return request
+            ? feats.filter(f => BlockView.coordFilter(f, request))
+            : feats;
+    }
+    parseBigBedBlock(data, startOffset, offset, request) {
+        const items = [];
+        let currOffset = startOffset;
+        while (currOffset < data.byteLength) {
+            const res = this.bigBedParser.parse(data.subarray(currOffset));
+            res.result.uniqueId = `bb-${offset + currOffset}`;
+            items.push(res.result);
+            currOffset += res.offset;
+        }
+        return request
+            ? items.filter((f) => BlockView.coordFilter(f, request))
+            : items;
+    }
+    parseBigWigBlock(bytes, startOffset, request) {
+        const data = bytes.subarray(startOffset);
+        const results = this.bigWigParser.parse(data).result;
+        const { items, itemSpan, itemStep, blockStart, blockType } = results;
+        if (blockType === BIG_WIG_TYPE_FSTEP) {
+            for (let i = 0; i < items.length; i++) {
+                items[i].start = blockStart + i * itemStep;
+                items[i].end = blockStart + i * itemStep + itemSpan;
+            }
+        }
+        else if (blockType === BIG_WIG_TYPE_VSTEP) {
+            for (let i = 0; i < items.length; i++) {
+                items[i].end = items[i].start + itemSpan;
+            }
+        }
+        return request
+            ? items.filter((f) => BlockView.coordFilter(f, request))
+            : items;
+    }
+    static coordFilter(f, range) {
+        return f.start < range.end && f.end >= range.start;
+    }
+    async readFeatures(observer, blocks, opts = {}) {
+        try {
+            const { blockType, isCompressed } = this;
+            const { signal, request } = opts;
+            const blockGroupsToFetch = (0, util_1.groupBlocks)(blocks);
+            (0, util_1.checkAbortSignal)(signal);
+            await Promise.all(blockGroupsToFetch.map(async (blockGroup) => {
+                (0, util_1.checkAbortSignal)(signal);
+                const { length, offset } = blockGroup;
+                const data = await this.featureCache.get(`${length}_${offset}`, blockGroup, signal);
+                blockGroup.blocks.forEach((block) => {
+                    (0, util_1.checkAbortSignal)(signal);
+                    let blockOffset = block.offset - blockGroup.offset;
+                    let resultData = data;
+                    if (isCompressed) {
+                        resultData = (0, unzip_1.unzip)(data.subarray(blockOffset));
+                        blockOffset = 0;
+                    }
+                    (0, util_1.checkAbortSignal)(signal);
+                    switch (blockType) {
+                        case 'summary':
+                            observer.next(this.parseSummaryBlock(resultData, blockOffset, request));
+                            break;
+                        case 'bigwig':
+                            observer.next(this.parseBigWigBlock(resultData, blockOffset, request));
+                            break;
+                        case 'bigbed':
+                            observer.next(this.parseBigBedBlock(resultData, blockOffset, 
+                            // eslint-disable-next-line no-bitwise
+                            block.offset * (1 << 8), request));
+                            break;
+                        default:
+                            console.warn(`Don't know what to do with ${blockType}`);
+                    }
+                });
+            }));
+            observer.complete();
+        }
+        catch (e) {
+            observer.error(e);
+        }
+    }
+}
+exports.BlockView = BlockView;

package/esm/index.d.ts

@@ -0,0 +1,3 @@
+export { BigWig } from './bigwig';
+export { BigBed } from './bigbed';
+export { Feature, Header, RequestOptions } from './bbi';

package/esm/index.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BigBed = exports.BigWig = void 0;
+var bigwig_1 = require("./bigwig");
+Object.defineProperty(exports, "BigWig", { enumerable: true, get: function () { return bigwig_1.BigWig; } });
+var bigbed_1 = require("./bigbed");
+Object.defineProperty(exports, "BigBed", { enumerable: true, get: function () { return bigbed_1.BigBed; } });

package/esm/range.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Adapted from a combination of Range and _Compound in the
+ * Dalliance Genome Explorer, (c) Thomas Down 2006-2010.
+ */
+export default class Range {
+    ranges: any;
+    constructor(arg1: any, arg2?: any);
+    min(): number;
+    max(): number;
+    contains(pos: number): boolean;
+    isContiguous(): boolean;
+    getRanges(): Range[];
+    toString(): string;
+    union(s1: Range): Range;
+    intersection(arg: Range): Range;
+    coverage(): number;
+    rangeOrder(tmpa: Range, tmpb: Range): number;
+}