@gmod/tabix 1.4.4 → 1.5.1

package/esm/csi.js

@@ -0,0 +1,254 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const long_1 = __importDefault(require("long"));
+const bgzf_filehandle_1 = require("@gmod/bgzf-filehandle");
+const virtualOffset_1 = __importStar(require("./virtualOffset"));
+const chunk_1 = __importDefault(require("./chunk"));
+const util_1 = require("./util");
+const indexFile_1 = __importDefault(require("./indexFile"));
+const CSI1_MAGIC = 21582659; // CSI\1
+const CSI2_MAGIC = 38359875; // CSI\2
+function lshift(num, bits) {
+    return num * 2 ** bits;
+}
+function rshift(num, bits) {
+    return Math.floor(num / 2 ** bits);
+}
+class CSI extends indexFile_1.default {
+    constructor(args) {
+        super(args);
+        this.maxBinNumber = 0;
+        this.depth = 0;
+        this.minShift = 0;
+    }
+    async lineCount(refName, opts = {}) {
+        const indexData = await this.parse(opts);
+        if (!indexData) {
+            return -1;
+        }
+        const refId = indexData.refNameToId[refName];
+        const idx = indexData.indices[refId];
+        if (!idx) {
+            return -1;
+        }
+        const { stats } = indexData.indices[refId];
+        if (stats) {
+            return stats.lineCount;
+        }
+        return -1;
+    }
+    async indexCov() {
+        throw new Error('CSI indexes do not support indexcov');
+        return [];
+    }
+    parseAuxData(bytes, offset, auxLength) {
+        if (auxLength < 30) {
+            return {
+                refIdToName: [],
+                refNameToId: {},
+            };
+        }
+        const formatFlags = bytes.readInt32LE(offset);
+        const coordinateType = formatFlags & 0x10000 ? 'zero-based-half-open' : '1-based-closed';
+        const format = { 0: 'generic', 1: 'SAM', 2: 'VCF' }[formatFlags & 0xf];
+        if (!format) {
+            throw new Error(`invalid Tabix preset format flags ${formatFlags}`);
+        }
+        const columnNumbers = {
+            ref: bytes.readInt32LE(offset + 4),
+            start: bytes.readInt32LE(offset + 8),
+            end: bytes.readInt32LE(offset + 12),
+        };
+        const metaValue = bytes.readInt32LE(offset + 16);
+        const metaChar = metaValue ? String.fromCharCode(metaValue) : '';
+        const skipLines = bytes.readInt32LE(offset + 20);
+        const nameSectionLength = bytes.readInt32LE(offset + 24);
+        const { refIdToName, refNameToId } = this._parseNameBytes(bytes.slice(offset + 28, offset + 28 + nameSectionLength));
+        return {
+            refIdToName,
+            refNameToId,
+            skipLines,
+            metaChar,
+            columnNumbers,
+            format,
+            coordinateType,
+        };
+    }
+    _parseNameBytes(namesBytes) {
+        let currRefId = 0;
+        let currNameStart = 0;
+        const refIdToName = [];
+        const refNameToId = {};
+        for (let i = 0; i < namesBytes.length; i += 1) {
+            if (!namesBytes[i]) {
+                if (currNameStart < i) {
+                    let refName = namesBytes.toString('utf8', currNameStart, i);
+                    refName = this.renameRefSeq(refName);
+                    refIdToName[currRefId] = refName;
+                    refNameToId[refName] = currRefId;
+                }
+                currNameStart = i + 1;
+                currRefId += 1;
+            }
+        }
+        return { refNameToId, refIdToName };
+    }
+    // fetch and parse the index
+    async _parse(opts = {}) {
+        const bytes = await (0, bgzf_filehandle_1.unzip)((await this.filehandle.readFile(opts)));
+        // check TBI magic numbers
+        let csiVersion;
+        if (bytes.readUInt32LE(0) === CSI1_MAGIC) {
+            csiVersion = 1;
+        }
+        else if (bytes.readUInt32LE(0) === CSI2_MAGIC) {
+            csiVersion = 2;
+        }
+        else {
+            throw new Error('Not a CSI file');
+            // TODO: do we need to support big-endian CSI files?
+        }
+        this.minShift = bytes.readInt32LE(4);
+        this.depth = bytes.readInt32LE(8);
+        this.maxBinNumber = ((1 << ((this.depth + 1) * 3)) - 1) / 7;
+        const maxRefLength = 2 ** (this.minShift + this.depth * 3);
+        const auxLength = bytes.readInt32LE(12);
+        let aux = {
+            refIdToName: [],
+            refNameToId: {},
+        };
+        if (auxLength) {
+            aux = this.parseAuxData(bytes, 16, auxLength);
+        }
+        const refCount = bytes.readInt32LE(16 + auxLength);
+        // read the indexes for each reference sequence
+        let firstDataLine;
+        let currOffset = 16 + auxLength + 4;
+        const indices = new Array(refCount).fill(0).map(() => {
+            // the binning index
+            const binCount = bytes.readInt32LE(currOffset);
+            currOffset += 4;
+            const binIndex = {};
+            let stats; // < provided by parsing a pseudo-bin, if present
+            for (let j = 0; j < binCount; j += 1) {
+                const bin = bytes.readUInt32LE(currOffset);
+                if (bin > this.maxBinNumber) {
+                    // this is a fake bin that actually has stats information
+                    // about the reference sequence in it
+                    stats = this.parsePseudoBin(bytes, currOffset + 4);
+                    currOffset += 4 + 8 + 4 + 16 + 16;
+                }
+                else {
+                    const loffset = (0, virtualOffset_1.fromBytes)(bytes, currOffset + 4);
+                    firstDataLine = this._findFirstData(firstDataLine, loffset);
+                    const chunkCount = bytes.readInt32LE(currOffset + 12);
+                    currOffset += 16;
+                    const chunks = new Array(chunkCount);
+                    for (let k = 0; k < chunkCount; k += 1) {
+                        const u = (0, virtualOffset_1.fromBytes)(bytes, currOffset);
+                        const v = (0, virtualOffset_1.fromBytes)(bytes, currOffset + 8);
+                        currOffset += 16;
+                        // this._findFirstData(data, u)
+                        chunks[k] = new chunk_1.default(u, v, bin);
+                    }
+                    binIndex[bin] = chunks;
+                }
+            }
+            return { binIndex, stats };
+        });
+        return {
+            ...aux,
+            csi: true,
+            refCount,
+            maxBlockSize: 1 << 16,
+            firstDataLine,
+            csiVersion,
+            indices,
+            depth: this.depth,
+            maxBinNumber: this.maxBinNumber,
+            maxRefLength,
+        };
+    }
+    parsePseudoBin(bytes, offset) {
+        const lineCount = (0, util_1.longToNumber)(long_1.default.fromBytesLE(Array.prototype.slice.call(bytes, offset + 28, offset + 36), true));
+        return { lineCount };
+    }
+    async blocksForRange(refName, min, max, opts = {}) {
+        if (min < 0) {
+            min = 0;
+        }
+        const indexData = await this.parse(opts);
+        if (!indexData) {
+            return [];
+        }
+        const refId = indexData.refNameToId[refName];
+        const ba = indexData.indices[refId];
+        if (!ba) {
+            return [];
+        }
+        // const { linearIndex, binIndex } = indexes
+        const overlappingBins = this.reg2bins(min, max); // List of bin #s that overlap min, max
+        const chunks = [];
+        // Find chunks in overlapping bins.  Leaf bins (< 4681) are not pruned
+        for (const [start, end] of overlappingBins) {
+            for (let bin = start; bin <= end; bin++) {
+                if (ba.binIndex[bin]) {
+                    const binChunks = ba.binIndex[bin];
+                    for (let c = 0; c < binChunks.length; ++c) {
+                        chunks.push(new chunk_1.default(binChunks[c].minv, binChunks[c].maxv, bin));
+                    }
+                }
+            }
+        }
+        return (0, util_1.optimizeChunks)(chunks, new virtualOffset_1.default(0, 0));
+    }
+    /**
+     * calculate the list of bins that may overlap with region [beg,end) (zero-based half-open)
+     */
+    reg2bins(beg, end) {
+        beg -= 1; // < convert to 1-based closed
+        if (beg < 1) {
+            beg = 1;
+        }
+        if (end > 2 ** 50) {
+            end = 2 ** 34;
+        } // 17 GiB ought to be enough for anybody
+        end -= 1;
+        let l = 0;
+        let t = 0;
+        let s = this.minShift + this.depth * 3;
+        const bins = [];
+        for (; l <= this.depth; s -= 3, t += lshift(1, l * 3), l += 1) {
+            const b = t + rshift(beg, s);
+            const e = t + rshift(end, s);
+            if (e - b + bins.length > this.maxBinNumber) {
+                throw new Error(`query ${beg}-${end} is too large for current binning scheme (shift ${this.minShift}, depth ${this.depth}), try a smaller query or a coarser index binning scheme`);
+            }
+            bins.push([b, e]);
+        }
+        return bins;
+    }
+}
+exports.default = CSI;

package/esm/index.d.ts

@@ -0,0 +1,4 @@
+import TabixIndexedFile from './tabixIndexedFile';
+import TBI from './tbi';
+import CSI from './csi';
+export { TabixIndexedFile, TBI, CSI };

package/esm/index.js

@@ -0,0 +1,12 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CSI = exports.TBI = exports.TabixIndexedFile = void 0;
+const tabixIndexedFile_1 = __importDefault(require("./tabixIndexedFile"));
+exports.TabixIndexedFile = tabixIndexedFile_1.default;
+const tbi_1 = __importDefault(require("./tbi"));
+exports.TBI = tbi_1.default;
+const csi_1 = __importDefault(require("./csi"));
+exports.CSI = csi_1.default;

package/esm/indexFile.d.ts

@@ -0,0 +1,32 @@
+import { GenericFilehandle } from 'generic-filehandle';
+import VirtualOffset from './virtualOffset';
+import Chunk from './chunk';
+export interface Options {
+    [key: string]: unknown;
+    signal?: AbortSignal;
+}
+export default abstract class IndexFile {
+    filehandle: GenericFilehandle;
+    renameRefSeq: (arg0: string) => string;
+    private _parseCache;
+    /**
+     * @param {filehandle} filehandle
+     * @param {function} [renameRefSeqs]
+     */
+    constructor({ filehandle, renameRefSeqs, }: {
+        filehandle: GenericFilehandle;
+        renameRefSeqs?: (a: string) => string;
+    });
+    abstract lineCount(refName: string, args: Options): Promise<number>;
+    protected abstract _parse(opts: Options): Promise<{
+        refNameToId: {
+            [key: string]: number;
+        };
+        refIdToName: string[];
+    }>;
+    getMetadata(opts?: Options): Promise<any>;
+    abstract blocksForRange(refName: string, start: number, end: number, opts: Options): Promise<Chunk[]>;
+    _findFirstData(currentFdl: VirtualOffset | undefined, virtualOffset: VirtualOffset): VirtualOffset;
+    parse(opts?: Options): Promise<any>;
+    hasRefSeq(seqId: number, opts?: Options): Promise<boolean>;
+}

package/esm/indexFile.js

@@ -0,0 +1,45 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const abortable_promise_cache_1 = __importDefault(require("abortable-promise-cache"));
+const quick_lru_1 = __importDefault(require("quick-lru"));
+class IndexFile {
+    /**
+     * @param {filehandle} filehandle
+     * @param {function} [renameRefSeqs]
+     */
+    constructor({ filehandle, renameRefSeqs = (n) => n, }) {
+        this.filehandle = filehandle;
+        this.renameRefSeq = renameRefSeqs;
+    }
+    async getMetadata(opts = {}) {
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const { indices, ...rest } = await this.parse(opts);
+        return rest;
+    }
+    _findFirstData(currentFdl, virtualOffset) {
+        if (currentFdl) {
+            return currentFdl.compareTo(virtualOffset) > 0
+                ? virtualOffset
+                : currentFdl;
+        }
+        else {
+            return virtualOffset;
+        }
+    }
+    async parse(opts = {}) {
+        if (!this._parseCache) {
+            this._parseCache = new abortable_promise_cache_1.default({
+                cache: new quick_lru_1.default({ maxSize: 1 }),
+                fill: () => this._parse(opts),
+            });
+        }
+        return this._parseCache.get('index', null, undefined);
+    }
+    async hasRefSeq(seqId, opts = {}) {
+        return !!((await this.parse(opts)).indices[seqId] || {}).binIndex;
+    }
+}
+exports.default = IndexFile;

package/esm/tabixIndexedFile.d.ts

@@ -0,0 +1,118 @@
+/// <reference types="node" />
+import { GenericFilehandle } from 'generic-filehandle';
+import { Options } from './indexFile';
+import Chunk from './chunk';
+declare type GetLinesCallback = (line: string, fileOffset: number) => Promise<void>;
+interface GetLinesOpts {
+    [key: string]: unknown;
+    signal?: AbortSignal;
+    lineCallback: GetLinesCallback;
+}
+export default class TabixIndexedFile {
+    private filehandle;
+    private index;
+    private chunkSizeLimit;
+    private renameRefSeq;
+    private chunkCache;
+    /**
+     * @param {object} args
+     * @param {string} [args.path]
+     * @param {filehandle} [args.filehandle]
+     * @param {string} [args.tbiPath]
+     * @param {filehandle} [args.tbiFilehandle]
+     * @param {string} [args.csiPath]
+     * @param {filehandle} [args.csiFilehandle]
+     * @param {chunkSizeLimit} default 50MiB
+     * @param {function} [args.renameRefSeqs] optional function with sig `string => string` to transform
+     * reference sequence names for the purpose of indexing and querying. note that the data that is returned is
+     * not altered, just the names of the reference sequences that are used for querying.
+     * @param {number} [args.chunkCacheSize] maximum size in bytes of the chunk cache. default 5MB
+     * @param {number} [args.blockCacheSize] maximum size in bytes of the block cache. default 5MB
+     */
+    constructor({ path, filehandle, tbiPath, tbiFilehandle, csiPath, csiFilehandle, chunkSizeLimit, renameRefSeqs, chunkCacheSize, }: {
+        path?: string;
+        filehandle?: GenericFilehandle;
+        tbiPath?: string;
+        tbiFilehandle?: GenericFilehandle;
+        csiPath?: string;
+        csiFilehandle?: GenericFilehandle;
+        chunkSizeLimit?: number;
+        renameRefSeqs?: (n: string) => string;
+        chunkCacheSize?: number;
+    });
+    /**
+     * @param {string} refName name of the reference sequence
+     * @param {number} start start of the region (in 0-based half-open coordinates)
+     * @param {number} end end of the region (in 0-based half-open coordinates)
+     * @param {function|object} lineCallback callback called for each line in the region. can also pass a object param containing obj.lineCallback, obj.signal, etc
+     * @returns {Promise} resolved when the whole read is finished, rejected on error
+     */
+    getLines(refName: string, start: number, end: number, opts: GetLinesOpts | GetLinesCallback): Promise<void>;
+    getMetadata(opts?: Options): Promise<any>;
+    /**
+     * get a buffer containing the "header" region of
+     * the file, which are the bytes up to the first
+     * non-meta line
+     *
+     * @returns {Promise} for a buffer
+     */
+    getHeaderBuffer(opts?: Options): Promise<Buffer>;
+    /**
+     * get a string containing the "header" region of the
+     * file, is the portion up to the first non-meta line
+     *
+     * @returns {Promise} for a string
+     */
+    getHeader(opts?: Options): Promise<string>;
+    /**
+     * get an array of reference sequence names, in the order in which
+     * they occur in the file.
+     *
+     * reference sequence renaming is not applied to these names.
+     *
+     * @returns {Promise} for an array of string sequence names
+     */
+    getReferenceSequenceNames(opts?: Options): Promise<any>;
+    /**
+     * @param {object} metadata metadata object from the parsed index,
+     * containing columnNumbers, metaChar, and format
+     * @param {string} regionRefName
+     * @param {number} regionStart region start coordinate (0-based-half-open)
+     * @param {number} regionEnd region end coordinate (0-based-half-open)
+     * @param {array[string]} line
+     * @returns {object} like `{startCoordinate, overlaps}`. overlaps is boolean,
+     * true if line is a data line that overlaps the given region
+     */
+    checkLine({ columnNumbers, metaChar, coordinateType, format, }: {
+        columnNumbers: {
+            ref: number;
+            start: number;
+            end: number;
+        };
+        metaChar: string;
+        coordinateType: string;
+        format: string;
+    }, regionRefName: string, regionStart: number, regionEnd: number, line: string): {
+        overlaps: boolean;
+        startCoordinate?: undefined;
+    } | {
+        startCoordinate: number;
+        overlaps: boolean;
+    };
+    _getVcfEnd(startCoordinate: number, refSeq: string, info: any): number;
+    /**
+     * return the approximate number of data lines in the given reference sequence
+     * @param {string} refSeq reference sequence name
+     * @returns {Promise} for number of data lines present on that reference sequence
+     */
+    lineCount(refName: string, opts?: Options): Promise<number>;
+    _readRegion(position: number, compressedSize: number, opts?: Options): Promise<Buffer>;
+    /**
+     * read and uncompress the data in a chunk (composed of one or more
+     * contiguous bgzip blocks) of the file
+     * @param {Chunk} chunk
+     * @returns {Promise} for a string chunk of the file
+     */
+    readChunk(chunk: Chunk, opts?: Options): Promise<any>;
+}
+export {};