@gmod/tabix 1.5.0 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/README.md +3 -2
- package/dist/chunk.js +33 -62
- package/dist/csi.d.ts +0 -1
- package/dist/csi.js +324 -533
- package/dist/index.js +11 -36
- package/dist/indexFile.d.ts +1 -1
- package/dist/indexFile.js +112 -174
- package/dist/tabixIndexedFile.d.ts +8 -4
- package/dist/tabixIndexedFile.js +476 -774
- package/dist/tbi.js +308 -485
- package/dist/util.js +132 -148
- package/dist/virtualOffset.js +44 -68
- package/esm/chunk.d.ts +18 -0
- package/esm/chunk.js +35 -0
- package/esm/csi.d.ts +71 -0
- package/esm/csi.js +254 -0
- package/esm/index.d.ts +4 -0
- package/esm/index.js +12 -0
- package/esm/indexFile.d.ts +32 -0
- package/esm/indexFile.js +45 -0
- package/esm/tabixIndexedFile.d.ts +118 -0
- package/esm/tabixIndexedFile.js +403 -0
- package/esm/tbi.d.ts +45 -0
- package/esm/tbi.js +240 -0
- package/esm/util.d.ts +25 -0
- package/esm/util.js +98 -0
- package/esm/virtualOffset.d.ts +10 -0
- package/esm/virtualOffset.js +41 -0
- package/package.json +23 -40
- package/dist/declare.d.js +0 -2
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
const abortable_promise_cache_1 = __importDefault(require("abortable-promise-cache"));
|
|
7
|
+
const quick_lru_1 = __importDefault(require("quick-lru"));
|
|
8
|
+
const generic_filehandle_1 = require("generic-filehandle");
|
|
9
|
+
const bgzf_filehandle_1 = require("@gmod/bgzf-filehandle");
|
|
10
|
+
const util_1 = require("./util");
|
|
11
|
+
const tbi_1 = __importDefault(require("./tbi"));
|
|
12
|
+
const csi_1 = __importDefault(require("./csi"));
|
|
13
|
+
function timeout(time) {
|
|
14
|
+
return new Promise(resolve => {
|
|
15
|
+
setTimeout(resolve, time);
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
class TabixIndexedFile {
|
|
19
|
+
/**
|
|
20
|
+
* @param {object} args
|
|
21
|
+
* @param {string} [args.path]
|
|
22
|
+
* @param {filehandle} [args.filehandle]
|
|
23
|
+
* @param {string} [args.tbiPath]
|
|
24
|
+
* @param {filehandle} [args.tbiFilehandle]
|
|
25
|
+
* @param {string} [args.csiPath]
|
|
26
|
+
* @param {filehandle} [args.csiFilehandle]
|
|
27
|
+
* @param {chunkSizeLimit} default 50MiB
|
|
28
|
+
* @param {function} [args.renameRefSeqs] optional function with sig `string => string` to transform
|
|
29
|
+
* reference sequence names for the purpose of indexing and querying. note that the data that is returned is
|
|
30
|
+
* not altered, just the names of the reference sequences that are used for querying.
|
|
31
|
+
* @param {number} [args.chunkCacheSize] maximum size in bytes of the chunk cache. default 5MB
|
|
32
|
+
* @param {number} [args.blockCacheSize] maximum size in bytes of the block cache. default 5MB
|
|
33
|
+
*/
|
|
34
|
+
constructor({ path, filehandle, tbiPath, tbiFilehandle, csiPath, csiFilehandle, chunkSizeLimit = 50000000, renameRefSeqs = n => n, chunkCacheSize = 5 * 2 ** 20, }) {
|
|
35
|
+
if (filehandle) {
|
|
36
|
+
this.filehandle = filehandle;
|
|
37
|
+
}
|
|
38
|
+
else if (path) {
|
|
39
|
+
this.filehandle = new generic_filehandle_1.LocalFile(path);
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
throw new TypeError('must provide either filehandle or path');
|
|
43
|
+
}
|
|
44
|
+
if (tbiFilehandle) {
|
|
45
|
+
this.index = new tbi_1.default({
|
|
46
|
+
filehandle: tbiFilehandle,
|
|
47
|
+
renameRefSeqs,
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
else if (csiFilehandle) {
|
|
51
|
+
this.index = new csi_1.default({
|
|
52
|
+
filehandle: csiFilehandle,
|
|
53
|
+
renameRefSeqs,
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
else if (tbiPath) {
|
|
57
|
+
this.index = new tbi_1.default({
|
|
58
|
+
filehandle: new generic_filehandle_1.LocalFile(tbiPath),
|
|
59
|
+
renameRefSeqs,
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
else if (csiPath) {
|
|
63
|
+
this.index = new csi_1.default({
|
|
64
|
+
filehandle: new generic_filehandle_1.LocalFile(csiPath),
|
|
65
|
+
renameRefSeqs,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
else if (path) {
|
|
69
|
+
this.index = new tbi_1.default({
|
|
70
|
+
filehandle: new generic_filehandle_1.LocalFile(`${path}.tbi`),
|
|
71
|
+
renameRefSeqs,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
else {
|
|
75
|
+
throw new TypeError('must provide one of tbiFilehandle, tbiPath, csiFilehandle, or csiPath');
|
|
76
|
+
}
|
|
77
|
+
this.chunkSizeLimit = chunkSizeLimit;
|
|
78
|
+
this.renameRefSeq = renameRefSeqs;
|
|
79
|
+
this.chunkCache = new abortable_promise_cache_1.default({
|
|
80
|
+
cache: new quick_lru_1.default({
|
|
81
|
+
maxSize: Math.floor(chunkCacheSize / (1 << 16)),
|
|
82
|
+
}),
|
|
83
|
+
fill: this.readChunk.bind(this),
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* @param {string} refName name of the reference sequence
|
|
88
|
+
* @param {number} start start of the region (in 0-based half-open coordinates)
|
|
89
|
+
* @param {number} end end of the region (in 0-based half-open coordinates)
|
|
90
|
+
* @param {function|object} lineCallback callback called for each line in the region. can also pass a object param containing obj.lineCallback, obj.signal, etc
|
|
91
|
+
* @returns {Promise} resolved when the whole read is finished, rejected on error
|
|
92
|
+
*/
|
|
93
|
+
async getLines(refName, start, end, opts) {
|
|
94
|
+
let signal;
|
|
95
|
+
let options = {};
|
|
96
|
+
let callback;
|
|
97
|
+
if (typeof opts === 'undefined') {
|
|
98
|
+
throw new TypeError('line callback must be provided');
|
|
99
|
+
}
|
|
100
|
+
if (typeof opts === 'function') {
|
|
101
|
+
callback = opts;
|
|
102
|
+
}
|
|
103
|
+
else {
|
|
104
|
+
options = opts;
|
|
105
|
+
callback = opts.lineCallback;
|
|
106
|
+
}
|
|
107
|
+
if (refName === undefined) {
|
|
108
|
+
throw new TypeError('must provide a reference sequence name');
|
|
109
|
+
}
|
|
110
|
+
if (!callback) {
|
|
111
|
+
throw new TypeError('line callback must be provided');
|
|
112
|
+
}
|
|
113
|
+
const metadata = await this.index.getMetadata(options);
|
|
114
|
+
(0, util_1.checkAbortSignal)(signal);
|
|
115
|
+
if (!start) {
|
|
116
|
+
start = 0;
|
|
117
|
+
}
|
|
118
|
+
if (!end) {
|
|
119
|
+
end = metadata.maxRefLength;
|
|
120
|
+
}
|
|
121
|
+
if (!(start <= end)) {
|
|
122
|
+
throw new TypeError('invalid start and end coordinates. start must be less than or equal to end');
|
|
123
|
+
}
|
|
124
|
+
if (start === end) {
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
const chunks = await this.index.blocksForRange(refName, start, end, options);
|
|
128
|
+
(0, util_1.checkAbortSignal)(signal);
|
|
129
|
+
// check the chunks for any that are over the size limit. if
|
|
130
|
+
// any are, don't fetch any of them
|
|
131
|
+
for (let i = 0; i < chunks.length; i += 1) {
|
|
132
|
+
const size = chunks[i].fetchedSize();
|
|
133
|
+
if (size > this.chunkSizeLimit) {
|
|
134
|
+
throw new Error(`Too much data. Chunk size ${size.toLocaleString()} bytes exceeds chunkSizeLimit of ${this.chunkSizeLimit.toLocaleString()}.`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
// now go through each chunk and parse and filter the lines out of it
|
|
138
|
+
let last = Date.now();
|
|
139
|
+
for (let chunkNum = 0; chunkNum < chunks.length; chunkNum += 1) {
|
|
140
|
+
let previousStartCoordinate;
|
|
141
|
+
const c = chunks[chunkNum];
|
|
142
|
+
const { buffer, cpositions, dpositions } = await this.chunkCache.get(c.toString(), c, signal);
|
|
143
|
+
const lines = (typeof TextDecoder !== 'undefined'
|
|
144
|
+
? new TextDecoder('utf-8').decode(buffer)
|
|
145
|
+
: buffer.toString()).split('\n');
|
|
146
|
+
lines.pop();
|
|
147
|
+
(0, util_1.checkAbortSignal)(signal);
|
|
148
|
+
let blockStart = c.minv.dataPosition;
|
|
149
|
+
let pos;
|
|
150
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
151
|
+
const line = lines[i];
|
|
152
|
+
for (pos = 0; blockStart >= dpositions[pos]; pos += 1) { }
|
|
153
|
+
// filter the line for whether it is within the requested range
|
|
154
|
+
const { startCoordinate, overlaps } = this.checkLine(metadata, refName, start, end, line);
|
|
155
|
+
// do a small check just to make sure that the lines are really sorted by start coordinate
|
|
156
|
+
if (previousStartCoordinate !== undefined &&
|
|
157
|
+
startCoordinate !== undefined &&
|
|
158
|
+
previousStartCoordinate > startCoordinate) {
|
|
159
|
+
throw new Error(`Lines not sorted by start coordinate (${previousStartCoordinate} > ${startCoordinate}), this file is not usable with Tabix.`);
|
|
160
|
+
}
|
|
161
|
+
previousStartCoordinate = startCoordinate;
|
|
162
|
+
if (overlaps) {
|
|
163
|
+
callback(line.trim(),
|
|
164
|
+
// cpositions[pos] refers to actual file offset of a bgzip block boundaries
|
|
165
|
+
//
|
|
166
|
+
// we multiply by (1 <<8) in order to make sure each block has a "unique"
|
|
167
|
+
// address space so that data in that block could never overlap
|
|
168
|
+
//
|
|
169
|
+
// then the blockStart-dpositions is an uncompressed file offset from
|
|
170
|
+
// that bgzip block boundary, and since the cpositions are multiplied by
|
|
171
|
+
// (1 << 8) these uncompressed offsets get a unique space
|
|
172
|
+
cpositions[pos] * (1 << 8) + (blockStart - dpositions[pos]));
|
|
173
|
+
}
|
|
174
|
+
else if (startCoordinate !== undefined && startCoordinate >= end) {
|
|
175
|
+
// the lines were overlapping the region, but now have stopped, so
|
|
176
|
+
// we must be at the end of the relevant data and we can stop
|
|
177
|
+
// processing data now
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
blockStart += line.length + 1;
|
|
181
|
+
// yield if we have emitted beyond the yield limit
|
|
182
|
+
if (last - Date.now() > 500) {
|
|
183
|
+
last = Date.now();
|
|
184
|
+
(0, util_1.checkAbortSignal)(signal);
|
|
185
|
+
await timeout(1);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
async getMetadata(opts = {}) {
|
|
191
|
+
return this.index.getMetadata(opts);
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* get a buffer containing the "header" region of
|
|
195
|
+
* the file, which are the bytes up to the first
|
|
196
|
+
* non-meta line
|
|
197
|
+
*
|
|
198
|
+
* @returns {Promise} for a buffer
|
|
199
|
+
*/
|
|
200
|
+
async getHeaderBuffer(opts = {}) {
|
|
201
|
+
const { firstDataLine, metaChar, maxBlockSize } = await this.getMetadata(opts);
|
|
202
|
+
(0, util_1.checkAbortSignal)(opts.signal);
|
|
203
|
+
const maxFetch = firstDataLine && firstDataLine.blockPosition
|
|
204
|
+
? firstDataLine.blockPosition + maxBlockSize
|
|
205
|
+
: maxBlockSize;
|
|
206
|
+
// TODO: what if we don't have a firstDataLine, and the header
|
|
207
|
+
// actually takes up more than one block? this case is not covered here
|
|
208
|
+
let bytes = await this._readRegion(0, maxFetch, opts);
|
|
209
|
+
(0, util_1.checkAbortSignal)(opts.signal);
|
|
210
|
+
try {
|
|
211
|
+
bytes = await (0, bgzf_filehandle_1.unzip)(bytes);
|
|
212
|
+
}
|
|
213
|
+
catch (e) {
|
|
214
|
+
console.error(e);
|
|
215
|
+
throw new Error(
|
|
216
|
+
//@ts-ignore
|
|
217
|
+
`error decompressing block ${e.code} at 0 (length ${maxFetch}) ${e}`);
|
|
218
|
+
}
|
|
219
|
+
// trim off lines after the last non-meta line
|
|
220
|
+
if (metaChar) {
|
|
221
|
+
// trim backward from the end
|
|
222
|
+
let lastNewline = -1;
|
|
223
|
+
const newlineByte = '\n'.charCodeAt(0);
|
|
224
|
+
const metaByte = metaChar.charCodeAt(0);
|
|
225
|
+
for (let i = 0; i < bytes.length; i += 1) {
|
|
226
|
+
if (i === lastNewline + 1 && bytes[i] !== metaByte) {
|
|
227
|
+
break;
|
|
228
|
+
}
|
|
229
|
+
if (bytes[i] === newlineByte) {
|
|
230
|
+
lastNewline = i;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
bytes = bytes.slice(0, lastNewline + 1);
|
|
234
|
+
}
|
|
235
|
+
return bytes;
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* get a string containing the "header" region of the
|
|
239
|
+
* file, is the portion up to the first non-meta line
|
|
240
|
+
*
|
|
241
|
+
* @returns {Promise} for a string
|
|
242
|
+
*/
|
|
243
|
+
async getHeader(opts = {}) {
|
|
244
|
+
const bytes = await this.getHeaderBuffer(opts);
|
|
245
|
+
(0, util_1.checkAbortSignal)(opts.signal);
|
|
246
|
+
return bytes.toString('utf8');
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* get an array of reference sequence names, in the order in which
|
|
250
|
+
* they occur in the file.
|
|
251
|
+
*
|
|
252
|
+
* reference sequence renaming is not applied to these names.
|
|
253
|
+
*
|
|
254
|
+
* @returns {Promise} for an array of string sequence names
|
|
255
|
+
*/
|
|
256
|
+
async getReferenceSequenceNames(opts = {}) {
|
|
257
|
+
const metadata = await this.getMetadata(opts);
|
|
258
|
+
return metadata.refIdToName;
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* @param {object} metadata metadata object from the parsed index,
|
|
262
|
+
* containing columnNumbers, metaChar, and format
|
|
263
|
+
* @param {string} regionRefName
|
|
264
|
+
* @param {number} regionStart region start coordinate (0-based-half-open)
|
|
265
|
+
* @param {number} regionEnd region end coordinate (0-based-half-open)
|
|
266
|
+
* @param {array[string]} line
|
|
267
|
+
* @returns {object} like `{startCoordinate, overlaps}`. overlaps is boolean,
|
|
268
|
+
* true if line is a data line that overlaps the given region
|
|
269
|
+
*/
|
|
270
|
+
checkLine({ columnNumbers, metaChar, coordinateType, format, }, regionRefName, regionStart, regionEnd, line) {
|
|
271
|
+
// skip meta lines
|
|
272
|
+
if (line.charAt(0) === metaChar) {
|
|
273
|
+
return { overlaps: false };
|
|
274
|
+
}
|
|
275
|
+
// check ref/start/end using column metadata from index
|
|
276
|
+
let { ref, start, end } = columnNumbers;
|
|
277
|
+
if (!ref) {
|
|
278
|
+
ref = 0;
|
|
279
|
+
}
|
|
280
|
+
if (!start) {
|
|
281
|
+
start = 0;
|
|
282
|
+
}
|
|
283
|
+
if (!end) {
|
|
284
|
+
end = 0;
|
|
285
|
+
}
|
|
286
|
+
if (format === 'VCF') {
|
|
287
|
+
end = 8;
|
|
288
|
+
}
|
|
289
|
+
const maxColumn = Math.max(ref, start, end);
|
|
290
|
+
// this code is kind of complex, but it is fairly fast.
|
|
291
|
+
// basically, we want to avoid doing a split, because if the lines are really long
|
|
292
|
+
// that could lead to us allocating a bunch of extra memory, which is slow
|
|
293
|
+
let currentColumnNumber = 1; // cols are numbered starting at 1 in the index metadata
|
|
294
|
+
let currentColumnStart = 0;
|
|
295
|
+
let refSeq = '';
|
|
296
|
+
let startCoordinate = -Infinity;
|
|
297
|
+
for (let i = 0; i < line.length + 1; i += 1) {
|
|
298
|
+
if (line[i] === '\t' || i === line.length) {
|
|
299
|
+
if (currentColumnNumber === ref) {
|
|
300
|
+
if (this.renameRefSeq(line.slice(currentColumnStart, i)) !==
|
|
301
|
+
regionRefName) {
|
|
302
|
+
return { overlaps: false };
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
else if (currentColumnNumber === start) {
|
|
306
|
+
startCoordinate = parseInt(line.slice(currentColumnStart, i), 10);
|
|
307
|
+
// we convert to 0-based-half-open
|
|
308
|
+
if (coordinateType === '1-based-closed') {
|
|
309
|
+
startCoordinate -= 1;
|
|
310
|
+
}
|
|
311
|
+
if (startCoordinate >= regionEnd) {
|
|
312
|
+
return { startCoordinate, overlaps: false };
|
|
313
|
+
}
|
|
314
|
+
if (end === 0 || end === start) {
|
|
315
|
+
// if we have no end, we assume the feature is 1 bp long
|
|
316
|
+
if (startCoordinate + 1 <= regionStart) {
|
|
317
|
+
return { startCoordinate, overlaps: false };
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
else if (format === 'VCF' && currentColumnNumber === 4) {
|
|
322
|
+
refSeq = line.slice(currentColumnStart, i);
|
|
323
|
+
}
|
|
324
|
+
else if (currentColumnNumber === end) {
|
|
325
|
+
let endCoordinate;
|
|
326
|
+
// this will never match if there is no end column
|
|
327
|
+
if (format === 'VCF') {
|
|
328
|
+
endCoordinate = this._getVcfEnd(startCoordinate, refSeq, line.slice(currentColumnStart, i));
|
|
329
|
+
}
|
|
330
|
+
else {
|
|
331
|
+
endCoordinate = parseInt(line.slice(currentColumnStart, i), 10);
|
|
332
|
+
}
|
|
333
|
+
if (endCoordinate <= regionStart) {
|
|
334
|
+
return { overlaps: false };
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
currentColumnStart = i + 1;
|
|
338
|
+
currentColumnNumber += 1;
|
|
339
|
+
if (currentColumnNumber > maxColumn) {
|
|
340
|
+
break;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
return { startCoordinate, overlaps: true };
|
|
345
|
+
}
|
|
346
|
+
_getVcfEnd(startCoordinate, refSeq, info) {
|
|
347
|
+
let endCoordinate = startCoordinate + refSeq.length;
|
|
348
|
+
// ignore TRA features as they specify CHR2 and END
|
|
349
|
+
// as being on a different chromosome
|
|
350
|
+
// if CHR2 is on the same chromosome, still ignore it
|
|
351
|
+
// because there should be another pairwise feature
|
|
352
|
+
// at the end of this one
|
|
353
|
+
const isTRA = info.indexOf('SVTYPE=TRA') !== -1;
|
|
354
|
+
if (info[0] !== '.' && !isTRA) {
|
|
355
|
+
let prevChar = ';';
|
|
356
|
+
for (let j = 0; j < info.length; j += 1) {
|
|
357
|
+
if (prevChar === ';' && info.slice(j, j + 4) === 'END=') {
|
|
358
|
+
let valueEnd = info.indexOf(';', j);
|
|
359
|
+
if (valueEnd === -1) {
|
|
360
|
+
valueEnd = info.length;
|
|
361
|
+
}
|
|
362
|
+
endCoordinate = parseInt(info.slice(j + 4, valueEnd), 10);
|
|
363
|
+
break;
|
|
364
|
+
}
|
|
365
|
+
prevChar = info[j];
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
else if (isTRA) {
|
|
369
|
+
return startCoordinate + 1;
|
|
370
|
+
}
|
|
371
|
+
return endCoordinate;
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* return the approximate number of data lines in the given reference sequence
|
|
375
|
+
* @param {string} refSeq reference sequence name
|
|
376
|
+
* @returns {Promise} for number of data lines present on that reference sequence
|
|
377
|
+
*/
|
|
378
|
+
async lineCount(refName, opts = {}) {
|
|
379
|
+
return this.index.lineCount(refName, opts);
|
|
380
|
+
}
|
|
381
|
+
async _readRegion(position, compressedSize, opts = {}) {
|
|
382
|
+
const { bytesRead, buffer } = await this.filehandle.read(Buffer.alloc(compressedSize), 0, compressedSize, position, opts);
|
|
383
|
+
return bytesRead < compressedSize ? buffer.slice(0, bytesRead) : buffer;
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* read and uncompress the data in a chunk (composed of one or more
|
|
387
|
+
* contiguous bgzip blocks) of the file
|
|
388
|
+
* @param {Chunk} chunk
|
|
389
|
+
* @returns {Promise} for a string chunk of the file
|
|
390
|
+
*/
|
|
391
|
+
async readChunk(chunk, opts = {}) {
|
|
392
|
+
// fetch the uncompressed data, uncompress carefully a block at a time,
|
|
393
|
+
// and stop when done
|
|
394
|
+
const compressedData = await this._readRegion(chunk.minv.blockPosition, chunk.fetchedSize(), opts);
|
|
395
|
+
try {
|
|
396
|
+
return (0, bgzf_filehandle_1.unzipChunkSlice)(compressedData, chunk);
|
|
397
|
+
}
|
|
398
|
+
catch (e) {
|
|
399
|
+
throw new Error(`error decompressing chunk ${chunk.toString()} ${e}`);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
exports.default = TabixIndexedFile;
|
package/esm/tbi.d.ts
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/// <reference types="node" />
|
|
2
|
+
import VirtualOffset from './virtualOffset';
|
|
3
|
+
import Chunk from './chunk';
|
|
4
|
+
import IndexFile, { Options } from './indexFile';
|
|
5
|
+
export default class TabixIndex extends IndexFile {
|
|
6
|
+
lineCount(refName: string, opts?: Options): Promise<any>;
|
|
7
|
+
_parse(opts?: Options): Promise<{
|
|
8
|
+
indices: {
|
|
9
|
+
binIndex: {
|
|
10
|
+
[key: number]: Chunk[];
|
|
11
|
+
};
|
|
12
|
+
linearIndex: any[];
|
|
13
|
+
stats: {
|
|
14
|
+
lineCount: number;
|
|
15
|
+
} | undefined;
|
|
16
|
+
}[];
|
|
17
|
+
metaChar: string | null;
|
|
18
|
+
maxBinNumber: number;
|
|
19
|
+
maxRefLength: number;
|
|
20
|
+
skipLines: any;
|
|
21
|
+
firstDataLine: VirtualOffset | undefined;
|
|
22
|
+
columnNumbers: {
|
|
23
|
+
ref: any;
|
|
24
|
+
start: any;
|
|
25
|
+
end: any;
|
|
26
|
+
};
|
|
27
|
+
coordinateType: string;
|
|
28
|
+
format: string;
|
|
29
|
+
refIdToName: string[];
|
|
30
|
+
refNameToId: {
|
|
31
|
+
[key: string]: number;
|
|
32
|
+
};
|
|
33
|
+
maxBlockSize: number;
|
|
34
|
+
}>;
|
|
35
|
+
parsePseudoBin(bytes: Buffer, offset: number): {
|
|
36
|
+
lineCount: number;
|
|
37
|
+
};
|
|
38
|
+
_parseNameBytes(namesBytes: Buffer): {
|
|
39
|
+
refNameToId: {
|
|
40
|
+
[key: string]: number;
|
|
41
|
+
};
|
|
42
|
+
refIdToName: string[];
|
|
43
|
+
};
|
|
44
|
+
blocksForRange(refName: string, min: number, max: number, opts?: Options): Promise<Chunk[]>;
|
|
45
|
+
}
|
package/esm/tbi.js
ADDED
|
@@ -0,0 +1,240 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
|
5
|
+
}) : (function(o, m, k, k2) {
|
|
6
|
+
if (k2 === undefined) k2 = k;
|
|
7
|
+
o[k2] = m[k];
|
|
8
|
+
}));
|
|
9
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
10
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
11
|
+
}) : function(o, v) {
|
|
12
|
+
o["default"] = v;
|
|
13
|
+
});
|
|
14
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
15
|
+
if (mod && mod.__esModule) return mod;
|
|
16
|
+
var result = {};
|
|
17
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
18
|
+
__setModuleDefault(result, mod);
|
|
19
|
+
return result;
|
|
20
|
+
};
|
|
21
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
22
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
23
|
+
};
|
|
24
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
25
|
+
const long_1 = __importDefault(require("long"));
|
|
26
|
+
const virtualOffset_1 = __importStar(require("./virtualOffset"));
|
|
27
|
+
const chunk_1 = __importDefault(require("./chunk"));
|
|
28
|
+
const bgzf_filehandle_1 = require("@gmod/bgzf-filehandle");
|
|
29
|
+
const util_1 = require("./util");
|
|
30
|
+
const indexFile_1 = __importDefault(require("./indexFile"));
|
|
31
|
+
const TBI_MAGIC = 21578324; // TBI\1
|
|
32
|
+
const TAD_LIDX_SHIFT = 14;
|
|
33
|
+
/**
|
|
34
|
+
* calculate the list of bins that may overlap with region [beg,end) (zero-based half-open)
|
|
35
|
+
*/
|
|
36
|
+
function reg2bins(beg, end) {
|
|
37
|
+
beg += 1; // < convert to 1-based closed
|
|
38
|
+
end -= 1;
|
|
39
|
+
return [
|
|
40
|
+
[0, 0],
|
|
41
|
+
[1 + (beg >> 26), 1 + (end >> 26)],
|
|
42
|
+
[9 + (beg >> 23), 9 + (end >> 23)],
|
|
43
|
+
[73 + (beg >> 20), 73 + (end >> 20)],
|
|
44
|
+
[585 + (beg >> 17), 585 + (end >> 17)],
|
|
45
|
+
[4681 + (beg >> 14), 4681 + (end >> 14)],
|
|
46
|
+
];
|
|
47
|
+
}
|
|
48
|
+
class TabixIndex extends indexFile_1.default {
|
|
49
|
+
async lineCount(refName, opts = {}) {
|
|
50
|
+
const indexData = await this.parse(opts);
|
|
51
|
+
if (!indexData) {
|
|
52
|
+
return -1;
|
|
53
|
+
}
|
|
54
|
+
const refId = indexData.refNameToId[refName];
|
|
55
|
+
const idx = indexData.indices[refId];
|
|
56
|
+
if (!idx) {
|
|
57
|
+
return -1;
|
|
58
|
+
}
|
|
59
|
+
const { stats } = indexData.indices[refId];
|
|
60
|
+
if (stats) {
|
|
61
|
+
return stats.lineCount;
|
|
62
|
+
}
|
|
63
|
+
return -1;
|
|
64
|
+
}
|
|
65
|
+
// memoize
|
|
66
|
+
// fetch and parse the index
|
|
67
|
+
async _parse(opts = {}) {
|
|
68
|
+
const bytes = await (0, bgzf_filehandle_1.unzip)((await this.filehandle.readFile(opts)));
|
|
69
|
+
(0, util_1.checkAbortSignal)(opts.signal);
|
|
70
|
+
// check TBI magic numbers
|
|
71
|
+
if (bytes.readUInt32LE(0) !== TBI_MAGIC /* "TBI\1" */) {
|
|
72
|
+
throw new Error('Not a TBI file');
|
|
73
|
+
// TODO: do we need to support big-endian TBI files?
|
|
74
|
+
}
|
|
75
|
+
// number of reference sequences in the index
|
|
76
|
+
const refCount = bytes.readInt32LE(4);
|
|
77
|
+
const formatFlags = bytes.readInt32LE(8);
|
|
78
|
+
const coordinateType = formatFlags & 0x10000 ? 'zero-based-half-open' : '1-based-closed';
|
|
79
|
+
const formatOpts = {
|
|
80
|
+
0: 'generic',
|
|
81
|
+
1: 'SAM',
|
|
82
|
+
2: 'VCF',
|
|
83
|
+
};
|
|
84
|
+
const format = formatOpts[formatFlags & 0xf];
|
|
85
|
+
if (!format) {
|
|
86
|
+
throw new Error(`invalid Tabix preset format flags ${formatFlags}`);
|
|
87
|
+
}
|
|
88
|
+
const columnNumbers = {
|
|
89
|
+
ref: bytes.readInt32LE(12),
|
|
90
|
+
start: bytes.readInt32LE(16),
|
|
91
|
+
end: bytes.readInt32LE(20),
|
|
92
|
+
};
|
|
93
|
+
const metaValue = bytes.readInt32LE(24);
|
|
94
|
+
const depth = 5;
|
|
95
|
+
const maxBinNumber = ((1 << ((depth + 1) * 3)) - 1) / 7;
|
|
96
|
+
const maxRefLength = 2 ** (14 + depth * 3);
|
|
97
|
+
const metaChar = metaValue ? String.fromCharCode(metaValue) : null;
|
|
98
|
+
const skipLines = bytes.readInt32LE(28);
|
|
99
|
+
// read sequence dictionary
|
|
100
|
+
const nameSectionLength = bytes.readInt32LE(32);
|
|
101
|
+
const { refNameToId, refIdToName } = this._parseNameBytes(bytes.slice(36, 36 + nameSectionLength));
|
|
102
|
+
// read the indexes for each reference sequence
|
|
103
|
+
let currOffset = 36 + nameSectionLength;
|
|
104
|
+
let firstDataLine;
|
|
105
|
+
const indices = new Array(refCount).fill(0).map(() => {
|
|
106
|
+
// the binning index
|
|
107
|
+
const binCount = bytes.readInt32LE(currOffset);
|
|
108
|
+
currOffset += 4;
|
|
109
|
+
const binIndex = {};
|
|
110
|
+
let stats;
|
|
111
|
+
for (let j = 0; j < binCount; j += 1) {
|
|
112
|
+
const bin = bytes.readUInt32LE(currOffset);
|
|
113
|
+
currOffset += 4;
|
|
114
|
+
if (bin > maxBinNumber + 1) {
|
|
115
|
+
throw new Error('tabix index contains too many bins, please use a CSI index');
|
|
116
|
+
}
|
|
117
|
+
else if (bin === maxBinNumber + 1) {
|
|
118
|
+
const chunkCount = bytes.readInt32LE(currOffset);
|
|
119
|
+
currOffset += 4;
|
|
120
|
+
if (chunkCount === 2) {
|
|
121
|
+
stats = this.parsePseudoBin(bytes, currOffset);
|
|
122
|
+
}
|
|
123
|
+
currOffset += 16 * chunkCount;
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
const chunkCount = bytes.readInt32LE(currOffset);
|
|
127
|
+
currOffset += 4;
|
|
128
|
+
const chunks = new Array(chunkCount);
|
|
129
|
+
for (let k = 0; k < chunkCount; k += 1) {
|
|
130
|
+
const u = (0, virtualOffset_1.fromBytes)(bytes, currOffset);
|
|
131
|
+
const v = (0, virtualOffset_1.fromBytes)(bytes, currOffset + 8);
|
|
132
|
+
currOffset += 16;
|
|
133
|
+
firstDataLine = this._findFirstData(firstDataLine, u);
|
|
134
|
+
chunks[k] = new chunk_1.default(u, v, bin);
|
|
135
|
+
}
|
|
136
|
+
binIndex[bin] = chunks;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
// the linear index
|
|
140
|
+
const linearCount = bytes.readInt32LE(currOffset);
|
|
141
|
+
currOffset += 4;
|
|
142
|
+
const linearIndex = new Array(linearCount);
|
|
143
|
+
for (let k = 0; k < linearCount; k += 1) {
|
|
144
|
+
linearIndex[k] = (0, virtualOffset_1.fromBytes)(bytes, currOffset);
|
|
145
|
+
currOffset += 8;
|
|
146
|
+
firstDataLine = this._findFirstData(firstDataLine, linearIndex[k]);
|
|
147
|
+
}
|
|
148
|
+
return { binIndex, linearIndex, stats };
|
|
149
|
+
});
|
|
150
|
+
return {
|
|
151
|
+
indices,
|
|
152
|
+
metaChar,
|
|
153
|
+
maxBinNumber,
|
|
154
|
+
maxRefLength,
|
|
155
|
+
skipLines,
|
|
156
|
+
firstDataLine,
|
|
157
|
+
columnNumbers,
|
|
158
|
+
coordinateType,
|
|
159
|
+
format,
|
|
160
|
+
refIdToName,
|
|
161
|
+
refNameToId,
|
|
162
|
+
maxBlockSize: 1 << 16,
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
parsePseudoBin(bytes, offset) {
|
|
166
|
+
const lineCount = (0, util_1.longToNumber)(long_1.default.fromBytesLE(bytes.slice(offset + 16, offset + 24), true));
|
|
167
|
+
return { lineCount };
|
|
168
|
+
}
|
|
169
|
+
_parseNameBytes(namesBytes) {
|
|
170
|
+
let currRefId = 0;
|
|
171
|
+
let currNameStart = 0;
|
|
172
|
+
const refIdToName = [];
|
|
173
|
+
const refNameToId = {};
|
|
174
|
+
for (let i = 0; i < namesBytes.length; i += 1) {
|
|
175
|
+
if (!namesBytes[i]) {
|
|
176
|
+
if (currNameStart < i) {
|
|
177
|
+
let refName = namesBytes.toString('utf8', currNameStart, i);
|
|
178
|
+
refName = this.renameRefSeq(refName);
|
|
179
|
+
refIdToName[currRefId] = refName;
|
|
180
|
+
refNameToId[refName] = currRefId;
|
|
181
|
+
}
|
|
182
|
+
currNameStart = i + 1;
|
|
183
|
+
currRefId += 1;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return { refNameToId, refIdToName };
|
|
187
|
+
}
|
|
188
|
+
async blocksForRange(refName, min, max, opts = {}) {
|
|
189
|
+
if (min < 0) {
|
|
190
|
+
min = 0;
|
|
191
|
+
}
|
|
192
|
+
const indexData = await this.parse(opts);
|
|
193
|
+
if (!indexData) {
|
|
194
|
+
return [];
|
|
195
|
+
}
|
|
196
|
+
const refId = indexData.refNameToId[refName];
|
|
197
|
+
const ba = indexData.indices[refId];
|
|
198
|
+
if (!ba) {
|
|
199
|
+
return [];
|
|
200
|
+
}
|
|
201
|
+
const minOffset = ba.linearIndex.length
|
|
202
|
+
? ba.linearIndex[min >> TAD_LIDX_SHIFT >= ba.linearIndex.length
|
|
203
|
+
? ba.linearIndex.length - 1
|
|
204
|
+
: min >> TAD_LIDX_SHIFT]
|
|
205
|
+
: new virtualOffset_1.default(0, 0);
|
|
206
|
+
if (!minOffset) {
|
|
207
|
+
console.warn('querying outside of possible tabix range');
|
|
208
|
+
}
|
|
209
|
+
// const { linearIndex, binIndex } = indexes
|
|
210
|
+
const overlappingBins = reg2bins(min, max); // List of bin #s that overlap min, max
|
|
211
|
+
const chunks = [];
|
|
212
|
+
// Find chunks in overlapping bins. Leaf bins (< 4681) are not pruned
|
|
213
|
+
for (const [start, end] of overlappingBins) {
|
|
214
|
+
for (let bin = start; bin <= end; bin++) {
|
|
215
|
+
if (ba.binIndex[bin]) {
|
|
216
|
+
const binChunks = ba.binIndex[bin];
|
|
217
|
+
for (let c = 0; c < binChunks.length; ++c) {
|
|
218
|
+
chunks.push(new chunk_1.default(binChunks[c].minv, binChunks[c].maxv, bin));
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
// Use the linear index to find minimum file position of chunks that could
|
|
224
|
+
// contain alignments in the region
|
|
225
|
+
const nintv = ba.linearIndex.length;
|
|
226
|
+
let lowest = null;
|
|
227
|
+
const minLin = Math.min(min >> 14, nintv - 1);
|
|
228
|
+
const maxLin = Math.min(max >> 14, nintv - 1);
|
|
229
|
+
for (let i = minLin; i <= maxLin; ++i) {
|
|
230
|
+
const vp = ba.linearIndex[i];
|
|
231
|
+
if (vp) {
|
|
232
|
+
if (!lowest || vp.compareTo(lowest) < 0) {
|
|
233
|
+
lowest = vp;
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
return (0, util_1.optimizeChunks)(chunks, lowest);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
exports.default = TabixIndex;
|