@gmod/tabix 3.5.1 → 3.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/csi.js +1 -1
- package/dist/csi.js.map +1 -1
- package/dist/tabix-bundle.js +1 -1
- package/dist/tabixIndexedFile.d.ts +1 -0
- package/dist/tabixIndexedFile.js +81 -3
- package/dist/tabixIndexedFile.js.map +1 -1
- package/dist/tbi.js +12 -15
- package/dist/tbi.js.map +1 -1
- package/dist/util.d.ts +11 -2
- package/dist/util.js +31 -5
- package/dist/util.js.map +1 -1
- package/esm/csi.js +2 -2
- package/esm/csi.js.map +1 -1
- package/esm/tabixIndexedFile.d.ts +1 -0
- package/esm/tabixIndexedFile.js +81 -3
- package/esm/tabixIndexedFile.js.map +1 -1
- package/esm/tbi.js +13 -16
- package/esm/tbi.js.map +1 -1
- package/esm/util.d.ts +11 -2
- package/esm/util.js +30 -4
- package/esm/util.js.map +1 -1
- package/package.json +2 -1
- package/src/csi.ts +2 -2
- package/src/tabixIndexedFile.ts +110 -3
- package/src/tbi.ts +13 -14
- package/src/util.ts +35 -8
package/esm/tbi.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { unzip } from '@gmod/bgzf-filehandle';
|
|
2
2
|
import Chunk from "./chunk.js";
|
|
3
3
|
import IndexFile from "./indexFile.js";
|
|
4
|
-
import { clampChunkEnds,
|
|
4
|
+
import { clampChunkEnds, memoizeByRefId, minVirtualOffset, optimizeChunks, parseAuxData, parsePseudoBin, } from "./util.js";
|
|
5
5
|
import { fromBytes } from "./virtualOffset.js";
|
|
6
6
|
const TBI_MAGIC = 21_578_324; // TBI\1
|
|
7
7
|
// TBI is always the fixed depth-5, 14-bit-leaf binning scheme
|
|
@@ -40,8 +40,15 @@ export default class TabixIndex extends IndexFile {
|
|
|
40
40
|
const { refNameToId, refIdToName, coordinateType, format, columnNumbers, metaChar, skipLines, } = parseAuxData(bytes, 8);
|
|
41
41
|
// nameSectionLength is at TBI offset 32; re-read to find where bin data starts
|
|
42
42
|
const nameSectionLength = dataView.getInt32(32, true);
|
|
43
|
-
// SYNC: ~/src/gmod/bam-js/src/
|
|
44
|
-
// First pass: record per-refId byte offsets and find firstDataLine
|
|
43
|
+
// SYNC: ~/src/gmod/bam-js/src/bai.ts _parse — two-pass structure
|
|
44
|
+
// First pass: record per-refId byte offsets and find firstDataLine.
|
|
45
|
+
//
|
|
46
|
+
// Only the linear index is consulted. Its entry for a window is the
|
|
47
|
+
// smallest virtual offset of any record overlapping that window, so the
|
|
48
|
+
// minimum over the linear index is already the minimum over the bin chunks
|
|
49
|
+
// — walking the chunks too only re-derives it. Checked against every .tbi
|
|
50
|
+
// in test/data: same answer on all 23, and no ref has bins without a
|
|
51
|
+
// linear index.
|
|
45
52
|
let curr = 36 + nameSectionLength;
|
|
46
53
|
let firstDataLine;
|
|
47
54
|
const offsets = [];
|
|
@@ -57,22 +64,12 @@ export default class TabixIndex extends IndexFile {
|
|
|
57
64
|
if (bin > maxBinNumber + 1) {
|
|
58
65
|
throw new Error('tabix index contains too many bins, please use a CSI index');
|
|
59
66
|
}
|
|
60
|
-
|
|
61
|
-
curr += 16 * chunkCount;
|
|
62
|
-
}
|
|
63
|
-
else {
|
|
64
|
-
for (let k = 0; k < chunkCount; k++) {
|
|
65
|
-
firstDataLine = findFirstData(firstDataLine, fromBytes(bytes, curr));
|
|
66
|
-
curr += 16;
|
|
67
|
-
}
|
|
68
|
-
}
|
|
67
|
+
curr += 16 * chunkCount;
|
|
69
68
|
}
|
|
70
69
|
const linearCount = dataView.getInt32(curr, true);
|
|
71
70
|
curr += 4;
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
curr += 8;
|
|
75
|
-
}
|
|
71
|
+
firstDataLine = minVirtualOffset(bytes, curr, linearCount, firstDataLine);
|
|
72
|
+
curr += 8 * linearCount;
|
|
76
73
|
}
|
|
77
74
|
function getIndices(refId) {
|
|
78
75
|
const start = offsets[refId];
|
package/esm/tbi.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tbi.js","sourceRoot":"","sources":["../src/tbi.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAA;AAE7C,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,SAAS,MAAM,gBAAgB,CAAA;AACtC,OAAO,EACL,cAAc,EACd,
|
|
1
|
+
{"version":3,"file":"tbi.js","sourceRoot":"","sources":["../src/tbi.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,uBAAuB,CAAA;AAE7C,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,SAAS,MAAM,gBAAgB,CAAA;AACtC,OAAO,EACL,cAAc,EACd,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,cAAc,GACf,MAAM,WAAW,CAAA;AAClB,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AAK9C,MAAM,SAAS,GAAG,UAAU,CAAA,CAAC,QAAQ;AAErC,8DAA8D;AAC9D,MAAM,SAAS,GAAG,CAAC,CAAA;AACnB,MAAM,kBAAkB,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,SAAS,GAAG,CAAC,CAAC,CAAA;AAEpD;;;GAGG;AACH,SAAS,QAAQ,CAAC,GAAW,EAAE,GAAW;IACxC,qEAAqE;IACrE,yEAAyE;IACzE,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAA;IACxD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,kBAAkB,CAAC,GAAG,CAAC,CAAA;IAC/C,MAAM,IAAI,GAAkC,EAAE,CAAA;IAC9C,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,IAAI,CACP,CAAC,CAAC,EAAE,CAAC,CAAC,EACN,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAC9B,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAC9B,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAChC,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAClC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CACrC,CAAA;IACH,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,MAAM,CAAC,OAAO,OAAO,UAAW,SAAQ,SAAS;IAC/C,gBAAgB;IAChB,KAAK,CAAC,MAAM,CAAC,OAAgB,EAAE;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YACzC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAA;QACF,MAAM,KAAK,GAAG,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,CAAe,CAAA;QAC9C,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAC3B,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,CACjB,CAAA;QAED,IAAI,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAA;QACnC,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;QAC5C,MAAM,YAAY,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;QAE3D,oEAAoE;QACpE,MAAM,EACJ,WAAW,EACX,WAAW,EACX,cAAc,EACd,MAAM,EACN,aAAa,EACb,QAAQ,EACR,SAAS,GACV,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC,CAAA;QAC1B,+EAA+E;QAC/E,MAAM,iBAAiB,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;QAErD,iEAAiE;QACjE,oEAAoE;QACpE,EAAE;QACF,oEAAoE;QACpE,wEAAwE;QACxE,2EAA2E;QAC3E,0EAA0E;QAC1E,qEAAqE;QACrE,gBAAgB;QAChB,IAAI,IAAI,GAAG,EAAE,GAAG,iBAAiB,CAAA;QACjC,IAAI,aAAwC,CAAA;QAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAClB,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YAC9C,IAAI,IAAI,CAAC,CAAA;YACT,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;gBAC1C,IAAI,IAAI,CAAC,CAAA;gBACT,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;gBAChD,IAAI,IAAI,CAAC,CAAA;gBACT,IAAI,GAAG,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;oBAC3B,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAA;gBACH,CAAC;gBACD,IAAI,IAAI,EAAE,GAAG,UAAU,CAAA;YACzB,CAAC;YACD,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YACjD,IAAI,IAAI,CAAC,CAAA;YACT,aAAa,GAAG,gBAAgB,CAAC,KAAK,EAAE,IAAI,EAAE,WAAW,EAAE,aAAa,CAAC,CAAA;YACzE,IAAI,IAAI,CAAC,GAAG,WAAW,CAAA;QACzB,CAAC;QAED,SAAS,UAAU,CAAC,KAAa;YAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;YAC5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,OAAO,SAAS,CAAA;YAClB,CAAC;YACD,IAAI,GAAG,GAAG,KAAK,CAAA;YACf,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YAC7C,GAAG,IAAI,CAAC,CAAA;YACR,MAAM,QAAQ,GAA4B,EAAE,CAAA;YAC5C,IAAI,KAAK,CAAA;YACT,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;gBACzC,GAAG,IAAI,CAAC,CAAA;gBACR,IAAI,GAAG,GAAG,YAAY,GAAG,CAAC,EAAE,CAAC;oBAC3B,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAA;gBACH,CAAC;qBAAM,IAAI,GAAG,KAAK,YAAY,GAAG,CAAC,EAAE,CAAC;oBACpC,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;oBAC/C,GAAG,IAAI,CAAC,CAAA;oBACR,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;wBACrB,KAAK,GAAG,cAAc,CAAC,KAAK,EAAE,GAAG,GAAG,EAAE,CAAC,CAAA;oBACzC,CAAC;oBACD,GAAG,IAAI,EAAE,GAAG,UAAU,CAAA;gBACxB,CAAC;qBAAM,CAAC;oBACN,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;oBAC/C,GAAG,IAAI,CAAC,CAAA;oBACR,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;oBACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;wBACpC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,CACnB,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,EACrB,SAAS,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,EACzB,GAAG,CACJ,CAAA;wBACD,GAAG,IAAI,EAAE,CAAA;oBACX,CAAC;oBACD,QAAQ,CAAC,GAAG,CAAC,GAAG,MAAM,CAAA;gBACxB,CAAC;YACH,CAAC;YACD,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YAChD,GAAG,IAAI,CAAC,CAAA;YACR,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAgB,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAA;YACtE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,WAAW,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;gBACtC,GAAG,IAAI,CAAC,CAAA;YACV,CAAC;YACD,cAAc,CACZ,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,EAC9B,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CACtC,CAAA;YACD,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,CAAA;QACzC,CAAC;QAED,OAAO;YACL,OAAO,EAAE,cAAc,CAAC,UAAU,CAAC;YACnC,QAAQ;YACR,YAAY;YACZ,YAAY,EAAE,kBAAkB;YAChC,SAAS;YACT,aAAa;YACb,aAAa;YACb,cAAc;YACd,MAAM;YACN,WAAW;YACX,WAAW;YACX,YAAY,EAAE,CAAC,IAAI,EAAE;SACtB,CAAA;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,OAAe,EACf,GAAW,EACX,GAAW,EACX,OAAgB,EAAE;QAElB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QAC5C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,CAAA;QACX,CAAC;QACD,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,OAAO,EAAE,CAAA;QACX,CAAC;QAED,IAAI,GAAG,GAAG,kBAAkB,EAAE,CAAC;YAC7B,OAAO,CAAC,IAAI,CAAC,0CAA0C,CAAC,CAAA;QAC1D,CAAC;QACD,qEAAqE;QACrE,sBAAsB;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAA;QAC1D,MAAM,eAAe,GAAG,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA,CAAC,uCAAuC;QAClF,MAAM,MAAM,GAAY,EAAE,CAAA;QAE1B,sEAAsE;QACtE,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,eAAe,EAAE,CAAC;YAC3C,KAAK,IAAI,GAAG,GAAG,KAAK,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;gBACxC,MAAM,SAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;gBAClC,IAAI,SAAS,EAAE,CAAC;oBACd,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,2EAA2E;QAC3E,sEAAsE;QACtE,oDAAoD;QACpD,MAAM,WAAW,GAAG,EAAE,CAAC,WAAW,CAAA;QAClC,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,EAAE,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAA;QAEzE,OAAO,cAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACvC,CAAC;CACF"}
|
package/esm/util.d.ts
CHANGED
|
@@ -1,8 +1,17 @@
|
|
|
1
1
|
import Chunk from './chunk.ts';
|
|
2
|
-
import
|
|
2
|
+
import VirtualOffset from './virtualOffset.ts';
|
|
3
3
|
export declare function optimizeChunks(chunks: Chunk[], lowest?: VirtualOffset): Chunk[];
|
|
4
4
|
export declare function clampChunkEnds(chunks: Chunk[], extraBoundaries?: number[]): void;
|
|
5
|
-
|
|
5
|
+
/**
|
|
6
|
+
* The smallest of `current` and the `count` packed virtual offsets starting at
|
|
7
|
+
* `offset`, allocating at most one VirtualOffset rather than one per entry.
|
|
8
|
+
*
|
|
9
|
+
* The index first pass exists only to find this minimum, and it visits every
|
|
10
|
+
* linear-index entry in the file to do it — 301k of them on
|
|
11
|
+
* test/data/failing_tabix.vcf.gz.tbi against 19k bin chunks. Building a
|
|
12
|
+
* VirtualOffset per entry to compare and discard it is the bulk of that pass.
|
|
13
|
+
*/
|
|
14
|
+
export declare function minVirtualOffset(bytes: Uint8Array, offset: number, count: number, current: VirtualOffset | undefined): VirtualOffset | undefined;
|
|
6
15
|
export declare function parseNameBytes(namesBytes: Uint8Array): {
|
|
7
16
|
refNameToId: Record<string, number>;
|
|
8
17
|
refIdToName: string[];
|
package/esm/util.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import LRU from '@jbrowse/quick-lru';
|
|
2
2
|
import Chunk from "./chunk.js";
|
|
3
3
|
import { longFromBytesToUnsigned } from "./long.js";
|
|
4
|
+
import VirtualOffset from "./virtualOffset.js";
|
|
4
5
|
// SYNC: ~/src/gmod/bam-js/src/util.ts optimizeChunks
|
|
5
6
|
export function optimizeChunks(chunks, lowest) {
|
|
6
7
|
const n = chunks.length;
|
|
@@ -97,10 +98,35 @@ export function clampChunkEnds(chunks, extraBoundaries = []) {
|
|
|
97
98
|
}
|
|
98
99
|
}
|
|
99
100
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
101
|
+
/**
|
|
102
|
+
* The smallest of `current` and the `count` packed virtual offsets starting at
|
|
103
|
+
* `offset`, allocating at most one VirtualOffset rather than one per entry.
|
|
104
|
+
*
|
|
105
|
+
* The index first pass exists only to find this minimum, and it visits every
|
|
106
|
+
* linear-index entry in the file to do it — 301k of them on
|
|
107
|
+
* test/data/failing_tabix.vcf.gz.tbi against 19k bin chunks. Building a
|
|
108
|
+
* VirtualOffset per entry to compare and discard it is the bulk of that pass.
|
|
109
|
+
*/
|
|
110
|
+
export function minVirtualOffset(bytes, offset, count, current) {
|
|
111
|
+
let minBlock = current ? current.blockPosition : Infinity;
|
|
112
|
+
let minData = current ? current.dataPosition : 0;
|
|
113
|
+
let found = false;
|
|
114
|
+
for (let i = 0; i < count; i++) {
|
|
115
|
+
const p = offset + i * 8;
|
|
116
|
+
const block = bytes[p + 7] * 0x1_00_00_00_00_00 +
|
|
117
|
+
bytes[p + 6] * 0x1_00_00_00_00 +
|
|
118
|
+
bytes[p + 5] * 0x1_00_00_00 +
|
|
119
|
+
bytes[p + 4] * 0x1_00_00 +
|
|
120
|
+
bytes[p + 3] * 0x1_00 +
|
|
121
|
+
bytes[p + 2];
|
|
122
|
+
const data = (bytes[p + 1] << 8) | bytes[p];
|
|
123
|
+
if (block < minBlock || (block === minBlock && data < minData)) {
|
|
124
|
+
minBlock = block;
|
|
125
|
+
minData = data;
|
|
126
|
+
found = true;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return found ? new VirtualOffset(minBlock, minData) : current;
|
|
104
130
|
}
|
|
105
131
|
export function parseNameBytes(namesBytes) {
|
|
106
132
|
const decoder = new TextDecoder('utf-8');
|
package/esm/util.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,oBAAoB,CAAA;AAEpC,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"util.js","sourceRoot":"","sources":["../src/util.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,oBAAoB,CAAA;AAEpC,OAAO,KAAK,MAAM,YAAY,CAAA;AAC9B,OAAO,EAAE,uBAAuB,EAAE,MAAM,WAAW,CAAA;AACnD,OAAO,aAAa,MAAM,oBAAoB,CAAA;AAE9C,qDAAqD;AACrD,MAAM,UAAU,cAAc,CAAC,MAAe,EAAE,MAAsB;IACpE,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IACvB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,OAAO,MAAM,CAAA;IACf,CAAC;IAED,uEAAuE;IACvE,wEAAwE;IACxE,2EAA2E;IAC3E,IAAI,QAAiB,CAAA;IACrB,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,WAAW,GAAG,MAAM,CAAC,aAAa,CAAA;QACxC,MAAM,UAAU,GAAG,MAAM,CAAC,YAAY,CAAA;QACtC,QAAQ,GAAG,EAAE,CAAA;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YACxB,MAAM,GAAG,GACP,KAAK,CAAC,IAAI,CAAC,aAAa,GAAG,WAAW;gBACtC,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,UAAU,CAAA;YACtC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACtB,CAAC;QACH,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,QAAQ,CAAA;QACjB,CAAC;IACH,CAAC;SAAM,CAAC;QACN,QAAQ,GAAG,MAAM,CAAA;IACnB,CAAC;IAED,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE;QACvB,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,IAAI,CAAC,aAAa,CAAA;QACzD,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAA;IACtE,CAAC,CAAC,CAAA;IAEF,4EAA4E;IAC5E,0DAA0D;IAC1D,oEAAoE;IACpE,MAAM,YAAY,GAAY,CAAC,QAAQ,CAAC,CAAC,CAAE,CAAC,CAAA;IAC5C,IAAI,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,aAAa,CAAA;IAClD,IAAI,YAAY,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,aAAa,CAAA;IAElD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAA;QAC1B,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAA;QAC9C,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAA;QAC9C,IACE,aAAa,GAAG,YAAY,GAAG,MAAM;YACrC,aAAa,GAAG,YAAY,GAAG,SAAS,EACxC,CAAC;YACD,MAAM,SAAS,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAA;YACtC,MAAM,GAAG,GACP,aAAa,GAAG,YAAY;gBAC5B,KAAK,CAAC,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC,IAAI,CAAC,YAAY,CAAA;YACvD,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACZ,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,KAAK,CAC/C,SAAS,CAAC,IAAI,EACd,KAAK,CAAC,IAAI,EACV,SAAS,CAAC,GAAG,EACb,KAAK,CAAC,WAAW,CAClB,CAAA;gBACD,YAAY,GAAG,aAAa,CAAA;YAC9B,CAAC;QACH,CAAC;aAAM,CAAC;YACN,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACxB,YAAY,GAAG,aAAa,CAAA;YAC5B,YAAY,GAAG,aAAa,CAAA;QAC9B,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAA;AACrB,CAAC;AAED,6EAA6E;AAC7E,kFAAkF;AAClF,kFAAkF;AAClF,iFAAiF;AACjF,kFAAkF;AAClF,iFAAiF;AACjF,yEAAyE;AACzE,MAAM,UAAU,cAAc,CAC5B,MAAe,EACf,kBAA4B,EAAE;IAE9B,uEAAuE;IACvE,wEAAwE;IACxE,qDAAqD;IACrD,MAAM,UAAU,GAAG,CAAC,GAAG,eAAe,CAAC,CAAA;IACvC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAC7D,CAAC;IACD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAEhC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,aAAa,CAAA;QAChC,2CAA2C;QAC3C,IAAI,EAAE,GAAG,CAAC,CAAA;QACV,IAAI,EAAE,GAAG,UAAU,CAAC,MAAM,CAAA;QAC1B,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAA;YAC3B,IAAI,UAAU,CAAC,GAAG,CAAE,GAAG,GAAG,EAAE,CAAC;gBAC3B,EAAE,GAAG,GAAG,CAAA;YACV,CAAC;iBAAM,CAAC;gBACN,EAAE,GAAG,GAAG,GAAG,CAAC,CAAA;YACd,CAAC;QACH,CAAC;QACD,IAAI,EAAE,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAC3B,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,EAAE,UAAU,CAAC,EAAE,CAAE,CAAC,CAAA;QAC1D,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAC9B,KAAiB,EACjB,MAAc,EACd,KAAa,EACb,OAAkC;IAElC,IAAI,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAA;IACzD,IAAI,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAA;IAChD,IAAI,KAAK,GAAG,KAAK,CAAA;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,CAAA;QACxB,MAAM,KAAK,GACT,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,GAAG,kBAAkB;YAClC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,GAAG,eAAe;YAC/B,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,GAAG,YAAY;YAC5B,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,GAAG,SAAS;YACzB,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,GAAG,MAAM;YACtB,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QACf,MAAM,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QAC7C,IAAI,KAAK,GAAG,QAAQ,IAAI,CAAC,KAAK,KAAK,QAAQ,IAAI,IAAI,GAAG,OAAO,CAAC,EAAE,CAAC;YAC/D,QAAQ,GAAG,KAAK,CAAA;YAChB,OAAO,GAAG,IAAI,CAAA;YACd,KAAK,GAAG,IAAI,CAAA;QACd,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,CAAC,IAAI,aAAa,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;AAC/D,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,UAAsB;IACnD,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAA;IACxC,MAAM,WAAW,GAAa,EAAE,CAAA;IAChC,MAAM,WAAW,GAA2B,EAAE,CAAA;IAC9C,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,OAAO,GAAG,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;QAC/B,MAAM,GAAG,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;QACtC,IAAI,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC;YACf,MAAK;QACP,CAAC;QACD,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;YAC7D,WAAW,CAAC,SAAS,CAAC,GAAG,OAAO,CAAA;YAChC,WAAW,CAAC,OAAO,CAAC,GAAG,SAAS,CAAA;QAClC,CAAC;QACD,GAAG,GAAG,GAAG,GAAG,CAAC,CAAA;QACb,SAAS,EAAE,CAAA;IACb,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,CAAA;AACrC,CAAC;AAED,MAAM,YAAY,GAA2B;IAC3C,CAAC,EAAE,SAAS;IACZ,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,KAAK;CACT,CAAA;AAED,MAAM,UAAU,YAAY,CAAC,KAAiB,EAAE,MAAc;IAC5D,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAC3B,KAAK,CAAC,MAAM,EACZ,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,CACjB,CAAA;IACD,MAAM,WAAW,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;IACnD,MAAM,cAAc,GAClB,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,gBAAgB,CAAA;IACrE,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,GAAG,GAAG,CAAC,CAAA;IAC9C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,qCAAqC,WAAW,EAAE,CAAC,CAAA;IACrE,CAAC;IACD,MAAM,aAAa,GAAG;QACpB,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC;QACxC,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC;QAC1C,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC;KAC1C,CAAA;IACD,MAAM,SAAS,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,CAAA;IACtD,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IACvE,MAAM,SAAS,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,CAAA;IACtD,MAAM,iBAAiB,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,IAAI,CAAC,CAAA;IAC9D,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,GAAG,cAAc,CACjD,KAAK,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,EAAE,MAAM,GAAG,EAAE,GAAG,iBAAiB,CAAC,CAC7D,CAAA;IACD,OAAO;QACL,WAAW;QACX,WAAW;QACX,SAAS;QACT,QAAQ;QACR,aAAa;QACb,MAAM;QACN,cAAc;KACf,CAAA;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,KAAiB,EAAE,MAAc;IAC9D,OAAO,EAAE,SAAS,EAAE,uBAAuB,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAA;AAC9D,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,cAAc,CAC5B,UAA4C,EAC5C,OAAO,GAAG,CAAC;IAEX,MAAM,GAAG,GAAG,IAAI,GAAG,CAAY,EAAE,OAAO,EAAE,CAAC,CAAA;IAC3C,OAAO,CAAC,KAAa,EAAE,EAAE;QACvB,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QAC7B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,MAAM,CAAA;QACf,CAAC;QACD,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QAChC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACzB,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACxB,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC,CAAA;AACH,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gmod/tabix",
|
|
3
|
-
"version": "3.5.
|
|
3
|
+
"version": "3.5.2",
|
|
4
4
|
"packageManager": "pnpm@11.15.1",
|
|
5
5
|
"description": "Read Tabix-indexed files, supports both .tbi and .csi indexes",
|
|
6
6
|
"type": "module",
|
|
@@ -37,6 +37,7 @@
|
|
|
37
37
|
"preversion": "pnpm lint && pnpm test --run && pnpm build",
|
|
38
38
|
"postversion": "git push --follow-tags",
|
|
39
39
|
"build": "pnpm build:esm && pnpm build:es5",
|
|
40
|
+
"differential": "vite-node scripts/differential.mjs",
|
|
40
41
|
"benchonly": "vitest bench",
|
|
41
42
|
"bench": "./scripts/build-both-branches.sh \"$BRANCH1\" \"$BRANCH2\" && vitest bench",
|
|
42
43
|
"build:docs": "typedoc && node scripts/build-docs.mjs",
|
package/src/csi.ts
CHANGED
|
@@ -4,8 +4,8 @@ import Chunk from './chunk.ts'
|
|
|
4
4
|
import IndexFile from './indexFile.ts'
|
|
5
5
|
import {
|
|
6
6
|
clampChunkEnds,
|
|
7
|
-
findFirstData,
|
|
8
7
|
memoizeByRefId,
|
|
8
|
+
minVirtualOffset,
|
|
9
9
|
optimizeChunks,
|
|
10
10
|
parseAuxData,
|
|
11
11
|
parsePseudoBin,
|
|
@@ -90,7 +90,7 @@ export default class CSI extends IndexFile {
|
|
|
90
90
|
if (bin > maxBinNumber) {
|
|
91
91
|
curr += 28 + 16 // skip pseudo-bin (loffset + nchunk + 2 chunks)
|
|
92
92
|
} else {
|
|
93
|
-
firstDataLine =
|
|
93
|
+
firstDataLine = minVirtualOffset(bytes, curr, 1, firstDataLine)
|
|
94
94
|
curr += 8 // loffset
|
|
95
95
|
const chunkCount = dataView.getInt32(curr, true)
|
|
96
96
|
curr += 4 + 16 * chunkCount
|
package/src/tabixIndexedFile.ts
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import AbortablePromiseCache from '@gmod/abortable-promise-cache'
|
|
2
2
|
import { unzip, unzipChunkSlice } from '@gmod/bgzf-filehandle'
|
|
3
|
-
import LRU from '@jbrowse/quick-lru'
|
|
4
3
|
import { LocalFile, RemoteFile } from 'generic-filehandle2'
|
|
5
4
|
|
|
6
5
|
import CSI from './csi.ts'
|
|
@@ -22,6 +21,113 @@ const SEMICOLON = 59
|
|
|
22
21
|
// going much above it buys nothing on the transport that matters.
|
|
23
22
|
const MAX_READ_AHEAD_CHUNKS = 6
|
|
24
23
|
|
|
24
|
+
// SYNC: ~/src/gmod/bam-js/src/bamFile.ts DEFAULT_MAX_CACHE_BYTES
|
|
25
|
+
//
|
|
26
|
+
// We fetch compressed and cache decompressed, and an entry is a whole chunk, so
|
|
27
|
+
// entry count says nothing about memory. How little is easy to underestimate:
|
|
28
|
+
// a dense VCF (test/data/1kg.chr1.subset.vcf.gz — 213MB over 600kb of chr1)
|
|
29
|
+
// has single index bins of 17MB compressed, 120MB decompressed. Panning it
|
|
30
|
+
// under the old 80-entry cache peaked at 2GB RSS.
|
|
31
|
+
const DEFAULT_CHUNK_CACHE_BYTES = 100 * 2 ** 20
|
|
32
|
+
|
|
33
|
+
// The entry type AbortablePromiseCache stores in its backing cache. Not exported
|
|
34
|
+
// by the package, so it is recovered from the constructor signature rather than
|
|
35
|
+
// restated (and left to drift) here.
|
|
36
|
+
type CacheEntry = NonNullable<
|
|
37
|
+
ReturnType<
|
|
38
|
+
ConstructorParameters<
|
|
39
|
+
typeof AbortablePromiseCache<Chunk, ReadChunk>
|
|
40
|
+
>[0]['cache']['get']
|
|
41
|
+
>
|
|
42
|
+
>
|
|
43
|
+
|
|
44
|
+
interface CachedChunk {
|
|
45
|
+
entry: CacheEntry
|
|
46
|
+
/** 0 until the read settles and the decompressed size is known */
|
|
47
|
+
bytes: number
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Backing store for the chunk cache, bounded by the decompressed size of the
|
|
52
|
+
* chunks it holds rather than by entry count.
|
|
53
|
+
*
|
|
54
|
+
* A chunk's size is only known once its read settles, so `set` records the
|
|
55
|
+
* entry immediately and charges it to the budget later. Unsettled entries are
|
|
56
|
+
* therefore free, which is what we want: they are reads a query is waiting on.
|
|
57
|
+
*/
|
|
58
|
+
class ByteBoundedChunkCache {
|
|
59
|
+
private entries = new Map<string, CachedChunk>()
|
|
60
|
+
private bytes = 0
|
|
61
|
+
private maxBytes: number
|
|
62
|
+
|
|
63
|
+
constructor(maxBytes: number) {
|
|
64
|
+
this.maxBytes = maxBytes
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
get byteSize() {
|
|
68
|
+
return this.bytes
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
get size() {
|
|
72
|
+
return this.entries.size
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
has(key: string) {
|
|
76
|
+
return this.entries.has(key)
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
get(key: string) {
|
|
80
|
+
const cached = this.entries.get(key)
|
|
81
|
+
if (cached) {
|
|
82
|
+
// re-insert so Map iteration order stays least-recently-used first
|
|
83
|
+
this.entries.delete(key)
|
|
84
|
+
this.entries.set(key, cached)
|
|
85
|
+
}
|
|
86
|
+
return cached?.entry
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
set(key: string, entry: CacheEntry) {
|
|
90
|
+
this.delete(key)
|
|
91
|
+
const cached = { entry, bytes: 0 }
|
|
92
|
+
this.entries.set(key, cached)
|
|
93
|
+
void entry.promise
|
|
94
|
+
.then(chunk => {
|
|
95
|
+
// a later set() may have replaced this key while the read was in
|
|
96
|
+
// flight; charging these bytes to it would then double-count
|
|
97
|
+
if (this.entries.get(key) === cached) {
|
|
98
|
+
cached.bytes = chunk.buffer.byteLength
|
|
99
|
+
this.bytes += cached.bytes
|
|
100
|
+
this.evict()
|
|
101
|
+
}
|
|
102
|
+
})
|
|
103
|
+
.catch(() => {
|
|
104
|
+
// a failed or aborted read caches nothing and costs nothing
|
|
105
|
+
})
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
delete(key: string) {
|
|
109
|
+
const cached = this.entries.get(key)
|
|
110
|
+
if (cached) {
|
|
111
|
+
this.entries.delete(key)
|
|
112
|
+
this.bytes -= cached.bytes
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
keys() {
|
|
117
|
+
return this.entries.keys()
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Evict from the least-recently-used end. The size > 1 guard means a single
|
|
121
|
+
// chunk larger than the whole budget is still kept: the caller needs it for
|
|
122
|
+
// the query in flight, and dropping it would only force a re-decompress.
|
|
123
|
+
private evict() {
|
|
124
|
+
const lru = this.entries.keys()
|
|
125
|
+
while (this.bytes > this.maxBytes && this.entries.size > 1) {
|
|
126
|
+
this.delete(lru.next().value!)
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
25
131
|
type GetLinesCallback = (
|
|
26
132
|
line: string,
|
|
27
133
|
fileOffset: number,
|
|
@@ -214,7 +320,7 @@ export default class TabixIndexedFile {
|
|
|
214
320
|
csiPath,
|
|
215
321
|
csiUrl,
|
|
216
322
|
csiFilehandle,
|
|
217
|
-
chunkCacheSize =
|
|
323
|
+
chunkCacheSize = DEFAULT_CHUNK_CACHE_BYTES,
|
|
218
324
|
}: {
|
|
219
325
|
path?: string
|
|
220
326
|
filehandle?: GenericFilehandle
|
|
@@ -225,6 +331,7 @@ export default class TabixIndexedFile {
|
|
|
225
331
|
csiPath?: string
|
|
226
332
|
csiUrl?: string
|
|
227
333
|
csiFilehandle?: GenericFilehandle
|
|
334
|
+
/** budget for the decompressed chunk cache, in bytes */
|
|
228
335
|
chunkCacheSize?: number
|
|
229
336
|
}) {
|
|
230
337
|
this.filehandle = resolveFilehandle(filehandle, path, url)
|
|
@@ -240,7 +347,7 @@ export default class TabixIndexedFile {
|
|
|
240
347
|
})
|
|
241
348
|
|
|
242
349
|
this.chunkCache = new AbortablePromiseCache<Chunk, ReadChunk>({
|
|
243
|
-
cache: new
|
|
350
|
+
cache: new ByteBoundedChunkCache(chunkCacheSize),
|
|
244
351
|
fill: (args: Chunk, signal?: AbortSignal) =>
|
|
245
352
|
this.readChunk(args, { signal }),
|
|
246
353
|
})
|
package/src/tbi.ts
CHANGED
|
@@ -4,8 +4,8 @@ import Chunk from './chunk.ts'
|
|
|
4
4
|
import IndexFile from './indexFile.ts'
|
|
5
5
|
import {
|
|
6
6
|
clampChunkEnds,
|
|
7
|
-
findFirstData,
|
|
8
7
|
memoizeByRefId,
|
|
8
|
+
minVirtualOffset,
|
|
9
9
|
optimizeChunks,
|
|
10
10
|
parseAuxData,
|
|
11
11
|
parsePseudoBin,
|
|
@@ -78,8 +78,15 @@ export default class TabixIndex extends IndexFile {
|
|
|
78
78
|
// nameSectionLength is at TBI offset 32; re-read to find where bin data starts
|
|
79
79
|
const nameSectionLength = dataView.getInt32(32, true)
|
|
80
80
|
|
|
81
|
-
// SYNC: ~/src/gmod/bam-js/src/
|
|
82
|
-
// First pass: record per-refId byte offsets and find firstDataLine
|
|
81
|
+
// SYNC: ~/src/gmod/bam-js/src/bai.ts _parse — two-pass structure
|
|
82
|
+
// First pass: record per-refId byte offsets and find firstDataLine.
|
|
83
|
+
//
|
|
84
|
+
// Only the linear index is consulted. Its entry for a window is the
|
|
85
|
+
// smallest virtual offset of any record overlapping that window, so the
|
|
86
|
+
// minimum over the linear index is already the minimum over the bin chunks
|
|
87
|
+
// — walking the chunks too only re-derives it. Checked against every .tbi
|
|
88
|
+
// in test/data: same answer on all 23, and no ref has bins without a
|
|
89
|
+
// linear index.
|
|
83
90
|
let curr = 36 + nameSectionLength
|
|
84
91
|
let firstDataLine: VirtualOffset | undefined
|
|
85
92
|
const offsets: number[] = []
|
|
@@ -97,21 +104,13 @@ export default class TabixIndex extends IndexFile {
|
|
|
97
104
|
throw new Error(
|
|
98
105
|
'tabix index contains too many bins, please use a CSI index',
|
|
99
106
|
)
|
|
100
|
-
} else if (bin === maxBinNumber + 1) {
|
|
101
|
-
curr += 16 * chunkCount
|
|
102
|
-
} else {
|
|
103
|
-
for (let k = 0; k < chunkCount; k++) {
|
|
104
|
-
firstDataLine = findFirstData(firstDataLine, fromBytes(bytes, curr))
|
|
105
|
-
curr += 16
|
|
106
|
-
}
|
|
107
107
|
}
|
|
108
|
+
curr += 16 * chunkCount
|
|
108
109
|
}
|
|
109
110
|
const linearCount = dataView.getInt32(curr, true)
|
|
110
111
|
curr += 4
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
curr += 8
|
|
114
|
-
}
|
|
112
|
+
firstDataLine = minVirtualOffset(bytes, curr, linearCount, firstDataLine)
|
|
113
|
+
curr += 8 * linearCount
|
|
115
114
|
}
|
|
116
115
|
|
|
117
116
|
function getIndices(refId: number): RefIndex | undefined {
|
package/src/util.ts
CHANGED
|
@@ -2,8 +2,7 @@ import LRU from '@jbrowse/quick-lru'
|
|
|
2
2
|
|
|
3
3
|
import Chunk from './chunk.ts'
|
|
4
4
|
import { longFromBytesToUnsigned } from './long.ts'
|
|
5
|
-
|
|
6
|
-
import type VirtualOffset from './virtualOffset.ts'
|
|
5
|
+
import VirtualOffset from './virtualOffset.ts'
|
|
7
6
|
|
|
8
7
|
// SYNC: ~/src/gmod/bam-js/src/util.ts optimizeChunks
|
|
9
8
|
export function optimizeChunks(chunks: Chunk[], lowest?: VirtualOffset) {
|
|
@@ -118,13 +117,41 @@ export function clampChunkEnds(
|
|
|
118
117
|
}
|
|
119
118
|
}
|
|
120
119
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
120
|
+
/**
|
|
121
|
+
* The smallest of `current` and the `count` packed virtual offsets starting at
|
|
122
|
+
* `offset`, allocating at most one VirtualOffset rather than one per entry.
|
|
123
|
+
*
|
|
124
|
+
* The index first pass exists only to find this minimum, and it visits every
|
|
125
|
+
* linear-index entry in the file to do it — 301k of them on
|
|
126
|
+
* test/data/failing_tabix.vcf.gz.tbi against 19k bin chunks. Building a
|
|
127
|
+
* VirtualOffset per entry to compare and discard it is the bulk of that pass.
|
|
128
|
+
*/
|
|
129
|
+
export function minVirtualOffset(
|
|
130
|
+
bytes: Uint8Array,
|
|
131
|
+
offset: number,
|
|
132
|
+
count: number,
|
|
133
|
+
current: VirtualOffset | undefined,
|
|
124
134
|
) {
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
135
|
+
let minBlock = current ? current.blockPosition : Infinity
|
|
136
|
+
let minData = current ? current.dataPosition : 0
|
|
137
|
+
let found = false
|
|
138
|
+
for (let i = 0; i < count; i++) {
|
|
139
|
+
const p = offset + i * 8
|
|
140
|
+
const block =
|
|
141
|
+
bytes[p + 7]! * 0x1_00_00_00_00_00 +
|
|
142
|
+
bytes[p + 6]! * 0x1_00_00_00_00 +
|
|
143
|
+
bytes[p + 5]! * 0x1_00_00_00 +
|
|
144
|
+
bytes[p + 4]! * 0x1_00_00 +
|
|
145
|
+
bytes[p + 3]! * 0x1_00 +
|
|
146
|
+
bytes[p + 2]!
|
|
147
|
+
const data = (bytes[p + 1]! << 8) | bytes[p]!
|
|
148
|
+
if (block < minBlock || (block === minBlock && data < minData)) {
|
|
149
|
+
minBlock = block
|
|
150
|
+
minData = data
|
|
151
|
+
found = true
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return found ? new VirtualOffset(minBlock, minData) : current
|
|
128
155
|
}
|
|
129
156
|
|
|
130
157
|
export function parseNameBytes(namesBytes: Uint8Array) {
|