@gmod/tabix 3.8.1 → 3.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -83
- package/dist/csi.js +1 -1
- package/dist/csi.js.map +1 -1
- package/dist/indexFile.d.ts +16 -0
- package/dist/indexFile.js +56 -1
- package/dist/indexFile.js.map +1 -1
- package/dist/tabix-bundle.js +1 -1
- package/dist/tabixIndexedFile.d.ts +12 -8
- package/dist/tabixIndexedFile.js +3 -2
- package/dist/tabixIndexedFile.js.map +1 -1
- package/esm/csi.js +1 -1
- package/esm/csi.js.map +1 -1
- package/esm/indexFile.d.ts +16 -0
- package/esm/indexFile.js +55 -1
- package/esm/indexFile.js.map +1 -1
- package/esm/tabixIndexedFile.d.ts +12 -8
- package/esm/tabixIndexedFile.js +3 -2
- package/esm/tabixIndexedFile.js.map +1 -1
- package/package.json +2 -2
- package/src/csi.ts +1 -1
- package/src/indexFile.ts +66 -1
- package/src/tabixIndexedFile.ts +15 -10
package/README.md
CHANGED
|
@@ -40,6 +40,25 @@ const custom = new TabixIndexedFile({
|
|
|
40
40
|
})
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
+
Over HTTP it is worth swapping in
|
|
44
|
+
[`@gmod/range-cache-filehandle`](https://github.com/GMOD/range-cache-filehandle).
|
|
45
|
+
A query fetches the index once, then reads the BGZF blocks it points at as byte
|
|
46
|
+
ranges spread through the file. Overlapping queries re-read the same blocks:
|
|
47
|
+
panning twenty half-overlapping windows across the 3.4 MB test BED file reads 11
|
|
48
|
+
MB. The cache serves those reads out of 256 KiB chunks, so neighboring blocks
|
|
49
|
+
share a request and each byte is fetched once.
|
|
50
|
+
|
|
51
|
+
```typescript
|
|
52
|
+
import { RemoteFileWithRangeCache } from '@gmod/range-cache-filehandle'
|
|
53
|
+
|
|
54
|
+
const cached = new TabixIndexedFile({
|
|
55
|
+
filehandle: new RemoteFileWithRangeCache('https://example.com/file.vcf.gz'),
|
|
56
|
+
tbiFilehandle: new RemoteFileWithRangeCache(
|
|
57
|
+
'https://example.com/file.vcf.gz.tbi',
|
|
58
|
+
),
|
|
59
|
+
})
|
|
60
|
+
```
|
|
61
|
+
|
|
43
62
|
### getLines
|
|
44
63
|
|
|
45
64
|
Fetches lines overlapping a region. `start`/`end` are 0-based half-open
|
|
@@ -73,19 +92,19 @@ await file.getLines('chr1', 200, 300, {
|
|
|
73
92
|
})
|
|
74
93
|
```
|
|
75
94
|
|
|
76
|
-
`onProgress` ticks once per
|
|
77
|
-
|
|
78
|
-
determinate progress bar.
|
|
95
|
+
`onProgress` ticks once per chunk — the run of BGZF blocks the index resolves a
|
|
96
|
+
query to — including instant ticks for chunks already cached, and the index
|
|
97
|
+
supplies `totalBytes` up front, which is enough for a determinate progress bar.
|
|
79
98
|
|
|
80
99
|
Notes:
|
|
81
100
|
|
|
82
|
-
-
|
|
101
|
+
- The scan skips meta/comment lines
|
|
83
102
|
- Line strings have no trailing whitespace
|
|
84
103
|
- Pass `undefined` for `end` to read to the end of the contig
|
|
85
104
|
- A `refName` that is not in the index yields no lines and no error, so a
|
|
86
105
|
`chr1`/`1` naming mismatch looks like an empty region. Check against
|
|
87
|
-
[`getReferenceSequenceNames`](#getreferencesequencenamesopts-promisestring)
|
|
88
|
-
a query comes back unexpectedly empty
|
|
106
|
+
[`getReferenceSequenceNames`](docs/api.md#getreferencesequencenamesopts-promisestring)
|
|
107
|
+
if a query comes back unexpectedly empty
|
|
89
108
|
- `start > end` throws a `TypeError`; `start === end` returns without reading
|
|
90
109
|
|
|
91
110
|
### Without NPM (CDN)
|
|
@@ -98,93 +117,77 @@ See [example/index.html](example/index.html) for a working demo. It fetches the
|
|
|
98
117
|
VCF over HTTP, so serve the directory (e.g. `npx serve example`) rather than
|
|
99
118
|
opening the file directly.
|
|
100
119
|
|
|
101
|
-
##
|
|
102
|
-
|
|
103
|
-
### `new TabixIndexedFile(args)`
|
|
104
|
-
|
|
105
|
-
| Arg | Type | Description |
|
|
106
|
-
| ------------------------- | -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
107
|
-
| `path` | `string?` | Local file path |
|
|
108
|
-
| `url` | `string?` | Remote URL |
|
|
109
|
-
| `filehandle` | `GenericFilehandle?` | Custom filehandle (from [generic-filehandle2](https://github.com/GMOD/generic-filehandle2)) |
|
|
110
|
-
| `tbiPath` | `string?` | TBI index path (defaults to `path + '.tbi'`) |
|
|
111
|
-
| `tbiUrl` | `string?` | TBI index URL |
|
|
112
|
-
| `tbiFilehandle` | `GenericFilehandle?` | TBI index filehandle |
|
|
113
|
-
| `csiPath` | `string?` | CSI index path |
|
|
114
|
-
| `csiUrl` | `string?` | CSI index URL |
|
|
115
|
-
| `csiFilehandle` | `GenericFilehandle?` | CSI index filehandle |
|
|
116
|
-
| `chunkCacheSize` | `number?` | Chunk LRU cache budget, in _decompressed_ bytes (default 1 GiB). A retention bound, not a bound on peak memory. Size it to hold several queries: below one query's working set the hit rate drops to zero while the memory is retained anyway |
|
|
117
|
-
| `chunkCacheIdleTimeoutMs` | `number?` | Drop a cached chunk once nothing has read it for this long (default 3 minutes, `0` disables). The only thing that lowers the cache while nothing is happening, and what makes the budget above a peak rather than a resting level |
|
|
118
|
-
|
|
119
|
-
### `getLines(refName, start, end, opts)`
|
|
120
|
-
|
|
121
|
-
Calls the line callback for each line overlapping `[start, end)`. `start`
|
|
122
|
-
defaults to `0` and `end` to the end of the contig when `undefined`. `opts` is
|
|
123
|
-
either the callback itself or an object:
|
|
124
|
-
|
|
125
|
-
| Option | Type | Description |
|
|
126
|
-
| -------------- | -------------------------------------------------------- | --------------------------------------- |
|
|
127
|
-
| `lineCallback` | `(line, fileOffset, start, end) => void` | Required |
|
|
128
|
-
| `signal` | `AbortSignal?` | Aborts the in-flight reads |
|
|
129
|
-
| `onProgress` | `(bytesDownloaded: number, totalBytes?: number) => void` | Called as compressed blocks are fetched |
|
|
130
|
-
|
|
131
|
-
### `getHeader(opts?): Promise<string>`
|
|
132
|
-
|
|
133
|
-
Returns all comment/meta lines before the first data line as a string, matching
|
|
134
|
-
what `tabix -H` prints. A header row that is not commented is therefore not
|
|
135
|
-
included, even when the index counted it as a line to skip — see
|
|
136
|
-
`getSkippedLines`.
|
|
120
|
+
## How a query flows
|
|
137
121
|
|
|
138
|
-
|
|
122
|
+
`getLines` turns a region into BGZF chunks through the index and decompresses
|
|
123
|
+
each one in wasm — index reads included, since `.tbi` and `.csi` are bgzipped
|
|
124
|
+
too. The rest is ordinary JS: it matches lines as bytes and decodes only the
|
|
125
|
+
ones you asked for. [docs/dataflow.md](docs/dataflow.md) has the diagram and
|
|
126
|
+
walks it through.
|
|
139
127
|
|
|
140
|
-
|
|
128
|
+
The file then holds on to those decompressed chunks, so overlapping and adjacent
|
|
129
|
+
queries reuse them instead of inflating again — up to 1GB per file, dropped
|
|
130
|
+
after three idle minutes. A consumer holding one file per track should bound
|
|
131
|
+
them together with a shared `chunkCacheBudget` rather than shrinking each file's
|
|
132
|
+
own ceiling: [docs/caching.md](docs/caching.md).
|
|
141
133
|
|
|
142
|
-
|
|
134
|
+
## Decompressing on a worker pool
|
|
143
135
|
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
which PLINK `.ld`, bedGraph and BED deflines routinely are.
|
|
136
|
+
BGZF blocks inflate independently, so that decompression can spread across
|
|
137
|
+
threads.
|
|
147
138
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
character, but `tabix -H` prints only the latter.
|
|
151
|
-
|
|
152
|
-
### `getHeaderLines(opts?): Promise<string[]>`
|
|
153
|
-
|
|
154
|
-
Returns the file's header lines however the file keeps them: the commented block
|
|
155
|
-
when there is one, and otherwise the rows the index counted. Empty lines are
|
|
156
|
-
dropped.
|
|
157
|
-
|
|
158
|
-
This is usually the one you want. Reading `getHeader` alone cannot tell a file
|
|
159
|
-
that has no header from one whose header is not commented — both come back as
|
|
160
|
-
the empty string — so callers fall back to an assumed column layout and quietly
|
|
161
|
-
mis-name columns. Both forms are parsed from a single read of the file's leading
|
|
162
|
-
blocks, which is also shared with `getHeader` and `getSkippedLines`.
|
|
163
|
-
|
|
164
|
-
### `getReferenceSequenceNames(opts?): Promise<string[]>`
|
|
165
|
-
|
|
166
|
-
Returns reference sequence names in index order.
|
|
167
|
-
|
|
168
|
-
### `lineCount(refName, opts?): Promise<number>`
|
|
169
|
-
|
|
170
|
-
Returns the number of data lines on the given reference, or `-1` if the
|
|
171
|
-
reference is not in the index.
|
|
172
|
-
|
|
173
|
-
### `bytesForRegions(regions, opts?): Promise<number>`
|
|
174
|
-
|
|
175
|
-
Estimates the compressed byte size of index chunks covering the given regions.
|
|
176
|
-
Useful for deciding whether a request is too large before calling `getLines`.
|
|
139
|
+
```typescript
|
|
140
|
+
import { getSharedWorkerPool } from '@gmod/bgzf-filehandle'
|
|
177
141
|
|
|
178
|
-
|
|
142
|
+
const file = new TabixIndexedFile({
|
|
143
|
+
url: 'https://example.com/yourfile.vcf.gz',
|
|
144
|
+
// the pending promise is fine — it is awaited at the point of use
|
|
145
|
+
bgzfWorkerPool: getSharedWorkerPool(),
|
|
146
|
+
})
|
|
147
|
+
```
|
|
179
148
|
|
|
180
|
-
|
|
149
|
+
Safe to pass unconditionally: `getSharedWorkerPool()` returns `undefined` under
|
|
150
|
+
node, or anywhere the host forbids Workers, which keeps the in-process path. No
|
|
151
|
+
cross-origin isolation needed. tabix-js never creates a pool on its own — the
|
|
152
|
+
consumer controls the thread budget.
|
|
153
|
+
|
|
154
|
+
**Worth about 1.4x here, against the 1.95x a BAM reader reports.** Measured in
|
|
155
|
+
jbrowse-components on `test/data/1kg.chr1.subset.vcf.gz` — 213MB of 1000
|
|
156
|
+
Genomes, headless Chrome, real HTTP, four workers, arms interleaved, both
|
|
157
|
+
returning the same record count: **1.34-1.46x** across five window sizes and a
|
|
158
|
+
twelve-step pan.
|
|
159
|
+
|
|
160
|
+
The decompression itself moves **1.83x**. A **28% floor of per-line byte
|
|
161
|
+
scanning and string decoding** holds the end-to-end figure below that; no worker
|
|
162
|
+
count reaches it, and the floor is at its worst on multi-sample VCF, whose
|
|
163
|
+
records carry a genotype field per sample and run to ~60KB a line. A format with
|
|
164
|
+
narrower lines sits closer to BAM. Getting more than ~1.5x on a multi-sample VCF
|
|
165
|
+
means attacking the scan, not the decompression.
|
|
166
|
+
|
|
167
|
+
Worker counts, lifecycle and benchmarks:
|
|
168
|
+
[bgzf-filehandle's worker pool docs](https://github.com/GMOD/bgzf-filehandle/blob/main/docs/worker-pool.md);
|
|
169
|
+
the end-to-end numbers above, and how to confirm a pool is really engaging in
|
|
170
|
+
production rather than quietly falling back, are in jbrowse-components'
|
|
171
|
+
[BGZF_WORKER_POOL.md](https://github.com/GMOD/jbrowse-components/blob/main/agent-docs/reference/BGZF_WORKER_POOL.md).
|
|
172
|
+
|
|
173
|
+
## Docs
|
|
174
|
+
|
|
175
|
+
- [docs/api.md](docs/api.md) — every constructor arg and method
|
|
176
|
+
- [docs/dataflow.md](docs/dataflow.md) — a query end to end, diagrammed
|
|
177
|
+
- [docs/optimizations.md](docs/optimizations.md) — why each step of that path
|
|
178
|
+
looks the way it does, and what measured it
|
|
179
|
+
- [docs/caching.md](docs/caching.md) — sizing the decompressed-chunk cache, and
|
|
180
|
+
bounding many files together
|
|
181
|
+
- [agent-docs/adr/](agent-docs/adr/) — the measurements behind those decisions
|
|
182
|
+
- [agent-docs/TODO.md](agent-docs/TODO.md) — what is worth doing next, and what
|
|
183
|
+
has to be measured before it
|
|
184
|
+
- [CONTRIBUTING.md](CONTRIBUTING.md) — development and release steps
|
|
181
185
|
|
|
182
186
|
## Academic Use
|
|
183
187
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
be linked from [jbrowse.org](http://jbrowse.org).
|
|
188
|
+
Written with [NHGRI](http://genome.gov) funding as part of
|
|
189
|
+
[JBrowse](http://jbrowse.org). If you use this in a publication, please cite the
|
|
190
|
+
most recent JBrowse paper at [jbrowse.org](http://jbrowse.org).
|
|
188
191
|
|
|
189
192
|
## License
|
|
190
193
|
|
package/dist/csi.js
CHANGED
|
@@ -34,7 +34,7 @@ class CSI extends indexFile_ts_1.default {
|
|
|
34
34
|
}
|
|
35
35
|
const minShift = dataView.getInt32(4, true);
|
|
36
36
|
const depth = dataView.getInt32(8, true);
|
|
37
|
-
const maxBinNumber = (
|
|
37
|
+
const maxBinNumber = (8 ** (depth + 1) - 1) / 7;
|
|
38
38
|
const maxRefLength = 2 ** (minShift + depth * 3);
|
|
39
39
|
const auxLength = dataView.getInt32(12, true);
|
|
40
40
|
const aux = auxLength >= 30
|
package/dist/csi.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"csi.js","sourceRoot":"","sources":["../src/csi.ts"],"names":[],"mappings":";;;;;AAAA,0DAA8B;AAC9B,kEAAsC;AACtC,uCAMkB;AAClB,yDAA8C;AAK9C,MAAM,UAAU,GAAG,UAAU,CAAA,CAAC,QAAQ;AACtC,MAAM,UAAU,GAAG,UAAU,CAAA,CAAC,QAAQ;AAEtC,gFAAgF;AAChF,wDAAwD;AACxD,SAAS,MAAM,CAAC,GAAW,EAAE,IAAY;IACvC,OAAO,GAAG,GAAG,CAAC,IAAI,IAAI,CAAA;AACxB,CAAC;AACD,SAAS,MAAM,CAAC,GAAW,EAAE,IAAY;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,CAAA;AACpC,CAAC;AAED,MAAqB,GAAI,SAAQ,sBAAS;IACxC,gBAAgB;IAChB,KAAK,CAAC,MAAM,CAAC,OAAgB,EAAE;QAC7B,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;QAE3D,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;QACzC,IAAI,UAAU,CAAA;QACd,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YACzB,UAAU,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YAChC,UAAU,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,yBAAyB,KAAK,GAAG,CAAC,CAAA;QACpD,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;QAC3C,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;QACxC,MAAM,YAAY,GAAG,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"csi.js","sourceRoot":"","sources":["../src/csi.ts"],"names":[],"mappings":";;;;;AAAA,0DAA8B;AAC9B,kEAAsC;AACtC,uCAMkB;AAClB,yDAA8C;AAK9C,MAAM,UAAU,GAAG,UAAU,CAAA,CAAC,QAAQ;AACtC,MAAM,UAAU,GAAG,UAAU,CAAA,CAAC,QAAQ;AAEtC,gFAAgF;AAChF,wDAAwD;AACxD,SAAS,MAAM,CAAC,GAAW,EAAE,IAAY;IACvC,OAAO,GAAG,GAAG,CAAC,IAAI,IAAI,CAAA;AACxB,CAAC;AACD,SAAS,MAAM,CAAC,GAAW,EAAE,IAAY;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,CAAA;AACpC,CAAC;AAED,MAAqB,GAAI,SAAQ,sBAAS;IACxC,gBAAgB;IAChB,KAAK,CAAC,MAAM,CAAC,OAAgB,EAAE;QAC7B,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAA;QAE3D,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;QACzC,IAAI,UAAU,CAAA;QACd,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YACzB,UAAU,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YAChC,UAAU,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,yBAAyB,KAAK,GAAG,CAAC,CAAA;QACpD,CAAC;QAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;QAC3C,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,CAAA;QACxC,MAAM,YAAY,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;QAC/C,MAAM,YAAY,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAA;QAChD,MAAM,SAAS,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC,CAAA;QAC7C,MAAM,GAAG,GACP,SAAS,IAAI,EAAE;YACb,CAAC,CAAC,IAAA,sBAAY,EAAC,KAAK,EAAE,EAAE,CAAC;YACzB,CAAC,CAAC;gBACE,WAAW,EAAE,EAAc;gBAC3B,WAAW,EAAE,EAA4B;gBACzC,QAAQ,EAAE,SAAS;gBACnB,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE;gBAC3C,cAAc,EAAE,sBAAsB;gBACtC,MAAM,EAAE,SAAS;aAClB,CAAA;QACP,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,EAAE,GAAG,SAAS,EAAE,IAAI,CAAC,CAAA;QAExD,iEAAiE;QACjE,gFAAgF;QAChF,IAAI,IAAI,GAAG,EAAE,GAAG,SAAS,GAAG,CAAC,CAAA;QAC7B,IAAI,aAAwC,CAAA;QAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAClB,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;YAC9C,IAAI,IAAI,CAAC,CAAA;YACT,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;gBAC1C,IAAI,IAAI,CAAC,CAAA;gBACT,IAAI,GAAG,GAAG,YAAY,EAAE,CAAC;oBACvB,IAAI,IAAI,EAAE,GAAG,EAAE,CAAA,CAAC,gDAAgD;gBAClE,CAAC;qBAAM,CAAC;oBACN,aAAa,GAAG,IAAA,0BAAgB,EAAC,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,CAAC,CAAA;oBAC/D,IAAI,IAAI,CAAC,CAAA,CAAC,UAAU;oBACpB,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;oBAChD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAA;gBAC7B,CAAC;YACH,CAAC;QACH,CAAC;QAED,SAAS,UAAU,CAAC,KAAa;YAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;YAC5B,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxB,OAAO,SAAS,CAAA;YAClB,CAAC;YACD,IAAI,GAAG,GAAG,KAAK,CAAA;YACf,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YAC7C,GAAG,IAAI,CAAC,CAAA;YACR,MAAM,QAAQ,GAA4B,EAAE,CAAA;YAC5C,MAAM,QAAQ,GAAkC,EAAE,CAAA;YAClD,IAAI,KAAK,CAAA;YACT,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;gBAClC,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;gBACzC,GAAG,IAAI,CAAC,CAAA;gBACR,IAAI,GAAG,GAAG,YAAY,EAAE,CAAC;oBACvB,KAAK,GAAG,IAAA,wBAAc,EAAC,KAAK,EAAE,GAAG,GAAG,EAAE,CAAC,CAAA;oBACvC,GAAG,IAAI,EAAE,GAAG,EAAE,CAAA;gBAChB,CAAC;qBAAM,CAAC;oBACN,QAAQ,CAAC,GAAG,CAAC,GAAG,IAAA,4BAAS,EAAC,KAAK,EAAE,GAAG,CAAC,CAAA;oBACrC,GAAG,IAAI,CAAC,CAAA;oBACR,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;oBAC/C,GAAG,IAAI,CAAC,CAAA;oBACR,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAA;oBACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;wBACpC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,kBAAK,CACnB,IAAA,4BAAS,EAAC,KAAK,EAAE,GAAG,CAAC,EACrB,IAAA,4BAAS,EAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,EACzB,GAAG,CACJ,CAAA;wBACD,GAAG,IAAI,EAAE,CAAA;oBACX,CAAC;oBACD,QAAQ,CAAC,GAAG,CAAC,GAAG,MAAM,CAAA;gBACxB,CAAC;YACH,CAAC;YACD,IAAA,wBAAc,EAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAA;QACtC,CAAC;QAED,OAAO;YACL,GAAG,GAAG;YACN,GAAG,EAAE,IAAI;YACT,QAAQ;YACR,YAAY,EAAE,CAAC,IAAI,EAAE;YACrB,aAAa;YACb,UAAU;YACV,OAAO,EAAE,IAAA,wBAAc,EAAC,UAAU,CAAC;YACnC,QAAQ;YACR,KAAK;YACL,YAAY;YACZ,YAAY;SACb,CAAA;IACH,CAAC;IAED;;;;;;OAMG;IACO,YAAY,CACpB,GAAa,EACb,GAAW,EACX,EAAE,QAAQ,EAAE,KAAK,EAAa;QAE9B,MAAM,EAAE,QAAQ,EAAE,GAAG,GAAG,CAAA;QACxB,IAAI,KAAgC,CAAA;QACpC,IAAI,QAAQ,EAAE,CAAC;YACb,0DAA0D;YAC1D,IAAI,GAAG,GACL,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAA;YACrE,OAAO,KAAK,KAAK,SAAS,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACtC,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;gBACrB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;oBACxC,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,CAAA;oBACjC,GAAG,GAAG,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;gBAC3C,CAAC;YACH,CAAC;YACD,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAA;QACvB,CAAC;QACD,OAAO,KAAK,CAAA;IACd,CAAC;IAES,QAAQ,CAChB,GAAW,EACX,GAAW,EACX,EAAE,QAAQ,EAAE,KAAK,EAAE,YAAY,EAAa;QAE5C,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QACtB,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,CAAA;QAC1C,IAAI,GAAG,GAAG,MAAM,EAAE,CAAC;YACjB,GAAG,GAAG,MAAM,CAAA;QACd,CAAC;QACD,GAAG,IAAI,CAAC,CAAA;QACR,IAAI,CAAC,GAAG,CAAC,CAAA;QACT,IAAI,CAAC,GAAG,CAAC,CAAA;QACT,IAAI,CAAC,GAAG,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAA;QAC5B,MAAM,IAAI,GAAkC,EAAE,CAAA;QAC9C,OAAO,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACzD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YAC5B,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,SAAS,GAAG,IAAI,GAAG,mDAAmD,QAAQ,WAAW,KAAK,0DAA0D,CACzJ,CAAA;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAU,CAAC,CAAA;QAC5B,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;CACF;AAxKD,sBAwKC"}
|
package/dist/indexFile.d.ts
CHANGED
|
@@ -1,6 +1,21 @@
|
|
|
1
1
|
import type Chunk from './chunk.ts';
|
|
2
2
|
import type VirtualOffset from './virtualOffset.ts';
|
|
3
3
|
import type { GenericFilehandle } from 'generic-filehandle2';
|
|
4
|
+
/**
|
|
5
|
+
* The virtual offset past which a coordinate-sorted file holds nothing
|
|
6
|
+
* overlapping `[.., end)`, from the binning index alone: htslib's `max_off`
|
|
7
|
+
* (`hts_itr_query` in hts.c).
|
|
8
|
+
*
|
|
9
|
+
* Walk right from the finest bin after the one holding `end - 1`, stepping up
|
|
10
|
+
* to the parent at every first child, so each bin visited begins at or past
|
|
11
|
+
* `end` and never overlaps the query. Every record in such a bin starts at or
|
|
12
|
+
* past `end`, so the first chunk of the first bin that exists is a record past
|
|
13
|
+
* the query, and in a sorted file so is every record after it.
|
|
14
|
+
*
|
|
15
|
+
* SYNC: ~/src/gmod/bam-js/src/indexFile.ts maxOffset, and see bam-js ADR 0023
|
|
16
|
+
* for why the caller drops whole merged chunks with it rather than trimming.
|
|
17
|
+
*/
|
|
18
|
+
export declare function maxOffset(binIndex: Record<number, Chunk[]>, end: number, minShift: number, depth: number): VirtualOffset | undefined;
|
|
4
19
|
export interface Options {
|
|
5
20
|
signal?: AbortSignal;
|
|
6
21
|
/**
|
|
@@ -111,6 +126,7 @@ export default abstract class IndexFile {
|
|
|
111
126
|
* The chunks of the data file that may hold records overlapping the region.
|
|
112
127
|
* The two index formats differ only in their binning scheme and in where
|
|
113
128
|
* they keep the pruning floor, which is what the two hooks above supply.
|
|
129
|
+
* The ceiling, `maxOffset`, needs only the bins, so it serves both.
|
|
114
130
|
*
|
|
115
131
|
* @internal
|
|
116
132
|
*/
|
package/dist/indexFile.js
CHANGED
|
@@ -1,8 +1,53 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.maxOffset = maxOffset;
|
|
3
4
|
const bgzf_filehandle_1 = require("@gmod/bgzf-filehandle");
|
|
4
5
|
const shared_read_cache_1 = require("@gmod/shared-read-cache");
|
|
5
6
|
const util_ts_1 = require("./util.js");
|
|
7
|
+
/**
|
|
8
|
+
* The virtual offset past which a coordinate-sorted file holds nothing
|
|
9
|
+
* overlapping `[.., end)`, from the binning index alone: htslib's `max_off`
|
|
10
|
+
* (`hts_itr_query` in hts.c).
|
|
11
|
+
*
|
|
12
|
+
* Walk right from the finest bin after the one holding `end - 1`, stepping up
|
|
13
|
+
* to the parent at every first child, so each bin visited begins at or past
|
|
14
|
+
* `end` and never overlaps the query. Every record in such a bin starts at or
|
|
15
|
+
* past `end`, so the first chunk of the first bin that exists is a record past
|
|
16
|
+
* the query, and in a sorted file so is every record after it.
|
|
17
|
+
*
|
|
18
|
+
* SYNC: ~/src/gmod/bam-js/src/indexFile.ts maxOffset, and see bam-js ADR 0023
|
|
19
|
+
* for why the caller drops whole merged chunks with it rather than trimming.
|
|
20
|
+
*/
|
|
21
|
+
function maxOffset(binIndex, end, minShift, depth) {
|
|
22
|
+
if (end > 2 ** (minShift + depth * 3)) {
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
const binCount = (8 ** (depth + 1) - 1) / 7;
|
|
26
|
+
let bin = (8 ** depth - 1) / 7 + Math.floor((end - 1) / 2 ** minShift) + 1;
|
|
27
|
+
if (bin >= binCount) {
|
|
28
|
+
bin = 0;
|
|
29
|
+
}
|
|
30
|
+
for (;;) {
|
|
31
|
+
while (bin % 8 === 1) {
|
|
32
|
+
bin = (bin - 1) / 8;
|
|
33
|
+
}
|
|
34
|
+
if (bin === 0) {
|
|
35
|
+
return undefined;
|
|
36
|
+
}
|
|
37
|
+
const chunks = binIndex[bin];
|
|
38
|
+
if (chunks?.length) {
|
|
39
|
+
let lowest = chunks[0].minv;
|
|
40
|
+
for (let i = 1; i < chunks.length; i++) {
|
|
41
|
+
const minv = chunks[i].minv;
|
|
42
|
+
if (minv.compareTo(lowest) < 0) {
|
|
43
|
+
lowest = minv;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return lowest;
|
|
47
|
+
}
|
|
48
|
+
bin++;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
6
51
|
class IndexFile {
|
|
7
52
|
filehandle;
|
|
8
53
|
/**
|
|
@@ -46,6 +91,7 @@ class IndexFile {
|
|
|
46
91
|
* The chunks of the data file that may hold records overlapping the region.
|
|
47
92
|
* The two index formats differ only in their binning scheme and in where
|
|
48
93
|
* they keep the pruning floor, which is what the two hooks above supply.
|
|
94
|
+
* The ceiling, `maxOffset`, needs only the bins, so it serves both.
|
|
49
95
|
*
|
|
50
96
|
* @internal
|
|
51
97
|
*/
|
|
@@ -71,7 +117,16 @@ class IndexFile {
|
|
|
71
117
|
}
|
|
72
118
|
}
|
|
73
119
|
}
|
|
74
|
-
|
|
120
|
+
const merged = (0, util_ts_1.optimizeChunks)(chunks, this.lowestOffset(ba, min, indexData));
|
|
121
|
+
const past = maxOffset(ba.binIndex, max, indexData.minShift, indexData.depth);
|
|
122
|
+
if (past) {
|
|
123
|
+
let n = merged.length;
|
|
124
|
+
while (n > 0 && merged[n - 1].minv.compareTo(past) >= 0) {
|
|
125
|
+
n--;
|
|
126
|
+
}
|
|
127
|
+
merged.length = n;
|
|
128
|
+
}
|
|
129
|
+
return merged;
|
|
75
130
|
}
|
|
76
131
|
// SYNC: ~/src/gmod/bam-js/src/indexFile.ts parse — same shape and the same
|
|
77
132
|
// reasoning below.
|
package/dist/indexFile.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"indexFile.js","sourceRoot":"","sources":["../src/indexFile.ts"],"names":[],"mappings":";;
|
|
1
|
+
{"version":3,"file":"indexFile.js","sourceRoot":"","sources":["../src/indexFile.ts"],"names":[],"mappings":";;AAuBA,8BAkCC;AAzDD,2DAA6C;AAC7C,+DAAyD;AAEzD,uCAA0C;AAM1C;;;;;;;;;;;;;GAaG;AACH,SAAgB,SAAS,CACvB,QAAiC,EACjC,GAAW,EACX,QAAgB,EAChB,KAAa;IAEb,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;QACtC,OAAO,SAAS,CAAA;IAClB,CAAC;IACD,MAAM,QAAQ,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;IAC3C,IAAI,GAAG,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAA;IAC1E,IAAI,GAAG,IAAI,QAAQ,EAAE,CAAC;QACpB,GAAG,GAAG,CAAC,CAAA;IACT,CAAC;IACD,SAAS,CAAC;QACR,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAA;QACrB,CAAC;QACD,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC;YACd,OAAO,SAAS,CAAA;QAClB,CAAC;QACD,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;YACnB,IAAI,MAAM,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAA;YAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAA;gBAC5B,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC/B,MAAM,GAAG,IAAI,CAAA;gBACf,CAAC;YACH,CAAC;YACD,OAAO,MAAM,CAAA;QACf,CAAC;QACD,GAAG,EAAE,CAAA;IACP,CAAC;AACH,CAAC;AA6CD,MAA8B,SAAS;IAC9B,UAAU,CAAmB;IACpC;;;OAGG;IACK,UAAU,GAAG,IAAI,mCAAe,CAAoB,EAAE,CAAC,CAAA;IAE/D,YAAY,EAAE,UAAU,EAAqC;QAC3D,IAAI,CAAC,UAAU,GAAG,UAAU,CAAA;IAC9B,CAAC;IAyBD;;;OAGG;IACO,KAAK,CAAC,cAAc,CAAC,IAAa;QAC1C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YACzC,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAA;QACF,MAAM,KAAK,GAAG,MAAM,IAAA,uBAAK,EAAC,GAAG,CAAC,CAAA;QAC9B,OAAO;YACL,KAAK;YACL,QAAQ,EAAE,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC;SACzE,CAAA;IACH,CAAC;IAED,gBAAgB;IACT,KAAK,CAAC,SAAS,CAAC,OAAe,EAAE,OAAgB,EAAE;QACxD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QAC5C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,CAAC,CAAC,CAAA;QACX,CAAC;QACD,OAAO,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,SAAS,IAAI,CAAC,CAAC,CAAA;IACzD,CAAC;IAED,gBAAgB;IACT,KAAK,CAAC,WAAW,CAAC,OAAgB,EAAE;QACzC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAC7D,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,cAAc,CACzB,OAAe,EACf,GAAW,EACX,GAAW,EACX,OAAgB,EAAE;QAElB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAA;QAC5C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,OAAO,EAAE,CAAA;QACX,CAAC;QACD,MAAM,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,OAAO,EAAE,CAAA;QACX,CAAC;QAED,6DAA6D;QAC7D,MAAM,MAAM,GAAY,EAAE,CAAA;QAC1B,KAAK,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,EAAE,SAAS,CAAC,EAAE,CAAC;YAC9D,KAAK,IAAI,GAAG,GAAG,KAAK,EAAE,GAAG,IAAI,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;gBACxC,MAAM,SAAS,GAAG,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;gBAClC,IAAI,SAAS,EAAE,CAAC;oBACd,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;wBAC1B,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,IAAA,wBAAc,EAAC,MAAM,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC,CAAA;QAC5E,MAAM,IAAI,GAAG,SAAS,CACpB,EAAE,CAAC,QAAQ,EACX,GAAG,EACH,SAAS,CAAC,QAAQ,EAClB,SAAS,CAAC,KAAK,CAChB,CAAA;QACD,IAAI,IAAI,EAAE,CAAC;YACT,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;YACrB,OAAO,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzD,CAAC,EAAE,CAAA;YACL,CAAC;YACD,MAAM,CAAC,MAAM,GAAG,CAAC,CAAA;QACnB,CAAC;QACD,OAAO,MAAM,CAAA;IACf,CAAC;IAED,2EAA2E;IAC3E,mBAAmB;IACnB;;;;;;;;;;;;;;;;;;;;;;;;;OAyBG;IACH,KAAK,CAAC,OAAgB,EAAE;QACtB,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CACxD,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,CAAC,CACjC,CAAA;IACH,CAAC;IAED,gBAAgB;IAChB,KAAK,CAAC,SAAS,CAAC,KAAa,EAAE,OAAgB,EAAE;QAC/C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAClC,OAAO,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAA;IACvC,CAAC;CACF;AAhKD,4BAgKC"}
|