minimap2 0.2.22.0 → 0.2.24.1

data/ext/minimap2/misc/README.md

@@ -0,0 +1,179 @@
+## <a name="started"></a>Getting Started
+
+```sh
+# install minimap2
+git clone https://github.com/lh3/minimap2
+cd minimap2 && make
+# install the k8 javascript shell
+curl -L https://github.com/attractivechaos/k8/releases/download/v0.2.4/k8-0.2.4.tar.bz2 | tar -jxf -
+cp k8-0.2.4/k8-`uname -s` k8              # or copy it to a directory on your $PATH
+# export PATH="$PATH:`pwd`:`pwd`/misc"    # run this if k8, minimap2 or paftools.js not on your $PATH
+minimap2 --cs test/MT-human.fa test/MT-orang.fa | paftools.js view -     # view alignment
+minimap2 -c test/MT-human.fa test/MT-orang.fa | paftools.js stat -       # basic alignment statistics
+minimap2 -c --cs test/MT-human.fa test/MT-orang.fa \
+  | sort -k6,6 -k8,8n | paftools.js call -L15000 -     # calling variants from asm-to-ref alignment
+minimap2 -c test/MT-human.fa test/MT-orang.fa \
+  | paftools.js liftover -l10000 - <(echo -e "MT_orang\t2000\t5000")     # liftOver
+# no test data for the following examples
+paftools.js junceval -e anno.gtf splice.sam > out.txt  # compare splice junctions to annotations
+paftools.js splice2bed anno.gtf > anno.bed             # convert GTF/GFF3 to BED12
+```
+
+## Table of Contents
+
+- [Getting Started](#started)
+- [Introduction](#intro)
+- [Evaluation](#eval)
+  - [Evaluating mapping accuracy with simulated reads](#mapeval)
+  - [Evaluating read overlap sensitivity](#oveval)
+- [Calling Variants from Assemblies](#asmvar)
+
+## <a name="intro"></a>Introduction
+
+paftools.js is a script that processes alignments in the [PAF format][paf],
+such as converting between formats, evaluating mapping accuracy, lifting over
+BED files based on alignment, and calling variants from assembly-to-assembly
+alignment. This script *requires* the [k8 Javascript shell][k8] to run. On
+Linux or Mac, you can download the precompiled k8 binary with:
+
+```sh
+curl -L https://github.com/attractivechaos/k8/releases/download/v0.2.4/k8-0.2.4.tar.bz2 | tar -jxf -
+cp k8-0.2.4/k8-`uname -s` $HOME/bin/k8  # assuming $HOME/bin in your $PATH
+```
+
+It is highly recommended to copy the executable `k8` to a directory on your
+`$PATH` such as `/usr/bin/env` can find it. Like python scripts, once you
+install `k8`, you can launch paftools.js in one of the two ways:
+
+```sh
+path/to/paftools.js             # only if k8 is on your $PATH
+k8 path/to/paftools.js
+```
+
+In a nutshell, paftools.js has the following commands:
+
+```
+Usage: paftools.js <command> [arguments]
+Commands:
+  view       convert PAF to BLAST-like (for eyeballing) or MAF
+  splice2bed convert spliced alignment in PAF/SAM to BED12
+  sam2paf    convert SAM to PAF
+  delta2paf  convert MUMmer's delta to PAF
+  gff2bed    convert GTF/GFF3 to BED12
+
+  stat       collect basic mapping information in PAF/SAM
+  liftover   simplistic liftOver
+  call       call variants from asm-to-ref alignment with the cs tag
+  bedcov     compute the number of bases covered
+
+  mapeval    evaluate mapping accuracy using mason2/PBSIM-simulated FASTQ
+  mason2fq   convert mason2-simulated SAM to FASTQ
+  pbsim2fq   convert PBSIM-simulated MAF to FASTQ
+  junceval   evaluate splice junction consistency with known annotations
+  ov-eval    evaluate read overlap sensitivity using read-to-ref mapping
+```
+
+paftools.js seamlessly reads both plain text files and gzip'd text files.
+
+## <a name="eval"></a>Evaluation
+
+### <a name="mapeval"></a>Evaluating mapping accuracy with simulated reads
+
+The **pbsim2fq** command of paftools.js converts the MAF output of [pbsim][pbsim]
+to FASTQ and encodes the true mapping position in the read name in a format like
+`S1_33!chr1!225258409!225267761!-`. Similarly, the **mason2fq** command
+converts [mason2][mason2] simulated SAM to FASTQ.
+
+Command **mapeval** evaluates mapped SAM/PAF. Here is example output:
+
+```
+Q       60      32478   0       0.000000000     32478
+Q       22      16      1       0.000030775     32494
+Q       21      43      1       0.000061468     32537
+Q       19      73      1       0.000091996     32610
+Q       14      66      1       0.000122414     32676
+Q       10      27      3       0.000214048     32703
+Q       8       14      1       0.000244521     32717
+Q       7       13      2       0.000305530     32730
+Q       6       46      1       0.000335611     32776
+Q       3       10      1       0.000366010     32786
+Q       2       20      2       0.000426751     32806
+Q       1       248     94      0.003267381     33054
+Q       0       31      17      0.003778147     33085
+U       3
+```
+
+where each Q-line gives the quality threshold, the number of reads mapped with
+mapping quality equal to or greater than the threshold, number of wrong
+mappings, accumulative mapping error rate and the accumulative number of
+mapped reads. The U-line, if present, gives the number of unmapped reads if
+they are present in the SAM file.
+
+Suppose the reported mapping coordinate overlap with the true coordinate like
+the following:
+
+```
+truth:   --------------------
+mapper:           ----------------------
+         |<- l1 ->|<-- o -->|<-- l2 -->|
+```
+
+Let `r=o/(l1+o+l2)`. The reported mapping is considered correct if `r>0.1` by
+default.
+
+### <a name="oveval"></a>Evaluating read overlap sensitivity
+
+Command **ov-eval** takes *sorted* read-to-reference alignment and read
+overlaps in PAF as input, and evaluates the sensitivity. For example:
+
+```sh
+minimap2 -cx map-pb ref.fa reads.fq.gz | sort -k6,6 -k8,8n > reads-to-ref.paf
+minimap2 -x ava-pb reads.fq.gz reads.fq.gz > ovlp.paf
+k8 ov-eval.js reads-to-ref.paf ovlp.paf
+```
+
+## <a name="asmvar"></a>Calling Variants from Haploid Assemblies
+
+The **call** command of paftools.js calls variants from coordinate-sorted
+assembly-to-reference alignment. It calls variants from the [cs tag][cs] and
+identifies confident/callable regions as those covered by exactly one contig.
+Here are example command lines:
+
+```sh
+minimap2 -cx asm5 -t8 --cs ref.fa asm.fa > asm.paf  # keeping this file is recommended; --cs required!
+sort -k6,6 -k8,8n asm.paf > asm.srt.paf             # sort by reference start coordinate
+k8 paftools.js call asm.srt.paf > asm.var.txt
+```
+
+Here is sample output:
+
+```
+V   chr1    2276040 2276041 1   60  c   g   LJII01000171.1  1217409 1217410 +
+V   chr1    2280409 2280410 1   60  a   g   LJII01000171.1  1221778 1221779 +
+V   chr1    2280504 2280505 1   60  a   g   LJII01000171.1  1221873 1221874 +
+R   chr1    2325140 2436340
+V   chr1    2325287 2325287 1   60  -   ct  LJII01000171.1  1272894 1272896 +
+V   chr1    2325642 2325644 1   60  tt  -   LJII01000171.1  1273251 1273251 +
+V   chr1    2326051 2326052 1   60  c   t   LJII01000171.1  1273658 1273659 +
+V   chr1    2326287 2326288 1   60  c   t   LJII01000171.1  1273894 1273895 +
+```
+
+where a line starting with `R` gives regions covered by one query contig, and a
+V-line encodes a variant in the following format: chr, start, end, query depth,
+mapping quality, REF allele, ALT allele, query name, query start, end and the
+query orientation. Generally, you should only look at variants where column 5
+is one.
+
+By default, when calling variants, "paftools.js call" ignores alignments 50kb
+or shorter; when deriving callable regions, it ignores alignments 10kb or
+shorter.  It uses two thresholds to avoid edge effects. These defaults are
+designed for long-read assemblies. For short reads, both should be reduced.
+
+
+
+[paf]: https://github.com/lh3/miniasm/blob/master/PAF.md
+[cs]: https://github.com/lh3/minimap2#cs
+[k8]: https://github.com/attractivechaos/k8
+[maf]: https://genome.ucsc.edu/FAQ/FAQformat#format5
+[pbsim]: https://github.com/pfaucon/PBSIM-PacBio-Simulator
+[mason2]: https://github.com/seqan/seqan/tree/master/apps/mason2

data/ext/minimap2/misc/mmphase.js

@@ -0,0 +1,335 @@
+#!/usr/bin/env k8
+
+var getopt = function(args, ostr) {
+	var oli; // option letter list index
+	if (typeof(getopt.place) == 'undefined')
+		getopt.ind = 0, getopt.arg = null, getopt.place = -1;
+	if (getopt.place == -1) { // update scanning pointer
+		if (getopt.ind >= args.length || args[getopt.ind].charAt(getopt.place = 0) != '-') {
+			getopt.place = -1;
+			return null;
+		}
+		if (getopt.place + 1 < args[getopt.ind].length && args[getopt.ind].charAt(++getopt.place) == '-') { // found "--"
+			++getopt.ind;
+			getopt.place = -1;
+			return null;
+		}
+	}
+	var optopt = args[getopt.ind].charAt(getopt.place++); // character checked for validity
+	if (optopt == ':' || (oli = ostr.indexOf(optopt)) < 0) {
+		if (optopt == '-') return null; //  if the user didn't specify '-' as an option, assume it means null.
+		if (getopt.place < 0) ++getopt.ind;
+		return '?';
+	}
+	if (oli+1 >= ostr.length || ostr.charAt(++oli) != ':') { // don't need argument
+		getopt.arg = null;
+		if (getopt.place < 0 || getopt.place >= args[getopt.ind].length) ++getopt.ind, getopt.place = -1;
+	} else { // need an argument
+		if (getopt.place >= 0 && getopt.place < args[getopt.ind].length)
+			getopt.arg = args[getopt.ind].substr(getopt.place);
+		else if (args.length <= ++getopt.ind) { // no arg
+			getopt.place = -1;
+			if (ostr.length > 0 && ostr.charAt(0) == ':') return ':';
+			return '?';
+		} else getopt.arg = args[getopt.ind]; // white space
+		getopt.place = -1;
+		++getopt.ind;
+	}
+	return optopt;
+}
+
+function read_fastx(file, buf)
+{
+	if (file.readline(buf) < 0) return null;
+	var m, line = buf.toString();
+	if ((m = /^([>@])(\S+)/.exec(line)) == null)
+		throw Error("wrong fastx format");
+	var is_fq = (m[1] == '@');
+	var name = m[2];
+	if (file.readline(buf) < 0)
+		throw Error("missing sequence line");
+	var seq = buf.toString();
+	if (is_fq) { // skip quality
+		file.readline(buf);
+		file.readline(buf);
+	}
+	return [name, seq];
+}
+
+function filter_paf(a, opt)
+{
+	if (a.length == 0) return;
+	var k = 0;
+	for (var i = 0; i < a.length; ++i) {
+		var ai = a[i];
+		if (ai[10] < opt.min_blen) continue;
+		if (ai[9] < ai[10] * opt.min_iden) continue;
+		var clip = [0, 0];
+		if (ai[4] == '+') {
+			clip[0] = ai[2] < ai[7]? ai[2] : ai[7];
+			clip[1] = ai[1] - ai[3] < ai[6] - ai[8]? ai[1] - ai[3] : ai[6] - ai[8];
+		} else {
+			clip[0] = ai[2] < ai[6] - ai[8]? ai[2] : ai[6] - ai[8];
+			clip[1] = ai[1] - ai[3] < ai[7]? ai[1] - ai[3] : ai[7];
+		}
+		if (clip[0] > opt.max_clip_len || clip[1] > opt.max_clip_len) continue;
+		a[k++] = ai;
+	}
+	a.length = k;
+}
+
+function parse_events(t, ev, id, buf)
+{
+	var re = /(:(\d+))|(([\+\-\*])([a-z]+))/g;
+	var m, cs = null;
+	for (var j = 12; j < t.length; ++j) {
+		if ((m = /^cs:Z:(\S+)/.exec(t[j])) != null) {
+			cs = m[1].toLowerCase();
+			break;
+		}
+	}
+	if (cs == null) {
+		warn("Warning: no cs tag for read '" + t[0] + "'");
+		return;
+	}
+	var st = t[2], en = t[3];
+	var x = st;
+	while ((m = re.exec(cs)) != null) {
+		var l;
+		if (m[2] != null) { // an identitcal match ":\d+"
+			l = parseInt(m[2]);
+			// [start, end, type, index, changed_base]
+			ev.push([x, x + l, 0, id]);
+		} else {
+			if (m[4] == '*') {
+				l = 1;
+				ev.push([x, x + 1, 1, id, m[5][0]]);
+			} else if (m[4] == '+') {
+				l = m[5].length;
+				ev.push([x, x + l, 2, id]);
+			} else if (m[4] == '-') {
+				l = 0;
+				ev.push([x, x, -1, id, m[5]]);
+			}
+		}
+		x += l;
+	}
+	if (x != en)
+		throw Error("inconsistent cs for read '" + t[0] + "'");
+}
+
+function find_het_sub(ev, a, opt)
+{
+	var n = a.length, last0_i = -1, h = [], d = [];
+	for (var i = 0; i < n; ++i) h[i] = [], d[i] = [];
+	for (var i = 0; i < ev.length; ++i) {
+		if (ev[i][2] == 0) {
+			if (last0_i < 0 || ev[i][0] != ev[last0_i][0]) last0_i = i;
+			else if (ev[i][1] > ev[last0_i][1])
+				last0_i = i;
+		} else if (ev[i][2] == 1 && last0_i >= 0 && ev[i][0] < ev[last0_i][1]) {
+			if (ev[last0_i][1] - ev[last0_i][0] >= opt.min_mlen) {
+				if (opt.dbg_ev) print("EV", ev[last0_i].join("\t"), "|", ev[i].join("\t"));
+				var e0 = ev[last0_i], hl = h[e0[3]];
+				if (hl.length == 0 || hl[hl.length-1][0] != e0[0])
+					hl.push([e0[0], e0[1]]);
+				d[ev[i][3]].push([ev[i][0], e0[1] - e0[0]]);
+			}
+		}
+	}
+	var b = [];
+	for (var i = 0; i < n; ++i) {
+		var sh = 0, dh = 0;
+		for (var j = 0; j < h[i].length; ++j)
+			sh += h[i][j][1] - h[i][j][0];
+		for (var j = 0; j < d[i].length; ++j)
+			dh += d[i][j][1];
+		// [start, end, index, #consistent, lenConsistent, #conflictive, lenConflictive, identity, mlen]
+		b[i] = [a[i][2], a[i][3], i, h[i].length, sh, d[i].length, dh, a[i][9] / a[i][10], a[i][9]];
+	}
+	return b;
+}
+
+function flt_utg_for_ec(b, opt)
+{
+	var k = 0;
+	for (var i = 0; i < b.length; ++i) {
+		var bi = b[i];
+		if (bi[4] == 0 && bi[6] == 0) b[k++] = bi; // entirely ambiguous
+		else if (bi[6] < (bi[4] + bi[6]) * opt.max_ratio0) b[k++] = bi;
+	}
+	b.length = k;
+	if (b.length == 0) return;
+	// find the longest contiguous segment
+	b.sort(function(x,y) { return x[0]-y[0] });
+	var st = b[0][0], en = b[0][1], max_st = 0, max_en = 0, max_max_en = en;
+	for (var i = 1; i < b.length; ++i) {
+		if (b[i][0] > en) {
+			if (en - st > max_en - max_st)
+				max_st = st, max_en = en;
+			st = b[i][0], en = b[i][1];
+		} else {
+			en = en > b[i][1]? en : b[i][1];
+		}
+		max_max_en = max_max_en > b[i][1]? max_max_en : b[i][1];
+	}
+	if (en - st > max_en - max_st)
+		max_st = st, max_en = en;
+	if (max_max_en != en || st != b[0][0]) {
+		var k = 0;
+		for (var i = 0; i < b.length; ++i)
+			if (b[i][0] < max_en && b[i][1] > max_st)
+				b[k++] = b[i];
+		b.length = k;
+	}
+}
+
+function flt_utg_for_bin(b, opt) // filter out alignments clearly on the wrong phase
+{
+	var k = 0;
+	for (var i = 0; i < b.length; ++i) {
+		var bi = b[i];
+		if (bi[4] + bi[6] == 0 || bi[4] >= (bi[4] + bi[6]) * opt.max_ratio0) b[k++] = bi;
+	}
+	b.length = k;
+}
+
+function ec_core(b, n_a, ev, buf, ecb) // error correction
+{
+	var intv = [];
+	for (var i = 0; i < n_a; ++i)
+		intv[i] = null;
+	intv[b[0][2]] = [b[0][0], b[0][1]];
+	var en = b[0][1];
+	for (var i = 1; i < b.length; ++i) {
+		if (b[i][1] <= en) continue;
+		intv[b[i][2]] = [en, b[i][1]];
+		en = b[i][1];
+	}
+	var k = 0;
+	ecb.capacity = buf.capacity;
+	ecb.length = 0;
+	for (var i = 0; i < ev.length; ++i) {
+		var e = ev[i], I = intv[e[3]];
+		if (I == null) continue;
+		if (e[0] >= I[0] && e[0] < I[1]) { // this is to reduce duplicated events around junctions
+			//print("X", e.join("\t"));
+			if (e[2] == 0) {
+				ecb.length += e[1] - e[0];
+				for (var j = e[0]; j < e[1]; ++j)
+					ecb[k++] = buf[j];
+			} else if (e[2] == 1) {
+				++ecb.length;
+				ecb[k++] = e[4].charCodeAt(0);
+			} else if (e[2] < 0) {
+				ecb.length += e[4].length;
+				for (var j = 0; j < e[4].length; ++j)
+					ecb[k++] = e[4].charCodeAt(j);
+			} // else, skip e[2] == 2
+		}
+	}
+	if (ecb.length != k) throw Error("BUG!");
+}
+
+function process_paf(a, opt, fp_seq, buf, ecb)
+{
+	if (a.length == 0) return;
+	var len = a[0][1], name = a[0][0], seq = null;
+	if (len < opt.min_rlen) return;
+	if (fp_seq) {
+		var ret;
+		while ((ret = read_fastx(fp_seq, buf)) != null)
+			if (ret[0] == a[0][0])
+				break;
+		if (ret == null)
+			throw Error("failed to find sequence for read '" + a[0][0] + "'");
+		name = ret[0], seq = ret[1];
+		if (seq.length != len)
+			throw Error("inconsistent length for read '" + name + "'");
+	}
+	filter_paf(a, opt);
+	if (a.length == 0) return;
+	var ev = [];
+	for (var i = 0; i < a.length; ++i)
+		parse_events(a[i], ev, i, buf);
+	ev.sort(function(x,y) { return x[0]!=y[0]? x[0]-y[0] : x[2]-y[2] });
+	if (seq == null) print("SQ", name, a[0][1], a.length);
+	var b = find_het_sub(ev, a, opt);
+	if (opt.ec) flt_utg_for_ec(b, opt);
+	else flt_utg_for_bin(b, opt);
+	if (seq == null) {
+		for (var i = 0; i < b.length; ++i) {
+			var m, ai = a[b[i][2]], score = 0;
+			for (var j = 10; j < ai.length; ++j)
+				if ((m = /^AS:i:(\d+)/.exec(ai[j])) != null)
+					score = m[1];
+			print("TS", b[i][2], b[i][0], b[i][1], ai.slice(5, 9).join("\t"), b[i].slice(3, 7).join("\t"), score);
+		}
+		print("//");
+	} else { // error correction
+		if (b.length == 0) return;
+		buf.set(seq, 0);
+		ec_core(b, a.length, ev, buf, ecb);
+		print(">" + name);
+		print(ecb);
+	}
+}
+
+function main(args)
+{
+	var c, opt = { min_rlen:5000, min_blen:5000, min_iden:0.8, min_mlen:5, max_clip_len:500, max_ratio0:0.25, dbg_ev:false };
+	while ((c = getopt(args, "l:b:d:m:c:r:E")) != null) {
+		if (c == 'l') opt.min_rlen = parseInt(getopt.arg);
+		else if (c == 'b') opt.min_blen = parseInt(getopt.arg);
+		else if (c == 'd') opt.min_iden = parseFloat(getopt.arg);
+		else if (c == 'm') opt.min_slen = parseInt(getopt.arg);
+		else if (c == 'c') opt.max_clip_len = parseInt(getopt.arg);
+		else if (c == 'r') opt.max_ratio0 = parseFloat(getopt.arg);
+		else if (c == 'E') opt.dbg_ev = true;
+	}
+	if (args.length - getopt.ind < 1) {
+		print("Usage: mmphase.js [options] <map-with-cs.paf> [reads.fa]");
+		print("Options:");
+		print("  -l INT    min read length [" + opt.min_rlen + "]");
+		print("  -b INT    min alignment length [" + opt.min_blen + "]");
+		print("  -d FLOAT  min identity [" + opt.min_iden + "]");
+		print("  -s INT    min match length [" + opt.min_mlen + "]");
+		print("  -c INT    max clip length [" + opt.max_clip_len + "]");
+		print("  -r FLOAT  initial ratio for haplotype filtering [" + opt.max_ratio0 + "]");
+		return 0;
+	}
+
+	opt.ec = args.length - getopt.ind < 2? false : true;
+	if (!opt.ec) {
+		print("CC");
+		print("CC", "SQ  qName  qLen    nHits");
+		print("CC", "TS  index  qStart  qEnd  tName  tLen  tStart  tEnd  nConsistent  lCons  nConflictive  lConf  score");
+		print("CC");
+	}
+
+	var buf = new Bytes(), ecb = new Bytes();
+	var fp_paf = new File(args[getopt.ind]);
+	var fp_seq = args.length - getopt.ind >= 2? new File(args[getopt.ind+1]) : null;
+	var a = [];
+	while (fp_paf.readline(buf) >= 0) {
+		var t = buf.toString().split("\t");
+		if (a.length > 0 && a[0][0] != t[0]) {
+			process_paf(a, opt, fp_seq, buf, ecb);
+			a.length = 0;
+		}
+		for (var i = 1; i <= 3; ++i) t[i] = parseInt(t[i]);
+		if (t[1] < opt.min_rlen) continue;
+		for (var i = 6; i <= 10; ++i) t[i] = parseInt(t[i]);
+		if (t[10] < opt.min_blen) continue;
+		a.push(t);
+	}
+	if (a.length >= 0)
+		process_paf(a, opt, fp_seq, buf, ecb);
+	if (fp_seq) fp_seq.close();
+	fp_paf.close();
+	ecb.destroy();
+	buf.destroy();
+}
+
+var ret = main(arguments)
+exit(ret)