bio-bwa 0.2.0

data/ext/bwase.h

@@ -0,0 +1,27 @@
+#ifndef BWASE_H
+#define BWASE_H
+
+#include "bntseq.h"
+#include "bwt.h"
+#include "bwtaln.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	// Initialize mapping tables in the bwa single-end mapper.
+	void bwase_initialize();
+	// Calculate the approximate position of the sequence from the specified bwt with loaded suffix array.
+	void bwa_cal_pac_pos_core(const bwt_t* forward_bwt, const bwt_t* reverse_bwt, bwa_seq_t* seq, const int max_mm, const float fnr);
+	// Refine the approximate position of the sequence to an actual placement for the sequence.
+	void bwa_refine_gapped(const bntseq_t *bns, int n_seqs, bwa_seq_t *seqs, ubyte_t *_pacseq, bntseq_t *ntbns);
+	// Backfill certain alignment properties mainly centering around number of matches.
+	void bwa_aln2seq(int n_aln, const bwt_aln1_t *aln, bwa_seq_t *s);
+	// Calculate the end position of a read given a certain sequence.
+	int64_t pos_end(const bwa_seq_t *p);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // BWASE_H

data/ext/bwaseqio.c

@@ -0,0 +1,222 @@
+#include <zlib.h>
+#include <ctype.h>
+#include "bwtaln.h"
+#include "utils.h"
+#include "bamlite.h"
+
+#include "kseq.h"
+KSEQ_INIT(gzFile, gzread)
+
+extern unsigned char nst_nt4_table[256];
+static char bam_nt16_nt4_table[] = { 4, 0, 1, 4, 2, 4, 4, 4, 3, 4, 4, 4, 4, 4, 4, 4 };
+
+struct __bwa_seqio_t {
+	// for BAM input
+	int is_bam, which; // 1st bit: read1, 2nd bit: read2, 3rd: SE
+	bamFile fp;
+	// for fastq input
+	kseq_t *ks;
+};
+
+bwa_seqio_t *bwa_bam_open(const char *fn, int which)
+{
+	bwa_seqio_t *bs;
+	bam_header_t *h;
+	bs = (bwa_seqio_t*)calloc(1, sizeof(bwa_seqio_t));
+	bs->is_bam = 1;
+	bs->which = which;
+	bs->fp = bam_open(fn, "r");
+	h = bam_header_read(bs->fp);
+	bam_header_destroy(h);
+	return bs;
+}
+
+bwa_seqio_t *bwa_seq_open(const char *fn)
+{
+	gzFile fp;
+	bwa_seqio_t *bs;
+	bs = (bwa_seqio_t*)calloc(1, sizeof(bwa_seqio_t));
+	fp = xzopen(fn, "r");
+	bs->ks = kseq_init(fp);
+	return bs;
+}
+
+void bwa_seq_close(bwa_seqio_t *bs)
+{
+	if (bs == 0) return;
+	if (bs->is_bam) bam_close(bs->fp);
+	else {
+		gzclose(bs->ks->f->f);
+		kseq_destroy(bs->ks);
+	}
+	free(bs);
+}
+
+void seq_reverse(int len, ubyte_t *seq, int is_comp)
+{
+	int i;
+	if (is_comp) {
+		for (i = 0; i < len>>1; ++i) {
+			char tmp = seq[len-1-i];
+			if (tmp < 4) tmp = 3 - tmp;
+			seq[len-1-i] = (seq[i] >= 4)? seq[i] : 3 - seq[i];
+			seq[i] = tmp;
+		}
+		if (len&1) seq[i] = (seq[i] >= 4)? seq[i] : 3 - seq[i];
+	} else {
+		for (i = 0; i < len>>1; ++i) {
+			char tmp = seq[len-1-i];
+			seq[len-1-i] = seq[i]; seq[i] = tmp;
+		}
+	}
+}
+
+int bwa_trim_read(int trim_qual, bwa_seq_t *p)
+{
+	int s = 0, l, max = 0, max_l = p->len - 1;
+	if (trim_qual < 1 || p->qual == 0) return 0;
+	for (l = p->len - 1; l >= BWA_MIN_RDLEN - 1; --l) {
+		s += trim_qual - (p->qual[l] - 33);
+		if (s < 0) break;
+		if (s > max) {
+			max = s; max_l = l;
+		}
+	}
+	p->clip_len = p->len = max_l + 1;
+	return p->full_len - p->len;
+}
+
+static bwa_seq_t *bwa_read_bam(bwa_seqio_t *bs, int n_needed, int *n, int is_comp, int trim_qual)
+{
+	bwa_seq_t *seqs, *p;
+	int n_seqs, l, i;
+	long n_trimmed = 0, n_tot = 0;
+	bam1_t *b;
+
+	b = bam_init1();
+	n_seqs = 0;
+	seqs = (bwa_seq_t*)calloc(n_needed, sizeof(bwa_seq_t));
+	while (bam_read1(bs->fp, b) >= 0) {
+		uint8_t *s, *q;
+		int go = 0;
+		if ((bs->which & 1) && (b->core.flag & BAM_FREAD1)) go = 1;
+		if ((bs->which & 2) && (b->core.flag & BAM_FREAD2)) go = 1;
+		if ((bs->which & 4) && !(b->core.flag& BAM_FREAD1) && !(b->core.flag& BAM_FREAD2))go = 1;
+		if (go == 0) continue;
+		l = b->core.l_qseq;
+		p = &seqs[n_seqs++];
+		p->tid = -1; // no assigned to a thread
+		p->qual = 0;
+		p->full_len = p->clip_len = p->len = l;
+		n_tot += p->full_len;
+		s = bam1_seq(b); q = bam1_qual(b);
+		p->seq = (ubyte_t*)calloc(p->len + 1, 1);
+		p->qual = (ubyte_t*)calloc(p->len + 1, 1);
+		for (i = 0; i != p->full_len; ++i) {
+			p->seq[i] = bam_nt16_nt4_table[(int)bam1_seqi(s, i)];
+			p->qual[i] = q[i] + 33 < 126? q[i] + 33 : 126;
+		}
+		if (bam1_strand(b)) { // then reverse 
+			seq_reverse(p->len, p->seq, 1);
+			seq_reverse(p->len, p->qual, 0);
+		}
+		if (trim_qual >= 1) n_trimmed += bwa_trim_read(trim_qual, p);
+		p->rseq = (ubyte_t*)calloc(p->full_len, 1);
+		memcpy(p->rseq, p->seq, p->len);
+		seq_reverse(p->len, p->seq, 0); // *IMPORTANT*: will be reversed back in bwa_refine_gapped()
+		seq_reverse(p->len, p->rseq, is_comp);
+		p->name = strdup((const char*)bam1_qname(b));
+		if (n_seqs == n_needed) break;
+	}
+	*n = n_seqs;
+	if (n_seqs && trim_qual >= 1)
+		fprintf(stderr, "[bwa_read_seq] %.1f%% bases are trimmed.\n", 100.0f * n_trimmed/n_tot);
+	if (n_seqs == 0) {
+		free(seqs);
+		bam_destroy1(b);
+		return 0;
+	}
+	bam_destroy1(b);
+	return seqs;
+}
+
+#define BARCODE_LOW_QUAL 13
+
+bwa_seq_t *bwa_read_seq(bwa_seqio_t *bs, int n_needed, int *n, int mode, int trim_qual)
+{
+	bwa_seq_t *seqs, *p;
+	kseq_t *seq = bs->ks;
+	int n_seqs, l, i, is_comp = mode&BWA_MODE_COMPREAD, is_64 = mode&BWA_MODE_IL13, l_bc = mode>>24;
+	long n_trimmed = 0, n_tot = 0;
+
+	if (l_bc > 15) {
+		fprintf(stderr, "[%s] the maximum barcode length is 15.\n", __func__);
+		return 0;
+	}
+	if (bs->is_bam) return bwa_read_bam(bs, n_needed, n, is_comp, trim_qual); // l_bc has no effect for BAM input
+	n_seqs = 0;
+	seqs = (bwa_seq_t*)calloc(n_needed, sizeof(bwa_seq_t));
+	while ((l = kseq_read(seq)) >= 0) {
+		if (is_64 && seq->qual.l)
+			for (i = 0; i < seq->qual.l; ++i) seq->qual.s[i] -= 31;
+		if (seq->seq.l <= l_bc) continue; // sequence length equals or smaller than the barcode length
+		p = &seqs[n_seqs++];
+		if (l_bc) { // then trim barcode
+			for (i = 0; i < l_bc; ++i)
+				p->bc[i] = (seq->qual.l && seq->qual.s[i]-33 < BARCODE_LOW_QUAL)? tolower(seq->seq.s[i]) : toupper(seq->seq.s[i]);
+			p->bc[i] = 0;
+			for (; i < seq->seq.l; ++i)
+				seq->seq.s[i - l_bc] = seq->seq.s[i];
+			seq->seq.l -= l_bc; seq->seq.s[seq->seq.l] = 0;
+			if (seq->qual.l) {
+				for (i = l_bc; i < seq->qual.l; ++i)
+					seq->qual.s[i - l_bc] = seq->qual.s[i];
+				seq->qual.l -= l_bc; seq->qual.s[seq->qual.l] = 0;
+			}
+			l = seq->seq.l;
+		} else p->bc[0] = 0;
+		p->tid = -1; // no assigned to a thread
+		p->qual = 0;
+		p->full_len = p->clip_len = p->len = l;
+		n_tot += p->full_len;
+		p->seq = (ubyte_t*)calloc(p->len, 1);
+		for (i = 0; i != p->full_len; ++i)
+			p->seq[i] = nst_nt4_table[(int)seq->seq.s[i]];
+		if (seq->qual.l) { // copy quality
+			p->qual = (ubyte_t*)strdup((char*)seq->qual.s);
+			if (trim_qual >= 1) n_trimmed += bwa_trim_read(trim_qual, p);
+		}
+		p->rseq = (ubyte_t*)calloc(p->full_len, 1);
+		memcpy(p->rseq, p->seq, p->len);
+		seq_reverse(p->len, p->seq, 0); // *IMPORTANT*: will be reversed back in bwa_refine_gapped()
+		seq_reverse(p->len, p->rseq, is_comp);
+		p->name = strdup((const char*)seq->name.s);
+		{ // trim /[12]$
+			int t = strlen(p->name);
+			if (t > 2 && p->name[t-2] == '/' && (p->name[t-1] == '1' || p->name[t-1] == '2')) p->name[t-2] = '\0';
+		}
+		if (n_seqs == n_needed) break;
+	}
+	*n = n_seqs;
+	if (n_seqs && trim_qual >= 1)
+		fprintf(stderr, "[bwa_read_seq] %.1f%% bases are trimmed.\n", 100.0f * n_trimmed/n_tot);
+	if (n_seqs == 0) {
+		free(seqs);
+		return 0;
+	}
+	return seqs;
+}
+
+void bwa_free_read_seq(int n_seqs, bwa_seq_t *seqs)
+{
+	int i, j;
+	for (i = 0; i != n_seqs; ++i) {
+		bwa_seq_t *p = seqs + i;
+		for (j = 0; j < p->n_multi; ++j)
+			if (p->multi[j].cigar) free(p->multi[j].cigar);
+		free(p->name);
+		free(p->seq); free(p->rseq); free(p->qual); free(p->aln); free(p->md); free(p->multi);
+		free(p->cigar);
+	}
+	free(seqs);
+}

data/ext/bwt.c

@@ -0,0 +1,250 @@
+/* The MIT License
+
+   Copyright (c) 2008 Genome Research Ltd (GRL).
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/* Contact: Heng Li <lh3@sanger.ac.uk> */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <stdint.h>
+#include "utils.h"
+#include "bwt.h"
+
+void bwt_gen_cnt_table(bwt_t *bwt)
+{
+	int i, j;
+	for (i = 0; i != 256; ++i) {
+		uint32_t x = 0;
+		for (j = 0; j != 4; ++j)
+			x |= (((i&3) == j) + ((i>>2&3) == j) + ((i>>4&3) == j) + (i>>6 == j)) << (j<<3);
+		bwt->cnt_table[i] = x;
+	}
+}
+
+// bwt->bwt and bwt->occ must be precalculated
+void bwt_cal_sa(bwt_t *bwt, int intv)
+{
+	bwtint_t isa, sa, i; // S(isa) = sa
+
+	xassert(bwt->bwt, "bwt_t::bwt is not initialized.");
+
+	if (bwt->sa) free(bwt->sa);
+	bwt->sa_intv = intv;
+	bwt->n_sa = (bwt->seq_len + intv) / intv;
+	bwt->sa = (bwtint_t*)calloc(bwt->n_sa, sizeof(bwtint_t));
+	// calculate SA value
+	isa = 0; sa = bwt->seq_len;
+	for (i = 0; i < bwt->seq_len; ++i) {
+		if (isa % intv == 0) bwt->sa[isa/intv] = sa;
+		--sa;
+		isa = bwt_invPsi(bwt, isa);
+	}
+	if (isa % intv == 0) bwt->sa[isa/intv] = sa;
+	bwt->sa[0] = (bwtint_t)-1; // before this line, bwt->sa[0] = bwt->seq_len
+}
+
+bwtint_t bwt_sa(const bwt_t *bwt, bwtint_t k)
+{
+	bwtint_t sa = 0;
+	while (k % bwt->sa_intv != 0) {
+		++sa;
+		k = bwt_invPsi(bwt, k);
+	}
+	/* without setting bwt->sa[0] = -1, the following line should be
+	   changed to (sa + bwt->sa[k/bwt->sa_intv]) % (bwt->seq_len + 1) */
+	return sa + bwt->sa[k/bwt->sa_intv];
+}
+
+static inline int __occ_aux(uint64_t y, int c)
+{
+	// reduce nucleotide counting to bits counting
+	y = ((c&2)? y : ~y) >> 1 & ((c&1)? y : ~y) & 0x5555555555555555ull;
+	// count the number of 1s in y
+	y = (y & 0x3333333333333333ull) + (y >> 2 & 0x3333333333333333ull);
+	return ((y + (y >> 4)) & 0xf0f0f0f0f0f0f0full) * 0x101010101010101ull >> 56;
+}
+
+inline bwtint_t bwt_occ(const bwt_t *bwt, bwtint_t k, ubyte_t c)
+{
+	bwtint_t n, l, j;
+	uint32_t *p;
+
+	if (k == bwt->seq_len) return bwt->L2[c+1] - bwt->L2[c];
+	if (k == (bwtint_t)(-1)) return 0;
+	if (k >= bwt->primary) --k; // because $ is not in bwt
+
+	// retrieve Occ at k/OCC_INTERVAL
+	n = (p = bwt_occ_intv(bwt, k))[c];
+	p += 4; // jump to the start of the first BWT cell
+
+	// calculate Occ up to the last k/32
+	j = k >> 5 << 5;
+	for (l = k/OCC_INTERVAL*OCC_INTERVAL; l < j; l += 32, p += 2)
+		n += __occ_aux((uint64_t)p[0]<<32 | p[1], c);
+
+	// calculate Occ
+	n += __occ_aux(((uint64_t)p[0]<<32 | p[1]) & ~((1ull<<((~k&31)<<1)) - 1), c);
+	if (c == 0) n -= ~k&31; // corrected for the masked bits
+
+	return n;
+}
+
+// an analogy to bwt_occ() but more efficient, requiring k <= l
+inline void bwt_2occ(const bwt_t *bwt, bwtint_t k, bwtint_t l, ubyte_t c, bwtint_t *ok, bwtint_t *ol)
+{
+	bwtint_t _k, _l;
+	if (k == l) {
+		*ok = *ol = bwt_occ(bwt, k, c);
+		return;
+	}
+	_k = (k >= bwt->primary)? k-1 : k;
+	_l = (l >= bwt->primary)? l-1 : l;
+	if (_l/OCC_INTERVAL != _k/OCC_INTERVAL || k == (bwtint_t)(-1) || l == (bwtint_t)(-1)) {
+		*ok = bwt_occ(bwt, k, c);
+		*ol = bwt_occ(bwt, l, c);
+	} else {
+		bwtint_t m, n, i, j;
+		uint32_t *p;
+		if (k >= bwt->primary) --k;
+		if (l >= bwt->primary) --l;
+		n = (p = bwt_occ_intv(bwt, k))[c];
+		p += 4;
+		// calculate *ok
+		j = k >> 5 << 5;
+		for (i = k/OCC_INTERVAL*OCC_INTERVAL; i < j; i += 32, p += 2)
+			n += __occ_aux((uint64_t)p[0]<<32 | p[1], c);
+		m = n;
+		n += __occ_aux(((uint64_t)p[0]<<32 | p[1]) & ~((1ull<<((~k&31)<<1)) - 1), c);
+		if (c == 0) n -= ~k&31; // corrected for the masked bits
+		*ok = n;
+		// calculate *ol
+		j = l >> 5 << 5;
+		for (; i < j; i += 32, p += 2)
+			m += __occ_aux((uint64_t)p[0]<<32 | p[1], c);
+		m += __occ_aux(((uint64_t)p[0]<<32 | p[1]) & ~((1ull<<((~l&31)<<1)) - 1), c);
+		if (c == 0) m -= ~l&31; // corrected for the masked bits
+		*ol = m;
+	}
+}
+
+#define __occ_aux4(bwt, b)											\
+	((bwt)->cnt_table[(b)&0xff] + (bwt)->cnt_table[(b)>>8&0xff]		\
+	 + (bwt)->cnt_table[(b)>>16&0xff] + (bwt)->cnt_table[(b)>>24])
+
+inline void bwt_occ4(const bwt_t *bwt, bwtint_t k, bwtint_t cnt[4])
+{
+	bwtint_t l, j, x;
+	uint32_t *p;
+	if (k == (bwtint_t)(-1)) {
+		memset(cnt, 0, 4 * sizeof(bwtint_t));
+		return;
+	}
+	if (k >= bwt->primary) --k; // because $ is not in bwt
+	p = bwt_occ_intv(bwt, k);
+	memcpy(cnt, p, 16);
+	p += 4;
+	j = k >> 4 << 4;
+	for (l = k / OCC_INTERVAL * OCC_INTERVAL, x = 0; l < j; l += 16, ++p)
+		x += __occ_aux4(bwt, *p);
+	x += __occ_aux4(bwt, *p & ~((1U<<((~k&15)<<1)) - 1)) - (~k&15);
+	cnt[0] += x&0xff; cnt[1] += x>>8&0xff; cnt[2] += x>>16&0xff; cnt[3] += x>>24;
+}
+
+// an analogy to bwt_occ4() but more efficient, requiring k <= l
+inline void bwt_2occ4(const bwt_t *bwt, bwtint_t k, bwtint_t l, bwtint_t cntk[4], bwtint_t cntl[4])
+{
+	bwtint_t _k, _l;
+	if (k == l) {
+		bwt_occ4(bwt, k, cntk);
+		memcpy(cntl, cntk, 4 * sizeof(bwtint_t));
+		return;
+	}
+	_k = (k >= bwt->primary)? k-1 : k;
+	_l = (l >= bwt->primary)? l-1 : l;
+	if (_l/OCC_INTERVAL != _k/OCC_INTERVAL || k == (bwtint_t)(-1) || l == (bwtint_t)(-1)) {
+		bwt_occ4(bwt, k, cntk);
+		bwt_occ4(bwt, l, cntl);
+	} else {
+		bwtint_t i, j, x, y;
+		uint32_t *p;
+		int cl[4];
+		if (k >= bwt->primary) --k; // because $ is not in bwt
+		if (l >= bwt->primary) --l;
+		cl[0] = cl[1] = cl[2] = cl[3] = 0;
+		p = bwt_occ_intv(bwt, k);
+		memcpy(cntk, p, 4 * sizeof(bwtint_t));
+		p += 4;
+		// prepare cntk[]
+		j = k >> 4 << 4;
+		for (i = k / OCC_INTERVAL * OCC_INTERVAL, x = 0; i < j; i += 16, ++p)
+			x += __occ_aux4(bwt, *p);
+		y = x;
+		x += __occ_aux4(bwt, *p & ~((1U<<((~k&15)<<1)) - 1)) - (~k&15);
+		// calculate cntl[] and finalize cntk[]
+		j = l >> 4 << 4;
+		for (; i < j; i += 16, ++p) y += __occ_aux4(bwt, *p);
+		y += __occ_aux4(bwt, *p & ~((1U<<((~l&15)<<1)) - 1)) - (~l&15);
+		memcpy(cntl, cntk, 16);
+		cntk[0] += x&0xff; cntk[1] += x>>8&0xff; cntk[2] += x>>16&0xff; cntk[3] += x>>24;
+		cntl[0] += y&0xff; cntl[1] += y>>8&0xff; cntl[2] += y>>16&0xff; cntl[3] += y>>24;
+	}
+}
+
+int bwt_match_exact(const bwt_t *bwt, int len, const ubyte_t *str, bwtint_t *sa_begin, bwtint_t *sa_end)
+{
+	bwtint_t k, l, ok, ol;
+	int i;
+	k = 0; l = bwt->seq_len;
+	for (i = len - 1; i >= 0; --i) {
+		ubyte_t c = str[i];
+		if (c > 3) return 0; // no match
+		bwt_2occ(bwt, k - 1, l, c, &ok, &ol);
+		k = bwt->L2[c] + ok + 1;
+		l = bwt->L2[c] + ol;
+		if (k > l) break; // no match
+	}
+	if (k > l) return 0; // no match
+	if (sa_begin) *sa_begin = k;
+	if (sa_end)   *sa_end = l;
+	return l - k + 1;
+}
+
+int bwt_match_exact_alt(const bwt_t *bwt, int len, const ubyte_t *str, bwtint_t *k0, bwtint_t *l0)
+{
+	int i;
+	bwtint_t k, l, ok, ol;
+	k = *k0; l = *l0;
+	for (i = len - 1; i >= 0; --i) {
+		ubyte_t c = str[i];
+		if (c > 3) return 0; // there is an N here. no match
+		bwt_2occ(bwt, k - 1, l, c, &ok, &ol);
+		k = bwt->L2[c] + ok + 1;
+		l = bwt->L2[c] + ol;
+		if (k > l) return 0; // no match
+	}
+	*k0 = k; *l0 = l;
+	return l - k + 1;
+}