ngs_server 0.4 → 0.5

data/ext/tabix/kstring.h

@@ -0,0 +1,68 @@
+#ifndef KSTRING_H
+#define KSTRING_H
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+
+#ifndef kroundup32
+#define kroundup32(x) (--(x), (x)|=(x)>>1, (x)|=(x)>>2, (x)|=(x)>>4, (x)|=(x)>>8, (x)|=(x)>>16, ++(x))
+#endif
+
+#ifndef KSTRING_T
+#define KSTRING_T kstring_t
+typedef struct __kstring_t {
+	size_t l, m;
+	char *s;
+} kstring_t;
+#endif
+
+int ksprintf(kstring_t *s, const char *fmt, ...);
+int ksplit_core(char *s, int delimiter, int *_max, int **_offsets);
+
+// calculate the auxiliary array, allocated by calloc()
+int *ksBM_prep(const uint8_t *pat, int m);
+
+/* Search pat in str and returned the list of matches. The size of the
+ * list is returned as n_matches. _prep is the array returned by
+ * ksBM_prep(). If it is a NULL pointer, ksBM_prep() will be called. */
+int *ksBM_search(const uint8_t *str, int n, const uint8_t *pat, int m, int *_prep, int *n_matches);
+
+static inline int kputsn(const char *p, int l, kstring_t *s)
+{
+	if (s->l + l + 1 >= s->m) {
+		s->m = s->l + l + 2;
+		kroundup32(s->m);
+		s->s = (char*)realloc(s->s, s->m);
+	}
+	strncpy(s->s + s->l, p, l);
+	s->l += l;
+	s->s[s->l] = 0;
+	return l;
+}
+
+static inline int kputs(const char *p, kstring_t *s)
+{
+	return kputsn(p, strlen(p), s);
+}
+
+static inline int kputc(int c, kstring_t *s)
+{
+	if (s->l + 1 >= s->m) {
+		s->m = s->l + 2;
+		kroundup32(s->m);
+		s->s = (char*)realloc(s->s, s->m);
+	}
+	s->s[s->l++] = c;
+	s->s[s->l] = 0;
+	return c;
+}
+
+static inline int *ksplit(kstring_t *s, int delimiter, int *n)
+{
+	int max = 0, *offsets = 0;
+	*n = ksplit_core(s->s, delimiter, &max, &offsets);
+	return offsets;
+}
+
+#endif

data/ext/tabix/main.c

@@ -0,0 +1,290 @@
+#include <string.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sys/stat.h>
+#include <errno.h>
+#include "bgzf.h"
+#include "tabix.h"
+
+#define PACKAGE_VERSION "0.2.5 (r964)"
+
+#define error(...) { fprintf(stderr,__VA_ARGS__); return -1; }
+
+int reheader_file(const char *header, const char *file, int meta)
+{
+    BGZF *fp = bgzf_open(file,"r");
+    if (bgzf_read_block(fp) != 0 || !fp->block_length)
+        return -1;
+    
+    char *buffer = fp->uncompressed_block;
+    int skip_until = 0;
+
+    if ( buffer[0]==meta )
+    {
+        skip_until = 1;
+
+        // Skip the header
+        while (1)
+        {
+            if ( buffer[skip_until]=='\n' )
+            {
+                skip_until++;
+                if ( skip_until>=fp->block_length )
+                {
+                    if (bgzf_read_block(fp) != 0 || !fp->block_length)
+                        error("no body?\n");
+                    skip_until = 0;
+                }
+                // The header has finished
+                if ( buffer[skip_until]!=meta ) break;
+            }
+            skip_until++;
+            if ( skip_until>=fp->block_length )
+            {
+                if (bgzf_read_block(fp) != 0 || !fp->block_length)
+                    error("no body?\n");
+                skip_until = 0;
+            }
+        }
+    }
+
+    FILE *fh = fopen(header,"r");
+    if ( !fh )
+        error("%s: %s", header,strerror(errno));
+    int page_size = getpagesize();
+    char *buf = valloc(page_size);
+    BGZF *bgzf_out = bgzf_fdopen(fileno(stdout), "w");
+    ssize_t nread;
+    while ( (nread=fread(buf,1,page_size-1,fh))>0 )
+    {
+        if ( nread<page_size-1 && buf[nread-1]!='\n' )
+            buf[nread++] = '\n';
+        if (bgzf_write(bgzf_out, buf, nread) < 0) error("Error: %s\n",bgzf_out->error);
+    }
+    fclose(fh);
+
+    if ( fp->block_length - skip_until > 0 )
+    {
+        if (bgzf_write(bgzf_out, buffer+skip_until, fp->block_length-skip_until) < 0) 
+            error("Error: %s\n",fp->error);
+    }
+    if (bgzf_flush(bgzf_out) < 0) 
+        error("Error: %s\n",bgzf_out->error);
+
+    while (1)
+    {
+#ifdef _USE_KNETFILE
+        nread = knet_read(fp->x.fpr, buf, page_size);
+#else
+        nread = fread(buf, 1, page_size, fp->file);
+#endif
+        if ( nread<=0 ) 
+            break;
+
+#ifdef _USE_KNETFILE
+        int count = fwrite(buf, 1, nread, bgzf_out->x.fpw);
+#else
+        int count = fwrite(buf, 1, nread, bgzf_out->file);
+#endif
+        if (count != nread)
+            error("Write failed, wrote %d instead of %d bytes.\n", count,(int)nread);
+    }
+
+    if (bgzf_close(bgzf_out) < 0) 
+        error("Error: %s\n",bgzf_out->error);
+   
+    return 0;
+}
+
+
+int main(int argc, char *argv[])
+{
+	int c, skip = -1, meta = -1, list_chrms = 0, force = 0, print_header = 0, bed_reg = 0;
+	ti_conf_t conf = ti_conf_gff;
+    const char *reheader = NULL;
+	while ((c = getopt(argc, argv, "p:s:b:e:0S:c:lhfBr:")) >= 0) {
+		switch (c) {
+		case 'B': bed_reg = 1; break;
+		case '0': conf.preset |= TI_FLAG_UCSC; break;
+		case 'S': skip = atoi(optarg); break;
+		case 'c': meta = optarg[0]; break;
+		case 'p':
+			if (strcmp(optarg, "gff") == 0) conf = ti_conf_gff;
+			else if (strcmp(optarg, "bed") == 0) conf = ti_conf_bed;
+			else if (strcmp(optarg, "sam") == 0) conf = ti_conf_sam;
+			else if (strcmp(optarg, "vcf") == 0 || strcmp(optarg, "vcf4") == 0) conf = ti_conf_vcf;
+			else if (strcmp(optarg, "psltbl") == 0) conf = ti_conf_psltbl;
+			else {
+				fprintf(stderr, "[main] unrecognized preset '%s'\n", optarg);
+				return 1;
+			}
+			break;
+		case 's': conf.sc = atoi(optarg); break;
+		case 'b': conf.bc = atoi(optarg); break;
+		case 'e': conf.ec = atoi(optarg); break;
+        case 'l': list_chrms = 1; break;
+        case 'h': print_header = 1; break;
+		case 'f': force = 1; break;
+        case 'r': reheader = optarg; break;
+		}
+	}
+	if (skip >= 0) conf.line_skip = skip;
+	if (meta >= 0) conf.meta_char = meta;
+	if (optind == argc) {
+		fprintf(stderr, "\n");
+		fprintf(stderr, "Program: tabix (TAB-delimited file InderXer)\n");
+		fprintf(stderr, "Version: %s\n\n", PACKAGE_VERSION);
+		fprintf(stderr, "Usage:   tabix <in.tab.bgz> [region1 [region2 [...]]]\n\n");
+		fprintf(stderr, "Options: -p STR     preset: gff, bed, sam, vcf, psltbl [gff]\n");
+		fprintf(stderr, "         -s INT     sequence name column [1]\n");
+		fprintf(stderr, "         -b INT     start column [4]\n");
+		fprintf(stderr, "         -e INT     end column; can be identical to '-b' [5]\n");
+		fprintf(stderr, "         -S INT     skip first INT lines [0]\n");
+		fprintf(stderr, "         -c CHAR    symbol for comment/meta lines [#]\n");
+	    fprintf(stderr, "         -r FILE    replace the header with the content of FILE [null]\n");
+		fprintf(stderr, "         -B         region1 is a BED file (entire file will be read)\n");
+		fprintf(stderr, "         -0         zero-based coordinate\n");
+		fprintf(stderr, "         -h         print the header lines\n");
+		fprintf(stderr, "         -l         list chromosome names\n");
+		fprintf(stderr, "         -f         force to overwrite the index\n");
+		fprintf(stderr, "\n");
+		return 1;
+	}
+    if (list_chrms) {
+		ti_index_t *idx;
+		int i, n;
+		const char **names;
+		idx = ti_index_load(argv[optind]);
+		if (idx == 0) {
+			fprintf(stderr, "[main] fail to load the index file.\n");
+			return 1;
+		}
+		names = ti_seqname(idx, &n);
+		for (i = 0; i < n; ++i) printf("%s\n", names[i]);
+		free(names);
+		ti_index_destroy(idx);
+		return 0;
+	}
+    if (reheader)
+        return reheader_file(reheader,argv[optind],conf.meta_char);
+
+	struct stat stat_tbi,stat_vcf;
+    char *fnidx = calloc(strlen(argv[optind]) + 5, 1);
+   	strcat(strcpy(fnidx, argv[optind]), ".tbi");
+
+	if (optind + 1 == argc) {
+		if (force == 0) {
+			if (stat(fnidx, &stat_tbi) == 0) 
+            {
+                // Before complaining, check if the VCF file isn't newer. This is a common source of errors,
+                //  people tend not to notice that tabix failed
+                stat(argv[optind], &stat_vcf);
+                if ( stat_vcf.st_mtime <= stat_tbi.st_mtime )
+                {
+                    fprintf(stderr, "[tabix] the index file exists. Please use '-f' to overwrite.\n");
+                    free(fnidx);
+                    return 1;
+                }
+			}
+		}
+        if ( bgzf_check_bgzf(argv[optind])!=1 )
+        {
+            fprintf(stderr,"[tabix] was bgzip used to compress this file? %s\n", argv[optind]);
+            free(fnidx);
+            return 1;
+        }
+		return ti_index_build(argv[optind], &conf);
+	}
+	{ // retrieve
+		tabix_t *t;
+        // Common source of errors: new VCF is used with an old index
+        stat(fnidx, &stat_tbi);
+        stat(argv[optind], &stat_vcf);
+        if ( force==0 && stat_vcf.st_mtime > stat_tbi.st_mtime )
+        {
+            fprintf(stderr, "[tabix] the index file is older than the vcf file. Please use '-f' to overwrite or reindex.\n");
+            free(fnidx);
+            return 1;
+        }
+        free(fnidx);
+
+		if ((t = ti_open(argv[optind], 0)) == 0) {
+			fprintf(stderr, "[main] fail to open the data file.\n");
+			return 1;
+		}
+		if (strcmp(argv[optind+1], ".") == 0) { // retrieve all
+			ti_iter_t iter;
+			const char *s;
+			int len;
+			iter = ti_query(t, 0, 0, 0);
+			while ((s = ti_read(t, iter, &len)) != 0) {
+				fputs(s, stdout); fputc('\n', stdout);
+			}
+			ti_iter_destroy(iter);
+		} else { // retrieve from specified regions
+			int i, len;
+            ti_iter_t iter;
+            const char *s;
+			const ti_conf_t *idxconf;
+
+			if (ti_lazy_index_load(t) < 0 && bed_reg == 0) {
+                fprintf(stderr,"[tabix] failed to load the index file.\n");
+                return 1;
+            }
+			idxconf = ti_get_conf(t->idx);
+
+            if ( print_header )
+            {
+                // If requested, print the header lines here
+                iter = ti_query(t, 0, 0, 0);
+                while ((s = ti_read(t, iter, &len)) != 0) {
+                    if ((int)(*s) != idxconf->meta_char) break;
+                    fputs(s, stdout); fputc('\n', stdout);
+                }
+                ti_iter_destroy(iter);
+            }
+			if (bed_reg) {
+				extern int bed_overlap(const void *_h, const char *chr, int beg, int end);
+				extern void *bed_read(const char *fn);
+				extern void bed_destroy(void *_h);
+
+				const ti_conf_t *conf_ = idxconf? idxconf : &conf; // use the index file if available
+				void *bed = bed_read(argv[optind+1]); // load the BED file
+				ti_interval_t intv;
+
+				if (bed == 0) {
+					fprintf(stderr, "[main] fail to read the BED file.\n");
+					return 1;
+				}
+				iter = ti_query(t, 0, 0, 0);
+				while ((s = ti_read(t, iter, &len)) != 0) {
+					int c;
+					ti_get_intv(conf_, len, (char*)s, &intv);
+					c = *intv.se; *intv.se = '\0';
+					if (bed_overlap(bed, intv.ss, intv.beg, intv.end)) {
+						*intv.se = c;
+						puts(s);
+					}
+					*intv.se = c;
+				}
+                ti_iter_destroy(iter);
+				bed_destroy(bed);
+			} else {
+				for (i = optind + 1; i < argc; ++i) {
+					int tid, beg, end;
+					if (ti_parse_region(t->idx, argv[i], &tid, &beg, &end) == 0) {
+						iter = ti_queryi(t, tid, beg, end);
+							while ((s = ti_read(t, iter, &len)) != 0) {
+							fputs(s, stdout); fputc('\n', stdout);
+						}
+						ti_iter_destroy(iter);
+					} 
+            	    // else fprintf(stderr, "[main] invalid region: unknown target name or minus interval.\n");
+				}
+			}
+		}
+		ti_close(t);
+	}
+	return 0;
+}

data/ext/tabix/perl/MANIFEST

@@ -0,0 +1,8 @@
+MANIFEST
+typemap
+Tabix.xs
+Tabix.pm
+TabixIterator.pm
+Makefile.PL
+t/01local.t
+t/02remote.t

data/ext/tabix/perl/Makefile.PL

@@ -0,0 +1,8 @@
+use ExtUtils::MakeMaker;
+WriteMakefile(
+			  NAME         => 'Tabix',
+			  VERSION_FROM => 'Tabix.pm',
+			  LIBS         => ['-lz -L.. -ltabix'],
+			  DEFINE       => '-D_FILE_OFFSET_BITS=64 -D_USE_KNETFILE',
+			  INC          => '-I..',
+			 );

data/ext/tabix/perl/Tabix.pm

@@ -0,0 +1,76 @@
+package Tabix;
+
+use strict;
+use warnings;
+use Carp qw/croak/;
+
+use TabixIterator;
+
+require Exporter;
+
+our @ISA = qw/Exporter/;
+our @EXPORT = qw/tabix_open tabix_close tabix_read tabix_query tabix_getnames tabix_iter_free/;
+
+our $VERSION = '0.2.0';
+
+require XSLoader;
+XSLoader::load('Tabix', $VERSION);
+
+sub new {
+  my $invocant = shift;
+  my %args = @_;
+  $args{-data} || croak("-data argument required");
+  my $class = ref($invocant) || $invocant;
+  my $self = {};
+  bless($self, $class);
+  $self->open($args{-data}, $args{-index});
+  return $self;
+}
+
+sub open {
+  my ($self, $fn, $fnidx) = @_;
+  $self->close;
+  $self->{_fn} = $fn;
+  $self->{_fnidx} = $fnidx;
+  $self->{_} = $fnidx? tabix_open($fn, $fnidx) : tabix_open($fn);
+}
+
+sub close {
+  my $self = shift;
+  if ($self->{_}) {
+	tabix_close($self->{_});
+	delete($self->{_}); delete($self->{_fn}); delete($self->{_fnidx});
+  }
+}
+
+sub DESTROY {
+  my $self = shift;
+  $self->close;
+}
+
+sub query {
+  my $self = shift;
+  my $iter;
+  if (@_) {
+	$iter = tabix_query($self->{_}, @_);
+  } else {
+	$iter = tabix_query($self->{_});
+  }
+  my $i = TabixIterator->new;
+  $i->set($iter);
+  return $i;
+}
+
+sub read {
+  my $self = shift;
+  my $iter = shift;
+  return tabix_read($self->{_}, $iter->get);
+}
+
+sub getnames {
+  my $self = shift;
+  return tabix_getnames($self->{_});
+}
+
+1;
+__END__

data/ext/tabix/perl/Tabix.xs

@@ -0,0 +1,71 @@
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+
+#include <stdlib.h>
+#include "tabix.h"
+
+MODULE = Tabix PACKAGE = Tabix
+
+tabix_t*
+tabix_open(fn, fnidx=0)
+	char *fn
+	char *fnidx
+  CODE:
+	RETVAL = ti_open(fn, fnidx);
+  OUTPUT:
+	RETVAL
+
+void
+tabix_close(t)
+	tabix_t *t
+  CODE:
+	ti_close(t);
+
+ti_iter_t
+tabix_query(t, seq=0, beg=0, end=0x7fffffff)
+	tabix_t *t
+	const char *seq
+	int beg
+	int end
+  PREINIT:
+  CODE:
+	RETVAL = ti_query(t, seq, beg, end);
+  OUTPUT:
+	RETVAL
+
+SV*
+tabix_read(t, iter)
+	tabix_t *t
+	ti_iter_t iter
+  PREINIT:
+	const char *s;
+	int len;
+  CODE:
+	s = ti_read(t, iter, &len);
+	if (s == 0)
+	   return XSRETURN_EMPTY;
+	RETVAL = newSVpv(s, len);
+  OUTPUT:
+	RETVAL
+
+void
+tabix_getnames(t)
+	tabix_t *t
+  PREINIT:
+	const char **names;
+	int i, n;
+  PPCODE:
+	ti_lazy_index_load(t);
+	names = ti_seqname(t->idx, &n);
+	for (i = 0; i < n; ++i)
+		XPUSHs(sv_2mortal(newSVpv(names[i], 0)));
+	free(names);
+
+MODULE = Tabix PACKAGE = TabixIterator
+
+void
+tabix_iter_free(iter)
+	ti_iter_t iter
+  CODE:
+	ti_iter_destroy(iter);

data/ext/tabix/perl/TabixIterator.pm

@@ -0,0 +1,41 @@
+package TabixIterator;
+
+use strict;
+use warnings;
+use Carp qw/croak/;
+
+require Exporter;
+
+our @ISA = qw/Exporter/;
+our @EXPORT = qw/tabix_iter_free/;
+
+our $VERSION = '0.2.0';
+
+require XSLoader;
+XSLoader::load('Tabix', $VERSION);
+
+sub new {
+  my $invocant = shift;
+  my $class = ref($invocant) || $invocant;
+  my $self = {};
+  bless($self, $class);
+  return $self;
+}
+
+sub set {
+  my ($self, $iter) = @_;
+  $self->{_} = $iter;
+}
+
+sub get {
+  my $self = shift;
+  return $self->{_};
+}
+
+sub DESTROY {
+  my $self = shift;
+  tabix_iter_free($self->{_}) if ($self->{_});
+}
+
+1;
+__END__

data/ext/tabix/perl/t/01local.t

@@ -0,0 +1,28 @@
+#-*-Perl-*-
+use Test::More tests => 9;
+BEGIN { use_ok('Tabix') };
+
+{ # C-like low-level interface
+	my $t = tabix_open("../example.gtf.gz");
+	ok($t);
+	my $iter = tabix_query($t, "chr1", 0, 2000);
+	ok($iter);
+	$_ = 0;
+	++$_ while (tabix_read($t, $iter));
+	is($_, 6);
+	tabix_iter_free($iter);
+	@_ = tabix_getnames($t);
+	is(scalar(@_), 2);
+}
+
+{ # OOP high-level interface
+	my $t = Tabix->new(-data=>"../example.gtf.gz");
+	ok($t);
+	my $iter = $t->query("chr1", 3000, 5000);
+	ok($iter);
+	$_ = 0;
+	++$_ while ($t->read($iter));
+	is($_, 27);
+	@_ = $t->getnames;
+	is($_[1], "chr2");
+}

data/ext/tabix/perl/t/02remote.t

@@ -0,0 +1,28 @@
+#-*-Perl-*-
+use Test::More tests => 9;
+BEGIN { use_ok('Tabix') };
+
+{ # FTP access
+	my $t = Tabix->new(-data=>"ftp://ftp.ncbi.nih.gov/1000genomes/ftp/pilot_data/release/2010_03/pilot1/CEU.SRP000031.2010_03.genotypes.vcf.gz");
+	ok($t);
+	my $iter = $t->query("1", 1000000, 1100000);
+	ok($iter);
+	$_ = 0;
+	++$_ while ($t->read($iter));
+	is($_, 306);
+	@_ = $t->getnames;
+	is(scalar(@_), 22);
+}
+
+{ # FTP access plus FTP index
+	my $t = Tabix->new(-data=>"ftp://ftp.ncbi.nih.gov/1000genomes/ftp/pilot_data/release/2010_03/pilot1/CEU.SRP000031.2010_03.genotypes.vcf.gz",
+					   -index=>"ftp://ftp.ncbi.nih.gov/1000genomes/ftp/pilot_data/release/2010_03/pilot1/CEU.SRP000031.2010_03.genotypes.vcf.gz.tbi");
+	ok($t);
+	my $iter = $t->query("19", 10000000, 10100000);
+	ok($iter);
+	$_ = 0;
+	++$_ while ($t->read($iter));
+	is($_, 268);
+	@_ = $t->getnames;
+	is(scalar(@_), 22);
+}

data/ext/tabix/perl/typemap

@@ -0,0 +1,3 @@
+TYPEMAP
+tabix_t*   T_PTROBJ
+ti_iter_t  T_PTROBJ

data/ext/tabix/python/setup.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+#
+# The MIT License
+#
+# Copyright (c) 2011 Seoul National University.
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+# Contact: Hyeshik Chang <hyeshik@snu.ac.kr>
+
+from distutils.core import setup, Extension
+
+# Change this to True when you need the knetfile support.
+USE_KNETFILE = False
+
+TABIX_SOURCE_FILES = [
+    '../bgzf.c', '../bgzip.c', '../index.c', '../knetfile.c', '../kstring.c'
+]
+
+define_options = [('_FILE_OFFSET_BITS', 64)]
+if USE_KNETFILE:
+    define_options.append(('_USE_KNETFILE', 1))
+
+ext_modules = [Extension("tabix", ["tabixmodule.c"] + TABIX_SOURCE_FILES,
+                         include_dirs=['..'],
+                         libraries=['z'],
+                         define_macros=define_options)]
+
+setup (name = 'tabix',
+       version = '1.0',
+       description = 'Python interface to tabix, a generic indexer '
+                     'for TAB-delimited genome position files',
+       author = 'Hyeshik Chang',
+       author_email = 'hyeshik@snu.ac.kr',
+       license = 'MIT',
+       ext_modules = ext_modules
+)