ngs_server 0.4 → 0.5

data/ext/tabix/tabix.h

@@ -0,0 +1,145 @@
+/* The MIT License
+
+   Copyright (c) 2009 Genome Research Ltd (GRL), 2010 Broad Institute
+
+   Permission is hereby granted, free of charge, to any person obtaining
+   a copy of this software and associated documentation files (the
+   "Software"), to deal in the Software without restriction, including
+   without limitation the rights to use, copy, modify, merge, publish,
+   distribute, sublicense, and/or sell copies of the Software, and to
+   permit persons to whom the Software is furnished to do so, subject to
+   the following conditions:
+
+   The above copyright notice and this permission notice shall be
+   included in all copies or substantial portions of the Software.
+
+   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+   NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+   BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+   ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+   CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+   SOFTWARE.
+*/
+
+/* Contact: Heng Li <lh3@live.co.uk> */
+
+#ifndef __TABIDX_H
+#define __TABIDX_H
+
+#include <stdint.h>
+#include "kstring.h"
+#include "bgzf.h"
+
+#define TI_PRESET_GENERIC 0
+#define TI_PRESET_SAM     1
+#define TI_PRESET_VCF     2
+
+#define TI_FLAG_UCSC      0x10000
+
+typedef int (*ti_fetch_f)(int l, const char *s, void *data);
+
+struct __ti_index_t;
+typedef struct __ti_index_t ti_index_t;
+
+struct __ti_iter_t;
+typedef struct __ti_iter_t *ti_iter_t;
+
+typedef struct {
+	BGZF *fp;
+	ti_index_t *idx;
+	char *fn, *fnidx;
+} tabix_t;
+
+typedef struct {
+	int32_t preset;
+	int32_t sc, bc, ec; // seq col., beg col. and end col.
+	int32_t meta_char, line_skip;
+} ti_conf_t;
+
+typedef struct {
+	int beg, end;
+	char *ss, *se;
+} ti_interval_t;
+
+extern ti_conf_t ti_conf_gff, ti_conf_bed, ti_conf_psltbl, ti_conf_vcf, ti_conf_sam; // preset
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+	/*******************
+	 * High-level APIs *
+	 *******************/
+
+	tabix_t *ti_open(const char *fn, const char *fnidx);
+	int ti_lazy_index_load(tabix_t *t);
+	void ti_close(tabix_t *t);
+	ti_iter_t ti_query(tabix_t *t, const char *name, int beg, int end);
+	ti_iter_t ti_queryi(tabix_t *t, int tid, int beg, int end);
+	ti_iter_t ti_querys(tabix_t *t, const char *reg);
+	const char *ti_read(tabix_t *t, ti_iter_t iter, int *len);
+
+	/* Destroy the iterator */
+	void ti_iter_destroy(ti_iter_t iter);
+
+	/* Get the list of sequence names. Each "char*" pointer points to a
+	 * internal member of the index, so DO NOT modify the returned
+	 * pointer; otherwise the index will be corrupted. The returned
+	 * pointer should be freed by a single free() call by the routine
+	 * calling this function. The number of sequences is returned at *n. */
+	const char **ti_seqname(const ti_index_t *idx, int *n);
+
+	/******************
+	 * Low-level APIs *
+	 ******************/
+
+	/* Build the index for file <fn>. File <fn>.tbi will be generated
+	 * and overwrite the file of the same name. Return -1 on failure. */
+	int ti_index_build(const char *fn, const ti_conf_t *conf);
+
+	/* Load the index from file <fn>.tbi. If <fn> is a URL and the index
+	 * file is not in the working directory, <fn>.tbi will be
+	 * downloaded. Return NULL on failure. */
+	ti_index_t *ti_index_load(const char *fn);
+
+	ti_index_t *ti_index_load_local(const char *fnidx);
+
+	/* Destroy the index */
+	void ti_index_destroy(ti_index_t *idx);
+
+	/* Parse a region like: chr2, chr2:100, chr2:100-200. Return -1 on failure. */
+	int ti_parse_region(const ti_index_t *idx, const char *str, int *tid, int *begin, int *end);
+
+	int ti_get_tid(const ti_index_t *idx, const char *name);
+
+	/* Get the iterator pointing to the first record at the current file
+	 * position. If the file is just openned, the iterator points to the
+	 * first record in the file. */
+	ti_iter_t ti_iter_first(void);
+
+	/* Get the iterator pointing to the first record in region tid:beg-end */
+	ti_iter_t ti_iter_query(const ti_index_t *idx, int tid, int beg, int end);
+
+	/* Get the data line pointed by the iterator and iterate to the next record. */
+	const char *ti_iter_read(BGZF *fp, ti_iter_t iter, int *len);
+
+	const ti_conf_t *ti_get_conf(ti_index_t *idx);
+	int ti_get_intv(const ti_conf_t *conf, int len, char *line, ti_interval_t *intv);
+
+	/*******************
+	 * Deprecated APIs *
+	 *******************/
+
+	/* The callback version for random access */
+	int ti_fetch(BGZF *fp, const ti_index_t *idx, int tid, int beg, int end, void *data, ti_fetch_f func);
+
+	/* Read one line. */
+	int ti_readline(BGZF *fp, kstring_t *str);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

data/ext/tabix/tabix.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+
+# Author: Heng Li and Aaron Quinlan
+# License: MIT/X11
+
+import sys
+from ctypes import *
+from ctypes.util import find_library
+import glob, platform
+
+def load_shared_library(lib, _path='.', ver='*'):
+    """Search for and load the tabix library. The
+    expectation is that the library is located in
+    the current directory (ie. "./")
+    """
+    # find from the system path
+    path = find_library(lib)
+    if (path == None): # if fail, search in the custom directory
+        s = platform.system()
+        if (s == 'Darwin'): suf = ver+'.dylib'
+        elif (s == 'Linux'): suf = '.so'+ver
+        candidates = glob.glob(_path+'/lib'+lib+suf);
+        if (len(candidates) == 1): path = candidates[0]
+        else: return None
+    cdll.LoadLibrary(path)
+    return CDLL(path)
+
+def tabix_init():
+    """Initialize and return a tabix reader object
+    for subsequent tabix_get() calls.  
+    """
+    tabix = load_shared_library('tabix')
+    if (tabix == None): return None
+    tabix.ti_read.restype = c_char_p
+    # on Mac OS X 10.6, the following declarations are required.
+    tabix.ti_open.restype = c_void_p
+    tabix.ti_querys.argtypes = [c_void_p, c_char_p]
+    tabix.ti_querys.restype = c_void_p
+    tabix.ti_query.argtypes = [c_void_p, c_char_p, c_int, c_int]
+    tabix.ti_query.restype = c_void_p
+    tabix.ti_read.argtypes = [c_void_p, c_void_p, c_void_p]
+    tabix.ti_iter_destroy.argtypes = [c_void_p]
+    tabix.ti_close.argtypes = [c_void_p]
+    # FIXME: explicit declarations for APIs not used in this script
+    return tabix
+
+# OOP interface
+class Tabix:
+    def __init__(self, fn, fnidx=0):
+        self.tabix = tabix_init();
+        if (self.tabix == None):
+            sys.stderr.write("[Tabix] Please make sure the shared library is compiled and available.\n")
+            return
+        self.fp = self.tabix.ti_open(fn, fnidx);
+
+    def __del__(self):
+        if (self.tabix): self.tabix.ti_close(self.fp)
+
+    def fetch(self, chr, start=-1, end=-1):
+        """Generator function that will yield each interval
+        within the requested range from the requested file.
+        """
+        if (self.tabix == None): return
+        if (start < 0): iter = self.tabix.ti_querys(self.fp, chr) # chr looks like: "chr2:1,000-2,000" or "chr2"
+        else: iter = self.tabix.ti_query(self.fp, chr, start, end) # chr must be a sequence name
+        if (iter == None):        
+            sys.stderr.write("[Tabix] Malformatted query or wrong sequence name.\n")
+            return
+        while (1): # iterate
+            s = self.tabix.ti_read(self.fp, iter, 0)
+            if (s == None): break
+            yield s   
+        self.tabix.ti_iter_destroy(iter)
+
+# command-line interface
+def main():
+    if (len(sys.argv) < 3):
+        sys.stderr.write("Usage: tabix.py <in.gz> <reg>\n")
+        sys.exit(1)
+    
+    # report the features in the requested interval
+    tabix = Tabix(sys.argv[1])
+    for line in tabix.fetch(sys.argv[2]):
+        print line
+
+if __name__ == '__main__':
+    main()

data/ext/tabix/tabix.tex

@@ -0,0 +1,121 @@
+\documentclass[10pt]{article}
+\usepackage{color}
+\definecolor{gray}{rgb}{0.7,0.7,0.7}
+
+\setlength{\topmargin}{0.0cm}
+\setlength{\textheight}{21.5cm}
+\setlength{\oddsidemargin}{0cm} 
+\setlength{\textwidth}{16.5cm}
+\setlength{\columnsep}{0.6cm}
+
+\begin{document}
+
+\title{The Tabix index file format}
+\author{Heng Li}
+\date{}
+
+\maketitle
+
+\begin{center}
+\begin{tabular}{|l|l|l|l|l|l|l|}
+\hline
+\multicolumn{4}{|c|}{\bf Field} & \multicolumn{1}{c|}{\bf Descrption} & \multicolumn{1}{c|}{\bf Type} & \multicolumn{1}{c|}{\bf Value} \\
+\hline\hline
+\multicolumn{4}{|l|}{\tt magic} & Magic string & {\tt char[4]} & TBI$\backslash$1 \\
+\hline
+\multicolumn{4}{|l|}{\tt n\_ref} & \# sequences & {\tt int32\_t} & \\
+\hline
+\multicolumn{4}{|l|}{\tt format} & Format (0: generic; 1: SAM; 2: VCF) & {\tt int32\_t} & \\
+\hline
+\multicolumn{4}{|l|}{\tt col\_seq} & Column for the sequence name & {\tt int32\_t} & \\
+\hline
+\multicolumn{4}{|l|}{\tt col\_beg} & Column for the start of a region & {\tt int32\_t} & \\
+\hline
+\multicolumn{4}{|l|}{\tt col\_end} & Column for the end of a region & {\tt int32\_t} & \\
+\hline
+\multicolumn{4}{|l|}{\tt meta} & Leading character for comment lines & {\tt int32\_t} & \\
+\hline
+\multicolumn{4}{|l|}{\tt skip} & \# lines to skip at the beginning & {\tt int32\_t} & \\
+\hline
+\multicolumn{4}{|l|}{\tt l\_nm} & Length of concatenated sequence names & {\tt int32\_t} & \\
+\hline
+\multicolumn{4}{|l|}{\tt names} & Concatenated names, each zero terminated & {\tt char[l\_nm]} & \\
+\hline
+\multicolumn{7}{|c|}{\textcolor{gray}{\it List of indices (n=n\_ref)}}\\
+\cline{2-7}
+\hspace{0.1cm} & \multicolumn{3}{l|}{\tt n\_bin} & \# distinct bins (for the binning index) & {\tt int32\_t} & \\
+\cline{2-7}
+ & \multicolumn{6}{c|}{\textcolor{gray}{\it List of distinct bins (n=n\_bin)}} \\
+\cline{3-7}
+ & \hspace{0.1cm} & \multicolumn{2}{l|}{\tt bin} & Distinct bin number & {\tt uint32\_t} & \\
+\cline{3-7}
+ & & \multicolumn{2}{l|}{\tt n\_chunk} & \# chunks & {\tt int32\_t} & \\
+\cline{3-7}
+ & & \multicolumn{5}{c|}{\textcolor{gray}{\it List of chunks (n=n\_chunk)}} \\
+\cline{4-7}
+ & & \hspace{0.1cm} & {\tt cnk\_beg} & Virtual file offset of the start of the chunk & {\tt uint64\_t} & \\
+\cline{4-7}
+ & & & {\tt cnk\_end} & Virtual file offset of the end of the chunk & {\tt uint64\_t} & \\
+\cline{2-7}
+ & \multicolumn{3}{l|}{\tt n\_intv} & \# 16kb intervals (for the linear index) & {\tt int32\_t} & \\
+\cline{2-7}
+ & \multicolumn{6}{c|}{\textcolor{gray}{\it List of distinct intervals (n=n\_intv)}} \\
+\cline{3-7}
+ & & \multicolumn{2}{l|}{\tt ioff} & File offset of the first record in the interval & {\tt uint64\_t} & \\
+\hline
+\end{tabular}
+\end{center}
+
+{\bf Notes:}
+
+\begin{itemize}
+\item The index file is BGZF compressed.
+\item All integers are little-endian.
+\item When {\tt (format\&0x10000)} is true, the coordinate follows the
+  {\tt BED} rule (i.e. half-closed-half-open and zero based); otherwise,
+  the coordinate follows the {\tt GFF} rule (closed and one based).
+\item For the SAM format, the end of a region equals {\tt POS} plus the
+  reference length in the alignment, inferred from {\tt CIGAR}. For the
+  VCF format, the end of a region equals {\tt POS} plus the size of the
+  deletion.
+\item Field {\tt col\_beg} may equal {\tt col\_end}, and in this case,
+  the end of a region is {\tt end}={\tt beg+1}.
+\item Example. For {\tt GFF}, {\tt format}=0, {\tt col\_seq}=1, {\tt
+    col\_beg}=4, {\tt col\_end}=5, {\tt meta}=`{\tt \#}' and {\tt
+    skip}=0. For {\tt BED}, {\tt format}=0x10000, {\tt col\_seq}=1, {\tt
+    col\_beg}=2, {\tt col\_end}=3, {\tt meta}=`{\tt \#}' and {\tt
+    skip}=0.
+\item Given a zero-based, half-closed and half-open region {\tt
+    [beg,end)}, the {\tt bin} number is calculated with the following C
+  function:
+\begin{verbatim}
+int reg2bin(int beg, int end) {
+  --end;
+  if (beg>>14 == end>>14) return ((1<<15)-1)/7 + (beg>>14);
+  if (beg>>17 == end>>17) return ((1<<12)-1)/7 + (beg>>17);
+  if (beg>>20 == end>>20) return  ((1<<9)-1)/7 + (beg>>20);
+  if (beg>>23 == end>>23) return  ((1<<6)-1)/7 + (beg>>23);
+  if (beg>>26 == end>>26) return  ((1<<3)-1)/7 + (beg>>26);
+  return 0;
+}
+\end{verbatim}
+\item The list of bins that may overlap a region {\tt [beg,end)} can be
+  obtained with the following C function.
+\begin{verbatim}
+#define MAX_BIN (((1<<18)-1)/7)
+int reg2bins(int rbeg, int rend, uint16_t list[MAX_BIN])
+{
+  int i = 0, k;
+  --rend;
+  list[i++] = 0;
+  for (k =    1 + (rbeg>>26); k <=    1 + (rend>>26); ++k) list[i++] = k;
+  for (k =    9 + (rbeg>>23); k <=    9 + (rend>>23); ++k) list[i++] = k;
+  for (k =   73 + (rbeg>>20); k <=   73 + (rend>>20); ++k) list[i++] = k;
+  for (k =  585 + (rbeg>>17); k <=  585 + (rend>>17); ++k) list[i++] = k;
+  for (k = 4681 + (rbeg>>14); k <= 4681 + (rend>>14); ++k) list[i++] = k;
+  return i; // #elements in list[]
+}
+\end{verbatim}
+\end{itemize}
+
+\end{document}

data/ext/vcftools/perl/Vcf.pm

@@ -72,6 +72,7 @@ use Carp;
 use Exporter;
 use Data::Dumper;
 use POSIX ":sys_wait_h";
+use FindBin;
 
 use vars qw/@ISA @EXPORT/;
 @ISA = qw/Exporter/;
@@ -157,6 +158,7 @@ sub new
     $$self{has_header} = 0;
     $$self{default_version} = '4.1';
     $$self{versions} = [ qw(Vcf3_2 Vcf3_3 Vcf4_0 Vcf4_1) ];
+    $$self{tabix} = "$FindBin::RealBin/../../tabix/tabix";
     if ( !exists($$self{max_line_len}) && exists($ENV{MAX_VCF_LINE_LEN}) ) { $$self{max_line_len} = $ENV{MAX_VCF_LINE_LEN} }
     $$self{fix_v40_AGtags} = $ENV{DONT_FIX_VCF40_AG_TAGS} ? 0 : 1;
     my %open_args = ();
@@ -203,7 +205,7 @@ sub _open
         {
             if ( exists($args{region}) && defined($args{region}) )
             {
-                $cmd = "tabix $tabix_args |";
+                $cmd = "$$self{tabix} $tabix_args |";
             }
             else { $cmd = "gunzip -c $$self{file} |"; } 
             $$self{check_exit_status} = 1;
@@ -211,7 +213,7 @@ sub _open
         elsif ( $$self{file}=~m{^(?:http|ftp)://} )
         {
             if ( !exists($args{region}) ) { $tabix_args .= ' .'; }
-            $cmd = "tabix $tabix_args |";
+            $cmd = "$$self{tabix} $tabix_args |";
             $$self{check_exit_status} = 1;
         }
         open($$self{fh},$cmd) or $self->throw("$cmd: $!");
@@ -2152,7 +2154,7 @@ sub get_chromosomes
 {
     my ($self) = @_;
     if ( !$$self{file} ) { $self->throw(qq[The parameter "file" not set.\n]); }
-    my (@out) = `tabix -l $$self{file}`;
+    my (@out) = `$$self{tabix} -l $$self{file}`;
     if ( $? ) 
     { 
         $self->throw(qq[The command "tabix -l $$self{file}" exited with an error. Is the file tabix indexed?\n]); 

data/ext/vcftools/perl/vcf-query

@@ -6,6 +6,8 @@
 use strict;
 use warnings;
 use Carp;
+use FindBin;
+use lib "$FindBin::Bin/../perl";
 use Vcf;
 
 my $opts = parse_params();

data/lib/ngs_server/version.rb

@@ -1,3 +1,3 @@
 module NgsServer
-  VERSION = "0.4"
+  VERSION = "0.5"
 end

data/lib/ngs_server.rb

@@ -4,23 +4,22 @@ require 'sinatra/base'
 require 'json'
 
 
+
 class MyNgsServer < Sinatra::Base
+
   set :server, 'thin'
 
   gempath = File.join(File.dirname(__FILE__), "../")
   datapath = File.join(gempath, 'data')
   bamtools_path = "#{gempath}/ext/bamtools/bin/bamtools"
   vcftools_path = "#{gempath}/ext/vcftools/bin/vcf-query"
-  get '/test' do
-
-    'hi'
-  end
-
   
   get '/json/bam/*' do |path|
-
-    content_type :json
-    response['Access-Control-Allow-Origin'] = '*';
+    
+    # turn off json content type so a preflight cors doesn't need to be done
+    #content_type :json
+    response['Access-Control-Allow-Origin'] = '*'; 
+    
 
     # invoke with eg: base_url/json/bam/subset22-sorted.bam?min=30000000&max=30010000&segment=22   
     json = `#{bamtools_path} convert -in #{datapath}/#{path} -format json -region #{params["segment"]}:#{params["min"]}..#{params["max"]}`
@@ -29,8 +28,9 @@ class MyNgsServer < Sinatra::Base
   end
 
   get '/json/vcf/*' do |path|
-
-    content_type :json
+    
+    # turn off json content type so a preflight cors doesn't need to be done
+    #content_type :json
     response['Access-Control-Allow-Origin'] = '*';
 
     # invoke with eg: base_url/json/vcf/ALL.2of4intersection.20100804.sites.vcf.gz?min=6992179&max=6992190&segment=1
@@ -41,7 +41,8 @@ class MyNgsServer < Sinatra::Base
 
   get '/json/sources/:extension' do
 
-    content_type :json
+    # turn off json content type so a preflight cors doesn't need to be done
+    # content_type :json
     response['Access-Control-Allow-Origin'] = '*';
 
     # invoke with eg: base_url/json/vcf/file=ALL.2of4intersection.20100804.sites.vcf.gz?min=6992179&max=6992190&segment=1

data/ngs_server.gemspec

@@ -20,7 +20,6 @@ Gem::Specification.new do |s|
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-#  s.executables   = "bin/ngs_"
-  s.extensions    = ["ext/bamtools/extconf.rb", "ext/vcftools/extconf.rb"]
+  s.extensions    = ["ext/bamtools/extconf.rb", "ext/vcftools/extconf.rb", "ext/tabix/extconf.rb"]
   s.require_paths = ["lib"]
 end

metadata

@@ -2,7 +2,7 @@
 name: ngs_server
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: "0.4"
+  version: "0.5"
 platform: ruby
 authors: 
 - Chase Miller
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-12-20 00:00:00 -05:00
+date: 2011-12-28 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -54,6 +54,7 @@ executables:
 extensions: 
 - ext/bamtools/extconf.rb
 - ext/vcftools/extconf.rb
+- ext/tabix/extconf.rb
 extra_rdoc_files: []
 
 files: 
@@ -338,6 +339,42 @@ files:
 - ext/bamtools/src/utils/bamtools_variant.h
 - ext/bamtools/src/utils/cmake_install.cmake
 - ext/bamtools/src/utils/utils_global.h
+- ext/tabix/ChangeLog
+- ext/tabix/Makefile
+- ext/tabix/NEWS
+- ext/tabix/TabixReader.java
+- ext/tabix/bam_endian.h
+- ext/tabix/bedidx.c
+- ext/tabix/bgzf.c
+- ext/tabix/bgzf.h
+- ext/tabix/bgzip.c
+- ext/tabix/example.gtf.gz
+- ext/tabix/example.gtf.gz.tbi
+- ext/tabix/extconf.rb
+- ext/tabix/index.c
+- ext/tabix/khash.h
+- ext/tabix/knetfile.c
+- ext/tabix/knetfile.h
+- ext/tabix/kseq.h
+- ext/tabix/ksort.h
+- ext/tabix/kstring.c
+- ext/tabix/kstring.h
+- ext/tabix/main.c
+- ext/tabix/perl/MANIFEST
+- ext/tabix/perl/Makefile.PL
+- ext/tabix/perl/Tabix.pm
+- ext/tabix/perl/Tabix.xs
+- ext/tabix/perl/TabixIterator.pm
+- ext/tabix/perl/t/01local.t
+- ext/tabix/perl/t/02remote.t
+- ext/tabix/perl/typemap
+- ext/tabix/python/setup.py
+- ext/tabix/python/tabixmodule.c
+- ext/tabix/python/test.py
+- ext/tabix/tabix.1
+- ext/tabix/tabix.h
+- ext/tabix/tabix.py
+- ext/tabix/tabix.tex
 - ext/vcftools/Makefile
 - ext/vcftools/README.txt
 - ext/vcftools/cpp/.svn/all-wcprops