bio-vcf 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: da9c14380c66a497089fef836a22d44c7651f264
-  data.tar.gz: 3291080afde13a1b7cd392d4f1f06ff275ec1f1b
+  metadata.gz: 90a933c33c683c1f0886a202fa5a9ee5ed2ad8ff
+  data.tar.gz: 1f769a89fcb3e3b44e22864ddf729ea3ac040260
 SHA512:
-  metadata.gz: f1584a14d8de45dc04115ae6458f3d1b6e9c69c9d19092dfefc18e411fc347d660ed1ca586e3e3b4327d4774e323a78eabe3434a4cd8eab7d87b1f4e37915c0e
-  data.tar.gz: d0a56668f17a272807167bde9caa066ba960ac597b9ce57a205fb2564b9944421e5ef4035f43ab855f72ab2c9ece6b963ef01043372db5601520f5593a1fe31b
+  metadata.gz: 308d93ca1bcb142fa9cd4be63d929edb0ad92b7ac0da0d4f2d51f4b363de6ef0b87ac3d2688af8c36257317e203f69355fdc3348c6a330adb7d997af7ab6714d
+  data.tar.gz: d7d328a13d90b209a6068f9d3f09d56e8a00262ccf7fba9d9d67ffe4935993b2ccbb2ddc2f9a4831dd8928258a9c5d468f050d9edfbb95737616f4bfaf184bb0

data/Gemfile

@@ -12,3 +12,4 @@ group :development do
   gem "jeweler", "~> 2.0.1" # , "~> 1.8.4", :git => "https://github.com/technicalpickles/jeweler.git"
   gem "regressiontest", "~> 0.0.3"
 end
+

data/README.md

@@ -16,7 +16,7 @@ RDF and JSON. Why would you use bio-vcf over other parsers?
 7. Bio-vcf allows for genotype processing
 8. Bio-vcf has support for set analysis
 9. Bio-vcf has sane error handling
-10. Bio-vcf can output tabular data, HTML, LaTeX, RDF, JSON and JSON-LD using templates
+10. Bio-vcf can output tabular data, HTML, LaTeX, RDF, JSON and JSON-LD and even other VCFs using (erb) templates
 
 Bio-vcf has better performance than other tools
 because of lazy parsing, multi-threading, and useful combinations of
@@ -52,7 +52,7 @@ a 16 core machine takes
   sys     0m5.039s
 ```
 
-which shows pretty decent core utilisation (10x). We are running 
+which shows decent core utilisation (10x). We are running 
 gzip compressed VCF files of 30+ Gb with similar performance gains.
 
 Use zcat to
@@ -633,6 +633,12 @@ To remove/select 3 samples:
   bio-vcf --samples 0,1,3 < mytest.vcf
 ```
 
+Filter on a BED file and annotate the gene name in the resulting VCF
+
+```sh
+  bio-vcf -v --bed test/data/input/test.bed --rewrite 'rec.info["gene"]=bed[3]' < test/data/input/somaticsniper.vcf
+```
+
 ## RDF output
 
 You can use --rdf for turtle RDF output from simple one-liners, note the use of --id and
@@ -691,10 +697,11 @@ template could be
 };
 ```
 
-To get JSON, run with something like
+To get JSON, run with something like (combining 
+with a filter)
 
 ```sh
-  bio-vcf --template template/vcf2json.erb --filter 'r.info.maf[0]<0.01' < dbsnp.vcf
+  bio-vcf --template template/vcf2json.erb --filter 'r.info.sao==1' < dbsnp.vcf
 ```
 
 which renders
@@ -713,7 +720,7 @@ which renders
 Likewise for RDF output:
 
 ```sh
-  bio-vcf --template template/vcf2rdf.erb --filter 'r.info.maf[0]<0.01' < dbsnp.vcf
+  bio-vcf --template template/vcf2rdf.erb --filter 'r.info.sao==1' < dbsnp.vcf
 ```
 
 renders the ERB template 
@@ -728,7 +735,7 @@ renders the ERB template
   seq:pos      <%= rec.pos %> ,
   seq:ref      "<%= rec.ref %>" ,
   seq:alt      "<%= rec.alt[0] %>" ,
-  seq:maf      <%= rec.info.maf[0] %> ,
+  seq:maf      <%= (rec.info.maf[0]*100).round %> ,
   seq:dp       <%= rec.info.dp %> ,
   db:vcf       true .
 ```
@@ -742,12 +749,72 @@ into
   seq:pos      33703698 ,
   seq:ref      "C" ,
   seq:alt      "A" ,
-  seq:maf      0.1567 ,
+  seq:maf      16 ,
   seq:dp       92 ,
   db:vcf       true .
 ```
 
-Be creative! You can write templates for csv, HTML, XML, LaTeX, RDF, JSON, YAML, JSON-LD, etc. etc.!
+Note the calculated field value for maf. Be creative! You can write templates for csv, HTML, XML, LaTeX, RDF, JSON, YAML, JSON-LD, etc. etc.!
+
+### Metadata
+
+Templates can also print data as a header of the JSON/YAML/RDF output. For this
+use the '=' prefix with HEADER, BODY, FOOTER keywords in the template. A small example
+can be
+
+```Javascript
+=HEADER
+<% require 'json' %>
+[
+  { "HEADER": {
+    "options":  <%= options.to_h.to_json %>,
+    "files":    <%= ARGV %>,
+    "version":  "<%= BIOVCF_VERSION %>"
+  },
+
+=BODY
+
+{
+  "seq:chr": "<%= rec.chrom %>" ,
+  "seq:pos": <%= rec.pos %> ,
+  "seq:ref": "<%= rec.ref %>" ,
+  "seq:alt": "<%= rec.alt[0] %>" ,
+  "dp":      <%= rec.info.dp %> ,
+},
+=FOOTER
+]
+```
+
+may generate something like
+
+```Javascript
+[
+  { "HEADER": {
+    "options":  {"show_help":false,"source":"https://github.com/CuppenResearch/bioruby-vcf","version":"0.8.1-pre3 (Pjotr Prins)","date":"2014-11-26 12:51:36 +0000","thread_lines":40000,"template":"template/vcf2json.erb","skip_header":true},
+    "files":    [],
+    "version":  "0.8.1-pre3"
+  },
+{
+  "seq:chr": "1" ,
+  "seq:pos": 883516 ,
+  "seq:ref": "G" ,
+  "seq:alt": "A" ,
+  "dp":       ,
+},
+{
+  "seq:chr": "1" ,
+  "seq:pos": 891344 ,
+  "seq:ref": "G" ,
+  "seq:alt": "A" ,
+  "dp":       ,
+},
+]
+```
+
+Note that the template is not smart enough to remove the final comma
+from the last BODY element. To make it valid JSON that needs to be
+removed. A future version may add a parameter to the BODY element or a
+global rewrite function for this purpose. YAML and RDF have no such issue.
 
 ## Statistics
 

data/Rakefile

@@ -21,6 +21,7 @@ Jeweler::Tasks.new do |gem|
   gem.description = %Q{Smart lazy multi-threaded parser for VCF format with useful filtering and output rewriting}
   gem.email = "pjotr.public01@thebird.nl"
   gem.authors = ["Pjotr Prins"]
+  gem.required_ruby_version = '>=2.0.0'
   # dependencies defined in Gemfile
 end
 Jeweler::RubygemsDotOrgTasks.new

data/VERSION

@@ -1 +1 @@
-0.8.0
+0.8.1

data/bin/bio-vcf

@@ -58,6 +58,10 @@ opts = OptionParser.new do |o|
     options[:efilter_samples] = l
   end
 
+  o.on("--bed bedfile", String, "Filter on BED elements") do |bed|
+    options[:bed] = bed
+  end
+
   o.on('-e cmd', '--eval cmd',String, 'Evaluate command on each record') do |cmd|
     options[:eval] = cmd
   end
@@ -145,7 +149,9 @@ end
 
 opts.parse!(ARGV)
 
-$stderr.print "vcf #{version} (biogem Ruby #{RUBY_VERSION}) by Pjotr Prins 2014\n" if !options[:quiet]
+BIOVCF_VERSION=version
+BIOVCF_BANNER = "vcf #{version} (biogem Ruby #{RUBY_VERSION}) by Pjotr Prins 2014\n" if !options[:quiet]
+$stderr.print BIOVCF_BANNER
 
 if options[:show_help] 
   print opts 
@@ -161,9 +167,11 @@ $stderr.print "Options: ",options,"\n" if !options[:quiet]
 
 if options[:template]
   include BioVcf::RDF
+  require 'bio-vcf/template'
   fn = options[:template]
   raise "No template #{fn}!" if not File.exist?(fn)
-  template = ERB.new(File.read(fn))
+  # template = ERB.new(File.read(fn))
+  template = Bio::Template.new(fn)
 end
 
 if options[:num_threads] != 1
@@ -227,7 +235,7 @@ def parse_header line, samples, options
 end
 
 # Parse a VCF line and return the result as a string
-def parse_line line,header,options,samples,template,stats=nil
+def parse_line line,header,options,bedfilter,samples,template,stats=nil
   fields = VcfLine.parse(line)
   rec = VcfRecord.new(fields,header)
   r = rec # alias
@@ -248,6 +256,11 @@ def parse_line line,header,options,samples,template,stats=nil
 
   # --------------------------
   # Filtering and set analysis
+  if bedfilter
+    bed = bedfilter.contains(rec)
+    return if not bed
+  end
+
   return if filter and not rec.filter(filter,ignore_missing_data: ignore_missing,quiet: quiet)
   
   if sfilter
@@ -315,7 +328,7 @@ def parse_line line,header,options,samples,template,stats=nil
     elsif options[:template]
       # Ruby ERB template
       begin
-        template.result(binding)
+        template.body(binding)
       rescue Exception => e
         $stderr.print e,": ",fields,"\n"
         $stderr.print e.backtrace.inspect if options[:verbose]
@@ -343,11 +356,15 @@ chunks = []
 lines = []
 line_number=0
 
+if options[:bed]
+  bedfilter = BedFilter.new(options[:bed])
+end 
+
 begin
   process = lambda { | lines |
     res = []
     lines.each do | line |
-      res << parse_line(line,header,options,samples,template,stats)
+      res << parse_line(line,header,options,bedfilter,samples,template,stats)
     end
     res
   }
@@ -357,6 +374,7 @@ begin
     end
   } # end output
 
+  print template.header(binding) if template
   # ---- Main loop
   STDIN.each_line do | line |
     line_number += 1
@@ -410,6 +428,7 @@ begin
       process.call(chunk)
     }
   end
+  print template.footer(binding) if template
   stats.print if stats
 
 rescue Exception => e

data/bio-vcf.gemspec

@@ -2,14 +2,16 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
+# stub: bio-vcf 0.8.1 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "bio-vcf"
-  s.version = "0.8.0"
+  s.version = "0.8.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib"]
   s.authors = ["Pjotr Prins"]
-  s.date = "2014-09-19"
+  s.date = "2014-11-26"
   s.description = "Smart lazy multi-threaded parser for VCF format with useful filtering and output rewriting"
   s.email = "pjotr.public01@thebird.nl"
   s.executables = ["bio-vcf"]
@@ -40,6 +42,8 @@ Gem::Specification.new do |s|
     "features/step_definitions/somaticsniper.rb",
     "features/support/env.rb",
     "lib/bio-vcf.rb",
+    "lib/bio-vcf/bedfilter.rb",
+    "lib/bio-vcf/template.rb",
     "lib/bio-vcf/utils.rb",
     "lib/bio-vcf/variant.rb",
     "lib/bio-vcf/vcf.rb",
@@ -53,6 +57,7 @@ Gem::Specification.new do |s|
     "template/gatk_vcf2rdf.erb",
     "template/vcf2json.erb",
     "template/vcf2rdf.erb",
+    "template/vcf2rdf_header.erb",
     "test/data/input/dbsnp.vcf",
     "test/data/input/multisample.vcf",
     "test/data/input/somaticsniper.vcf",
@@ -70,8 +75,8 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/pjotrp/bioruby-vcf"
   s.licenses = ["MIT"]
-  s.require_paths = ["lib"]
-  s.rubygems_version = "2.0.3"
+  s.required_ruby_version = Gem::Requirement.new(">= 2.0.0")
+  s.rubygems_version = "2.2.2"
   s.summary = "Fast multi-threaded VCF parser"
 
   if s.respond_to? :specification_version then

data/features/multisample.feature

@@ -63,3 +63,16 @@ Feature: Multi-sample VCF
     And I expect rec.sample.s3t2? to be true
     And I expect rec.missing_samples? to be true
 
+    # Phased genotype
+    Given multisample vcf line
+    """
+1 10723 . C G 73.85 . AC=4;AF=0.667;AN=6;BaseQRankSum=1.300;DP=18;Dels=0.00;FS=3.680;HaplotypeScore=0.0000;MLEAC=4;MLEAF=0.667;MQ=20.49;MQ0=11;MQRankSum=1.754;QD=8.21;ReadPosRankSum=0.000 GT:AD:DP:GQ:PL  0|1 ./. 1/1:2,2:4:6:66,6,0  1/1:4,1:5:3:36,3,0  ./. ./.  0/0:6,0:6:3:0,3,33
+    """
+    When I parse the record
+    Then I expect rec.pos to contain 10723
+    Then I expect rec.valid? to be true
+    And I expect r.original? to be true
+    And I expect r.original.gts? to be true
+    And I expect r.original.gts to be ["C","G"]
+    And I expect r.original.gts[0] to be "C"
+    And I expect r.original.gts[1] to be "G"

data/lib/bio-vcf.rb

@@ -17,3 +17,4 @@ require 'bio-vcf/vcfgenotypefield'
 require 'bio-vcf/vcfrecord'
 require 'bio-vcf/variant'
 require 'bio-vcf/vcfstatistics'
+require 'bio-vcf/bedfilter'

data/lib/bio-vcf/bedfilter.rb

@@ -0,0 +1,43 @@
+module BioVcf
+
+  class BedFilter
+    def initialize bedfilen
+      require 'binary_search/native'
+
+      # Parse Bed file and build up search array
+      chrs = {}
+      info = {}
+      File.open(bedfilen).each_line { | line |
+        (chr,start,stop,gene) = line.strip.split(/\t/)[0..3]
+        chrs[chr] ||= []
+        chrs[chr].push(stop.to_i)
+        info[chr+':'+stop] = [chr,start.to_i,stop.to_i,gene]
+      }
+      # Make sure chrs is sorted
+      @chrs = {}
+      chrs.each { | k,list |
+        @chrs[k] = list.sort
+      }
+      @info = info
+    end
+
+    def contains(rec)
+      stop_list = @chrs[rec.chrom]
+      if stop_list
+        pos = rec.pos
+        stop = stop_list.bsearch { |bedstop| bedstop >= pos }
+        if stop
+          rinfo = @info[rec.chrom+':'+stop.to_s]
+          raise "Unexpected error in BED record for #{rec.chrom}:#{stop} position" if rinfo == nil
+          start = rinfo[1]
+          if pos >= start
+            # p [rec.chrom,rec.pos,rinfo]
+            return rinfo
+          end
+        end
+      end
+      nil
+    end
+  end
+
+end

data/lib/bio-vcf/template.rb

@@ -0,0 +1,75 @@
+require 'erb'
+
+module Bio
+
+  class Template
+    
+    def initialize fn
+      raise "Can not find template #{fn}!" if not File.exist?(fn)
+      parse(File.read(fn))
+    end
+
+    def parse buf
+      header = []
+      body = []
+      footer = []
+      where = :header
+      buf.split("\n").each do | line |
+        case where
+          when :header
+            next if line =~ /=HEADER/
+            if line =~ /=BODY/
+              body = []
+              where = :body
+              next
+            end
+            header << line
+          when :body
+            if line =~ /=FOOTER/
+              footer = []
+              where = :footer
+              next
+            end
+            body << line
+          else
+            footer << line
+        end
+      end
+      if body == []
+        body = header
+        header = []
+      end
+      @erb_header = ERB.new(header.join("\n")) if header.size
+      @erb_body   = ERB.new(body.join("\n")) if body.size
+      @erb_footer = ERB.new(footer.join("\n")) if footer.size
+    end
+    
+    def result env
+      @erb.result(env)
+    end
+
+    def header env
+      if @erb_header
+        @erb_header.result(env)
+      else
+        ""
+      end
+    end
+
+    def body env
+      if @erb_body
+        @erb_body.result(env)
+      else
+        ""
+      end
+    end
+
+    def footer env
+      if @erb_footer
+        @erb_footer.result(env)
+      else
+        ""
+      end
+    end
+  end
+end

data/lib/bio-vcf/vcfgenotypefield.rb

@@ -156,7 +156,7 @@ module BioVcf
     end
 
     def gti
-      gt.split('/').map { |g| g.to_i }
+      gt.split(/[\/\|]/).map { |g| g.to_i }
     end
 
     def gts?
@@ -218,7 +218,12 @@ module BioVcf
     end
 
     def [] name
-      @samples[name] ||= VcfGenotypeField.new(@fields[@sample_index[name]],@format,@header,@ref,@alt)
+      begin
+        @samples[name] ||= VcfGenotypeField.new(@fields[@sample_index[name]],@format,@header,@ref,@alt)
+      rescue TypeError
+        $stderr.print "Unknown field name <#{name}> in record, did you mean r.info.#{name}?\n"
+        raise 
+      end
     end
 
     def method_missing(m, *args, &block) 

data/lib/bio-vcf/vcfheader.rb

@@ -60,13 +60,21 @@ module BioVcf
     end
 
     def samples
-      @samples ||= column_names[9..-1]
+      @samples ||= if column_names.size > 8
+                     column_names[9..-1]
+                   else
+                     []
+                   end
     end
 
     def samples_index_array
       @all_samples_index ||= column_names[9..-1].fill{|i| i}
     end
 
+    def num_samples
+      @num_samples ||= ( samples == nil ? 0 : samples.size )
+    end
+
     def sample_index
       return @sample_index if @sample_index
       index = {}

data/lib/bio-vcf/vcfsample.rb

@@ -40,7 +40,7 @@ module BioVcf
       # Split GT into index values
       def gti
         v = fetch_values("GT")
-        v.split(/\//).map{ |v| (v=='.' ? nil : v.to_i) }
+        v.split(/[\/\|]/).map{ |v| (v=='.' ? nil : v.to_i) }
       end
 
       # Split GT into into a nucleode sequence

data/template/vcf2json.erb

@@ -1,8 +1,20 @@
+=HEADER
+<% require 'json' %>
+[
+  { "HEADER": {
+    "options":  <%= options.to_h.to_json %>,
+    "files":    <%= ARGV %>,
+    "version":  "<%= BIOVCF_VERSION %>"
+  },
+
+=BODY
+
 {
   "seq:chr": "<%= rec.chrom %>" ,
   "seq:pos": <%= rec.pos %> ,
   "seq:ref": "<%= rec.ref %>" ,
   "seq:alt": "<%= rec.alt[0] %>" ,
-  "seq:maf": <%= rec.info.maf[0] %> ,
   "dp":      <%= rec.info.dp %> ,
-};
+},
+=FOOTER
+]

data/template/vcf2rdf_header.erb

@@ -0,0 +1,24 @@
+=HEADER
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix dc: <http://purl.org/dc/elements/1.1/> .
+@prefix hgnc: <http://identifiers.org/hgnc.symbol/> .
+@prefix doi: <http://dx.doi.org/> .
+@prefix seq: <http://biobeat.org/rdf/seq#> .
+@prefix db: <http://biobeat.org/rdf/db#> .
+@prefix : <http://biobeat.org/rdf/dbsnp#> .
+
+=BODY
+<%
+  id = Turtle::mangle_identifier(['ch'+rec.chrom,rec.pos,rec.alt.join('')].join('_')) 
+%>
+:<%= id %>
+  :query_id    "<%= id %>";
+  seq:chr      "<%= rec.chrom %>" ;
+  seq:pos      <%= rec.pos %> ;
+  seq:ref      "<%= rec.ref %>" ;
+  seq:alt      "<%= rec.alt[0] %>" ;
+  seq:dp       <%= rec.info.dp %> ;
+  db:vcf       true .
+
+=FOOTER

data/test/data/regression/thread4_4_failed_filter-stderr.ref

@@ -1 +1,2 @@
 Error: Missing 'parallel' module. Install with command 'gem install parallel' if you want multiple threads
+.Unknown field name <t> in record, did you mean r.info.t?

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bio-vcf
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Pjotr Prins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-09-19 00:00:00.000000000 Z
+date: 2014-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -98,6 +98,8 @@ files:
 - features/step_definitions/somaticsniper.rb
 - features/support/env.rb
 - lib/bio-vcf.rb
+- lib/bio-vcf/bedfilter.rb
+- lib/bio-vcf/template.rb
 - lib/bio-vcf/utils.rb
 - lib/bio-vcf/variant.rb
 - lib/bio-vcf/vcf.rb
@@ -111,6 +113,7 @@ files:
 - template/gatk_vcf2rdf.erb
 - template/vcf2json.erb
 - template/vcf2rdf.erb
+- template/vcf2rdf_header.erb
 - test/data/input/dbsnp.vcf
 - test/data/input/multisample.vcf
 - test/data/input/somaticsniper.vcf
@@ -137,7 +140,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -145,7 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Fast multi-threaded VCF parser