bio-vcf 0.0.3 → 0.7.0

data/bio-vcf.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "bio-vcf"
-  s.version = "0.0.3"
+  s.version = "0.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Pjotr Prins"]
-  s.date = "2014-05-24"
+  s.date = "2014-06-24"
   s.description = "Smart parser for VCF format"
   s.email = "pjotr.public01@thebird.nl"
   s.executables = ["bio-vcf"]
@@ -30,11 +30,13 @@ Gem::Specification.new do |s|
     "features/cli.feature",
     "features/diff_count.feature",
     "features/multisample.feature",
+    "features/sfilter.feature",
     "features/somaticsniper.feature",
     "features/step_definitions/bio-vcf_steps.rb",
     "features/step_definitions/cli-feature.rb",
     "features/step_definitions/diff_count.rb",
     "features/step_definitions/multisample.rb",
+    "features/step_definitions/sfilter.rb",
     "features/step_definitions/somaticsniper.rb",
     "features/support/env.rb",
     "lib/bio-vcf.rb",
@@ -55,7 +57,9 @@ Gem::Specification.new do |s|
     "test/data/regression/rewrite.info.sample.ref",
     "test/data/regression/s.dp.ref",
     "test/data/regression/seval_s.dp.ref",
-    "test/data/regression/sfilter001.ref",
+    "test/data/regression/sfilter_seval_s.dp.ref",
+    "test/data/regression/thread4.ref",
+    "test/data/regression/thread4_4.ref",
     "test/performance/metrics.md"
   ]
   s.homepage = "http://github.com/pjotrp/bioruby-vcf"

data/features/cli.feature

@@ -8,6 +8,16 @@ Feature: Command-line interface (CLI)
     When I execute "./bin/bio-vcf -i --filter 'r.info.dp>100'"
     Then I expect the named output to match the named output "r.info.dp"
 
+  Scenario: Test the info filter using dp and threads
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --num-threads 4 --filter 'r.info.dp>2'"
+    Then I expect the named output to match the named output "thread4"
+
+  Scenario: Test the info filter using dp and threads with lines
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --num-threads 4 --thread-lines 4 --filter 'r.info.dp>2'"
+    Then I expect the named output to match the named output "thread4_4"
+
   Scenario: Test the sample filter using dp
     Given I have input file(s) named "test/data/input/multisample.vcf"
     When I execute "./bin/bio-vcf -i --sfilter 's.dp>20'"
@@ -23,6 +33,12 @@ Feature: Command-line interface (CLI)
     When I execute "./bin/bio-vcf -i --seval 's.dp'"
     Then I expect the named output to match the named output "seval_s.dp"
 
+  Scenario: Test the sample filter + eval using dp
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --sfilter 's.dp>10' --seval 's.dp'"
+    Then I expect the named output to match the named output "sfilter_seval_s.dp"
+
+
   Scenario: Rewrite an info field
     Given I have input file(s) named "test/data/input/multisample.vcf"
     When I execute "./bin/bio-vcf --rewrite rec.info[\'sample\']=\'XXXXX\'"

data/features/multisample.feature

@@ -36,11 +36,21 @@ Feature: Multi-sample VCF
     And I expect rec.sample.original.gt to be [0,1]
     And I expect rec.sample.s3t2.pl to be [20,0,522]
     # And the even better
+    And I expect r.original.gt? to be true
     And I expect rec.original.gt to be [0,1]
     And I expect rec.s3t2.pl to be [20,0,522]
     # Check for missing data
     And I expect test rec.missing_samples? to be false 
     And I expect test rec.original? to be true
+    # Special functions
+    And I expect r.original? to be true
+    And I expect r.original.gti? to be true
+    And I expect r.original.gti to be [0,1]
+    And I expect r.original.gti[1] to be 1
+    And I expect r.original.gts? to be true
+    And I expect r.original.gts to be ["C","T"]
+    And I expect r.original.gts[1] to be "T"
+
     Given multisample vcf line
     """
 1 10723 . C G 73.85 . AC=4;AF=0.667;AN=6;BaseQRankSum=1.300;DP=18;Dels=0.00;FS=3.680;HaplotypeScore=0.0000;MLEAC=4;MLEAF=0.667;MQ=20.49;MQ0=11;MQRankSum=1.754;QD=8.21;ReadPosRankSum=0.000 GT:AD:DP:GQ:PL  ./. ./. 1/1:2,2:4:6:66,6,0  1/1:4,1:5:3:36,3,0  ./. ./.  0/0:6,0:6:3:0,3,33

data/features/sfilter.feature

@@ -0,0 +1,60 @@
+@sfilter
+Feature: Sample filters
+
+  Bio-vcf supports sample filters, where every sample is evaluated
+  independently, though they have the rec information (chrom, pos, info)
+  available.
+
+  Scenario: Example of a sample
+  
+    Given the VCF line
+    """
+1 10723 . C G 73.85 . AC=4;AF=0.667;AN=6;BaseQRankSum=1.300;DP=18;Dels=0.00;FS=3.680;HaplotypeScore=0.0000;MLEAC=4;MLEAF=0.667;MQ=20.49;MQ0=11;MQRankSum=1.754;QD=8.21;ReadPosRankSum=0.000 GT:AD:DP:GQ:PL
+    """
+    When I evaluate '0/0:6,0:6:3:0,3,33'
+    Then I expect s.empty? to be false
+    Then I expect s.dp? to be true
+    Then I expect s.dp to be 6
+    And sfilter 's.dp>4' to be true
+
+  # Scenario: Sample with missing data
+    When I evaluate missing '0/0:6,0:.:3:0,3,33'
+    Then I expect s.empty? to be false
+    Then I expect s.dp? to be false
+    Then I expect s.dp to be nil
+    And sfilter 's.dp>4' to throw an error
+
+  # Scenario: Sample with missing data with ignore missing set
+    When I evaluate missing '0/0:6,0:.:3:0,3,33' with ignore missing
+    Then I expect s.empty? to be false
+    Then I expect s.dp? to be false
+    Then I expect s.dp to be nil
+    And sfilter 's.dp>4' to be false
+
+  # Scenario: Missing sample
+    When I evaluate empty './.'
+    Then I expect s.empty? to be true
+    Then I expect s.dp? to be false
+    Then I expect s.dp to throw an error
+    And sfilter 's.dp>4' to throw an error
+
+  # Scenario: Missing sample with ignore missing set
+    When I evaluate empty './.' with ignore missing
+    Then I expect s.empty? to be true
+    Then I expect s.dp? to be false
+    Then I expect s.dp to be nil
+    And sfilter 's.dp>4' to be false
+
+  # Scenario: Wrong field name in sample
+    When I evaluate '0/0:6,0:6:3:0,3,33'
+    Then I expect s.empty? to be false
+    Then I expect s.dp? to be true
+    Then I expect s.what? to throw an error
+    And I expect s.what to throw an error
+
+  # Scenario: Get other information for a sample
+    When I evaluate '0/0:6,0:6:3:0,3,33'
+    Then I expect r.chrom to be "1"
+    And I expect r.alt to be ["G"]
+    And I expect r.info.af to be 0.667
+

data/features/step_definitions/cli-feature.rb

@@ -8,5 +8,5 @@ When /^I execute "(.*?)"$/ do |arg1|
 end
 
 Then(/^I expect the named output to match the named output "(.*?)"$/) do |arg1|
-  RegressionTest::CliExec::exec(@cmd,arg1).should be_true
+  RegressionTest::CliExec::exec(@cmd,arg1,ignore: '##BioVcf=').should be_true
 end

data/features/step_definitions/multisample.rb

@@ -117,3 +117,35 @@ Then(/^I expect rec\.valid\? to be true$/) do
   expect(@rec1.valid?).to eq true
 end
 
+Then(/^I expect r\.original\.gt\? to be true$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect r\.original\? to be true$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect r\.original\.gti\? to be true$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect r\.original\.gti to be \[(\d+),(\d+)\]$/) do |arg1, arg2|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect r\.original\.gti\[(\d+)\] to be (\d+)$/) do |arg1, arg2|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect r\.original\.gts\? to be true$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect r\.original\.gts to be \["(.*?)","(.*?)"\]$/) do |arg1, arg2|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect r\.original\.gts\[(\d+)\] to be "(.*?)"$/) do |arg1, arg2|
+  pending # express the regexp above with the code you wish you had
+end
+

data/features/step_definitions/sfilter.rb

@@ -0,0 +1,90 @@
+Given(/^the VCF line$/) do |string|
+  @header = nil
+  @vcfline = string
+end
+
+When(/^I evaluate '([^']+)'$/) do |arg1|
+  @fields = VcfLine.parse((@vcfline.split(/\s+/)+[arg1]).join("\t"))
+  @rec = VcfRecord.new(@fields,@header)
+  p @rec
+end
+
+Then(/^I expect s\.empty\? to be false$/) do
+  p @rec.sample[0]
+  expect(@s.empty?).to be false
+end
+
+
+Then(/^I expect s\.dp to be (\d+)$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^sfilter 's\.dp>(\d+)' to be true$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+
+When(/^I evaluate missing '(\d+)\/(\d+):(\d+),(\d+):\.:(\d+):(\d+),(\d+),(\d+)'$/) do |arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect s\.dp to be nil$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^sfilter 's\.dp>(\d+)' to be false$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+
+When(/^I evaluate empty '\.\/\.'$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^sfilter 's\.dp>(\d+)' to throw an error$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+
+When(/^I evaluate missing '(\d+)\/(\d+):(\d+),(\d+):\.:(\d+):(\d+),(\d+),(\d+)' with ignore missing$/) do |arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect s\.empty\? to be true$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect s\.dp to throw an error$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+When(/^I evaluate empty '\.\/\.' with ignore missing$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect s\.dp\? to be true$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect s\.dp\? to be false$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect s\.what\? to throw an error$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect s\.what to throw an error$/) do
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect r\.chrom to be "(.*?)"$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect r\.alt to be \["(.*?)"\]$/) do |arg1|
+  pending # express the regexp above with the code you wish you had
+end
+
+Then(/^I expect r\.info\.af to be (\d+)\.(\d+)$/) do |arg1, arg2|
+  pending # express the regexp above with the code you wish you had
+end
+
+

data/lib/bio-vcf/utils.rb

@@ -9,14 +9,20 @@ module BioVcf
       !!Float(str) rescue false
     end
 
-    def self::convert v
-      if integer?(v)  # the common case
-        v = v.to_i
+    def self::convert str
+      if str =~ /,/
+        str.split(/,/).map { |item| convert(item) }
       else
-        # 150.268 or 9.68463e-05
-        v = v.to_f if float?(v)
+        if integer?(str)
+          str.to_i 
+        else
+          if float?(str)
+            str.to_f 
+          else
+            str
+          end
+        end
       end
-      v
     end
   end
 

data/lib/bio-vcf/vcfgenotypefield.rb

@@ -103,6 +103,10 @@ module BioVcf
       @alt = alt
     end
 
+    def to_s
+      @original_s
+    end
+
     def values
       @cache_values ||= @original_s.split(/:/)
     end
@@ -164,7 +168,6 @@ module BioVcf
       v.split(',').map{|i| i.to_i} 
     end
 
-
   end
 
   # Holds all samples

data/lib/bio-vcf/vcfheader.rb

@@ -27,6 +27,16 @@ module BioVcf
       @lines << line.strip
     end
 
+    # Add a key value list to the header
+    def tag h
+      h2 = h.dup
+      [:show_help,:skip_header,:verbose,:quiet,:debug].each { |key| h2.delete(key) }
+      info = h2.map { |k,v| k.to_s.capitalize+'='+'"'+v.to_s+'"' }.join(',')
+      line = '##BioVcf=<'+info+'>'
+      @lines.insert(-2,line)
+      line
+    end
+
     def version
       @version ||= lines[0].scan(/##fileformat=VCFv(\d+\.\d+)/)[0][0]
     end
@@ -39,10 +49,24 @@ module BioVcf
       @column ||= column_names.size
     end
 
+    def printable_header_line(fields)
+      fields.map { | field |
+        if field == '#samples'
+          samples
+        else
+          field
+        end
+      }.join("\t")
+    end
+
     def samples
       @samples ||= column_names[9..-1]
     end
 
+    def samples_index_array
+      @all_samples_index ||= column_names[9..-1].fill{|i| i} 
+    end
+
     def sample_index
       return @sample_index if @sample_index
       index = {}

data/lib/bio-vcf/vcfrdf.rb

@@ -11,20 +11,27 @@ module BioVcf
 @prefix dc: <http://purl.org/dc/elements/1.1/> .
 @prefix hgnc: <http://identifiers.org/hgnc.symbol/> .
 @prefix doi: <http://dx.doi.org/> .
-@prefix : <http://biobeat.org/rdf/ns#> .
+@prefix db: <http://biobeat.org/rdf/db#> .
+@prefix seq: <http://biobeat.org/rdf/seq#> .
+@prefix : <http://biobeat.org/rdf/vcf#> .
 EOB
     end
 
-    def VcfRdf::record id,rec,hash = {}
-      id2 = [id,'ch'+rec.chrom,rec.pos].join('_')
+    def VcfRdf::record id,rec,tags = "{}"
+      id2 = [id,'ch'+rec.chrom,rec.pos,rec.alt.join('')].join('_')
       print <<OUT
-:#{id2} :chr \"#{rec.chrom}\" .
-:#{id2} :pos #{rec.pos} .
-:#{id2} :vcf true .
+:#{id2} seq:chr \"#{rec.chrom}\" .
+:#{id2} seq:pos #{rec.pos} .
+:#{id2} seq:alt \"#{rec.alt[0]}\" .
+:#{id2} db:vcf true .
 OUT
-      hash.each do |k,v|
-        print ":#{id2} :#{k} #{v} .\n"
+      hash = eval(tags)
+      if hash
+        hash.each do |k,v|
+          print ":#{id2} #{k} #{v} .\n"
+        end
       end
+      print "\n"
     end
   end
 end

data/lib/bio-vcf/vcfrecord.rb

@@ -27,9 +27,10 @@ module BioVcf
       v = if @h
             @h[m.to_s.upcase]
           else
-            @info =~ /#{m.to_s.upcase}=([^;]+)/
+            @info =~ /#{m.to_s}=([^;]+)/i
             value = $1
-            # m = @info.match(/#{m.to_s.upcase}=(?<value>[^;]+)/)
+            # p [m,value]
+            # m = @info.match(/#{m.to_s.upcase}=(?<value>[^;]+)/) slower!
             # value = m[:value]
             if value == nil
               split_fields # no option but to split
@@ -117,6 +118,7 @@ module BioVcf
     def initialize fields, header
       @fields = fields
       @header = header
+      @sample_by_index = []
     end
 
     def chrom
@@ -176,14 +178,15 @@ module BioVcf
       sample[name]
     end
 
+    def sample_by_index i
+      # p [i,@fields[i+9]]
+      @sample_by_index[i] ||= VcfGenotypeField.new(@fields[i+9],format,@header,alt)
+    end
+
+    # Walk the samples. list contains an Array of int (the index)
     def each_sample(list = nil)
-      samples = @header.column_names[9..-1]
-      raise "Empty sample list, can not execute query!" if not samples
-      samples.each_with_index { |name,i|
-        # p [i,list]
-        next if list and not list.index(i.to_s)
-        yield VcfSample::Sample.new(self,sample[name]) 
-      }
+      list = @header.samples_index_array() if not list 
+      list.each { |i| yield VcfSample::Sample.new(self,sample_by_index(i)) }
     end
 
     def missing_samples?
@@ -230,6 +233,39 @@ module BioVcf
       end
     end
 
+    def filter expr, ignore_missing_data, quiet
+      begin
+        if not respond_to?(:call_cached_filter)
+          code =
+          """
+          def call_cached_filter(rec,fields)
+            r = rec
+            #{expr}
+          end
+          """
+          self.class.class_eval(code)
+        end
+        res = call_cached_filter(self,@fields)
+        if res.kind_of?(Array)
+          res.join("\t")
+        else
+          res
+        end
+      rescue NoMethodError => e
+        if not quiet
+          $stderr.print "RECORD ERROR!\n"
+          $stderr.print [@fields],"\n"
+          $stderr.print expr,"\n"
+        end
+        if ignore_missing_data
+          $stderr.print e.message if not quiet
+          return false
+        else
+          raise
+        end
+      end
+    end
+
     # Return the sample
     def method_missing(m, *args, &block)  
       name = m.to_s