bio-vcf 0.9.0 → 0.9.2

data/bio-vcf.gemspec

@@ -3,12 +3,12 @@
 
 Gem::Specification.new do |s|
   s.name = "bio-vcf"
-  s.version = "0.9.0"
+  s.version = File.read("VERSION")
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Pjotr Prins"]
   # s.date = "2015-12-28"
-  s.description = "Smart lazy multi-threaded parser for VCF format with useful filtering and output rewriting"
+  s.description = "Smart lazy multi-threaded parser for VCF format with useful filtering and output rewriting (JSON, RDF etc.)"
   s.email = "pjotr.public01@thebird.nl"
   s.executables = ["bio-vcf"]
   s.extra_rdoc_files = [
@@ -35,25 +35,8 @@ Gem::Specification.new do |s|
   s.licenses = ["MIT"]
   s.require_paths = ["lib"]
   s.required_ruby_version = Gem::Requirement.new(">= 2.0.0")
-  s.rubygems_version = "2.0.3"
-  s.summary = "Fast multi-threaded VCF parser"
+  # s.rubygems_version = "2.0.3"
+  s.summary = "Fast multi-purpose multi-threaded VCF parser"
 
-  # if s.respond_to? :specification_version then
-  #   s.specification_version = 4
-
-  #   if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-  #     s.add_development_dependency(%q<rspec>, [">= 2.14.0"])
-  #     s.add_development_dependency(%q<cucumber>, [">= 1.3.11"])
-  #     s.add_development_dependency(%q<regressiontest>, [">= 0.0.3"])
-  #   else
-  #     s.add_dependency(%q<rspec>, [">= 2.14.0"])
-  #     s.add_dependency(%q<cucumber>, [">= 1.3.11"])
-  #     s.add_dependency(%q<regressiontest>, [">= 0.0.3"])
-  #   end
-  # else
-  #   s.add_dependency(%q<rspec>, [">= 2.14.0"])
-  #   s.add_dependency(%q<cucumber>, [">= 1.3.11"])
-  #   s.add_dependency(%q<regressiontest>, [">= 0.0.3"])
-  # end
 end
 

data/features/#cli.feature#

@@ -0,0 +1,71 @@
+@cli
+Feature: Command-line interface (CLI)
+
+  bio-vcf has a powerful command line interface. Here we regression test features.
+
+  Scenario: Test the info filter using dp
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --filter 'r.info.dp>100'"
+    Then I expect the named output to match the named output "r.info.dp"
+
+  Scenario: Test the info filter using dp and threads
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --num-threads 4 --filter 'r.info.dp>2'"
+    Then I expect the named output to match the named output "thread4"
+
+  Scenario: Test the info filter using dp and threads with lines
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --num-threads 4 --thread-lines 4 --filter 'r.info.dp>2'"
+    Then I expect the named output to match the named output "thread4_4"
+
+  Scenario: Test the sample filter using dp
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --sfilter 's.dp>20'"
+    Then I expect the named output to match the named output "s.dp"
+
+  Scenario: Test the include sample filter using dp
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --ifilter 's.dp>100' --seval s.dp"
+    Then I expect the named output to match the named output "ifilter_s.dp"
+
+  Scenario: Test the info eval using dp
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --eval 'r.info.dp'"
+    Then I expect the named output to match the named output "eval_r.info.dp"
+
+  Scenario: Test the sample eval using dp
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --seval 's.dp'"
+    Then I expect the named output to match the named output "seval_s.dp"
+
+  Scenario: Test the sample filter + eval using dp
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -i --sfilter 's.dp>10' --seval 's.dp'"
+    Then I expect the named output to match the named output "sfilter_seval_s.dp"
+
+  Scenario: Rewrite an info field
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf --rewrite rec.info[\'sample\']=\'XXXXX\'"
+    Then I expect the named output to match the named output "rewrite.info.sample"
+
+  Scenario: Test eval-once
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf --eval-once header.meta[\'GATKCommandLine\']"
+    Then I expect the named output to match the named output "eval_once"
+
+  Scenario: Test JSON output with header meta data
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf --template template/vcf2json_full_header.erb"
+    Then I expect the named output to match the named output "vcf2json_full_header"
+
+  Scenario: Test JSON output with header meta data and query samples
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf --template template/vcf2json_use_meta.erb"
+    Then I expect the named output to match the named output "vcf2json_use_meta"
+
+  Scenario: Test deadlock on failed filter with threads
+    Given I have input file(s) named "test/data/input/multisample.vcf"
+    When I execute "./bin/bio-vcf -q --timeout 4 --num-threads 4 --thread-lines 4 --filter 't.info.dp>2'"
+    Then I expect an error and the named output to match the named output "thread4_4_failed_filter" in under 30 seconds
+
+

data/features/cli.feature

@@ -11,12 +11,12 @@ Feature: Command-line interface (CLI)
   Scenario: Test the info filter using dp and threads
     Given I have input file(s) named "test/data/input/multisample.vcf"
     When I execute "./bin/bio-vcf -i --num-threads 4 --filter 'r.info.dp>2'"
-    Then I expect the named output to match the named output "thread4"
+    Then I expect the named output to match the named output "thread4" in under 30 seconds
 
   Scenario: Test the info filter using dp and threads with lines
     Given I have input file(s) named "test/data/input/multisample.vcf"
     When I execute "./bin/bio-vcf -i --num-threads 4 --thread-lines 4 --filter 'r.info.dp>2'"
-    Then I expect the named output to match the named output "thread4_4"
+    Then I expect the named output to match the named output "thread4_4" in under 30 seconds
 
   Scenario: Test the sample filter using dp
     Given I have input file(s) named "test/data/input/multisample.vcf"
@@ -65,7 +65,7 @@ Feature: Command-line interface (CLI)
 
   Scenario: Test deadlock on failed filter with threads
     Given I have input file(s) named "test/data/input/multisample.vcf"
-    When I execute "./bin/bio-vcf -q --timeout 2 --num-threads 4 --thread-lines 4 --filter 't.info.dp>2'"
+    When I execute "./bin/bio-vcf -q --timeout 4 --num-threads 4 --thread-lines 4 --filter 't.info.dp>2'"
     Then I expect an error and the named output to match the named output "thread4_4_failed_filter" in under 30 seconds
 
 

data/features/filter.feature

@@ -0,0 +1,12 @@
+@filter
+Feature: Adding filters
+
+  bio-vcf can add soft filters. Rather than removing failing items we can
+  inject filter state into the FILTER field. To add state such as PASS or
+  LowDepth simply use a filter and the --set-filter switch. If a filter already
+  has state the new one is appended with a semi-colon.
+  
+  Scenario: Test the info filter using dp and threads
+    Given I have input file(s) named "test/data/input/somaticsniper.vcf"
+    When I execute "./bin/bio-vcf --add-filter PASS --filter 'r.normal.dp>5 and r.tumor.dp>7'"
+    Then I expect the named output to match the named output "pass1"

data/features/filter.feature~

@@ -0,0 +1,35 @@
+@meta
+Feature: Parsing VCF meta information from the header
+
+  Take a header and parse that information as defined by the VCF standard.
+  
+  Scenario: When parsing a header line
+
+    Given the VCF header lines
+    """
+##fileformat=VCFv4.1
+##fileDate=20140121
+##phasing=none
+##reference=file:///data/GENOMES/human_GATK_GRCh37/GRCh37_gatk.fasta
+##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
+##FORMAT=<ID=DP,Number=1,Type=Integer,Description="Total read depth">
+##FORMAT=<ID=DP4,Number=4,Type=Integer,Description="# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases">
+##INFO=<ID=PM,Number=0,Type=Flag,Description="Variant is Precious(Clinical,Pubmed Cited)">
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	NORMAL	TUMOR
+    """
+    When I parse the VCF header
+    Then I expect vcf.columns to be [CHROM','POS','ID','REF','ALT','QUAL','FILTER','INFO','FORMAT','NORMAL','TUMOR']
+    And I expect vcf.fileformat to be "VCFv4.1"
+    And I expect vcf.fileDate to be "20140121"
+    And I expect vcf.field['fileDate'] to be "20140121"
+    And I expect vcf.phasing to be "none"
+    And I expect vcf.reference to be "file:///data/GENOMES/human_GATK_GRCh37/GRCh37_gatk.fasta"
+    And I expect vcf.format['GT'] to be {"ID"=>"GT", "Number"=>"1", "Type"=>"String", "Description"=>"Genotype"}
+    And I expect vcf.format['DP'] to be {"ID"=>"DP", "Number"=>"1", "Type"=>"Integer", "Description"=>"Total read depth"}
+    And I expect vcf.format['DP4'] to be {"ID"=>"DP4", "Number"=>"4", "Type"=>"Integer", "Description"=>"# high-quality ref-forward bases, ref-reverse, alt-forward and alt-reverse bases"}
+    And I expect vcf.info['PM'] to be {"ID"=>"PM", "Number"=>"0", "Type"=>"Flag", "Description"=>"Variant is Precious(Clinical,Pubmed Cited)"}'
+    And I expect vcf.meta to contain all header meta information
+
+  Scenario: When parsing the header of somatic_sniper.vcf
+
+    Do something

data/features/somaticsniper.feature

@@ -46,6 +46,8 @@ Feature: VCF for Somatic Sniper
     And I expect rec.tumor.amq.to_ary to be [37,37]
     And I expect rec.tumor.mq to be 37
     And I expect rec.tumor.ss to be 2
+    And I expect rec.tumor.ssc to be 33
+    And I expect rec.normal.ssc to be nil
     # The following are additional functions
     And I expect rec.call_diff to be [-4,2,-2,0]
     And I expect rec.call_nuc to be "C"

data/features/step_definitions/cli-feature.rb

@@ -11,6 +11,11 @@ Then(/^I expect the named output to match the named output "(.*?)"$/) do |arg1|
   RegressionTest::CliExec::exec(@cmd,arg1,ignore: '(##BioVcf|date|"version":)').should be_truthy
 end
 
+Then(/^I expect the named output to match the named output "([^"]*)" in under (\d+) seconds$/) do |arg1, arg2|
+  RegressionTest::CliExec::exec(@cmd,arg1,ignore: '(##BioVcf|date|"version":)',timeout: arg2.to_i).should be_truthy
+end
+
+
 Then(/^I expect an error and the named output to match the named output "(.*?)" in under (\d+) seconds$/) do |arg1,arg2|
   RegressionTest::CliExec::exec(@cmd,arg1,ignore: '(FATAL|Waiting|from|vcf|Options|Final pid)',should_fail: true,timeout:arg2.to_i).should be_truthy
 end

data/features/step_definitions/somaticsniper.rb

@@ -99,6 +99,14 @@ Then(/^I expect rec\.tumor\.ss to be (\d+)$/) do |arg1|
 end
 
 
+Then(/^I expect rec\.tumor\.ssc to be (\d+)$/) do |arg1|
+  expect(@rec.tumor.ssc).to be 33
+end
+
+Then(/^I expect rec\.normal\.ssc to be nil$/) do
+  expect(@rec.normal.ssc).to be nil
+end
+
 Then(/^I expect rec.call_diff to be \[(\-\d+),(\d+),(\-\d+),(\d+)\]$/) do |arg1, arg2, arg3, arg4|
   expect(@rec.call_diff).to eq [arg1.to_i,arg2.to_i,arg3.to_i,arg4.to_i]
 end

data/lib/bio-vcf/pcows.rb

@@ -4,25 +4,31 @@ require 'tempfile'
 
 class PCOWS
 
-  RUNNINGEXT = 'part'
+  RUNNINGEXT = 'part' # file extension
   
-  def initialize(num_threads,name=File.basename(__FILE__),timeout=180)
+  def initialize(num_threads,chunk_size,name=File.basename(__FILE__),timeout=180,quiet=false,debug=false)
     num_threads = cpu_count() if not num_threads # FIXME: set to cpu_num by default
     # $stderr.print "Using ",num_threads,"threads \n"
     @num_threads = num_threads
+    @chunk_size = chunk_size
     @pid_list = []
     @name = name
     @timeout = timeout
+    @quiet = quiet
+    @debug = debug
+    if @debug
+      $stderr.print "PCOWS running in DEBUG MODE\n"
+    end
     if multi_threaded
       @tmpdir =  Dir::mktmpdir(@name+'_')
     end
     @last_output = 0 # counter
-    @output_locked = nil
+    @output_locked = false
   end
 
-  # Feed the worker func and state to COWS. Note that func is a lambda
-  # closure so it can pick up surrounding scope at invocation in
-  # addition to the data captured in 'state'.
+  # Feed the worker 'func and state' to COWS. Note that func is a
+  # lambda closure so it can pick up surrounding scope at invocation
+  # in addition to the data captured in 'state'.
   
   def submit_worker(func,state)
     pid = nil
@@ -36,17 +42,30 @@ class PCOWS
         func.call(state).each { | line | print line }
         STDOUT.flush
         STDOUT.close
+        # sleep 0.1
+        # f.flush
+        # f.close
+        # sleep 0.2  # interval to make sure we are done writing,
+                   # otherwise there may be misses at the end of a
+                   # block (maybe the f.close fixed it)
+
         FileUtils::mv(tempfn,fn)
-        exit 0
+        exit(0)
       end
+      Process.detach(pid)
     else
-      # ---- Call in main process and output immediately
+      # ---- Single threaded: call in main process and output immediately
       func.call(state).each { | line | print line }
     end
     @pid_list << [ pid,count,fn ]
     return true
   end
 
+  def submit_final_worker(func,state)
+    @final_worker = true
+    submit_worker(func,state)
+  end
+  
   # Make sure no more than num_threads are running at the same time -
   # this is achieved by checking the PID table and the running files
   # in the tmpdir
@@ -54,7 +73,7 @@ class PCOWS
   def wait_for_worker_slot()
     return if single_threaded
     Timeout.timeout(@timeout) do
-    
+      printed_timeout_message = false
       while true
         # ---- count running pids
         running = @pid_list.reduce(0) do | sum, info |
@@ -66,9 +85,11 @@ class PCOWS
           end
         end
         return if running < @num_threads
-        $stderr.print "Waiting for slot (timeout=#{@timeout})\n"
-        sleep 0.1
-        
+        if not printed_timeout_message
+          $stderr.print "Waiting for slot (timeout=#{@timeout})\n" if not @quiet
+          printed_timeout_message = true
+        end
+        sleep 0.1        
       end
     end
   end
@@ -81,8 +102,7 @@ class PCOWS
   #
   #      In this implementation type==:by_line will call func for
   #      each line. Otherwise it is called once with the filename.
-
-  def process_output(func=nil,type = :by_line, blocking=false)
+  def process_output(func=nil,type=:by_line, blocking=false)
     return if single_threaded
     output = lambda { |fn|
       if type == :by_line
@@ -92,53 +112,87 @@ class PCOWS
       else
         func.call(fn)
       end
-      File.unlink(fn)
     }
     if @output_locked
+      # ---- is the other thread still running?
       (pid,count,fn) = @output_locked
-      return if File.exist?(fn)  # still processing
-      # on to the next one
-      @last_output += 1
-      @output_locked = nil
+      $stderr.print "Checking for output_lock on existing #{fn}\n" if not @quiet
+      return if File.exist?(fn)  # continue because thread still processing
+      # Now we should remove the .keep file
+      if not @debug
+        sleep 0.1 # give it a little time
+        keep = fn+'.keep'
+        if File.exist?(keep)
+          $stderr.print "Removing #{keep}\n" if not @quiet
+          File.unlink(keep)
+        end
+      end
+      @last_output += 1          # get next one in line
+      @output_locked = false
     end
+    # Still processing
     if info = @pid_list[@last_output]
       (pid,count,fn) = info
-      $stderr.print "Processing #{fn}\n"
+      $stderr.print "Testing for output file ",[info],"\n" if @debug
       if File.exist?(fn)
         # Yes! We have the next output, create outputter
+        @output_locked = info
+        $stderr.print "Set lock on ",[info],"\n" if not @quiet
         if not blocking
+          $stderr.print "Processing output file #{fn} (non-blocking)\n" if not @quiet
           pid = fork do
             output.call(fn)
+            FileUtils::mv(fn,fn+'.keep')
+            # if not @debug
+            #   $stderr.print "Removing #{fn}\n" if not @quiet
+            #   File.unlink(fn)
+            # else
+            #   FileUtils::mv(fn,fn+'.keep')
+            # end
+
             exit(0)
           end
-          @output_locked = info
+          Process.detach(pid)
         else
+          $stderr.print "Processing output file #{fn} (blocking)\n" if not @quiet
           output.call(fn)
+          if not @debug
+            $stderr.print "Removing #{fn}\n" if not @quiet
+            File.unlink(fn)
+          else
+            FileUtils::mv(fn,fn+'.keep')
+          end
         end
+      else
+        sleep 0.2
       end
     end
   end
 
+  # Wait for a worker slot to appear. When working the pid is writing
+  # a file with extension .part(ial). After completion the file is
+  # renamed without .part and a slot is free.
   def wait_for_worker(info)
     (pid,count,fn) = info
     if pid_or_file_running?(pid,fn)
-      $stderr.print "Waiting up to #{@timeout} seconds for pid=#{pid} to complete\n"
+      $stderr.print "Waiting up to #{@timeout} seconds for pid=#{pid} to complete #{fn}\n" if not @quiet
       begin
         Timeout.timeout(@timeout) do
           while not File.exist?(fn)  # wait for the result to appear
             sleep 0.2
+            return if not pid_or_file_running?(pid,fn) # worker is gone
           end
         end
-        # Thread file should have gone:
-        raise "FATAL: child process appears to have crashed #{fn}" if not File.exist?(fn)
-        $stderr.print "OK pid=#{pid}, processing #{fn}\n"
+        # Partial file should have been renamed:
+        raise "FATAL: child process #{pid} appears to have crashed #{fn}" if not File.exist?(fn)
+        $stderr.print "OK pid=#{pid}, processing output of #{fn}\n" if not @quiet
       rescue Timeout::Error
-        if pid_running?(pid)
-          # Kill it to speed up exit
-          Process.kill 9, pid
-          Process.wait pid
-        end
-        $stderr.print "FATAL: child process killed because it stopped responding, pid = #{pid}\n"
+        # Kill it to speed up exit
+        Process.kill 9, pid
+        Process.wait pid
+        $stderr.print "FATAL: child process killed because it stopped responding, pid = #{pid}, fn = #{fn}, count = #{count}\n"
+        $stderr.print "Bailing out"
+        raise
       end
     end
   end
@@ -155,15 +209,41 @@ class PCOWS
 
   def process_remaining_output()
     return if single_threaded
+    $stderr.print "Processing remaining output...\n" if not @quiet
     while @output_locked
       sleep 0.2
-      process_output()
+      process_output() # keep trying
     end
     @pid_list.each do |info|
-      process_output(nil,:by_line,true)
+      (pid,count,fn) = info
+      while pid_or_file_running?(pid,fn) or File.exist?(fn)
+        $stderr.print "Trying: ",[info],"\n" if not @quiet
+        process_output(nil,:by_line,true)
+        sleep 0.2
+      end
+    end
+    cleanup_tmpdir()
+  end
+
+  def cleanup()
+    @pid_list.each do |info|
+      (pid,count,fn) = info
+      if pid_running?(pid)
+        $stderr.print "Killing child ",[info],"\n"
+        begin
+          Process.kill 9, pid
+          Process.wait pid
+        rescue Errno::ENOENT
+          $stdout.puts "INFO: #{pidfile} did not exist: Errno::ENOENT" if not @quiet
+        rescue Errno::ESRCH
+          $stdout.puts "INFO: The process #{opid} did not exist: Errno::ESRCH" if not @quiet
+        end
+      end
+      File.unlink(fn) if File.exist?(fn)
+      tempfn = fn+'.'+RUNNINGEXT
+      File.unlink(tempfn) if File.exist?(tempfn)
     end
-    # final cleanup
-    Dir.unlink(@tmpdir) if @tmpdir
+    cleanup_tmpdir()
   end
   
   private
@@ -203,8 +283,15 @@ class PCOWS
       # Count on MAC
       return Integer `sysctl -n hw.ncpu 2>/dev/null`
     end
-    $stderr.print "Could not determine number of CPUs"
+    $stderr.print "Could not determine number of CPUs" if not @quiet
     1
   end
 
+  def cleanup_tmpdir
+    if not @debug
+      $stderr.print "Removing dir #{@tmpdir}\n" if not @quiet
+      Dir.unlink(@tmpdir) if @tmpdir
+    end
+  end
+  
 end