macroape 3.3.3 → 3.3.4

data/.gitignore

@@ -15,3 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
+benchmark/

data/Rakefile.rb

@@ -1,34 +1,19 @@
 #!/usr/bin/env rake
 require "bundler/gem_tasks"
 require 'rspec/core/rake_task'
+require 'rake/testtask'
 
 namespace :spec do
-  task :find_threshold do
-    system("ruby -I ./test test/find_threshold_test.rb")
-  end
-  task :find_pvalue do
-    system("ruby -I ./test test/find_pvalue_test.rb")
-  end
-  task :eval_similarity do
-    system("ruby -I ./test test/eval_similarity_test.rb")
-  end
-  task :eval_alignment_similarity do
-    system("ruby -I ./test test/eval_alignment_similarity_test.rb")
-  end
-  task :preprocess_collection do
-    system("ruby -I ./test test/preprocess_collection_test.rb")
-  end
-  task :scan_collection do
-    system("ruby -I ./test test/scan_collection_test.rb")
-  end
-  task :tests => [:find_threshold, :find_pvalue, :eval_similarity,
-                :eval_alignment_similarity, :scan_collection, :preprocess_collection]
-  
+  Rake::TestTask.new do |t|
+    t.libs << "test"
+    t.test_files = FileList['test/*_test.rb']
+    t.verbose = true
+  end  
   RSpec::Core::RakeTask.new
 end
 
 desc 'Test all functionality of gem executables'
-task :spec => ['spec:tests', 'spec:spec']
+task :spec => ['spec:test', 'spec:spec']
 
 namespace :benchmark do
   task :run do

data/TODO.txt

@@ -1,16 +1,13 @@
 Absolutely necessary:
-  Repair obtaining matrix not only from files but from stdin
-  Make it available to load collections in preprocess_collection from single file (and from stdin of certainly)
-  Make it available to load PCM files with (it should be first preprocessed to PWMs in a standardized way) -- may be it's better to use pipeline
+  (already work in preprocess_colleсtion and align_motifs) Make it available to load PCM files with (it should be first preprocessed to PWMs in a standardized way) -- may be it's better to use pipeline 
 
 Specs and tests:
   create spec on use of MaxHashSize, MaxHashSizeDouble
   create spec for testing case when {real_pvalue == 0, threshold == best_score + 1}
-  create test for getting PWMs from stdin
   create test for nonuniform word-wise background([1,1,1,1]) and for different backgrounds
 
-Ideas to inctrease perfomance:
-  - Add shifting matrix elements to zero after discreeting - in such case worst suffix is zero at all positions
+Ideas to increase perfomance:
+  - Add shifting matrix elements to zero after discreeting - in such case worst suffix is zero at all positions (??! it can significantly obscure code because thresholds will be changed too, and I can't tell what is better: slight perfomance optimization or conciseness of code)
   - (?) Make rearrangment of rows by DIC decreasing in aligned pair of matrices before counting
   - Create JAVA extension for alignment_intersection methods in order to increase perfomance
   - Possibly algorithm shouldn't use hash but had two iterations: at first it determines possible hash scores for every length(if worst suffix is always zero, its flat space of scores at all pwm prefix lengths) of each pwm separately. And after that we can work with arrays which use such scores as indices via additional substructure
@@ -18,3 +15,7 @@ Ideas to inctrease perfomance:
 Usability issues:  
   review Collection class. Now its completely unuseful. May be it should be even in another gem (with blackjack and clustering)
 
+remove .stdin placeholder. Use tty? method instead
+
+use OptionParser (??? can OptionParser get stub ARGV ???)
+make options more uniform so that some of them were reusable(and the question: can I apply two option parsers consequently?)

data/bin/align_motifs

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'macroape/cli/align_motifs'
+Macroape::CLI::AlignMotifs.main(ARGV)

data/bin/eval_alignment

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
 
-require 'macroape/exec/eval_alignment.rb'
+require 'macroape/cli/eval_alignment'
+Macroape::CLI::EvalAlignment.main(ARGV)

data/bin/eval_similarity

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
 
-require 'macroape/exec/eval_similarity.rb'
+require 'macroape/cli/eval_similarity'
+Macroape::CLI::EvalSimilarity.main(ARGV)

data/bin/find_pvalue

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
 
-require 'macroape/exec/find_pvalue.rb'
+require 'macroape/cli/find_pvalue'
+Macroape::CLI::FindPValue.main(ARGV)

data/bin/find_threshold

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
 
-require 'macroape/exec/find_threshold.rb'
+require 'macroape/cli/find_threshold'
+Macroape::CLI::FindThreshold.main(ARGV)

data/bin/preprocess_collection

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
 
-require 'macroape/exec/preprocess_collection.rb'
+require 'macroape/cli/preprocess_collection'
+Macroape::CLI::PreprocessCollection.main(ARGV)

data/bin/scan_collection

@@ -1,3 +1,4 @@
 #!/usr/bin/env ruby
 
-require 'macroape/exec/scan_collection.rb'
+require 'macroape/cli/scan_collection'
+Macroape::CLI::ScanCollection.main(ARGV)

data/lib/macroape/aligned_pair_intersection.rb

@@ -1,6 +1,5 @@
 module Macroape
   class PWMCompareAligned
-    
     # unoptimized version of this and related methods
     def counts_for_two_matrices(threshold_first, threshold_second)
       # just not to call method each time
@@ -27,11 +26,11 @@ module Macroape
       scores = { 0 => {0 => 1} }
       length.times do |column|
         new_scores = recalc_score_hash(scores,
-                          @first.matrix[column], @second.matrix[column],
+                          first.matrix[column], second.matrix[column],
                           threshold_first - first.best_suffix(column + 1),
                           threshold_second - second.best_suffix(column + 1), &count_contribution_block)
         scores.replace(new_scores)
-        if defined?(MaxHashSizeDouble) && scores.inject(0){|sum,hsh|sum + hsh.size} > MaxHashSizeDouble
+        if max_hash_size && scores.inject(0){|sum,hsh|sum + hsh.size} > max_hash_size
           raise 'Hash overflow in Macroape::AlignedPairIntersection#counts_for_two_matrices_with_different_probabilities'
         end
       end

data/lib/macroape/cli/align_motifs.rb

@@ -0,0 +1,49 @@
+require 'macroape'
+
+module Macroape
+  module CLI
+    module AlignMotifs
+    
+      def self.main(argv)
+        help_string = %q{
+        Usage:
+          ruby align_motifs pwm1_file pwm2_file pwm3_file
+          ruby align_motifs pcm1_file pcm2_file pcm3_file --pcm
+        Output:
+          pwm_1_file  shift_1  orientation_1
+          pwm_2_file  shift_2  orientation_2
+          pwm_3_file  shift_3  orientation_3
+        }
+
+        if argv.empty? || ['-h', '--h', '-help', '--help'].any?{|help_option| argv.include?(help_option)}
+          STDERR.puts help_string
+          exit
+        end
+
+
+        data_model = argv.delete('--pcm') ? Bioinform::PCM : Bioinform::PWM
+        leader = argv.shift
+        background = [1,1,1,1]
+        discretization = 10
+        pvalue = 0.0005
+        
+        shifts = {leader => [0,:direct]}
+        pwm_first = data_model.new(File.read(leader)).to_pwm.background!(background).discrete!(discretization)
+        argv.each do |motif_name|
+          pwm_second = data_model.new(File.read(motif_name)).to_pwm.background!(background).discrete!(discretization)
+          cmp = Macroape::PWMCompare.new(pwm_first, pwm_second)
+          info = cmp.jaccard_by_pvalue(pvalue)
+          shifts[motif_name] = [info[:shift], info[:orientation]]
+        end
+        
+        shifts.each do |motif_name, (shift,orientation)|
+          puts "#{motif_name}\t#{shift}\t#{orientation}"
+        end
+
+      rescue => err
+        STDERR.puts "\n#{err}\n#{err.backtrace.first(5).join("\n")}\n\nUse -help option for help\n"
+      end
+      
+    end
+  end
+end

data/lib/macroape/cli/eval_alignment.rb

@@ -0,0 +1,124 @@
+require 'macroape'
+
+module Macroape
+  module CLI
+    module EvalAlignment
+    
+      def self.main(argv)
+        help_string = %q{
+        Command-line format:
+        ruby eval_alignment.rb <1st matrix pat-file> <2nd matrix pat-file> <shift> <orientation(direct/revcomp)> [options]
+        type <1st matrix pat-file> <2nd matrix pat-file> | ruby eval_alignment.rb .stdin .stdin <shift> <orientation(direct/revcomp)> [options]
+             or in linux
+        cat <1st matrix pat-file> <2nd matrix pat-file> | ruby eval_alignment.rb .stdin .stdin <shift> <orientation(direct/revcomp)> [options]
+
+        Options:
+          [-p <P-value>]
+          [-d <discretization level>]
+          [-b <background probabilities, ACGT - 4 numbers, space-delimited, sum should be equal to 1>]
+
+        Output format:
+          <jaccard similarity coefficient>
+          <number of words recognized by both 1st and 2nd matrices | probability to draw a word recognized by both 1st and 2nd matrices> <length of the given alignment>
+          <aligned 1st matrix>
+          <aligned 2nd matrix>
+          <shift> <orientation>
+
+        Examples:
+          ruby eval_alignment.rb motifs/KLF4_f2.pat motifs/SP1_f1.pat -1 direct -p 0.0005 -d 100 -b 0.4 0.3 0.2 0.1
+             or on windows
+          type motifs/SP1.pat | ruby eval_alignment.rb motifs/KLF4.pat .stdin 0 revcomp -p 0.0005 -d 100 -b 0.4 0.3 0.2 0.1
+             or in linux
+          cat motifs/KLF4.pat motifs/SP1.pat | ruby eval_alignment.rb .stdin .stdin 3 direct -p 0.0005 -d 100 -b 0.4 0.3 0.2 0.1
+        }
+
+        if argv.empty? || ['-h', '--h', '-help', '--help'].any?{|help_option| argv.include?(help_option)}
+          STDERR.puts help_string
+          exit
+        end
+
+        pvalue = 0.0005
+        discretization = 10
+
+        first_background = [1,1,1,1]
+        second_background = [1,1,1,1]
+        max_hash_size = 1000000
+        max_pair_hash_size = 1000
+
+
+        first_file = argv.shift
+        second_file = argv.shift
+
+        shift = argv.shift
+        orientation = argv.shift
+
+        raise "You'd specify two input sources (each is filename or .stdin)" unless first_file and second_file
+        raise 'You\'d specify shift' unless shift
+        raise 'You\'d specify orientation' unless orientation
+
+        shift = shift.to_i
+        orientation = orientation.to_sym
+
+        case orientation
+          when :direct
+            reverse = false
+          when :revcomp
+            reverse = true
+          else
+            raise 'Unknown orientation(direct/revcomp)'
+        end
+
+
+        until argv.empty?
+          case argv.shift
+            when '-p'
+              pvalue = argv.shift.to_f
+            when '-d'
+              discretization = argv.shift.to_f
+            when '-m'
+              max_hash_size = argv.shift.to_i
+            when '-md'
+              max_pair_hash_size = argv.shift.to_i
+            when '-b'
+              second_background = first_background = argv.shift(4).map(&:to_f)
+            when '-b1'
+              first_background = argv.shift(4).map(&:to_f)
+            when '-b2'
+              second_background = argv.shift(4).map(&:to_f)
+          end
+        end
+        raise 'background should be symmetric: p(A)=p(T) and p(G) = p(C)' unless first_background == first_background.reverse
+        raise 'background should be symmetric: p(A)=p(T) and p(G) = p(C)' unless second_background == second_background.reverse
+
+        parser = Bioinform::StringParser.new($stdin.read)  if first_file == '.stdin' || second_file == '.stdin'
+      
+        if first_file == '.stdin'
+          pwm_first = Bioinform::PWM.new( parser.parse )
+        else
+          raise "Error! File #{first_file} don't exist" unless File.exist?(first_file)
+          pwm_first = Bioinform::PWM.new(File.read(first_file))
+        end
+
+        if second_file == '.stdin'
+          pwm_second = Bioinform::PWM.new( parser.parse )
+        else
+          raise "Error! File #{second_file} don't exist" unless File.exist?(second_file)
+          pwm_second = Bioinform::PWM.new(File.read(second_file))
+        end
+        
+        pwm_first.background!(first_background).max_hash_size!(max_hash_size).discrete!(discretization)
+        pwm_second.background!(second_background).max_hash_size!(max_hash_size).discrete!(discretization)
+
+        cmp = Macroape::PWMCompareAligned.new(pwm_first, pwm_second, shift, orientation).max_hash_size(max_pair_hash_size)
+
+        info = cmp.alignment_infos.merge( cmp.jaccard_by_pvalue(pvalue) )
+
+        puts "#{info[:similarity]}\n#{info[:recognized_by_both]}\t#{info[:alignment_length]}\n#{info[:text]}\n#{info[:shift]}\t#{info[:orientation]}"
+
+      rescue => err
+        STDERR.puts "\n#{err}\n#{err.backtrace.first(5).join("\n")}\n\nUse -help option for help\n"
+      end
+
+    end
+  end
+end

data/lib/macroape/cli/eval_similarity.rb

@@ -0,0 +1,107 @@
+require 'macroape'
+
+module Macroape
+  module CLI
+    module EvalSimilarity
+    
+      def self.main(argv)
+        help_string = %q{
+        Command-line format:
+        ruby eval_similarity.rb <1st matrix pat-file> <2nd matrix pat-file> [options]
+             or on windows
+        type <1st matrix pat-file> <2nd matrix pat-file> | ruby eval_similarity.rb .stdin .stdin [options]
+             or in linux
+        cat <1st matrix pat-file> <2nd matrix pat-file> | ruby eval_similarity.rb .stdin .stdin [options]
+
+        Options:
+          [-p <P-value>]
+          [-d <discretization level>]
+          [-b <background probabilities, ACGT - 4 numbers, space-delimited, sum should be equal to 1>]
+
+        Output has format:
+          <jaccard similarity coefficient>
+          <number of words recognized by both 1st and 2nd matrices | probability to draw a word recognized by both 1st and 2nd matrices> <length of the optimal alignment>
+          <optimal alignment, the 1st matrix>
+          <optimal alignment, the 2nd matrix>
+          <shift> <orientation>
+
+        Examples:
+          ruby eval_similarity.rb motifs/KLF4.pat motifs/SP1.pat -p 0.0005 -d 100 -b 0.4 0.3 0.2 0.1
+             or on windows
+          type motifs/SP1.pat | ruby eval_similarity.rb motifs/KLF4.pat .stdin -p 0.0005 -d 100 -b 0.4 0.3 0.2 0.1
+             or in linux
+          cat motifs/KLF4.pat motifs/SP1.pat | ruby eval_similarity.rb .stdin .stdin -p 0.0005 -d 100 -b 0.4 0.3 0.2 0.1
+        }
+
+        if argv.empty? || ['-h', '--h', '-help', '--help'].any?{|help_option| argv.include?(help_option)}
+          STDERR.puts help_string
+          exit
+        end
+
+        pvalue = 0.0005
+        discretization = 10
+
+        first_background = [1,1,1,1]
+        second_background = [1,1,1,1]
+
+        max_hash_size = 1000000
+        max_pair_hash_size = 1000
+
+      
+        first_file = argv.shift
+        second_file = argv.shift
+        raise "You'd specify two input sources (each is filename or .stdin)" unless first_file and second_file
+
+        until argv.empty?
+          case argv.shift
+            when '-p'
+              pvalue = argv.shift.to_f
+            when '-d'
+              discretization = argv.shift.to_f
+            when '-m'
+              max_hash_size = argv.shift.to_i
+            when '-md'
+              max_pair_hash_size = argv.shift.to_i
+            when '-b'
+              second_background = first_background = argv.shift(4).map(&:to_f)
+            when '-b1'
+              first_background = argv.shift(4).map(&:to_f)
+            when '-b2'
+              second_background = argv.shift(4).map(&:to_f)
+          end
+        end
+        raise 'background should be symmetric: p(A)=p(T) and p(G) = p(C)' unless first_background == first_background.reverse
+        raise 'background should be symmetric: p(A)=p(T) and p(G) = p(C)' unless second_background == second_background.reverse
+
+        parser = Bioinform::StringParser.new($stdin.read)  if first_file == '.stdin' || second_file == '.stdin'
+        
+        if first_file == '.stdin'
+          pwm_first = Bioinform::PWM.new( parser.parse )
+        else
+          raise "Error! File #{first_file} don't exist" unless File.exist?(first_file)
+          pwm_first = Bioinform::PWM.new(File.read(first_file))
+        end
+
+        if second_file == '.stdin'
+          pwm_second = Bioinform::PWM.new( parser.parse )
+        else
+          raise "Error! File #{second_file} don't exist" unless File.exist?(second_file)
+          pwm_second = Bioinform::PWM.new(File.read(second_file))
+        end
+        
+        pwm_first.background!(first_background).max_hash_size!(max_hash_size).discrete!(discretization)
+        pwm_second.background!(second_background).max_hash_size!(max_hash_size).discrete!(discretization)
+
+        cmp = Macroape::PWMCompare.new(pwm_first, pwm_second).max_hash_size(max_pair_hash_size)
+
+        info = cmp.jaccard_by_pvalue(pvalue)
+
+        puts "#{info[:similarity]}\n#{info[:recognized_by_both]}\t#{info[:alignment_length]}\n#{info[:text]}\n#{info[:shift]}\t#{info[:orientation]}"
+
+      rescue => err
+        STDERR.puts "\n#{err}\n#{err.backtrace.first(5).join("\n")}\n\nUse -help option for help\n"      
+      end
+      
+    end
+  end
+end

data/lib/macroape/cli/find_pvalue.rb

@@ -0,0 +1,89 @@
+require 'macroape'
+
+module Macroape
+  module CLI
+    module FindPValue
+    
+      def self.main(argv)
+        help_string = %q{
+          Command-line format:
+          ruby find_pvalue.rb <pat-file> <threshold list> [options]
+                    or in linux
+          cat <pat-file> | ruby find_pvalue.rb .stdin <threshold> [options]
+            or on windows
+          type <pat-file> | ruby find_pvalue.rb .stdin <threshold> [options]
+
+          Options:
+            [-d <discretization level>]
+            [-b <background probabilities, ACGT - 4 numbers, space-delimited, sum should be equal to 1>]
+
+          Output format:
+            threshold_1 count_1  pvalue_1
+            threshold_2 count_2  pvalue_2
+            threshold_3 count_3  pvalue_3
+          The results are printed out in the same order as in the given threshold list.
+
+          Examples:
+            ruby find_pvalue.rb motifs/KLF4.pat 7.32 -d 1000 -b 0.2 0.3 0.2 0.3
+                     or on windows
+            type motifs/KLF4.pat | ruby find_pvalue.rb .stdin 7.32 4.31 5.42
+                     or in linux
+            cat motifs/KLF4.pat | ruby find_pvalue.rb .stdin 7.32 4.31 5.42
+        }
+      
+        if argv.empty? || ['-h', '--h', '-help', '--help'].any?{|help_option| argv.include?(help_option)}
+          STDERR.puts help_string
+          exit
+        end
+
+        discretization = 10000
+        background = [1,1,1,1]
+        thresholds = []
+        max_hash_size = 1000000
+
+        filename = argv.shift
+
+        loop do
+          begin
+            Float(argv.first)
+            thresholds << argv.shift.to_f
+          rescue
+            raise StopIteration
+          end
+        end
+
+        raise "No input. You'd specify input source: filename or .stdin" unless filename
+        raise 'You should specify at least one threshold' if thresholds.empty?
+
+        until argv.empty?
+          case argv.shift
+            when '-b'
+              background = argv.shift(4).map(&:to_f)
+            when '-d'
+              discretization = argv.shift.to_f
+            when '-m'
+              max_hash_size = argv.shift.to_i
+          end
+        end
+
+        
+        if filename == '.stdin'
+          pwm = Bioinform::PWM.new( $stdin.read )
+        else
+          raise "Error! File #{filename} doesn't exist" unless File.exist?(filename)
+          pwm = Bioinform::PWM.new( File.read(filename) )
+        end
+        pwm.background!(background).max_hash_size!(max_hash_size).discrete!(discretization)
+
+        counts = pwm.counts_by_thresholds(* thresholds.map{|count| count * discretization})
+        pvalues = counts.map{|count| count.to_f / pwm.vocabulary_volume}
+        pvalues.zip(thresholds,counts).each{|pvalue,threshold,count|
+          puts "#{threshold}\t#{count}\t#{pvalue}"
+        }
+      rescue => err
+        STDERR.puts "\n#{err}\n#{err.backtrace.first(5).join("\n")}\n\nUse -help option for help\n"
+      end
+      
+    end
+  end
+end

data/lib/macroape/cli/find_threshold.rb

@@ -0,0 +1,84 @@
+require 'macroape'
+
+module Macroape
+  module CLI
+    module FindThreshold
+    
+      def self.main(argv)
+        help_string = %q{
+          Command-line format::
+          ruby find_threshold.rb <pat-file> [options]
+                    or in linux
+          cat <pat-file> | ruby find_threshold.rb .stdin [options]
+                      or on windows
+          type <pat-file> | ruby find_threshold.rb .stdin [options]
+
+          Options:
+            [-p <list of P-values>]
+            [-d <discretization level>]
+            [-b <background probabilities, ACGT - 4 numbers, space-delimited, sum should be equal to 1>]
+
+          Output format:
+            requested_pvalue_1 threshold_1 achieved_pvalue_1
+            requested_pvalue_2 threshold_2 achieved_pvalue_2
+
+
+          Example:
+            ruby find_threshold.rb motifs/KLF4.pat -p 0.001 0.0001 0.0005 -d 1000 -b 0.4 0.3 0.2 0.1
+        }
+      
+        if argv.empty? || ['-h', '--h', '-help', '--help'].any?{|help_option| argv.include?(help_option)}
+          STDERR.puts help_string
+          exit
+        end
+        
+        background = [1,1,1,1]
+        default_pvalues = [0.0005]
+        discretization = 10000
+        max_hash_size = 1000000
+
+        filename = argv.shift
+        raise "No input. You'd specify input source: filename or .stdin" unless filename
+
+        pvalues = []
+        until argv.empty?
+          case argv.shift
+            when '-b'
+              background = argv.shift(4).map(&:to_f)
+            when '-m'
+              max_hash_size = argv.shift.to_i
+            when '-p'
+              loop do
+                begin
+                  Float(argv.first)
+                  pvalues << argv.shift.to_f
+                rescue
+                  raise StopIteration
+                end
+              end
+            when '-d'
+              discretization = argv.shift.to_f
+            end
+        end
+        pvalues = default_pvalues if pvalues.empty?
+
+        if filename == '.stdin'
+          pwm = Bioinform::PWM.new( $stdin.read )
+        else
+          raise "Error! File #{filename} doesn't exist" unless File.exist?(filename)
+          pwm = Bioinform::PWM.new( File.read(filename) )
+        end
+
+        pwm.background!(background).max_hash_size!(max_hash_size).discrete!(discretization)
+
+        pwm.thresholds(*pvalues) do |pvalue, threshold, real_pvalue|
+          puts "#{pvalue}\t#{threshold / discretization}\t#{real_pvalue}"
+        end
+        
+      rescue => err
+        STDERR.puts "\n#{err}\n#{err.backtrace.first(5).join("\n")}\n\nUse -help option for help\n"
+      end
+      
+    end
+  end
+end