macroape 3.2.2 → 3.3.0

data/Rakefile

@@ -1,7 +1,28 @@
-#!/usr/bin/env rake
-require "bundler/gem_tasks"
-
-desc 'Test all functionality of gem executables'
-task :test do
-  system("ruby test/macroape_test.rb")
-end
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+namespace :spec do
+  task :find_threshold do
+    system("ruby -I ./test test/find_threshold_test.rb")
+  end
+  task :find_pvalue do
+    system("ruby -I ./test test/find_pvalue_test.rb")
+  end
+  task :eval_similarity do
+    system("ruby -I ./test test/eval_similarity_test.rb")
+  end
+  task :eval_alignment_similarity do
+    system("ruby -I ./test test/eval_alignment_similarity_test.rb")
+  end
+  task :preprocess_collection do
+    system("ruby -I ./test test/preprocess_collection_test.rb")
+  end
+  task :scan_collection do
+    system("ruby -I ./test test/scan_collection_test.rb")
+  end
+  task :all => [:find_threshold, :find_pvalue, :eval_similarity,
+                :eval_alignment_similarity, :scan_collection, :preprocess_collection]
+end
+
+desc 'Test all functionality of gem executables'
+task :spec => ['spec:all']

data/lib/macroape.rb

@@ -1,26 +1,14 @@
-require 'macroape/version'
-require 'yaml'
-
-require 'macroape/extract_pwm'
-require 'macroape/support'
-require 'macroape/matrix_transformations'
-require 'macroape/matrix_information'
-require 'macroape/gauss_estimation'
-require 'macroape/threshold_by_pvalue'
-require 'macroape/single_matrix'
-require 'macroape/count_by_threshold'
-require 'macroape/matrix_on_background'
-
-require 'macroape/aligned_pair_transformations'
-require 'macroape/aligned_pair_metrics'
-require 'macroape/aligned_pair_intersection'
-require 'macroape/pwm_compare_aligned'
-
-require 'macroape/pair_transformations'
-require 'macroape/pair_metrics'
-require 'macroape/pwm_compare'
-require 'macroape/collection'
-
-module Macroape
-  # Your code goes here...
-end
+require 'macroape/version'
+
+require 'bioinform'
+require 'macroape/threshold_by_pvalue'
+require 'macroape/count_by_threshold'
+
+require 'macroape/aligned_pair_intersection'
+require 'macroape/pwm_compare_aligned'
+require 'macroape/pwm_compare'
+require 'macroape/collection'
+
+module Macroape
+  # Your code goes here...
+end

data/lib/macroape/aligned_pair_intersection.rb

@@ -1,9 +1,9 @@
-module PWMCompare
-  module AlignedPairIntersection
+module Macroape
+  class PWMCompareAligned
 
     def counts_for_two_matrices(threshold_first, threshold_second)
-      if first.probabilities == second.probabilities
-        if first.probabilities == [1,1,1,1]
+      if first.background == second.background
+        if first.background == [1,1,1,1]
           common_words_for_two_matrices(threshold_first, threshold_second)
         else
           counts_for_two_matrices_with_same_probabilities(threshold_first, threshold_second) 
@@ -18,8 +18,8 @@ module PWMCompare
       result_first = 0.0
       result_second = 0.0
       length.times do |column|    
-        ending_weight_first =  first.sum_of_probabilities ** (length - column - 1)
-        ending_weight_second = second.sum_of_probabilities ** (length - column - 1)
+        ending_weight_first =  first.background_sum ** (length - column - 1)
+        ending_weight_second = second.background_sum ** (length - column - 1)
         already_enough_first  = threshold_first  - first.worst_suffix[column + 1]
         already_enough_second = threshold_second - second.worst_suffix[column + 1]
         least_sufficient_first  = threshold_first  - first.best_suffix[column + 1]
@@ -33,23 +33,23 @@ module PWMCompare
               if new_score_first >= already_enough_first 
                 new_score_second = score_second + second.matrix[column][letter]
                 if new_score_second >= already_enough_second
-                  result_first += count[0] * first.probabilities[letter] * ending_weight_first
-                  result_second += count[1] * second.probabilities[letter] * ending_weight_second
+                  result_first += count[0] * first.background[letter] * ending_weight_first
+                  result_second += count[1] * second.background[letter] * ending_weight_second
                 elsif new_score_second >= least_sufficient_second 
-                  new_scores[new_score_first][new_score_second][0] += count[0] * first.probabilities[letter]
-                  new_scores[new_score_first][new_score_second][1] += count[1] * second.probabilities[letter]
+                  new_scores[new_score_first][new_score_second][0] += count[0] * first.background[letter]
+                  new_scores[new_score_first][new_score_second][1] += count[1] * second.background[letter]
                 end
               elsif new_score_first >= least_sufficient_first
                 new_score_second = score_second + second.matrix[column][letter]
                 if new_score_second >= least_sufficient_second
-                  new_scores[new_score_first][new_score_second][0] += count[0] * first.probabilities[letter]
-                  new_scores[new_score_first][new_score_second][1] += count[1] * second.probabilities[letter]
+                  new_scores[new_score_first][new_score_second][0] += count[0] * first.background[letter]
+                  new_scores[new_score_first][new_score_second][1] += count[1] * second.background[letter]
                 end
               end
             end
           end
         end
-        raise 'Hash overflow in PWMCompare::AlignedPairIntersection#counts_for_two_matrices_with_different_probabilities' if new_scores.inject(0){|sum,hsh|sum+hsh.size} > MaxHashSize
+        raise 'Hash overflow in Macroape::AlignedPairIntersection#counts_for_two_matrices_with_different_probabilities' if new_scores.inject(0){|sum,hsh|sum+hsh.size} > MaxHashSizeDouble
         scores = new_scores
       end
       [result_first, result_second]
@@ -58,9 +58,9 @@ module PWMCompare
     def counts_for_two_matrices_with_same_probabilities(threshold_first, threshold_second)
       scores = { 0 => {0 => 1} } # scores_on_first_pwm, scores_on_second_pwm --> count_on_first_probabilities, count_on_second_probabilities
       result = 0.0
-      probabilities = first.probabilities
+      background = first.background
       length.times do |column|    
-        ending_weight =  first.sum_of_probabilities ** (length - column - 1)
+        ending_weight =  first.background_sum ** (length - column - 1)
         already_enough_first  = threshold_first  - first.worst_suffix[column + 1]
         already_enough_second = threshold_second - second.worst_suffix[column + 1]
         least_sufficient_first  = threshold_first  - first.best_suffix[column + 1]
@@ -74,20 +74,20 @@ module PWMCompare
               if new_score_first >= already_enough_first 
                 new_score_second = score_second + second.matrix[column][letter]
                 if new_score_second >= already_enough_second
-                  result += count * probabilities[letter] * ending_weight
+                  result += count * background[letter] * ending_weight
                 elsif new_score_second >= least_sufficient_second 
-                  new_scores[new_score_first][new_score_second] += count * probabilities[letter]
+                  new_scores[new_score_first][new_score_second] += count * background[letter]
                 end
               elsif new_score_first >= least_sufficient_first
                 new_score_second = score_second + second.matrix[column][letter]
                 if new_score_second >= least_sufficient_second
-                  new_scores[new_score_first][new_score_second] += count * probabilities[letter]
+                  new_scores[new_score_first][new_score_second] += count * background[letter]
                 end
               end
             end
           end
         end
-        raise 'Hash overflow in PWMCompare::AlignedPairIntersection#counts_for_two_matrices_with_same_probabilities' if new_scores.inject(0){|sum,hsh|sum+hsh.size} > MaxHashSize
+        raise 'Hash overflow in Macroape::AlignedPairIntersection#counts_for_two_matrices_with_same_probabilities' if new_scores.inject(0){|sum,hsh|sum+hsh.size} > MaxHashSizeDouble
         scores = new_scores
       end
       [result, result]
@@ -97,7 +97,7 @@ module PWMCompare
     def common_words_for_two_matrices(threshold_first, threshold_second)
       scores = { 0 => {0 => 1} } # scores_on_first_pwm, scores_on_second_pwm --> count_on_first_probabilities, count_on_second_probabilities
       result = 0
-      length.times do |column|    
+      length.times do |column|
         ending_weight =  4 ** (length - column - 1)
         already_enough_first  = threshold_first  - first.worst_suffix[column + 1]
         already_enough_second = threshold_second - second.worst_suffix[column + 1]
@@ -107,13 +107,13 @@ module PWMCompare
         new_scores = Hash.new{|h,k| h[k]=Hash.new{|h2,k2| h2[k2]=0} }
         scores.each do |score_first, second_scores|
           second_scores.each do |score_second, count|
-            4.times do |letter|          
+            4.times do |letter|
               new_score_first = score_first + first.matrix[column][letter]
-              if new_score_first >= already_enough_first 
+              if new_score_first >= already_enough_first
                 new_score_second = score_second + second.matrix[column][letter]
                 if new_score_second >= already_enough_second
                   result += count * ending_weight
-                elsif new_score_second >= least_sufficient_second 
+                elsif new_score_second >= least_sufficient_second
                   new_scores[new_score_first][new_score_second] += count
                 end
               elsif new_score_first >= least_sufficient_first
@@ -126,7 +126,7 @@ module PWMCompare
           end
         end
         
-        raise 'Hash overflow in PWMCompare::AlignedPairIntersection#common_words_for_two_matrices' if defined? MaxHashSize and new_scores.inject(0){|sum,hsh|sum+hsh.size} > MaxHashSize
+        raise 'Hash overflow in Macroape::AlignedPairIntersection#common_words_for_two_matrices' if defined? MaxHashSizeDouble and new_scores.inject(0){|sum,hsh|sum+hsh.size} > MaxHashSizeDouble
         scores = new_scores
       end
       [result, result]

data/lib/macroape/collection.rb

@@ -1,4 +1,4 @@
-module PWM
+module Macroape
   class Collection
     attr_reader :rough_discretization, :precise_discretization, :background, :pvalues, :pwms, :infos
     def initialize(rough_discretization, precise_discretization, background, pvalues)
@@ -7,7 +7,6 @@ module PWM
       @infos={}
     end
     def add_pwm(pwm,info)
-      #@pwms[pwm] = info
       @pwms[pwm.name] = pwm
       @infos[pwm.name] = info
     end

data/lib/macroape/count_by_threshold.rb

@@ -1,34 +1,16 @@
-module PWM
-  module CountByThreshold
+require 'macroape/threshold_by_pvalue'
+
+module Bioinform
+  class PWM
     def counts_by_thresholds(*thresholds)
-      scores = calculate_count_distribution_after_threshold(thresholds.min)
-=begin
-       thresholds.map{ |threshold|
-       #scores.select{|score,count| score >= threshold}.map{|score,count| count}.inject(0){|sum,val|sum+val}
-        scores.inject(0){|sum,(score,count)|  (score >= threshold) ? sum + count : sum}
+      scores = count_distribution_after_threshold(thresholds.min)
+      thresholds.map{ |threshold|
+        scores.inject(0.0){|sum,(score,count)|  (score >= threshold) ? sum + count : sum}
       }
-=end
-      s_thr= thresholds.map.with_index{|threshold,index|[threshold,index]}.sort_by{|threshold,index| threshold}
-      
-      cnt = 0
-      thr_cnts=[]
-      
-      scores.sort.reverse.each do |score,count|
-        while !s_thr.empty? and score < s_thr.last[0]
-          thr_cnts.push(cnt)
-          s_thr.pop
-        end
-        cnt += count
-      end
-      s_thr = thresholds.map.with_index{|threshold,index|[threshold,index]}.sort_by{|threshold,index| threshold}
-      while thr_cnts.size < s_thr.size
-        thr_cnts.push(cnt)
-      end
-      s_thr.reverse.zip(thr_cnts).sort_by{|(threshold,index), count| index}.map{|(threshold,index), count| count.to_f}
     end
 
     def pvalue_by_threshold(threshold)
-      counts_by_thresholds(threshold).first / number_of_words
+      counts_by_thresholds(threshold).first / vocabulary_volume
     end
   end
 end

data/lib/macroape/exec/eval_alignment.rb

@@ -24,7 +24,7 @@ Examples:
      or in linux
   cat motifs/KLF4.pat motifs/SP1.pat | ruby eval_alignment.rb .stdin .stdin 3 direct -p 0.0005 -d 100 -b 0.4 0.3 0.2 0.1
 }
-
+$:.unshift File.join(File.dirname(__FILE__),'./../../')
 require 'macroape'
 
 if ARGV.empty? or ARGV.include? '-h' or ARGV.include? '-help' or ARGV.include? '--help' or ARGV.include? '--h'
@@ -69,9 +69,9 @@ begin
       when '-d'
         discretization = ARGV.shift.to_f
       when '-m'
-        PWM::MaxHashSize = ARGV.shift.to_f
+        Macroape::MaxHashSizeSingle = ARGV.shift.to_f
       when '-md'
-        PWMCompare::MaxHashSize = ARGV.shift.to_f
+        Macroape::MaxHashSizeDouble = ARGV.shift.to_f
       when '-b'
         second_background = first_background = ARGV.shift(4).map(&:to_f)
       when '-b1'
@@ -84,35 +84,35 @@ begin
   raise 'background should be symmetric: p(A)=p(T) and p(G) = p(C)' unless second_background == second_background.reverse
 
   
-  PWM::MaxHashSize = 1000000 unless defined? PWM::MaxHashSize
-  PWMCompare::MaxHashSize = 1000 unless defined? PWMCompare::MaxHashSize
+  Macroape::MaxHashSizeSingle = 1000000 unless defined? Macroape::MaxHashSizeSingle
+  Macroape::MaxHashSizeDouble = 1000 unless defined? Macroape::MaxHashSizeDouble
   
-  if first_file == '.stdin' || second_file == '.stdin'
-    r_stream, w_stream = IO.pipe
-    STDIN.readlines.each{|line| w_stream.write(line)}
-    w_stream.close
-  end
+#  if first_file == '.stdin' || second_file == '.stdin'
+#    r_stream, w_stream = IO.pipe
+#    STDIN.readlines.each{|line| w_stream.write(line)}
+#    w_stream.close
+#  end
   
   if first_file == '.stdin'
-    r_stream, w_stream, extracted_pwm = extract_pwm(r_stream, w_stream)
-    pwm_first = PWM::SingleMatrix.load_from_line_array(extracted_pwm).with_background(first_background).discrete(discretization)
+#    r_stream, w_stream, extracted_pwm = extract_pwm(r_stream, w_stream)
+#    pwm_first = Macroape::SingleMatrix.load_from_line_array(extracted_pwm).with_background(first_background).discrete(discretization)
   else
     raise "Error! File #{first_file} don't exist" unless File.exist?(first_file)
-    pwm_first = PWM::SingleMatrix.load_pat(first_file).with_background(first_background).discrete(discretization)
+    pwm_first = Bioinform::PWM.new(File.read(first_file)).background(first_background).discrete(discretization)
   end
   
   if second_file == '.stdin'
-    r_stream, w_stream, extracted_pwm = extract_pwm(r_stream, w_stream)
-    pwm_second = PWM::SingleMatrix.load_from_line_array(extracted_pwm).with_background(second_background).discrete(discretization)
+#    r_stream, w_stream, extracted_pwm = extract_pwm(r_stream, w_stream)
+#    pwm_second = Macroape::SingleMatrix.load_from_line_array(extracted_pwm).with_background(second_background).discrete(discretization)
   else
     raise "Error! File #{second_file} don't exist" unless File.exist?(second_file)
-    pwm_second = PWM::SingleMatrix.load_pat(second_file).with_background(second_background).discrete(discretization)
+    pwm_second = Bioinform::PWM.new(File.read(second_file)).background(second_background).discrete(discretization)
   end
   
-  r_stream.close if first_file == '.stdin' || second_file == '.stdin'
+#  r_stream.close if first_file == '.stdin' || second_file == '.stdin'
   
   
-  pwm_second = pwm_second.reverse_complement if reverse
+  pwm_second.reverse_complement!  if reverse
   
   first_pwm_alignment = '.' * [-shift, 0].max + '>' * pwm_first.length
   second_pwm_alignment = '.' * [shift, 0].max + (orientation == :direct ? '>' : '<') * pwm_second.length
@@ -121,7 +121,7 @@ begin
   (first_pwm_alignment.length...alignment_length).each{|i| first_pwm_alignment[i] = '.'}
   (second_pwm_alignment.length...alignment_length).each{|i| second_pwm_alignment[i] = '.'}
 
-  cmp = PWMCompare::PWMCompareAligned.new(pwm_first.left_augment([-shift,0].max), 
+  cmp = Macroape::PWMCompareAligned.new(pwm_first.left_augment([-shift,0].max), 
                               pwm_second.left_augment([shift,0].max))
 
   first_threshold = pwm_first.threshold(pvalue)

data/lib/macroape/exec/eval_similarity.rb

@@ -26,7 +26,9 @@ Examples:
   cat motifs/KLF4.pat motifs/SP1.pat | ruby eval_similarity.rb .stdin .stdin -p 0.0005 -d 100 -b 0.4 0.3 0.2 0.1
 }
 
+$:.unshift File.join(File.dirname(__FILE__),'./../../')
 require 'macroape'
+require 'bioinform'
 
 if ARGV.empty? or ARGV.include? '-h' or ARGV.include? '-help' or ARGV.include? '--help' or ARGV.include? '--h'
   STDERR.puts help_string
@@ -51,9 +53,9 @@ begin
       when '-d'
         discretization = ARGV.shift.to_f
       when '-m'
-        PWM::MaxHashSize = ARGV.shift.to_f
+        Macroape::MaxHashSizeSingle = ARGV.shift.to_f
       when '-md'
-        PWMCompare::MaxHashSize = ARGV.shift.to_f
+        Macroape::MaxHashSizeDouble = ARGV.shift.to_f
       when '-b'
         second_background = first_background = ARGV.shift(4).map(&:to_f)
       when '-b1'
@@ -65,35 +67,35 @@ begin
   raise 'background should be symmetric: p(A)=p(T) and p(G) = p(C)' unless first_background == first_background.reverse
   raise 'background should be symmetric: p(A)=p(T) and p(G) = p(C)' unless second_background == second_background.reverse
   
-  PWM::MaxHashSize = 1000000 unless defined? PWM::MaxHashSize
-  PWMCompare::MaxHashSize = 1000 unless defined? PWMCompare::MaxHashSize
+  Macroape::MaxHashSizeSingle = 1000000 unless defined? Macroape::MaxHashSizeSingle
+  Macroape::MaxHashSizeDouble = 1000 unless defined? Macroape::MaxHashSizeDouble
   
   
-  if first_file == '.stdin' || second_file == '.stdin'
-    r_stream, w_stream = IO.pipe
-    STDIN.readlines.each{|line| w_stream.write(line)}
-    w_stream.close
-  end
+#  if first_file == '.stdin' || second_file == '.stdin'
+#    r_stream, w_stream = IO.pipe
+#    STDIN.readlines.each{|line| w_stream.write(line)}
+#    w_stream.close
+#  end
   
   if first_file == '.stdin'
-    r_stream, w_stream, extracted_pwm = extract_pwm(r_stream, w_stream)
-    pwm_first = PWM::SingleMatrix.load_from_line_array(extracted_pwm).with_background(first_background).discrete(discretization)
+#    r_stream, w_stream, extracted_pwm = extract_pwm(r_stream, w_stream)
+#    pwm_first = Macroape::SingleMatrix.load_from_line_array(extracted_pwm).with_background(first_background).discrete(discretization)
   else
     raise "Error! File #{first_file} don't exist" unless File.exist?(first_file)
-    pwm_first = PWM::SingleMatrix.load_pat(first_file).with_background(first_background).discrete(discretization)
+    pwm_first = Bioinform::PWM.new(File.read(first_file)).background(first_background).discrete(discretization)
   end
   
   if second_file == '.stdin'
-    r_stream, w_stream, extracted_pwm = extract_pwm(r_stream, w_stream)
-    pwm_second = PWM::SingleMatrix.load_from_line_array(extracted_pwm).with_background(second_background).discrete(discretization)
+#    r_stream, w_stream, extracted_pwm = extract_pwm(r_stream, w_stream)
+#    pwm_second = Macroape::SingleMatrix.load_from_line_array(extracted_pwm).with_background(second_background).discrete(discretization)
   else
     raise "Error! File #{second_file} don't exist" unless File.exist?(second_file)
-    pwm_second = PWM::SingleMatrix.load_pat(second_file).with_background(second_background).discrete(discretization)
+    pwm_second = Bioinform::PWM.new(File.read(second_file)).background(second_background).discrete(discretization)
   end
   
   r_stream.close if first_file == '.stdin' || second_file == '.stdin'
 
-  cmp = PWMCompare::PWMCompare.new(pwm_first, pwm_second)
+  cmp = Macroape::PWMCompare.new(pwm_first, pwm_second)
 
   first_threshold = pwm_first.threshold(pvalue)
   second_threshold = pwm_second.threshold(pvalue)

data/lib/macroape/exec/find_pvalue.rb

@@ -24,7 +24,9 @@ Examples:
   cat motifs/KLF4.pat | ruby find_pvalue.rb .stdin 7.32 4.31 5.42
 }
 
+$:.unshift File.join(File.dirname(__FILE__),'./../../')
 require 'macroape'
+require 'bioinform'
 
 if ARGV.empty? or ARGV.include? '-h' or ARGV.include? '-help' or ARGV.include? '--help' or ARGV.include? '--h'
   STDERR.puts help_string
@@ -56,22 +58,22 @@ begin
       when '-d'
         discretization = ARGV.shift.to_f
       when '-m'
-        PWM::MaxHashSize = ARGV.shift.to_f
+        Macroape::MaxHashSizeSingle = ARGV.shift.to_f
     end
   end
-  PWM::MaxHashSize = 1000000 unless defined? PWM::MaxHashSize
+  Macroape::MaxHashSizeSingle = 1000000 unless defined? Macroape::MaxHashSizeSingle
   
   
   if filename == '.stdin'
-    pwm = PWM::SingleMatrix.load_from_stdin(STDIN)
+#  TODO
   else
     raise "Error! File #{filename} doesn't exist" unless File.exist?(filename)
-    pwm = PWM::SingleMatrix.load_pat(filename)
+    pwm = Bioinform::PWM.new( File.read(filename) )
   end
-  pwm = pwm.with_background(background)
+  pwm.background(background)
 
   counts = pwm.discrete(discretization).counts_by_thresholds(* thresholds.map{|count| count * discretization})
-  pvalues = counts.map{|count| count.to_f / pwm.number_of_words}
+  pvalues = counts.map{|count| count.to_f / pwm.vocabulary_volume}
   pvalues.zip(thresholds,counts).each{|pvalue,threshold,count|
     puts "#{threshold}\t#{count}\t#{pvalue}"
   }