macroape 3.2.2 → 3.3.0

data/lib/macroape/exec/find_threshold.rb

@@ -20,7 +20,9 @@ Example:
   ruby find_threshold.rb motifs/KLF4.pat -p 0.001 0.0001 0.0005 -d 1000 -b 0.4 0.3 0.2 0.1
 }
 
+$:.unshift File.join(File.dirname(__FILE__),'./../../')
 require 'macroape'
+require 'bioinform'
 
 if ARGV.empty? or ARGV.include? '-h' or ARGV.include? '-help' or ARGV.include? '--help' or ARGV.include? '--h'
   STDERR.puts help_string
@@ -41,7 +43,7 @@ begin
       when '-b'
         background = ARGV.shift(4).map(&:to_f)
       when '-m'
-        PWM::MaxHashSize = ARGV.shift.to_f
+        Macroape::MaxHashSizeSingle = ARGV.shift.to_f
       when '-p'
         loop do
           begin
@@ -57,16 +59,16 @@ begin
   end
   pvalues = default_pvalues if pvalues.empty?
   
-  PWM::MaxHashSize = 1000000 unless defined? PWM::MaxHashSize
+  Macroape::MaxHashSizeSingle = 1000000 unless defined? Macroape::MaxHashSizeSingle
   
   if filename == '.stdin'
-    pwm = PWM::SingleMatrix.load_from_stdin(STDIN)
+##  TODO
   else
     raise "Error! File #{filename} doesn't exist" unless File.exist?(filename)
-    pwm = PWM::SingleMatrix.load_pat(filename)
+    pwm = Bioinform::PWM.new( File.read(filename) )
   end
   
-  pwm = pwm.with_background(background)
+  pwm.background(background)
   
   pwm.discrete(discretization).thresholds(*pvalues) do |pvalue, threshold, real_pvalue|
     puts "#{pvalue}\t#{threshold / discretization}\t#{real_pvalue}"

data/lib/macroape/exec/preprocess_collection.rb

@@ -9,13 +9,16 @@ Options:
   [-o <output file>]
   [--silent] - don't show current progress information during scan (by default this information's written into stderr)
 
-The tool stores preprocessed PWM collection to the specified YAML-file.
+The tool stores preprocessed Macroape collection to the specified YAML-file.
  
 Example:
   ruby preprocess_collection.rb ./motifs -p 0.001 0.0005 0.0001 -d 1 10 -b 0.2 0.3 0.2 0.3 -o collection.yaml
 }
 
+$:.unshift File.join(File.dirname(__FILE__),'./../../')
 require 'macroape'
+require 'bioinform'
+require 'yaml'
 
 if ARGV.empty? or ARGV.include? '-h' or ARGV.include? '-help' or ARGV.include? '--help' or ARGV.include? '--h'
   STDERR.puts help_string
@@ -54,24 +57,24 @@ begin
       when '-o'
         output_file = ARGV.shift
       when '-m'
-        PWM::MaxHashSize = ARGV.shift.to_f
+        Macroape::MaxHashSizeSingle = ARGV.shift.to_f
       when '-md'
-        PWMCompare::MaxHashSize = ARGV.shift.to_f
+        Macroape::MaxHashSizeDouble = ARGV.shift.to_f
       when '--silent'
         silent = true
       end
   end
   pvalues = default_pvalues if pvalues.empty?
   
-  PWM::MaxHashSize = 1000000 unless defined? PWM::MaxHashSize
-  PWMCompare::MaxHashSize = 1000 unless defined? PWMCompare::MaxHashSize
+  Macroape::MaxHashSizeSingle = 1000000 unless defined? Macroape::MaxHashSizeSingle
+  Macroape::MaxHashSizeDouble = 1000 unless defined? Macroape::MaxHashSizeDouble
 
-  collection = PWM::Collection.new(rough_discretization, precise_discretization, background, pvalues)
+  collection = Macroape::Collection.new(rough_discretization, precise_discretization, background, pvalues)
 
   current_dir = File.dirname(__FILE__)
   Dir.glob(File.join(folder,'*')) do |filename|
     STDERR.puts filename unless silent
-    pwm = PWM::SingleMatrix.load_pat(filename)
+    pwm = Bioinform::PWM.new(File.read(filename))
     info = {rough: {}, precise: {}}
     output = `ruby "#{File.join current_dir,'find_threshold.rb'}" #{filename} -p #{pvalues.join(' ')} -b #{background.join(' ')} -d #{rough_discretization}`.split("\n")
     output.each do |line|

data/lib/macroape/exec/scan_collection.rb

@@ -23,7 +23,10 @@ Example:
   cat motifs/KLF4.pat | ruby scan_collection.rb .stdin collection.yaml -p 0.005 --precise 0.03
 }
 
+$:.unshift File.join(File.dirname(__FILE__),'./../../')
 require 'macroape'
+require 'bioinform'
+require 'yaml'
 
 if ARGV.empty? or ARGV.include? '-h' or ARGV.include? '-help' or ARGV.include? '--help' or ARGV.include? '--h'
   STDERR.puts help_string
@@ -52,9 +55,9 @@ begin
       when '-p'
         pvalue = ARGV.shift.to_f
       when '-m'
-        PWM::MaxHashSize = ARGV.shift.to_f
+        Macroape::MaxHashSizeSingle = ARGV.shift.to_f
       when '-md'
-        PWMCompare::MaxHashSize = ARGV.shift.to_f
+        Macroape::MaxHashSizeDouble = ARGV.shift.to_f
       when '-c'
         cutoff = ARGV.shift.to_f
       when '--all'
@@ -71,21 +74,21 @@ begin
         end
     end
   end
-  PWM::MaxHashSize = 1000000 unless defined? PWM::MaxHashSize
-  PWMCompare::MaxHashSize = 1000 unless defined? PWMCompare::MaxHashSize
+  Macroape::MaxHashSizeSingle = 1000000 unless defined? Macroape::MaxHashSizeSingle
+  Macroape::MaxHashSizeDouble = 1000 unless defined? Macroape::MaxHashSizeDouble
   
   raise "Thresholds for pvalue #{pvalue} aren't presented in collection (#{collection.pvalues.join(', ')}). Use one of listed pvalues or recalculate the collection with needed pvalue" unless collection.pvalues.include? pvalue
   
   if filename == '.stdin'
-    query_pwm = PWM::SingleMatrix.load_from_stdin(STDIN)
+#    query_pwm = Macroape::SingleMatrix.load_from_stdin(STDIN)
   else
     raise "Error! File #{filename} doesn't exist" unless File.exist?(filename)
-    query_pwm = PWM::SingleMatrix.load_pat(filename)
+    query_pwm = Bioinform::PWM.new(File.read(filename))
   end
   
   
-  query_pwm_rough = query_pwm.with_background(background_query).discrete(collection.rough_discretization)
-  query_pwm_precise = query_pwm.with_background(background_query).discrete(collection.precise_discretization)
+  query_pwm_rough = query_pwm.background(background_query).discrete(collection.rough_discretization)
+  query_pwm_precise = query_pwm.background(background_query).discrete(collection.precise_discretization)
   
   threshold = query_pwm_rough.threshold(pvalue)
   threshold_precise = query_pwm_precise.threshold(pvalue)
@@ -98,13 +101,13 @@ begin
     pwm = collection.pwms[name]
     pwm_info = collection.infos[name]
     STDERR.puts pwm.name unless silent
-    cmp = PWMCompare::PWMCompare.new(query_pwm_rough, pwm.with_background(collection.background).discrete(collection.rough_discretization))
+    cmp = Macroape::PWMCompare.new(query_pwm_rough, pwm.background(collection.background).discrete(collection.rough_discretization))
     info = cmp.jaccard(threshold, pwm_info[:rough][pvalue] * collection.rough_discretization)
     name = pwm.name || "Unnamed #{unnamed_index += 1}"
     precision_file_mode[name] = :rough
     
     if precision_mode == :precise and info[:similarity] >= minimal_similarity
-      cmp = PWMCompare::PWMCompare.new(query_pwm_precise, pwm.with_background(collection.background).discrete(collection.precise_discretization))
+      cmp = Macroape::PWMCompare.new(query_pwm_precise, pwm.background(collection.background).discrete(collection.precise_discretization))
       info = cmp.jaccard(threshold_precise, pwm_info[:precise][pvalue] * collection.precise_discretization)
       precision_file_mode[name] = :precise
     end

data/lib/macroape/pwm_compare.rb

@@ -1,10 +1,41 @@
-module PWMCompare
+module Macroape
   class PWMCompare
     attr_reader :first, :second
     def initialize(first, second)
       @first = first
       @second = second
     end
-    include PairTransformations, PairMetrics
+
+    def jaccard(threshold_first, threshold_second)
+      self.map_each_align do |align, alignment_info|
+        align.jaccard(threshold_first, threshold_second).merge(alignment_info)
+      end.max_by {|alignment_info| alignment_info[:similarity]}
+    end
+    
+    
+    def each
+      second_rc = second.reverse_complement
+      (-second.length..first.length).to_a.product([:direct,:revcomp]) do |shift, orientation|
+        first_pwm_alignment = '.' * [-shift, 0].max + '>' * first.length
+        second_pwm_alignment = '.' * [shift, 0].max + (orientation == :direct ? '>' : '<') * second.length
+        overlap = [first.length + [-shift,0].max, second.length + [shift,0].max].min - shift.abs
+        alignment_length = [first_pwm_alignment.length, second_pwm_alignment.length].max
+        (first_pwm_alignment.length...alignment_length).each{|i| first_pwm_alignment[i] = '.'}
+        (second_pwm_alignment.length...alignment_length).each{|i| second_pwm_alignment[i] = '.'}
+        
+        yield(PWMCompareAligned.new(first.left_augment([-shift,0].max), 
+                                    (orientation == :direct ? second : second_rc).left_augment([shift,0].max)),
+              text: "#{first_pwm_alignment}\n#{second_pwm_alignment}",
+              shift: shift,
+              orientation: orientation,
+              overlap: overlap,
+              alignment_length: alignment_length
+              )
+      end
+    end
+    include Enumerable
+    alias :each_align :each
+    alias :map_each_align :map
+    
   end
 end

data/lib/macroape/pwm_compare_aligned.rb

@@ -1,4 +1,6 @@
-module PWMCompare
+require 'macroape/aligned_pair_intersection'
+
+module Macroape
   class PWMCompareAligned
     attr_reader :first, :second, :length
     def initialize(first, second)
@@ -7,7 +9,41 @@ module PWMCompare
       @second = second.right_augment(@length - second.length)
     end
     
-    include AlignedPairTransformations, AlignedPairMetrics, AlignedPairIntersection
+    #def discrete(rate)
+    #  PWMCompareAligned.new(first.discrete(rate), second.discrete(rate))
+    #end
+
+    def sort_pair_of_matrices_by(&block)
+      mat = first.pwm.zip(second.pwm).sort_by(&block).transpose
+      PWMCompareAligned.new(SinglePWM(mat[0],first.probabilities), SinglePWM(mat[1], second.probabilities))
+    end
+    def sort_decreasing_max
+      PWMCompareAligned.new(*sort_pair_of_matrices_by{|col_pair| -col_pair[0].max} )
+    end
+    def sort_increasing_min
+      PWMCompareAligned.new(*sort_pair_of_matrices_by{|col_pair| col_pair[0].min} )
+    end
+    def permute_columns(permutation_index)
+      PWMCompareAligned.new(first.permute(permutation_index), second.permute(permutation_index))
+    end
+
+    def jaccard(first_threshold, second_threshold)
+      f = first.counts_by_thresholds(first_threshold).first
+      s = second.counts_by_thresholds(second_threshold).first
+      if f == 0 or s == 0
+        return {similarity: -1, tanimoto: -1, recognized_by_both: 0,
+              recognized_by_first: f,
+              recognized_by_second: s,
+            }
+      end
+      
+      intersect = counts_for_two_matrices(first_threshold, second_threshold)
+      intersect = Math.sqrt(intersect[0] * intersect[1])
+      union = f + s - intersect
+      similarity = intersect.to_f / union
+      { similarity: similarity,  tanimoto: 1.0 - similarity,  recognized_by_both: intersect,
+        recognized_by_first: f,  recognized_by_second: s }
+    end
     
   end
 end

data/lib/macroape/threshold_by_pvalue.rb

@@ -1,67 +1,72 @@
-module PWM
-  module ThresholdByPvalue
+module Bioinform
+  class PWM
     def threshold(pvalue)
       thresholds(pvalue){|_, thresh, _| return thresh }
     end
     
     def thresholds(*pvalues)
-      thresholds_by_pvalues(*pvalues).each do |pvalue,(left_threshold, left_count, right_threshold, right_count)|
-        threshold = left_threshold + 0.1
-        real_pvalue = right_count.to_f / number_of_words
+      thresholds_by_pvalues(*pvalues).each do |pvalue,(thresholds, counts)|
+        threshold = thresholds.begin + 0.1 * (thresholds.end - thresholds.begin)
+        real_pvalue = counts.end.to_f / vocabulary_volume
         yield pvalue, threshold, real_pvalue
       end
     end
   
-    # ret-value: hash {pvalue => [left_threshold, left_count, right_threshold, right_count]}
-    def thresholds_by_pvalues(*pvalues)
-      max_pvalue = pvalues.max
-      max_look_for_count = max_pvalue * sum_of_probabilities ** length
-      scores={}
-      until scores.inject(0){|sum,(score,count)| sum + count} >= max_look_for_count
-        scores = calculate_count_distribution_after_threshold(threshold_gauss_estimation(max_pvalue))
+    def count_distribution_under_pvalue(max_pvalue)
+      count_distribution={}
+      look_for_count = max_pvalue * vocabulary_volume
+      until count_distribution.inject(0.0){|sum,(score,count)| sum + count} >= look_for_count
+        count_distribution = count_distribution_after_threshold(threshold_gauss_estimation(max_pvalue))
         max_pvalue *=2 # if estimation counted too small amount of words - try to lower threshold estimation by doubling pvalue
       end
-      pvalue_counts = pvalues.sort.inject(Hash.new){|h, pvalue| h.merge pvalue => pvalue * sum_of_probabilities**length }
-      look_for_counts = pvalue_counts.to_a
-      sum_count = 0
-      scores = scores.sort.reverse
+      
+      count_distribution
+    end
+  
+  
+    # ret-value: hash {pvalue => [thresholds, counts]}
+    # thresholds = left_threshold .. right_threshold  (left_threshold < right_threshold)
+    # counts = left_count .. right_count  (left_count > right_count)
+    def thresholds_by_pvalues(*pvalues)
+      count_distribution = count_distribution_under_pvalue(pvalues.max)
+      sorted_scores = count_distribution.sort.reverse
+      scores = sorted_scores.map{|score,count| score}
+      counts = sorted_scores.map{|score,count| count}
+      partial_sums = counts.partial_sums
+      
       results = {}
-      scores.size.times do |i|
-        while !look_for_counts.empty? and sum_count + scores[i][1] > look_for_counts.first[1] # usually this 'while' works as 'if'
-          cnt = look_for_counts.shift
-          pval = cnt[0]
-          score = cnt[1]
-          
-          threshold_2 = scores[i][0]
-          sum_count_2 = sum_count + scores[i][1]
-          if i>0
-            threshold = scores[i-1][0]
-            results[pval] = [threshold_2.to_f, sum_count_2, threshold.to_f, sum_count.to_f]
-          else          
-            results[pval] = [threshold_2.to_f, sum_count_2.to_f, best_score + 1.0, 0.0]
-          end
-        end
-        sum_count += scores[i][1]
+      
+      pvalue_counts = pvalues.sort.collect_hash{|pvalue| [pvalue, pvalue * vocabulary_volume] }
+      pvalue_counts.map do |pvalue,look_for_count|
+        ind = partial_sums.index{|sum| sum >= look_for_count}
+        minscore, count_at_minscore = scores[ind], partial_sums[ind]
+        maxscore, count_at_maxscore = ind > 0  ?  [ scores[ind-1],  partial_sums[ind-1] ]  :  [ best_score + 1.0, 0.0 ]
+        results[pvalue] = [(minscore .. maxscore), (count_at_minscore .. count_at_maxscore)]
       end
+
       results
     end
     
-    def calculate_count_distribution_after_threshold(threshold)
+    def count_distribution_after_threshold(threshold)
       scores = { 0 => 1 }
       length.times do |column|
-        new_scores = Hash.new(0);
-        scores.each do |score, count|
-          4.times do |letter|
-            new_score = score + matrix[column][letter]
-            if new_score + best_suffix[column + 1] >= threshold
-              new_scores[new_score] += count * probabilities[letter]
-            end
+        scores.replace recalc_score_hash(scores, @matrix[column], threshold - best_suffix[column + 1])
+        raise 'Hash overflow in PWM::ThresholdByPvalue#count_distribution_after_threshold' if defined? MaxHashSizeSingle and scores.size > MaxHashSizeSingle
+      end
+      scores
+    end
+    
+    def recalc_score_hash(scores, column, least_sufficient)
+      new_scores = Hash.new(0)
+      scores.each do |score, count|
+        4.times do |letter|
+          new_score = score + column[letter]
+          if new_score >= least_sufficient
+            new_scores[new_score] += count * @background[letter]
           end
         end
-        raise 'Hash overflow in PWM::ThresholdByPvalue#calculate_count_distribution_after_threshold' if defined? MaxHashSize and new_scores.size > MaxHashSize
-        scores = new_scores
       end
-      scores
+      new_scores
     end
     
   end

data/lib/macroape/version.rb

@@ -1,3 +1,3 @@
-module Macroape
-  VERSION = "3.2.2"
-end
+module Macroape
+  VERSION = "3.3.0"
+end

data/macroape.gemspec

@@ -14,4 +14,6 @@ Gem::Specification.new do |gem|
   gem.name          = "macroape"
   gem.require_paths = ["lib"]
   gem.version       = Macroape::VERSION
+  
+  gem.add_dependency('bioinform', '~> 0.1.0')
 end

data/test/data/test_collection.yaml

@@ -1,4 +1,4 @@
---- !ruby/object:PWM::Collection
+--- !ruby/object:Macroape::Collection
 pvalues:
 - 0.0005
 - 0.0001
@@ -11,7 +11,27 @@ background:
 precise_discretization: 10
 rough_discretization: 1
 pwms:
-  GABPA_f1: !ruby/object:PWM::SingleMatrix
+  GABPA_f1: !ruby/object:Bioinform::PWM
+    background:
+    - 1
+    - 1
+    - 1
+    - 1
+    input: ! "GABPA_f1\n-0.1106670158341858 0.013801606113892391 0.6054596108973699
+      -1.3518085041421573 \n0.37030668921643345 0.15761121480429963 0.009069314183831202
+      -0.9888619717703562 \n0.47526546359546684 -0.3011678534572083 0.4031522994412777
+      -1.8638752827041059 \n-1.5544255540164373 1.1082369687811506 -0.2814091552834454
+      -5.30708531823271 \n-0.6362037835776368 1.235338189985594 -3.5801322928552253
+      -5.717323067092849 \n-5.852906870733575 -5.852906870733575 1.3841383838057746
+      -5.852906870733575 \n-5.852906870733575 -5.852906870733575 1.3841383838057746
+      -5.852906870733575 \n1.3835219739184708 -5.2341956006430985 -5.852906870733575
+      -5.852906870733575 \n1.3756340514956562 -5.394962755562375 -5.394962755562375
+      -3.401117964959733 \n-1.2176198315414444 -3.109079898175411 1.2964067931472216
+      -5.717323067092849 \n-1.3716559438167257 -0.2761401935045069 -1.8504445165866068
+      1.0404320473626856 \n-0.5440863133031895 -0.48103682561971345 0.907381908447086
+      -1.1280642594012078 \n0.10557340209290218 -0.01814819455289191 0.4381106695354074
+      -1.0304105539540915 \n"
+    parser: !ruby/class 'Bioinform::StringParser'
     matrix:
     - - -0.1106670158341858
       - 0.013801606113892391
@@ -66,7 +86,24 @@ pwms:
       - 0.4381106695354074
       - -1.0304105539540915
     name: GABPA_f1
-  KLF4_f2: !ruby/object:PWM::SingleMatrix
+  KLF4_f2: !ruby/object:Bioinform::PWM
+    background:
+    - 1
+    - 1
+    - 1
+    - 1
+    input: ! "KLF4_f2\n0.30861857265872605 -2.254321000121579 0.13505703522674192
+      0.3285194224375633 \n-1.227018967707036 -4.814127713368663 1.3059890687390967
+      -4.908681463544344 \n-2.443469374521196 -4.648238485031404 1.3588686548279805
+      -4.441801801188402 \n-2.7177827948276123 -3.8073538975356565 1.356272809724262
+      -3.504104725510225 \n-0.5563232977367343 0.5340697765121405 -3.61417723090579
+      0.5270259776377405 \n-1.8687622060887386 -4.381483976582316 1.337932245336098
+      -3.815629658877517 \n-2.045671123823928 -2.384975142213679 0.7198551207724355
+      0.5449254135616948 \n-1.373157530374372 -3.0063112097748217 1.285188335493552
+      -2.5026044231773543 \n-2.1030513122772208 -1.8941348100402244 1.249265758393991
+      -1.4284210948906104 \n-1.3277128628152939 0.8982415633049462 -0.8080773665408135
+      -0.18161647647456935 \n"
+    parser: !ruby/class 'Bioinform::StringParser'
     matrix:
     - - 0.30861857265872605
       - -2.254321000121579
@@ -109,7 +146,36 @@ pwms:
       - -0.8080773665408135
       - -0.18161647647456935
     name: KLF4_f2
-  SP1_f1: !ruby/object:PWM::SingleMatrix
+  SP1_f1: !ruby/object:Bioinform::PWM
+    background:
+    - 1
+    - 1
+    - 1
+    - 1
+    input: ! '> SP1_f1
+
+      -0.24435707885585334  -0.6748234046937317  0.8657012535789861  -1.1060188862599292
+
+      -1.0631255752097801  -2.1119259694238686  1.0960627561110399  -0.6138563775211981
+
+      -0.387227623476054  -2.973985191321805  1.1807800242010371  -4.338927525031567
+
+      -4.563896055436894  -2.916163300253228  1.3684371349982631  -5.077972423609655
+
+      -2.2369752892820087  -3.719643631330185  1.3510439136452728  -4.8899306705082335
+
+      -0.07473964149330914  0.9449196547620103  -2.624685764808605  -0.851098348782244
+
+      -1.9643526491643326  -2.9784027708801153  1.3113096718240569  -2.3243342594990253
+
+      -4.015548413965584  -3.138426807809667  1.338748858978805  -2.0846739035376483
+
+      -0.4450938582835542  -2.2510053061629707  1.126543157436868  -1.7780413702431377
+
+      -1.1896356092245055  -1.2251832285630033  1.163676006374752  -1.6080243648157357
+
+      -0.5166047365590577  0.7641033353626651  -0.28626775700282125  -0.6825482097865606'
+    parser: !ruby/class 'Bioinform::StringParser'
     matrix:
     - - -0.24435707885585334
       - -0.6748234046937317