mspire 0.2.4 → 0.3.0

data/lib/validator/background.rb

@@ -0,0 +1,73 @@
+require 'validator'
+require 'vec'
+require 'enumerator'
+
+class Validator ; end
+class Validator::Background
+
+  attr_accessor :data
+
+  def initialize(data=nil)
+    @data = data
+  end
+
+  def delete_nan!(vec)
+    vec.each_with_index do |v,i|
+      if v.nan?
+        vec[i] = 0
+      end
+    end
+  end
+
+  def stdev_plus_spread(stdev_factor=2.0, stdev_points=15, min_window_pre=5, min_window_post=5)
+    data_vec = VecD[*@data]
+    delete_nan!(data_vec)
+    stdev_transform = data_vec.transform(9) {|vec| (stdev_factor * vec.sample_stats[1]) + vec.spread  } 
+    smoothed_stdev = stdev_transform.transform(9) {|vec| vec.avg }
+    smoothed_stdev_derivs = smoothed_stdev.chim
+    last_0_index = index_of_last_0(smoothed_stdev_derivs)
+    min_in_window(data_vec, last_0_index, min_window_pre, min_window_post)
+  end
+
+  # not really working right currently
+  def derivs(avg_points=15, min_window_pre=5, min_window_post=5)
+    data_vec = VecD[*@data]
+    delete_nan!(data_vec)
+    drvs = data_vec.chim
+    # absolute value
+    drvs.each_with_index {|x,i| drvs[i] = x.abs }
+    mv_avg = drvs.transform(avg_points) {|v| v.avg }
+    last_0_index = index_of_last_0(mv_avg.chim)
+    min_in_window(data_vec, last_0_index, min_window_pre, min_window_post)
+  end
+
+  def index_of_last_0(vec)
+    last_0_index = nil
+    vec.each_with_index do |v,i|
+      if v == 0
+        last_0_index = i
+      end
+    end
+    last_0_index
+  end
+
+  # returns the minimum value in the window centered on index
+  def min_in_window(vec, index, pre, post)
+    last_index = vec.size - 1
+    start = index - pre
+    stop = index + post
+    start = 0 if start < 0
+    stop = last_index if stop > last_index
+    vec[start..stop].min
+  end
+
+  # very simple, should work
+  def min_mesa(start, stop, points=3)
+    data_vec = VecD[*@data]
+    delete_nan!(data_vec)
+    smoothed = data_vec.transform(3) {|v| v.avg }
+    smoothed[start..stop].min
+  end
+  
+end
+

data/lib/validator/bias.rb

@@ -0,0 +1,95 @@
+require 'validator'
+require 'validator/digestion_based'
+
+# class for any generic kind of bias.  For instance, a list of high abundance
+# proteins we would expect to see, or a list of low abundance proteins we
+# would not expect to see, or proteins that have been filtered out in some
+# way, etc.
+class Validator::Bias < Validator::DigestionBased
+  include Precision::Calculator
+
+  # a fasta object (by default containing proteins expected to be in the
+  # sample [see proteins_expected to modify that behavior])
+  attr_reader :fasta
+
+  # correct_wins means that only a single protein from a pep.aaseq must match
+  # the fasta object for the pep hit to be considered valid.  Otherwise, all
+  # must be a match (logic negated by proteins_expected)
+  attr_accessor :correct_wins
+
+  # proteins_expected==true means we expect to see the proteins in the sample
+  # proteins_expected==false means we do not expect to see these proteins in
+  # the sample
+  attr_accessor :proteins_expected
+
+  # a hash made by taking each fasta reference in fasta_object, (everything
+  # until a space) and setting the value to true.  It can be queried with the
+  # start of an fasta sequence
+  attr_accessor :short_reference_hash
+
+  DEFAULTS = Validator::DigestionBased::DEFAULTS.merge( {
+    :proteins_expected => true, 
+    :correct_wins => true,
+  } )
+
+  # options: 
+  #   (t = true, f = false, '*'= default)
+  #   :proteins_expected => *t/f  we expect to see the fasta proteins in our hit list
+  #   :correct_wins => *t/f  a single peptide hit from one of these proteins
+  #                    constitutes a true positive 
+  #   :background => Float  (*0.0-1.0)
+  #   :false_to_total_ratio => Float (*nil by default)
+  def initialize(fasta_object, options={})
+    opts = DEFAULTS.merge(options)
+    (@proteins_expected, @correct_wins, @background, @false_to_total_ratio) = opts.values_at(:proteins_expected, :correct_wins, :background, :false_to_total_ratio)
+    @fasta = fasta_object
+    @header_split_hash = @fasta.prots.map {|prot| prot.reference }
+    @short_reference_hash = self.class.make_short_reference_hash(fasta_object)
+  end
+
+  def self.make_short_reference_hash(fasta_object)
+    hash = {}
+    fasta_object.each do |prot|
+      hash[prot.first_entry] = true
+    end
+    hash
+  end
+
+  def partition(peps)
+    klass = self.class
+    cw = 
+      if !@proteins_expected
+        !@correct_wins
+      else
+        @correct_wins
+      end
+
+    (tp, fp) = 
+      if cw
+        peps.partition do |pep|
+          pep.prots.any? do |pepprot| 
+            @short_reference_hash.key?( pepprot.first_entry )
+          end
+        end
+      else
+        peps.partition do |pep|
+          pep.prots.any? do |pepprot| 
+            !@short_reference_hash.key?( pepprot.first_entry )
+          end
+        end
+      end
+
+    if !@correct_wins
+      tp, fp = fp, tp
+    end
+
+    [tp, fp]
+  end
+
+  # pephit_precision is done through inheritance
+
+  def to_param_string
+    "abundance=" +  ["{fasta=#{@fasta.filename}", "proteins_expected=#{@proteins_expected}", "correct_wins=#{@correct_wins}", "background=#{@background}}"].join(", ")
+  end
+
+end

data/lib/validator/cmdline.rb

@@ -0,0 +1,260 @@
+require 'validator'
+
+class Validator::Cmdline
+
+  Validator_symbols_to_classes = {
+    :tmm => Validator::Transmem::Protein,
+    :decoy => Validator::Decoy,
+    :bad_aa => Validator::AA,
+    :tps => Validator::TruePos,
+    :bias => Validator::Bias,
+    :prob => Validator::Probability,
+  }
+  # was VAL_DEFAULTS
+  DEFAULTS = {
+    :tmm => 
+    {
+      # file                      
+      :min_num_tmm_seqs => 1,     
+      :expect_soluble => true,   
+      :no_include_tm_peps => 0.8, 
+      :bkg => 0.0,               
+    },
+    :decoy => 
+    {
+      :hits_together => true,
+      :decoy_on_match => true,
+    },
+    :bad_aa => 
+    {
+      :false_if_found => true,
+      :estimate => true,
+      :bkg => 0.0, 
+    },
+    :bias =>
+    {
+      :bkg => 0.0,
+      :proteins_expected => true,
+    },
+    :ties => true,
+  }
+  COMMAND_LINE = {
+    :decoy => ["--decoy /REGEXP/|FILENAME[DOM]", Array, "REGEXP for decoy proteins (catenated searches) or a",
+                                                "FILENAME of separate search on decoys.", 
+                                                "All regular expressions must be surrounded by '/'",
+                                                "(no extended options [trailing modifiers]).",
+                                                "e.g., a run using concatenated reversed proteins that",
+                                                "includes 'REVERSE' in the fasta heading:",
+                                                "    --decoy /REVERSE/",
+                                                "Anything fancier should be quoted:",
+                                                "    --decoy '/^\\s*REVERSE/'",
+                                                "If decoys proteins were searched in a separate file,",
+                                                "then give the FILENAME (e.g., --decoy decoy.srg)",
+                                                "DOM = *true/false, decoy on match",],
+        :tps => ["--tps <fasta>", "for a completely defined sample, this is the",
+                                  "fasta file containing the true protein hits"],
+         # may require digestion:
+        :digestion => ["--digestion ORIG_FASTA,PARAMS", Array, "The following validators require additional",
+                                                         "information (that is shared between them).",
+                                                         "ORIG_FASTA = the fasta file used to do the run",
+                                                         "PARAMS = the params file used to do the run",],
+        :bias => ["--bias FASTA[,PE,BKG]", Array, "FASTA contains proteins expected to be in the sample",
+                                                  "PE = *true|false proteins in fasta file expected in sample",
+                                                  "BKG = Background frequency of fps (d: #{DEFAULTS[:bias][:bkg]})",],
+        :bad_aa => ["--bad_aa AA,[EST,BKG]", Array, "An amino acid expected (or not expected) in legitimate hits",
+                                                        "AA = The amino acid (e.g., 'C')",
+                                                        "EST = true|false (def: #{DEFAULTS[:bad_aa][:estimate]})",
+                                                        "BKG = Background frequency of genuine pephits (d: #{DEFAULTS[:bad_aa][:bkg]}):",],
+
+        :tmm => ["--tmm <TM[,MIN,SOL,PEPS,BKG]>", Array, "TM = phobius.small or toppred.out file",
+                                                         "phobius.small:",
+                                                         "http://phobius.cgb.ki.se/",
+                                                         "(select 'Short' output, and save output as file)",
+                                                         "toppred.out:",
+                                                         "http://bioweb.pasteur.fr/seqanal/interfaces/toppred.html",
+                                                         "(output 'toppred.out' in 'New' or 'Xml' format)",
+                                                         "MIN = Int, minimum number transmembrane seqs (def: #{DEFAULTS[:tmm][:min_num_tmm_seqs]})",
+                                                         "SOL = true|false, this is a soluble fraction( def: #{DEFAULTS[:tmm][:expect_soluble]})",
+                                                         "PEPS = Float | false, don't consider tm peps (>= fraction",
+                                                         "                   tm content) (false skips) (def: #{DEFAULTS[:tmm][:no_include_tm_peps]})",
+                                                         "BKG = Float , background contaminating insoluble (def: #{DEFAULTS[:tmm][:bkg]})"],
+
+
+        # VALIDATION MODIFIERS
+        :false_on_tie => ["--false_on_tie", "if peptide belongs to correct AND incorrect proteins",
+                                            "it will be counted as correct"],
+
+  }
+
+      def self.boolean(arg, default)
+        case arg
+        when 'true' ; true
+        when 'false' ; false
+        else ; default
+        end
+      end
+
+      PrepArgs = {
+        :prob => lambda {|ar, opts|
+        mthd = 
+          if ar
+            if ar == 'nsp'
+              :probability
+            elsif ar == 'init'
+              :initial_probability
+            else
+              raise ArgumentError, "--prob [arg], optional arg can only be 'nsp' or 'init'!"
+            end
+          else
+            :probability
+          end
+        opts[:validators].push([:prob, mthd])
+      },
+        :decoy => lambda {|ar, opts| 
+        myargs = [:decoy] 
+        first_arg = ar[0]
+        myargs[1] =
+          if first_arg[0,1] == '/' and first_arg[-1,1] == '/'
+            Regexp.new(first_arg[1...-1])
+          else
+            first_arg
+          end
+        myargs[2] = self.boolean(ar[1], DEFAULTS[:decoy][:decoy_on_match])
+        opts[:validators].push(myargs) 
+      },
+        :digestion => lambda {|ar, opts|
+        raise(ArgumentError, "need fasta and sequest params!") if ar.size != 2
+        opts[:digestion] = ar.dup
+        opts[:digestion_objects] = [Fasta.new(ar[0]), Sequest::Params.new(ar[1])]
+      },
+        :bias => lambda {|ar, opts|
+        myargs = [:bias]
+        myargs.push( Fasta.new(ar[0]) )
+        val_opts = {}
+        val_opts[:proteins_expected] = self.boolean(ar[1], DEFAULTS[:bias][:proteins_expected])
+        val_opts[:background] = 
+          if ar[2]
+            ar[2].to_f
+          else
+            DEFAULTS[:bias][:bkg]
+          end
+        myargs.push(val_opts)
+        opts[:validators].push(myargs)
+      },
+        :bad_aa => lambda {|ar, opts|
+        ## GET the FREQUENCY
+        myargs = [:bad_aa] 
+        myargs.push( ar[0] )
+        val_opts = {}
+        val_opts[:estimate] = self.boolean(ar[1], DEFAULTS[:bad_aa][:est])
+        val_opts[:background] = 
+          if ar[2]
+            ar[2].to_f
+          else
+            DEFAULTS[:bad_aa][:bkg]
+          end
+        myargs.push(val_opts)
+        opts[:validators].push(myargs)
+      },
+        :tmm =>  lambda {|ar, opts|
+        myargs = [:tmm]
+        myargs.push( ar[0] )
+        val_opts = {}
+        val_opts[:min_num_tms] =
+          if ar[1] ; ar[1].to_i
+          else ; DEFAULTS[:tmm][:min_num_tmm_seqs]
+          end
+        val_opts[:soluble_fraction] = self.boolean(ar[2], DEFAULTS[:tmm][:expect_soluble])
+        val_opts[:no_include_tm_peps] =
+          if ar[3]
+            case ar[3]
+            when 'false' ; false
+            else ; ar[3].to_f
+            end
+          else ; DEFAULTS[:tmm][:no_include_tm_peps]
+          end
+        val_opts[:background] =
+          if ar[4] ; ar[4].to_f
+          else ; DEFAULTS[:tmm][:bkg]
+          end
+        myargs.push(val_opts)
+        opts[:validators].push( myargs )
+      },
+      :tps => lambda {|v,opts| opts[:validators].push([:tps, Fasta.new(v)]) },
+      :false_on_tie => lambda {|v,opts| opts[:ties] = false },
+      }
+
+      # remove the keys from opts involved in validators and return an array
+      # of validators
+      def self.prepare_validators(opts, false_on_tie, interactive, spec_id)      
+        validator_args = opts[:validators]
+        correct_wins = !false_on_tie
+        need_false_to_total_ratio = []
+        need_frequency = []
+        transmem_vals = []
+        validators = validator_args.map do |args|
+          tp = args.shift
+          val_args = args.dup # protect the original keys
+          val_args = 
+            case tp
+            when :tmm
+              val_args[1][:correct_wins] = correct_wins
+              val_args[1][:fasta] = opts[:digestion_objects][0]
+              val_args
+            when :bias
+              val_args[1][:correct_wins] = correct_wins
+              val_args
+            when :tps
+              val_args = [val_args[0], correct_wins]
+              val_args
+            when :decoy
+              val_args = [val_args[0], val_args[1], correct_wins]
+              # don't delete the key here since we need the decoy = regexp key
+              val_args
+            else ## bad_aa and prob are represented here:
+              val_args
+            end
+          val = Validator_symbols_to_classes[tp].new( *val_args )
+          # make some lists of validators based on pre-processing needs:
+          if tp == :tmm
+            transmem_vals << val
+          end
+          potential_digestion_classes = /Transmem|AA|Bias/
+          if val.class.to_s =~ potential_digestion_classes
+            if val_args[1][:estimate] == true
+              need_frequency << val
+            else
+              need_false_to_total_ratio << val
+            end
+          end
+          val
+        end
+
+        if need_false_to_total_ratio.size > 0
+          raise ArgumentError, "requires --digestion fasta,params argument!" if !opts.key?(:digestion_objects)
+          peps = Digestor.digest( *(opts[:digestion_objects]) )
+          need_false_to_total_ratio.each do |val|
+            val.set_false_to_total_ratio( peps )
+          end
+        end
+        if need_frequency.size > 0
+          raise ArgumentError, "requires --digestion fasta,params argument!" if !opts.key?(:digestion_objects)
+          need_frequency.each do |val|
+            val.set_frequency( opts[:digestion_objects][0] )
+          end
+        end
+        opts.delete(:digestion_objects)
+
+        if (transmem_vals.size > 0)   #  and interactive   ## we'd like to just run this for interactive
+          # This is overkill if we are doing a single filtering job, but it
+          # ensures that it works in all the ways I'm doing it.  Should
+          # refactor eventually !!
+          transmem_vals.each do |val|                      ## but, prob uses it too!
+            val.transmem_status_hash = val.create_transmem_status_hash(spec_id.peps)
+          end
+        end
+        validators
+
+      end
+
+end

data/lib/validator/decoy.rb

@@ -0,0 +1,94 @@
+require 'validator'
+
+class Validator::Decoy < Validator
+  include Precision::Calculator::Decoy
+
+  attr_accessor :constraint
+
+  attr_accessor :decoy_on_match
+  attr_accessor :correct_wins
+
+  attr_accessor :last_pep_was_decoy
+
+  attr_accessor :increment_normal
+  attr_accessor :increment_decoy
+  attr_accessor :increment_total_submitted
+
+  attr_reader :normal_peps_just_submitted
+
+  def initialize(constraint=nil, decoy_on_match = true, correct_wins = true)
+    @decoy_on_match = decoy_on_match
+    @correct_wins = correct_wins
+    @constraint = constraint
+  end
+
+  # returns [normal, decoy] (?? I think ??)
+  def partition(peps)
+    if @decoy_on_match 
+      if @correct_wins
+        peps.partition do |pep|
+          !(pep.prots.all? {|prot| prot.reference.match(@constraint) })
+        end
+      else  # fp wins
+        peps.partition do |pep|
+          !(pep.prots.any? {|prot| prot.reference.match(@constraint) })
+        end
+      end
+    else 
+      if @correct_wins
+        peps.partition do |pep|
+          pep.prots.any? {|prot| prot.reference.match(@constraint) }
+        end
+      else
+        peps.partition do |pep|
+          pep.prots.all? {|prot| prot.reference.match(@constraint) }
+        end
+      end
+    end
+  end
+
+  def initialize_increment
+    @increment_normal = 0
+    @increment_decoy = 0
+    @increment_total_submitted = 0
+    @increment_initialized = true
+  end
+
+
+  # does not deal in separate_peps right now!!
+  # will take an array or single peptide
+  def increment_pephits_precision(peps)
+    tmp = $VERBOSE; $VERBOSE = nil
+    initialize_increment unless @increment_initialized
+    $VERBOSE = tmp
+
+    to_submit = 
+      if peps.is_a? SpecID::Pep
+        [peps]
+      else
+        peps
+      end
+
+    @increment_total_submitted += to_submit.size
+    (normal, decoy) = partition(to_submit)
+    @normal_peps_just_submitted = normal
+    @increment_normal += normal.size
+    @increment_decoy += decoy.size
+    calc_precision(@increment_normal, @increment_decoy)
+  end
+
+  def pephit_precision(peps, separate_peps=nil)
+    if separate_peps
+      calc_precision(peps.size, separate_peps.size)
+    else
+      (norm, decoy) = partition(peps)
+      calc_precision(norm.size, decoy.size)
+    end
+  end
+
+  def to_param_string
+    "decoy="+ ["{constraint=#{(constraint ? constraint.inspect : '')}", "decoy_on_match=#{@decoy_on_match}", "correct_wins=#{@correct_wins}}"].join(", ")
+  end
+end
+
+

data/lib/validator/digestion_based.rb

@@ -0,0 +1,69 @@
+require 'validator'
+require 'fasta'
+require 'spec_id/sequest/params'
+
+# objects of this class can calculate pephit_precision given an array of
+# SpecID::Pep objects using the pephit_precision method.
+class Validator::DigestionBased < Validator
+  DEFAULTS = {
+    :false_to_total_ratio => 1.0,
+    :background => 0.0,
+  }
+
+  # the number of tps
+  attr_accessor :increment_tps
+  # the number of fps
+  attr_accessor :increment_fps
+
+  # the total peptides submitted to the validator (regardless of tp, fp, or
+  # nil)
+  attr_accessor :increment_total_submitted
+
+  # the ratio of false hits to total peptides in the fasta file
+  attr_accessor :false_to_total_ratio
+
+  # the false_to_total_ratio calculated (but not applied)
+  attr_reader :calculated_background
+
+  # For a sample with no false hits in it, (under defaults) this is the
+  # fraction of peptides with the constraint over the total number of peptides
+  # from which these hits are derived.
+  attr_accessor :background
+
+
+  # expects that classes define a partition method, and a @background
+  def pephit_precision(peps)
+    ## this gives us the fraction that are transmembrane (under defaults):
+    (tps, fps) = partition(peps)
+    (num_tps, num_fps) = calc_precision_prep(tps.size, fps.size)
+    calc_precision(num_tps, num_fps)
+  end
+
+  # returns [num_tps, num_fps]
+  def calc_precision_prep(num_tps, num_fps)
+    total_peps_passing_partition = num_tps + num_fps
+    num_fps = adjust_fps_for_background(num_tps, num_fps, @background)
+    ## we must use the false_to_total_ratio to estimate how many are really
+    ## incorrect!
+    # FALSE/TOTAL  = FALSE(found)/TOTAL(found)
+    # TOTAL(found) = FALSE(found) * TOTAL/FALSE
+    #              = FALSE(found) / (FALSE/TOTAL)
+    total_false = num_fps / @false_to_total_ratio
+    # NOTE: the partition algorithm drops peptides that are transmembrane
+    # under certain options.  Thus, the total false estimate must be tempered
+    # by this lower number of total peptides.
+    adjusted_tps = total_peps_passing_partition.to_f - total_false
+    [adjusted_tps, total_false]
+  end
+
+  # returns self
+  # assumes partition returns (tps, fps)
+  def set_false_to_total_ratio(peps)
+    (tps, fps) = partition(peps)
+    @false_to_total_ratio = fps.size.to_f / (tps.size + fps.size) 
+    self
+  end
+
+end
+
+

data/lib/validator/probability.rb

@@ -0,0 +1,48 @@
+
+class Validator::Probability
+
+  attr_accessor :prob_method
+
+  def initialize(prob_method=:probability)
+    @prob_method = prob_method
+  end
+
+  # objs should respond_to probability
+  def precision(objs)
+    return 1.0 if objs.size == 0
+
+    current_sum_one_minus_prob = 0.0
+
+    # this should work!
+    #objs.inject(0.0) {|sum,obj| sum + (1.0 - obj.probability) }
+
+    objs.each do |obj|
+      # SUM(1-probX)/#objs
+      current_sum_one_minus_prob += 1.0 - obj.send(@prob_method)
+    end
+    prec = 1.0 - (current_sum_one_minus_prob / objs.size)
+  end
+
+
+  # objs should respond_to probability
+  # These should be added from high probability(1.0) to low (0.0)
+  def increment_precision(objs)
+    if objs.is_a?(SpecID::Pep) or objs.is_a?(SpecID::Prot)
+      objs = [objs]
+    end
+
+    @total_objs ||= 0
+    @current_sum_one_minus_prob ||= 0.0
+
+    @total_objs += objs.size
+    objs.each do |obj|
+      @current_sum_one_minus_prob += 1.0 - obj.send(@prob_method)
+    end
+    prec = 1.0 - (@current_sum_one_minus_prob / @total_objs)
+  end
+
+
+  alias_method :pephit_precision, :precision
+  alias_method :prothit_precision, :precision
+  alias_method :increment_pephits_precision, :increment_precision
+end