mspire 0.2.4 → 0.3.0

data/lib/spec_id/precision/filter.rb

@@ -0,0 +1,623 @@
+require 'sort_by_attributes'
+require 'validator'
+require 'spec_id'
+require 'merge_deep'
+require 'spec_id/precision/filter/interactive'
+require 'spec_id/precision/filter/output'
+
+
+class Filter
+  
+  # filters using previously passed in methods and options
+  def filter(group)
+    if @opts
+      send(@method, group, *@opts)
+    else
+      send(@method, group)
+    end
+  end
+
+  # replaces the contents of group with what passed
+  def filter!(group)
+    group.replace(filter(group))
+  end
+end
+
+
+
+# we have to require this after we setup our defaults hash
+# require 'filter/spec_id/cmdline'
+
+class SpecID::Precision::Filter 
+  FV_DEFAULTS = {
+    :sequest => 
+    {
+      :xcorr1 => 1.0, 
+      :xcorr2 => 1.5,
+      :xcorr3 => 2.0,
+      :deltacn => 0.1,
+      :ppm => 1000,
+      :include_deltacnstar => true,
+    },
+
+
+    # output
+    :proteins => false,
+    :output => [],
+
+    # general
+    :top_hit_by => :xcorr,
+    :postfilter => :top_per_scan,
+    :prefilter => false,
+    :hits_together => true,
+
+    # These are also defaulted in the commandline because they are necessary
+    # for the validators...  could this introduce conflicts somehow?
+    :decoy_on_match => true,
+    :ties => true,
+
+    # UNLISTED FOR NOW:
+    :include_ties_in_top_hit_prefilter => true,
+    :include_ties_in_top_hit_postfilter => false,
+  }
+
+  require 'spec_id/precision/filter/cmdline'
+
+  def filter_and_validate_cmdline(args)
+    (spec_id_obj, options, option_parser) = CmdlineParser.new.parse(args)
+    if spec_id_obj == nil
+      puts option_parser
+      return
+    end
+    final_answer = SpecID::Precision::Filter.new.filter_and_validate(spec_id_obj, options)
+  end
+
+    #  # output_array has doublets of [format, handle]
+    #  # answer is the answer one gets out of filter_and_validate
+    #  def output(answer, output_array)
+    #    output_array.each do |format, handle|
+    #      SpecID::Precision::Filter::Output.new(format, handle)
+    #    end
+    #  end
+
+  # Very high level method that takes simple parameters.
+  # spec_id may be a filename or a SpecID object (containing peps)
+  # Default values may be queried from SpecID::Precision::Filter::FV_DEFAULTS
+  # Returns a structured hash: 
+  #  Fl = Float ; Ar = Array
+  # { :params => <Hash of filtering params>,
+  #   :pephits => <Ar of pephits>,
+  #   :pephits_precision => [<array of precision>]
+  #      # if :proteins => true
+  #   :prothits => <Array of prothits>,
+  #   :prothits_precision => [ Array of hashes where each hash =   
+  #                             { :worst => Fl, :normal => Fl,
+  #                             :normal_stdev => Fl } ]
+  # }
+  #
+  # NOTE: Brackets [] indicate an Array! The Bar '|' indicates another option.
+  # The asterik '*' is the default option.
+  #
+  # :sequest => {
+  #   :xcorr1 -> >= (xcorr +1 charge state)
+  #   :xcorr2 -> >= (xcorr +2 charge state)
+  #   :xcorr3 -> >= (xcorr +3 charge state)
+  #   :deltacn -> >= (delta cn)
+  #   :ppm -> <= parts per million (Float)
+  #   :include_deltacnstar => *true | false  include deltacn (given at 1.1) of
+  #                                          top hit with no 2nd hit
+  #
+  # }
+  # OUTPUT:
+  #   :proteins => true | *false    gives proteins (and validation)
+  #   :output => [[format, FILENAME=nil],...]  formats to output filtering results.
+  #                                            can be used multiple times
+  #                                            FILENAME is the filename to use
+  #                                            if nil, then outputs to $stdout
+  #                                            valid formats are:
+  #                                            :text_table   (default)
+  #                                            :yaml         (need to implement)
+  #                                            :protein_summary (need to implement)
+  #                                            :html_table   (need to implement)
+  #                                            default value =>
+  #                                            [[:text_table,nil]]
+  #
+  # VALIDATION:
+  #   :validators => [Array]  objects that respond to pephit_precision
+  #                           usually of base class Validator
+  #                           NOTE: if you have decoy peptides, you MUST have
+  #                           a Validator::Decoy object to separate them out.
+  #                           NOTE: if transmem validator passed in, the
+  #                           proteins in spec_id must already be granted
+  #                           transmem status!
+  #                           
+  #
+  # OTHER:
+  #   :top_hit_by -> *:xcorr | :probability
+  #                   probabilities only in bioworks.xml files right now (if
+  #                   they were calculated).
+  #   :postfilter -> *:top_per_scan | :top_per_aaseq | :top_per_aaseq_charge 
+  #                   :top_per_scan hashes by filename + scan
+  #                   :top_per_aaseq hashes by top_per_scan + aaseq
+  #                   :top_per_aaseq_charge hashes by top_per_aaseq + charge
+  #   :prefilter -> true | *false    Takes top hit per file+scan+charge
+  #   :interactive => interactive_object
+  #        # should behave like this:
+  #        # interactive_object.filter_args(currentopts) -> args_for_filtering | nil (done)
+  #        
+  #        # interactive_object.passing(final_answer)
+
+  # The defaults for filter_and_validate
+
+  def filter_and_validate(spec_id_obj, options={})
+    # NOTE:
+    # This is a fairly complicated method.  The complication comes in doing
+    # top hit filters on separate/cat searches wanted them to be either
+    # together or separate.  I opt for fewer conversions between the two, but
+    # that means keeping track of more things...
+    
+    opts = FV_DEFAULTS.merge_deep(options)
+
+    spec_id = spec_id_obj
+
+    peps = spec_id.peps
+    filename = spec_id.filename
+
+    #######################################
+    # DEFAULTS:
+    interactive_changing_keys = [:xcorr1, :xcorr2, :xcorr3, :deltacn, :ppm, :include_deltacnstar, :postfilter]
+    interactive_shortcut_map = {
+      :xcorr1 => 'x1',
+      :xcorr2 => 'x2',
+      :xcorr3 => 'x3',
+      :deltacn => 'dcn',
+      :ppm => 'ppm',
+      :include_deltacnstar => 'dcns',
+      :postfilter => 'pf',
+    }
+    to_float = proc {|x| x.to_f}
+    to_bool = proc do |x|
+      case x
+      when /^t/io
+        true
+      when /^f/io
+        false
+      when true
+        true
+      when false
+        false
+      else
+        nil
+      end
+    end
+    to_postfilter = proc do |x|
+      case x
+      when 's'
+        :top_per_scan
+      when 'a'
+        :top_per_aaseq
+      when 'ac'
+        :top_per_aaseq_charge
+      when Symbol
+        x
+      end
+    end
+    casting_map = {
+      :xcorr1 => to_float,
+      :xcorr2 => to_float,
+      :xcorr3 => to_float,
+      :deltacn => to_float,
+      :ppm => to_float,
+      :include_deltacnstar => to_bool,
+      :postfilter => to_postfilter,
+    }
+   
+    # output:  
+    # NOTE: BOOLEANS that are by default false do not need a default!!
+    # They will yield false on key lookup if no key or false!
+    # BOOLEANS that by default are true should be queried like this
+    # !(opts[:<option>] == false)
+
+    # open up each of the files for writing
+    if opts[:output]
+      outputs = opts[:output].map do |format, where|
+        if where == nil
+          where = $stdout
+        end
+        SpecID::Precision::Filter::Output.new(format, where)
+      end
+    end
+
+    postfilters_per_hash = {
+      :top_per_scan => [:base_name, :first_scan],
+      :top_per_aaseq => [:aaseq],  # first by top_per_scan, then this guy
+      :top_per_aaseq_charge => [:aaseq, :charge], # first by top_per_scan, then this one
+    }
+
+    top_hit_by__to_sort_by = {
+      :xcorr => [:xcorr, {:down=> [:xcorr]}],
+      :probability => [:probability, (spec_id.hi_prob_best ? {:down=> [:probability]} : {})],
+    }
+    sort_by_att_opts = top_hit_by__to_sort_by[opts[:top_hit_by]]
+    opts_for_top_hit_prefilter = {
+      :per => [:base_name, :first_scan, :charge], 
+      :by => sort_by_att_opts, 
+      :include_ties => opts[:include_ties_in_top_hit_prefilter]
+    }
+    # PRIVATE DEFAULTS:
+    merge_prefix = 'DECOY_'
+    unmerge_regexp = /^DECOY_/
+
+    #######################################
+
+
+    # opts_decoy = opts[:decoy]
+
+
+    
+    # if we have a Validator::Decoy object, we will use its defaults to split
+    # peptides.
+    decoy_validator = 
+      if opts[:validators]
+        decoy_vals = opts[:validators].select {|v| v.class == Validator::Decoy }
+        if decoy_vals.size == 0
+          nil
+        elsif decoy_vals.size == 1
+          decoy_vals.first
+        else
+          raise ArgumentError, "can only have one Validator::Decoy object"
+        end
+
+        ### suck out the relevant parameters
+        #sep_params = [:decoy_on_match, :correct_wins].inject({}) do |hash,k|
+        #  hash[k] = decoy_validator.send(k)
+        #  hash
+        #end
+      else
+        nil
+      end
+
+    decoy_validator_to_split_with = nil
+
+    pep_sets = 
+      if decoy_validator
+        if decoy_validator.constraint.is_a?(Regexp)
+          if opts[:hits_together]
+            decoy_validator_to_split_with = decoy_validator
+            [peps]
+          else
+            (target, decoy) = decoy_validator.partition(peps)
+            #(target, decoy) = SpecID.classify_by_prot(peps, opts_decoy, sep_params[:decoy_on_match], sep_params[:correct_wins])
+            [target, decoy]
+          end
+        elsif decoy_validator.constraint.is_a?(String)  ## a Filename 
+          decoy_peps = SpecID.new(decoy_validator.constraint).peps
+          
+          if opts[:hits_together]
+            # we fake that the protein sets are together
+            decoy_validator_to_split_with = Validator::Decoy.new(unmerge_regexp)
+            decoy_peps.each do |pep|
+              pep.prots.each {|prt| prt.reference = merge_prefix + prt.reference }
+            end
+            [peps + decoy_peps] # wrap them so we get the target out
+          else
+            [peps, decoy_peps]
+          end
+        else 
+          raise ArgumentError, "Decoy::Validator#constraint must be a Regexp or valid SpecID file"
+        end
+      else
+        [peps]  # no decoy
+      end
+
+    if opts[:proteins]
+      protein_validator = Validator::ProtFromPep.new
+    end
+
+    ### TOP HITS PREFILTER < < TOP_HITS_TOGETHER > >
+    ###########################
+    # TOP HITS FILTER:
+    ###########################
+    # REALLY, this guy only exists for speed and memory consumption
+    # If we prefilter, we don't have to filter as many hits in every
+    # interactive round.  I'd leave this guy out if I were doing only a
+    # sequest filter.  (I should compare results with this filter and w/o)
+    # This guy is very tricky since we need to consider whether they are to be
+    # run together or separately and not do more work than we need
+    # get passed_target for any case (and passed_decoy if opts[:decoy])
+
+    
+    top_hit_prefilter = SpecID::Precision::Filter::Peps.new(:top_hit, opts_for_top_hit_prefilter)  if opts[:prefilter]
+
+    if top_hit_prefilter
+      pep_sets.map! do |pep_set|
+        top_hit_prefilter.filter(pep_set)
+      end
+    end
+
+    # prepare our top hit filter:
+    # since we are now modulating this guy, we need to create it fresh every
+    # time
+    top_per_scan_postfilter = SpecID::Precision::Filter::Peps.new(:top_hit, 
+                                                  :per => postfilters_per_hash[:top_per_scan],
+                                                  :by => sort_by_att_opts, 
+                                                  :include_ties => opts[:include_ties_in_top_hit_postfilter])
+
+
+
+    # Prepare to loop
+    # Give interactive help once here if necessary
+    interactive = opts[:interactive]
+    if interactive
+      ARGV.clear
+      interactive.out(interactive.interactive_help(interactive_changing_keys, interactive_shortcut_map)) if interactive.verbose
+    end
+
+    # the loop is for if we are interactive
+    final_answer = nil
+    loop do  
+
+      if interactive #interactive
+        # a bit of a hack, but we shove on the postfilter param to modulate
+        opts[:sequest][:postfilter] = opts[:postfilter]
+        response = interactive.filter_args(opts[:sequest], interactive_changing_keys, interactive_shortcut_map, casting_map)
+        opts[:postfilter] = opts[:sequest].delete(:postfilter)
+        break if response == nil
+      end
+
+      # prepare our top hit filter:
+      # since we are now modulating this guy, we need to create it fresh every
+      # time
+      
+      sub_postfilter = 
+        if opts[:postfilter] == :top_per_scan
+          nil
+        else
+          postfilter_per_args = postfilters_per_hash[opts[:postfilter]]
+          SpecID::Precision::Filter::Peps.new(:top_hit, 
+                                   :per => postfilter_per_args,
+                                   :by => sort_by_att_opts, 
+                                   :include_ties => opts[:include_ties_in_top_hit_postfilter]
+                                  )
+        end
+
+      pep_sets_to_be_filtered = pep_sets.map 
+
+      ### SEQUEST < EITHER >
+      ###########################
+      # SEQUEST FILTER:
+      ###########################
+      # This guy is immune to the trickiness of top hits, so we just filter
+      # separately since validation is best done without decoys (except decoy)
+      sequest_args = opts[:sequest].values_at( :xcorr1, :xcorr2, :xcorr3, :deltacn, :ppm, :include_deltacnstar )
+      sequest_filter = SpecID::Precision::Filter::Peps.new(:standard_sequest_filter, *sequest_args)
+
+      pep_sets_filtered = pep_sets_to_be_filtered.map do |pep_set|
+        sequest_filter.filter(pep_set)
+      end
+
+      ### FINAL HIT PER SCAN < < TOP_HITS_TOGETHER > >
+      ##########################
+      # FINAL HIT PER SCAN
+      ##########################
+      # Why not just do the top hit filter in the top hits pre filter before?
+      # Good question.  Answer: We may have instances when the top hit (by
+      # xcorr) has some other poorer attribute than the hit at the other charge.
+      # In this case, we'd end up with no passing peptide. 
+      # Also, the xcorr filter is per charge, so we may filter out the higher
+      # scoring peptide hit even though the other would pass based on its charge
+      # state, etc., etc....
+      # ###################################################
+      # NOTE THIS WELL:
+      # IF IT IS SUPPOSE TO be separate it's *ALREADY* separate, if together its
+      # *ALREADY* together!!!!
+      # the implication is that we don't need to do any merging or
+      # separating before we do this last filter!!!!
+      # ###################################################
+
+      # TODO: We need to add this guy in!
+      #if opts[:uniq_aa]
+      #  pep_sets_filtered.map do |pep_set|
+      #  end
+      #end
+
+      pep_sets_filtered.map! do |pep_set|
+        top_per_scan_postfilter.filter!(pep_set)
+        if sub_postfilter
+          sub_postfilter.filter!(pep_set) 
+        else
+          pep_set
+        end
+      end
+
+      normal_post_filtered_peps = pep_sets_filtered.first
+
+      # separate the decoy's out if they are together
+      if decoy_validator_to_split_with  # only set if opts[:hits_together]!!
+        (target, decoy) = decoy_validator_to_split_with.partition(normal_post_filtered_peps)
+        pep_sets_filtered = [target, decoy]
+      end
+
+      ### VALIDATION < SEPARATE >
+      pephit_precision_array = get_pephit_precision(opts[:validators], *pep_sets_filtered) if opts[:validators]
+
+      final_answer = {
+        :params => opts,
+        :pephits => pep_sets_filtered.first, 
+      }
+      if pephit_precision_array
+        final_answer[:pephits_precision] = pephit_precision_array
+      end
+
+      if opts[:proteins]       
+        protein_precision_array = peptide_precision_to_protein_precision(protein_validator, normal_post_filtered_peps, pephit_precision_array)
+        # this could be factored out (since we do it in protein_precision)
+
+        # merge the final prots into a unique set:
+        final_answer[:prothits] = normal_post_filtered_peps.inject(Set.new) do |protset, pep|
+          protset.merge(pep.prots)
+        end
+        final_answer[:prothits_precision] = protein_precision_array
+      end
+
+      ## output the output
+      outputs.each {|output| output.print(final_answer) }
+
+      if interactive
+        interactive.passing(opts, final_answer)
+      end
+
+      if !interactive
+        break
+      end
+    end
+    # Close the filehandles
+    outputs.each { |output| output.close } if opts[:output]
+    final_answer
+  end
+
+  # takes peps and a peptide_precision_hash.  Returns a hash with the same
+  # keys of peptide_precision_hash where the value is a hash with these keys:
+  #   :worst => worstcase protein precision
+  #   :normal => estimaton by binomial/gaussian method (optimistic)
+  #   :normal_stdev => the stdev of the normal method
+  def peptide_precision_to_protein_precision(protein_validator, peps, peptide_precision_array, round_num_false=:ceil)
+    peptide_precision_array.map do |precision|
+      num_false = ((1.0 - precision) * peps.size).ceil
+      reply = protein_validator.prothit_precision(peps, num_false)
+      hash = {}
+      %w(worst normal normal_stdev).zip(reply) do |label, answer|
+        hash[label.to_sym] = answer
+      end
+      hash
+    end
+  end
+
+  # takes an array of validator objects and peps (already separated out from
+  # decoys; the decoy's can be passed in
+  # returns an array of results
+  def get_pephit_precision(validators, peps, decoy_peps=nil, grant_transmem_status=false)
+    validators.map do |validator|
+      if validator.class == Validator::Decoy
+        validator.pephit_precision(peps, decoy_peps)
+      else
+        validator.pephit_precision(peps)
+      end
+    end
+  end
+end
+
+class SpecID::Precision::Filter::Peps < Filter
+
+  # can pass in the method to call.  If you have static options and you will
+  # reuse your filter, you can pass them in here.
+  # BEWARE: this will override any passed into the method at filter time.
+  # If you need to do that, make a new, blank filter and pass in your args
+  # at filter time
+  def initialize(meth=nil, *opts)
+    @method = meth
+    if opts.size > 0
+      @opts = opts
+    else
+      @opts = nil
+    end
+  end
+
+  # passes the top peptide hits per attributes that it is hashed by
+  # all hits with same score as top score are returned
+  # assumes that all attributes are cast properly: Float,Integer, etc
+  # converts xcorr, deltacn, deltamass, mass, and charge into numerical types
+  # deletes the protein array (but not relevant proteins)
+  # hashes on [pep.basename, pep.first_scan.to_i, pep.charge.to_i]
+  # returns self for chaining
+  # opts 
+  #   :per => Array of attributes e.g. [:first_scan, :charge]   # TODO: allow lambda
+  #   :by  => an array for sort_by_attributes 
+  #           e.g. [:xcorr, :deltacn, :ppm, {:down => [:xcorr, :deltacn]}]
+  #   :ties => *false | true | :as_array
+  #             false -     one top hit is selected by random (by sorting)
+  #             true  -     all ties are included in final answer
+  #             :as_array - ties are included as an array 
+  def top_hit(peps, opts = {})
+
+    # get the top peptide by firstscan/charge (equivalent to .out files)
+    top_peps = []
+    #hash = peps.hash_by(*(opts[:per]))
+    per_array = opts[:per]
+    hash = peps.hash_by(*per_array)
+    ties = opts[:ties]
+    if ties == :as_array
+      as_array = true
+    end
+    hash.values.each do |v|
+      best_to_worst = v.sort_by_attributes(*(opts[:by]))
+      if ties
+
+        best_hit = best_to_worst.first
+        ## get the values that matter for the top hit 
+        # here get the attributes we are considering
+        atts =
+          if opts[:by].last.is_a? Hash
+            opts[:by][0...-1]
+          else
+            opts[:by].dup
+          end
+        # find the best hits values
+        top_hit_vals = atts.map do |att| 
+          best_hit.send(att)        
+        end
+
+        tying_peps = []
+        best_to_worst.each do |pep|
+          tie = true
+          atts.each_with_index do |att,i|
+            unless (pep.send(att) == top_hit_vals[i])
+              tie = false
+              break 
+            end
+          end
+          if tie
+            tying_peps << pep
+          else
+            break
+          end
+        end
+        if as_array
+          if tying_peps.size == 1
+            top_peps.push( *tying_peps )
+          else
+            top_peps.push( tying_peps )
+          end
+        else
+          top_peps.push( *tying_peps )
+        end
+      else
+        top_peps << best_to_worst.first
+      end
+    end
+    top_peps
+  end
+
+  # returns self for chaining
+  def standard_sequest_filter(peps, x1,x2,x3,deltacn,ppm,include_deltacnstar=true)
+    peps.select do |pep|
+      pep_deltacn = pep.deltacn
+      pep_charge = pep.charge
+
+      ## The outer parentheses are critical to getting the correct answer!
+      _passing = ( (pep_deltacn >= deltacn) and ((pep_charge == 1 && pep.xcorr >= x1) or (pep_charge == 2 && pep.xcorr >= x2) or (pep_charge == 3 && pep.xcorr >= x3)) and ( pep.ppm <= ppm ))
+
+      if _passing
+        if ((!include_deltacnstar) && (pep_deltacn > 1.0))
+          false
+        else
+          true
+        end
+      else
+        false
+      end
+    end
+  end
+
+end
+

data/lib/spec_id/precision/output.rb

@@ -0,0 +1,60 @@
+
+
+module SpecID ; end
+module SpecID::Precision ; end
+
+module SpecID::Precision::Output
+
+  # takes a format type (as symbol) and the handle to write to
+  # if handle_or_file is a file, will open it and close (on calling close)
+  # if it is a handle, will not close it
+  def initialize(format, handle_or_file)
+    @handle = 
+      if handle_or_file.is_a? String
+        @need_to_close = true
+        File.open(handle_or_file, 'w')
+      else
+        @need_to_close = false
+        handle_or_file
+      end
+    @format = format
+  end
+
+  # returns self
+  def print(answer)
+    send( @format, @handle, answer )
+    self
+  end
+
+  # turns all keys that are symbols into strings (recursively into *Hashes*)
+  def self.symbol_keys_to_string(hash)
+    new_hash = {}
+    hash.each do |k,v|
+      new_value = 
+        if v.is_a? Hash
+          symbol_keys_to_string(v)
+        else
+          v
+        end
+      if k.is_a? Symbol
+        new_hash[k.to_s] = new_value
+      else
+        new_hash[k] = new_value
+      end
+    end
+    new_hash
+  end
+
+    # TODO: implement recursively, this has just grown and grown terribly
+  def hash_as_string(hash)
+    hash.inspect
+  end
+
+   # will close the handle if it is a File object
+  def close
+    if @need_to_close
+      @handle.close 
+    end
+  end
+
+end