mspire 0.4.5 → 0.4.7

data/changelog.txt

@@ -213,3 +213,11 @@ difference between good hits and bad hits
 1. using pi_zero instead of decoy_to_target_ratio.  While all tests are
 passing, this release should be considered experimental with the use of any
 target-decoy validation.
+
+## version 0.4.6
+1. added NOTE to --to_qvalues option to include all results (no low prob
+filter)
+
+## version 0.4.7
+1. Added ability to quickly grab sequest params out of a .SRF file
+2. Added helpful runtime error if print_duplicate_references is 0.

data/lib/mspire.rb

@@ -1,4 +1,4 @@
 
 module Mspire
-  Version = '0.4.5'
+  Version = '0.4.7'
 end

data/lib/pi_zero.rb

@@ -1,5 +1,4 @@
 require 'rsruby'
-require 'gsl'
 require 'vec'
 require 'vec/r'
 require 'enumerator'
@@ -21,7 +20,8 @@ module PiZero
       pi_zeros = []                # pi_0
       total = sorted_pvals.size  # m 
 
-      # totally retarded implementation with correct logic:
+      # totally inefficient implementation (with correct logic):
+      # TODO: implement this efficiently
       start.step(stop, step) do |lam|
         lambdas << lam
         (greater, less) = sorted_pvals.partition {|pval| pval > lam }
@@ -30,6 +30,36 @@ module PiZero
       [lambdas, pi_zeros]
     end
 
+=begin
+    def plateau_height_with_gsl(x, y)
+      require 'gsl'
+      x_deltas = (0...(x.size-1)).to_a.map do |i|
+        x[i+1] - x[i]
+      end
+      y_deltas = (0...(y.size-1)).to_a.map do |i|
+        y[i+1] - y[i]
+      end
+      new_xs = x.dup
+      new_ys = y.dup
+      x_deltas.reverse.each do |delt|
+        new_xs.push( new_xs.last + delt )
+      end
+
+      y_cnt = y.size
+      y_deltas.reverse.each do |delt|
+        y_cnt -= 1
+        new_ys.push( y[y_cnt] - delt )
+      end
+
+      x_vec = GSL::Vector.alloc(new_xs)
+      y_vec = GSL::Vector.alloc(new_ys)
+      coef, cov, chisq, status = GSL::Poly.fit(x_vec,y_vec, 3)
+      coef.eval(x.last)
+      #x2 = GSL::Vector::linspace(0,2.4,20)
+      #graph([x_vec,y_vec], [x2, coef.eval(x2)], "-C -g 3 -S 4")
+    end
+=end
+
     # expecting x and y to make a scatter plot descending to a plateau on the
     # right side (which is assumed to be of increasing noise as it goes to the
     # right)
@@ -42,46 +72,19 @@ module PiZero
     #          ** ***         *    *
     #                    ***** **** ***
     def plateau_height(x, y)
-=begin
-    require 'gsl'
-    x_deltas = (0...(x.size-1)).to_a.map do |i|
-      x[i+1] - x[i]
-    end
-    y_deltas = (0...(y.size-1)).to_a.map do |i|
-      y[i+1] - y[i]
-    end
-    new_xs = x.dup
-    new_ys = y.dup
-    x_deltas.reverse.each do |delt|
-      new_xs.push( new_xs.last + delt )
-    end
-
-    y_cnt = y.size
-    y_deltas.reverse.each do |delt|
-      y_cnt -= 1
-      new_ys.push( y[y_cnt] - delt )
-    end
-
-    x_vec = GSL::Vector.alloc(new_xs)
-    y_vec = GSL::Vector.alloc(new_ys)
-    coef, cov, chisq, status = GSL::Poly.fit(x_vec,y_vec, 3)
-    coef.eval(x.last)
-    #x2 = GSL::Vector::linspace(0,2.4,20)
-    #graph([x_vec,y_vec], [x2, coef.eval(x2)], "-C -g 3 -S 4")
-=end
-
       r = RSRuby.instance
       answ = r.smooth_spline(x,y, :df => 3)
       ## to plot it!
-      #r.plot(x,y, :ylab=>"instantaneous pi_zeros")
-      #r.lines(answ['x'], answ['y'])
-      #r.points(answ['x'], answ['y'])
-      #sleep(8)
+      r.plot(x,y, :ylab=>"pi_zeros or frit")
+      r.lines(answ['x'], answ['y'])
+      r.points(answ['x'], answ['y'])
+      sleep(4)
 
       answ['y'].last
     end
 
     def plateau_exponential(x,y)
+      require 'gsl'
       xvec = GSL::Vector.alloc(x)
       yvec = GSL::Vector.alloc(y)
       a2, b2, = GSL::Fit.linear(xvec, GSL::Sf::log(yvec))
@@ -93,9 +96,10 @@ module PiZero
 
     end
 
-    # returns a conservative (but close) estimate of pi_0 given sorted p-values
+    # returns a conservative (but close) estimate of pi_0 given p-values
     # following Storey et al. 2003, PNAS.
-    def pi_zero(sorted_pvals)
+    def pi_zero(pvals)
+      sorted_pvals = pvals.sort
       plateau_height( *(pi_zero_hats(sorted_pvals)) )
     end
 
@@ -161,6 +165,8 @@ module PiZero
       p_values(target_hits.map {|v| v.xcorr}, new_decoy_vals )
     end
 
+#### NEED TO VERIFY if this is PIT or PI_ZERO!
+=begin
     # takes a list of booleans with true being a target hit and false being a
     # decoy hit and returns the pi_zero using the smooth method
     # Should be ordered from best to worst (i.e., one expects more true values
@@ -184,12 +190,32 @@ module PiZero
       ys.reverse!
       plateau_height(xs, ys)
     end
+=end
+
+    # returns fraction of incorrect target hits (frit) (this is the percent
+    # incorrect targets [PIT] expressed as a fraction rather than percent)
+    # takes two parallel arrays consisting of the total number of hits (this
+    # will typically be the total # target hits) at that point and the
+    # precision (ranging from: [0,1]) (typically determined by counting the
+    # number of decoy hits).  Expects the number of total hits to be
+    # monotonically increasing and the precision to roughly start high and
+    # decrease as more hits (of lesser quality) are added.
+    def frit_from_precision(total_num_hits_ar, precision_ar)
+      instant_pi_zeros = []
+      total_num_hits_ar.reverse.zip(precision_ar.reverse).each_cons(2) do |dp1, dp0|
+        (x1, y1) = dp1
+        (x0, y0) = dp0
+        instant_pi_zeros << ((x1 * (1.0 - y1)) - (x0 * (1.0 - y0) )) / (x1 - x0)
+      end
+      instant_pi_zeros.reverse!
+      plateau_height(total_num_hits_ar[1..-1], instant_pi_zeros)
+    end
 
     # Takes an array of doublets ([[int, int], [int, int]...]) where the first
     # value is the number of target hits and the second is the number of decoy
     # hits.  Expects that best hits are at the beginning of the list.  Assumes
-    # that each sum is a subset
-    # of the following group (shown as actual hits rather than number of hits):
+    # that each sum is a subset of the following group (shown as actual hits
+    # rather than number of hits):
     #
     #    [[target, target, target, decoy], [target, target, target, decoy,
     #    target, decoy, target], [target, target, target, decoy, target,
@@ -197,31 +223,22 @@ module PiZero
     #
     # This assumption may be relaxed somewhat and should still give good
     # results.
-    def pi_zero_from_groups(array_of_doublets)
-      pi_zeros = []
+    def frit_from_groups(array_of_doublets)
+      frits = []
       array_of_doublets.reverse.each_cons(2) do |two_doublets|
         bigger, smaller = two_doublets
-        bigger[0] = bigger[0] - smaller[0]
-        bigger[1] = bigger[1] - smaller[1]
-        bigger.map! {|v| v < 0 ? 0 : v }
-        if bigger[1] > 0
-          pi_zeros << (bigger[0].to_f / bigger[1])
+        num_targets = bigger[0] - smaller[0] 
+        num_decoy = bigger[1] - smaller[1]
+        num_targets = 0 if num_targets < 0
+        num_decoy = 0 if num_targets < 0
+        if num_decoy > 0
+          frits << (num_targets.to_f / num_decoy)
         end
       end
-      pi_zeros.reverse!
-      xs = (0...(pi_zeros.size)).to_a
-      plateau_height(xs, pi_zeros)
+      frits.reverse!
+      xs = (0...(frits.size)).to_a
+      plateau_height(xs, frits)
     end
 
   end
-
-
-end
-
-if $0 == __FILE__
-  #xcorrs = IO.readlines("/home/jtprince/xcorr_hist/all_xcorrs.yada").first.chomp.split(/\s+/).map {|v| v.to_f }
-  #PiZero.p_values_for_sequest(
-  #File.open("newtail.yada", 'w') {|out| out.puts new_dist.join(" ") }
-  
-
 end

data/lib/qvalue.rb

@@ -79,41 +79,50 @@ class VecD
     end
 
     pi_zeros = lambda_vals.map {|val| self.pi_zero_at_lambda(val) }
-    if lambda_vals.size == 1
-      pi_zeros.first
-    else
-      case method
-      when :smooth
-        r = RSRuby.instance
-        calc_pi_zero = lambda do |_pi_zeros| 
-          hash = r.smooth_spline(lambda_vals, _pi_zeros, :df => Default_smooth_df) 
-          hash['y'][VecD.new(lambda_vals).max_indices.max]
-        end
-        if log_transform
-          pi_zeros.log_space {|log_vals| calc_pi_zero.call(log_vals) }
-        else
-          calc_pi_zero.call(pi_zeros)
-        end
-      when :bootstrap
-        min_pi0 = pi_zeros.min
-        lsz = lambda_vals.size
-        mse = VecD.new(lsz, 0)
-        pi0_boot = VecD.new(lsz, 0)
-        sz = self.size
-        100.times do   #  for(i in 1:100) {
-          p_boot = self.shuffle
-          (0...lsz).each do |i|
-            pi0_boot[i] = ( p_boot.select{|v| v > lambda_vals[i] }.size.to_f/p_boot.size ) / (1-lambda_vals[i])
+
+    r = RSRuby.instance
+    r.plot(lambda_vals,pi_zeros, :ylab=>"instantaneous pi_zeros")
+    answ = r.smooth_spline(lambda_vals, pi_zeros, :df => Default_smooth_df)
+    r.lines(answ['x'], answ['y'])
+    r.points(answ['x'], answ['y'])
+    sleep(20)
+
+    answer = 
+      if lambda_vals.size == 1
+        pi_zeros.first
+      else
+        case method
+        when :smooth
+          r = RSRuby.instance
+          calc_pi_zero = lambda do |_pi_zeros| 
+            hash = r.smooth_spline(lambda_vals, _pi_zeros, :df => Default_smooth_df) 
+            hash['y'][VecD.new(lambda_vals).max_indices.max]
+          end
+          if log_transform
+            pi_zeros.log_space {|log_vals| calc_pi_zero.call(log_vals) }
+          else
+            calc_pi_zero.call(pi_zeros)
           end
-          mse = mse + ( (pi0_boot-min_pi0)**2 )
+        when :bootstrap
+          min_pi0 = pi_zeros.min
+          lsz = lambda_vals.size
+          mse = VecD.new(lsz, 0)
+          pi0_boot = VecD.new(lsz, 0)
+          sz = self.size
+          100.times do   #  for(i in 1:100) {
+            p_boot = self.shuffle
+            (0...lsz).each do |i|
+              pi0_boot[i] = ( p_boot.select{|v| v > lambda_vals[i] }.size.to_f/p_boot.size ) / (1-lambda_vals[i])
+            end
+            mse = mse + ( (pi0_boot-min_pi0)**2 )
+          end
+          #  pi0 <- min(pi0[mse==min(mse)])
+          pi_zero = pi_zeros.values_at(*(mse.min_indices)).min
+          [pi_zero,1].min
+        else 
+          raise ArgumentError, ":pi_zero_method must be :smooth or :bootstrap!"
         end
-        #  pi0 <- min(pi0[mse==min(mse)])
-        pi_zero = pi_zeros.values_at(*(mse.min_indices)).min
-        [pi_zero,1].min
-      else 
-        raise ArgumentError, ":pi_zero_method must be :smooth or :bootstrap!"
       end
-    end
   end
 
   # Returns a VecD filled with parallel q-values

data/lib/spec_id/precision/prob/cmdline.rb

@@ -16,7 +16,10 @@ module SpecID
           :to_qvalues => ['--to_qvalues', "transform probabilities into q-values",
                                        "(includes pi_0 correction)",
                                        "uses PROB [TYPE] if given and supercedes",
-                                       "the prob validation type"],
+                                       "the prob validation type",
+                                       "*NOTE: include all PeptideProphet results",
+                                       "(don't use any low prob cutoff) for",
+                                       "accurate results!"],
           :prob => ['--prob [TYPE]', "use prophet probabilites to calculate precision",
                                      "TYPE = nsp [default] prophet nsp",
                                      "     (nsp also should be used for PeptideProphet results)",

data/lib/spec_id/precision/prob.rb

@@ -226,7 +226,7 @@ class SpecID::Precision::Prob
     else
       out[:probabilities] = probabilities
     end
-    out[:pephits] = ordered_peps  # just in case they want to see
+    # out[:pephits] = ordered_peps  # just in case they want to see
     out[:count] = num_pephits
     out[:aaseqs] = pepstrings
     out[:charges] = pepcharges

data/lib/spec_id/srf.rb

@@ -73,6 +73,7 @@ class SRFGroup
     IO.readlines(file).grep(/\w/).map {|v| v.chomp } 
   end
 
+
   # if srfs were read in separately, then the proteins will need to be merged
   # by their reference
   def merge_different_sets(srfs)
@@ -184,6 +185,20 @@ class SRF
 
   attr_accessor :filtered_by_precursor_mass_tolerance 
 
+  # returns a Sequest::Params object
+  def self.get_sequest_params(filename)
+    # split the file in half and only read the second half (since we can be
+    # confident that the params file will be there!)
+    File.open(filename) do |handle|
+      halfway = handle.stat.size / 2
+      handle.seek halfway
+      last_half = handle.read
+      params_start_index = last_half.rindex('[SEQUEST]') + halfway
+      handle.seek(params_start_index)
+      Sequest::Params.new.parse_handle(handle)
+    end
+  end
+
   def dta_start_byte
     case @version
     when '3.2' ; 3260
@@ -468,10 +483,24 @@ class SRF
 
   # returns self
   def from_file(filename, peps, global_ref_hash)
+    dups = SRF.get_sequest_params(filename).print_duplicate_references
+    if dups == '0'
+      raise RuntimeError, <<END
+
+***************************************************************************
+Sorry, but the SRF reader cannot read this file!
+.srf files must currently be created with print_duplicate_references > 0
+(This is how the srf object can link peptides with proteins!)
+To capture all duplicate references, set the sequest parameter
+'print_duplicate_references' to 100 or greater.
+***************************************************************************
+END
+    end
 
     File.open(filename, "rb") do |fh|
       @header = SRF::Header.new.from_handle(fh)      
       @version = @header.version
+      
       unpack_35 = case @version
                   when '3.2'
                     false
@@ -824,6 +853,7 @@ class SRF::OUT::Pep
   end
 
   def self.read_extra_references(fh, num_extra_references, pep_hits, global_ref_hash)
+    p num_extra_references
     num_extra_references.times do
       # 80 bytes total (with index number)
       pep = pep_hits[fh.read(8).unpack('x4I').first - 1]

data/lib/validator/cmdline.rb

@@ -41,7 +41,7 @@ class Validator::Cmdline
     {
       :hits_together => true,
       :decoy_on_match => true,
-      :pi_zero => 1.0,
+      :frit => 1.0, # fraction incorrect targets (like PIT)
     },
     :bad_aa => 
     {
@@ -61,7 +61,7 @@ class Validator::Cmdline
     :ties => true,
   }
   COMMAND_LINE = {
-    :decoy => ["--decoy /REGEXP/|FILENAME[,PI0,DOM]", Array, "REGEXP for decoy proteins (catenated searches) or a",
+    :decoy => ["--decoy /REGEXP/|FILENAME[,PIT,DOM]", Array, "REGEXP for decoy proteins (catenated searches) or a",
                                                 "FILENAME of separate search on decoys.", 
                                                 "All regular expressions must be surrounded by '/'",
                                                 "(no extended options [trailing modifiers]).",
@@ -72,7 +72,8 @@ class Validator::Cmdline
                                                 "    --decoy '/^\\s*REVERSE/'",
                                                 "If decoys proteins were searched in a separate file,",
                                                 "then give the FILENAME (e.g., --decoy decoy.srg)",
-                                                "PI0 = Incorrect Targets to Decoy Ratio (default: #{DEFAULTS[:decoy][:pi_zero]})",
+                                                "FRIT = Fraction Incorrect Targets (like",
+                                                "the PIT as a fraction) (default: #{DEFAULTS[:decoy][:frit]})",
                                                 "DOM = *true/false, decoy on match",],
         :tps => ["--tps <fasta>", "for a completely defined sample, this is the",
                                   "fasta file containing the true protein hits"],
@@ -156,7 +157,7 @@ class Validator::Cmdline
             raise ArgumentError, "File does not exist: #{first_arg}\n(was this supposed to be a regular expression? if so, should be given: /#{first_arg}/)" unless File.exist?(first_arg)
             first_arg
           end
-        val_opts[:pi_zero] = (ar[1] || DEFAULTS[:decoy][:pi_zero]).to_f
+        val_opts[:frit] = (ar[1] || DEFAULTS[:decoy][:frit]).to_f
         val_opts[:decoy_on_match] = self.boolean(ar[2], DEFAULTS[:decoy][:decoy_on_match])
         myargs.push(val_opts)
         opts[:validators].push(myargs)
@@ -295,6 +296,7 @@ class Validator::Cmdline
             val = 1e-9 if val == 0
             val
           end
+          File.open("TMP_PVALUES.txt", 'w') {|v| v.puts pvals.sort.join(" ") }
           pvals = VecD.new(pvals)
           #qvals = pvals.qvalues(false, :lambda_vals => 0.30 )
           qvals = pvals.qvalues

data/lib/validator/decoy.rb

@@ -9,10 +9,11 @@ class Validator::Decoy < Validator
   attr_accessor :decoy_on_match
   attr_accessor :correct_wins
   # This is the the number of incorrect target hits over the total decoy hits
-  # The very rough, conservative ballpark estimate is the ratio of target hits
-  # to decoy hits.  This can be refined by removing the number of true target
+  # The percent incorrect targets (PIT) expressed as a fraction (== 1 - PI_0).
+  # The rough, conservative ballpark estimate is the ratio of target hits to
+  # decoy hits.  This can be refined by removing the number of true target
   # hits from the targets used to calculate it.
-  attr_accessor :pi_zero
+  attr_accessor :frit
 
   attr_accessor :last_pep_was_decoy
 
@@ -25,12 +26,12 @@ class Validator::Decoy < Validator
   DEFAULTS = {
     :decoy_on_match => true,
     :correct_wins => true,
-    :pi_zero => 1.0,
+    :frit => 1.0,
   }
 
   def initialize(opts={})
     merged = DEFAULTS.merge(opts)
-    @constraint, @decoy_on_match, @correct_wins, @pi_zero = merged.values_at(:constraint, :decoy_on_match, :correct_wins, :pi_zero)
+    @constraint, @decoy_on_match, @correct_wins, @frit = merged.values_at(:constraint, :decoy_on_match, :correct_wins, :frit)
   end
 
   # returns [normal, decoy] (?? I think ??)
@@ -86,15 +87,15 @@ class Validator::Decoy < Validator
     @normal_peps_just_submitted = normal
     @increment_normal += normal.size
     @increment_decoy += decoy.size
-    calc_precision(@increment_normal, @increment_decoy, @pi_zero)
+    calc_precision(@increment_normal, @increment_decoy, @frit)
   end
 
   def pephit_precision(peps, separate_peps=nil)
     if separate_peps
-      calc_precision(peps.size, separate_peps.size, @pi_zero)
+      calc_precision(peps.size, separate_peps.size, @frit)
     else
       (norm, decoy) = partition(peps)
-      calc_precision(norm.size, decoy.size, @pi_zero)
+      calc_precision(norm.size, decoy.size, @frit)
     end
   end
 

data/lib/validator.rb

@@ -167,11 +167,11 @@ end
 # normal hits (which may be true or false) and the second are decoy hits.
 # edge case:  if num_normal.to_f == 0.0 then if num_decoy.to_f > 0 ; 0, else 1
 module Precision::Calculator::Decoy
-  def calc_precision(num_normal, num_decoy, pi_zero=1.0)
+  def calc_precision(num_normal, num_decoy, frit=1.0)
     # will calculate as floats in case fractional amounts passed in for
     # whatever reason
     num_normal_f = num_normal.to_f
-    num_true_pos = num_normal_f - (num_decoy.to_f * pi_zero)
+    num_true_pos = num_normal_f - (num_decoy.to_f * frit)
     precision =
       if num_normal_f == 0.0
         if num_decoy.to_f > 0.0

data/specs/pi_zero_spec.rb

@@ -18,7 +18,7 @@ describe PiZero do
     @sorted_pvals = [0.0, 0.1, 0.223, 0.24, 0.55, 0.68, 0.68, 0.90, 0.98, 1.0]
   end
 
-  it 'calculates instantaneous pi_0 hats' do
+  xit 'calculates instantaneous pi_0 hats' do
     answ = PiZero.pi_zero_hats(@sorted_pvals, :step => 0.1)
     exp_lambdas =       [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
     passing_threshold = [9,   8,   8,   6,   6,   6,   5,   3,   3,   2]
@@ -36,7 +36,7 @@ describe PiZero do
     # still working on this one
   end
 
-  it 'can find a plateau height' do
+  xit 'can find a plateau height' do
     x = [0.0, 0.01, 0.012, 0.13, 0.2, 0.3, 0.4, 0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2]
     y = [1.0, 0.95, 0.92, 0.8, 0.7, 0.6, 0.55, 0.58, 0.62, 0.53, 0.54, 0.59, 0.4, 0.72]
     z = PiZero.plateau_height(x,y)
@@ -47,7 +47,7 @@ describe PiZero do
     #sleep(8)
   end
 
-  it 'can calculate p values for SEQUEST hits' do
+  xit 'can calculate p values for SEQUEST hits' do
     class FakeSequest ; attr_accessor :xcorr ; def initialize(xcorr) ; @xcorr = xcorr ; end ; end
 
     target = []
@@ -68,13 +68,13 @@ describe PiZero do
     end
   end
 
-  it 'can calculate pi zero for target/decoy booleans' do
+  xit 'can calculate pi zero for target/decoy booleans' do
     pi_zero = PiZero.pi_zero_from_booleans(@bools)
     # frozen
     pi_zero.should be_close(0.03522869, 0.0001)
   end
 
-  it 'can calculate pi zero for groups of hits' do
+  it 'can calculate frit for groups of hits' do
     # setup
     targets = [4,3,8,3,5,3,4,5,4]
     decoys = [0,2,2,3,5,7,8,8,8]
@@ -95,9 +95,20 @@ describe PiZero do
       decoys_summed << sum
     end
     zipped = targets_summed.zip(decoys_summed)
-    pi_zero = PiZero.pi_zero_from_groups(zipped)
+    frit = PiZero.frit_from_groups(zipped)
     # frozen
-    pi_zero.should be_close(0.384064, 0.00001)
+    frit.should be_close(0.384064, 0.00001)
+  end
+
+  xit 'can calcuate pi zero for total number of hits and precision' do
+    tot_hits = [1,10,20,30,50,200]
+    precision = [1.0, 1.0, 0.85, 0.80, 0.7, 0.5]
+    reply = PiZero.frit_from_precision(tot_hits, precision)
+    puts "ANSER"
+    # frozen
+    puts reply
+    #reply.should be_close()
+
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: mspire
 version: !ruby/object:Gem::Version 
-  version: 0.4.5
+  version: 0.4.7
 platform: ruby
 authors: 
 - John Prince
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-09-25 00:00:00 -06:00
+date: 2008-11-06 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency