mspire 0.2.4 → 0.3.0

data/lib/spec_id/srf.rb

@@ -1,7 +1,12 @@
+require 'spec_id'
 require 'spec_id/sequest'
+require 'fasta'
+require 'mspire'
+require 'set'
+require 'fasta'
 
 module BinaryReader
-  Null_char = "\0"[0]  ## change for ruby 1.9 or 2.0
+  Null_char = "\0"[0]  ## TODO: change for ruby 1.9 or 2.0
   # extracts a string with all empty chars at the end stripped
   # expects the filehandle to be at the proper location
   def get_null_padded_string(fh,bytes)
@@ -16,62 +21,6 @@ module BinaryReader
 end
 
 # class to extract information from <file>_dta.log files
-class DTALog
-  # returns an array indexed by the dta file number (starting at 0)
-  # each entry is an array [first_scan, last_scan, dta_filename_noext]
-  # this is now obsolete since I found the scan # index at the end of the srf
-  # files
-  def self.dta_and_scans_by_dta_index(file)
-    dta_index = nil
-    final_scan = nil
-    dta_cnt = 0
-    re = /^ m/o
-    scan_line_re = /scan: (\d+) - (\d+), Datafile: (.*?) (.*)/o
-    other_dta_re = /Datafile: (.*?) /o
-    File.open(file) do |fh|
-      10.times { fh.readline }
-      scan_range_line = fh.readline
-      if scan_range_line =~ /scan range\s+= \d+ - (\d+)/
-        # this is an overestimate (since MS scans have no dta, but that's OK)
-        dta_index = Array.new($1.to_i)
-      else
-        dta_index = []
-      end
-      3.times { fh.readline }
-      fh.each do |line|
-        if line =~ re
-          if line =~ scan_line_re
-            first_scan = $1.to_i
-            last_scan = $2.to_i
-            the_rest = $4.dup
-            dta_index[dta_cnt] = [first_scan, last_scan, $3.sub(/\.dta/,'')]
-            dta_cnt += 1
-            if the_rest =~ other_dta_re 
-              dta_index[dta_cnt] = [first_scan, last_scan, $1.sub(/\.dta/,'')]
-              dta_cnt += 1
-            end
-          end
-          break
-        end
-      end
-      fh.each do |line|
-        if line =~ scan_line_re
-          first_scan = $1.to_i
-          last_scan = $2.to_i
-          the_rest = $4.dup
-          dta_index[dta_cnt] = [first_scan, last_scan, $3.sub(/\.dta/,'')]
-          dta_cnt += 1
-          if the_rest =~ other_dta_re 
-            dta_index[dta_cnt] = [first_scan, last_scan, $1.sub(/\.dta/,'')]
-            dta_cnt += 1
-          end
-        end
-      end
-    end
-    dta_index.compact! # remove those trailing nils
-    dta_index
-  end
-end
 
 class SRFGroup
   include SpecID
@@ -83,12 +32,15 @@ class SRFGroup
   # takes an array of filenames
   # or a single .srg filename
   # see from_srg to load a single .srg file
-  def initialize(filenames=nil)
+  # by default, the hits will be returned filtered by sequest params values.
+  # [The raw SRF data is unfiltered!]
+  def initialize(filenames=nil, filter_hits_by_params=true)
     @filenames = filenames
     @peps = []
     @prots = []
-    @global_ref_hash = {}
     @srfs = []
+
+    global_ref_hash = {}
     if filenames
       if filenames.is_a?(String) && filenames =~ /\.srg$/
         srg_filename = filenames.dup
@@ -103,16 +55,99 @@ class SRFGroup
         end
       end
       filenames.each do |file|
-        @srfs << SRF.new(file, @peps, @global_ref_hash)
+        @srfs << SRF.new(file, @peps, global_ref_hash)
+      end
+      @prots = global_ref_hash.values
+      if filter_hits_by_params
+        filter_by_peptide_mass_tolerance
+      end
+    end
+  end
+
+  # if srfs were read in separately, then the proteins will need to be merged
+  # by their reference
+  def merge_different_sets(srfs)
+    raise NotImplementedError, "need to implement?"
+  end
+
+  # 1. sets @prots and returns it: a new list of proteins based on which
+  # peptides passed.
+  # 2. updates the out_file's list of hits based on passing peptides (but not
+  # the original hit id; rank is implicit in array ordering)
+  # 3. updates each protein to only include peptides passing thresholds.
+  # [Note, this process is how .out files are generated!]
+  # 4. recalculates deltacn values completely if number of hits changed (does
+  # not touch deltacn orig)
+  # ASSUMES: 
+  # A. all srfs have identical params objects and each has a
+  # peptide_mass_tolerance filter attribute.
+  # B. proteins are already unique (peptides referencing the same protein
+  # reference the same object already) In practice, this means all srfs were
+  # read in together.
+  def filter_by_peptide_mass_tolerance
+    prots_in_set = Set.new
+    params = @srfs.first.params
+    pmt = params.peptide_mass_tolerance.to_f
+    methd = nil  # the method to 
+
+    case params.peptide_mass_units
+    when '0'
+      amu_based = true
+      milli_amu = false
+    when '1'
+      amu_based = true
+      milli_amu = true
+    when '2'
+      amu_based = false
+    end
+
+    @srfs.each do |srf|
+      srf.out_files.each do |out_file|
+        hits = out_file.hits
+        before = hits.size
+        hits.reject! do |pep|
+          do_not_keep =
+            if amu_based
+              if milli_amu
+                (pep.deltamass.abs > (pmt/1000))
+              else
+                (pep.deltamass.abs > pmt)
+              end
+            else
+              (pep.ppm.abs > pmt)
+            end
+          unless do_not_keep
+            pep.prots.each do |prot|
+              if prots_in_set.include?(prot)
+                prot.peps << pep
+              else
+                prots_in_set.add(prot)
+                prot.peps = [pep]
+              end
+            end
+          end
+          do_not_keep
+        end
+        if hits.size != before
+          SRF::OUT::Pep.set_deltacn_from_xcorr(hits)
+        end
       end
     end
+    @prots = prots_in_set.to_a
+
   end
 
   # returns the filename used
+  # if the file exists, the name will be expanded to full path, otherwise just
+  # what is given
   def to_srg(srg_filename='bioworks.srg')
     File.open(srg_filename, 'w') do |v|
       @filenames.each do |srf_file|
-        v.puts File.expand_path(srf_file)
+        if File.exist? srf_file
+          v.puts File.expand_path(srf_file)
+        else
+          v.puts srf_file
+        end
       end
     end
     srg_filename
@@ -132,15 +167,25 @@ class SRF
   # [first_scan, last_scan, charge]
   attr_accessor :index
   attr_accessor :base_name
+  # this is the global peptides array
+  attr_accessor :peps
+  # the global reference hash that allows...
+  attr_accessor :global_ref_hash
 
   def dta_start_byte
     case @version
     when '3.2' ; 3260
     when '3.3' ; 3644
+    when '3.5' ; 3644
     end
   end
 
-  # peps and
+  # peps and global_ref_hash are created as the srf files is read.  If the
+  # file is read as part of a group, then these should be passed in. 
+  # NOTE: if you want the hits filtered by precursor tolerance (the way they
+  # might be displayed in .out files) you should probably use SRFGroup (which
+  # does this by default)
+  # SRF is meant to be a low level read of the file.
   def initialize(filename=nil, peps=[], global_ref_hash={})
     @dta_files = []
     @out_files = []
@@ -149,17 +194,203 @@ class SRF
     end
   end
 
+  def round(float, decimal_places)
+    sprintf("%.#{decimal_places}f", float)
+  end
+
+  # the out_filename will be the base_name + .sqt unless 'out_filename' is
+  # defined
+  # :round => round floating point numbers
+  # etc...
+  def to_sqt(out_filename=nil, opts={})
+    tic_dp = 2
+    mh_dp = 7
+    xcorr_dp = 5
+    sp_dp = 2
+    dcn_dp = 5
+
+    defaults = {:db_info=>false, :new_db_path=>nil, :update_db_path=>false, :round=>false}
+    opt = defaults.merge(opts)
+
+    outfile =
+      if out_filename
+        out_filename
+      else
+        base_name + '.sqt'
+      end
+    invariant_ordering = %w(SQTGenerator SQTGeneratorVersion Database FragmentMasses PrecursorMasses StartTime) # just for readability and consistency
+    fmt = 
+      if params.fragment_mass_type == 'average' ; 'AVG'
+      else ; 'MONO'
+      end
+    pmt =
+      if params.precursor_mass_type == 'average' ; 'AVG'
+      else ; 'MONO'
+      end
+
+    mass_table = params.mass_table
+    static_mods = params.static_mods.map do |k,v|
+      key =  k.split(/_/)[1]
+      if key.size == 1
+        key + '=' + (mass_table[key.to_sym] + v.to_f).to_s
+      else
+        key + '=' + v
+      end
+    end
+
+    dynamic_mods = []
+    header.modifications.scan(/\((.*?)\)/) do |match|
+      dynamic_mods << match.first.sub(/ /,'=')
+    end
+    plural = {
+      'StaticMod' => static_mods,
+      'DynamicMod' => dynamic_mods,  # example as diff mod
+      'Comment' => ['Created from Bioworks .srf file']
+    }
+
+
+    db_filename = header.db_filename
+    db_filename_in_sqt = db_filename
+    if opt[:new_db_path]
+      db_filename = File.join(opt[:new_db_path], File.basename(db_filename.gsub('\\', '/')))
+      if opt[:update_db_path]
+        db_filename_in_sqt = File.expand_path(db_filename)
+        warn "writing Database #{db_filename} to sqt, but it does not exist on this file system" unless File.exist?(db_filename) 
+      end
+    end
+
+    apmu = 
+      case params.peptide_mass_units 
+      when '0' : 'amu' 
+      when '1' : 'mmu'
+      when '2' : 'ppm'
+      end
+
+    hh =  {
+      'SQTGenerator' => 'mspire',
+      'SQTGeneratorVersion' => Mspire::Version,
+      'Database' => db_filename_in_sqt,
+      'FragmentMasses' => fmt,
+      'PrecursorMasses' => pmt,
+      'StartTime' => '',  # Bioworks 3.2 also leaves this blank...
+      'Alg-PreMassTol' => params.peptide_mass_tolerance,
+      'Alg-FragMassTol' => params.fragment_ion_tolerance,
+      'Alg-PreMassUnits' => apmu, ## mine
+      'Alg-IonSeries' => header.ion_series.split(':').last.lstrip,
+      'Alg-Enzyme' => header.enzyme.split(':').last,
+      'Alg-MSModel' => header.model,
+    }
+
+    if opt[:db_info]
+      if File.exist?(db_filename)
+        reply = get_db_info_for_sqt(db_filename)
+        %w(DBSeqLength DBLocusCount DBMD5Sum).zip(reply) do |label,val|
+          hh[label] = val
+        end
+      else
+        warn "file #{db_filename} does not exist, no extra db info in header!"
+      end
+    end
+
+    has_hits = (self.out_files.size > 0)
+    if has_hits
+      # somewhat redundant with above, but we can get this without a db present!
+      hh['DBLocusCount'] = self.out_files.first.db_locus_count
+    end
+
+    File.open(outfile, 'w') do |out|
+      # print the header:
+      invariant_ordering.each do |iv|
+        out.puts ['H', iv, hh.delete(iv)].join("\t")
+      end
+      hh.each do |k,v|
+        out.puts ['H', k, v].join("\t")
+      end
+      plural.each do |k,vals|
+        vals.each do |val|
+          out.puts ['H', k, val].join("\t")
+        end
+      end
+
+      ##### SPECTRA
+      time_to_process = '0.0'
+      #########################################
+      # NEED TO FIGURE OUT: (in spectra guy)
+      #    * Lowest Sp value for top 500 spectra
+      #    * Number of sequences matching this precursor ion 
+      #########################################
+
+
+      manual_validation_status = 'U'
+      self.out_files.zip(dta_files) do |out_file, dta_file|
+        # don't have the time to process (using 0.0 like bioworks 3.2)
+        dta_file_mh = dta_file.mh
+        out_file_total_inten = out_file.total_inten
+        out_file_lowest_sp = out_file.lowest_sp
+        if opt[:round]
+          dta_file_mh = round(dta_file_mh, mh_dp)
+          out_file_total_inten = round(out_file_total_inten, tic_dp)
+          out_file_lowest_sp = round(out_file_lowest_sp, sp_dp)
+        end
+
+        out.puts ['S', out_file.first_scan, out_file.last_scan, out_file.charge, time_to_process, out_file.computer, dta_file_mh, out_file_total_inten, out_file_lowest_sp, out_file.num_matched_peptides].join("\t")
+        out_file.hits.each_with_index do |hit,index|
+          hit_mh = hit.mh
+          hit_deltacn_orig = hit.deltacn_orig
+          hit_xcorr = hit.xcorr
+          hit_sp = hit.sp
+          if opt[:round]
+            hit_mh = round(hit_mh, mh_dp)
+            hit_deltacn_orig = round(hit_deltacn_orig, dcn_dp)
+            hit_xcorr = round(hit_xcorr, xcorr_dp)
+            hit_sp = round(hit_sp, sp_dp)
+          end
+          # note that the rank is determined by the order..
+          out.puts ['M', index+1, hit.rsp, hit_mh, hit_deltacn_orig, hit_xcorr, hit_sp, hit.ions_matched, hit.ions_total, hit.sequence, manual_validation_status].join("\t")
+          hit.prots.each do |prot|
+            out.puts ['L', prot.first_entry].join("\t")
+          end
+        end
+      end
+    end # close the filehandle
+
+  end
+
+  # assumes the file exists and is readable
+  # returns [DBSeqLength, DBLocusCount, DBMD5Sum] or nil if no file
+  def get_db_info_for_sqt(dbfile)
+      fasta = Fasta.new(dbfile)
+      [fasta.aa_seq_length, fasta.size, fasta.md5_sum]
+  end
+
+
   # returns self
   def from_file(filename, peps, global_ref_hash)
 
     File.open(filename, "rb") do |fh|
       @header = SRF::Header.new.from_handle(fh)      
       @version = @header.version
-      @dta_files, measured_mhs = read_dta_files(fh,@header.num_dta_files)
-      @out_files = read_out_files(fh,@header.num_dta_files, global_ref_hash, measured_mhs)
-      @params = Sequest::Params.new.parse_handle(fh)
-      fh.read(12)  ## gap between last params entry and index
-      @index = read_scan_index(fh,@header.num_dta_files)
+      unpack_35 = case @version
+                  when '3.2'
+                    false
+                  when '3.3'
+                    false
+                  when '3.5'
+                    true
+                  end
+      @dta_files, measured_mhs = read_dta_files(fh,@header.num_dta_files, unpack_35)
+      
+      @out_files = read_out_files(fh,@header.num_dta_files, global_ref_hash, measured_mhs, unpack_35)
+      if fh.eof?
+        warn "FILE: '#{filename}' appears to be an abortive run (no params in srf file)\nstill continuing..."
+        @params = nil
+        @index = []
+      else
+        @params = Sequest::Params.new.parse_handle(fh)
+        # This is very sensitive to the grab_params method in sequest params
+        fh.read(12)  ## gap between last params entry and index
+        @index = read_scan_index(fh,@header.num_dta_files)
+      end
     end
 
     ### UPDATE SOME THINGS ON SINGLE PASS:
@@ -168,14 +399,15 @@ class SRF
     @index.each_with_index do |ind,i|
       mass_measured = @dta_files[i][0]
       #puts @out_files[i].join(", ")
-      pep_hits = @out_files[i][3]
+      @out_files[i][0,3] = *ind
+      pep_hits = @out_files[i][6]
       peps.push( *pep_hits )
       pep_hits.each do |pep_hit|
-        pep_hit[13,3] = @base_name, *ind
+        pep_hit[14,4] = @base_name, *ind
         # add the deltamass
-        pep_hit[10] = pep_hit[0] - mass_measured  # real - measured (deltamass)
-        pep_hit[11] = 1.0e6 * pep_hit[10].abs / mass_measured ## ppm
-        pep_hit[17] = self  ## link with the srf object
+        pep_hit[11] = pep_hit[0] - mass_measured  # real - measured (deltamass)
+        pep_hit[12] = 1.0e6 * pep_hit[11].abs / mass_measured ## ppm
+        pep_hit[18] = self  ## link with the srf object
       end
     end
     self
@@ -195,27 +427,17 @@ class SRF
     index
   end
 
-  # given a zero indexed list where each entry is [first_scan, last_scan,
-  # dta_filename] updates the out info
-  # returns self
-  def update_out_scan_info_from_dta_log(dta_log)
-    index = DTALog.dta_and_scans_by_dta_index(dta_log) 
-    @out_files.each_with_index do |ot,i|
-      ot[4,3] = index[i]  #contingent on implementation of ot
-    end
-    self
-  end
-
   # returns an array of dta_files
-  def read_dta_files(fh, num_files)
+  def read_dta_files(fh, num_files, unpack_35)
     measured_mhs = Array.new(num_files) ## A parallel array to capture the actual mh
     dta_files = Array.new(num_files)
     start = dta_start_byte
     unless fh.pos == start
       fh.pos = start
     end
+
     header.num_dta_files.times do |i|
-      dta_file = SRF::DTA.new.from_handle(fh) 
+      dta_file = SRF::DTA.new.from_handle(fh, unpack_35) 
       measured_mhs[i] = dta_file[0]
       dta_files[i] = dta_file
     end
@@ -224,10 +446,10 @@ class SRF
 
   # filehandle (fh) must be at the start of the outfiles.  'read_dta_files'
   # will put the fh there.
-  def read_out_files(fh,number_files, global_ref_hash, measured_mhs)
+  def read_out_files(fh,number_files, global_ref_hash, measured_mhs, unpack_35)
     out_files = Array.new(number_files)
     header.num_dta_files.times do |i|
-      out_files[i] = SRF::OUT.new.from_handle(fh, global_ref_hash)
+      out_files[i] = SRF::OUT.new.from_handle(fh, global_ref_hash, unpack_35)
     end
     out_files
   end
@@ -326,31 +548,44 @@ class SRF::DTAGen
   end
 end
 
+# total_num_possible_charge_states is not correct under 3.5 (Bioworks 3.3.1)
+# unknown is, well unknown...
 SRF::DTA = ArrayClass.new(%w(mh dta_tic num_peaks charge ms_level unknown total_num_possible_charge_states peaks))
 
 class SRF::DTA 
-  Unpack = "EeIvvvv"
-
-  # is this universal?
-  First_record_start_byte = 3644 
+  # original
+  # Unpack = "EeIvvvv"
+  Unpack_32 = "EeIvvvv"
+  Unpack_35 = "Ex8eVx2vvvv"
 
   # note on peaks (self[7])
   # this is a byte array of floats, you can get the peaks out with
   # unpack("e*")
 
+  undef_method :inspect
   def inspect
     peaks_st = 'nil'
     if self[7] ; peaks_st = "[#{self[7].size} bytes]" end
     "<SRF::DTA @mh=#{mh} @dta_tic=#{dta_tic} @num_peaks=#{num_peaks} @charge=#{charge} @ms_level=#{ms_level} @total_num_possible_charge_states=#{total_num_possible_charge_states} @peaks=#{peaks_st} >"
   end
     
-  def from_handle(fh)
-    st = fh.read(24)
+  def from_handle(fh, unpack_35)
+    if unpack_35
+      @unpack = Unpack_35
+      @read_header = 34
+      @read_spacer = 22
+    else
+      @unpack = Unpack_32
+      @read_header = 24
+      @read_spacer = 24
+    end
+
+    st = fh.read(@read_header)
     # get the bulk of the data in single unpack
-    self[0,7] = st.unpack(Unpack)
+    self[0,7] = st.unpack(@unpack)
     
-    # Scan numbers possibly hidden in this next sequence of bytes (I think)
-    st2 = fh.read(24)
+    # Scan numbers are given at the end in an index!
+    st2 = fh.read(@read_spacer)
 
     num_bytes_to_read = num_peaks * 8  
     st3 = fh.read(num_bytes_to_read)
@@ -360,135 +595,176 @@ class SRF::DTA
 
 end
 
-SRF::OUT = ArrayClass.new( %w(num_hits computer date_time hits) )
-# 0=num_hits 1=charge 2=computer 3=date_time 4=hits
+SRF::OUT =  ArrayClass.new( %w(first_scan last_scan charge num_hits computer date_time hits total_inten lowest_sp num_matched_peptides db_locus_count) )
+# 0=first_scan, 1=last_scan, 2=charge, 3=num_hits, 4=computer, 5=date_time, 6=hits, 7=total_inten, 8=lowest_sp, 9=num_matched_peptides, 10=db_locus_count
 
 class SRF::OUT
-  Unpack = '@36vx2Z*@60Z*'
+  Unpack_32 = '@36vx2Z*@60Z*'
+  Unpack_35 = '@36vx4Z*@62Z*'
 
+  undef_method :inspect
   def inspect
-    if first_scan
-      ins = "@first_scan=#{first_scan}, @last_scan=#{last_scan}, @filename_noext=#{filename_noext}, "
-    end
-    "<SRF::OUT  @num_hits=#{num_hits}, @computer=#{computer}, @date_time=#{date_time}, #{ins}@hits=#{hits.inspect}>"
+    hits_s = 
+      if self[6]
+        ", @hits(#)=#{hits.size}"
+      else
+        ''
+      end
+    "<SRF::OUT  first_scan=#{first_scan}, last_scan=#{last_scan}, charge=#{charge}, num_hits=#{num_hits}, computer=#{computer}, date_time=#{date_time}#{hits_s}>"
   end
     
-  def from_handle(fh, global_ref_hash)
+  def from_handle(fh, global_ref_hash, unpack_35)
     ## EMPTY out file is 96 bytes
     ## each hit is 320 bytes
     ## num_hits and charge:
     st = fh.read(96)
-    self[0,3] = st.unpack(Unpack)
-    num_hits = self[0]
+
+    self[3,3] = st.unpack( (unpack_35 ? Unpack_35 : Unpack_32) )
+    self[7,4] = st.unpack('@8eex4Ix4I')
+    num_hits = self[3]
 
     ar = Array.new(num_hits)
     if ar.size > 0
+      num_extra_references = 0
       num_hits.times do |i|
-        ar[i] = SRF::OUT::Pep.new.from_handle(fh, global_ref_hash)
+        ar[i] = SRF::OUT::Pep.new.from_handle(fh, global_ref_hash, unpack_35)
+        num_extra_references += ar[i].num_other_loci
       end
+      SRF::OUT::Pep.read_extra_references(fh, num_extra_references, ar, global_ref_hash)
       ## The xcorrs are already ordered by best to worst hit
       ## ADJUST the deltacn's to be meaningful for the top hit:
       ## (the same as bioworks and prophet)
-      (1...ar.size).each {|i| ar[i-1].deltacn = ar[i].deltacn }
-      ar.last.deltacn = 1.1
+      SRF::OUT::Pep.set_deltacn_from_deltacn_orig(ar)
+      #puts ar.map  {|a| a.deltacn }.join(", ")
     end
-    self[3] = ar
-
+    self[6] = ar
     self
   end
 
+
+
 end
 
+
 # deltacn is modified to be that of the next best hit (by xcorr).
+# deltacn_orig is the one that sequest originally reports
 # if there is no next best hit, then it will be 1.1 (like bioworks)
 # mh is the theoretical mass + h
 # prots are created as SRF prot objects with a reference and linked to their
 # peptides (from global hash by reference)
 # ppm = 10^6 * ∆m_accuracy / mass_measured  [ where ∆m_accuracy = mass_real – mass_measured ]
 # This is calculated for the M+H mass!
+# num_other_loci is the number of other loci that the peptide matches beyond
+# the first one listed
 # srf = the srf object this scan came from
-SRF::OUT::Pep = ArrayClass.new(%w( mh deltacn sp xcorr id rsp ions_matched ions_total sequence prots deltamass ppm aaseq base_name first_scan last_scan charge srf) )
 
-# 0=mh 1=deltacn 2=sp 3=xcorr 4=id 5=rsp 6=ions_matched 7=ions_total 8=sequence 9=prots 10=deltamass 11=ppm 12=aaseq 13=base_name 14=first_scan 15=last_scan 16=charge 17=srf
+SRF::OUT::Pep = ArrayClass.new(%w( mh deltacn_orig sp xcorr id num_other_loci rsp ions_matched ions_total sequence prots deltamass ppm aaseq base_name first_scan last_scan charge srf deltacn) )
+
+# 0=mh 1=deltacn 2=sp 3=xcorr 4=id 5=num_other_loci 6=rsp 7=ions_matched 8=ions_total 9=sequence 10=prots 11=deltamass 12=ppm 13=aaseq 14=base_name 15=first_scan 16=last_scan 17=charge 18=srf 19=deltacn
 
 class SRF::OUT::Pep
   include SpecID::Pep
 
-  Unpack = '@64Ex8ex12eeIx18vvvx8Z*@240Z*'
+  # creates the deltacn that is meaningful for the top hit (the deltacn_orig
+  # or the second best hit and so on).
+  # assumes sorted
+  def self.set_deltacn_from_deltacn_orig(ar)
+    (1...ar.size).each {|i| ar[i-1].deltacn = ar[i].deltacn_orig }
+    ar[-1].deltacn = 1.1
+  end
+
+  # same as set_deltacn_from_deltacn_orig except calculates with xcorr.
+  # assumes sorted
+  def self.set_deltacn_from_xcorr(ar)
+    if ar.size > 0
+      top_score = ar.first[3]
+      other_scores = (1...(ar.size)).to_a.map do |i|
+        (top_score - ar[i][3])/top_score
+      end
+      (0...(ar.size-1)).each do |i|
+        ar[i][19] = other_scores[i]
+      end
+      ar.last[19] = 1.1
+    end
+  end
+
+  def self.read_extra_references(fh, num_extra_references, pep_hits, global_ref_hash)
+    num_extra_references.times do
+      # 80 bytes total (with index number)
+      pep = pep_hits[fh.read(8).unpack('x4I').first - 1]
+
+      ref = fh.read(80).unpack('A*').first
+      pep[10] << pep.new_protein(ref[0,38], pep, global_ref_hash)
+    end
+    #  fh.read(6) if unpack_35
+  end
+
+  # x2=???
+  #Unpack_35 = '@64Ex8ex12eeIx22vx2vvx8Z*@246Z*'
+  ### NOTE: 
+  # I need to verify that this is correct (I mean the 'I' after x18)
+  Unpack_35 = '@64Ex8ex12eeIx18Ivx2vvx8Z*@246Z*'
+  # translation: @64=(64 bytes in to the record), E=mH, x8=8unknown bytes, e=deltacn,
+  # x12=12unknown bytes, e=sp, e=xcorr, I=ID#, x18=18 unknown bytes, v=rsp,
+  # v=ions_matched, v=ions_total, x8=8unknown bytes, Z*=sequence, 240Z*=at
+  # byte 240 grab the string (which is proteins).
+  #Unpack_32 = '@64Ex8ex12eeIx18vvvx8Z*@240Z*'
+  Unpack_32 = '@64Ex8ex12eeIx14Ivvvx8Z*@240Z*'
   Unpack_four_null_bytes = 'a*'
   Unpack_Zstar = 'Z*'
+  Read_35 = 426
+  Read_32 = 320
 
   FourNullBytes_as_string = "\0\0\0\0"
   #NewRecordStart = "\0\0" + 0x3a.chr + 0x1a.chr + "\0\0"
   NewRecordStart = 0x01.chr + 0x00.chr
   Sequest_record_start = "[SEQUEST]"
 
-  tmp = $VERBOSE ; $VERBOSE = nil
-  def prots() self[9] end
-  $VERBOSE = tmp
-
+  undef_method :inspect
   def inspect
-    st = %w(aaseq sequence mh deltacn sp xcorr id rsp ions_matched ions_total prots deltamass ppm base_name first_scan last_scan charge).map do |v| 
-      if v.is_a? Array
+    st = %w(aaseq sequence mh deltacn_orig sp xcorr id rsp ions_matched ions_total prots deltamass ppm base_name first_scan last_scan charge deltacn).map do |v| 
+      if v == 'prots'
+        "#{v}(#)=#{send(v.to_sym).size}"
+      elsif v.is_a? Array
         "##{v}=#{send(v.to_sym).size}"
       else
-        "@#{v}=#{send(v.to_sym)}"
+        "#{v}=#{send(v.to_sym).inspect}"
       end
     end
     st.unshift("<#{self.class}")
     if srf
-      st.push("@srf(base_name)=#{srf.base_name}")
+      st.push("srf(base_name)=#{srf.base_name.inspect}")
     end
     st.push('>')
     st.join(' ')
     #"<SRF::OUT::Pep @mh=#{mh}, @deltacn=#{deltacn}, @sp=#{sp}, @xcorr=#{xcorr}, @id=#{id}, @rsp=#{rsp}, @ions_matched=#{ions_matched}, @ions_total=#{ions_total}, @sequence=#{sequence}, @prots(count)=#{prots.size}, @deltamass=#{deltamass}, @ppm=#{ppm} @aaseq=#{aaseq}, @base_name=#{base_name}, @first_scan=#{first_scan}, @last_scan=#{last_scan}, @charge=#{charge}, @srf(base_name)=#{srf.base_name}>"
   end
+  # extra_references_array is an array that grows with peptides as extra
+  # references are discovered.
+  def from_handle(fh, global_ref_hash, unpack_35)
+    unpack = 
+      if unpack_35 ; Unpack_35
+      else ; Unpack_32
+      end
 
-  ## There must be a better way to do this.
-  ## We are checking that there are no additional protein references only
-  ## so that we are in register for the next reading
-  def read_extra_references(fh, global_ref_hash)
-    $SRF_OUT_HIT_FH_POS = fh.pos
-    st = fh.read(4)
-    #puts "HHH: " + st.unpack("H*").first
-    ## if we see 0000 0000 we are done
-    if st.unpack(Unpack_four_null_bytes).first == FourNullBytes_as_string
-      fh.pos = $SRF_OUT_HIT_FH_POS 
-      return nil
-    end
-    # read in context of 4 bytes read above:
-
-    ## NOTE: in context of 4 bytes read above!
-    st = fh.read(36)
-    if st[34,2] == NewRecordStart
-      fh.pos = $SRF_OUT_HIT_FH_POS 
-      return nil
-    end
+    ## get the first part of the info
+    st = fh.read(( unpack_35 ? Read_35 : Read_32) ) ## read all the hit data
 
-    # is this the end of the outfiles?
-    ## BACK to beginning of this section
-    fh.pos = $SRF_OUT_HIT_FH_POS
-    if fh.read(9) == Sequest_record_start 
-      fh.pos = $SRF_OUT_HIT_FH_POS
-      return
-    end
+    self[0,10] = st.unpack(unpack)
 
-    ## we have extra references
-    ## original read was fh.read(79)
-    fh.seek(-1, IO::SEEK_CUR)
-    self[9].push( new_protein(fh.read(80).unpack(Unpack_Zstar).first, self, global_ref_hash ) )
+    # we are slicing the reference to 38 chars to be the same length as
+    # duplicate references
+    self[10] = [new_protein(self[10][0,38], self, global_ref_hash)]
     
-    #p self.prots
-    #puts self.prots.size
-    #$glob ||= 0
-    #$glob += 1
-    #if $glob == 20
-    #  abort 
-    #end
-
-    read_extra_references(fh,global_ref_hash)
+    self[13] = SpecID::Pep.sequence_to_aaseq(self[9])
+
+    fh.read(6) if unpack_35
+
+    self
   end
 
+
+
   def new_protein(reference, peptide, global_ref_hash)
     if global_ref_hash.key? reference
       global_ref_hash[reference].peps << peptide
@@ -498,29 +774,20 @@ class SRF::OUT::Pep
     global_ref_hash[reference] 
   end
 
-  def from_handle(fh, global_ref_hash)
-    ## get the first part of the info
-    st = fh.read(320) ## read all the hit data
-    self[0,10] = st.unpack(Unpack)
-    # we are slicing the reference to 38 chars to be the same length as
-    # duplicate references
-    self[9] = [new_protein(self[9][0,38], self, global_ref_hash)]
-    self[12] = SpecID::Pep.sequence_to_aaseq(self[8])
-    read_extra_references(fh, global_ref_hash) 
-    
-    self
-  end
-    
-end
+ end
 
 SRF::OUT::Prot = ArrayClass.new( %w(reference peps) )
 
 class SRF::OUT::Prot
   include SpecID::Prot
+  # we shouldn't have to do this because this is inlcuded in SpecID::Prot, but
+  # under some circumstances it won't work without explicitly calling it.
+  include ProteinReferenceable 
 
   tmp = $VERBOSE ; $VERBOSE = nil
   def initialize(reference=nil, peps=[])
-    super(@@arr_size)
+    #super(@@arr_size)
+    super(size)
     #@reference = reference
     #@peps = peps
     self[0,2] = reference, peps
@@ -529,6 +796,7 @@ class SRF::OUT::Prot
 
   #  "<SRF::OUT::Prot reference=\"#{@reference}\">"
   
+  undef_method :inspect
   def inspect
     "<SRF::OUT::Prot @reference=#{reference}, @peps(#)=#{peps.size}>"
   end