ms-error_rate 0.0.9 → 0.0.10

data/lib/transmembrane.rb

@@ -0,0 +1,157 @@
+
+# A transmemIndex is a hash that takes a fasta reference as key and returns
+# a structured hash containing the transmembrane information.
+module TransmembraneIndex
+  
+  # returns :toppred or :phobius
+  def self.filetype(file)
+    tp = nil
+    File.open(file) do |fh|
+      while (line = fh.gets)
+        case line
+        when /SEQENCE/
+          tp = :phobius 
+          break
+        when /    0  0 i/
+          tp = :phobius  # if they don't have the headers, 
+                         # this will pick it up if they have a 
+                         # single prot without tm or signal peptide.
+          break
+        when /Algorithm specific parameters/
+          tp = :toppred  # New text
+          break
+        when /<parameters>/
+          tp = :toppred  # XML
+          break
+        end
+      end
+    end
+    tp
+  end
+
+  def reference_to_key(reference)
+    # needs to be subclassed or written
+  end
+  
+  # right now accepts toppred.out files
+  # Phobius objects can use the fasta object to update their hash for methods
+  # like avg_overlap
+  def self.new(file)
+    case x = filetype(file)
+    when :toppred
+      require 'transmembrane/toppred'
+      TopPred::Index.new(file)
+    when :phobius
+      require 'transmembrane/phobius'
+      # warn "WARNING: You have NO fasta object with Phobius based TransmembraneIndex! (which needs one to do proper indexing!)" unless fasta
+      Phobius::Index.new(file)
+    else 
+      raise ArgumentError, "#{x} filetype for #{file} not recognized!"
+    end
+  end
+
+  # returns a hash of key -> num certain transmembrane segments
+  def num_certain_index
+    hash = {}
+    self.each do |k,v|
+      hash[k] = v[:num_certain_transmembrane_segments] || 0
+    end
+    hash
+  end
+
+  # tp = :number or :fraction which is the fraction of the sequence size
+  # returns the average number of overlapping amino acids with transmembrane
+  # segments
+  # returns nil if there is no protein by that key
+  def avg_overlap(key, sequence, tp=:number)
+    if self.key? key
+      numbers = num_transmem_aa(self[key], sequence)
+      if numbers.size > 0
+        sum = 0
+        numbers.each {|num| sum += num}
+        avg_num = sum.to_f / numbers.size
+        # the one line way to do it
+        #avg_num = numbers.inject(0) {|memo,num| num + memo }.to_f / numbers.size
+        if tp == :fraction
+          avg_num / sequence.size
+          # this is the same as doing this:
+          #numbers.inject(0.0) {|memo,num| (num.to_f/seq_size + memo) } / numbers.size
+        else
+          avg_num
+        end
+      else
+        0.0
+      end
+    else  # what to do if the protein isn't there?? which happens on occasion
+      nil
+    end
+  end
+
+  # returns an array (usually length of 1) of the number of amino acids
+  # contained inside transmembrane spanning segments.
+  # assumes that tmhash has the key 'transmembrane_segments'
+  # if there are no transmembrane segments, returns empty array.
+  def num_transmem_aa(tmhash, sequence)
+    if tmhash.key? :transmembrane_segments
+      ranges = tmhash[:transmembrane_segments].map do |tmseg|
+        Range.new( tmseg[:start]-1, tmseg[:stop]-1 )
+      end
+      num_overlapping_chars(tmhash[:aaseq], ranges, sequence)
+    else
+      []
+    end
+  end
+
+  # returns an array of the number of overlapping sequences in substring with
+  # the substrings defined in start_stop_doublets within full_sequence
+  # start_stop_doublets should be 0 indexed!!!
+  # the span includes the 'stop' position i.e., full_sequence[start..stop]
+  def num_overlapping_chars(full_sequence, ranges, substring)
+    #start_positions = aaseq.enum_for(:scan, substring).map { $~.offset(0)[0]}
+    if ranges.size == 0
+      []
+      #full_sequence.enum_for(:scan, substring).map { 0 }
+    else
+      substring_ranges = []
+      pos = 0
+      slen = substring.size
+      while i=full_sequence.index(substring,pos)
+        substring_ranges << Range.new(i, i+slen-1)
+        pos = i + slen
+      end
+      # brute force way
+      last_tm_range = ranges.last.last
+      to_return = substring_ranges.map do |sb|
+        overlap = 0
+        # there's got to be a much simpler way to do this, but this does work...
+        ranges.each do |tm|
+          (frst, lst) = 
+            if tm.include?( sb.first )
+              [tm, sb]
+            elsif tm.include?( sb.last )
+              [sb, tm]
+            else
+              nil
+            end
+          if frst
+            if lst.last <= frst.last
+              overlap += (frst.last+1 - frst.first) - (lst.first - frst.first) - (frst.last - lst.last)
+            else 
+              overlap += (frst.last+1 - frst.first) - (lst.first - frst.first)
+            end
+          end
+        end
+        overlap
+      end
+    end
+  end
+
+
+end
+
+
+#substring_ranges = full_sequence.enum_for(:scan, substring).map do 
+#        (ofirst, olast) = $~.offset(0) 
+#        Range.new(ofirst, olast - 1)
+#      end
+

data/schema/peptide_hit_qvalues.pqh.tsv

@@ -0,0 +1,5 @@
+# structure of a very simple file for holding peptide hit qvalues
+# entries should be separated by a tab!!!
+aaseq  charge  qvalue
+String Integer Float
+...    ...     ...

data/script/expert_addition.rb

@@ -0,0 +1,26 @@
+#!/usr/bin/ruby
+
+require 'yaml'
+require 'set'
+
+if ARGV.size == 0
+  puts "usage: prog summary__<setname>__name_to_gene_id.yml"
+  exit
+end
+
+file = ARGV.shift
+
+hash = YAML.load_file(file)
+
+previous_hits = Set.new
+results = []
+hash.sort.each do |fdr, hits|
+  new_hits = hits - previous_hits.to_a
+  previous_hits.merge(new_hits)
+  results << [fdr, hits.size, *new_hits]
+end
+
+results.shift.zip(*results) do |row|
+  puts row.join("\t")
+end
+

data/script/expert_list.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/ruby
+
+require 'orderedhash'
+require 'yaml'
+require 'set'
+
+if ARGV.size != 2
+  puts "usage: #{File.basename(__FILE__)} <gene_ids>.txt summary.yml"
+  puts "writes a yml file with unique proteins per qvalue cutoff"
+  puts "for each set"
+  puts "summary__<setname>__<gene_ids>.yml"
+  exit
+end
+
+(gene_ids, summary) = ARGV
+
+globs = IO.readlines(gene_ids).reject{|v| v[0,1] == '#'}.map{|v| v.chomp }.select {|v| v =~ /\w/ }
+
+hash = YAML.load_file(summary)
+protein_info = hash['protein_info']
+results = hash['results']
+output_hashes = OrderedHash.new
+results.each do |result|
+
+  qvalue_cutoff = result['qvalue_cutoff']
+  result['sets'].each do |setname, sethash|
+    matches = Set.new
+    output_hashes[setname] ||= OrderedHash.new
+    proteins = sethash['proteins']
+    proteins.each do |ipi,info|
+      if info['num_hits_minimal'].first > 0
+        all_proteins = [ipi, *info['indistinguishable']]
+        all_proteins.each do |id|
+          globs.each do |glob|
+            if File.fnmatch?(glob, protein_info[id]['Gene_Symbol'])
+              matches << protein_info[id]['Gene_Symbol']
+            end
+          end
+        end
+      end
+    end
+    output = matches.to_a.sort
+    output_hashes[setname][qvalue_cutoff] = output
+  end
+end
+
+output_hashes.each do |setname, output|
+  gene_ids_base = File.basename(gene_ids, '.*')
+  summary_base = summary.chomp(File.extname(summary))
+  output_file = [summary_base, setname, gene_ids_base].join("__") + ".yml"
+
+  File.open(output_file, 'w') {|out| out.print output.to_yaml }
+end

data/script/fasta_ipi_to_ipi_decoy.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/ruby
+
+if ARGV.size == 0
+  puts "usage: #{File.basename(__FILE__)} <IPI_based>.fasta ..."
+  puts "moves any leading \"><.*_>\" to the IPI value"
+  puts "for example:"
+  puts ">DCY_IPI:IPI0032311.1|STUFF ->  >IPI:DCY_IPI0032311.1|STUFF"
+  exit
+end
+
+ARGV.each do |file|
+  tmp = file + '.tmp'
+  if File.exist?(tmp) ; warn "Skipping #{file} since #{tmp} exists" ; next  end
+  File.open(tmp, 'w') do |out|
+    IO.foreach(file) do |line|
+      if line =~ />([^\:\|]+_)/
+        line.sub!("#{$1}IPI:IPI", "IPI:#{$1}IPI")
+      end
+      out.print line
+    end
+  end
+  FileUtils.mv tmp, file
+end

data/script/minimal_protein_set.rb

@@ -0,0 +1,366 @@
+#!/usr/bin/ruby
+
+require 'yaml'
+require 'set'
+require 'optparse'
+require 'ms/fasta'
+require 'ms/fasta/ipi'
+
+SET_RE = /Set\s+(.*)/i
+QVALUE_EXT = ".qval.yml"
+
+# returns [sets_to_paths_hash, sets_order]
+  def sets_compare_to_paths(file, ext=QVALUE_EXT)
+  dirname = File.dirname(File.expand_path(file))
+  lines = IO.readlines(file).map {|v| v.chomp }.select {|v| v =~ /\w/}
+  sets = {}
+  current_set = nil
+  sets_order = []
+  lines.each do |line|
+    if line =~ SET_RE
+      current_set = $1.dup
+      sets[current_set] = [] 
+      sets_order << current_set
+    else
+      full_path = (File.join(dirname,(line + ext)))
+      raise RuntimeError, "file #{full_path} does not exist!!" unless File.exist?(full_path)
+      sets[current_set] << full_path
+    end
+  end
+  [sets, sets_order]
+end
+
+# returns [minimal_protein_to_uniq_peps_hash, indistinguishable_protein_hash] 
+# takes a hash of proteins to aaseqs. Uses a greedy algorithm where
+# things are sorted first by the number of uniq amino acid sequences and total
+# aa length.  if a block is given, then will yield the prot and the
+# peptide_array and sort by the returned value.  The greedy algorithm acts on
+# the REVERSE of the sorted proteins.  indistinguishable_protein_hash is keyed
+# on the proteins in the minimal_protein_array and gives an array of other
+# proteins.  
+def minimal_protein_set(proteins_to_aaseqs)
+  blk_given = block_given?
+  #STDERR.puts "using block for minimal_protein_set" if blk_given
+  proteins_and_uniq_peps = []
+
+  sorted_most_to_least = proteins_to_aaseqs.sort_by do |k,v| 
+    if blk_given
+      yield(k,v)
+    else
+      [ v.size, v.inject(0){|m,s| m+s.size} ]
+    end
+  end.reverse
+
+  found_seq = Set.new
+
+  same_peptide_hits = {}
+
+  last_peps = nil
+  last_uniq_prot = nil
+  sorted_most_to_least.each do |prot, peps|
+    sorted_peps = peps.sort # is it necessary to SORT?????????
+    uniq_peps = peps.select do |pep|
+      if found_seq.include?(pep)
+        false
+      else
+        found_seq.add pep
+        true
+      end
+    end
+    if uniq_peps.size > 0
+      proteins_and_uniq_peps << [prot, uniq_peps]
+      same_peptide_hits[prot] = []
+      last_peps = sorted_peps
+      last_uniq_prot = prot
+    else
+      if sorted_peps == last_peps 
+        same_peptide_hits[last_uniq_prot] << prot
+      end
+    end
+  end
+  prot_to_uniq_peps_hash = {}
+  proteins_and_uniq_peps.each do |prot, uniq_peps|
+    prot_to_uniq_peps_hash[prot] = uniq_peps
+  end
+
+  [prot_to_uniq_peps_hash, same_peptide_hits]
+end
+
+def cutoffs_to_floats(ar)
+  ar.map do |v|
+    if v == 'nil' || v == '-'
+      nil
+    else
+      answ = v.to_f
+    end
+  end
+end
+
+# returns a hash keyed on protein id that yields an array:
+#   [#aaseq, #aaseq_and_charge, #total_hits]
+def stats_per_prot(prot_to_peps, seq_to_hits)
+  per_protein_hash = {}
+  prot_to_peps.each do |prot, uniq_pep_seqs|
+    all = Set.new
+    aaseqcharges = Set.new
+    aaseqs = Set.new
+
+    uniq_pep_seqs.each do |pep_seq|
+      all_hits = seq_to_hits[pep_seq]
+      all.merge( all_hits )
+      all_hits.each do |hit|
+        aaseq = hit.sequence
+        aaseqs.add( aaseq )
+        aaseqcharges.add( aaseq + '_' + hit.charge.to_s )
+      end
+      per_protein_hash[prot] = [aaseqs.size, aaseqcharges.size, all.size]
+
+    end
+  end
+  per_protein_hash
+end
+
+opt = {
+  :cutoffs => [nil],
+  :outfile => "summary.yml",
+}
+
+opts = OptionParser.new do |op|
+  op.banner = "USAGE: #{File.basename(__FILE__)} sets_compare.txt"
+  op.separator "OUTPUT: #{opt[:outfile]}"
+  op.separator ""
+  op.separator "INPUT: "
+  op.separator "    each <file> referenced in sets_compare.txt should have a"
+  op.separator "    <file>.qval.yml file"
+  op.separator ""
+  op.separator "OPTIONS:"
+  op.on("-q", "--qvalue <0-1[,...]>", Array, "only take qvalues < given ['-' for no threshold]") {|v| opt[:cutoffs] = cutoffs_to_floats(v)}
+  op.separator ""
+  op.on("--proteins <fasta>,<pep-db>", Array, "path to fasta and peptide centric DB", "peptide_centric_db is in the format: ", "<PEPTIDE>: <ID>-<ID>-<ID>") {|v| opt[:proteins] = v }
+  op.separator "FORMATS:"
+  op.on("--output-format", "prints the output yaml scheme and exits") {|v| opt[:output_format] = v }
+  op.on("--input-format", "prints sets_compare.txt format and exits") {|v| opt[:input_format] = v }
+end
+
+# later on we could implement full isoform resolution like IsoformResolver
+# for now we will generate a report, realizing that some isoforms may not be
+# reported
+# it is implemented by using a pre-made map from sequence to protein groups
+# then, a set of sequences allows one to deduce all the relationships from the 
+# protein groups.
+
+opts.parse!
+
+if opt[:output_format]
+  yaml = <<SKEL
+results: 
+- qvalue_cutoff: <Float>
+  sets: 
+    <set_name>: 
+      num_uniq_aaseqs: <Integer>
+      num_aaseqs_not_in_pep_db: <Integer>
+      num_uniq_aaseqs_charge: <Integer>
+      proteins: 
+        <IPI_ID>: 
+          num_hits_all: 
+          - <Integer> # total num aaseqs
+          - <Integer> # total num aaseq+charge
+          - <Integer> # total num hits
+          num_hits_minimal: 
+          - <Integer> # total num aaseqs
+          - <Integer> # total num aaseq+charge
+          - <Integer> # total num hits
+          indistinguishable: 
+          - <IPI_ID>
+          - <IPI_ID>
+          aaseqs: 
+          - <String>
+          - <String>
+sets_order:
+- <String>
+- <String>
+protein_info: 
+  <IPI_ID>: 
+    Gene_Symbol: <String>
+    IPI: <IPI_ID>
+    Tax_Id: <String>
+    SWISS-PROT: <String>
+    description: <String>
+    ENSEMBL: <String>
+SKEL
+  print yaml
+  exit
+end
+
+if opt[:input_format]
+  string =<<EXPLANATION
+# the sets_compare.yml format is very simple:
+
+Set <some_name_for_set1>
+filename1_no_ext
+filename2_no_ext
+Set <some_name_for_set2>
+filename3_no_ext
+filename4_no_ext
+...
+EXPLANATION
+  puts string
+  exit
+end
+
+if ARGV.size != 1
+  puts opts.to_s
+  exit
+end
+
+
+results = {}
+
+protein_info = {}
+results['protein_info'] = protein_info
+results['results'] = []
+
+(sets_hash, sets_order) = sets_compare_to_paths(ARGV.shift)
+results['sets_order'] = sets_order
+
+if opt[:proteins]
+  (fasta, pep_db_file) = opt[:proteins]
+  
+  # a hash indexed on ipi containing all info
+  prot_header_hash = {}
+
+  STDERR.print "Loading information from fasta file..."
+  start = Time.now
+  prot_sizes_hash = {}
+  Ms::Fasta.open(fasta, 'rb', :io_index => []) do |obj|
+    obj.each do |entry|
+      hash = Ms::Fasta::Ipi.parse(entry.header)
+      ipi = hash['IPI']
+      prot_header_hash[ipi] = hash
+      prot_sizes_hash[ipi] = entry.sequence.size
+    end
+  end
+  STDERR.puts "#{Time.now - start} seconds."
+
+  STDERR.print "Loading peptide centric DB (this takes about a minute)..."
+  start = Time.now
+  pep_db = YAML.load_file(pep_db_file)
+  STDERR.puts "#{Time.now - start} seconds."
+
+end
+
+opt[:cutoffs].each do |cutoff|
+
+  cutoff_results = {'qvalue_cutoff' => cutoff}
+  results_sets_hash = {}
+  cutoff_results['sets'] = results_sets_hash
+  results['results'] << cutoff_results
+
+  #########################
+  # FOR EACH SET:
+  #########################
+  pep_klass = nil
+  sets_hash.each do |set, files|
+    set_results = {}
+    results_sets_hash[set] = set_results
+
+    # assumes the indices are the same into each data file
+
+    # get the complete set of passing hits
+    all_passing_hits = files.inject([]) do |all_passing_hits, file|
+      hash = YAML.load_file(file)
+
+      header_hash = hash['headers']
+      pep_klass ||= Struct.new(*(header_hash.map {|v| v.to_sym }))
+      hits = hash['data'].map {|v| pep_klass.new(*v) }
+
+      passing_hits = 
+        if cutoff
+          # assumes monotonic qvalues values!
+          (above, below) = hits.partition {|hit| hit.qvalue <= cutoff }
+          above
+        else
+          hits
+        end
+      all_passing_hits.push(*passing_hits)
+    end
+
+    
+    # create an index from aaseq to hits
+    seq_to_hits = Hash.new {|h,k| h[k] = []}
+    uniq_seqcharge = Set.new
+    all_passing_hits.each do |hit|
+      seq_to_hits[hit.sequence] << hit
+      uniq_seqcharge.add( hit.sequence + '_' + hit.charge.to_s )
+    end
+
+
+    # determine the number of uniq aaseqs
+    uniq_seqs = seq_to_hits.size
+
+    num_uniq_seqcharges = uniq_seqcharge.size
+
+    set_results.merge!( { 'num_peptide_hits' => all_passing_hits.size,
+      'num_uniq_aaseqs' => uniq_seqs,
+      'num_uniq_aaseqs_charge' => num_uniq_seqcharges,
+    })
+
+    if opt[:proteins]
+
+      # create an index from proteins to peptides
+      prots_to_peps = Hash.new {|h,k| h[k] = [] }
+      peptides_not_found = []
+      seq_to_hits.keys.each do |seq|
+        if pep_db.key?(seq)
+          pep_db[seq].split('-').each do |prot|
+            prots_to_peps[prot] << seq
+          end
+        else
+          peptides_not_found << seq
+        end
+      end
+
+      # Determine the number of 1) hits, 2) aaseqs, 3) aaseqcharges per protein BEFORE minimization
+      stats_per_protein_before = stats_per_prot(prots_to_peps, seq_to_hits)
+
+      # get the minimal protein set
+      (prot_to_uniq_peps_hash, indistinguishable_protein_hash) = minimal_protein_set(prots_to_peps) do |prot,peps|
+        # will sort with lowest 
+        [ peps.size, peps.inject(0){|m,s| m+s.size}, -(prot_sizes_hash[prot])]
+      end
+
+      prot_to_uniq_peps_hash.each do |prot, peps|
+        [prot, *indistinguishable_protein_hash[prot]].each do |prot|
+          protein_info[prot] = prot_header_hash[prot]
+        end
+      end
+
+      stats_per_protein_minimal = stats_per_prot(prot_to_uniq_peps_hash, seq_to_hits)
+
+      # create a hash of data for each protein
+      protein_data_hashes_hash = {}
+      prot_to_uniq_peps_hash.each do |prot, peps|
+        protein_data_hashes_hash[prot] = { 
+            'aaseqs' => peps,
+            # this will be a triplet 
+            'num_hits_minimal' => stats_per_protein_minimal[prot],
+            'indistinguishable' => indistinguishable_protein_hash[prot],
+            'num_hits_all' => stats_per_protein_before[prot],
+        } 
+      end
+
+      set_results['proteins'] = protein_data_hashes_hash
+      set_results['num_proteins'] = prot_to_uniq_peps_hash.size
+      set_results['num_aaseqs_not_in_pep_db'] = peptides_not_found.size
+      if peptides_not_found.size > 0
+        warn "Did not find in peptide centric db: #{peptides_not_found.join(', ')}"
+      end
+    end
+  end
+end
+
+File.open(opt[:outfile], 'w') do |out|
+  out.print results.to_yaml
+end
+
+

data/script/unique_seq_stats.rb

@@ -0,0 +1,72 @@
+#!/usr/bin/ruby -w
+
+require 'set'
+require 'yaml'
+require 'optparse'
+
+opt = {}
+opts = OptionParser.new do |op|
+  op.banner = "usage: #{File.basename(__FILE__)} <precision_file>.yml ..."
+  op.separator "outputs information collected by combining hits from files:"
+  op.separator "---"
+  op.separator "filenames: "
+  op.separator "- <pathgiven>"
+  op.separator "num_unique_aaseqs: <Int>"
+  op.separator "num_unique_aaseqs_charge: <Int>"
+  op.separator "num_peptide_hits: <Int>"
+  op.separator ""
+  op.separator "NOTE: if a precision cutoff is given, all hits that have a better"
+  op.separator "score than the worst score at the cutoff are included, even if "
+  op.separator "the precision for that hit was below the cutoff"
+  op.separator "this prevents early, local aberrations in precision from messing"
+  op.separator "up the analysis"
+  op.separator ""
+  op.on("-p", "--precision <0-1>", Float, "precision cutoff") {|v| opt[:cutoff] = v }
+  op.on("-f", "--fdr <0-1>", Float, "false discovery rate cutoff (1-precision)") {|v| opt[:cutoff] = 1.0 - v }
+end
+
+opts.parse!
+
+if ARGV.size == 0
+  puts opts.to_s  
+  exit
+end
+
+unique_sequences = Set.new
+unique_ions = Set.new
+all_hits = []
+
+ARGV.each do |file|
+  hash = YAML.load_file(file)
+
+  prec_index = hash['headers'].index('precision')
+  mowse_index = hash['headers'].index('mowse')
+  aaseq_index = hash['headers'].index('aaseq')
+  charge_index = hash['headers'].index('charge')
+
+  above_cutoff.each do |ar|
+    sequence = ar[aaseq_index]
+    seq_plus_charge = sequence + ar[charge_index]
+    unique_sequences.add sequence
+    unique_ions.add  seq_plus_charge
+  end
+end
+
+prec_k = 'precision cutoff'
+fn_k = 'filenames'
+uniq_aaseq_k = 'num unique aaseqs'
+uniq_ions_k = 'num unique aaseqs+charge'
+num_hits_k = 'num peptide hits'
+
+order = [fn_k, prec_k, num_hits_k, uniq_ions_k, uniq_aaseq_k]
+
+results = {}
+results[fn_k] = '[' + ARGV.join(", ") + ']'
+results[prec_k] = opt[:cutoff]
+results[uniq_aaseq_k] = unique_sequences.size
+results[uniq_ions_k] = unique_ions.size
+results[num_hits_k] = all_hits.size
+
+order.each do |key|
+  puts "#{key}: #{results[key]}"
+end