mspire 0.8.4 → 0.8.5

data/README.md

@@ -19,7 +19,7 @@ Prince JT, Marcotte EM. <b>mspire: mass spectrometry proteomics in Ruby.</b> *Bi
 
 ### imzml
 
-Mspire is the *only* converter from mzml into imzml.  
+Mspire is the *only* commandline converter from mzml into imzml (also see [imzMLConverter](http://www.cs.bham.ac.uk/~ibs/imzMLConverter/)) 
 
 * handles both processed and continuous modes
 * gracefully handles SIM data

data/VERSION

@@ -1 +1 @@
-0.8.4
+0.8.5

data/lib/mspire/fasta.rb

@@ -58,33 +58,11 @@ module Mspire
       Bio::FlatFile.new(Bio::FastaFormat, io)
     end
 
-=begin
-    # returns two hashes [id_to_length, id_to_description]
-    # faster (~4x) than official route.
-    def self.protein_lengths_and_descriptions(file)
-      protid_to_description = {}
-      protid_to_length = {}
-      re = /^>([^\s]+) (.*)/
-        ids = []
-      lengths = []
-      current_length = nil
-      IO.foreach(file) do |line|
-        line.chomp!
-        if md=re.match(line)  
-          lengths << current_length
-          current_id = md[1]
-          ids << current_id
-          current_length = 0
-          protid_to_description[current_id] = md[2]
-        else
-          current_length += line.size
-        end
-      end
-      lengths << current_length
-      lengths.shift # remove the first nil entry
-      [Hash[ids.zip(lengths).to_a], protid_to_description]
+    # takes the header string and returns the uniprot id
+    #
+    #     'sp|Q04917|1433F_HUMAN' #=> 'Q04917'
+    def self.uniprot_id(header)
+      header[/^[^\|]+\|([^\|]+)\|/, 1]
     end
-=end
-
   end
 end

data/lib/mspire/ident/peptide/db/creator.rb

@@ -0,0 +1,248 @@
+require 'optparse'
+require 'mspire/digester'
+require 'mspire/fasta'
+require 'mspire/ident/peptide/db'
+
+class Mspire::Ident::Peptide::Db::Creator
+  MAX_NUM_AA_EXPANSION = 3
+
+  # the twenty standard amino acids
+  STANDARD_AA = %w(A C D E F G H I K L M N P Q R S T V W Y)
+  EXPAND_AA = {'X' => STANDARD_AA}
+
+  DEFAULT_PEPTIDE_CENTRIC_DB = {
+    missed_cleavages: 2, 
+    min_length: 4,
+    enzyme: Mspire::Digester[:trypsin], 
+    remove_digestion_file: true, 
+    cleave_initiator_methionine: true, 
+    expand_aa: true, 
+    uniprot: true 
+  }
+
+  def self.cmdline(argv)
+
+    opt = {
+      :remove_digestion_file => true,
+      :enzyme => Mspire::Digester[:trypsin]
+    }
+    opts = OptionParser.new do |op|
+      op.banner = "usage: #{File.basename($0)} <file>.fasta ..."
+      op.separator "output: "
+      op.separator "    <file>.msd_clvg<missed_cleavages>.min_aaseq<min_length>.yml"
+      op.separator "format:"
+      op.separator "    PEPTIDE: ID1<tab>ID2<tab>ID3..."
+      op.separator ""    
+      op.separator "    Initiator Methionines - by default, will generate two peptides"
+      op.separator "    for any peptide found at the N-termini starting with 'M'"
+      op.separator "    (i.e., one with and one without the leading methionine)"
+      op.separator ""
+      op.on("--missed-cleavages <#{opt[:missed_cleavages]}>", Integer, "max num of missed cleavages") {|v| opt[:missed_cleavages] = v }
+      op.on("--min-length <#{opt[:min_length]}>", Integer, "the minimum peptide aaseq length") {|v| opt[:min_length] = v }
+      op.on("--no-cleaved-methionine", "does not cleave off initiator methionine") { opt[:cleave_initiator_methionine] = false }
+      op.on("--no-expand-x", "don't enumerate aa possibilities", "(removes these peptides)") { opt[:expand_aa] = false }
+      op.on("--no-uniprot", "use entire protid section of fasta header", "for non-uniprot fasta files") { opt[:uniprot] = false }
+      op.on("--trie", "use a trie (for very large uniprot files)", "must have fast_trie gem installed") {|v| opt[:trie] = v }
+      op.on("-e", "--enzyme <name>", "enzyme for digestion") {|v| opt[:enzyme] = Mspire::Insilico::Digester.const_get(v.upcase) }
+      op.on("--list-enzymes", "lists approved enzymes and exits") do
+        puts Mspire::Digester::ENZYMES.keys.join("\n")
+        exit
+      end
+    end
+
+    opts.parse!(argv)
+
+    if argv.size == 0
+      puts opts || exit
+    end
+
+    argv.map do |file|
+      creator = Mspire::Ident::Peptide::Db::Creator.new
+      creator.create(file, opt)
+    end
+  end
+
+  # returns the name of the digestion file that was written
+  def create_digestion_file(fasta_file, opts={})
+    opts = DEFAULT_PEPTIDE_CENTRIC_DB.merge(opts)
+
+    (missed_cleavages, enzyme, cleave_initiator_methionine, expand_aa) = opts.values_at(:missed_cleavages, :enzyme, :cleave_initiator_methionine, :expand_aa) 
+    start_time = Time.now
+    print "Digesting #{fasta_file} ..." if $VERBOSE
+
+    letters_to_expand_re = Regexp.new("[" << Regexp.escape(EXPAND_AA.keys.join) << "]")
+
+    base = fasta_file.chomp(File.extname(fasta_file))
+    digestion_file = base + ".msd_clvg#{missed_cleavages}.peptides"
+    File.open(digestion_file, "w") do |fh|
+      Mspire::Fasta.open(fasta_file) do |fasta|
+        fasta.each do |prot|
+          peptides = enzyme.digest(prot.sequence, missed_cleavages)
+          if (cleave_initiator_methionine && (prot.sequence[0,1] == "M"))
+            m_peps = []
+            init_methionine_peps = []
+            peptides.each do |pep|
+              # if the peptide is at the beginning of the protein sequence
+              if prot.sequence[0,pep.size] == pep
+                m_peps << pep[1..-1]
+              end
+            end
+            peptides.push(*m_peps)
+          end
+          peptides = 
+            if expand_aa
+              peptides.flat_map do |pep|
+                (pep =~ letters_to_expand_re) ? expand_peptides(pep, EXPAND_AA) : pep
+              end
+            else
+              peptides.map {|pep| pep =~ letters_to_expand_re }.compact
+            end
+          header = prot.header
+          id = opts[:uniprot] ? Mspire::Fasta.uniprot_id(header) : header.split(/\s+/).first
+          fh.puts( id + "\t" + peptides.join(" ") )
+        end
+      end
+    end
+    puts "#{Time.now - start_time} sec" if $VERBOSE
+    digestion_file
+  end
+
+  # returns the full path of the created file
+  def db_from_fasta_digestion_file(digestion_file, opts={})
+    opts = DEFAULT_PEPTIDE_CENTRIC_DB.merge(opts)
+
+    start_time = Time.now
+    puts "Organizing raw digestion #{digestion_file} ..." if $VERBOSE
+
+    puts "#{Time.now - start_time} sec" if $VERBOSE
+    hash_like = hash_like_from_digestion_file(digestion_file, opts[:min_length], opts[:trie])
+
+    base = digestion_file.chomp(File.extname(digestion_file))
+    final_outfile = base + ".min_aaseq#{opts[:min_length]}" + ".yml"
+
+    start_time = Time.now
+    print "Writing #{hash_like.size} peptides to #{} ..." if $VERBOSE
+
+    File.open(final_outfile, 'w') do |out|
+      hash_like.each do |k,v|
+        #out.puts( [k, v.join(Mspire::Ident::Peptide::Db::PROTEIN_DELIMITER)].join(Mspire::Ident::Peptide::Db::KEY_VALUE_DELIMITER) )
+        out.puts "#{k}#{Mspire::Ident::Peptide::Db::KEY_VALUE_DELIMITER}#{v}"       
+      end
+    end
+    puts "#{Time.now - start_time} sec" if $VERBOSE
+
+    if opts[:remove_digestion_file]
+      File.unlink(digestion_file)
+    end
+    File.expand_path(final_outfile)
+  end
+
+  def hash_like_tree
+    require 'trie'
+    trie = Trie.new
+    def trie.[](key)
+      val = self.get(key) 
+      if val.nil?
+        self.add(key,"")
+        self.get(key)
+      else
+        val
+      end
+    end
+    trie
+  end
+
+  def hash_like_from_digestion_file(digestion_file, min_length, use_trie=false)
+    cnt = 0
+    if use_trie
+      raise NotImplementedError
+      #puts "using trie" if $VERBOSE
+      #trie = hash_like_tree
+      #line_cnt = 0
+      #::IO.foreach(digestion_file) do |line|
+        #line_cnt += 1
+        ##puts "LINE COUND"
+        ##p line_cnt
+        #(prot, *peps) = line.chomp!.split(/\s+/)
+        ##p peps
+        ##p peps.class
+        ## prot is something like this: "P31946"
+        #puts line
+        #peps.each do |pep|
+          #if pep.size >= min_length
+            #to_set = 
+              #if val = trie.get(pep)
+                #val +  Mspire::Ident::Peptide::Db::PROTEIN_DELIMITER + prot
+              #else
+                #prot
+              #end
+            #p to_set.size
+            #trie.add(pep, to_set)
+          #end
+        #end
+        #cnt += 1
+        #puts cnt if (cnt % 1000) == 0
+      #end
+      #abort "HERE"
+      #trie
+    else
+      hash = {}
+      ::IO.foreach(digestion_file) do |line|
+        (prot, *peps) = line.chomp!.split(/\s+/)
+        # prot is something like this: "P31946"
+        peps.each do |pep|
+          if pep.size >= min_length
+            if val = hash[pep]
+              val << Mspire::Ident::Peptide::Db::PROTEIN_DELIMITER << prot
+            else
+              val = prot
+            end
+            hash[pep] = val
+          end
+        end
+        cnt += 1
+        puts cnt if (cnt % 1000) == 0
+      end
+      hash
+    end
+  end
+
+  # writes a new file with the added 'min_aaseq<Integer>'
+  # creates a temporary digestion file that contains all peptides digesting
+  # with certain missed_cleavages (i.e., min_seq_length is not applied to
+  # this file but on the final peptide centric db)
+  # returns the full name of the written file.
+  def create(fasta_file, opts={})
+    opts = DEFAULT_PEPTIDE_CENTRIC_DB.merge(opts)
+    digestion_file = create_digestion_file(fasta_file, opts)
+    db_from_fasta_digestion_file(digestion_file, opts)
+  end
+
+  # does combinatorial expansion of all letters requesting it.
+  # expand_aa is hash like: {'X'=>STANDARD_AA}
+  # returns nil if there are more than MAX_NUM_AA_EXPANSION amino acids to
+  # be expanded
+  # returns an empty array if there is no expansion
+  def expand_peptides(peptide, expand_aa_hash)
+    letters_in_order = expand_aa_hash.keys.sort
+    index_and_key = []
+    peptide.split('').each_with_index do |char,i| 
+      if let_index = letters_in_order.index(char) 
+        index_and_key << [i, letters_in_order[let_index]]
+      end
+    end
+    if index_and_key.size > MAX_NUM_AA_EXPANSION
+      return nil
+    end
+    to_expand = [peptide]
+    index_and_key.each do |i,letter|
+      new_peps = []
+      while current_pep = to_expand.shift do
+        new_peps << expand_aa_hash[letter].map {|v| dp = current_pep.dup ; dp[i] = v ; dp }
+      end
+      to_expand = new_peps.flatten
+    end
+    to_expand
+  end
+end
+

data/lib/mspire/ident/peptide/db/io.rb

@@ -0,0 +1,62 @@
+require 'mspire/ident/peptide/db'
+
+class Mspire::Ident::Peptide::Db::IO
+  # an object for on disk retrieval of db entries
+  # proteins are returned as an array.
+  # behaves like a hash once it is opened.
+  include Enumerable
+  def self.open(filename, &block)
+    raise ArgumentError unless block
+    File.open(filename) do |io|
+      block.call(self.new(io))
+    end
+  end
+
+  attr_accessor :io
+  attr_accessor :index
+
+  def initialize(io)
+    @io = io
+    @index = {}
+    re = /^(\w+)#{Regexp.escape(Mspire::Ident::Peptide::Db::KEY_VALUE_DELIMITER)}/
+      prev_io_pos = io.pos
+    triplets = io.each_line.map do |line|
+      key = re.match(line)[1]
+      [key, prev_io_pos + key.bytesize+Mspire::Ident::Peptide::Db::KEY_VALUE_DELIMITER.bytesize, prev_io_pos=io.pos]
+    end
+    triplets.each do |key, start, end_pos|
+      @index[key] = [start, end_pos-start]
+    end
+  end
+
+  # returns an array of proteins for the given key (peptide aaseq)
+  def [](key)
+    (start, length) = @index[key]
+    return nil unless start
+    @io.seek(start)
+    string = @io.read(length)
+    string.chomp!
+    string.split(Mspire::Ident::Peptide::Db::PROTEIN_DELIMITER)
+  end
+
+  # number of entries
+  def size ; @index.size end
+  alias_method :length, :size
+
+  def keys
+    @index.keys
+  end
+
+  # all the protein lists
+  def values
+    keys.map {|key| self[key] }
+  end
+
+  # yields a pair of aaseq and protein array
+  def each(&block)
+    @index.each do |key, start_length|
+      block.call([key, self[key]])
+    end
+  end
+end
+

data/lib/mspire/ident/peptide/db.rb

@@ -1,243 +1,36 @@
-require 'mspire/digester'
-require 'mspire/fasta'
-require 'optparse'
+require 'yaml'
 
 module Mspire ; end
 module Mspire::Ident ; end
 module Mspire::Ident::Peptide ; end
 
-# the object itself is a modified Hash.
-# It is initialized with the database file and a protein array can be
-# retrieved with the #[] method given an amino acid sequence.  All other
-# methods are untested at this time and should be avoided!
-class Mspire::Ident::Peptide::Db < Hash
-  MAX_NUM_AA_EXPANSION = 3
-
-  # the twenty standard amino acids
-  STANDARD_AA = %w(A C D E F G H I K L M N P Q R S T V W Y)
-
-  DEFAULT_PEPTIDE_CENTRIC_DB = {:missed_cleavages => 2, :min_length => 4, :enzyme => Mspire::Digester[:trypsin], :id_regexp => nil, :remove_digestion_file => true, :cleave_initiator_methionine => true, :expand_aa => {'X' => STANDARD_AA}}
-
+# Very simple object for protein retrieval from a peptide-centric database
+# See Mspire::Ident::Peptide::Db::IO for an on-disc version for larger files.
+class Mspire::Ident::Peptide::Db
   PROTEIN_DELIMITER = "\t"
-  KEY_VALUE_DELIMITER = ": "
-
-  def self.cmdline(argv)
-
-    opt = {
-      :remove_digestion_file => true,
-      :enzyme => Mspire::Digester[:trypsin]
-    }
-    opts = OptionParser.new do |op|
-      op.banner = "usage: #{File.basename($0)} <file>.fasta ..."
-      op.separator "output: "
-      op.separator "    <file>.msd_clvg<missed_cleavages>.min_aaseq<min_length>.yml"
-      op.separator "format:"
-      op.separator "    PEPTIDE: ID1<tab>ID2<tab>ID3..."
-      op.separator ""    
-      op.separator "    Initiator Methionines - by default, will generate two peptides"
-      op.separator "    for any peptide found at the N-termini starting with 'M'"
-      op.separator "    (i.e., one with and one without the leading methionine)"
-      op.separator ""
-      op.on("--missed-cleavages <#{opt[:missed_cleavages]}>", Integer, "max num of missed cleavages") {|v| opt[:missed_cleavages] = v }
-      op.on("--min-length <#{opt[:min_length]}>", Integer, "the minimum peptide aaseq length") {|v| opt[:min_length] = v }
-      op.on("--no-cleaved-methionine", "does not cleave off initiator methionine") { opt[:cleave_initiator_methionine] = false }
-      op.on("--no-expand-x", "don't enumerate aa 'X' possibilities") { opt[:expand_aa] = nil }
-      op.on("-e", "--enzyme <name>", "enzyme for digestion") {|v| opt[:enzyme] = Mspire::Insilico::Digester.const_get(v.upcase) }
-      op.on("--list-enzymes", "lists approved enzymes and exits") do
-        puts Mspire::Digester::ENZYMES.keys.join("\n")
-        exit
-      end
-    end
-
-    opts.parse!(argv)
-
-    if argv.size == 0
-      puts opts || exit
-    end
-
-    argv.map do |file|
-      Mspire::Ident::Peptide::Db.peptide_centric_db(file, opt)
-    end
-  end
-
-  # writes a new file with the added 'min_aaseq<Integer>'
-  # creates a temporary digestion file that contains all peptides digesting
-  # with certain missed_cleavages (i.e., min_seq_length is not applied to
-  # this file but on the final peptide centric db)
-  # returns the full name of the written file.
-  def self.peptide_centric_db(fasta_file, opts={})
-    opts = DEFAULT_PEPTIDE_CENTRIC_DB.merge(opts)
-
-    (missed_cleavages, min_length, enzyme, id_regexp, remove_digestion_file, cleave_initiator_methionine, expand_aa) = opts.values_at(:missed_cleavages, :min_length, :enzyme, :id_regexp, :remove_digestion_file, :cleave_initiator_methionine, :expand_aa) 
-    start_time = Time.now
-    print "Digesting #{fasta_file} ..." if $VERBOSE
-
-    if expand_aa
-      letters_to_expand_re = Regexp.new("[" << Regexp.escape(expand_aa.keys.join) << "]")
-    end
-
-    base = fasta_file.chomp(File.extname(fasta_file))
-    digestion_file = base + ".msd_clvg#{missed_cleavages}.peptides"
-    File.open(digestion_file, "w") do |fh|
-      Mspire::Fasta.open(fasta_file) do |fasta|
-        fasta.each do |prot|
-          peptides = enzyme.digest(prot.sequence, missed_cleavages)
-          if (cleave_initiator_methionine && (prot.sequence[0,1] == "M"))
-            m_peps = []
-            init_methionine_peps = []
-            peptides.each do |pep|
-              # if the peptide is at the beginning of the protein sequence
-              if prot.sequence[0,pep.size] == pep
-                m_peps << pep[1..-1]
-              end
-            end
-            peptides.push(*m_peps)
-          end
-          if expand_aa
-            peptides = peptides.map do |pep|
-              if pep =~ letters_to_expand_re
-                expand_peptides(pep, expand_aa)
-              else
-                pep
-              end
-            end.flatten
-          end
-          fh.puts( prot.header.split(/\s+/).first + "\t" + peptides.join(" ") )
-        end
-      end
-    end
-    puts "#{Time.now - start_time} sec" if $VERBOSE
-
-
-    start_time = Time.now
-    print "Organizing raw digestion #{digestion_file} ..." if $VERBOSE
+  KEY_VALUE_DELIMITER = ': '
 
-    hash = Hash.new {|h,k| h[k] = [] }
-    ::IO.foreach(digestion_file) do |line|
-      (prot, *peps) = line.chomp!.split(/\s+/)
-      # prot is something like this: "sp|P31946|1433B_HUMAN" in uniprot 
-      peps.each do |pep|
-        if pep.size >= min_length
-          hash[pep] << prot
-        end
-      end
-    end
-    puts "#{Time.now - start_time} sec" if $VERBOSE
-
-    base = digestion_file.chomp(File.extname(digestion_file))
-    final_outfile = base + ".min_aaseq#{min_length}" + ".yml"
-
-    start_time = Time.now
-    print "Writing #{hash.size} peptides to #{} ..." if $VERBOSE
-
-    File.open(final_outfile, 'w') do |out|
-      hash.each do |k,v|
-        out.puts( [k, v.join(PROTEIN_DELIMITER)].join(KEY_VALUE_DELIMITER) )
-      end
-    end
-    puts "#{Time.now - start_time} sec" if $VERBOSE
-
-    if remove_digestion_file
-      File.unlink(digestion_file)
-    end
-    File.expand_path(final_outfile)
-  end
-
-  # does combinatorial expansion of all letters requesting it.
-  # expand_aa is hash like: {'X'=>STANDARD_AA}
-  # returns nil if there are more than MAX_NUM_AA_EXPANSION amino acids to
-  # be expanded
-  # returns an empty array if there is no expansion
-  def self.expand_peptides(peptide, expand_aa)
-    letters_in_order = expand_aa.keys.sort
-    index_and_key = []
-    peptide.split('').each_with_index do |char,i| 
-      if let_index = letters_in_order.index(char) 
-        index_and_key << [i, letters_in_order[let_index]]
-      end
-    end
-    if index_and_key.size > MAX_NUM_AA_EXPANSION
-      return nil
-    end
-    to_expand = [peptide]
-    index_and_key.each do |i,letter|
-      new_peps = []
-      while current_pep = to_expand.shift do
-        new_peps << expand_aa[letter].map {|v| dp = current_pep.dup ; dp[i] = v ; dp }
-      end
-      to_expand = new_peps.flatten
-    end
-    to_expand
-  end
+  attr_accessor :data
 
   def initialize(db_file)
-    self.replace(YAML.load_file(db_file))
+    @data = YAML.load_file(db_file)
   end
 
-  alias_method :old_bracket, '[]'.to_sym
-
-  # returns the protein id's as an array
+  # returns protein id's as an array
   def [](key)
-    old_bracket(key).chomp.split(PROTEIN_DELIMITER)
+    val=@data[key]
+    val.chomp.split(PROTEIN_DELIMITER) if val
   end
 
-  # an object for on disk retrieval of db entries
-  # proteins are returned as an array.
-  # behaves much like a hash once it is opened.
-  class IO
-    include Enumerable
-    def self.open(filename, &block)
-      raise ArgumentError unless block
-      File.open(filename) do |io|
-        block.call(self.new(io))
-      end
-    end
-
-    attr_accessor :io
-    attr_accessor :index
-
-    def initialize(io)
-      @io = io
-      @index = {}
-      re = /^(\w+)#{Regexp.escape(KEY_VALUE_DELIMITER)}/
-        prev_io_pos = io.pos
-      triplets = io.each_line.map do |line|
-        key = re.match(line)[1]
-        [key, prev_io_pos + key.bytesize+KEY_VALUE_DELIMITER.bytesize, prev_io_pos=io.pos]
-      end
-      triplets.each do |key, start, end_pos|
-        @index[key] = [start, end_pos-start]
-      end
-    end
-
-    # returns an array of proteins for the given key (peptide aaseq)
-    def [](key)
-      (start, length) = @index[key]
-      return nil unless start
-      @io.seek(start)
-      string = @io.read(length)
-      string.chomp!
-      string.split("\t")
-    end
-
-    # number of entries
-    def size ; @index.size end
-    alias_method :length, :size
-
-    def keys
-      @index.keys
-    end
+  def keys
+    @data.keys
+  end
 
-    # all the protein lists
-    def values
-      keys.map {|key| self[key] }
-    end
+  def values
+    @data.values
+  end
 
-    # yields a pair of aaseq and protein array
-    def each(&block)
-      @index.each do |key, start_length|
-        block.call([key, self[key]])
-      end
-    end
+  def size
+    @data.size
   end
 end

data/lib/mspire/ident/peptide_hit/qvalue.rb

@@ -19,12 +19,13 @@ class Mspire::Ident::PeptideHit
 
       # writes the peptide hits to a phq.tsv file. qvalues is a parallel array
       # to hits that can provide qvalues if not inherent to the hits
-      # returns the filename.
+      # returns the filename.  Expects each hit to implement #search_id, #id,
+      # #aaseq and #charge
       def to_file(filename, hits, qvalues=[])
         File.open(filename,'w') do |out|
           out.puts HEADER.join(FILE_DELIMITER)
           hits.zip(qvalues) do |hit, qvalue|
-            out.puts [hit.search.id, hit.id, hit.aaseq, hit.charge, qvalue || hit.qvalue].join(FILE_DELIMITER)
+            out.puts [hit.search_id, hit.id, hit.aaseq, hit.charge, qvalue || hit.qvalue].join(FILE_DELIMITER)
           end
         end
         filename

data/schema/peptide_hit_qvalues.pqh.tsv

@@ -0,0 +1,5 @@
+# structure of a very simple file for holding peptide hit qvalues
+# entries should be separated by a tab!!!
+aaseq  charge  qvalue
+String Integer Float
+...    ...     ...