mzid 0.0.1

data/lib/mzid/parser_sax.rb

@@ -0,0 +1,292 @@
+require 'ox'
+require 'progressbar'
+require 'csv'
+
+module MzID
+  #
+  # class to parse an mzIdentML file (.mzid) in a memory efficient manner.
+  # can parse large files that a DOM parser will fail on, e.g., most mzid 
+  # parsers. The caveat is that it must be written to a csv file.
+  #
+  class ParserSax
+    #
+    # counts the different element types
+    #
+    class CounterHandler < Ox::Sax
+      ATTR = [:DBSequence, :Peptide, :PeptideEvidence, :SpectrumIdentificationItem]
+      #
+      def initialize()
+        @dbseq_count = 0
+        @pep_count = 0
+        @pepev_count = 0
+        @spec_count = 0
+      end
+      attr_accessor :dbseq_count, :pep_count, :pepev_count, :spec_count
+      #
+      def start_element(name)
+        return unless ATTR.include?(name)
+        case name
+        when :DBSequence 
+          @dbseq_count += 1 
+        when :Peptide 
+          @pep_count += 1
+        when :PeptideEvidence 
+          @pepev_count += 1 
+        when :SpectrumIdentificationItem 
+          @spec_count += 1
+        end 
+      end 
+    end 
+    #
+    # handler for DBSequence elements
+    #
+    class DBSequenceHandler < Ox::Sax
+      ATTR = [:DBSequence]
+      #
+      def initialize(num_dbseq=nil)
+        @dbseq_h = Hash.new
+        @pbar = num_dbseq.nil? ? nil : ProgressBar.new("DBSeq", num_dbseq)
+      end 
+      #
+      attr_accessor :dbseq_h, :pbar
+      #
+      def start_element(name)
+        @h = {} if name == :DBSequence
+        @curr_node = name
+      end
+      #
+      def attr(name, value)
+        return unless ATTR.include?(@curr_node)
+        @h[name] = value if name == :accession || name == :id
+      end
+      #
+      def value(value)
+        return unless ATTR.include?(@curr_node)
+      end 
+      #
+      def end_element(name)
+        return unless name == :DBSequence
+        @pbar.inc if !@pbar.nil?
+        @dbseq_h[@h[:id].to_sym] = @h[:accession]
+      end 
+    end
+    #
+    # handler for Peptide elements
+    #
+    class PeptideHandler < Ox::Sax
+      ATTR = [:Peptide, :PeptideSequence, :Modification]
+      
+      def initialize(num_pep=nil)
+        @pbar = num_pep.nil? ? nil : ProgressBar.new("Peptides", num_pep)
+        @pep_h = Hash.new
+        @mod_h = Hash.new
+      end 
+
+      attr_accessor :pep_h, :mod_h, :pbar
+      #
+      def start_element(name)
+        @h = {} if name == :Peptide
+        @mh = [] if name == :Peptide
+        @curr_node = name
+      end
+      #
+      def attr(name, value)
+        return unless ATTR.include?(@curr_node)
+        @h[name] = value if @curr_node != :Modification
+        @mh.push(value) if @curr_node == :Modification
+      end
+      #
+      def text(value)
+        return unless ATTR.include?(@curr_node)
+        @h[@curr_node] = value
+      end 
+      #
+      def end_element(name)
+        return unless name == :Peptide
+        @pbar.inc if !@pbar.nil?
+        @pep_h[@h[:id].to_sym] = @h[:PeptideSequence]
+        if !@mh.empty? then
+          @mod_h[@h[:id].to_sym] = {}
+          (1..@mh.size-1).step(2) do |i|
+            @mod_h[@h[:id].to_sym][@mh[i].to_i] = @mh[i-1]
+          end
+        end
+      end 
+    end
+    #
+    # handler for PeptideEvent elements
+    #
+    class PeptideEventHandler < Ox::Sax
+      ATTR_MAP = [:post, :pre, :start, :end, :peptide_ref, :dBSequence_ref, :id, :isDecoy]
+      ATTR = [:PeptideEvidence]
+      def initialize(dbseq_h, num_pepev=nil)
+        @dbseq_h = dbseq_h
+        @pep_ev_h = Hash.new
+        @pbar = num_pepev.nil? ? nil : ProgressBar.new("PepEv", num_pepev)
+      end 
+      
+      attr_accessor :pep_ev_h, :pbar
+      
+      def start_element(name)
+        @h = {} if name == :PeptideEvidence
+        @curr_node = name
+      end
+
+      def attr(name, value)
+        return unless ATTR.include?(@curr_node)
+        @h[name] = value if ATTR_MAP.include?(name)
+      end
+      
+      def end_element(name)
+        return unless name == :PeptideEvidence
+        @pbar.inc if !@pbar.nil?
+        @pep_ev_h[@h[:id].to_sym] =  
+          PeptideEvidence.new(:db_seq_ref => @h[:dBSequence_ref].to_sym,
+                              :pep_id => @h[:peptide_ref].to_sym,
+                              :start_pos => @h[:start],
+                              :end_pos => @h[:end],
+                              :prot_id => @dbseq_h[@h[:dBSequence_ref].to_sym],
+                              :is_decoy => @h[:isDecoy])
+      end 
+
+    end 
+    #
+    # handler for SpectrumIDItem elements
+    #
+    class SpectraIDHandler < Ox::Sax
+      ATTR = [:SpectrumIdentificationItem, :PeptideEvidenceRef]  
+      SPEC_ATTR_MAP = [:peptide_ref, :id, :passThreshold]
+      SPEC_PROB_ATTR_MAP = [:accession, :value]
+      SPEC_PROB_ACC = "MS:1002052"  # code for spec-prob
+      def initialize(dbseq_h, pep_h, pep_ev_h, block, num_spec=nil)
+        @yield_to = block
+        @dbseq_h = dbseq_h
+        @pep_h = pep_h
+        @pep_ev_h = pep_ev_h
+        @spec_h = Hash.new
+        @pbar = num_spec.nil? ? nil : ProgressBar.new("Spectra", num_spec)
+      end 
+
+      attr_accessor :spec_h, :pbar
+
+      def start_element(name)
+        @h = {} if name == :SpectrumIdentificationItem
+        @curr_node = name
+        @h_param = nil if name == :cvParam
+      end
+
+      def attr(name, value)
+        return unless ATTR.include?(@curr_node) || 
+          (@curr_node == :cvParam && SPEC_PROB_ATTR_MAP.include?(name))
+        
+        @h_param[name] = value if !@h_param.nil?
+        @h_param = {} if name == :accession && value == SPEC_PROB_ACC    
+        if name == :peptideEvidence_ref then # if peptideEvidence, force into list
+          @h[name].nil? ? @h[name] = [value.to_sym] : @h[name].push(value.to_sym)
+        end 
+        @h[name] = value.to_sym if SPEC_ATTR_MAP.include?(name)
+        @h[name] = value.split("_")[1].to_i if name == :id
+      end
+      
+      def attrs_done()
+        return unless (!@h_param.nil? && !@h.nil?)
+        @h[:spec_prob] = @h_param[:value].to_f 
+      end 
+      
+      def end_element(name)
+        return unless name == :SpectrumIdentificationItem
+        @yield_to.call(@h)
+        @pbar.inc if !@pbar.nil?
+      end 
+    end
+    
+    def initialize(file, use_pbar = nil, tda_flag = true)
+      @use_pbar = use_pbar
+      @mzid_file = file
+      @tda_flag = tda_flag
+      #
+      # get counts
+      if @use_pbar then
+        count_handler = CounterHandler.new
+        File.open(@mzid_file){|f| Ox.sax_parse(count_handler, f)}     
+        @num_spec = count_handler.spec_count
+      end 
+      
+      #puts "DBSeq:\t#{count_handler.dbseq_count}"
+      #puts "Peptides:\t#{count_handler.pep_count}"
+      #puts "PepEv:\t#{count_handler.pepev_count}"
+      #puts "Spectra:\t#{count_handler.spec_count}"
+      #
+      # cache DBSequence elements
+      dbseq_handler = DBSequenceHandler.new(@use_pbar.nil? ? nil : count_handler.dbseq_count)
+      File.open(@mzid_file){|f| Ox.sax_parse(dbseq_handler, f)}
+      dbseq_handler.pbar.finish if !dbseq_handler.pbar.nil?
+      @dbseq_h = dbseq_handler.dbseq_h
+      #      
+      # cache Peptide elements
+      pep_handler = PeptideHandler.new(@use_pbar.nil? ? nil : count_handler.pep_count)
+      File.open(@mzid_file){|f| Ox.sax_parse(pep_handler, f)}
+      pep_handler.pbar.finish if !pep_handler.pbar.nil?
+      @pep_h = pep_handler.pep_h
+      @mod_h = pep_handler.mod_h
+      #
+      # create/cache PeptideEvent elements
+      pep_ev_handler = PeptideEventHandler.new(@dbseq_h, @use_pbar.nil? ? nil : count_handler.pepev_count)
+      File.open(@mzid_file){|f| Ox.sax_parse(pep_ev_handler, f)}
+      pep_ev_handler.pbar.finish if !pep_ev_handler.pbar.nil?
+      @pep_ev_h = pep_ev_handler.pep_ev_h
+      
+    end
+    #
+    # write output to specified csv file
+    #
+    def write_to_csv(outfile="result.csv", show_mods=true)
+      CSV.open(outfile, "w", {:col_sep => "\t"}) do |csv|
+        headerAry = ["#spec_num", "peptide", "spec_prob", "decoy", "prot_ids", "start", "end", "num_prot"]
+        headerAry.push("mods") if show_mods
+        headerAry.delete("decoy") if !@tda_flag
+        csv << headerAry
+        
+        proc = Proc.new do |spec_h|
+          # peptide reference/seq
+          pep_ref = spec_h[:peptide_ref].to_sym
+          pep_seq = @pep_h[pep_ref]
+          mods = @mod_h[pep_ref]
+          # peptide evidence list
+          pep_ev_ref_lst = spec_h[:peptideEvidence_ref]
+          # number of proteins with matching peptide
+          num_prot = pep_ev_ref_lst.size
+          # for each PeptideEvidence entry ...
+          pep_ev_ref_lst.each do |pep_ev_ref|
+            pep_ev = @pep_ev_h[pep_ev_ref]
+            # start/end pos within protein
+            start_pos = pep_ev.get_start_pos
+            end_pos = pep_ev.get_end_pos
+            # get protein ID
+            prot_id = pep_ev.get_prot_id
+            # get decoy flag
+            is_decoy = pep_ev.get_is_decoy
+            # write to file
+            ary = [spec_h[:id], pep_seq, spec_h[:spec_prob], is_decoy, prot_id, start_pos, end_pos, num_prot]
+            ary.delete_at(3) if !@tda_flag
+            if show_mods then
+              modstr = if !mods.nil? then
+                         mods.keys.map{|loc| val = mods[loc].to_i; [loc, val > 0 ? "+#{val}" : "-#{val}"].join(";")}.join("|")
+                       else
+                         nil
+                       end
+              ary.push(modstr)
+            end
+            csv << ary
+          end 
+          
+        end
+        spec_handler = SpectraIDHandler.new(@dbseq_h, @pep_h, @pep_ev_h, proc, @use_pbar.nil? ? nil : @num_spec)
+        File.open(@mzid_file){|f| Ox.sax_parse(spec_handler, f)}
+        spec_handler.pbar.finish if !spec_handler.pbar.nil?
+      end
+    end
+    
+  end
+  
+end

data/lib/mzid/peptide_evidence.rb

@@ -0,0 +1,39 @@
+module MzID
+  #
+  # class to represent peptide evidence entries in file
+  #
+  class PeptideEvidence
+    def initialize(h={})
+      @id = h.has_key?(:id) ? h[:id] : nil
+      @db_seq_ref = h.has_key?(:db_seq_ref) ? h[:db_seq_ref] : nil
+      @pep_id = h.has_key?(:pep_id) ? h[:pep_id] : nil
+      @start_pos = h.has_key?(:start_pos) ? h[:start_pos] : nil
+      @end_pos = h.has_key?(:end_pos) ? h[:end_pos] : nil
+      @pre = h.has_key?(:pre) ? h[:pre] : nil
+      @post = h.has_key?(:post) ? h[:post] : nil
+      @is_decoy = h.has_key?(:is_decoy) ? h[:is_decoy] : nil
+      @prot_id = h.has_key?(:prot_id) ? h[:prot_id] : nil
+    end
+    #
+    # get methods
+    #
+    def get_id() @id end
+    def get_db_seq_ref() @db_seq_ref end
+    def get_pep_id() @pep_id end
+    def get_start_pos() @start_pos end
+    def get_end_pos() @end_pos end
+    def get_pre() @pre end
+    def get_post() @post end
+    def get_is_decoy() @is_decoy end
+    def get_prot_id() @prot_id end
+    #
+    # represent as string
+    #
+    def to_s() 
+      "[#{@id}, #{@pep_id}; #{@start_pos}:#{@end_pos}, #{@pre}...#{@post}]" 
+    end
+    
+  end
+
+  
+end

data/lib/mzid/psm.rb

@@ -0,0 +1,61 @@
+
+module MzID
+  #
+  # class to represent a single peptide-spectrum match (PSM)
+  #
+  class PSM
+    
+    def initialize(h={})
+      @id = h.has_key?(:id) ? h[:id] : nil
+      @pep = h.has_key?(:pep) ? h[:pep] : nil
+      @spec_prob = h.has_key?(:spec_prob) ? h[:spec_prob] : nil
+      @peptide_ref= h.has_key?(:pep_ref) ? h[:pep_ref] : nil
+      @spec_ref = h.has_key?(:spec_ref) ? h[:spec_ref] : nil
+      @spec_num = h.has_key?(:spec_num) ? h[:spec_num] : nil
+      @mods = h.has_key?(:mods) ? h[:mods] : nil
+      @pep_evidence = h.has_key?(:pep_ev) ? h[:pep_ev] : nil
+      @pass_thresh = h.has_key?(:pass_threshold) ? h[:pass_threshold] : nil
+    end
+    #
+    #--
+    # get methods
+    #++
+    #
+    # get ID
+    def get_id() @id end
+    # get peptide sequence
+    def get_pep() @pep end
+    # get spectral probability
+    def get_spec_prob() @spec_prob end
+    # get peptide reference
+    def get_pep_ref() @peptide_ref end
+    # get spectrum reference
+    def get_spec_ref() @spec_ref end
+    # get spectrum number
+    def get_spec_num() @spec_num end
+    # get modifications
+    def get_mods() @mods end
+    # get peptide evidence
+    def get_pep_ev() @pep_evidence end
+    # get pass threshold flag
+    def get_pass_threshold() @pass_thresh end
+    #
+    #--
+    # set methods
+    #++
+    #
+    # set the peptide sequence
+    def set_pep(pep) @pep = pep end
+    # set the spectral probability
+    def set_spec_prob(prob) @spec_prob = prob end
+    # set peptide
+    def set_pep(pep_seq) @pep = pep_seq end
+    # add the peptide evidence
+    def add_pep_ev(pep_ev) @pep_evidence.nil? ? @pep_evidence = [pep_ev] : @pep_evidence.push(pep_ev) end    
+    #
+    # output PSM as string
+    #
+    def to_s() "[#{@spec_num}; Pep: #{@pep}; SpecProb: #{@spec_prob}; Mods #{@mods.to_s}]" end
+  end
+
+end

data/lib/mzid/streaming_parser.rb

@@ -0,0 +1,177 @@
+require 'nokogiri'
+require 'progressbar'
+require 'mzid/base_parser'
+require 'mzid/batch_parser'
+
+module MzID
+  #
+  # class to parse an mzIdentML file in a streaming (i.e., mem-efficient) manner
+  #
+  class StreamingParser < BatchParser
+    
+    def initialize(file, use_pbar = nil)
+      @use_pbar = use_pbar
+      @num_spec = 0
+      super(file)
+    end
+    #
+    # first pass thru file just counting element types
+    #
+    def get_num_elements(use_pbar=@use_pbar)
+      num_pep = 0
+      num_db_seq = 0
+      num_pep_ev = 0
+      # once through file to count
+      tmp_reader = Nokogiri::XML::Reader(File.open(@mzid_file))
+      tmp_reader.each do |node|
+        @num_spec += 1 if node.name == "SpectrumIdentificationResult"
+        num_pep += 1 if node.name == "Peptide"
+        num_db_seq += 1 if node.name == "DBSequence"
+        num_pep_ev += 1 if node.name == "PeptideEvidence"
+      end
+      [num_pep, num_db_seq, num_pep_ev]
+    end
+    #
+    # store peptide sequences in hash for lookup
+    #
+    def cache_ids(use_pbar = @use_pbar)
+      # num_pep = 0
+      # num_db_seq = 0
+      # num_pep_ev = 0
+      # # once through file to count
+      # tmp_reader = Nokogiri::XML::Reader(File.open(@mzid_file))
+      # tmp_reader.each do |node|
+      #   @num_spec += 1 if node.name == "SpectrumIdentificationResult"
+      #   num_pep += 1 if node.name == "Peptide"
+      #   num_db_seq += 1 if node.name == "DBSequence"
+      #   num_pep_ev += 1 if node.name == "PeptideEvidence"
+      # end
+      num_pep, num_db_seq, num_pep_ev = get_num_elements(nil)
+      # puts "SPEC:\t#{@num_spec}"
+      # puts "PEP:\t#{num_pep}"
+      # puts "DB:\t#{num_db_seq}"
+      # puts "PEPEV:\t#{num_pep_ev}"
+
+      @pep_h = Hash.new
+      @mod_h = Hash.new
+      pbar = ProgressBar.new("Caching", num_pep+num_db_seq+num_pep_ev) if use_pbar
+      reader = Nokogiri::XML::Reader(File.open(@mzid_file))
+      reader.each do |node|
+        # @num_spec += 1 if node.name == "SpectrumIdentificationResult"
+        
+        if node.name == "Peptide" then
+          # parse local peptide entry
+          tmp_node = Nokogiri::XML.parse(node.outer_xml)
+          tmp_node.remove_namespaces!
+          root = tmp_node.root
+          
+          pep_id = root["id"]
+          # skip if already handled PepID
+          next if @pep_h.has_key?(pep_id)
+          # parse sequence/mods if haven't seen it yet
+          pep_seq = get_peptide_sequence(root)
+          mod_line = get_modifications(root)
+          @pep_h[pep_id] = pep_seq 
+          @mod_h[pep_id] = mod_line 
+          pbar.inc if use_pbar
+        end
+        #
+        if node.name == "DBSequence" then
+          # parse local DBSequence entry
+          tmp_node = Nokogiri::XML.parse(node.outer_xml)
+          tmp_node.remove_namespaces!
+          root = tmp_node.root
+          cache_db_seq_entries(root)
+          pbar.inc if use_pbar
+        end
+        #
+        if node.name == "PeptideEvidence" then
+          # parse local DBSequence entry
+          tmp_node = Nokogiri::XML.parse(node.outer_xml)
+          tmp_node.remove_namespaces!
+          root = tmp_node.root
+          cache_pep_ev(root)
+          pbar.inc if use_pbar
+        end 
+
+      end
+      pbar.finish if use_pbar
+    end
+    #
+    # store peptide evidence sequences in hash for lookup
+    #
+    def cache_pep_ev(root)
+      pep_ev_lst = root.xpath('//PeptideEvidence')
+      pep_ev_lst.each do |pnode|
+        id = pnode["id"]
+        @pep_ev_h[id] = 
+          PeptideEvidence.new(#:id => pnode["id"],
+                              :db_seq_ref => pnode["dBSequence_ref"],
+                              #:pep_id => pnode["peptide_ref"],
+                              :start_pos => pnode["start"].to_i,
+                              :end_pos => pnode["end"].to_i,
+                              #:pre => pnode["pre"],
+                              #:post => pnode["post"],
+                              :prot_id => @db_seq_h[pnode["dBSequence_ref"]].to_sym)
+        # @pep_ev_h[id] = pnode["dBSequence_ref"]
+      end
+    end
+    #
+    # iterate through each psm
+    #
+    def each_psm(use_pbar=@use_pbar)
+      reader = Nokogiri::XML::Reader(File.open(@mzid_file))
+      pbar = ProgressBar.new("PSMs", @num_spec) if use_pbar
+      reader.each do |node|
+        next if node.name != "SpectrumIdentificationResult"        
+        # parse local spec result entry
+        tmp_node = Nokogiri::XML.parse(node.outer_xml)
+        tmp_node.remove_namespaces!
+        root = tmp_node.root
+        # parse spectrum id item
+        psms_of_spec = root.xpath('.//SpectrumIdentificationItem')
+        psms_of_spec.each do |psm_node|
+          # get PSM object
+          psm = get_psm(psm_node)
+          # yield psm object
+          yield psm
+        end
+        pbar.inc if use_pbar
+      end
+      pbar.finish if use_pbar
+    end
+    #
+    # given a xml node of a psm, return the PSM 
+    #
+    def get_psm(psm_node)
+      # get peptide evidence list
+      pep_ev_raw_lst = psm_node.xpath('.//PeptideEvidenceRef')
+      pep_ev_lst = pep_ev_raw_lst.map do |penode|
+        pep_ev_ref_id = penode["peptideEvidence_ref"]
+        #@db_seq_h[@pep_ev_h[pep_ev_ref_id]]  # if use simpler hash of prot ID
+        @pep_ev_h[pep_ev_ref_id]  # if use PeptideEvidence object
+      end 
+      # get cvparams
+      cvlst = psm_node.xpath('.//cvParam')
+      # find spectral prob
+      tmp_lst = cvlst.select{|v| v['name'] == "MS-GF:SpecEValue"}
+      spec_prob = tmp_lst[0]['value']
+      # get peptide
+      pep_seq = @pep_h[psm_node['peptide_ref']]
+      # get spectrum id/ref number
+      spec_id = psm_node['id']
+      spec_num = spec_id.split("_")[1].to_i
+      spec_ref = spec_id.split("_")[-1].to_i
+      # store in object
+      psm = PSM.new(:spec_num => spec_num, 
+                    :spec_ref => spec_ref, 
+                    :pep => pep_seq, 
+                    :spec_prob => spec_prob.to_f,
+                    :mods => (@mod_h.has_key?(psm_node['peptide_ref']) ? @mod_h[psm_node['peptide_ref']] : nil),
+                    :pep_ev => pep_ev_lst)
+    end
+    
+    private :get_psm, :get_num_elements
+  end
+
+end