ms-msrun 0.0.1 → 0.1.0

data/README

@@ -4,7 +4,7 @@ A library for working with LC/MS runs.
 
 == Examples
 
-The following example will work on *mzXML*, *mzData* ( and *mzML* files, shortly)!
+The following example will work on *mzXML*, *mzData* ( and *mzML* files in near future)!
 
     require "ms/msrun" 
 
@@ -55,8 +55,7 @@ The following example will work on *mzXML*, *mzData* ( and *mzML* files, shortly
     gem install ms-msrun
 
 The library currently relies on xmlparser (though LibXML is close to being
-supported).  After installation of ms-msrun (which will automatically install
+supported).  After installation of ms-msrun (which should automatically install
 `axml`) issue this command to get instructions on installing xmlparser:
 
-    ruby -rubygems -e 'require "axml" \ 
-        puts AXML::Autoload.install_instructions(:xmlparser)'
+    ruby -rubygems -e 'require "axml"; puts AXML::Autoload.install_instructions(:xmlparser)'

data/Rakefile

@@ -35,11 +35,26 @@ Rake::RDocTask.new do |rd|
   rd.options.push( *rdoc_options )
 end
 
-desc "create and upload docs to server"
-task :upload_docs => [:rdoc] do
-  sh "scp -r #{rdoc_dir}/* jtprince@rubyforge.org:/var/www/gforge-projects/mspire/ms-msrun/"
+
+desc "Publish RDoc to RubyForge"
+task :publish_rdoc => [:rdoc] do
+  require 'yaml'
+  
+  config = YAML.load(File.read(File.expand_path("~/.rubyforge/user-config.yml")))
+  host = "#{config["username"]}@rubyforge.org"
+  
+  rsync_args = "-v -c -r"
+  remote_dir = "/var/www/gforge-projects/mspire/projects/#{NAME}"
+  local_dir = "rdoc"
+ 
+  sh %{rsync #{rsync_args} #{local_dir}/ #{host}:#{remote_dir}}
 end
 
+#desc "create and upload docs to server"
+#task :upload_docs => [:rdoc] do
+#  sh "scp -r #{rdoc_dir}/* jtprince@rubyforge.org:/var/www/gforge-projects/mspire/projects/ms-msrun/"
+#end
+
 ###############################################
 # TESTS
 ###############################################
@@ -77,12 +92,12 @@ gemspec = Gem::Specification.new do |t|
   t.has_rdoc = true
   t.authors = ["John Prince"]
   t.files = dist_files
-  t.add_dependency 'axml', '~> 0.0.5'
+  t.add_dependency 'ms-core'
+  t.add_dependency 'nokogiri'
   t.add_dependency 'runarray'
   t.rdoc_options = rdoc_options
   t.extra_rdoc_files = rdoc_extra_includes
   t.executables = FL["bin/*"].map {|file| File.basename(file) }
-  t.requirements << 'xmlparser (preferrably) or libxml'
   t.test_files = FL["spec/**/*_spec.rb"]
 end
 

data/bin/base64_to_array.rb

@@ -1,5 +1,6 @@
+#!/usr/bin/ruby
 
-require 'ms/spectrum'
+require 'ms/data/lazy_string'
 
 if ARGV.size == 0
   puts "usage: #{File.basename(__FILE__)} <base64_string>"
@@ -7,7 +8,5 @@ if ARGV.size == 0
   exit
 end
 
-precision = 32
-network_order = true
-ar = Ms::Spectrum.base64_to_array(ARGV.shift, precision, network_order)
+ar = Ms::Data::LazyString.new(ARGV.shift).to_a
 puts "[ " + ar.join(", ") + " ]"

data/bin/ms_to_obiwarp.rb

@@ -21,8 +21,8 @@ opts = OptionParser.new do |op|
   op.separator "(sums m/z values that round to the same bin)"
   op.separator ""
   op.on("--mz-inc N", Float, "m/z increment (def: 1.0)") {|n| opt[:mz_inc] = n.to_f}
-  op.on("--mz-start N", Float, "m/z start (def: start of 1st full scan)") {|n| opt[:start_mz] = n.to_f}
-  op.on("--mz-end N", Float, "m/z end (def: end of 1st full scan)") {|n| opt[:end_mz] = n.to_f}
+  op.on("--mz-start N", Float, "m/z start (otherwise auto)") {|n| opt[:start_mz] = n.to_f}
+  op.on("--mz-end N", Float, "m/z end (otherwise auto)") {|n| opt[:end_mz] = n.to_f}
   op.on("--baseline N", Float, "value for missing indices (def: #{opt[:baseline]})") {|n| opt[:baseline] = n.to_f}
   op.on("--ascii", "generates an lmata file instead") {opt[:ascii] = true}
   op.on("-v", "--verbose") {$VERBOSE = true}
@@ -36,21 +36,12 @@ end
 ARGV.each do |file|
   Ms::Msrun.open(file) do |msrun|
     mslevel = 1
-    (start_mz, end_mz) = msrun.start_and_end_mz(mslevel)
-    (times, spectra) = msrun.times_and_spectra(mslevel)
-    args = {
-      :start_mz => start_mz,
-      :end_mz => end_mz,
-
-      :start_tm => times.first,
-      :end_tm => times.last,
-      :inc_tm => nil,
-    }
-    args.merge!(opt)
-    lmat = Ms::Msrun::Lmat.new.from_times_and_spectra(times, spectra, args)
+    lmat = Lmat.new
+    t = Time.now
+    lmat.from_msrun(msrun, opt)
     ext = File.extname(file)
     outfile = file.sub(/#{Regexp.escape(ext)}$/, opt[:newext])
-      if args[:ascii]
+      if opt[:ascii]
         outfile << "a"
         lmat.print(outfile)
       else

data/bin/ms_to_search.rb

@@ -1,37 +1,30 @@
 #!/usr/bin/ruby
 
 require 'rubygems'
-require 'tap'
+require 'optparse'
+require 'ms/msrun/search'
 
+opts = OptionParser.new do |op|
+  op.banner = "usage: #{File.basename(__FILE__)} <file>.mzXML ..."
+  op.separator "outputs: <file>.mgf"
+  op.separator "[by default outputs .mgf file]"
 
-module Ms ; end
-class Ms::Msrun ; end
-
-# Documentation here
-class Ms::Msrun::Search < Tap::Task
-  
-  #config :first_scan, 0, :short => 'F', &c.integer # first scan
-  #config :last_scan, 1e12, :short => 'L', &c.integer  # last scan
-  ## if not determined to be +1, then create these charge states
-  #config( :charge_states, [2,3], :short => 'c') {|v| v.split(',') }
-  #config :bottom_mh, 0, :short => 'B', &c.float # bottom MH+ 
-  #config :top_mh, -1.0, :short => 'T', &c.float # top MH+
-  #config :min_peaks, 0, :short => 'P', &c.integer # minimum peak count
-  #config :ms_levels, 2..-1, :short => 'M', &c.range  # ms levels to export
+end
 
+if ARGV.size == 0
+  puts opts.to_s
+  exit
+end
 
-  def process(filename)
-    Ms::Msrun.open(filename) do |ms|
-      ms.to_mgf(ms.filename.chomp(File.extname(ms.filename)))
-    end
+ARGV.each do |file|
+  Ms::Msrun.open(file) do |ms|
+    outfile = ms.to_mgf(file.sub(/\.mzxml/i, '.mgf'))
+    ms.to_mgf(outfile)
   end
 end
 
-Ms::Msrun::Search.execute
-
 # extract_msn.exe -M0.2 -B85 -T4500 -S0 -G1 -I35 -C0 -P2 -D output smallraw.RAW
 
-
   #config :group_mass_tol, 1.4, :short => 'M', &c.float # prec. mass tolerance for grouping
   #config :bottom_mw, 0.0, :short => 'B', &c.float # bottom MW for data file creation
   #config :top_mw, 999999.0, :short => 'T', &c.float # top MW for data file creation

data/lib/lmat.rb

@@ -64,49 +64,65 @@ class Lmat
     self
   end
 
-  # converts raw times and spectrum to a labeled matrix
-  # times is an array (or VecI object)
-  # where each row = [mz,inten,mz,inten...]
+  # converts msrun object to a labeled matrix
   # takes hash with symbols as keys
-  # if inc_tm is undefined, then times from the times array will be used
-  def from_times_and_spectra(times, spectra, args)
+  def from_msrun(msrun, args)
     opt = {
-      :start_mz => 400.0, 
-      :end_mz => 1500.0,
       :inc_mz => 1.0,
       :behave_mz => 'sum', 
-
-      :start_tm => 0.0, 
-      :end_tm => 3600.0,
-      :inc_tm => nil,
-
       :baseline=> 0.0,
+
+      #:start_tm => 0.0, 
+      #:end_tm => 3600.0,
+      #:inc_tm => nil,
+      
+      #:start_mz => 400.0, 
+      #:end_mz => 1500.0,
     }
     opt.merge!(args)
-    unless opt[:start_tm] then opt[:start_tm] = times.first end
-    unless opt[:end_tm] then opt[:end_tm] = times.last end
-    
+
+    (st, en) = msrun.start_and_end_mz
+    unless st && en
+      msg = ["scanning spectrum for start and end m/z values"]
+      msg << "(use :start_mz and :end_mz options to avoid this)"
+      warn msg
+      (st, en) = msrun.start_and_end_mz_brute_force
+    end
+    opt[:start_mz] ||= st
+    opt[:end_mz] ||= en
+
+    #unless opt[:start_tm] then opt[:start_tm] = times.first end
+    #unless opt[:end_tm] then opt[:end_tm] = times.last end
+
     if opt[:inc_tm]
       raise NotImplementedError, "haven't implemented interpolation in ruby yet! (#{File.basename(__FILE__)}: #{__LINE__})"
     else ## No interpolation
-      if times.first != opt[:start_tm] || times.last != opt[:end_tm]
-        abort "haven't implemented yet! (#{File.basename(__FILE__)}: #{__LINE__})"
-      else
-        @mvec = NArray.new(times)
-        give_vecs = true
-        vecs = spectra.map do |spectrum|
-          #(mz,inten) = spectrum_to_mz_and_inten(spectrum, VecD)
-          # TODO: Figure out a shallow copy here:
-          # perhaps we'll make spectra Vec objects by default in future and
-          # then we'd be set...
-          mzs = NArray.new(spectrum.mzs)
-          intens = NArray.new(spectrum.intensities)
-          (x,y) = mzs.inc_x(intens, opt[:start_mz], opt[:end_mz], opt[:inc_mz], opt[:baseline], opt[:behave_mz])
-          @nvec = x # ends up being the last one, but that's OK
-          y
+      times = []
+      @nvec = nil
+      vecs = []
+      num_scans = msrun.scan_count
+      printf "Reading #{num_scans} spectra [.=100]" if $VERBOSE
+      spectrum_cnt = 0
+      msrun.each do |scan|
+        spectrum = scan.spectrum
+        times << scan.time
+        #(mz,inten) = spectrum_to_mz_and_inten(spectrum, VecD)
+        # TODO: Figure out a shallow copy here:
+        # perhaps we'll make spectra Vec objects by default in future and
+        # then we'd be set...
+        mzs = NArray.new(spectrum.mzs)
+        intens = NArray.new(spectrum.intensities)
+        (x,y) = mzs.inc_x(intens, opt[:start_mz], opt[:end_mz], opt[:inc_mz], opt[:baseline], opt[:behave_mz])
+        spectrum_cnt += 1
+        if spectrum_cnt % 100 == 0
+          printf "." if $VERBOSE ; $stdout.flush
         end
-        @mat = vecs
+        @nvec ||= x # just need the first one for the x values
+        vecs << y
       end
+      puts "DONE!" if $VERBOSE
+      @mvec = NArray.new(times)
+      @mat = vecs
     end
     self  
   end
@@ -124,28 +140,6 @@ class Lmat
      other != nil && self.class == other.class && @nvec == other.nvec && @mvec == other.mvec && @mat == other.mat
   end
 
-  # converts a single array of alternating m/z intensity values to two
-  # separate arrays
-  # (maybe implement in Ruby::Inline?)
-  # the answer is given in terms of arrs_as (object of class "arrs_as" must
-  # respond to "[]" and create a certain sized array with arrs_as.new(size))
-  def spectrum_to_mz_and_inten(spectrum, arrs_as=Array)
-    half_size = spectrum.size / 2
-    mzs = arrs_as.new(half_size)
-    intens = arrs_as.new(half_size)
-    mz = true 
-    spectrum.each_index do |i|
-      if mz
-        mzs[i/2] = spectrum[i]
-        mz = false
-      else
-        mz = true
-        intens[(i-1)/2] = spectrum[i]
-      end
-    end
-    [mzs, intens]
-  end
-
   def write(file=nil)
     handle = $>
     if file; handle = File.open(file, "wb") end

data/lib/ms/msrun.rb

@@ -3,9 +3,15 @@ require 'ms/scan'
 require 'ms/precursor'
 require 'ms/spectrum'
 require 'ms/msrun/search'
+require 'ms/msrun/index'
 
 module Ms; end
 class Ms::Msrun
+  include Enumerable
+
+  #DEFAULT_PARSER = 'axml'
+  #DEFAULT_PARSER = 'regexp'
+  DEFAULT_PARSER = 'nokogiri'
 
   # the retention time in seconds of the first scan (regardless of any
   # meta-data written in the header)
@@ -13,8 +19,6 @@ class Ms::Msrun
   # the retention time in seconds of the last scan (regardless of any
   # meta-data written in the header)
   attr_accessor :end_time
-  # an array of scans
-  attr_accessor :scans
 
   # The filetype. Valid types (for parsing) are:
   #   :mzxml
@@ -39,6 +43,15 @@ class Ms::Msrun
   # this will be nil.
   attr_accessor :parent_location
 
+  # an array of doublets, [start_byte, length] for each scan element
+  attr_accessor :index
+
+  # holds the class that parses the file
+  attr_accessor :parser
+
+  # an array holding the scan numbers found in the run
+  attr_accessor :scan_nums
+
   # Opens the filename 
   def self.open(filename, &block)
     File.open(filename) {|io| block.call( self.new(io, filename) ) }
@@ -50,10 +63,13 @@ class Ms::Msrun
   def initialize(io, filename=nil)
     @scan_counts = nil
     @filename = filename
-    @filetype, @version = Utils.filetype_and_version(io)
-    parser = Utils.get_parser(@filetype, @version)
-    parser.new.parse(self, io, @version)
-    @scan_counts = nil  # <- to keep warnings away
+    @filetype, @version = Ms::Msrun.filetype_and_version(io)
+    parser_klass = Ms::Msrun.get_parser(@filetype, @version)
+
+    @parser = parser_klass.new(self, io, @version)
+    @index = Ms::Msrun::Index.new(io)
+    @scan_nums = @index.scan_nums
+    @parser.parse_header(@index.header_length)
   end
 
   def parent_basename_noext
@@ -61,32 +77,59 @@ class Ms::Msrun
   end
 
   # returns each scan
-  def each(&block)
-    scans.each(&block)
+  # options:
+  #     :spectrum => true | false (default is true)
+  #     :precursor => true | false (default is true)
+  #     :ms_level => Integer or Array return only scans of that level
+  #     :reverse => true | false (default is false) goes backwards
+  def each_scan(parse_opts={}, &block)
+    ms_levels = 
+      if msl = parse_opts[:ms_level]
+        if msl.is_a?(Integer) ; [msl]
+        else ; msl  
+        end
+      end
+    snums = @index.scan_nums
+    snums = snums.reverse if parse_opts[:reverse]
+    snums.each do |scan_num|
+      if ms_levels
+        next unless ms_levels.include?(ms_level(scan_num))
+      end
+      block.call(scan(scan_num, parse_opts))
+    end
   end
+  alias_method :each, :each_scan
 
   # opens the file and yields each scan in the block
-  def self.foreach(filename, &block)
+  # see each_scan for parsing options
+  def self.foreach(filename, parse_opts={}, &block)
     self.open(filename) do |obj|
-      obj.each(&block)
+      obj.each_scan(parse_opts, &block)
     end
   end
 
-  def scans_by_ms_level
-    by_level = []
-    scans.each do |scan|
-      by_level[scan.ms_level] = scan
-    end
-    by_level
+  # a very fast method to only query the ms_level of a scan
+  def ms_level(num)
+    @parser.parse_ms_level(@index[num].first, @index[num].last)
   end
 
+  # returns a Ms::Scan object for the scan at that number
+  #
+  def scan(num, parse_opts={})
+    #@parser.parse_scan(*(@index[num]), parse_opts)
+    @parser.parse_scan(@index[num].first, @index[num].last, parse_opts)
+  end
+
+  #bracket_method = '[]'.to_sym
+  #alias_method bracket_method, :scan
+
   # returns an array, whose indices provide the number of scans in each index level the ms_levels, [0] = all the scans, [1] = mslevel 1, [2] = mslevel 2,
   # ...
   def scan_counts
     return @scan_counts if @scan_counts
     ar = []
     ar[0] = 0
-    scans.each do |sc|
+    each_scan do |sc|
       level = sc.ms_level
       unless ar[level]
         ar[level] = 0
@@ -104,113 +147,57 @@ class Ms::Msrun
       if mslevel == 0
         @scan_count 
       else
-        num = 0
-        scans.each do |sc|
-          if sc.ms_level == mslevel
-            num += 1
-          end
-        end
-        num
+        scan_counts[mslevel]
       end
     end
   end
 
-  # for level 1, finds first scan and asks if it has start_mz/end_mz
-  # attributes.  for other levels, asks for start_mz/ end_mz and takes the
-  # min/max.  If start_mz and end_mz are not found, goes through every scan
-  # finding the max/min first and last m/z. returns [start_mz (rounded down to
-  # nearest int), end_mz (rounded up to nearest int)]
-  def start_and_end_mz(mslevel=1)
-    if mslevel == 1
-      # special case for mslevel 1 (where we expect scans to be same length)
-      scans.each do |sc|
-        if sc.ms_level == mslevel
-          if sc.start_mz && sc.end_mz
-            return [sc.start_mz, sc.end_mz]
-          end
-          break
-        end
-      end
-    end
-    hi_mz = nil
-    lo_mz = nil
-    # see if we have start_mz and end_mz for the level we want
-    # set the initial hi_mz and lo_mz in any case
-    have_start_end_mz = false
-    scans.each do |sc|
-      if sc.ms_level == mslevel
-        if sc.start_mz && sc.end_mz
-          lo_mz = sc.start_mz
-          hi_mz = sc.end_mz
-        else
-          mz_ar = sc.spectrum.mzs
-          hi_mz = mz_ar.last
-          lo_mz = mz_ar.first
-        end
-        break
-      end
-    end
-    if have_start_end_mz
-      scans.each do |sc|
-        if sc.ms_level == mslevel
-          if sc.start_mz < lo_mz
-            lo_mz = sc.start_mz
-          end
-          if sc.end_mz > hi_mz
-            hi_mz = sc.end_mz
-          end
-        end
+
+  # returns [start_mz, end_mz] or [nil,nil] if unknown
+  def start_and_end_mz
+    scan = first(:ms_level => 1, :spectrum => false, :precursor => false)
+    [scan.start_mz, scan.end_mz]
+  end
+
+  # goes through every scan and gets the first and last m/z, then returns the
+  # max.ceil and min.floor
+  def start_and_end_mz_brute_force
+    first_scan = first(:ms_level => 1, :precursor => false)
+    first_mzs = first_scan.spectrum.mzs
+
+    lo_mz = first_mzs[0]
+    hi_mz = first_mzs[-1]
+
+    each_scan(:ms_level => 1, :precursor => false) do |sc|
+      mz_ar = sc.spectrum.mzs
+      if mz_ar.last > hi_mz
+        hi_mz = mz_ar.last
       end
-    else
-      # didn't have the attributes (find by brute force)
-      scans.each do |sc|
-        if sc.ms_level == mslevel
-          mz_ar = sc.spectrum.mzs
-          if mz_ar.last > hi_mz
-            hi_mz = mz_ar.last
-          end
-          if mz_ar.last < lo_mz
-            lo_mz = mz_ar.last
-          end
-        end
+      if mz_ar.last < lo_mz
+        lo_mz = mz_ar.last
       end
     end
     [lo_mz.floor, hi_mz.ceil]
   end
 
-  # returns an array of times and parallel array of spectra objects.
-  # ms_level = 0  then all spectra and times
-  # ms_level = 1 then all spectra of ms_level 1
-  def times_and_spectra(ms_level=0)
-    spectra = []
-    if ms_level == 0
-      times = @scans.map do |scan|
-        spectra << scan.spectrum  
-        scan.time
-      end
-      [times, spectra]
-    else  # choose a particular ms_level
-      times = []
-      @scans.each do |scan|
-        if ms_level == scan.ms_level
-          spectra << scan.spectrum  
-          times << scan.time
-        end
-      end
-      [times, spectra]
+  def first(opts={})
+    the_first = nil
+    each_scan(opts) do |scan|
+      the_first = scan
+      break
     end
+    the_first
   end
-end
-
 
-module Ms::Msrun::Axml ; end  # so we can get our parser
-
-module Ms::Msrun::Utils
+  def last(opts={})
+    opts[:reverse] = true
+    first(opts)
+  end
 
   def self.get_parser(filetype, version)
-    require "ms/msrun/axml/#{filetype}"
+    require "ms/msrun/#{DEFAULT_PARSER}/#{filetype}"
     parser_class = filetype.to_s.capitalize
-    base_class = Ms::Msrun::Axml
+    base_class = Ms::Msrun.const_get( DEFAULT_PARSER.capitalize )
     if base_class.const_defined? parser_class
       base_class.const_get parser_class
     else
@@ -247,6 +234,7 @@ module Ms::Msrun::Utils
     end
   end
 
+
   Mzxml_regexp = /http:\/\/sashimi.sourceforge.net\/schema(_revision)?\/([\w\d_\.]+)/o
   # 'http://sashimi.sourceforge.net/schema/MsXML.xsd' # version 1
   # 'http://sashimi.sourceforge.net/schema_revision/mzXML_X.X' # others
@@ -258,6 +246,7 @@ module Ms::Msrun::Utils
     if file_or_io.is_a? IO
       io = file_or_io
       found = nil
+      io.rewind
       # Test for RAW file:
       header = io.read(18).unpack(Raw_header_unpack_code).join
       if header == 'Finnigan'
@@ -294,4 +283,5 @@ module Ms::Msrun::Utils
       end
     end
   end
+
 end