bio-maf 1.0.0-java → 1.0.1-java

data/lib/bio/maf/jobs.rb

@@ -0,0 +1,168 @@
+require 'set'
+require 'java' if RUBY_PLATFORM == 'java'
+
+module Bio::MAF
+  
+  module JobRunner
+    def JobRunner.create(n_parallel)
+      if RUBY_PLATFORM == 'java'
+        JThreadRunner.new(n_parallel)
+      else
+        ForkRunner.new(n_parallel)
+      end
+    end
+  end
+
+  class ForkRunner
+
+    def initialize(n_parallel)
+      @n_parallel = n_parallel
+      @jobs = []
+      @kids = Set.new
+    end
+
+    def add(&proc)
+      @jobs << proc
+    end
+
+    def run
+      until @jobs.empty? && @kids.empty?
+        while can_start?
+          start_job
+        end
+        await
+      end
+    end
+
+    private
+    
+    def can_start?
+      (! @jobs.empty?) && @kids.size < @n_parallel
+    end
+
+    def start_job
+      job = @jobs.shift
+      pid = fork()
+      if pid
+        # parent
+        @kids << pid
+      else
+        # child
+        begin
+          job.call()
+          exit 0
+        rescue SystemExit
+          raise
+        rescue Exception
+          LOG.error $!
+          exit 1
+        end
+      end
+    end
+
+    def await
+      pid = Process.wait
+      unless @kids.delete?(pid)
+        raise "Completion of unexpected job #{pid}!"  
+      end
+      if ! $?.success?
+        raise "Job #{pid} failed with status #{status.exitstatus}!"
+      end
+    end
+
+  end
+
+  class JThreadRunner
+
+    def initialize(n_parallel)
+      @n_parallel = n_parallel
+      @exec = java.util.concurrent.Executors.newFixedThreadPool(n_parallel)
+      @ecs = java.util.concurrent.ExecutorCompletionService.new(@exec)
+      @n = 0
+    end
+
+    def add(&blk)
+      @ecs.submit(&blk)
+      @n += 1
+    end
+
+    def run
+      seen = 0
+      until seen == @n
+        f = @ecs.take()
+        begin
+          f.get()
+        rescue Exception => e
+          LOG.error e
+          @exec.shutdownNow()
+          raise
+        end
+        seen += 1
+      end
+      @exec.shutdown()
+    end
+
+  end
+
+  module Executor
+    def Executor.create
+      if RUBY_PLATFORM == 'java'
+        JExecutor.new
+      else
+        DummyExecutor.new
+      end
+    end
+  end
+
+  class JExecutor
+
+    def initialize
+      queue = java.util.concurrent.LinkedBlockingQueue.new(8)
+      policy = java.util.concurrent.ThreadPoolExecutor::CallerRunsPolicy.new
+      @exec = java.util.concurrent.ThreadPoolExecutor.new(1, 1, 1,
+                                                          java.util.concurrent.TimeUnit::MINUTES,
+                                                          queue,
+                                                          policy)
+      @ecs = java.util.concurrent.ExecutorCompletionService.new(@exec)
+      @submitted = 0
+      @completed = 0
+    end
+
+    def submit(&blk)
+      @ecs.submit(&blk)
+      @submitted += 1
+      check_for_errors
+    end
+
+    def check_for_errors
+      while f = @ecs.poll
+        f.get
+        @completed += 1
+      end
+    end
+
+    def shutdown
+      @exec.shutdown
+      until @completed == @submitted
+        f = @ecs.take
+        f.get
+        @completed += 1
+      end
+    end
+  end
+
+  class DummyExecutor
+
+    def initialize
+    end
+
+    def submit
+      yield
+    end
+
+    def shutdown
+    end
+    
+  end
+
+end

data/lib/bio/maf/maf.rb

@@ -58,10 +58,14 @@ module Bio
       attr_reader :sequences
       # Offset of the alignment block within the MAF file, in bytes.
       # @return [Integer]
-      attr_reader :offset
+      attr_accessor :offset
       # Size of the alignment block within the MAF file, in bytes.
       # @return [Integer]
       attr_reader :size
+      # Original text of the MAF block. Only available if the
+      # :retain_text parser option is set.
+      # @return [String]
+      attr_accessor :orig_text
 
       def initialize(vars, sequences, offset, size, filtered)
         @vars = vars
@@ -90,6 +94,10 @@ module Bio
         sequences.first.text.size
       end
 
+      def upcase!
+        sequences.each { |s| s.upcase! }
+      end
+
       # Whether this block has been modified by a parser filter.
       # @return [Boolean]
       def filtered?
@@ -101,6 +109,13 @@ module Bio
         Bio::BioAlignment::Alignment.new(ba_seq)
       end
 
+      def to_s
+        buf = StringIO.new
+        writer = Writer.new(buf)
+        writer.write_block(self)
+        return buf.string
+      end
+
       GAP = /-+/
 
       # Find gaps present in all sequences. These would generally
@@ -356,6 +371,10 @@ module Bio
         end
       end
 
+      def upcase!
+        text.upcase!
+      end
+
       def to_bio_alignment
         Bio::BioAlignment::Sequence.new(source, text)
       end
@@ -471,6 +490,10 @@ module Bio
         true
       end
 
+      def upcase!
+        # no-op
+      end
+
       def write_fasta(writer)
         raise "empty sequence output not implemented!"
       end

data/lib/bio/maf/parser.rb

@@ -1,4 +1,5 @@
 require 'strscan'
+require 'zlib'
 require 'java' if RUBY_PLATFORM == 'java'
 require 'bio-bgzf'
 
@@ -104,6 +105,7 @@ module Bio
       
       # Spawn a read-ahead thread. Called from {#initialize}.
       def start_read_ahead
+        LOG.debug { "Starting read-ahead thread." }
         @read_thread = Thread.new { read_ahead }
       end
 
@@ -169,6 +171,7 @@ module Bio
       BLOCK_START = /^(?=a)/
       BLOCK_START_OR_EOS = /(?:^(?=a))|\z/
       EOL_OR_EOF = /\n|\z/
+      JRUBY_P = (RUBY_PLATFORM == 'java')
 
       def set_last_block_pos!
         @last_block_pos = s.string.rindex(BLOCK_START)
@@ -333,14 +336,22 @@ module Bio
           elsif [I, E, Q, COMMENT, nil].include? first
             next
           else
-            parse_error "unexpected line: '#{line}'"
+            if opts[:strict]
+              parse_error "unexpected line: '#{line}'"
+            else
+              LOG.warn "Ignoring invalid MAF line: '#{line}'"
+            end
           end
         end
-        Block.new(block_vars,
-                  seqs,
-                  block_offset,
-                  s.pos - block_start_pos,
-                  filtered)
+        b = Block.new(block_vars,
+                      seqs,
+                      block_offset,
+                      s.pos - block_start_pos,
+                      filtered)
+        if opts[:retain_text]
+          b.orig_text = s.string.slice(block_start_pos...(s.pos))
+        end
+        return b
       end
 
       # Parse an 's' line.
@@ -504,12 +515,16 @@ module Bio
     #  * `:parse_extended`: whether to parse 'i' and 'q' lines
     #  * `:parse_empty`: whether to parse 'e' lines
     #  * `:remove_gaps`: remove gaps left after filtering sequences
+    #  * `:join_blocks`: join blocks where possible
+    #  * `:upcase`: fold sequence data to upper case
     #  * `:chunk_size`: read MAF file in chunks of this many bytes
     #  * `:random_chunk_size`: as above, but for random access ({#fetch_blocks})
     #  * `:merge_max`: merge up to this many bytes of blocks for
     #    random access
     #  * `:threads`: number of threads to use for parallel
     #    parsing. Only useful under JRuby.
+    #  * `:strict`: abort on un-parseable lines instead of continuing with
+    #    a warning.
     # @api public
     
     class Parser
@@ -519,8 +534,12 @@ module Bio
       attr_reader :header
       # @return [String] path of MAF file being parsed.
       attr_reader :file_spec
-      # @return [File] file handle for MAF file.
+      # @return [IO] file handle for MAF file.
       attr_reader :f
+      # May be gzip-compressed.
+      # @return [IO] file handle for physical MAF file.
+      # @api private
+      attr_reader :phys_f
       # @return [StringScanner] scanner for parsing.
       attr_reader :s
       # @return [ChunkReader] ChunkReader.
@@ -547,33 +566,47 @@ module Bio
       RANDOM_CHUNK_SIZE = 4096
       MERGE_MAX = SEQ_CHUNK_SIZE
 
+      DEFAULT_OPTS = {
+        :chunk_size => SEQ_CHUNK_SIZE,
+        :random_chunk_size => RANDOM_CHUNK_SIZE,
+        :merge_max => MERGE_MAX,
+        :parse_extended => false,
+        :parse_empty => false,
+        :readahead_thread => true,
+        :seq_parse_thread => true
+      }
+      if JRUBY_P
+        DEFAULT_OPTS[:threads] = java.lang.Runtime.runtime.availableProcessors
+      end
+
       # Create a new parser instance.
       #
       # @param [String] file_spec path of file to parse.
-      # @param [Hash] opts parser options.
+      # @param [Hash] parse_opts parser options.
       # @api public
-      def initialize(file_spec, opts={})
+      def initialize(file_spec, parse_opts={})
+        opts = DEFAULT_OPTS.merge(parse_opts)
         @opts = opts
-        if RUBY_PLATFORM == 'java'
-          opts[:threads] ||= java.lang.Runtime.runtime.availableProcessors
-        end
-        chunk_size = opts[:chunk_size] || SEQ_CHUNK_SIZE
-        @random_access_chunk_size = opts[:random_chunk_size] || RANDOM_CHUNK_SIZE
-        @merge_max = opts[:merge_max] || MERGE_MAX
-        @parse_extended = opts[:parse_extended] || false
-        @parse_empty = opts[:parse_empty] || false
+        @random_access_chunk_size = opts[:random_chunk_size]
+        @merge_max = opts[:merge_max]
+        @parse_extended = opts[:parse_extended]
+        @parse_empty = opts[:parse_empty]
         @chunk_start = 0
         if file_spec.respond_to? :flush
+          # an IO object
           # guess what, Pathnames respond to :read...
           @f = file_spec
           @file_spec = @f.path if @f.respond_to?(:path)
-          # TODO: gzip?
+          # TODO: test for gzip?
         else
+          # a pathname (or Pathname)
           @file_spec = file_spec
+          @phys_f = File.open(file_spec)
           if file_spec.to_s.end_with?(".maf.gz")
-            @f = IO.popen("gzip -dc #{file_spec}")
+            @f = Zlib::GzipReader.new(@phys_f)
+            @compression = :gzip
           else
-            @f = File.open(file_spec)
+            @f = @phys_f
           end
         end
         if @file_spec.to_s =~ /\.bgzf?$/
@@ -582,8 +615,9 @@ module Bio
         else
           @base_reader = ChunkReader
         end
-        @cr = base_reader.new(@f, chunk_size)
-        if RUBY_PLATFORM == 'java'
+        @cr = base_reader.new(@f, opts[:chunk_size])
+        if JRUBY_P && opts[:readahead_thread]
+          LOG.debug "Using ThreadedChunkReaderWrapper."
           @cr = ThreadedChunkReaderWrapper.new(@cr)
         end
         @s = StringScanner.new(cr.read_chunk())
@@ -649,7 +683,7 @@ module Bio
       def fetch_blocks(fetch_list, &blk)
         if blk
           merged = merge_fetch_list(fetch_list)
-          if RUBY_PLATFORM == 'java' && @opts.fetch(:threads, 1) > 1
+          if JRUBY_P && @opts.fetch(:threads, 1) > 1
             fun = lambda { |&b2| fetch_blocks_merged_parallel(merged, &b2) }
           else
             fun = lambda { |&b2| fetch_blocks_merged(merged, &b2) }
@@ -667,15 +701,17 @@ module Bio
       def fetch_blocks_merged(fetch_list, &blk)
         start = Time.now
         total_size = fetch_list.collect { |e| e[1] }.reduce(:+)
+        count = 0
         with_context(@random_access_chunk_size) do |ctx|
           fetch_list.each do |e|
             ctx.fetch_blocks(*e, &blk)
+            count += 1
           end
         end
         elapsed = Time.now - start
         rate = (total_size / 1048576.0) / elapsed
-        LOG.debug { sprintf("Fetched blocks in %.3fs, %.1f MB/s.",
-                            elapsed, rate) }
+        LOG.debug { sprintf("Fetched %d blocks in %.3fs, %.1f MB/s.",
+                            count, elapsed, rate) }
       end
 
       # Fetch and parse the blocks given by the merged fetch list, in
@@ -807,7 +843,9 @@ module Bio
           end
         end
         @header = Header.new(vars, align_params)
-        s.skip_until BLOCK_START || parse_error("Cannot find block start!")
+        if ! s.skip_until(BLOCK_START)
+          @at_end = true
+        end
       end
 
       # Parse all alignment blocks until EOF.
@@ -820,7 +858,7 @@ module Bio
       # @api public
       def each_block(&blk)
         if block_given?
-          if RUBY_PLATFORM == 'java' && @opts.has_key?(:threads)
+          if JRUBY_P && opts[:seq_parse_thread]
             fun = method(:parse_blocks_parallel)
           else
             fun = method(:each_block_seq)
@@ -847,11 +885,12 @@ module Bio
         b
       end
 
-      WRAP_OPTS = [:as_bio_alignment, :join_blocks, :remove_gaps]
+      WRAP_OPTS = [:as_bio_alignment, :join_blocks, :remove_gaps, :upcase]
 
       def wrap_block_seq(fun, &blk)
         opts = WRAP_OPTS.find_all { |o| @opts[o] }
         opts << :sequence_filter if sequence_filter && (! sequence_filter.empty?)
+        LOG.debug { "wrapping #{fun} with #{opts.inspect}" }
         _wrap(opts, fun, &blk)
       end
 
@@ -873,6 +912,12 @@ module Bio
                     fun,
                     :to_bio_alignment,
                     &blk)
+        when :upcase
+          conv_send(options,
+                    fun,
+                    :upcase!,
+                    true,
+                    &blk)
         when :remove_gaps
           conv_map(options,
                    fun,
@@ -910,10 +955,14 @@ module Bio
         end
       end
 
-      def conv_send(options, search, sym)
+      def conv_send(options, search, sym, always_yield_block=false)
         _wrap(options, search) do |block|
           v = block.send(sym)
-          yield v if v
+          if always_yield_block
+            yield block
+          else
+            yield v if v
+          end
         end
       end
 
@@ -925,14 +974,17 @@ module Bio
         queue = java.util.concurrent.LinkedBlockingQueue.new(128)
         worker = Thread.new do
           begin
+            LOG.debug "Starting parse worker."
             until at_end
               block = _parse_block()
               queue.put(block) if block
             end
             queue.put(:eof)
-          rescue
-            LOG.error "worker exiting: #{$!.class}: #{$!}"
+            LOG.debug { "Parse worker reached EOF." }
+          rescue Exception
             LOG.error $!
+            Thread.current[:exception] = $!
+            raise
           end
         end
         saw_eof = false
@@ -946,12 +998,15 @@ module Bio
             yield block
           else
             # timed out
-            n_final_poll += 1 unless worker.alive?
+            unless worker.alive?
+              LOG.debug "Worker has exited."
+              n_final_poll += 1
+            end
           end
           break if n_final_poll > 1
         end
         unless saw_eof
-          raise "worker exited unexpectedly!"
+          raise "worker exited unexpectedly from #{worker[:exception]}!"
         end
       end
 
@@ -1000,7 +1055,7 @@ module Bio
 
     def handle_logging_options(opts)
       opts.on("--logger filename", String,
-              "Log to file (default STDOUT)") do |name|
+              "Log to file (default STDERR)") do |name|
         Bio::Log::CLI.logger(name)
       end
       opts.on("--trace options", String,