bio-maf 0.2.0 → 0.3.0

data/lib/bio/maf/maf.rb

@@ -1,4 +1,15 @@
+require 'bio-alignment'
+
 module Bio
+  class GenomicInterval
+    def intersection(other)
+      raise ArgumentError unless self.chrom == other.chrom
+      GenomicInterval.new(self.chrom,
+                          [self.chr_start, other.chr_start].max,
+                          [self.chr_end, other.chr_end].min)
+    end
+  end
+
   module MAF
 
     # A MAF header, containing the variable-value pairs from the first
@@ -12,6 +23,12 @@ module Bio
       # @return [Hash]
       attr_accessor :alignment_params
 
+      # Create a default header with version=1.
+      # @return [Header]
+      def Header.default
+        Header.new({:version => 1}, nil)
+      end
+
       def initialize(vars, params)
         @vars = vars
         @alignment_params = params
@@ -47,6 +64,7 @@ module Bio
       attr_reader :size
 
       def initialize(vars, sequences, offset, size, filtered)
+        #raise ArgumentError, "no sequences given for block at offset #{offset}!" unless sequences && sequences.first
         @vars = vars
         @sequences = sequences
         @offset = offset
@@ -79,6 +97,11 @@ module Bio
         @filtered
       end
 
+      def to_bio_alignment
+        ba_seq = sequences.collect { |s| s.to_bio_alignment }
+        Bio::BioAlignment::Alignment.new(ba_seq)
+      end
+
       GAP = /-+/
 
       # Remove gaps present in all sequences. These would generally
@@ -116,6 +139,96 @@ module Bio
         gaps.size
       end
 
+      # Returns a new Block covering only the region where it overlaps
+      # the given interval.
+      # @param [Bio::GenomicInterval] interval to slice the block with
+      # @return [Block] block covering intersection with interval
+      def slice(interval)
+        case interval.compare(ref_seq.interval)
+        when :equal
+          return self
+        when :contains, :contained_by, :left_overlapped, :right_overlapped
+          _slice(interval.intersection(ref_seq.interval))
+        when :left_adjacent, :right_adjacent, :left_off, :right_off
+          raise "Cannot slice a block with a non-overlapping interval! Block #{ref_seq.interval}, interval #{interval}"
+        when :different_chrom
+          raise "Cannot slice a block with reference sequence #{ref_seq.source} using an interval on #{interval.chrom}!"
+        else
+          raise "Unhandled comparison result: #{interval.compare(ref_seq.interval)}"
+        end
+      end
+
+      def _slice(interval)
+        range = _slice_text_range(interval)
+        s2 = sequences.collect { |s| s.slice(range) }
+        v2 = vars.dup
+        #v2[:score] = '0.0'
+        # TODO: should the filtered param be #modified? instead?
+        Block.new(v2, s2, offset, size, @filtered)
+      end
+
+      def _slice_text_range(interval)
+        i_start  = interval.zero_start
+        i_end    = interval.zero_end
+        g_pos    = ref_seq.start
+        t_start  = nil
+        t_end    = nil
+        ref_seq.text.each_char.each_with_index do |c, t_pos|
+          if c != '-'
+            # non-gap
+            if g_pos == i_start
+              t_start = t_pos
+            end
+            g_pos += 1
+            if t_start && g_pos == i_end
+              t_end = t_pos + 1
+              break
+            end
+          end
+        end
+        unless t_start && t_end
+          raise "did not find start and end for #{interval} in #{ref_seq.inspect}!"
+        end
+        return t_start...t_end
+      end
+
+      def joinable_with?(other)
+        if sequences.size == other.sequences.size
+          r1 = ref_seq
+          r2 = other.ref_seq
+          return false if r1.source != r2.source
+          return false if r1.end != r2.start
+          rest = sequences.each_with_index
+          rest.next
+          mismatch = rest.find do |s1, i|
+            s2 = other.seq_from(s1.source, i)
+            (! s2) || ! s1.joinable_with?(s2)
+          end
+          return (! mismatch)
+        else
+          return false
+        end
+      end
+
+      def join(other)
+        nseq = sequences.each_with_index.collect do |s1, i|
+          s2 = other.seq_from(s1.source, i)
+          s1.join(s2)
+        end
+        v2 = vars.dup
+        v2[:score] = '0.0'
+        Block.new(v2, nseq, offset, nil, @filtered)
+      end
+
+      def seq_from(src, pos_guess)
+        sg = sequences[pos_guess]
+        if sg.source == src
+          sg
+        else
+          sequences.find { |s| s.source == src }
+        end
+      end
+
     end
 
     # A sequence within an alignment block.
@@ -153,6 +266,29 @@ module Bio
         start + size
       end
 
+      def interval
+        GenomicInterval.zero_based(self.source, self.start, self.end)
+      end
+
+      def slice(range)
+        before = text.slice(0...(range.begin))
+        non_gap_prev = before.delete("-").size
+        new_text = text.slice(range)
+        unless new_text
+          raise "could not extract slice #{range} from #{self.inspect}!"
+        end
+        non_gap_text = new_text.delete("-").size
+        s2 = Sequence.new(source,
+                          start + non_gap_prev,
+                          non_gap_text,
+                          strand,
+                          src_size,
+                          new_text)
+        s2.quality = quality.slice(range) if quality
+        # TODO: what to do with synteny data?
+        s2
+      end
+
       # Whether this sequence is empty. Only true for {EmptySequence}
       # instances from 'e' lines.
       def empty?
@@ -163,6 +299,43 @@ module Bio
         size != text.size
       end
 
+      I_STATUS = {
+        'C' => :contiguous,
+        'I' => :intervening,
+        'N' => :first,
+        'n' => :first_bridged,
+        'M' => :missing_data,
+        'T' => :tandem
+      }
+
+      def decode_status_char(c)
+        I_STATUS[c] || raise("Unsupported status character #{c}!")
+      end
+
+      def left_status_char
+        i_data && i_data[0]
+      end
+
+      def left_status
+        i_data && decode_status_char(left_status_char())
+      end
+
+      def left_count
+        i_data && i_data[1].to_i
+      end
+
+      def right_status_char
+        i_data && i_data[2]
+      end
+
+      def right_status
+        i_data && decode_status_char(right_status_char())
+      end
+
+      def right_count
+        i_data && i_data[3].to_i
+      end
+
       def species
         parts = source.split('.', 2)
         parts.size == 2 ? parts[0] : nil
@@ -177,11 +350,34 @@ module Bio
         end
       end
 
+      def to_bio_alignment
+        Bio::BioAlignment::Sequence.new(source, text)
+      end
+
       def write_fasta(writer)
         writer.write("#{source}:#{start}-#{start + size}",
                      text)
       end
 
+      def joinable_with?(o)
+        (self.end == o.start) \
+        && (self.strand == o.strand) \
+        && (self.empty? == o.empty?)
+      end
+
+      def join(o)
+        s2 = Sequence.new(source,
+                          start,
+                          size + o.size,
+                          strand,
+                          src_size,
+                          text + o.text)
+        if quality && o.quality
+          s2.quality = quality + o.quality
+        end
+        s2
+      end
+
       # Maps the given zero-based genomic range onto a range of string
       # offsets, suitable for extracting the text for the given range
       # from #text.
@@ -253,6 +449,19 @@ module Bio
         ''
       end
 
+      def slice(offset, len)
+        self
+      end
+
+      def join(o)
+        EmptySequence.new(source,
+                          start,
+                          size + o.size,
+                          strand,
+                          src_size,
+                          @status)
+      end
+
       def empty?
         true
       end

data/lib/bio/maf/parser.rb

@@ -150,7 +150,7 @@ module Bio
       #
       # @return [Block] alignment block
       # @api public
-      def parse_block
+      def _parse_block
         return nil if at_end
         if s.pos != last_block_pos
           # in non-trailing block
@@ -296,19 +296,11 @@ module Bio
             parse_error "unexpected line: '#{line}'"
           end
         end
-        block = Block.new(block_vars,
-                          seqs,
-                          block_offset,
-                          s.pos - block_start_pos,
-                          filtered)
-        postprocess_block(block)
-      end
-
-      def postprocess_block(block)
-        if block.filtered? && opts[:remove_gaps]
-          block.remove_gaps!
-        end
-        block
+        Block.new(block_vars,
+                  seqs,
+                  block_offset,
+                  s.pos - block_start_pos,
+                  filtered)
       end
 
       # Parse an 's' line.
@@ -419,19 +411,21 @@ module Bio
       # @param [Array] block_offsets Offsets of blocks to parse.
       # @return [Array<Block>]
       def fetch_blocks(offset, len, block_offsets)
-        start_chunk_read_if_needed(offset, len)
-        # read chunks until we have the entire merged set of
-        # blocks ready to parse
-        # to avoid fragment joining
-        append_chunks_to(len)
-        # parse the blocks
-        Enumerator.new do |y|
+        if block_given?
+          start_chunk_read_if_needed(offset, len)
+          # read chunks until we have the entire merged set of
+          # blocks ready to parse
+          # to avoid fragment joining
+          append_chunks_to(len)
+          # parse the blocks
           block_offsets.each do |expected_offset|
-            block = parse_block
-            ctx.parse_error("expected a block at offset #{expected_offset} but could not parse one!") unless block
-            ctx.parse_error("got block with offset #{block.offset}, expected #{expected_offset}!") unless block.offset == expected_offset
-            y << block
+            block = _parse_block
+            parse_error("expected a block at offset #{expected_offset} but could not parse one!") unless block
+            parse_error("got block with offset #{block.offset}, expected #{expected_offset}!") unless block.offset == expected_offset
+            yield block
           end
+        else
+          enum_for(:fetch_blocks, offset, len, block_offsets)
         end
       end
 
@@ -530,6 +524,10 @@ module Bio
         _parse_header()
       end
 
+      def close
+        f.close
+      end
+
       # Create a {ParseContext} for random access, using the given
       # chunk size.
       #
@@ -574,13 +572,19 @@ module Bio
       # `fetch_list` should be an array of `[offset, length]` tuples.
       #
       # @param [Array] fetch_list the fetch list
-      # @return [Array<Block>] the requested alignment blocks
-      def fetch_blocks(fetch_list)
-        merged = merge_fetch_list(fetch_list)
-        if RUBY_PLATFORM == 'java' && @opts.fetch(:threads, 1) > 1
-          fetch_blocks_merged_parallel(merged)
+      # @yield [block] each block matched, in turn
+      # @return [Enumerable<Block>] each matching {Block}, if no block given
+      def fetch_blocks(fetch_list, &blk)
+        if blk
+          merged = merge_fetch_list(fetch_list)
+          if RUBY_PLATFORM == 'java' && @opts.fetch(:threads, 1) > 1
+            fun = lambda { |&b2| fetch_blocks_merged_parallel(merged, &b2) }
+          else
+            fun = lambda { |&b2| fetch_blocks_merged(merged, &b2) }
+          end
+          wrap_block_seq(fun, &blk)
         else
-          fetch_blocks_merged(merged)
+          enum_for(:fetch_blocks, fetch_list)
         end
       end
 
@@ -588,23 +592,19 @@ module Bio
       #
       # @param [Array] fetch_list merged fetch list from {#merge_fetch_list}.
       # @return [Array<Block>] the requested alignment blocks
-      def fetch_blocks_merged(fetch_list)
-        Enumerator.new do |y|
-          start = Time.now
-          total_size = fetch_list.collect { |e| e[1] }.reduce(:+)
-          with_context(@random_access_chunk_size) do |ctx|
-            fetch_list.each do |e|
-              ctx.fetch_blocks(*e).each do |block|
-                y << block
-                #total_size += block.size
-              end
-            end
+      def fetch_blocks_merged(fetch_list, &blk)
+        start = Time.now
+        total_size = fetch_list.collect { |e| e[1] }.reduce(:+)
+        with_context(@random_access_chunk_size) do |ctx|
+          fetch_list.each do |e|
+            ctx.fetch_blocks(*e, &blk)
           end
-          elapsed = Time.now - start
-          rate = (total_size / 1048576.0) / elapsed
-          $stderr.printf("Fetched blocks in %.3fs, %.1f MB/s.\n",
-                         elapsed, rate)
         end
+        elapsed = Time.now - start
+        # TODO: debug log
+        # rate = (total_size / 1048576.0) / elapsed
+        # $stderr.printf("Fetched blocks in %.3fs, %.1f MB/s.\n",
+        #                elapsed, rate)
       end
 
       # Fetch and parse the blocks given by the merged fetch list, in
@@ -614,40 +614,38 @@ module Bio
       # @param [Array] fetch_list merged fetch list from {#merge_fetch_list}.
       # @return [Array<Block>] the requested alignment blocks
       def fetch_blocks_merged_parallel(fetch_list)
-        Enumerator.new do |y|
-          total_size = fetch_list.collect { |e| e[1] }.reduce(:+)
-          start = Time.now
-          n_threads = @opts.fetch(:threads, 1)
-          # TODO: break entries up into longer runs for more
-          # sequential I/O
-          jobs = java.util.concurrent.ConcurrentLinkedQueue.new(fetch_list)
-          ct = CompletionTracker.new(fetch_list)
-          completed = ct.queue
-          threads = []
-          n_threads.times { threads << make_worker(jobs, ct) }
-
-          n_res = 0
-          while n_res < fetch_list.size
-            c = completed.poll(1, java.util.concurrent.TimeUnit::SECONDS)
-            unless c
-              raise "Worker failed!" if threads.find { |t| t.status.nil? }
-              next
-            end
-            c.each do |block|
-              y << block
-            end
-            n_res += 1
+        total_size = fetch_list.collect { |e| e[1] }.reduce(:+)
+        start = Time.now
+        n_threads = @opts.fetch(:threads, 1)
+        # TODO: break entries up into longer runs for more
+        # sequential I/O
+        jobs = java.util.concurrent.ConcurrentLinkedQueue.new(fetch_list)
+        ct = CompletionTracker.new(fetch_list)
+        completed = ct.queue
+        threads = []
+        n_threads.times { threads << make_worker(jobs, ct) }
+
+        n_res = 0
+        while n_res < fetch_list.size
+          c = completed.poll(1, java.util.concurrent.TimeUnit::SECONDS)
+          unless c
+            raise "Worker failed!" if threads.find { |t| t.status.nil? }
+            next
           end
-          threads.each { |t| t.join }
-          elapsed = Time.now - start
-          $stderr.printf("Fetched blocks from %d threads in %.1fs.\n",
-                         n_threads,
-                         elapsed)
-          mb = total_size / 1048576.0
-          $stderr.printf("%.3f MB processed (%.1f MB/s).\n",
-                         mb,
-                         mb / elapsed)
+          c.each do |block|
+            yield block
+          end
+          n_res += 1
         end
+        threads.each { |t| t.join }
+        elapsed = Time.now - start
+        $stderr.printf("Fetched blocks from %d threads in %.1fs.\n",
+                       n_threads,
+                       elapsed)
+        mb = total_size / 1048576.0
+        $stderr.printf("%.3f MB processed (%.1f MB/s).\n",
+                       mb,
+                       mb / elapsed)
       end
 
       # Create a worker thread for parallel parsing.
@@ -721,30 +719,119 @@ module Bio
       # Delegates to {#parse_blocks_parallel} if `:threads` is set
       # under JRuby.
       #
-      # @return [Enumerator<Block>] enumerator of alignment blocks.
+      # @return [Enumerator<Block>] enumerator of {Block}s if no block given.
+      # @yield [block] Passes each {Block} in turn to a block
       # @api public
-      def parse_blocks
-        if RUBY_PLATFORM == 'java' && @opts.has_key?(:threads)
-          parse_blocks_parallel
+      def each_block(&blk)
+        if block_given?
+          if RUBY_PLATFORM == 'java' && @opts.has_key?(:threads)
+            fun = method(:parse_blocks_parallel)
+          else
+            fun = method(:each_block_seq)
+          end
+          wrap_block_seq(fun, &blk)
         else
-          Enumerator.new do |y|
-            until at_end
-              y << parse_block()
-            end
+          enum_for(:each_block)
+        end
+      end
+      alias_method :parse_blocks, :each_block
+
+      def each_block_seq
+        until at_end
+          block = _parse_block()
+          yield block if block
+        end
+      end
+
+      def parse_block
+        b = nil
+        wrap_block_seq(lambda { |&blk| blk.call(_parse_block()) }) do |block|
+          b = block
+        end
+        b
+      end
+
+      WRAP_OPTS = [:as_bio_alignment, :join_blocks, :remove_gaps]
+
+      def wrap_block_seq(fun, &blk)
+        opts = WRAP_OPTS.find_all { |o| @opts[o] }
+        opts << :sequence_filter if sequence_filter && (! sequence_filter.empty?)
+        _wrap(opts, fun, &blk)
+      end
+
+      # options should be [:outer, ..., :inner]
+      def _wrap(options, fun, &blk)
+        first = options.shift
+        case first
+        when nil
+          fun.call(&blk)
+        when :sequence_filter
+          conv_map(options,
+                   fun,
+                   lambda { |b| b if b.sequences.size > 1 },
+                   &blk)
+        when :join_blocks
+          block_joiner(options, fun, &blk)
+        when :as_bio_alignment
+          conv_send(options,
+                    fun,
+                    :to_bio_alignment,
+                    &blk)
+        when :remove_gaps
+          conv_map(options,
+                   fun,
+                   lambda { |b| b.remove_gaps! if b.filtered?; b },
+                   &blk)
+        else
+          raise "unhandled wrapper mode: #{first}"
+        end
+      end
+
+      def filter_seq_count(fun)
+        fun.call() do |block|
+          yield block if block.filtered? && block.sequences.size > 1
+        end
+      end
+
+      def block_joiner(options, fun)
+        prev = nil
+        _wrap(options, fun) do |cur|
+          if prev && (prev.filtered? || cur.filtered?) \
+            && prev.joinable_with?(cur)
+            prev = prev.join(cur)
+          else
+            yield prev if prev
+            prev = cur
           end
         end
+        yield prev if prev
+      end
+
+      def conv_map(options, search, fun)
+        _wrap(options, search) do |block|
+          v = fun.call(block)
+          yield v if v
+        end
+      end
+
+      def conv_send(options, search, sym)
+        _wrap(options, search) do |block|
+          v = block.send(sym)
+          yield v if v
+        end
       end
 
       # Parse alignment blocks with a worker thread.
       #
-      # @return [Enumerator<Block>] enumerator of alignment blocks.
+      # @block block handler
       # @api private
       def parse_blocks_parallel
         queue = java.util.concurrent.LinkedBlockingQueue.new(128)
         worker = Thread.new do
           begin
             until at_end
-              queue.put(parse_block())
+              block = _parse_block()
+              queue.put(block) if block
             end
             queue.put(:eof)
           rescue
@@ -752,31 +839,23 @@ module Bio
             $stderr.puts $!.backtrace.join("\n")
           end
         end
-        Enumerator.new do |y|
-          saw_eof = false
-          n_final_poll = 0
-          while true
-            block = queue.poll(1, java.util.concurrent.TimeUnit::SECONDS)
-            if block == :eof
-              saw_eof = true
-              break
-            elsif block
-              y << block
-            else
-              # timed out
-              n_final_poll += 1 unless worker.alive?
-            end
-            break if n_final_poll > 1
-          end
-          unless saw_eof
-            raise "worker exited unexpectedly!"
+        saw_eof = false
+        n_final_poll = 0
+        while true
+          block = queue.poll(1, java.util.concurrent.TimeUnit::SECONDS)
+          if block == :eof
+            saw_eof = true
+            break
+          elsif block
+            yield block
+          else
+            # timed out
+            n_final_poll += 1 unless worker.alive?
           end
+          break if n_final_poll > 1
         end
-      end
-
-      def each_block
-        until at_end
-          yield parse_block()
+        unless saw_eof
+          raise "worker exited unexpectedly!"
         end
       end