bio-maf 0.2.0 → 0.3.0

data/features/maf-indexing.feature

@@ -29,3 +29,26 @@ Feature: Indexed access to MAF files
     Then 2 blocks are obtained
     And sequence mm8.chr7 of block 0 has start 80082592
     And sequence mm8.chr7 of block 1 has start 80082713
+
+  @no_jruby
+  Scenario: Build MAF index with CLI tool
+    Given test files:
+    | mm8_chr7_tiny.maf |
+    When I run `maf_index mm8_chr7_tiny.maf mm8_chr7_tiny.kct`
+    Then it should pass with:
+    """
+    """
+    And a file named "mm8_chr7_tiny.kct" should exist
+
+  @no_jruby
+  Scenario: Dump MAF index with CLI tool
+    Given test files:
+    | mm8_chr7_tiny.maf |
+    | mm8_chr7_tiny.kct |
+    When I run `maf_index -d mm8_chr7_tiny.kct`
+    Then it should pass with regex:
+    """
+    0 \[bin 1195\] 80082334:80082368
+    """
+
+  

data/features/maf-to-fasta.feature

@@ -48,3 +48,12 @@ Feature: Convert MAF file to FASTA
     
     """
 
+  @no_jruby
+  Scenario: Convert MAF to FASTA with CLI tool
+    Given test files:
+    | mm8_chr7_tiny.maf |
+    When I run `maf_to_fasta mm8_chr7_tiny.maf mm8_chr7_tiny.fa`
+    Then it should pass with:
+    """
+    """
+    And the file "mm8_chr7_tiny.fa" should contain ">rn4.chr1:136011785-136011819"

data/features/slice.feature

@@ -0,0 +1,54 @@
+Feature: MAF slicing
+  In order to obtain just the alignment data covering a given region
+  I want to be able to take slices of alignment blocks over
+  A given interval
+
+  Scenario: Interval covering two blocks
+    Given a MAF source file "mm8_chr7_tiny.maf"
+    And a Kyoto Cabinet index file "mm8_chr7_tiny.kct"
+    When I open it with a MAF reader
+    And I enable the :remove_gaps parser option
+    And open a new MAF writer
+    And write the header from the original MAF file
+    And filter for only the species
+      | mm8 |
+      | rn4 |
+    And search for blocks between positions 80082350 and 80082380 of mm8.chr7
+    And slice the resulting blocks according to the given interval
+    And write all the matched blocks
+    Then the output should match, except whitespace, "mm8_chr7_tiny_slice1.maf"
+
+  Scenario: Interval covering two blocks, using directory access
+    Given indexed MAF files in "test/data"
+    When I enable the :remove_gaps parser option
+    And open a new MAF writer
+    And write a default header
+    And filter for only the species
+      | mm8 |
+      | rn4 |
+    And I extract a slice over the genomic interval
+      | chrom    |    start |      end |
+      | mm8.chr7 | 80082350 | 80082380 |
+    And write all the matched blocks
+    Then the output should match, except whitespace, "mm8_chr7_tiny_slice1.maf"
+    
+  Scenario: Interval in block subset
+    Given indexed MAF files in "test/data"
+    When I open a new MAF writer
+    And write a default header
+    And I extract a slice over the genomic interval
+      | chrom    |    start |      end |
+      | mm8.chr7 | 80082718 | 80082728 |
+    And write all the matched blocks
+    Then the output should match, except whitespace, "mm8_chr7_tiny_slice2.maf"
+    
+  Scenario: Interval to end of block
+    Given indexed MAF files in "test/data"
+    When I open a new MAF writer
+    And write a default header
+    And I extract a slice over the genomic interval
+      | chrom    |    start |      end |
+      | mm8.chr7 | 80082757 | 80082767 |
+    And write all the matched blocks
+    Then the output should match, except whitespace, "mm8_chr7_tiny_slice3.maf"
+    

data/features/step_definitions/dir-access_steps.rb

@@ -0,0 +1,15 @@
+Given /^indexed MAF files in "(.*?)"$/ do |dir|
+  @opts ||= {}
+  @access = Bio::MAF::Access.maf_dir(dir, @opts)
+end
+
+When /^I query for the genomic intervals$/ do |table|
+  # table is a Cucumber::Ast::Table
+  intervals = table.hashes.collect do |row|
+    Bio::GenomicInterval.zero_based(row['chrom'],
+                                    row['start'].to_i,
+                                    row['end'].to_i)
+  end
+  @access.block_filter = @block_filter
+  @blocks = @access.find(intervals).to_a
+end

data/features/step_definitions/file_steps.rb

@@ -0,0 +1,7 @@
+Given /^test files:$/ do |table|
+  Pathname.new("tmp/aruba").mkpath
+  table.raw.collect { |row| $test_data + row[0] }.each do |path|
+    $stderr.puts "staging #{path}"
+    system("cp #{path} tmp/aruba/")
+  end
+end

data/features/step_definitions/gap_removal_steps.rb

@@ -17,3 +17,7 @@ end
 Then /^the text size of the block is (\d+)$/ do |e_text_size|
   @block.text_size.should == e_text_size.to_i
 end
+
+Then /^the text size of block (\d+) is (\d+)$/ do |n, e_text_size|
+  @blocks[n.to_i].text_size.should == e_text_size.to_i
+end

data/features/step_definitions/index_steps.rb

@@ -11,10 +11,10 @@ Then /^the index has at least (\d+) entries$/ do |size_spec|
 end
 
 When /^search for blocks between positions (\d+) and (\d+) of (\S+)$/ do |i_start, i_end, chr|
-  int = Bio::GenomicInterval.zero_based(chr, i_start.to_i, i_end.to_i)
-  @blocks = @idx.find([int], @parser, @block_filter).to_a
+  @interval = Bio::GenomicInterval.zero_based(chr, i_start.to_i, i_end.to_i)
+  @blocks = @idx.find([@interval], @parser, @block_filter).to_a
 end
 
 Then /^(\d+) blocks? (?:is|are) obtained$/ do |num|
-  @blocks.size.should == num.to_i
+  @blocks.count.should == num.to_i
 end

data/features/step_definitions/output_steps.rb

@@ -1,4 +1,4 @@
-When /^open a new MAF writer$/ do
+When /^(?:I )?open a new MAF writer$/ do
   @dst = Tempfile.new(["cuke", ".maf"])
   @writer = Bio::MAF::Writer.new(@dst)
 end
@@ -7,10 +7,18 @@ When /^write the header from the original MAF file$/ do
   @writer.write_header(@parser.header)
 end
 
+When /^write a default header$/ do
+  @writer.write_header(Bio::MAF::Header.default)
+end
+
 When /^write all the parsed blocks$/ do
   @writer.write_blocks(@parser.parse_blocks)
 end
 
+When /^write all the matched blocks$/ do
+  @writer.write_blocks(@blocks)
+end
+
 RSpec::Matchers.define :match_except_ws do |expected|
   match do |actual|
     system("diff --ignore-space-change --brief #{expected} #{actual} >/dev/null 2>&1")

data/features/step_definitions/parse_steps.rb

@@ -4,8 +4,15 @@ When /^I open it with a MAF reader$/ do
 end
 
 When /^I enable the :(\S+) parser option$/ do |opt_s|
-  @opts ||= {}
-  @opts[opt_s.to_sym] = true
+  if @parser
+    opts = @parser.opts
+  elsif @access
+    opts = @access.parse_options
+  else
+    @opts ||= {}
+    opts = @opts
+  end
+  opts[opt_s.to_sym] = true
 end
 
 Then /^the MAF version should be "(.*?)"$/ do |v_spec|
@@ -29,6 +36,10 @@ Then /^the alignment block has (\d+) sequences$/ do |n_seq|
   @block.sequences.size.should == n_seq.to_i
 end
 
+Then /^block (\d+) has (\d+) sequences$/ do |block_n, n_seq|
+  @blocks[block_n.to_i].sequences.size.should == n_seq.to_i
+end
+
 Then /^sequence (\d+) has (\w.*?) "(.*?)"$/ do |i, method, str|
   method_sym = method.gsub(/ /, '_').to_sym
   @block.raw_seq(i.to_i).send(method_sym).should == str

data/features/step_definitions/query_steps.rb

@@ -1,20 +1,21 @@
-When /^filter for only the species$/ do |table|
+When /^(?:I )?filter for only the species$/ do |table|
   # table is a Cucumber::Ast::Table
   sp = table.raw.collect { |row| row[0] }
-  @parser.sequence_filter = { :only_species => sp }
+  thing = @access || @parser
+  thing.sequence_filter = { :only_species => sp }
 end
 
-When /^filter for blocks with the species$/ do |table|
+When /^(?:I )?filter for blocks with the species$/ do |table|
   # table is a Cucumber::Ast::Table
   sp = table.raw.collect { |row| row[0] }
   @block_filter = { :with_all_species => sp }
 end
 
-When /^filter for blocks with at least (\d+) sequences$/ do |n|
+When /^(?:I )?filter for blocks with at least (\d+) sequences$/ do |n|
   @block_filter = { :at_least_n_sequences => n.to_i }
 end
 
-When /^filter for blocks with text size at (least|most) (\d+)$/ do |op, len|
+When /^(?:I )?filter for blocks with text size at (least|most) (\d+)$/ do |op, len|
   constraint = case op
                when 'least' then :min_size
                when 'most' then :max_size
@@ -23,7 +24,7 @@ When /^filter for blocks with text size at (least|most) (\d+)$/ do |op, len|
   @block_filter = { constraint => len.to_i}
 end
 
-When /^filter for blocks with text size between (\d+) and (\d+)$/ do |min, max|
+When /^(?:I )?filter for blocks with text size between (\d+) and (\d+)$/ do |min, max|
   @block_filter = {
     :min_size => min.to_i,
     :max_size => max.to_i

data/features/step_definitions/slice_steps.rb

@@ -0,0 +1,15 @@
+When /^slice the resulting blocks according to the given interval$/ do
+  # @blocks and @interval
+  @blocks = @blocks.collect { |b| b.slice(@interval) }
+end
+
+When /^I extract a slice over the genomic interval$/ do |table|
+  # table is a Cucumber::Ast::Table
+  intervals = table.hashes.collect do |row|
+    Bio::GenomicInterval.zero_based(row['chrom'],
+                                    row['start'].to_i,
+                                    row['end'].to_i)
+  end
+  intervals.size.should == 1
+  @blocks = @access.slice(intervals[0])
+end

data/features/support/aruba.rb

@@ -0,0 +1 @@
+require 'aruba/cucumber'

data/features/support/env.rb

@@ -9,7 +9,9 @@ end
 require 'pathname'
 require 'tempfile'
 
-$LOAD_PATH << File.expand_path('../../../lib', __FILE__)
+lib_dir = File.expand_path('../../../lib', __FILE__)
+$LOAD_PATH << lib_dir
+ENV['RUBYLIB'] = lib_dir
 
 require 'bio-maf'
 

data/features/{gap-filling.feature → tiling.feature}

@@ -154,5 +154,90 @@ Feature: Join alignment blocks with reference data
     **********AGGTTTAGGG******************************
     """
 
+  @no_jruby
+  Scenario: Tile with CLI tool and reference seq
+    Given test files:
+    | gap-sp1.fa.gz |
+    | gap-1.maf     |
+    | gap-1.kct     |
+    When I run `maf_tile --reference gap-sp1.fa.gz --interval 0:50 -s sp1:mouse -s sp2:nautilus -s sp3:jaguar gap-1.maf gap-1.kct`
+    Then it should pass with:
+    """
+    >mouse
+    CCAGGATGCTGGGCTGAGGGC--AGTTGTGTCAGGGCGGTCCGGTGCAGGCA
+    >nautilus
+    **********GGGCTGACGGC--AG*******AGGGCGGTGC**********
+    >jaguar
+    **********AGGTTTAGGGCAGAG***************************
+    """
+
+  @no_jruby
+  Scenario: Tile with CLI tool and no reference seq
+    Given test files:
+    | gap-1.maf     |
+    | gap-1.kct     |
+    When I run `maf_tile --interval 0:50 -s sp1:mouse -s sp2:nautilus -s sp3:jaguar gap-1.maf gap-1.kct`
+    Then it should pass with:
+    """
+    >mouse
+    NNNNNNNNNNGGGCTGAGGGC--AGNNNNNNNAGGGCGGTCCNNNNNNNNNN
+    >nautilus
+    **********GGGCTGACGGC--AG*******AGGGCGGTGC**********
+    >jaguar
+    **********AGGTTTAGGGCAGAG***************************
+    """
+
+  @no_jruby
+  Scenario: Tile with CLI tool and BED intervals
+    Given test files:
+    | gap-1.maf     |
+    | gap-1.kct     |
+    | gap-sp1.fa.gz |
+    And a file named "example.bed" with:
+    """
+    sp1.chr1 12 36
+    """
+    When I run `maf_tile -s sp1:mouse -s sp2:nautilus -s sp3:jaguar --output-base selected --bed example.bed --reference gap-sp1.fa.gz gap-1.maf gap-1.kct`
+    Then the file "selected_12-36.fa" should contain exactly:
+    """
+    >mouse
+    GCTGAGGGC--AGTTGTGTCAGGGCG
+    >nautilus
+    GCTGACGGC--AG*******AGGGCG
+    >jaguar
+    GTTTAGGGCAGAG*************
+    
+    """
 
+  @no_jruby
+  Scenario: Tile with CLI tool and implicit index
+    Given test files:
+    | mm8_chr7_tiny.maf |
+    | mm8_chr7_tiny.kct |
+    When I run `maf_tile -s mm8 -s rn4 -s hg18 --interval 80082334:80082344 mm8_chr7_tiny.maf`
+    Then it should pass with:
+    """
+    >mm8
+    GGGCTGAGGG
+    >rn4
+    GGGCTGAGGG
+    >hg18
+    --------GG
+    """
+
+  @no_jruby
+  Scenario: Tile with CLI tool and directory
+    Given test files:
+    | mm8_chr7_tiny.maf |
+    | mm8_chr7_tiny.kct |
+    When I run `maf_tile -s mm8 -s rn4 -s hg18 --interval mm8.chr7:80082334:80082344 .`
+    Then it should pass with:
+    """
+    >mm8
+    GGGCTGAGGG
+    >rn4
+    GGGCTGAGGG
+    >hg18
+    --------GG
+    """
 

data/lib/bio/maf/index.rb

@@ -61,6 +61,204 @@ module Bio
       end
     end
 
+    # Top-level class for working with a set of indexed MAF
+    # files. Provides a higher-level alternative to working with
+    # {Parser} and {KyotoIndex} objects directly.
+    #
+    # Instantiate with {Access.maf_dir} and {Access.file} methods.
+    class Access
+
+      # Parser options.
+      # @return [Hash]
+      # @see Parser
+      attr_accessor :parse_options
+      # Sequence filter to apply.
+      # @return [Hash]
+      # @see Parser#sequence_filter
+      attr_accessor :sequence_filter
+      # Block filter to apply.
+      # @return [Hash]
+      # @see KyotoIndex#find
+      attr_accessor :block_filter
+      attr_reader :indices
+
+      # Provides access to a directory of indexed MAF files. Any files
+      # with .maf suffixes and accompanying .kct indexes in the given
+      # directory will be accessible.
+      # @param [String] dir directory to scan
+      # @param [Hash] options parser options
+      # @return [Access]
+      def self.maf_dir(dir, options={})
+        o = options.dup
+        o[:dir] = dir
+        self.new(o)
+      end
+
+      # Provides access to a single MAF file. If this file is not
+      # indexed, it will be fully parsed to create a temporary
+      # in-memory index. For large MAF files or ones which will be
+      # used multiple times, this is inefficient, and an index file
+      # should be created with maf_index(1).
+      #
+      # @param [String] maf path to MAF file
+      # @param [String] index Kyoto Cabinet index file
+      # @param [Hash] options parser options
+      # @return [Access]
+      def self.file(maf, index=nil, options={})
+        o = options.dup
+        o[:maf] = maf
+        o[:index] = index if index
+        self.new(o)
+      end
+
+      # Close all open resources, in particular Kyoto Cabinet database
+      # handles.
+      def close
+        @indices.values.each { |ki| ki.close }
+      end
+
+      # Find all alignment blocks in the genomic regions in the list
+      # of Bio::GenomicInterval objects, and parse them with the given
+      # parser.
+      #
+      # @param [Enumerable<Bio::GenomicInterval>] intervals genomic
+      #  intervals to parse.
+      # @yield [block] each {Block} matched, in turn
+      # @return [Enumerable<Block>] each matching {Block}, if no block given
+      # @api public
+      # @see KyotoIndex#find
+      def find(intervals, &blk)
+        if block_given?
+          by_chrom = intervals.group_by { |i| i.chrom }
+          by_chrom.keys.each do |chrom|
+            unless @indices.has_key? chrom
+              raise "No index available for chromosome #{chrom}!"
+            end
+          end
+          by_chrom.each do |chrom, c_intervals|
+            index = @indices[chrom]
+            with_parser(chrom) do |parser|
+              index.find(c_intervals, parser, block_filter, &blk)
+            end
+          end
+        else
+          enum_for(:find, intervals)
+        end
+      end
+
+      # Find and parse all alignment blocks in the genomic region
+      # given by a Bio::GenomicInterval, and combine them to
+      # synthesize a single alignment covering that interval
+      # exactly.
+      #
+      # @param [Bio::GenomicInterval] interval interval to search
+      # @yield [tiler] a {Tiler} ready to operate on the given interval
+      # @api public
+      def tile(interval)
+        index = chrom_index(interval.chrom)
+        with_parser(interval.chrom) do |parser|
+          tiler = Tiler.new
+          tiler.index = index
+          tiler.parser = parser
+          tiler.interval = interval
+          yield tiler
+        end
+      end
+
+      # Find and parse all alignment blocks in the genomic region
+      # given by a Bio::GenomicInterval, and truncate them to just the
+      # region intersecting that interval.
+      #
+      # @param [Bio::GenomicInterval] interval interval to search
+      # @yield [block] each {Block} matched, in turn
+      # @return [Enumerable<Block>] each matching {Block}, if no block given
+      # @api public
+      # @see KyotoIndex#slice
+      def slice(interval, &blk)
+        index = chrom_index(interval.chrom)
+        with_parser(interval.chrom) do |parser|
+          index.slice(interval, parser, &blk)
+        end
+      end
+
+      #### Internals
+
+      # @api private
+      def initialize(options)
+        @parse_options = options
+        @indices = {}
+        @maf_by_chrom = {}
+        if options[:dir]
+          @dir = options[:dir]
+          @maf_files = Dir.glob("#{@dir}/*.maf")
+        elsif options[:maf]
+          @maf_files = [options[:maf]]
+          if options[:index]
+            register_index(KyotoIndex.open(options[:index]),
+                           options[:maf])
+          end
+        else
+          raise "Must specify :dir or :maf!"
+        end
+        scan_indices!
+        if options[:maf] && @indices.empty?
+          # MAF file explicitly given but no index
+          # build a temporary one
+          # (could build a real one, too...)
+          maf = options[:maf]
+          parser = Parser.new(maf, @parse_options)
+          # $stderr.puts "WARNING: building temporary index on #{maf}."
+          index = KyotoIndex.build(parser, '%')
+          register_index(index, maf)
+        end
+      end
+
+      # @api private
+      def find_index_file(maf)
+        base = File.basename(maf, '.maf')
+        index_f = "#{@dir}/#{base}.kct"
+        File.exists?(index_f) ? index_f : nil
+      end
+
+      # @api private
+      def register_index(index, maf)
+        @indices[index.ref_seq] = index
+        @maf_by_chrom[index.ref_seq] = maf
+      end
+
+      # @api private
+      def scan_indices!
+        @maf_files.each do |maf|
+          index_f = find_index_file(maf)
+          if index_f
+            index = KyotoIndex.open(index_f)
+            register_index(index, maf)
+          end
+        end
+      end
+
+      # @api private
+      def chrom_index(chrom)
+        unless @indices.has_key? chrom
+          raise "No index available for chromosome #{chrom}!"
+        end
+        @indices[chrom]
+      end
+
+      # @api private
+      def with_parser(chrom)
+        # $stderr.puts "Creating parser with options #{@parse_options.inspect}"
+        parser = Parser.new(@maf_by_chrom[chrom], @parse_options)
+        parser.sequence_filter = self.sequence_filter
+        begin
+          yield parser
+        ensure
+          parser.close
+        end
+      end
+
+    end
+
     class KyotoIndex
       include KVHelpers
 
@@ -189,15 +387,22 @@ module Bio
       # @param [Parser] parser MAF parser for file to fetch blocks
       #  from.
       # @param [Hash] filter Block filter expression.
-      # @return [Array<Block>]
+      # @yield [block] each {Block} matched, in turn
+      # @return [Enumerable<Block>] each matching {Block}, if no block given
       # @api public
-      def find(intervals, parser, filter={})
-        start = Time.now
+      def find(intervals, parser, filter={}, &blk)
+        # start = Time.now
         fl = fetch_list(intervals, filter)
-        $stderr.printf("Built fetch list of %d items in %.3fs.\n",
-                       fl.size,
-                       Time.now - start)
-        parser.fetch_blocks(fl)
+        # $stderr.printf("Built fetch list of %d items in %.3fs.\n",
+        #                fl.size,
+        #                Time.now - start)
+        if ! fl.empty?
+          parser.fetch_blocks(fl, &blk)
+        else
+          if ! block_given?
+           []
+          end
+        end
       end
 
       # Close the underlying Kyoto Cabinet database handle.
@@ -205,6 +410,16 @@ module Bio
         db.close
       end
 
+      def slice(interval, parser, filter={})
+        if block_given?
+          find([interval], parser, filter) do |block|
+            yield block.slice(interval)
+          end
+        else
+          enum_for(:slice, interval, parser, filter)
+        end
+      end
+
       #### KyotoIndex Internals
       # @api private
 
@@ -288,7 +503,6 @@ module Bio
       # Build a fetch list of alignment blocks to read, given an array
       # of Bio::GenomicInterval objects
       def fetch_list(intervals, filter_spec={})
-        start = Time.now
         filter_spec ||= {}
         filters = Filters.build(filter_spec, self)
         chrom = intervals.first.chrom
@@ -309,9 +523,7 @@ module Bio
         bin_intervals.values.each do |intervals|
           intervals.sort_by! {|i| i.begin}
         end
-        ready = Time.now
-        $stderr.puts "bin intervals computed after #{ready - start} seconds."
-        matches = if RUBY_PLATFORM == 'java'
+        matches = if RUBY_PLATFORM == 'java' && bin_intervals.size > 4
                     scan_bins_parallel(chrom_id, bin_intervals, filters)
                   else
                     scan_bins(chrom_id, bin_intervals, filters)