bio-maf 1.0.0-java → 1.0.1-java

data/bio-maf.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = "bio-maf"
-  s.version = "1.0.0"
+  s.version = "1.0.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Clayton Wheeler"]
-  s.date = "2012-08-02"
+  s.date = "2012-08-08"
   s.description = "Multiple Alignment Format parser for BioRuby."
   s.email = "cswh@umich.edu"
   s.extra_rdoc_files = [
@@ -32,7 +32,7 @@ Gem::Specification.new do |s|
   end
 
   s.add_runtime_dependency('bio-alignment', ["~> 0.0.7"])
-  s.add_runtime_dependency('bio-bgzf', ["~> 0.2.0"])
+  s.add_runtime_dependency('bio-bgzf', ["~> 0.2.1"])
   s.add_runtime_dependency('bio-genomic-interval', ["~> 0.1.2"])
   s.add_runtime_dependency('bio-logger', ["~> 1.0.1"])
   if RUBY_PLATFORM == 'java'

data/features/bgzf.feature

@@ -60,3 +60,48 @@ Feature: BGZF compression
     And a file named "mm8_chr7_tiny.maf.bgz" should exist
     And a file named "mm8.chrM.maf.bgz" should exist
     
+  @no_jruby
+  Scenario: Don't overwrite MAF files
+    Given test files:
+    | mm8.chrM.maf      |
+    | mm8.chrM.maf.bgz  |
+    When I run `maf_bgzip mm8.chrM.maf`
+    Then it should fail with:
+    """
+    exists
+    """
+
+  @no_jruby
+  Scenario: Don't overwrite indexes
+    Given test files:
+    | mm8_chr7_tiny.maf |
+    When I run `maf_bgzip --index mm8_chr7_tiny.maf`
+     And I run `rm mm8_chr7_tiny.maf.bgz`
+     And I run `maf_bgzip --index mm8_chr7_tiny.maf`
+    Then it should fail with:
+    """
+    exists
+    """
+
+  @no_jruby
+  Scenario: Overwrite MAF files with --force
+    Given test files:
+    | mm8.chrM.maf      |
+    | mm8.chrM.maf.bgz  |
+    When I run `maf_bgzip --force mm8.chrM.maf`
+    Then it should pass with:
+    """
+    """
+
+  @no_jruby
+  Scenario: Overwrite indexes with --force
+    Given test files:
+    | mm8_chr7_tiny.maf |
+    When I run `maf_bgzip --index mm8_chr7_tiny.maf`
+     And I run `rm mm8_chr7_tiny.maf.bgz`
+     And I run `maf_bgzip --force --index mm8_chr7_tiny.maf`
+    Then it should pass with:
+    """
+    """
+
+    

data/features/maf-indexing.feature

@@ -39,6 +39,12 @@ Feature: Indexed access to MAF files
     And sequence mm8.chr7 of block 0 has start 80082368
     And sequence mm8.chr7 of block 1 has start 80082471
 
+  Scenario: Index MAF file with extended bin positions
+    Given a MAF source file "ext-bin.maf"
+    When I open it with a MAF reader
+    And build an index on all sequences
+    Then the index has at least 18 entries
+    
   @no_jruby
   Scenario: Build MAF index with CLI tool
     Given test files:

data/features/maf-parsing.feature

@@ -42,3 +42,20 @@ Feature: Parse MAF files
     And sequence 0 has text "ACA-TTACT"
     And sequence 1 has strand :-
 
+  Scenario: Read alignment block, folded to upper case
+    Given MAF data:
+    """
+    ##maf version=1 scoring=humor.v4
+    # humor.v4 R=30 M=10 /cluster/data/hg15/bed/blastz.mm3/axtNet300/chr1.maf
+    # /cluster/data/hg15/bed/blastz.rn3/axtNet300/chr1.maf
+
+    a score=0.128
+    s human_hoxa 100  8 + 100257 aca-ttact
+    s horse_hoxa 120  9 -  98892 acaattgct
+    s fugu_hoxa   88  7  + 90788 aca--tgct
+    """
+    When I enable the :upcase parser option
+    And I open it with a MAF reader
+    Then an alignment block can be obtained
+    And the alignment block has 3 sequences
+    And sequence 0 has text "ACA-TTACT"

data/features/maf-querying.feature

@@ -82,3 +82,14 @@ Feature: Filter results from MAF files
     And I run `maf_extract -m mm8.chrM.maf.bgz --interval mm8.chrM:6938-13030 -o m2.maf`
     And I run `diff m1.maf m2.maf`
     Then the exit status should be 0
+
+  @no_jruby
+  Scenario: One-based indexing with maf_extract
+    Given test files:
+    | mm8_chr7_tiny.maf |
+    | mm8_chr7_tiny.kct |
+    When I run `sh -c 'maf_extract -d . --one-based --interval mm8.chr7:80082592-80082713 | grep "^a" | wc -l'`
+    Then it should pass with:
+    """
+    2
+    """

data/features/slice.feature

@@ -18,6 +18,17 @@ Feature: MAF slicing
     And write all the matched blocks
     Then the output should match, except whitespace, "mm8_chr7_tiny_slice1.maf"
 
+  Scenario: Interval covering two blocks, using directory access, counting
+    Given indexed MAF files in "test/data"
+    When I enable the :remove_gaps parser option
+    And filter for only the species
+      | mm8 |
+      | rn4 |
+    And I extract a slice over the genomic interval
+      | chrom    |    start |      end |
+      | mm8.chr7 | 80082350 | 80082380 |
+    Then 2 blocks are obtained
+
   Scenario: Interval covering two blocks, using directory access
     Given indexed MAF files in "test/data"
     When I enable the :remove_gaps parser option

data/features/step_definitions/parse_steps.rb

@@ -30,6 +30,7 @@ end
 Then /^an alignment block can be obtained$/ do
   @block = @parser.parse_block
   @block.should_not be_nil
+  @block.is_a?(Bio::MAF::Block).should be_true
 end
 
 Then /^the alignment block has (\d+) sequences$/ do |n_seq|

data/features/tiling.feature

@@ -160,7 +160,7 @@ Feature: Join alignment blocks with reference data
     | gap-sp1.fa.gz |
     | gap-1.maf     |
     | gap-1.kct     |
-    When I run `maf_tile --reference gap-sp1.fa.gz --interval 0:50 -s sp1:mouse -s sp2:nautilus -s sp3:jaguar gap-1.maf gap-1.kct`
+    When I run `maf_tile --reference gap-sp1.fa.gz --interval 0-50 -s sp1:mouse -s sp2:nautilus -s sp3:jaguar gap-1.maf gap-1.kct`
     Then it should pass with:
     """
     >mouse
@@ -176,7 +176,7 @@ Feature: Join alignment blocks with reference data
     Given test files:
     | gap-1.maf     |
     | gap-1.kct     |
-    When I run `maf_tile --interval 0:50 -s sp1:mouse -s sp2:nautilus -s sp3:jaguar gap-1.maf gap-1.kct`
+    When I run `maf_tile --interval 0-50 -s sp1:mouse -s sp2:nautilus -s sp3:jaguar gap-1.maf gap-1.kct`
     Then it should pass with:
     """
     >mouse
@@ -198,7 +198,10 @@ Feature: Join alignment blocks with reference data
     sp1.chr1 12 36
     """
     When I run `maf_tile -s sp1:mouse -s sp2:nautilus -s sp3:jaguar --output-base selected --bed example.bed --reference gap-sp1.fa.gz gap-1.maf gap-1.kct`
-    Then the file "selected_12-36.fa" should contain exactly:
+    Then it should pass with:
+    """
+    """
+    And the file "selected_12-36.fa" should contain exactly:
     """
     >mouse
     GCTGAGGGC--AGTTGTGTCAGGGCG
@@ -214,7 +217,7 @@ Feature: Join alignment blocks with reference data
     Given test files:
     | mm8_chr7_tiny.maf |
     | mm8_chr7_tiny.kct |
-    When I run `maf_tile -s mm8 -s rn4 -s hg18 --interval 80082334:80082344 mm8_chr7_tiny.maf`
+    When I run `maf_tile -s mm8 -s rn4 -s hg18 --interval 80082334-80082344 mm8_chr7_tiny.maf`
     Then it should pass with:
     """
     >mm8
@@ -230,7 +233,7 @@ Feature: Join alignment blocks with reference data
     Given test files:
     | mm8_chr7_tiny.maf |
     | mm8_chr7_tiny.kct |
-    When I run `maf_tile -s mm8 -s rn4 -s hg18 --interval mm8.chr7:80082334:80082344 .`
+    When I run `maf_tile -s mm8 -s rn4 -s hg18 --interval mm8.chr7:80082334-80082344 .`
     Then it should pass with:
     """
     >mm8
@@ -241,3 +244,18 @@ Feature: Join alignment blocks with reference data
     --------GG
     """
 
+  @no_jruby
+  Scenario: Tile with CLI tool and directory, 1-based
+    Given test files:
+    | mm8_chr7_tiny.maf |
+    | mm8_chr7_tiny.kct |
+    When I run `maf_tile -s mm8 -s rn4 -s hg18 --one-based --interval mm8.chr7:80082335-80082344 .`
+    Then it should pass with:
+    """
+    >mm8
+    GGGCTGAGGG
+    >rn4
+    GGGCTGAGGG
+    >hg18
+    --------GG
+    """

data/lib/bio-maf.rb

@@ -11,7 +11,11 @@
 require 'bio-logger'
 log = Bio::Log::LoggerPlus.new('bio-maf')
 log.outputters = Bio::Log::Outputter.stderr
-log.level = Bio::Log::WARN
+log.level = if ENV['BIO_MAF_DEBUG']
+              Bio::Log::DEBUG
+            else
+              Bio::Log::WARN
+            end
 
 require 'bio/ucsc'
 require 'bio/maf'

data/lib/bio/maf.rb

@@ -4,3 +4,4 @@ require 'bio/maf/index'
 require 'bio/maf/parser'
 require 'bio/maf/writer'
 require 'bio/maf/tiler'
+require 'bio/maf/jobs'

data/lib/bio/maf/index.rb

@@ -125,7 +125,7 @@ module Bio
       # @param [Enumerable<Bio::GenomicInterval>] intervals genomic
       #  intervals to parse.
       # @yield [block] each {Block} matched, in turn
-      # @return [Enumerable<Block>] each matching {Block}, if no block given
+      # @return [Array<Block>] each matching {Block}, if no block given
       # @api public
       # @see KyotoIndex#find
       def find(intervals, &blk)
@@ -137,13 +137,16 @@ module Bio
             end
           end
           by_chrom.each do |chrom, c_intervals|
-            index = @indices[chrom]
-            with_parser(chrom) do |parser|
-              index.find(c_intervals, parser, block_filter, &blk)
+            with_index(chrom) do |index|
+              with_parser(chrom) do |parser|
+                index.find(c_intervals, parser, block_filter, &blk)
+              end
             end
           end
         else
-          enum_for(:find, intervals)
+          acc = []
+          self.find(intervals) { |block| acc << block }
+          acc
         end
       end
 
@@ -156,13 +159,14 @@ module Bio
       # @yield [tiler] a {Tiler} ready to operate on the given interval
       # @api public
       def tile(interval)
-        index = chrom_index(interval.chrom)
-        with_parser(interval.chrom) do |parser|
-          tiler = Tiler.new
-          tiler.index = index
-          tiler.parser = parser
-          tiler.interval = interval
-          yield tiler
+        with_index(interval.chrom) do |index|
+          with_parser(interval.chrom) do |parser|
+            tiler = Tiler.new
+            tiler.index = index
+            tiler.parser = parser
+            tiler.interval = interval
+            yield tiler
+          end
         end
       end
 
@@ -172,13 +176,15 @@ module Bio
       #
       # @param [Bio::GenomicInterval] interval interval to search
       # @yield [block] each {Block} matched, in turn
-      # @return [Enumerable<Block>] each matching {Block}, if no block given
+      # @return [Array<Block>] each matching {Block}, if no block given
       # @api public
       # @see KyotoIndex#slice
       def slice(interval, &blk)
-        index = chrom_index(interval.chrom)
-        with_parser(interval.chrom) do |parser|
-          index.slice(interval, parser, &blk)
+        with_index(interval.chrom) do |index|
+          with_parser(interval.chrom) do |parser|
+            s = index.slice(interval, parser, block_filter, &blk)
+            block_given? ? s : s.to_a
+          end
         end
       end
 
@@ -193,12 +199,17 @@ module Bio
           scan_dir(options[:dir])
         elsif options[:maf]
           if options[:index]
-            register_index(KyotoIndex.open(options[:index]),
+            LOG.debug { "Opening index file #{options[:index]}" }
+            index = KyotoIndex.open(options[:index])
+            register_index(index,
                            options[:maf])
+            index.close
           else
-            idx = find_index_file(options[:maf])
-            if idx
-              register_index(KyotoIndex.open(idx), options[:maf])
+            idx_f = find_index_file(options[:maf])
+            if idx_f
+              index = KyotoIndex.open(idx_f)
+              register_index(index, options[:maf])
+              index.close
             end
           end
         else
@@ -229,7 +240,11 @@ module Bio
         unless index.maf_file == File.basename(maf)
           raise "Index #{index.path} was created for #{index.maf_file}, not #{File.basename(maf)}!"
         end
-        @indices[index.ref_seq] = index
+        if index.path.to_s.start_with? '%'
+          @indices[index.ref_seq] = index
+        else
+          @indices[index.ref_seq] = index.path.to_s
+        end
         @maf_by_chrom[index.ref_seq] = maf
       end
 
@@ -241,6 +256,7 @@ module Bio
           if File.exist? maf
             register_index(index, maf)
           end
+          index.close
         end
       end
 
@@ -249,7 +265,23 @@ module Bio
         unless @indices.has_key? chrom
           raise "No index available for chromosome #{chrom}!"
         end
-        @indices[chrom]
+        index = @indices[chrom]
+        if index.is_a? KyotoIndex
+          # temporary
+          index
+        else
+          KyotoIndex.open(index)
+        end
+      end
+
+      def with_index(chrom)
+        index = chrom_index(chrom)
+        LOG.debug { "Selected index #{index} for sequence #{chrom}." }
+        begin
+          yield index
+        ensure
+          index.close unless index.path.to_s.start_with? '%'
+        end
       end
 
       # @api private
@@ -403,7 +435,7 @@ module Bio
       def find(intervals, parser, filter={}, &blk)
         start = Time.now
         fl = fetch_list(intervals, filter)
-        LOG.debug { sprintf("Built fetch list of %d items in %.3fs.\n",
+        LOG.debug { sprintf("Built fetch list of %d items in %.3fs.",
                             fl.size,
                             Time.now - start) }
         if ! fl.empty?
@@ -426,6 +458,7 @@ module Bio
             yield block.slice(interval)
           end
         else
+          LOG.debug { "accumulating results of #slice" }
           enum_for(:slice, interval, parser, filter)
         end
       end
@@ -436,6 +469,7 @@ module Bio
       def initialize(path, db_arg=nil)
         @species = {}
         @species_max_id = -1
+        @index_sequences = {}
         @max_sid = -1
         if db_arg || ((path.size > 1) and File.exist?(path))
           mode = KyotoCabinet::DB::OREADER
@@ -444,15 +478,25 @@ module Bio
         end
         @db = db_arg || KyotoCabinet::DB.new
         @path = path
-        unless db_arg || db.open(path.to_s, mode)
+        path_str = "#{path.to_s}#opts=ls#dfunit=100000"
+        unless db_arg || db.open(path_str, mode)
           raise "Could not open DB file!"
         end
         if mode == KyotoCabinet::DB::OREADER
+          version = db[FORMAT_VERSION_KEY].to_i
+          if version != FORMAT_VERSION
+            raise "Index #{path} is version #{version}, expecting version #{FORMAT_VERSION}!"
+          end
           @maf_file = db[FILE_KEY]
           self.ref_seq = db[REF_SEQ_KEY]
           load_index_sequences
           load_species
         end
+        @mutex = Mutex.new
+      end
+
+      def to_s
+        "#<KyotoIndex path=#{path}>"
       end
 
       # Reopen the same DB handle read-only. Only useful for unit tests.
@@ -576,6 +620,11 @@ module Bio
       end
 
       def scan_bins_parallel(chrom_id, bin_intervals, filters)
+        LOG.debug {
+          sprintf("Beginning scan of %d bin intervals %s filters.",
+                  bin_intervals.size,
+                  filters.empty? ? "without" : "with")
+        }
         start = Time.now
         n_threads = ENV['profile'] ? 1 : java.lang.Runtime.runtime.availableProcessors
         jobs = java.util.concurrent.ConcurrentLinkedQueue.new(bin_intervals.to_a)
@@ -603,7 +652,7 @@ module Bio
           n_completed += 1
         end
         threads.each { |t| t.join }
-        LOG.debug { sprintf("Matched %d index records with %d threads in %.3f seconds.\n",
+        LOG.debug { sprintf("Matched %d index records with %d threads in %.3f seconds.",
                             to_fetch.size, n_threads, Time.now - start) }
         to_fetch
       end
@@ -676,30 +725,55 @@ module Bio
          || gi.include?(i_start)
       end
 
-      def build(parser, ref_only=true)
-        db[FILE_KEY] = File.basename(parser.file_spec)
+      CHUNK_THRESHOLD_BYTES = 50 * 1024 * 1024
+      CHUNK_THRESHOLD_BLOCKS = 1000
+
+      def prep(file_spec, compression, ref_only)
+        db[FORMAT_VERSION_KEY] = FORMAT_VERSION
+        db[FILE_KEY] = File.basename(file_spec)
         @maf_file = db[FILE_KEY]
-        if parser.compression
-          db[COMPRESSION_KEY] = parser.compression.to_s
+        if compression
+          db[COMPRESSION_KEY] = compression.to_s
         end
-        first_block = parser.parse_block
-        self.ref_seq = first_block.sequences.first.source
         @ref_only = ref_only
-        db[REF_SEQ_KEY] = ref_seq
-        db[FORMAT_VERSION_KEY] = FORMAT_VERSION
-        @index_sequences = {}
-        index_blocks([first_block])
+        @seen_first = false
+      end
+        
+      def build(parser, ref_only=true)
+        prep(parser.file_spec,
+             parser.compression,
+             ref_only)
+
         n = 0
-        parser.each_block.each_slice(1000).each do |blocks|
-          index_blocks(blocks)
-          n += blocks.size
+        acc = []
+        acc_bytes = 0
+        parser.each_block do |block|
+          acc << block
+          acc_bytes += block.size
+          if acc_bytes > CHUNK_THRESHOLD_BYTES \
+            || acc.size > CHUNK_THRESHOLD_BLOCKS
+            index_blocks(acc)
+            acc = []
+            acc_bytes = 0
+          end
+          n += 1
         end
+        index_blocks(acc)
         LOG.debug { "Created index for #{n} blocks and #{@index_sequences.size} sequences." }
         db.synchronize(true)
       end
 
       def index_blocks(blocks)
-        h = blocks.map { |b| entries_for(b) }.reduce(:merge!)
+        h = @mutex.synchronize do
+          if ! @seen_first
+            # set the reference sequence from the first block
+            first_block = blocks.first
+            self.ref_seq = first_block.sequences.first.source
+            db[REF_SEQ_KEY] = ref_seq
+            @seen_first = true
+          end
+          blocks.map { |b| entries_for(b) }.reduce(:merge!)
+        end
         db.set_bulk(h, false)
       end
 
@@ -719,8 +793,11 @@ module Bio
         if ! sid
           @max_sid += 1
           sid = @max_sid
-          db.set("sequence:#{name}", sid.to_s)
-          index_sequences[name] = sid
+          # "" << foo is hideous but apparently what it takes to get a
+          # non-shared copy of a string on JRuby...
+          name_copy = "" << name
+          db.set("sequence:#{name_copy}", sid.to_s)
+          index_sequences[name_copy] = sid
         end
         return sid
       end
@@ -739,22 +816,24 @@ module Bio
         # example: otoGar1.scaffold_104707.1-93001
         parts = seq.split('.', 2)
         if parts.size == 2
-          species_name = parts[0]
-          if species.has_key? species_name
-            return species[species_name]
-          else
-            species_id = @species_max_id + 1
-            if species_id >= MAX_SPECIES
-              raise "cannot index MAF file with more than #{MAX_SPECIES} species"
-            end
-            species[species_name] = species_id
-            db["species:#{species_name}"] = species_id
-            @species_max_id = species_id
-            return species_id
-          end
+          # "" << foo is hideous but apparently what it takes to get a
+          # non-shared copy of a string on JRuby...
+          species_name = "" << parts[0]
         else
           # not in species.sequence format, apparently
-          return nil
+          species_name = "" << seq
+        end
+        if species.has_key? species_name
+          return species[species_name]
+        else
+          species_id = @species_max_id + 1
+          if species_id >= MAX_SPECIES
+            raise "cannot index MAF file with more than #{MAX_SPECIES} species"
+          end
+          species[species_name] = species_id
+          db["species:#{species_name}"] = species_id
+          @species_max_id = species_id
+          return species_id
         end
       end
 
@@ -769,20 +848,27 @@ module Bio
       end
 
       def entries_for(block)
-        unless block.ref_seq.source == @ref_seq
-          raise "Inconsistent reference sequence: expected #{@ref_seq}, got #{block.ref_seq.source}"
-        end
-        h = {}
-        val = build_block_value(block)
-        to_index = ref_only ? [block.sequences.first] : block.sequences
-        to_index.each do |seq|
-          seq_id = seq_id_for(seq.source)
-          seq_end = seq.start + seq.size
-          bin = Bio::Ucsc::UcscBin.bin_from_range(seq.start, seq_end)
-          key = [255, seq_id, bin, seq.start, seq_end].pack(KEY_FMT)
-          h[key] = val
+        begin
+          unless block.ref_seq.source == @ref_seq
+            raise "Inconsistent reference sequence: expected #{@ref_seq}, got #{block.ref_seq.source}"
+          end
+          h = {}
+          val = build_block_value(block)
+          to_index = ref_only ? [block.sequences.first] : block.sequences
+          to_index.each do |seq|
+            seq_id = seq_id_for(seq.source)
+            # size 0 occurs in e.g. upstream1000.maf.gz
+            next if seq.size == 0
+            seq_end = seq.start + seq.size
+            bin = Bio::Ucsc::UcscBin.bin_from_range(seq.start, seq_end)
+            key = [255, seq_id, bin, seq.start, seq_end].pack(KEY_FMT)
+            h[key] = val
+          end
+          return h
+        rescue Exception => e
+          LOG.error "Failed to index block at offset #{block.offset}:\n#{block}"
+          raise e
         end
-        return h
       end
     end # class KyotoIndex
 
@@ -861,6 +947,10 @@ module Bio
         @l = l
       end
 
+      def empty?
+        @l.empty?
+      end
+
       def match(entry)
         return ! @l.find { |f| ! f.call(entry) }
       end