bio-maf 0.3.0 → 0.3.1

data/bin/maf_index

@@ -13,29 +13,35 @@ PRINTERS = {
 
 $options = OpenStruct.new
 $options.mode = :build
-$options.reader = Bio::MAF::ChunkReader
+$options.ref_only = true
+$options.reader = if RUBY_PLATFORM == 'java'
+                    Bio::MAF::ThreadedChunkReader
+                  else
+                    Bio::MAF::ChunkReader
+                  end
 
 def build_index(maf, index)
   parser = Bio::MAF::Parser.new(maf,
                                 :chunk_reader => $options.reader,
                                 :parse_extended => false)
-  idx = Bio::MAF::KyotoIndex.build(parser, index)
+  idx = Bio::MAF::KyotoIndex.build(parser, index, $options.ref_only)
   idx.close
 end
 
 op = OptionParser.new do |opts|
   opts.banner = "Usage: maf_index [options] <maf> <index>"
-  #opts.separator ""
-  #opts.separator "Options:"
+  opts.separator ""
+  opts.separator "Options:"
+  opts.on("-a", "--all", "Index all sequences, not just reference seq") do
+    $options.ref_only = false
+  end
+  Bio::MAF::handle_logging_options(opts)
   opts.on("--time", "print elapsed time") do
     $options.bench = true
   end
-  opts.on("-d", "--dump") do
+  opts.on("-d", "--dump", "Dump contents of given INDEX") do
     $options.mode = :dump
   end
-  opts.on("-t", "--threaded") do
-    $options.reader = Bio::MAF::ThreadedChunkReader
-  end
   opts.on("--ruby-prof PATH", "Profile with ruby-prof") do |pspec|
     require 'ruby-prof'
     if pspec =~ /(\w+):(.+)/
@@ -49,6 +55,7 @@ op = OptionParser.new do |opts|
 end
 
 op.parse!(ARGV)
+Bio::Log::CLI.configure('bio-maf')
 
 maf_p = ARGV.shift if $options.mode == :build
 index_p = ARGV.shift

data/bin/maf_tile

@@ -65,9 +65,11 @@ o_parser = OptionParser.new do |opts|
           "(requires --output-base)") do |bed|
     options.bed = bed
   end
+  Bio::MAF::handle_logging_options(opts)
 end
 
 o_parser.parse!(ARGV)
+Bio::Log::CLI.configure('bio-maf')
 
 maf_p = ARGV.shift
 index_p = ARGV.shift

data/bin/maf_to_fasta

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
 
 require 'bio-maf'
-require 'bigbio'
 require 'optparse'
 require 'ostruct'
 
@@ -42,16 +41,14 @@ if options.profile_gc
 end
 
 parser = options.parser.new(src_path)
-writer = FastaWriter.new(dst_path)
+File.open(dst_path, 'w') do |outf|
+  writer = Bio::MAF::FASTAWriter.new(outf)
 
-parser.each_block do |block|
-  block.each_raw_seq do |seq|
-    seq.write_fasta(writer)
+  parser.each_block do |block|
+    writer.write_block(block)
   end
 end
 
-writer.close
-
 if options.profile_gc
   $stderr.puts GC::Profiler.result
   GC::Profiler.disable

data/bio-maf.gemspec

@@ -2,14 +2,13 @@
 
 Gem::Specification.new do |s|
   s.name = "bio-maf"
-  s.version = "0.3.0"
+  s.version = "0.3.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Clayton Wheeler"]
-  s.date = "2012-07-18"
+  s.date = "2012-07-26"
   s.description = "Multiple Alignment Format parser for BioRuby."
   s.email = "cswh@umich.edu"
-  s.executables = ["maf_count", "maf_dump_blocks", "maf_extract_ranges_count", "maf_index", "maf_parse_bench", "maf_to_fasta", "maf_write", "random_ranges"]
   s.extra_rdoc_files = [
     "LICENSE.txt",
     "README.md"
@@ -33,8 +32,8 @@ Gem::Specification.new do |s|
   end
 
   s.add_runtime_dependency('bio-alignment', ["~> 0.0.7"])
-  s.add_runtime_dependency('bio-bigbio', [">= 0"])
   s.add_runtime_dependency('bio-genomic-interval', ["~> 0.1.2"])
+  s.add_runtime_dependency('bio-logger', ["~> 1.0.1"])
   if RUBY_PLATFORM == 'java'
     s.add_runtime_dependency('kyotocabinet-java', ["~> 0.3.0"])
   else    

data/features/maf-indexing.feature

@@ -30,6 +30,15 @@ Feature: Indexed access to MAF files
     And sequence mm8.chr7 of block 0 has start 80082592
     And sequence mm8.chr7 of block 1 has start 80082713
 
+  Scenario: Extract alignment blocks by chromosomal range on non-ref sequence
+    Given a MAF source file "mm8_chr7_tiny.maf"
+    When I open it with a MAF reader
+    And build an index on all sequences
+    And search for blocks between positions 136011819 and 136012026 of rn4.chr1
+    Then 2 blocks are obtained
+    And sequence mm8.chr7 of block 0 has start 80082368
+    And sequence mm8.chr7 of block 1 has start 80082471
+
   @no_jruby
   Scenario: Build MAF index with CLI tool
     Given test files:
@@ -40,6 +49,17 @@ Feature: Indexed access to MAF files
     """
     And a file named "mm8_chr7_tiny.kct" should exist
 
+  @no_jruby
+  Scenario: Build MAF index on all sequences with CLI tool
+    Given test files:
+    | mm8_chr7_tiny.maf |
+    When I run `maf_index --all mm8_chr7_tiny.maf mm8_chr7_tiny.kct`
+    And I run `maf_index -d mm8_chr7_tiny.kct`
+    Then it should pass with regex:
+    """
+    9 \[bin 585\] 594:631
+    """
+
   @no_jruby
   Scenario: Dump MAF index with CLI tool
     Given test files:
@@ -51,4 +71,4 @@ Feature: Indexed access to MAF files
     0 \[bin 1195\] 80082334:80082368
     """
 
-  
+

data/features/step_definitions/convert_steps.rb

@@ -1,5 +1,3 @@
-require 'bigbio'                # FASTA support
-
 Given /^a MAF source file "(.*?)"$/ do |src|
   @src_f = $test_data + src
   @src_f.exist?.should be_true
@@ -13,15 +11,12 @@ end
 
 When /^I select FASTA output$/ do
   @dst = Tempfile.new(['cuke', ".#{@out_fmt.to_s}"])
-  @dst.close
-  @writer = FastaWriter.new(@dst.path)
+  @writer = Bio::MAF::FASTAWriter.new(@dst)
 end
 
 When /^process the file$/ do
   @parser.each_block do |block|
-    block.each_raw_seq do |seq|
-      seq.write_fasta(@writer)
-    end
+    @writer.write_block(block)
   end
   @writer.close
 end

data/features/step_definitions/index_steps.rb

@@ -2,6 +2,10 @@ When /^build an index on the reference sequence$/ do
   @idx = Bio::MAF::KyotoIndex.build(@parser, '%')
 end
 
+When /^build an index on all sequences$/ do
+  @idx = Bio::MAF::KyotoIndex.build(@parser, '%', false)
+end
+
 Given /^a Kyoto Cabinet index file "(.*?)"$/ do |name|
   @idx = Bio::MAF::KyotoIndex.open($test_data + name)
 end

data/lib/bio-maf.rb

@@ -8,5 +8,10 @@
 #
 # In this file only require other files. Avoid other source code.
 
+require 'bio-logger'
+log = Bio::Log::LoggerPlus.new('bio-maf')
+log.outputters = Bio::Log::Outputter.stderr
+log.level = Bio::Log::WARN
+
 require 'bio/ucsc'
 require 'bio/maf'

data/lib/bio/maf/index.rb

@@ -207,7 +207,7 @@ module Bio
           # (could build a real one, too...)
           maf = options[:maf]
           parser = Parser.new(maf, @parse_options)
-          # $stderr.puts "WARNING: building temporary index on #{maf}."
+          LOG.warn { "WARNING: building temporary index on #{maf}." }
           index = KyotoIndex.build(parser, '%')
           register_index(index, maf)
         end
@@ -247,7 +247,7 @@ module Bio
 
       # @api private
       def with_parser(chrom)
-        # $stderr.puts "Creating parser with options #{@parse_options.inspect}"
+        LOG.debug { "Creating parser with options #{@parse_options.inspect}" }
         parser = Parser.new(@maf_by_chrom[chrom], @parse_options)
         parser.sequence_filter = self.sequence_filter
         begin
@@ -262,7 +262,7 @@ module Bio
     class KyotoIndex
       include KVHelpers
 
-      attr_reader :db, :species, :species_max_id
+      attr_reader :db, :species, :species_max_id, :ref_only
       attr_accessor :index_sequences, :ref_seq
 
       FORMAT_VERSION_KEY = 'bio-maf:index-format-version'
@@ -353,9 +353,9 @@ module Bio
       # @param [Parser] parser MAF parser for file to index
       # @param [String] path path to index file to create
       # @return [KyotoIndex]
-      def self.build(parser, path)
+      def self.build(parser, path, ref_only=true)
         idx = self.new(path)
-        idx.build_default(parser)
+        idx.build(parser, ref_only)
         return idx
       end
 
@@ -391,11 +391,11 @@ module Bio
       # @return [Enumerable<Block>] each matching {Block}, if no block given
       # @api public
       def find(intervals, parser, filter={}, &blk)
-        # start = Time.now
+        start = Time.now
         fl = fetch_list(intervals, filter)
-        # $stderr.printf("Built fetch list of %d items in %.3fs.\n",
-        #                fl.size,
-        #                Time.now - start)
+        LOG.debug { sprintf("Built fetch list of %d items in %.3fs.\n",
+                            fl.size,
+                            Time.now - start) }
         if ! fl.empty?
           parser.fetch_blocks(fl, &blk)
         else
@@ -426,6 +426,7 @@ module Bio
       def initialize(path, db_arg=nil)
         @species = {}
         @species_max_id = -1
+        @max_sid = -1
         if db_arg || ((path.size > 1) and File.exist?(path))
           mode = KyotoCabinet::DB::OREADER
         else
@@ -585,8 +586,8 @@ module Bio
           n_completed += 1
         end
         threads.each { |t| t.join }
-        $stderr.printf("Matched %d index records with %d threads in %.3f seconds.\n",
-                       to_fetch.size, n_threads, Time.now - start)
+        LOG.debug { sprintf("Matched %d index records with %d threads in %.3f seconds.\n",
+                            to_fetch.size, n_threads, Time.now - start) }
         to_fetch
       end
 
@@ -602,8 +603,8 @@ module Bio
                   completed.put(result)
                 rescue Exception => e
                   completed.put(e)
-                  $stderr.puts "Worker failing: #{e.class}: #{e}"
-                  $stderr.puts e.backtrace.join("\n")
+                  LOG.error "Worker failing: #{e.class}: #{e}"
+                  LOG.error e
                   raise e
                 end
               end
@@ -658,17 +659,20 @@ module Bio
          || gi.include?(i_start)
       end
 
-      def build_default(parser)
+      def build(parser, ref_only=true)
         first_block = parser.parse_block
         self.ref_seq = first_block.sequences.first.source
+        @ref_only = ref_only
         db[REF_SEQ_KEY] = ref_seq
         db[FORMAT_VERSION_KEY] = FORMAT_VERSION
-        @index_sequences = { ref_seq => 0 }
-        store_index_sequences!
+        @index_sequences = {}
         index_blocks([first_block])
-        parser.enum_for(:each_block).each_slice(1000).each do |blocks|
+        n = 0
+        parser.each_block.each_slice(1000).each do |blocks|
           index_blocks(blocks)
+          n += blocks.size
         end
+        LOG.debug { "Created index for #{n} blocks and #{@index_sequences.size} sequences." }
         db.synchronize(true)
       end
 
@@ -685,12 +689,18 @@ module Bio
           h[name] = id
         end
         @index_sequences = h
+        @max_sid = @index_sequences.values.max
       end
 
-      def store_index_sequences!
-        index_sequences.each do |name, id|
-          db.set("sequence:#{name}", id.to_s)
+      def seq_id_for(name)
+        sid = index_sequences[name]
+        if ! sid
+          @max_sid += 1
+          sid = @max_sid
+          db.set("sequence:#{name}", sid.to_s)
+          index_sequences[name] = sid
         end
+        return sid
       end
 
       def load_species
@@ -742,9 +752,9 @@ module Bio
         end
         h = {}
         val = build_block_value(block)
-        block.sequences.each do |seq|
-          seq_id = index_sequences[seq.source]
-          next unless seq_id
+        to_index = ref_only ? [block.sequences.first] : block.sequences
+        to_index.each do |seq|
+          seq_id = seq_id_for(seq.source)
           seq_end = seq.start + seq.size
           bin = Bio::Ucsc::UcscBin.bin_from_range(seq.start, seq_end)
           key = [255, seq_id, bin, seq.start, seq_end].pack(KEY_FMT)

data/lib/bio/maf/maf.rb

@@ -64,7 +64,6 @@ module Bio
       attr_reader :size
 
       def initialize(vars, sequences, offset, size, filtered)
-        #raise ArgumentError, "no sequences given for block at offset #{offset}!" unless sequences && sequences.first
         @vars = vars
         @sequences = sequences
         @offset = offset
@@ -258,8 +257,13 @@ module Bio
       attr_accessor :quality
       alias_method :source_size, :src_size
 
-      def initialize(*args)
-        @source, @start, @size, @strand, @src_size, @text = args
+      def initialize(source, start, size, strand, src_size, text)
+        @source = source
+        @start = start
+        @size = size
+        @strand = strand
+        @src_size = src_size
+        @text = text
       end
 
       def end
@@ -354,9 +358,8 @@ module Bio
         Bio::BioAlignment::Sequence.new(source, text)
       end
 
-      def write_fasta(writer)
-        writer.write("#{source}:#{start}-#{start + size}",
-                     text)
+      def fasta_desc
+        "#{source}:#{start}-#{start + size}"
       end
 
       def joinable_with?(o)
@@ -441,7 +444,7 @@ module Bio
       attr_reader :status
 
       def initialize(*args)
-        super(*args[0..4])
+        super(*(args[0..4] << nil))
         @status = args[5]
       end
 

data/lib/bio/maf/parser.rb

@@ -5,7 +5,8 @@ require 'java' if RUBY_PLATFORM == 'java'
 module Bio
   # @api public
   module MAF
-
+    LOG = Bio::Log::LoggerPlus['bio-maf']
+    
     # @api public
     class ParseError < Exception; end
 
@@ -601,10 +602,9 @@ module Bio
           end
         end
         elapsed = Time.now - start
-        # TODO: debug log
-        # rate = (total_size / 1048576.0) / elapsed
-        # $stderr.printf("Fetched blocks in %.3fs, %.1f MB/s.\n",
-        #                elapsed, rate)
+        rate = (total_size / 1048576.0) / elapsed
+        LOG.debug { sprintf("Fetched blocks in %.3fs, %.1f MB/s.",
+                            elapsed, rate) }
       end
 
       # Fetch and parse the blocks given by the merged fetch list, in
@@ -639,13 +639,13 @@ module Bio
         end
         threads.each { |t| t.join }
         elapsed = Time.now - start
-        $stderr.printf("Fetched blocks from %d threads in %.1fs.\n",
-                       n_threads,
-                       elapsed)
+        LOG.debug { sprintf("Fetched blocks from %d threads in %.1fs.",
+                            n_threads,
+                            elapsed) }
         mb = total_size / 1048576.0
-        $stderr.printf("%.3f MB processed (%.1f MB/s).\n",
-                       mb,
-                       mb / elapsed)
+        LOG.debug { sprintf("%.3f MB processed (%.1f MB/s).",
+                            mb,
+                            mb / elapsed) }
       end
 
       # Create a worker thread for parallel parsing.
@@ -667,8 +667,8 @@ module Bio
               end
             end
           rescue Exception => e
-            $stderr.puts "Worker failing: #{e.class}: #{e}"
-            $stderr.puts e.backtrace.join("\n")
+            LOG.error "Worker failing: #{e.class}: #{e}"
+            LOG.error e
             raise e
           end
         end
@@ -835,8 +835,8 @@ module Bio
             end
             queue.put(:eof)
           rescue
-            $stderr.puts "worker exiting: #{$!.class}: #{$!}"
-            $stderr.puts $!.backtrace.join("\n")
+            LOG.error "worker exiting: #{$!.class}: #{$!}"
+            LOG.error $!
           end
         end
         saw_eof = false
@@ -902,6 +902,28 @@ module Bio
       include MAFParsing
     end
 
+    def handle_logging_options(opts)
+      opts.on("--logger filename", String,
+              "Log to file (default STDOUT)") do |name|
+        Bio::Log::CLI.logger(name)
+      end
+      opts.on("--trace options", String,
+              "Set log level",
+              "(default INFO, see bio-logger)") do |s|
+        Bio::Log::CLI.trace(s)
+      end
+      opts.on("-q", "--quiet", "Run quietly") do
+        Bio::Log::CLI.trace('error')
+      end
+      opts.on("-v", "--verbose", "Run verbosely") do
+        Bio::Log::CLI.trace('info')
+      end
+      opts.on("--debug", "Run with extra debugging output") do
+        Bio::Log::CLI.trace('debug')
+      end
+    end
+    module_function :handle_logging_options
+
   end
   
 end