bio-maf 0.1.0

data/bin/maf_count

@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+require 'bio-maf'
+require 'bigbio'
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+options.parser = Bio::MAF::Parser
+options.reader = Bio::MAF::ChunkReader
+
+PRINTERS = {
+  'flat' => :FlatPrinter,
+  'stack' => :CallStackPrinter
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: maf_count [options] <maf>"
+  opts.separator ""
+  opts.separator "Options:"
+  opts.on("-p", "--profile PROF", "Profile with PerfTools") do |prof|
+    options.prof = prof
+  end
+  opts.on("--ruby-prof PATH", "Profile with ruby-prof") do |pspec|
+    if pspec =~ /(\w+):(.+)/
+      require 'ruby-prof'
+      options.ruby_prof_printer = RubyProf.const_get(PRINTERS.fetch($1))
+      options.ruby_prof_path = $2
+    else
+      options.ruby_prof_printer = RubyProf::FlatPrinter
+      options.ruby_prof_path = pspec
+    end
+  end
+  opts.on("--profile-gc", "Profile GC") do |prof|
+    options.profile_gc = true
+  end
+  opts.on("--parser PARSER", "parser") do |name|
+    options.parser = Bio::MAF.const_get(name)
+  end
+  opts.on("-t", "--threaded") do
+    options.reader = Bio::MAF::ThreadedChunkReader
+  end
+end.parse!(ARGV)
+
+src_path = ARGV.shift
+
+if options.prof
+  require 'perftools'
+  PerfTools::CpuProfiler.start(options.prof)
+elsif options.ruby_prof_path
+  require 'ruby-prof'
+  RubyProf.start
+end
+
+if options.profile_gc
+  GC::Profiler.enable
+end
+
+parser = options.parser.new(src_path,
+                            :chunk_reader => options.reader,
+                            :parse_extended => false)
+
+n = 0
+parser.each_block do |block|
+  n += 1
+end
+puts "Parsed #{n} MAF alignment blocks."
+
+if options.profile_gc
+  $stderr.puts GC::Profiler.result
+  GC::Profiler.disable
+end
+
+if options.prof
+  PerfTools::CpuProfiler.stop
+elsif options.ruby_prof_path
+  res = RubyProf.stop
+  printer = options.ruby_prof_printer.new(res)
+  File.open(options.ruby_prof_path, 'w') do |f|
+    printer.print(f)
+  end
+end

data/bin/maf_dump_blocks

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require 'bio-maf'
+require 'bigbio'
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+options.parser = Bio::MAF::Parser
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: maf_dump_blocks [options] <maf>"
+  opts.separator ""
+  opts.separator "Options:"
+  opts.on("--parser PARSER", "parser") do |name|
+    options.parser = Bio::MAF.const_get(name)
+  end
+end.parse!(ARGV)
+
+src_path = ARGV.shift
+
+parser = options.parser.new(src_path)
+
+parser.each_block do |block|
+  $stdout.printf("%12d\t%7d\n", block.offset, block.size)
+end
+

data/bin/maf_extract_ranges_count

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'ostruct'
+
+require 'bio-maf'
+require 'bio-genomic-interval'
+
+options = OpenStruct.new
+options.p = { :threads => 1 }
+options.passes = 1
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: maf_extract_ranges_count [options] <maf> <index>"
+  opts.separator ""
+  opts.separator "Options:"
+  opts.on("-t", "--threads N", "Parser threads") do |n|
+    options.p[:threads] = n.to_i
+  end
+  opts.on("-p", "--passes N", "Number of passes") do |n|
+    options.passes = n.to_i
+  end
+end.parse!(ARGV)
+
+maf_p = ARGV.shift
+index_p = ARGV.shift
+
+parser = Bio::MAF::Parser.new(maf_p, options.p)
+index = Bio::MAF::KyotoIndex.open(index_p)
+
+def parse_interval(line)
+  src, r_start_s, r_end_s, _ = line.split(nil, 4)
+  r_start = r_start_s.to_i
+  r_end = r_end_s.to_i
+  return Bio::GenomicInterval.zero_based(src, r_start, r_end)
+end
+
+intervals = []
+$stdin.each_line { |line| intervals << parse_interval(line) }
+
+options.passes.times do
+  blocks = index.find(intervals, parser)
+  puts "TOTAL: #{blocks.count} blocks parsed."
+end

data/bin/maf_index

@@ -0,0 +1,88 @@
+#!/usr/bin/env ruby
+
+require 'benchmark'
+require 'bio-maf'
+require 'optparse'
+require 'ostruct'
+
+PRINTERS = {
+  'flat' => :FlatPrinter,
+  'stack' => :CallStackPrinter,
+  'graph' => :GraphHtmlPrinter
+}
+
+$options = OpenStruct.new
+$options.mode = :build
+$options.reader = Bio::MAF::ChunkReader
+
+def build_index(maf, index)
+  parser = Bio::MAF::Parser.new(maf,
+                                :chunk_reader => $options.reader,
+                                :parse_extended => false)
+  idx = Bio::MAF::KyotoIndex.build(parser, index)
+  idx.close
+end
+
+op = OptionParser.new do |opts|
+  opts.banner = "Usage: maf_index [options] <maf> <index>"
+  #opts.separator ""
+  #opts.separator "Options:"
+  opts.on("--time", "print elapsed time") do
+    $options.bench = true
+  end
+  opts.on("-d", "--dump") do
+    $options.mode = :dump
+  end
+  opts.on("-t", "--threaded") do
+    $options.reader = Bio::MAF::ThreadedChunkReader
+  end
+  opts.on("--ruby-prof PATH", "Profile with ruby-prof") do |pspec|
+    require 'ruby-prof'
+    if pspec =~ /(\w+):(.+)/
+      $options.ruby_prof_printer = RubyProf.const_get(PRINTERS.fetch($1))
+      $options.ruby_prof_path = $2
+    else
+      $options.ruby_prof_printer = Ruby_Prof::FlatPrinter
+      $options.ruby_prof_path = pspec
+    end
+  end
+end
+
+op.parse!(ARGV)
+
+maf_p = ARGV.shift if $options.mode == :build
+index_p = ARGV.shift
+
+unless (maf_p || $options.mode == :dump) && index_p
+  $stderr.puts op
+  exit 1
+end
+
+if $options.ruby_prof_path
+  RubyProf.start
+end
+
+case $options.mode
+when :build
+  if ! $options.bench
+    build_index(maf_p, index_p)
+  else
+    bm_res = Benchmark.measure do
+      build_index(maf_p, index_p)
+    end
+    puts bm_res
+  end
+when :dump
+  idx = Bio::MAF::KyotoIndex.open(index_p)
+  idx.dump
+else
+  raise "Unsupported mode: #{$options.mode}"
+end
+
+if $options.ruby_prof_path
+  res = RubyProf.stop
+  printer = $options.ruby_prof_printer.new(res)
+  File.open($options.ruby_prof_path, 'w') do |f|
+    printer.print(f)
+  end
+end

data/bin/maf_parse_bench

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+
+require 'benchmark'
+require 'bio-maf'
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+options.parser = Bio::MAF::Parser
+options.runs = 100_000
+options.warmup = false
+
+PRINTERS = {
+  'flat' => :FlatPrinter,
+  'stack' => :CallStackPrinter
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: maf_parse_bench [options] <maf>"
+  opts.separator ""
+  opts.separator "Options:"
+  opts.on("-p", "--profile PROF", "Profile with PerfTools") do |prof|
+    options.prof = prof
+  end
+  opts.on("--ruby-prof PATH", "Profile with ruby-prof") do |pspec|
+    if pspec =~ /(\w+):(.+)/
+      require 'ruby-prof'
+      options.ruby_prof_printer = RubyProf.const_get(PRINTERS.fetch($1))
+      options.ruby_prof_path = $2
+    else
+      options.ruby_prof_printer = :FlatPrinter
+      options.ruby_prof_path = pspec
+    end
+  end
+  opts.on("--profile-gc", "Profile GC") do |prof|
+    options.profile_gc = true
+  end
+  opts.on("--parser PARSER", "parser") do |name|
+    options.parser = Bio::MAF.const_get(name)
+  end
+  opts.on("-w", "--warmup", "perform warmup run") do 
+    options.warmup = true
+  end
+end.parse!(ARGV)
+
+src_path = ARGV.shift
+
+if options.prof
+  require 'perftools'
+  PerfTools::CpuProfiler.start(options.prof)
+elsif options.ruby_prof_path
+  require 'ruby-prof'
+  RubyProf.start
+end
+
+if options.profile_gc
+  GC::Profiler.enable
+end
+
+parser = options.parser.new(src_path)
+parser.parse_block
+parser.parse_block
+pos = parser.s.pos
+
+if options.warmup
+  options.runs.times do
+    parser.parse_block
+    parser.s.pos = pos
+  end
+end
+
+bm_res = Benchmark.measure do
+  options.runs.times do
+    parser.parse_block
+    parser.s.pos = pos
+  end
+end
+
+if options.profile_gc
+  $stderr.puts GC::Profiler.result
+  GC::Profiler.disable
+end
+
+if options.prof
+  PerfTools::CpuProfiler.stop
+elsif options.ruby_prof_path
+  res = RubyProf.stop
+  printer = options.ruby_prof_printer.new(res)
+  File.open(options.ruby_prof_path, 'w') do |f|
+    printer.print(f)
+  end
+end
+
+puts bm_res / options.runs

data/bin/maf_to_fasta

@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+
+require 'bio-maf'
+require 'bigbio'
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+options.parser = Bio::MAF::Parser
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: maf_to_fasta [options] <maf> <fasta>"
+  opts.separator ""
+  opts.separator "Options:"
+  opts.on("-p", "--profile PROF", "Profile with PerfTools") do |prof|
+    options.prof = prof
+  end
+  opts.on("--ruby-prof PATH", "Profile with ruby-prof") do |path|
+    options.ruby_prof = path
+  end
+  opts.on("--profile-gc", "Profile GC") do |prof|
+    options.profile_gc = true
+  end
+  opts.on("--parser PARSER", "parser") do |name|
+    options.parser = Bio::MAF.const_get(name)
+  end
+end.parse!(ARGV)
+
+src_path = ARGV.shift
+dst_path = ARGV.shift
+
+if options.prof
+  require 'perftools'
+  PerfTools::CpuProfiler.start(options.prof)
+elsif options.ruby_prof
+  require 'ruby-prof'
+  RubyProf.start
+end
+
+if options.profile_gc
+  GC::Profiler.enable
+end
+
+parser = options.parser.new(src_path)
+writer = FastaWriter.new(dst_path)
+
+parser.each_block do |block|
+  block.each_raw_seq do |seq|
+    seq.write_fasta(writer)
+  end
+end
+
+writer.close
+
+if options.profile_gc
+  $stderr.puts GC::Profiler.result
+  GC::Profiler.disable
+end
+
+if options.prof
+  PerfTools::CpuProfiler.stop
+elsif options.ruby_prof
+  res = RubyProf.stop
+  printer = RubyProf::FlatPrinter.new(res)
+  File.open(options.ruby_prof, 'w') do |f|
+    printer.print(f)
+  end
+end

data/bin/maf_write

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+require 'bio-maf'
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+options.parser = Bio::MAF::Parser
+options.opts = {
+  :chunk_reader => Bio::MAF::ChunkReader,
+  :parse_extended => false
+}
+
+PRINTERS = {
+  'flat' => :FlatPrinter,
+  'stack' => :CallStackPrinter
+}
+
+OptionParser.new do |opts|
+  opts.banner = "Usage: maf_write [options] <maf>"
+  opts.separator ""
+  opts.separator "Options:"
+  opts.on("-p", "--profile PROF", "Profile with PerfTools") do |prof|
+    options.prof = prof
+  end
+  opts.on("--ruby-prof PATH", "Profile with ruby-prof") do |pspec|
+    if pspec =~ /(\w+):(.+)/
+      require 'ruby-prof'
+      options.ruby_prof_printer = RubyProf.const_get(PRINTERS.fetch($1))
+      options.ruby_prof_path = $2
+    else
+      options.ruby_prof_printer = RubyProf::FlatPrinter
+      options.ruby_prof_path = pspec
+    end
+  end
+  opts.on("--profile-gc", "Profile GC") do |prof|
+    options.profile_gc = true
+  end
+  opts.on("--parser PARSER", "parser") do |name|
+    options.parser = Bio::MAF.const_get(name)
+  end
+  opts.on("-t", "--threaded") do
+    options.opts[:chunk_reader] = Bio::MAF::ThreadedChunkReader
+    options.opts[:threads] = 1
+  end
+  opts.on("-e", "--extended") do
+    options.opts[:parse_extended] = true
+    options.opts[:parse_empty] = true
+  end
+end.parse!(ARGV)
+
+src_path = ARGV.shift
+
+if options.prof
+  require 'perftools'
+  PerfTools::CpuProfiler.start(options.prof)
+elsif options.ruby_prof_path
+  require 'ruby-prof'
+  RubyProf.start
+end
+
+if options.profile_gc
+  GC::Profiler.enable
+end
+
+parser = options.parser.new(src_path, options.opts)
+writer = Bio::MAF::Writer.new($stdout)
+writer.write_header(parser.header)
+writer.write_blocks(parser.parse_blocks)
+
+if options.profile_gc
+  $stderr.puts GC::Profiler.result
+  GC::Profiler.disable
+end
+
+if options.prof
+  PerfTools::CpuProfiler.stop
+elsif options.ruby_prof_path
+  res = RubyProf.stop
+  printer = options.ruby_prof_printer.new(res)
+  File.open(options.ruby_prof_path, 'w') do |f|
+    printer.print(f)
+  end
+end

data/bin/random_ranges

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+
+op = OptionParser.new do |opts|
+  opts.banner = "Usage: random_ranges [options]"
+  opts.on("-r", "--range START:END", "range") do |range|
+    s, e = range.split(':')
+    options.start = s.to_i
+    options.end = e.to_i
+  end
+  opts.on("-l", "--length LEN", "block length") do |len|
+    options.length = len.to_i
+  end
+  opts.on("-n", "--number NUM", "number of blocks") do |num|
+    options.num = num.to_i
+  end
+  opts.on("-s", "--sequence SEQ", "sequence") do |seq|
+    options.seq = seq
+  end
+end.parse!(ARGV)
+
+rand = Random.new
+range = options.end - options.start
+block_range = range / options.num
+block_start_range = block_range - options.length
+(0...options.num).each do |n|
+  block_offset = rand.rand(block_start_range)
+  b_start = options.start + (block_range * n) + block_offset
+  b_end = b_start + options.length
+  puts "#{options.seq}\t#{b_start}\t#{b_end}\tx"
+end

data/features/maf-indexing.feature

@@ -0,0 +1,31 @@
+@milestone_2
+Feature: Indexed access to MAF files
+  In order to extract alignment blocks from MAF files
+  By chromosomal ranges matching a source sequence
+  I want to have a way to build indexes on MAF files
+  And use indexes to efficiently find alignment blocks
+  Because linear searches of a 200 GB file are impractical
+
+  Scenario: Index a MAF file
+    Given a MAF source file "mm8_chr7_tiny.maf"
+    When I open it with a MAF reader
+    And build an index on the reference sequence
+    Then the index has at least 8 entries
+    
+  Scenario: Extract alignment blocks by chromosomal range
+    Given a MAF source file "mm8_chr7_tiny.maf"
+    When I open it with a MAF reader
+    And build an index on the reference sequence
+    And search for blocks between positions 80082592 and 80082766 of mm8.chr7
+    Then 2 blocks are obtained
+    And sequence mm8.chr7 of block 0 has start 80082592
+    And sequence mm8.chr7 of block 1 has start 80082713
+    
+  Scenario: Extract alignment blocks by chromosomal range from index file
+    Given a MAF source file "mm8_chr7_tiny.maf"
+    And a Kyoto Cabinet index file "mm8_chr7_tiny.kct"
+    When I open it with a MAF reader
+    And search for blocks between positions 80082592 and 80082766 of mm8.chr7
+    Then 2 blocks are obtained
+    And sequence mm8.chr7 of block 0 has start 80082592
+    And sequence mm8.chr7 of block 1 has start 80082713

data/features/maf-output.feature

@@ -0,0 +1,29 @@
+Feature: MAF output
+  In order to output modified MAF files or subsets of them
+  I want to be able to write out parsed MAF data
+
+  Scenario: Reproduce simple test data
+    Given a MAF source file "mm8_single.maf"
+    When I open it with a MAF reader
+    And open a new MAF writer
+    And write the header from the original MAF file
+    And write all the parsed blocks
+    Then the output should match, except whitespace, "mm8_single.maf"
+
+  Scenario: Reproduce longer test data
+    Given a MAF source file "mm8_chr7_tiny.maf"
+    When I open it with a MAF reader
+    And open a new MAF writer
+    And write the header from the original MAF file
+    And write all the parsed blocks
+    Then the output should match, except whitespace, "mm8_chr7_tiny.maf"
+
+  Scenario: Reproduce test data with i, e, q lines
+    Given a MAF source file "chr22_ieq.maf"
+    When I enable the :parse_extended parser option
+    And I enable the :parse_empty parser option
+    And I open it with a MAF reader
+    And open a new MAF writer
+    And write the header from the original MAF file
+    And write all the parsed blocks
+    Then the output should match, except whitespace, "chr22_ieq.maf"

data/features/maf-parsing.feature

@@ -0,0 +1,44 @@
+Feature: Parse MAF files
+  In order to extract information from a MAF file
+  I want to read it and pull out information
+
+  Scenario: Read MAF header
+    Given MAF data:
+    """
+    ##maf version=1 scoring=humor.v4
+    # humor.v4 R=30 M=10 /cluster/data/hg15/bed/blastz.mm3/axtNet25/chr22.maf /cluster/data/hg15/bed/blastz.rn3/axtNet25/chr22.maf
+
+    a score=0.128
+    s human_hoxa 100  8 + 100257 ACA-TTACT
+    s horse_hoxa 120  9 -  98892 ACAATTGCT
+    s fugu_hoxa   88  7  + 90788 ACA--TGCT
+    """
+    When I open it with a MAF reader
+    Then the MAF version should be "1"
+    And the scoring scheme should be "humor.v4"
+    # third line a continuation
+    And the alignment parameters should be "humor.v4 R=30 M=10 /cluster/data/hg15/bed/blastz.mm3/axtNet25/chr22.maf /cluster/data/hg15/bed/blastz.rn3/axtNet25/chr22.maf" 
+
+  Scenario: Read alignment block
+    Given MAF data:
+    """
+    ##maf version=1 scoring=humor.v4
+    # humor.v4 R=30 M=10 /cluster/data/hg15/bed/blastz.mm3/axtNet300/chr1.maf
+    # /cluster/data/hg15/bed/blastz.rn3/axtNet300/chr1.maf
+
+    a score=0.128
+    s human_hoxa 100  8 + 100257 ACA-TTACT
+    s horse_hoxa 120  9 -  98892 ACAATTGCT
+    s fugu_hoxa   88  7  + 90788 ACA--TGCT
+    """
+    When I open it with a MAF reader
+    Then an alignment block can be obtained
+    And the alignment block has 3 sequences
+    And sequence 0 has source "human_hoxa"
+    And sequence 0 has start 100
+    And sequence 0 has size 8
+    And sequence 0 has strand :+
+    And sequence 0 has source size 100257
+    And sequence 0 has text "ACA-TTACT"
+    And sequence 1 has strand :-
+