bio-maf 1.0.0-java → 1.0.1-java

data/bin/maf_bgzip

@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 
+require 'optparse'
 require 'ostruct'
 
 require 'bio-maf'
@@ -8,6 +9,9 @@ require 'bio-bgzf'
 $options = OpenStruct.new
 $options.dir = '.'
 $options.ref_only = true
+$options.n_jobs = 1
+$options.force = false
+$options.level = 2
 
 op = OptionParser.new do |opts|
   opts.banner = "Usage: maf_bgzip [options] [<maf> ...]"
@@ -26,31 +30,155 @@ op = OptionParser.new do |opts|
           "(has no effect without --index)") do
     $options.ref_only = false
   end
+  opts.on("-l", "--level LEVEL", Integer,
+          "gzip compression level for BGZF (1-9)") do |level|
+    unless 1 <= level && level <= 9
+      $stderr.puts "Invalid compression level: #{level}"
+      $stderr.puts opts
+      exit 2
+    end
+    $options.level = level
+  end
+  opts.on("-f", "--force",
+          "Replace output files if they already exist") do
+    $options.force = true
+  end
+  opts.on("-j", "--jobs N", Integer,
+          "Run N concurrent jobs (default 1)") do |n|
+    $options.n_jobs = n
+  end
+  Bio::MAF::handle_logging_options(opts)
 end
 
 op.parse!(ARGV)
+Bio::Log::CLI.configure('bio-maf')
 
-until ARGV.empty?
-  maf_path = ARGV.shift
-  maf_base = File.basename(maf_path)
+INTERVAL = 10
+LOG = Bio::MAF::LOG
+
+def make_processing_task(maf)
+  maf_base = File.basename(maf)
   base = maf_base.gsub(/\.maf.*/, '')
   bgz_path = "#{$options.dir}/#{base}.maf.bgz"
+  if File.exist?(bgz_path) && ! $options.force
+    LOG.error "#{bgz_path} already exists, refusing to overwrite " \
+    "without --force!"
+    exit 1
+  end
+  idx_path = nil
+  if $options.index
+    idx_path = "#{$options.dir}/#{base}.kct"
+    if File.exist?(idx_path) && ! $options.force
+      LOG.error "#{idx_path} already exists, refusing to overwrite " \
+      "without --force!"
+      exit 1
+    end
+  end
+  lambda { process_maf(maf, bgz_path, idx_path) }
+end
+
+def process_maf(maf_path, bgz_path, idx_path)
+  maf_base = File.basename(maf_path)
+  LOG.debug { "Processing #{maf_base}." }
   p = Bio::MAF::Parser.new(maf_path,
-                           :parse_extended => true,
-                           :parse_empty => true)
-  File.open(bgz_path, 'w') do |out_f|
-    Bio::BGZF::Writer.new(out_f) do |bgz_w|
+                           :retain_text => true)
+  if idx_path
+    if File.exists?(idx_path)
+      File.unlink(idx_path)
+    end
+    idx = Bio::MAF::KyotoIndex.new(idx_path)
+    idx.prep(bgz_path, :bgzf, $options.ref_only)
+    exec = Bio::MAF::Executor.create
+  end
+  start_t = Time.now
+  last_t = start_t
+  last_pos = 0
+  n_blocks = 0
+  maf_size = File.size(maf_path)
+  File.open(bgz_path, 'wb') do |out_f|
+    Bio::BGZF::Writer.new(out_f, $options.level) do |bgz_w|
       maf_w = Bio::MAF::Writer.new(bgz_w)
       maf_w.write_header(p.header)
       p.each_block do |block|
-        maf_w.write_block(block)
+        bgz_w.write(block.orig_text)
+        if idx
+          block.offset = bgz_w.last_write_pos
+          exec.submit do
+            idx.index_blocks([block])
+          end
+        end
+        n_blocks += 1
+        if n_blocks % 100 == 0
+          cur_t = Time.now
+          delta_t = cur_t - last_t
+          if delta_t > INTERVAL
+            cur_pos = p.phys_f.tell
+            LOG.debug {
+              pos_mb = cur_pos.to_f / 1048576
+              delta_bytes = cur_pos - last_pos
+              rate = delta_bytes.to_f / delta_t
+              mb_rate = rate / 1048576
+              pct = cur_pos.to_f / maf_size * 100
+              elapsed = cur_t - start_t
+              sprintf("%s: processed %.1f MB (%.1f%%) in %ds, %.2f MB/s.",
+                                maf_base,
+                                pos_mb,
+                                pct,
+                                elapsed,
+                      mb_rate)
+            }
+            last_t = cur_t
+            last_pos = cur_pos
+          end
+        end
       end
     end
   end
+  unc = p.f.tell if p.f != p.phys_f
   p.close
-  if $options.index
-    p2 = Bio::MAF::Parser.new(bgz_path)
-    idx_path = "#{$options.dir}/#{base}.kct"
-    Bio::MAF::KyotoIndex.build(p2, idx_path, $options.ref_only)
+  if idx
+    exec.shutdown
+    idx.db.synchronize(true)
+  end
+  elapsed = Time.now - start_t
+  mb = maf_size.to_f / 1048576
+  mb_rate = mb / elapsed
+  LOG.info { sprintf("Processed %s (%.1f MB) in %ds, %.2f MB/s",
+                     maf_base,
+                     mb,
+                     elapsed,
+                     mb_rate) }
+  if unc
+    LOG.info {
+      unc_mb = unc / 1048576
+      unc_rate = unc_mb / elapsed
+      sprintf("  Uncompressed: %.1f MB, %.2f MB/s",
+              unc_mb, unc_rate)
+    }
   end
+  LOG.info {
+    raw_size = unc || maf_size
+    avg_block_kb = raw_size.to_f / n_blocks / 1024
+    sprintf("  %d alignment blocks, average size %.2f KB",
+            n_blocks, avg_block_kb)
+  }
+  LOG.info {
+    orig_size = unc ? unc : maf_size
+    bgzf_size = File.size(bgz_path).to_f
+    ratio = bgzf_size / orig_size
+    sprintf("  Compressed with BGZF (level=%d) to %.1f MB (%.1fx)",
+            $options.level,
+            bgzf_size / 1048576,
+            ratio)
+  }
+end
+
+runner = Bio::MAF::JobRunner.create($options.n_jobs)
+LOG.debug "Created #{runner.class} set for #{$options.n_jobs} concurrent jobs."
+ARGV.each do |maf|
+  task = make_processing_task(maf)
+  runner.add(&task)
 end
+LOG.debug "Running jobs."
+runner.run
+LOG.debug "Finished processing."

data/bin/maf_extract

@@ -6,12 +6,13 @@ require 'ostruct'
 
 include Bio::MAF
 
-options = OpenStruct.new
-options.mode = :intersect
-options.format = :maf
-options.seq_filter = {}
-options.block_filter = {}
-options.parse_options = {}
+$options = OpenStruct.new
+$options.mode = :intersect
+$options.format = :maf
+$options.one_based = false
+$options.seq_filter = {}
+$options.block_filter = {}
+$options.parse_options = {}
 
 def handle_list_spec(spec)
   if spec =~ /^@(.+)/
@@ -23,7 +24,11 @@ end
 
 def handle_interval_spec(int)
   if int =~ /(.+):(\d+)-(\d+)/
-    Bio::GenomicInterval.zero_based($1, $2.to_i, $3.to_i)
+    if $options.one_based
+      Bio::GenomicInterval.new($1, $2.to_i, $3.to_i)
+    else
+      Bio::GenomicInterval.zero_based($1, $2.to_i, $3.to_i)
+    end
   else
     raise "Invalid interval specification: #{int}"
   end
@@ -34,13 +39,13 @@ $op = OptionParser.new do |opts|
   opts.separator ""
   opts.separator "MAF source options (either --maf or --maf-dir must be given):"
   opts.on("-m", "--maf MAF", "MAF file") do |maf|
-    options.maf = maf
+    $options.maf = maf
   end
   opts.on("-i", "--index INDEX", "MAF index") do |idx|
-    options.idx = idx
+    $options.idx = idx
   end
   opts.on("-d", "--maf-dir DIR", "MAF directory") do |dir|
-    options.maf_dir = dir
+    $options.maf_dir = dir
   end
   opts.separator ""
   opts.separator "Extraction options:"
@@ -49,21 +54,26 @@ $op = OptionParser.new do |opts|
           "blocks intersecting the given region,",
           "or 'slice' to extract subsets covering ",
           "given regions") do |mode|
-    options.mode = mode
+    $options.mode = mode
   end
   opts.on("--bed BED", "Use intervals from the given BED file") do |bed|
-    options.bed = bed
+    $options.bed = bed
   end
   opts.on("--interval SEQ:START:END", "Zero-based genomic interval to match") do |int|
-    options.interval = handle_interval_spec(int)
+    $options.interval = handle_interval_spec(int)
+  end
+  opts.on("--one-based",
+          "Treat all intervals as one-based",
+          "(even from BED files, contrary to the standard)") do
+    $options.one_based = true
   end
   opts.separator ""
   opts.separator "Output options:"
   opts.on("-f", "--format FMT", [:maf, :fasta], "Output format") do |fmt|
-    options.format = fmt
+    $options.format = fmt
   end
   opts.on("-o", "--output OUT", "Write output to file OUT") do |out|
-    options.out_path = out
+    $options.out_path = out
   end
   opts.separator ""
   opts.separator "Filtering options:"
@@ -71,41 +81,41 @@ $op = OptionParser.new do |opts|
           "Filter out all but the species in the",
           "given comma-separated list",
           "(or @FILE to read from a file)") do |spec|
-    options.seq_filter[:only_species] = handle_list_spec(spec)
+    $options.seq_filter[:only_species] = handle_list_spec(spec)
   end
   opts.on("--with-all-species SPECIES",
           "Only match blocks with all the given",
           "species, comma-separated",
           "(or @FILE to read from a file)") do |spec|
-    options.block_filter[:with_all_species] = handle_list_spec(spec)
+    $options.block_filter[:with_all_species] = handle_list_spec(spec)
   end
   opts.on("--min-sequences N", Integer,
           "Match only blocks with at least N sequences") do |n|
-    options.block_filter[:at_least_n_sequences] = n
+    $options.block_filter[:at_least_n_sequences] = n
   end
   opts.on("--min-text-size N", Integer,
           "Match only blocks with minimum text size N") do |n|
-    options.block_filter[:min_size] = n
+    $options.block_filter[:min_size] = n
   end
   opts.on("--max-text-size N", Integer,
           "Match only blocks with maximum text size N") do |n|
-    options.block_filter[:max_size] = n
+    $options.block_filter[:max_size] = n
   end
   opts.separator ""
   opts.separator "Block processing options:"
   opts.on("--join-blocks",
           "Join blocks if appropriate after filtering",
           "out sequences") do
-    options.parse_options[:join_blocks] = true
+    $options.parse_options[:join_blocks] = true
   end
   opts.on("--remove-gaps", "Remove gaps after filtering out sequences") do
-    options.parse_options[:remove_gaps] = true
+    $options.parse_options[:remove_gaps] = true
   end
   opts.on("--parse-extended", "Parse 'extended' MAF data (i, q lines)") do
-    options.parse_options[:parse_extended] = true
+    $options.parse_options[:parse_extended] = true
   end
   opts.on("--parse-empty", "Parse empty (e) lines of MAF data") do
-    options.parse_options[:parse_empty] = true
+    $options.parse_options[:parse_empty] = true
   end
   opts.separator ""
   opts.separator "Logging options:"
@@ -120,24 +130,24 @@ def usage(msg)
   exit 2
 end
 
-if options.maf
-  access = Access.file(options.maf, options.idx, options.parse_options)
-elsif options.maf_dir
-  access = Access.maf_dir(options.maf_dir, options.parse_options)
+if $options.maf
+  access = Access.file($options.maf, $options.idx, $options.parse_options)
+elsif $options.maf_dir
+  access = Access.maf_dir($options.maf_dir, $options.parse_options)
 else
   usage "Must supply --maf or --maf-dir!"
 end
 
 begin
-  access.sequence_filter = options.seq_filter unless options.seq_filter.empty?
-  access.block_filter = options.block_filter unless options.block_filter.empty?
-  if options.out_path
-    outf = File.open(options.out_path, 'w')
+  access.sequence_filter = $options.seq_filter unless $options.seq_filter.empty?
+  access.block_filter = $options.block_filter unless $options.block_filter.empty?
+  if $options.out_path
+    outf = File.open($options.out_path, 'w')
   else
     outf = $stdout
   end
 
-  case options.format
+  case $options.format
   when :maf
     writer = Writer.new(outf)
   when :fasta
@@ -146,20 +156,20 @@ begin
     raise "unsupported output format #{format}!"
   end
 
-  if options.bed
-    intervals = read_bed_intervals(options.bed)
-  elsif options.interval
-    intervals = [options.interval]
+  if $options.bed
+    intervals = read_bed_intervals($options.bed)
+  elsif $options.interval
+    intervals = [$options.interval]
   else
     usage "Must supply --interval or --bed!"
   end
 
   # TODO: provide access to original MAF header?
-  if options.format == :maf
+  if $options.format == :maf
     writer.write_header(Header.default)
   end
 
-  case options.mode
+  case $options.mode
   when :intersect
     access.find(intervals) do |block|
       writer.write_block(block)
@@ -172,7 +182,7 @@ begin
       end
     end
   else
-    raise "Unsupported mode #{options.mode}!"
+    raise "Unsupported mode #{$options.mode}!"
   end
   
 ensure

data/bin/maf_index

@@ -14,10 +14,10 @@ PRINTERS = {
 $options = OpenStruct.new
 $options.mode = :build
 $options.ref_only = true
+$options.parser_opts = { :parse_extended => false }
 
 def build_index(maf, index)
-  parser = Bio::MAF::Parser.new(maf,
-                                :parse_extended => false)
+  parser = Bio::MAF::Parser.new(maf, $options.parser_opts)
   idx = Bio::MAF::KyotoIndex.build(parser, index, $options.ref_only)
   idx.close
 end
@@ -36,6 +36,15 @@ op = OptionParser.new do |opts|
   opts.on("-d", "--dump", "Dump contents of given INDEX") do
     $options.mode = :dump
   end
+  opts.on("-O", "--parser-option OPT") do |opt|
+    if opt =~ /(-?)(.+)/
+      val = ! ($1 == "-")
+      option = $2.to_sym
+      $options.parser_opts[option] = val
+    else
+      raise "malformed parser option #{opt}!"
+    end
+  end
   opts.on("--ruby-prof PATH", "Profile with ruby-prof") do |pspec|
     require 'ruby-prof'
     if pspec =~ /(\w+):(.+)/

data/bin/maf_tile

@@ -10,7 +10,16 @@ def parse_interval(line)
   src, r_start_s, r_end_s, _ = line.split(nil, 4)
   r_start = r_start_s.to_i
   r_end = r_end_s.to_i
-  return Bio::GenomicInterval.zero_based(src, r_start, r_end)
+  i_src = if $options.bed_species
+            "#{$options.bed_species}.#{src}"
+          else
+            src
+          end
+  if $options.one_based
+    Bio::GenomicInterval.new(i_src, r_start, r_end)
+  else
+    Bio::GenomicInterval.zero_based(i_src, r_start, r_end)
+  end
 end
 
 def target_for(base, interval, &blk)
@@ -18,52 +27,96 @@ def target_for(base, interval, &blk)
   File.open(path, 'w', &blk)
 end
 
-def apply_options(options, tiler)
-  tiler.reference = options.ref if options.ref
-  tiler.species = options.species
-  tiler.species_map = options.species_map
+def apply_options(tiler)
+  tiler.reference = $options.ref if $options.ref
+  tiler.species = $options.species
+  tiler.species_map = $options.species_map
+  tiler.fill_char = $options.fill_char if $options.fill_char
 end
 
-options = OpenStruct.new
-options.p = { :threads => 1 }
-options.species = []
-options.species_map = {}
-options.usage = false
+def each_tiler(access, intervals)
+  intervals.each do |int|
+    access.tile(int) do |tiler|
+      apply_options(tiler)
+      yield tiler
+    end
+  end
+end
+
+$options = OpenStruct.new
+$options.p = { :threads => 1 }
+$options.species = []
+$options.species_map = {}
+$options.usage = false
 
 o_parser = OptionParser.new do |opts|
   opts.banner = "Usage: maf_tile [options] <maf> [index]"
   opts.separator ""
   opts.separator "Options:"
   opts.on("-r", "--reference SEQ", "FASTA reference sequence") do |ref|
-    options.ref = ref
-  end
-  opts.on("-i", "--interval [CHR:]BEGIN:END", "Genomic interval, zero-based") do |int|
-    if int =~ /(.+):(\d+):(\d+)/
-      gi = Bio::GenomicInterval.zero_based($1, ($2.to_i), ($3.to_i))
-      options.genomic_interval = gi
-    elsif int =~ /(\d+):(\d+)/
-      options.interval = ($1.to_i)...($2.to_i)
+    $options.ref = ref
+  end
+  opts.on("-i", "--interval [CHR:]BEGIN-END", "Genomic interval, zero-based") do |int|
+    if int =~ /(.+):(\d+)-(\d+)/
+      $options.genomic_interval_spec = [$1, $2.to_i, $3.to_i]
+    elsif int =~ /(\d+)-(\d+)/
+      $options.interval = ($1.to_i)...($2.to_i)
     else
       $stderr.puts "Invalid interval specification #{int}!"
-      options.usage = true
+      $options.usage = true
     end
   end
-  opts.on("-s", "--species SPECIES[:NAME]", "Species to use (with mapped name)") do |sp|
+  opts.on("--one-based",
+          "Treat all intervals as one-based",
+          "(even from BED files, contrary to the standard)") do
+    $options.one_based = true
+  end
+  opts.on("-s", "--species SPECIES[:NAME]",
+          "Species to use (mapped name optional)",
+          "(can be a comma-separated list)") do |sp|
     if sp =~ /:/
       species, mapped = sp.split(/:/)
-      options.species << species
-      options.species_map[species] = mapped
+      $options.species << species
+      $options.species_map[species] = mapped
+    elsif sp =~ /,/
+      $options.species.concat(sp.split(/,/))
     else
-      options.species << sp
+      $options.species << sp
+    end
+  end
+  opts.on("--species-file FILE", "File specifying species and optionally mapped names") do |file|
+    File.open(file) do |f|
+      f.each_line do |line|
+        next if line =~ /^#/
+        parts = line.split
+        next unless parts.size > 0
+        $options.species << parts[0]
+        $options.species_map[parts[0]] = parts[1] if parts[1]
+      end
     end
   end
   opts.on("-o", "--output-base BASE", "Base name for output files",
           "Use stdout for a single interval if not given") do |base|
-    options.output_base = base
+    $options.output_base = base
   end
   opts.on("--bed BED", "BED file specifying intervals",
           "(requires --output-base)") do |bed|
-    options.bed = bed
+    $options.bed = bed
+  end
+  opts.on("--bed-species SPECIES",
+          "Species to prepend to BED chromosome specs") do |species|
+    $options.bed_species = species
+  end
+  opts.on("--fill-char C",
+          "Fill gaps with character C",
+          "(default is *)") do |char|
+    $options.fill_char = char
+  end
+  opts.on("--upcase", "Fold all sequence data to upper case") do
+    $options.p[:upcase] = true
+  end
+  opts.on("--concat", "Concatenate result blocks") do
+    $options.concat = true
   end
   Bio::MAF::handle_logging_options(opts)
 end
@@ -74,52 +127,96 @@ Bio::Log::CLI.configure('bio-maf')
 maf_p = ARGV.shift
 index_p = ARGV.shift
 
-unless (! options.usage) \
-  && maf_p && (! options.species.empty?) \
-  && (options.output_base \
-      ? options.bed \
-      : options.interval || options.genomic_interval)
+unless maf_p
+  $stderr.puts "Must specify MAF file to process!"
+  $options.usage = true
+end
+
+if $options.species.empty?
+  $stderr.puts "Must specify species to tile with --species!"
+  $options.usage = true
+end
+
+unless $options.bed || $options.interval || $options.genomic_interval_spec
+  $stderr.puts "Must specify --bed or --interval!"
+  $options.usage = true
+end
+
+if $options.bed && ! ($options.output_base || $options.concat)
+  $stderr.puts "Must specify --output-base or --concat when specifying --bed!"
+  $options.usage = true
+end
+
+if (! $options.output_base) && ! ($options.interval || $options.genomic_interval_spec || ($options.bed && $options.concat))
+  $stderr.puts "Must specify --interval or --bed with --concat if --output-base is not given!"
+  $options.usage = true
+end
+
+if $options.usage
   $stderr.puts o_parser
   exit 2
 end
 
 access = if File.directory? maf_p
-           Bio::MAF::Access.maf_dir(maf_p, options.p)
+           Bio::MAF::Access.maf_dir(maf_p, $options.p)
          else
-           Bio::MAF::Access.file(maf_p, index_p, options.p)
+           Bio::MAF::Access.file(maf_p, index_p, $options.p)
          end
 
-if options.bed
+if $options.bed
   intervals = []
-  File.open(options.bed) do |bed_f|
+  File.open($options.bed) do |bed_f|
     bed_f.each_line { |line| intervals << parse_interval(line) }
   end
-  intervals.sort_by! { |int| int.zero_start }
-  intervals.each do |int|
-    access.tile(int) do |tiler|
-      apply_options(options, tiler)
-      target_for(options.output_base, int) do |target|
+  #intervals.sort_by! { |int| int.zero_start }
+  if $options.concat
+    # concatenate, as with exons
+    concat = Array.new($options.species.size)
+    concat.fill { '' }
+    non_fill = nil
+    each_tiler(access, intervals) do |tiler|
+      non_fill = tiler.non_fill_re if ! non_fill
+      concat.zip(tiler.build_bio_alignment) do |buf, seq|
+        buf << seq.to_s
+      end
+    end
+    fh = $options.output_base ? File.open($options.output_base, 'wb') : $stdout
+    $options.species.zip(concat) do |species, seq|
+      if non_fill.match(seq)
+        sp_out_name = $options.species_map[species] || species
+        fh.puts ">#{sp_out_name}", seq.scan(/.{1,70}/)
+      end
+    end
+  else
+    # output each interval separately
+    each_tiler(access, intervals) do |tiler|
+      target_for($options.output_base, tiler.interval) do |target|
         tiler.write_fasta(target)
       end
     end
   end
 else
   # single interval
-  if options.genomic_interval
-    interval = options.genomic_interval
+  if $options.genomic_interval_spec
+    spec = $options.genomic_interval_spec
+    if $options.one_based
+      interval = Bio::GenomicInterval.new(*spec)
+    else
+      interval = Bio::GenomicInterval.zero_based(*spec)
+    end
   else
     if access.indices.size != 1
       raise "Must explicitly specify sequence in --interval argument with multiple candidate MAF files!"
     end
     ref_seq = access.indices.keys.first
     interval = Bio::GenomicInterval.zero_based(ref_seq,
-                                               options.interval.begin,
-                                               options.interval.end)
+                                               $options.interval.begin,
+                                               $options.interval.end)
   end
   access.tile(interval) do |tiler|
-    apply_options(options, tiler)
-    if options.output_base
-      target = target_for(options.output_base, tiler.interval)
+    apply_options(tiler)
+    if $options.output_base
+      target = target_for($options.output_base, tiler.interval)
     else
       target = $stdout
     end