scout-gear 1.2.0 → 5.1.1

data/bin/scout

@@ -0,0 +1,235 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.join(__dir__, '../lib')
+
+require 'scout-gear'
+
+class CmdStop < Exception
+  attr_accessor :exit_status
+  def initialize(exit_status = 0)
+    @exit_status = exit_status
+  end
+end
+
+# Add paths to scout repos under --dev to LOAD_PATHS
+dev_dir = nil
+if _i = ARGV.index("--dev")
+  dev_dir = ARGV[_i+1]
+  ARGV.delete "--dev"
+  ARGV.delete dev_dir
+end
+
+if dev_dir.nil? 
+  _s = nil
+  ARGV.each_with_index do |s,i|
+    if s.match(/^--dev(?:=(.*))?/)
+      dev_dir = $1
+      _s = s
+      next
+    end
+  end
+  ARGV.delete _s if _s
+end
+
+if dev_dir
+  ['scout-*/lib'].each do |pattern|
+    Dir.glob(File.join(File.expand_path(dev_dir), pattern)).each do |f|
+      $LOAD_PATH.unshift f
+    end
+  end
+end
+
+require 'scout/simple_opt'
+
+options = SOPT.setup <<EOF
+Ruby bioinformatics toolkit
+
+$ #{$0} <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value'
+
+
+--log*                  #{Log.color :yellow, "Log level from 0 (debug) 6 (errors)"}
+--dev*                  #{Log.color :yellow, "Find development libraries in the directory specified"}
+--nocolor               #{Log.color :yellow, "Disable colored output"}
+--nobar                 #{Log.color :yellow, "Disable progress report"}
+--locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
+EOF
+
+Log.nocolor = true if options[:nocolor]
+
+locate = options.delete :locate_file
+
+if options[:log_file]
+  Log.logfile(options[:log_file])
+end
+
+if options[:log]
+  Log.severity = options[:log].to_i
+else
+  global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
+  if ENV["SCOUT_LOG"]
+    Log.severity = ENV["SCOUT_LOG"].to_i 
+  else
+    global_severity = Log.get_level(Scout.etc.log_severity.read.strip) if Scout.etc.log_severity.exists?
+    Log.severity = global_severity.to_i if global_severity
+  end
+end
+
+$scout_command_dir = Scout.bin.scout
+$scout_command_dir.path_maps[:scout_commands] = File.join(File.dirname(__dir__), "{PATH/bin\\/scout/scout_commands}")
+
+SOPT.description =<<EOF
+This command controls many aspects of the Scout framework, from configuration tasks to running applications. 
+
+Commands are implemented in separate files under the Scout path '#{$scout_command_dir}'. 
+Known locations are: #{([$scout_command_dir] + $scout_command_dir.find_all) * ", " }. 
+You can place your own commads at #{$scout_command_dir.find(:user)}.
+EOF
+
+if options[:profile]
+  require 'ruby-prof'
+  RubyProf.start
+end
+
+
+def prev_dir(prev)
+    scout_command_dir = $scout_command_dir
+
+    prev.each do |previous_command|
+        scout_command_dir = scout_command_dir[previous_command]
+    end
+
+    scout_command_dir
+end
+
+def commands(prev)
+    scout_command_dir = prev_dir(prev)
+
+    command_file_dirs = scout_command_dir.find_all
+    command_files = command_file_dirs.collect{|d| d.glob('*') }.flatten
+    command_files.collect{|p| File.basename(p) }.uniq.reject{|p| p =~ /\.desc$/}.sort
+end
+
+def scout_usage(prev = nil)
+  puts
+  puts SOPT.doc
+
+  if prev
+    puts
+    puts Log.color :magenta, "## COMMANDS"
+    puts
+    puts Log.color :magenta, "Command:"
+    puts 
+    puts "  #{File.basename($0)} #{prev * " "}"
+    puts 
+    puts Log.color :magenta, "Subcommands:"
+    puts 
+    prev_dir = prev_dir(prev)
+    commands(prev).each do |command|
+      directory = File.directory? prev_dir[command].find
+      if directory
+        puts "  " << Log.color(:blue, command)
+      else
+        puts "  " << command
+      end
+    end
+  end
+  puts
+  true
+end
+
+alias usage scout_usage 
+
+def print_error(error, backtrace = nil)
+  puts Log.color :magenta, "## ERROR"
+  puts
+  if backtrace
+    puts Log.color :red, "Backtrace: "
+    puts
+    puts Log.color_stack(backtrace.reverse) * "\n"
+    puts
+  end
+  puts Log.color :red, error
+  puts
+end
+
+def aliases
+  @aliases ||= Scout.etc.cmd_alias.exists? ? Scout.etc.cmd_alias.yaml : {}
+end
+
+def tokenize_cmd_params(str)
+ return str if Array === str
+ str.scan(/
+          (?:["']([^"']*?)["']) |
+          ([^"'\s]+)
+          /x).flatten.compact
+end
+
+def cmd_alias
+  while ARGV[0] && aliases.include?(ARGV[0])
+    ARGV.replace tokenize_cmd_params(aliases[ARGV[0]]) + ARGV[1..-1]
+  end
+end
+    
+dir = $scout_command_dir
+$previous_commands = []
+
+cmd_alias
+
+exit_status = 0
+begin
+  while ARGV.any?
+    $command = ARGV.shift
+    case
+    when File.directory?(dir[$command].find)
+      $previous_commands << $command
+      dir = dir[$command]
+    when dir[$command].exists?
+      if locate
+        puts dir[$command].find
+        exit_status = 0
+        exit exit_status
+      else
+        load dir[$command].find
+        exit_status = 0
+        exit exit_status
+      end
+    when File.exist?($command)
+      load $command
+      exit_status = 0
+      exit exit_status
+    else
+      error = "Command '#{$command }' not understood"
+      scout_usage($previous_commands)
+      print_error(error)
+      exit_status = -1
+      exit exit_status
+    end
+  end
+
+  scout_usage($previous_commands)
+  exit_status = 0
+  exit exit_status
+
+rescue ParameterException
+  puts
+  scout_usage
+  print_error($!.message, $!.backtrace)
+  puts
+  exit_status = -1
+  exit exit_status
+rescue SystemExit,CmdStop
+  exit_status = $!.status
+  exit exit_status
+rescue Exception
+  Log.exception $!
+  exit_status = -1
+  exit exit_status
+ensure
+  if options[:profile]
+    result = RubyProf.stop
+    printer = RubyProf::FlatPrinter.new(result)
+    printer.print(STDOUT, :min_percent => 10)
+  end
+end
+
+exit exit_status

data/lib/scout/cmd.rb

@@ -0,0 +1,344 @@
+require_relative 'indiferent_hash'
+require_relative 'concurrent_stream'
+require_relative 'log'
+require_relative 'open/stream'
+require 'stringio'
+require 'open3'
+
+module CMD
+
+  TOOLS = IndiferentHash.setup({})
+  def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
+    TOOLS[tool] = [claim, test, block, cmd]
+  end
+
+  def self.conda(tool, env = nil, channel = 'bioconda')
+    if env
+      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")      
+    else
+      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")      
+    end
+  end
+
+  def self.get_tool(tool)
+    return tool.to_s unless TOOLS[tool]
+
+    @@init_cmd_tool ||= IndiferentHash.setup({})
+    if !@@init_cmd_tool[tool]
+      claim, test, block, cmd = TOOLS[tool]
+      begin
+        if test
+          CMD.cmd(test + " ")
+        else
+          CMD.cmd("#{tool} --help")
+        end
+      rescue
+        if claim
+          claim.produce
+        else
+          block.call
+        end
+      end
+      version_txt = ""
+      version = nil
+      ["--version", "-version", "--help", ""].each do |f|
+        begin
+          version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
+          version = CMD.scan_version_text(version_txt, tool)
+          break if version
+        rescue
+          Log.exception $!
+        end
+      end
+
+      @@init_cmd_tool[tool] = version || true
+
+      return cmd if cmd
+    end
+
+    tool.to_s
+  end
+
+  def self.scan_version_text(text, cmd = nil)
+    cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
+    text.split("\n").each do |line|
+      next unless line =~ /\W#{cmd}\W/i
+      m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+      return m[1] if m
+      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?v)/i)
+      return m[1] if m
+      next unless line =~ /\Wversion\W/i
+      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+      return m[1] if m
+    end
+    m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+    return m[1] if m
+    m = text.match(/(?:#{cmd}.*(v.*|.*v))/i)
+    return m[1] if m
+    nil
+  end
+  def self.versions
+    return {} unless defined? @@init_cmd_tool
+    @@init_cmd_tool.select{|k,v| v =~ /\d+\./ }
+  end
+
+  def self.bash(cmd)
+    cmd = %Q(bash <<EOF\n#{cmd}\nEOF\n)
+    CMD.cmd(cmd, :autojoin => true)
+  end
+
+  def self.process_cmd_options(options = {})
+    add_dashes = IndiferentHash.process_options options, :add_option_dashes
+
+    string = ""
+    options.each do |option, value|
+      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
+      #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
+      value = value.gsub("'","\\'") if value.to_s.include? "'"
+
+      option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
+
+      case 
+      when value.nil? || FalseClass === value 
+        next
+      when TrueClass === value
+        string << "#{option} "
+      else
+        if option.to_s.chars.to_a.last == "="
+          string << "#{option}'#{value}' "
+        else
+          string << "#{option} '#{value}' "
+        end
+      end
+    end
+
+    string.strip
+  end
+
+  def self.cmd(tool, cmd = nil, options = {}, &block)
+    options, cmd = cmd, nil if Hash === cmd
+
+    options    = IndiferentHash.add_defaults options, :stderr => Log::DEBUG
+    in_content = options.delete(:in)
+    stderr     = options.delete(:stderr)
+    post       = options.delete(:post)
+    pipe       = options.delete(:pipe)
+    log        = options.delete(:log)
+    no_fail    = options.delete(:no_fail)
+    no_fail    = options.delete(:nofail) if no_fail.nil?
+    no_wait    = options.delete(:no_wait)
+    xvfb       = options.delete(:xvfb)
+    bar        = options.delete(:progress_bar)
+    save_stderr = options.delete(:save_stderr)
+    autojoin   = options.delete(:autojoin)
+    autojoin   = no_wait if autojoin.nil?
+
+    dont_close_in  = options.delete(:dont_close_in)
+
+    log = true if log.nil?
+    
+    if cmd.nil? && ! Symbol === tool 
+      cmd = tool
+    else
+      tool = get_tool(tool)
+      if cmd.nil?
+        cmd = tool
+      else
+        cmd = tool + ' ' + cmd
+      end
+
+    end
+
+    case xvfb
+    when TrueClass
+      cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
+    when String
+      cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
+    when String
+    end
+
+    if stderr == true
+      stderr = Log::HIGH
+    end
+
+    cmd_options = process_cmd_options options
+    if cmd =~ /'\{opt\}'/
+      cmd.sub!('\'{opt}\'', cmd_options) 
+    else
+      cmd << " " << cmd_options
+    end
+
+    in_content = StringIO.new in_content if String === in_content
+
+    sin, sout, serr, wait_thr = begin
+                                  Open3.popen3(ENV, cmd)
+                                rescue
+                                  Log.warn $!.message
+                                  raise ProcessFailed, nil, cmd unless no_fail
+                                  return
+                                end
+    pid = wait_thr.pid
+
+    Log.debug{"CMD: [#{pid}] #{cmd}" if log}
+
+    if in_content.respond_to?(:read)
+      in_thread = Thread.new(Thread.current) do |parent|
+        Thread.current.report_on_exception = false if no_fail
+        begin
+          begin
+            while c = in_content.readpartial(Open::BLOCK_SIZE)
+              sin << c 
+            end
+          rescue EOFError
+          end
+          sin.close  unless sin.closed?
+
+          unless dont_close_in
+            in_content.close unless in_content.closed? 
+            in_content.join if in_content.respond_to? :join 
+          end
+        rescue
+          Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
+          raise $!
+        end
+      end
+    else
+      in_thread = nil
+      sin.close
+    end
+
+    pids = [pid]
+
+    if pipe
+
+      ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail 
+
+      sout.callback = post if post
+
+      if (Integer === stderr and log) || bar
+        err_thread = Thread.new do
+          Thread.current["name"] = "Error log: [#{pid}] #{ cmd }"
+          begin
+          while line = serr.gets
+            bar.process(line) if bar
+            sout.log = line
+            sout.std_err << line if save_stderr
+            Log.log "STDERR [#{pid}]: " +  line, stderr if log
+          end 
+          serr.close
+          rescue
+            Log.exception $!
+            raise $!
+          end
+        end
+      else
+        err_thread = Open.consume_stream(serr, true)
+      end
+
+      sout.threads = [in_thread, err_thread, wait_thr].compact
+
+      sout
+    else
+
+      if bar
+        err = ""
+        err_thread = Thread.new do
+          while not serr.eof?
+            line = serr.gets 
+            bar.process(line) 
+            err << line if Integer === stderr and log
+          end
+          serr.close
+        end
+      elsif log and Integer === stderr
+        err = ""
+        err_thread = Thread.new do
+          while not serr.eof?
+            err << serr.gets 
+          end
+          serr.close
+        end
+      else
+        Open.consume_stream(serr, true)
+        #serr.close 
+        err_thread = nil
+        err = ""
+      end
+
+      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail 
+
+      begin
+        out = StringIO.new sout.read
+        sout.close unless sout.closed?
+
+        status = wait_thr.value
+        if not status.success? and not no_fail
+          if !err.empty?
+            raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
+          else
+            raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
+          end
+        else
+          Log.log err, stderr if Integer === stderr and log
+        end
+        out
+      ensure
+        post.call if post
+      end
+    end
+  end
+
+  def self.cmd_pid(*args)
+    all_args = *args
+
+    bar = all_args.last[:progress_bar] if Hash === all_args.last
+
+    all_args << {} unless Hash === all_args.last
+
+    level = all_args.last[:log] || 0
+    level = 0 if TrueClass === level
+    level = 10 if FalseClass === level
+    level = level.to_i
+
+    all_args.last[:log] = true
+    all_args.last[:pipe] = true
+
+    io = cmd(*all_args)
+    pid = io.pids.first
+
+    line = "" if bar
+    starting = true
+    while c = io.getc
+      if starting
+        if pid
+          Log.logn "STDOUT [#{pid}]: ", level
+        else
+          Log.logn "STDOUT: ", level
+        end
+        starting = false
+      end
+      STDERR << c if Log.severity <= level
+      line << c if bar
+      if c == "\n"
+        bar.process(line) if bar
+        starting = true
+        line = "" if bar
+      end
+    end 
+    begin
+      io.join
+      bar.remove if bar
+    rescue
+      bar.remove(true) if bar
+      raise $!
+    end
+
+    nil
+  end
+
+  def self.cmd_log(*args)
+    cmd_pid(*args)
+    nil
+  end
+
+end