scout-gear 2.0.0 → 5.2.0

data/lib/scout/cmd.rb

@@ -0,0 +1,344 @@
+require_relative 'indiferent_hash'
+require_relative 'concurrent_stream'
+require_relative 'log'
+require_relative 'open/stream'
+require 'stringio'
+require 'open3'
+
+module CMD
+
+  TOOLS = IndiferentHash.setup({})
+  def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
+    TOOLS[tool] = [claim, test, block, cmd]
+  end
+
+  def self.conda(tool, env = nil, channel = 'bioconda')
+    if env
+      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")      
+    else
+      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")      
+    end
+  end
+
+  def self.get_tool(tool)
+    return tool.to_s unless TOOLS[tool]
+
+    @@init_cmd_tool ||= IndiferentHash.setup({})
+    if !@@init_cmd_tool[tool]
+      claim, test, block, cmd = TOOLS[tool]
+      begin
+        if test
+          CMD.cmd(test + " ")
+        else
+          CMD.cmd("#{tool} --help")
+        end
+      rescue
+        if claim
+          claim.produce
+        else
+          block.call
+        end
+      end
+      version_txt = ""
+      version = nil
+      ["--version", "-version", "--help", ""].each do |f|
+        begin
+          version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
+          version = CMD.scan_version_text(version_txt, tool)
+          break if version
+        rescue
+          Log.exception $!
+        end
+      end
+
+      @@init_cmd_tool[tool] = version || true
+
+      return cmd if cmd
+    end
+
+    tool.to_s
+  end
+
+  def self.scan_version_text(text, cmd = nil)
+    cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
+    text.split("\n").each do |line|
+      next unless line =~ /\W#{cmd}\W/i
+      m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+      return m[1] if m
+      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?v)/i)
+      return m[1] if m
+      next unless line =~ /\Wversion\W/i
+      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+      return m[1] if m
+    end
+    m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+    return m[1] if m
+    m = text.match(/(?:#{cmd}.*(v.*|.*v))/i)
+    return m[1] if m
+    nil
+  end
+  def self.versions
+    return {} unless defined? @@init_cmd_tool
+    @@init_cmd_tool.select{|k,v| v =~ /\d+\./ }
+  end
+
+  def self.bash(cmd)
+    cmd = %Q(bash <<EOF\n#{cmd}\nEOF\n)
+    CMD.cmd(cmd, :autojoin => true)
+  end
+
+  def self.process_cmd_options(options = {})
+    add_dashes = IndiferentHash.process_options options, :add_option_dashes
+
+    string = ""
+    options.each do |option, value|
+      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
+      #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
+      value = value.gsub("'","\\'") if value.to_s.include? "'"
+
+      option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
+
+      case 
+      when value.nil? || FalseClass === value 
+        next
+      when TrueClass === value
+        string << "#{option} "
+      else
+        if option.to_s.chars.to_a.last == "="
+          string << "#{option}'#{value}' "
+        else
+          string << "#{option} '#{value}' "
+        end
+      end
+    end
+
+    string.strip
+  end
+
+  def self.cmd(tool, cmd = nil, options = {}, &block)
+    options, cmd = cmd, nil if Hash === cmd
+
+    options    = IndiferentHash.add_defaults options, :stderr => Log::DEBUG
+    in_content = options.delete(:in)
+    stderr     = options.delete(:stderr)
+    post       = options.delete(:post)
+    pipe       = options.delete(:pipe)
+    log        = options.delete(:log)
+    no_fail    = options.delete(:no_fail)
+    no_fail    = options.delete(:nofail) if no_fail.nil?
+    no_wait    = options.delete(:no_wait)
+    xvfb       = options.delete(:xvfb)
+    bar        = options.delete(:progress_bar)
+    save_stderr = options.delete(:save_stderr)
+    autojoin   = options.delete(:autojoin)
+    autojoin   = no_wait if autojoin.nil?
+
+    dont_close_in  = options.delete(:dont_close_in)
+
+    log = true if log.nil?
+    
+    if cmd.nil? && ! Symbol === tool 
+      cmd = tool
+    else
+      tool = get_tool(tool)
+      if cmd.nil?
+        cmd = tool
+      else
+        cmd = tool + ' ' + cmd
+      end
+
+    end
+
+    case xvfb
+    when TrueClass
+      cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
+    when String
+      cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
+    when String
+    end
+
+    if stderr == true
+      stderr = Log::HIGH
+    end
+
+    cmd_options = process_cmd_options options
+    if cmd =~ /'\{opt\}'/
+      cmd.sub!('\'{opt}\'', cmd_options) 
+    else
+      cmd << " " << cmd_options
+    end
+
+    in_content = StringIO.new in_content if String === in_content
+
+    sin, sout, serr, wait_thr = begin
+                                  Open3.popen3(ENV, cmd)
+                                rescue
+                                  Log.warn $!.message
+                                  raise ProcessFailed, nil, cmd unless no_fail
+                                  return
+                                end
+    pid = wait_thr.pid
+
+    Log.debug{"CMD: [#{pid}] #{cmd}" if log}
+
+    if in_content.respond_to?(:read)
+      in_thread = Thread.new(Thread.current) do |parent|
+        Thread.current.report_on_exception = false if no_fail
+        begin
+          begin
+            while c = in_content.readpartial(Open::BLOCK_SIZE)
+              sin << c 
+            end
+          rescue EOFError
+          end
+          sin.close  unless sin.closed?
+
+          unless dont_close_in
+            in_content.close unless in_content.closed? 
+            in_content.join if in_content.respond_to? :join 
+          end
+        rescue
+          Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
+          raise $!
+        end
+      end
+    else
+      in_thread = nil
+      sin.close
+    end
+
+    pids = [pid]
+
+    if pipe
+
+      ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail 
+
+      sout.callback = post if post
+
+      if (Integer === stderr and log) || bar
+        err_thread = Thread.new do
+          Thread.current["name"] = "Error log: [#{pid}] #{ cmd }"
+          begin
+          while line = serr.gets
+            bar.process(line) if bar
+            sout.log = line
+            sout.std_err << line if save_stderr
+            Log.log "STDERR [#{pid}]: " +  line, stderr if log
+          end 
+          serr.close
+          rescue
+            Log.exception $!
+            raise $!
+          end
+        end
+      else
+        err_thread = Open.consume_stream(serr, true)
+      end
+
+      sout.threads = [in_thread, err_thread, wait_thr].compact
+
+      sout
+    else
+
+      if bar
+        err = ""
+        err_thread = Thread.new do
+          while not serr.eof?
+            line = serr.gets 
+            bar.process(line) 
+            err << line if Integer === stderr and log
+          end
+          serr.close
+        end
+      elsif log and Integer === stderr
+        err = ""
+        err_thread = Thread.new do
+          while not serr.eof?
+            err << serr.gets 
+          end
+          serr.close
+        end
+      else
+        Open.consume_stream(serr, true)
+        #serr.close 
+        err_thread = nil
+        err = ""
+      end
+
+      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail 
+
+      begin
+        out = StringIO.new sout.read
+        sout.close unless sout.closed?
+
+        status = wait_thr.value
+        if not status.success? and not no_fail
+          if !err.empty?
+            raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
+          else
+            raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
+          end
+        else
+          Log.log err, stderr if Integer === stderr and log
+        end
+        out
+      ensure
+        post.call if post
+      end
+    end
+  end
+
+  def self.cmd_pid(*args)
+    all_args = *args
+
+    bar = all_args.last[:progress_bar] if Hash === all_args.last
+
+    all_args << {} unless Hash === all_args.last
+
+    level = all_args.last[:log] || 0
+    level = 0 if TrueClass === level
+    level = 10 if FalseClass === level
+    level = level.to_i
+
+    all_args.last[:log] = true
+    all_args.last[:pipe] = true
+
+    io = cmd(*all_args)
+    pid = io.pids.first
+
+    line = "" if bar
+    starting = true
+    while c = io.getc
+      if starting
+        if pid
+          Log.logn "STDOUT [#{pid}]: ", level
+        else
+          Log.logn "STDOUT: ", level
+        end
+        starting = false
+      end
+      STDERR << c if Log.severity <= level
+      line << c if bar
+      if c == "\n"
+        bar.process(line) if bar
+        starting = true
+        line = "" if bar
+      end
+    end 
+    begin
+      io.join
+      bar.remove if bar
+    rescue
+      bar.remove(true) if bar
+      raise $!
+    end
+
+    nil
+  end
+
+  def self.cmd_log(*args)
+    cmd_pid(*args)
+    nil
+  end
+
+end

data/lib/scout/concurrent_stream.rb

@@ -0,0 +1,259 @@
+require_relative 'indiferent_hash'
+
+module AbortedStream
+  attr_accessor :exception
+  def self.setup(obj, exception = nil)
+    obj.extend AbortedStream
+    obj.exception = exception
+  end
+end
+
+module ConcurrentStream
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread, :stream_exception, :log, :std_err
+
+  def self.setup(stream, options = {}, &block)
+    
+    threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair
+    stream.extend ConcurrentStream unless ConcurrentStream === stream
+
+    stream.threads ||= []
+    stream.pids ||= []
+    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil? 
+    stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
+    stream.autojoin = autojoin unless autojoin.nil?
+    stream.no_fail = no_fail unless no_fail.nil?
+    stream.std_err = ""
+
+    stream.pair = pair unless pair.nil?
+
+    callback = block if block_given?
+    if callback
+      if stream.callback
+        old_callback = stream.callback
+        stream.callback = Proc.new do
+          old_callback.call
+          callback.call
+        end
+      else
+        stream.callback = callback 
+      end
+    end
+
+    if abort_callback
+      if stream.abort_callback
+        old_abort_callback = stream.abort_callback
+        stream.abort_callback = Proc.new do
+          old_abort_callback.call
+          abort_callback.call
+        end
+      else
+        stream.abort_callback = abort_callback 
+      end
+    end
+
+    stream.filename = filename.nil? ? stream.inspect.split(":").last[0..-2] : filename
+
+    stream.lockfile = lockfile unless lockfile.nil?
+
+    stream.aborted = false
+
+    stream
+  end
+
+  def annotate(stream)
+    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lockfile => lockfile)
+    stream
+  end
+
+  def clear
+    @threads = @pids = @callback = @abort_callback = @joined = nil
+  end
+
+  def joined?
+    @joined
+  end
+
+  def aborted?
+    @aborted
+  end
+
+  def join_threads
+    if @threads
+      @threads.each do |t| 
+        next if t == Thread.current
+        begin
+          t.join
+          if Process::Status === t.value
+            if ! (t.value.success? || no_fail)
+
+              if log
+                msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
+              else
+                msg = "Error joining #{self.filename || self.inspect}"
+              end
+
+              raise ConcurrentStreamProcessFailed.new t.pid, msg, self
+            end
+          end
+        rescue Exception
+          if no_fail
+            Log.low "Not failing on exception joining thread in ConcurrenStream - #{filename} - #{$!.message}"
+          else
+            Log.low "Exception joining thread in ConcurrenStream #{Log.fingerprint self} - #{Log.fingerprint t} - #{$!.message}"
+            stream_raise_exception $! 
+          end
+        end
+      end
+    end
+    @threads = []
+  end
+
+  def join_pids
+    if @pids and @pids.any?
+      @pids.each do |pid| 
+        begin
+          Process.waitpid(pid, Process::WUNTRACED)
+          stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
+        rescue Errno::ECHILD
+        end
+      end 
+      @pids = []
+    end
+  end
+
+  def join_callback
+    if @callback and not joined?
+      begin
+        @callback.call
+      ensure
+        @callback = nil
+      end
+    end
+  end
+
+  def join
+    begin
+      join_threads
+      join_pids
+      join_callback
+      close unless closed?
+    ensure
+      @joined = true
+      lockfile.unlock if lockfile and lockfile.locked?
+      raise stream_exception if stream_exception
+    end
+  end
+
+  def abort_threads(exception = nil)
+    return unless @threads and @threads.any?
+    name = Log.fingerprint(Thread.current)
+    name += " - file:#{filename}" if filename
+    Log.low "Aborting threads (#{name}) - #{@threads.collect{|t| Log.fingerprint(t) } * ", "}"
+
+    @threads.each do |t| 
+      next if t == Thread.current
+      Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
+      t.raise((exception.nil? ? Aborted.new : exception))
+    end 
+
+    @threads.each do |t|
+      next if t == Thread.current
+      begin
+        t.join unless t == Thread.current
+      rescue Aborted
+      rescue Exception
+        Log.debug "Thread (#{name}) exception: #{$!.message}"
+      end
+    end
+  end
+
+  def abort_pids
+    @pids.each do |pid|
+      begin 
+        Log.low "Killing PID #{pid} in ConcurrentStream #{filename}"
+        Process.kill :INT, pid 
+      rescue Errno::ESRCH
+      end
+    end if @pids
+    @pids = []
+  end
+
+  def abort(exception = nil)
+    self.stream_exception ||= exception
+    if @aborted
+      Log.medium "Already aborted stream #{Log.fingerprint self} [#{@aborted}]"
+      return
+    else
+      Log.medium "Aborting stream #{Log.fingerprint self} [#{@aborted}]"
+    end
+    AbortedStream.setup(self, exception)
+    @aborted = true 
+    begin
+      @abort_callback.call exception if @abort_callback
+
+      abort_threads(exception)
+      abort_pids
+
+      @callback = nil
+      @abort_callback = nil
+
+      if @pair && @pair.respond_to?(:abort) && ! @pair.aborted?
+        Log.medium "Aborting pair stream #{Log.fingerprint self}: #{Log.fingerprint @pair }"
+        @pair.abort exception
+      end
+    ensure
+      close unless closed?
+
+      if lockfile and lockfile.locked?
+        lockfile.unlock 
+      end
+    end
+  end
+
+  def close(*args)
+    if autojoin
+      begin
+        super(*args)
+      rescue
+        Log.exception $!
+        self.abort
+        self.join 
+        stream_raise_exception $!
+      ensure
+        self.join if self.closed? or self.eof? 
+      end
+    else
+      super(*args)
+    end
+  end
+
+  def read(*args)
+    begin
+      super(*args)
+    ensure
+      begin
+        close unless closed?
+      rescue Exception
+        raise $! if ConcurrentStreamProcessFailed === $!
+      end if autojoin && ! closed? && eof?
+    end
+  end
+
+  def add_callback(&block)
+    old_callback = callback
+    @callback = Proc.new do 
+      old_callback.call if old_callback
+      block.call
+    end
+  end
+
+  def stream_raise_exception(exception)
+    threads.each do |thread|
+      thread.raise exception
+    end
+    self.stream_exception = exception
+
+    self.abort
+  end
+
+end

data/lib/scout/exceptions.rb

@@ -13,6 +13,12 @@ end
 
 class Aborted < StandardError; end
 
+class ParameterException < ScoutException; end
+class MissingParameterException < ParameterException
+  def initialize(parameter)
+    super("Missing parameter '#{parameter}'")
+  end
+end
 class ProcessFailed < StandardError; 
   attr_accessor :pid, :msg
   def initialize(pid = Process.pid, msg = nil)
@@ -41,7 +47,13 @@ end
 
 class OpenURLError < StandardError; end
 
-class DontClose < Exception; end
+class DontClose < Exception
+  attr_accessor :payload
+  def initialize(payload = nil)
+    @payload = payload
+  end
+end
+
 
 class KeepLocked < Exception
   attr_accessor :payload
@@ -57,13 +69,9 @@ class KeepBar < Exception
   end
 end
 
-#class ParameterException < ScoutException; end
+class LockInterrupted < TryAgain; end
+
 #
-#class MissingParameterException < ParameterException
-#  def initialize(parameter)
-#    super("Missing parameter '#{parameter}'")
-#  end
-#end
 #class ClosedStream < StandardError; end
 #class OpenGzipError < StandardError; end
 #
@@ -76,7 +84,6 @@ end
 #end
 #
 #class SemaphoreInterrupted < TryAgain; end
-#class LockInterrupted < TryAgain; end
 #
 #class RemoteServerError < StandardError; end
 #

data/lib/scout/indiferent_hash/options.rb

@@ -1,32 +1,24 @@
 module IndiferentHash
   def self.add_defaults(options, defaults = {})
-    options ||= {}
+    options = string2hash options if String === options
     IndiferentHash.setup(options)
-    case
-    when Hash === options
-      new_options = options.dup
-    when String === options
-      new_options = string2hash options
-    else
-      raise "Format of '#{options.inspect}' not understood. It should be a hash"
-    end
+
+    defaults = string2hash defaults if String === defaults
 
     defaults.each do |key, value|
       next if options.include? key
 
-      new_options[key] = value 
+      options[key] = value 
     end
 
-    new_options
-
-    options.replace new_options
+    options
   end
 
   def self.process_options(hash, *keys)
     IndiferentHash.setup(hash)
 
     defaults = keys.pop if Hash === keys.last
-    hahs = IndiferentHash.add_defaults hash, defaults if defaults
+    hash = IndiferentHash.add_defaults hash, defaults if defaults
 
     if keys.length == 1
       hash.include?(keys.first.to_sym) ? hash.delete(keys.first.to_sym) : hash.delete(keys.first.to_s) 
@@ -102,7 +94,7 @@ module IndiferentHash
     options = {}
 
     string.split('#').each do |str|
-      key, sep, value = str.partition "="
+      key, _, value = str.partition "="
 
       key = key[1..-1].to_sym if key[0] == ":"
 
@@ -114,17 +106,7 @@ module IndiferentHash
       options[key] = value.to_f and next if value =~ /^\d*\.\d+$/
       options[key] = true and next if value == "true"
       options[key] = false and next if value == "false"
-      options[key] = value and next 
-
-      options[key] = begin
-                       saved_safe = $SAFE
-                       $SAFE = 0
-                       eval(value)
-                     rescue Exception
-                       value
-                     ensure
-                       $SAFE = saved_safe
-                     end
+      options[key] = value
     end
 
     IndiferentHash.setup(options)