scout-gear 9.1.0 → 10.0.1

data/lib/scout/cmd.rb

@@ -1,347 +0,0 @@
-require_relative 'indiferent_hash'
-require_relative 'concurrent_stream'
-require_relative 'log'
-require_relative 'open/stream'
-require 'stringio'
-require 'open3'
-
-module CMD
-
-  TOOLS = IndiferentHash.setup({})
-  def self.tool(tool, claim = nil, test = nil, cmd = nil, &block)
-    TOOLS[tool] = [claim, test, block, cmd]
-  end
-
-  def self.conda(tool, env = nil, channel = 'bioconda')
-    if env
-      CMD.cmd("bash -l -c '(conda activate #{env} && conda install #{tool} -c #{channel})'")
-    else
-      CMD.cmd("bash -l -c 'conda install #{tool} -c #{channel}'")
-    end
-  end
-
-
-  def self.get_tool(tool)
-    return tool.to_s unless TOOLS[tool]
-
-    @@init_cmd_tool ||= IndiferentHash.setup({})
-
-    claim, test, block, cmd = TOOLS[tool]
-    cmd = tool.to_s if cmd.nil?
-
-    if !@@init_cmd_tool[tool]
-
-      begin
-        if test
-          CMD.cmd(test + " ")
-        else
-          CMD.cmd("#{cmd} --help")
-        end
-      rescue
-        if claim
-          claim.produce
-        else
-          block.call
-        end
-      end
-      version_txt = ""
-      version = nil
-      ["--version", "-version", "--help", ""].each do |f|
-        begin
-          version_txt += CMD.cmd("#{cmd} #{f} 2>&1", :nofail => true).read
-          version = CMD.scan_version_text(version_txt, tool)
-          break if version
-        rescue
-          Log.exception $!
-        end
-      end
-
-      @@init_cmd_tool[tool] = version || true
-
-      return cmd if cmd
-    end
-
-    cmd
-  end
-  def self.scan_version_text(text, cmd = nil)
-    cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
-    text = Misc.fixutf8 text
-    text.split("\n").each do |line|
-      next unless line =~ /\W#{cmd}\W/i
-      m = line.match(/(v(?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
-      return m[1] if m
-      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?v)/i)
-      return m[1] if m
-      next unless line =~ /\Wversion\W/i
-      m = line.match(/((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
-      return m[1] if m
-    end
-    m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
-    return m[1] if m
-    m = text.match(/(?:#{cmd}.*(v.*|.*v))/i)
-    return m[1] if m
-    nil
-  end
-  def self.versions
-    return {} unless defined? @@init_cmd_tool
-    @@init_cmd_tool.select{|k,v| v =~ /\d+\./ }
-  end
-
-  def self.bash(cmd)
-    cmd = %Q(bash <<EOF\n#{cmd}\nEOF\n)
-    CMD.cmd(cmd, :autojoin => true)
-  end
-
-  def self.process_cmd_options(options = {})
-    add_dashes = IndiferentHash.process_options options, :add_option_dashes
-
-    string = ""
-    options.each do |option, value|
-      raise "Invalid option key: #{option.inspect}" if option.to_s !~ /^[a-z_0-9\-=.]+$/i
-      #raise "Invalid option value: #{value.inspect}" if value.to_s.include? "'"
-      value = value.gsub("'","\\'") if value.to_s.include? "'"
-
-      option = "--" << option.to_s if add_dashes and option.to_s[0] != '-'
-
-      case
-      when value.nil? || FalseClass === value
-        next
-      when TrueClass === value
-        string << "#{option} "
-      else
-        if option.to_s.chars.to_a.last == "="
-          string << "#{option}'#{value}' "
-        else
-          string << "#{option} '#{value}' "
-        end
-      end
-    end
-
-    string.strip
-  end
-
-  def self.cmd(tool, cmd = nil, options = {}, &block)
-    options, cmd = cmd, nil if Hash === cmd
-
-    options    = IndiferentHash.add_defaults options, :stderr => Log::DEBUG
-    in_content = options.delete(:in)
-    stderr     = options.delete(:stderr)
-    post       = options.delete(:post)
-    pipe       = options.delete(:pipe)
-    log        = options.delete(:log)
-    no_fail    = options.delete(:no_fail)
-    no_fail    = options.delete(:nofail) if no_fail.nil?
-    no_wait    = options.delete(:no_wait)
-    xvfb       = options.delete(:xvfb)
-    bar        = options.delete(:progress_bar)
-    save_stderr = options.delete(:save_stderr)
-    autojoin   = options.delete(:autojoin)
-    autojoin   = no_wait if autojoin.nil?
-
-    dont_close_in  = options.delete(:dont_close_in)
-
-    log = true if log.nil?
-
-    if cmd.nil? && ! Symbol === tool
-      cmd = tool
-    else
-      tool = get_tool(tool)
-      if cmd.nil?
-        cmd = tool
-      else
-        cmd = tool + ' ' + cmd
-      end
-
-    end
-
-    case xvfb
-    when TrueClass
-      cmd = "xvfb-run --server-args='-screen 0 1024x768x24' --auto-servernum #{cmd}"
-    when String
-      cmd = "xvfb-run --server-args='#{xvfb}' --auto-servernum --server-num=1 #{cmd}"
-    when String
-    end
-
-    if stderr == true
-      stderr = Log::HIGH
-    end
-
-    cmd_options = process_cmd_options options
-    if cmd =~ /'\{opt\}'/
-      cmd.sub!('\'{opt}\'', cmd_options)
-    else
-      cmd << " " << cmd_options
-    end
-
-    in_content = StringIO.new in_content if String === in_content
-
-    sin, sout, serr, wait_thr = begin
-                                  Open3.popen3(ENV, cmd)
-                                rescue
-                                  Log.warn $!.message
-                                  raise ProcessFailed, nil, cmd unless no_fail
-                                  return
-                                end
-    pid = wait_thr.pid
-
-    Log.debug{"CMD: [#{pid}] #{cmd}".strip if log}
-
-    if in_content.respond_to?(:read)
-      in_thread = Thread.new(Thread.current) do |parent|
-        begin
-          Thread.current.report_on_exception = false if no_fail
-          Thread.current["name"] = "CMD in"
-          while c = in_content.read(Open::BLOCK_SIZE)
-            sin << c
-          end
-          sin.close  unless sin.closed?
-
-          unless dont_close_in
-            in_content.close unless in_content.closed?
-            in_content.join if in_content.respond_to? :join
-          end
-        rescue
-          Log.error "Error in CMD  [#{pid}] #{cmd}: #{$!.message}" unless no_fail
-          raise $!
-        end
-      end
-      Thread.pass until in_thread["name"]
-    else
-      in_thread = nil
-      sin.close
-    end
-
-    pids = [pid]
-
-    if pipe
-
-      ConcurrentStream.setup sout, :pids => pids, :autojoin => autojoin, :no_fail => no_fail
-
-      sout.callback = post if post
-
-      if (Integer === stderr and log) || bar
-        err_thread = Thread.new do
-          Thread.current["name"] = "Error log: [#{pid}] #{ cmd }"
-          begin
-          while line = serr.gets
-            bar.process(line) if bar
-            sout.log = line
-            sout.std_err << line if save_stderr
-            Log.log "STDERR [#{pid}]: " +  line, stderr if log
-          end
-          serr.close
-          rescue
-            Log.exception $!
-            raise $!
-          end
-        end
-      else
-        err_thread = Open.consume_stream(serr, true)
-      end
-
-      sout.threads = [in_thread, err_thread, wait_thr].compact
-
-      sout
-    else
-
-      if bar
-        err = ""
-        err_thread = Thread.new do
-          while not serr.eof?
-            line = serr.gets
-            bar.process(line)
-            err << line if Integer === stderr and log
-          end
-          serr.close
-        end
-      elsif log and Integer === stderr
-        err = ""
-        err_thread = Thread.new do
-          while not serr.eof?
-            err << serr.gets
-          end
-          serr.close
-        end
-      else
-        Open.consume_stream(serr, true)
-        err_thread = nil
-        err = ""
-      end
-
-      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => autojoin, :no_fail => no_fail
-
-      begin
-        out = StringIO.new sout.read
-        sout.close unless sout.closed?
-
-        status = wait_thr.value
-        if status && ! status.success? && ! no_fail
-          if !err.empty?
-            raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}.\n#{err}"
-          else
-            raise ProcessFailed.new pid, "#{cmd} failed with error status #{status.exitstatus}"
-          end
-        else
-          Log.log err, stderr if Integer === stderr and log
-        end
-        out
-      ensure
-        post.call if post
-      end
-    end
-  end
-
-  def self.cmd_pid(*args)
-    all_args = *args
-
-    bar = all_args.last[:progress_bar] if Hash === all_args.last
-
-    all_args << {} unless Hash === all_args.last
-
-    level = all_args.last[:log] || 0
-    level = 0 if TrueClass === level
-    level = 10 if FalseClass === level
-    level = level.to_i
-
-    all_args.last[:log] = true
-    all_args.last[:pipe] = true
-
-    io = cmd(*all_args)
-    pid = io.pids.first
-
-    line = "" if bar
-    starting = true
-    while c = io.getc
-      if starting
-        if pid
-          Log.logn "STDOUT [#{pid}]: ", level
-        else
-          Log.logn "STDOUT: ", level
-        end
-        starting = false
-      end
-      STDERR << c if Log.severity <= level
-      line << c if bar
-      if c == "\n"
-        bar.process(line) if bar
-        starting = true
-        line = "" if bar
-      end
-    end
-    begin
-      io.join
-      bar.remove if bar
-    rescue
-      bar.remove(true) if bar
-      raise $!
-    end
-
-    nil
-  end
-
-  def self.cmd_log(*args)
-    cmd_pid(*args)
-    nil
-  end
-
-end

data/lib/scout/concurrent_stream.rb

@@ -1,284 +0,0 @@
-require_relative 'indiferent_hash'
-
-module AbortedStream
-  attr_accessor :exception
-  def self.setup(obj, exception = nil)
-    obj.extend AbortedStream
-    obj.exception = exception
-  end
-end
-
-module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lock, :no_fail, :pair, :thread, :stream_exception, :log, :std_err, :next
-
-  def self.setup(stream, options = {}, &block)
-    threads, pids, callback, abort_callback, filename, autojoin, lock, no_fail, pair, next_stream = IndiferentHash.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lock, :no_fail, :pair, :next
-    stream.extend ConcurrentStream unless ConcurrentStream === stream
-
-    stream.threads ||= []
-    stream.pids ||= []
-    stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
-    stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
-    stream.autojoin = autojoin unless autojoin.nil?
-    stream.no_fail = no_fail unless no_fail.nil?
-    stream.std_err = ""
-
-    stream.next = next_stream unless next_stream.nil?
-    stream.pair = pair unless pair.nil?
-
-    callback = block if block_given?
-    if callback
-      if stream.callback
-        old_callback = stream.callback
-        stream.callback = Proc.new do
-          old_callback.call
-          callback.call
-        end
-      else
-        stream.callback = callback
-      end
-    end
-
-    if abort_callback
-      if stream.abort_callback
-        old_abort_callback = stream.abort_callback
-        stream.abort_callback = Proc.new do
-          old_abort_callback.call
-          abort_callback.call
-        end
-      else
-        stream.abort_callback = abort_callback
-      end
-    end
-
-    stream.filename = filename.nil? ? stream.inspect.split(":").last[0..-2] : filename
-
-    stream.lock = lock unless lock.nil?
-
-    stream.aborted = false
-
-    stream
-  end
-
-  def annotate(stream)
-    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lock => lock)
-    stream
-  end
-
-  def clear
-    @threads = @pids = @callback = @abort_callback = @joined = nil
-  end
-
-  def joined?
-    @joined
-  end
-
-  def aborted?
-    @aborted
-  end
-
-  def join_threads
-    if @threads
-      @threads.each do |t|
-        next if t == Thread.current
-        begin
-          t.join
-          if Process::Status === t.value
-            if ! (t.value.success? || no_fail)
-
-              if log
-                msg = "Error joining #{self.filename || self.inspect}. Last log line: #{log}"
-              else
-                msg = "Error joining #{self.filename || self.inspect}"
-              end
-
-              raise ConcurrentStreamProcessFailed.new t.pid, msg, self
-            end
-          end
-        rescue Exception
-          if no_fail
-            Log.low "Not failing on exception joining thread in ConcurrenStream - #{filename} - #{$!.message}"
-          else
-            Log.low "Exception joining thread in ConcurrenStream #{Log.fingerprint self} - #{Log.fingerprint t} - #{$!.message}"
-            stream_raise_exception $!
-          end
-        end
-      end
-    end
-    @threads = []
-  end
-
-  def join_pids
-    if @pids and @pids.any?
-      @pids.each do |pid|
-        begin
-          Process.waitpid(pid, Process::WUNTRACED)
-          stream_raise_exception ConcurrentStreamProcessFailed.new(pid, "Error in waitpid", self) unless $?.success? or no_fail
-        rescue Errno::ECHILD
-        end
-      end
-      @pids = []
-    end
-  end
-
-  def join_callback
-    if @callback and not joined?
-      begin
-        @callback.call
-      ensure
-        @callback = nil
-      end
-    end
-  end
-
-  def join
-    begin
-      join_threads
-      join_pids
-      raise stream_exception if stream_exception
-      join_callback
-      close unless closed?
-    ensure
-      @joined = true
-      begin
-        lock.unlock if lock && lock.locked?
-      rescue
-        Log.exception $!
-      end
-      raise stream_exception if stream_exception
-    end
-  end
-
-  def abort_threads(exception = nil)
-    return unless @threads and @threads.any?
-    name = Log.fingerprint(Thread.current)
-    name += " - file:#{filename}" if filename
-    Log.low "Aborting threads (#{name}) - #{@threads.collect{|t| Log.fingerprint(t) } * ", "}"
-
-    threads = @threads.dup
-    @threads.clear
-    threads.each do |t|
-      next if t == Thread.current
-      next if t["aborted"]
-      t["aborted"] = true
-      exception = exception.nil? ? Aborted.new : exception
-      Log.debug "Aborting thread #{Log.fingerprint(t)} with exception: #{exception}"
-      t.raise(exception)
-      t.join
-    end
-  end
-
-  def abort_pids
-    @pids.each do |pid|
-      begin
-        Log.low "Killing PID #{pid} in ConcurrentStream #{filename}"
-        Process.kill :INT, pid
-      rescue Errno::ESRCH
-      end
-    end if @pids
-    @pids = []
-  end
-
-  def abort(exception = nil)
-    self.stream_exception ||= exception
-    if @aborted
-      Log.medium "Already aborted stream #{Log.fingerprint self} [#{@aborted}]"
-      return
-    else
-      Log.medium "Aborting stream #{Log.fingerprint self} [#{@aborted}]"
-    end
-    AbortedStream.setup(self, exception)
-    @aborted = true
-    begin
-      @abort_callback.call exception if @abort_callback
-
-      abort_threads(exception)
-      abort_pids
-
-      @callback = nil
-      @abort_callback = nil
-
-      if @pair && @pair.respond_to?(:abort) && ! @pair.aborted?
-        Log.medium "Aborting pair stream #{Log.fingerprint self}: #{Log.fingerprint @pair }"
-        @pair.abort exception
-      end
-    ensure
-      close unless closed?
-
-      if lock and lock.locked?
-        lock.unlock
-      end
-    end
-  end
-
-  def close(*args)
-    if autojoin
-      begin
-        super(*args)
-      rescue
-        self.abort
-        self.join
-        stream_raise_exception $!
-      ensure
-        self.join if ! @stream_exception && (self.closed? || self.eof?)
-      end
-    else
-      super(*args)
-    end
-  end
-
-  def read(*args)
-    begin
-      super(*args)
-    rescue Exception
-      @stream_exception ||= $!
-      raise @stream_exception
-    ensure
-      if ! @stream_exception && autojoin && ! closed?
-        begin
-          done = eof?
-        rescue Exception
-          self.abort($!)
-          raise $!
-        end
-        close if done
-      end
-    end
-  end
-
-  def add_callback(&block)
-    old_callback = callback
-    @callback = Proc.new do
-      old_callback.call if old_callback
-      block.call
-    end
-  end
-
-  def stream_raise_exception(exception)
-    self.stream_exception = exception
-    threads.each do |thread|
-      thread.raise exception
-    end
-    self.abort
-  end
-
-  def self.process_stream(stream, close: true, join: true, message: "process_stream", **kwargs, &block)
-    ConcurrentStream.setup(stream, **kwargs)
-    begin
-      begin
-        yield
-      ensure
-        stream.close if close && stream.respond_to?(:close) && ! (stream.respond_to?(:closed?) && stream.closed?)
-        stream.join if join && stream.respond_to?(:join) && ! stream.joined?
-      end
-    rescue Aborted
-      Log.low "Aborted #{message}: #{$!.message}"
-      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
-      raise $!
-    rescue Exception
-      Log.low "Exception #{message}: #{$!.message}"
-      stream.abort($!) if stream.respond_to?(:abort) && ! stream.aborted?
-      raise $!
-    end
-  end
-end